From 38bc131598d5d993295f97b7001811209a1b16a5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:10:12 -0800 Subject: [PATCH 01/79] fix(code-index): report skipped nested repositories in unmounted audit Keep parent repository source admission unchanged; use one shared walker for exclusion evidence instead of a second filesystem scan. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 64 +++++++++++++++- .../src/unmounted_files.rs | 76 +++++++++++++++++-- 2 files changed, 132 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 63d4be6e407..a02ecb2e20e 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -9,8 +9,9 @@ //! walk is therefore public rather than crate-private, the audit in the root //! crate reuses this policy instead of restating it. +use std::collections::BTreeSet; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{Arc, Mutex, PoisonError}; use ignore::overrides::{Override, OverrideBuilder}; use ignore::{Walk, WalkBuilder}; @@ -103,11 +104,51 @@ impl PathGlobScope { } } +/// Diagnostics emitted by the same source walker that enforces the nested +/// repository boundary. Only directories the walk actually reaches are named. +#[derive(Clone, Default)] +pub(crate) struct SourceWalkDiagnostics { + skipped_nested_repositories: Arc>>, +} + +impl SourceWalkDiagnostics { + pub(crate) fn skipped_nested_repositories(&self) -> Vec { + self.skipped_nested_repositories + .lock() + .unwrap_or_else(PoisonError::into_inner) + .iter() + .cloned() + .collect() + } +} + #[tracing::instrument(name = "code_index.capture.source_walk", level = "trace", skip_all)] pub fn source_walk( project_root: &Path, path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, +) -> Result { + source_walk_inner(project_root, path_glob, path_policy, None) +} + +/// Expose excluded nested repository roots without changing source admission. +/// The unmounted-file audit uses this to avoid falsely implying completeness. +/// It uses the same traversal rather than an independent repository walker. +pub(crate) fn source_walk_with_diagnostics( + project_root: &Path, + path_glob: Option<&str>, + path_policy: &IndexPathPolicyV1, +) -> Result<(Walk, SourceWalkDiagnostics), SourceWalkError> { + let diagnostics = SourceWalkDiagnostics::default(); + let walk = source_walk_inner(project_root, path_glob, path_policy, Some(diagnostics.clone()))?; + Ok((walk, diagnostics)) +} + +fn source_walk_inner( + project_root: &Path, + path_glob: Option<&str>, + path_policy: &IndexPathPolicyV1, + diagnostics: Option, ) -> Result { let overrides = build_overrides(project_root, path_glob)?; let has_positive_override = overrides @@ -171,6 +212,13 @@ pub fn source_walk( // sources. Linked worktrees and nested clones share this boundary. // The project root is deliberately exempt at depth zero above. if std::fs::symlink_metadata(entry.path().join(".git")).is_ok() { + if let Some(diagnostics) = &diagnostics { + diagnostics + .skipped_nested_repositories + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(relative); + } return false; } !path_policy.excludes_directory(&relative) @@ -220,7 +268,7 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::source_walk; + use super::{source_walk, source_walk_with_diagnostics}; fn walked_files(root: &std::path::Path, path_policy: &IndexPathPolicyV1) -> Vec { let mut files = source_walk(root, None, path_policy) @@ -435,6 +483,18 @@ mod tests { assert!(index.stdout.ends_with(b" 0\tapp\0")); assert!(root.join("app/src/feature.rs").is_file()); + let (walk, diagnostics) = source_walk_with_diagnostics( + &root, + None, + &IndexPathPolicyV1::new(vec![], vec![]).unwrap(), + ) + .expect("source walk with nested repository diagnostics"); + let _visited = walk.map(|entry| entry.expect("source entry")).count(); + assert_eq!( + diagnostics.skipped_nested_repositories(), + vec!["app".to_owned()] + ); + for rule in ["app/**", "app/src/feature.rs", "app/src/fe*.rs"] { let policy = IndexPathPolicyV1::new(vec![], vec![rule.into()]).unwrap(); let files = walked_files(&root, &policy); diff --git a/crates/tracedecay-code-index/src/unmounted_files.rs b/crates/tracedecay-code-index/src/unmounted_files.rs index 1d1cb19e497..3f140bdebf8 100644 --- a/crates/tracedecay-code-index/src/unmounted_files.rs +++ b/crates/tracedecay-code-index/src/unmounted_files.rs @@ -152,16 +152,19 @@ pub struct ProjectAudit { pub(super) struct ProjectFiles { root: PathBuf, files: Vec, + skipped_nested_repositories: Vec, } impl ProjectFiles { fn collect(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Result { - let walk = - crate::source_walk::source_walk(project_root, None, path_policy).map_err(|error| { - TraceDecayError::Config { - message: format!("source walk rejected its own scope: {}", error.message), - } - })?; + let (walk, diagnostics) = crate::source_walk::source_walk_with_diagnostics( + project_root, + None, + path_policy, + ) + .map_err(|error| TraceDecayError::Config { + message: format!("source walk rejected its own scope: {}", error.message), + })?; let mut files = Vec::new(); for entry in walk { let Ok(entry) = entry else { continue }; @@ -174,6 +177,7 @@ impl ProjectFiles { Ok(Self { root: project_root.to_path_buf(), files, + skipped_nested_repositories: diagnostics.skipped_nested_repositories(), }) } @@ -258,6 +262,32 @@ pub fn audit_project(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Re }; let mut ecosystems = vec![rust, typescript]; ecosystems.extend(unmodelled_ecosystems(&files)); + if !files.skipped_nested_repositories.is_empty() { + // Preserve existing ecosystem caveats and bound the diagnostic text. + // Skipped child repositories are not indexed as parent source. + let named = files + .skipped_nested_repositories + .iter() + .take(8) + .cloned() + .collect::>(); + let remaining = files.skipped_nested_repositories.len() - named.len(); + let summary = format!( + "Skipped nested repositories outside parent index: {}{}", + named.join(", "), + if remaining == 0 { + String::new() + } else { + format!(" (and {remaining} more)") + } + ); + for ecosystem in &mut ecosystems { + ecosystem.note = Some(match ecosystem.note.take() { + Some(existing) => format!("{existing} {summary}"), + None => summary.clone(), + }); + } + } Ok(ProjectAudit { ecosystems }) } @@ -395,6 +425,40 @@ mod tests { ); } + #[test] + fn audit_reports_nested_repository_exclusion_without_claiming_child_code() { + let root = tempfile::tempdir().expect("project fixture"); + write( + root.path(), + "Cargo.toml", + "[package]\nname = \"parent\"\nversion = \"0.0.0\"\n", + ); + write(root.path(), "src/lib.rs", "pub fn parent() {}\n"); + write(root.path(), "app/.git", "gitdir: ../.git/modules/app\n"); + write(root.path(), "app/src/child.rs", "pub fn child() {}\n"); + write(root.path(), "unrelated/.git/HEAD", "ref: refs/heads/main\n"); + write(root.path(), "unrelated/src/foreign.rs", "pub fn foreign() {}\n"); + + let audit = audit_project(root.path(), &no_exclusions()).expect("audit"); + let rust = ecosystem(&audit, "rust"); + assert_eq!( + rust.scanned_file_count, 1, + "child repositories are not parent source" + ); + assert!(rust.unmounted.iter().all(|file| { + !file.file.starts_with("app/") && !file.file.starts_with("unrelated/") + })); + let note = rust.note.as_deref().expect("explicit scope diagnostic"); + assert!(note.contains("Skipped nested repositories"), "{note}"); + assert!(note.contains("app") && note.contains("unrelated"), "{note}"); + assert!(audit.ecosystems.iter().all(|ecosystem| { + ecosystem + .note + .as_deref() + .is_some_and(|note| note.contains("Skipped nested repositories")) + })); + } + /// Both modelled ecosystems report in one pass over a polyglot repository. #[test] fn a_polyglot_project_reports_one_section_per_ecosystem() { From 1f8c1997090e4631ee38f50a32af4186e8153c12 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:14:21 -0800 Subject: [PATCH 02/79] style(code-index): format nested-repository diagnostics --- .../tracedecay-code-index/src/source_walk.rs | 7 ++++++- .../src/unmounted_files.rs | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index a02ecb2e20e..c0eebaeda39 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -140,7 +140,12 @@ pub(crate) fn source_walk_with_diagnostics( path_policy: &IndexPathPolicyV1, ) -> Result<(Walk, SourceWalkDiagnostics), SourceWalkError> { let diagnostics = SourceWalkDiagnostics::default(); - let walk = source_walk_inner(project_root, path_glob, path_policy, Some(diagnostics.clone()))?; + let walk = source_walk_inner( + project_root, + path_glob, + path_policy, + Some(diagnostics.clone()), + )?; Ok((walk, diagnostics)) } diff --git a/crates/tracedecay-code-index/src/unmounted_files.rs b/crates/tracedecay-code-index/src/unmounted_files.rs index 3f140bdebf8..5e5ee2e690d 100644 --- a/crates/tracedecay-code-index/src/unmounted_files.rs +++ b/crates/tracedecay-code-index/src/unmounted_files.rs @@ -157,14 +157,11 @@ pub(super) struct ProjectFiles { impl ProjectFiles { fn collect(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Result { - let (walk, diagnostics) = crate::source_walk::source_walk_with_diagnostics( - project_root, - None, - path_policy, - ) - .map_err(|error| TraceDecayError::Config { - message: format!("source walk rejected its own scope: {}", error.message), - })?; + let (walk, diagnostics) = + crate::source_walk::source_walk_with_diagnostics(project_root, None, path_policy) + .map_err(|error| TraceDecayError::Config { + message: format!("source walk rejected its own scope: {}", error.message), + })?; let mut files = Vec::new(); for entry in walk { let Ok(entry) = entry else { continue }; @@ -437,7 +434,11 @@ mod tests { write(root.path(), "app/.git", "gitdir: ../.git/modules/app\n"); write(root.path(), "app/src/child.rs", "pub fn child() {}\n"); write(root.path(), "unrelated/.git/HEAD", "ref: refs/heads/main\n"); - write(root.path(), "unrelated/src/foreign.rs", "pub fn foreign() {}\n"); + write( + root.path(), + "unrelated/src/foreign.rs", + "pub fn foreign() {}\n", + ); let audit = audit_project(root.path(), &no_exclusions()).expect("audit"); let rust = ecosystem(&audit, "rust"); From 8d6f07e1bb326630a313e1911d8bd271f337dd6c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:16:18 -0800 Subject: [PATCH 03/79] fix(code-index): bound repository audits and report walk failures Limit skipped-repository samples during traversal, honor directory policy before collecting diagnostics, and quote bounded path labels. Propagate source-walk failures instead of returning an empty successful audit. Add behavioral coverage for bounded sampling, policy scope, preserved ecosystem caveats, Unicode/control-character paths, and missing roots. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 85 +++++++-- .../source_walk/nested_repository_tests.rs | 169 ++++++++++++++++++ .../src/unmounted_files.rs | 29 +-- 3 files changed, 246 insertions(+), 37 deletions(-) create mode 100644 crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index c0eebaeda39..e360e4e6807 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -104,21 +104,72 @@ impl PathGlobScope { } } +const MAX_REPOSITORY_NAMES: usize = 8; +const MAX_REPOSITORY_PATH_CHARS: usize = 160; + /// Diagnostics emitted by the same source walker that enforces the nested -/// repository boundary. Only directories the walk actually reaches are named. +/// repository boundary. Only directories within the walk's policy are named. #[derive(Clone, Default)] pub(crate) struct SourceWalkDiagnostics { - skipped_nested_repositories: Arc>>, + skipped_nested_repositories: Arc>, +} + +#[derive(Default)] +struct NestedRepositorySummary { + total: usize, + names: BTreeSet, } impl SourceWalkDiagnostics { - pub(crate) fn skipped_nested_repositories(&self) -> Vec { - self.skipped_nested_repositories + fn record_skipped_nested_repository(&self, relative: String) { + let mut summary = self + .skipped_nested_repositories .lock() - .unwrap_or_else(PoisonError::into_inner) + .unwrap_or_else(PoisonError::into_inner); + // A pruned directory is visited once. Keep only the smallest names, + // independent of filesystem iteration order, rather than collecting + // every path and truncating only after the walk has finished. + summary.total += 1; + summary.names.insert(relative); + if summary.names.len() > MAX_REPOSITORY_NAMES { + let _ = summary.names.pop_last(); + } + } + + pub(crate) fn skipped_nested_repositories_note(&self) -> Option { + let summary = self + .skipped_nested_repositories + .lock() + .unwrap_or_else(PoisonError::into_inner); + if summary.total == 0 { + return None; + } + let names = summary + .names .iter() - .cloned() - .collect() + .map(|path| { + // Bound by characters before escaping, so truncation cannot + // split UTF-8 or an escape. Quoting prevents a filename from + // injecting new lines or delimiters into an agent's report. + let mut chars = path.chars(); + let prefix = chars + .by_ref() + .take(MAX_REPOSITORY_PATH_CHARS) + .collect::(); + let suffix = if chars.next().is_some() { "…" } else { "" }; + format!("{prefix:?}{suffix}") + }) + .collect::>(); + let remaining = summary.total - summary.names.len(); + let suffix = if remaining == 0 { + String::new() + } else { + format!(" (and {remaining} more)") + }; + Some(format!( + "Skipped nested repositories outside parent index: {}{suffix}", + names.join(", ") + )) } } @@ -211,6 +262,11 @@ fn source_walk_inner( if !is_dir { return !path_policy.excludes(&relative); } + // Excluded directories are outside this audit, not missing source + // caused by the repository boundary. Do not probe or name them. + if path_policy.excludes_directory(&relative) { + return false; + } // Includes select paths, not repository authority. A parent // gitlink does not capture the child's files or prove their // freshness, so even registered submodules remain separate @@ -218,15 +274,11 @@ fn source_walk_inner( // The project root is deliberately exempt at depth zero above. if std::fs::symlink_metadata(entry.path().join(".git")).is_ok() { if let Some(diagnostics) = &diagnostics { - diagnostics - .skipped_nested_repositories - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(relative); + diagnostics.record_skipped_nested_repository(relative); } return false; } - !path_policy.excludes_directory(&relative) + true }); if let Some(overrides) = overrides { builder.overrides(overrides); @@ -265,6 +317,9 @@ fn build_overrides( } } +#[cfg(test)] +mod nested_repository_tests; + #[cfg(test)] mod tests { use std::fs; @@ -496,8 +551,8 @@ mod tests { .expect("source walk with nested repository diagnostics"); let _visited = walk.map(|entry| entry.expect("source entry")).count(); assert_eq!( - diagnostics.skipped_nested_repositories(), - vec!["app".to_owned()] + diagnostics.skipped_nested_repositories_note().as_deref(), + Some("Skipped nested repositories outside parent index: \"app\"") ); for rule in ["app/**", "app/src/feature.rs", "app/src/fe*.rs"] { diff --git a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs new file mode 100644 index 00000000000..5242c8305fa --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs @@ -0,0 +1,169 @@ +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use tracedecay_domain::IndexPathPolicyV1; +use tracedecay_domain::errors::TraceDecayError; + +use super::{SourceWalkDiagnostics, source_walk_with_diagnostics}; +use crate::unmounted_files::audit_project; + +fn write(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + std::fs::create_dir_all(path.parent().expect("fixture parent")).unwrap(); + std::fs::write(path, contents).unwrap(); +} + +#[test] +fn repository_diagnostic_sample_is_bounded_and_order_independent() { + let expected = (0..8) + .map(|index| format!("repo-{index:02}")) + .collect::>(); + let orders = [ + (0..24).collect::>(), + (0..24).rev().collect::>(), + ]; + for order in orders { + let diagnostics = SourceWalkDiagnostics::default(); + assert!(diagnostics.skipped_nested_repositories_note().is_none()); + for index in order { + diagnostics.record_skipped_nested_repository(format!("repo-{index:02}")); + let summary = diagnostics.skipped_nested_repositories.lock().unwrap(); + assert!(summary.names.len() <= 8, "collection must be bounded too"); + } + let summary = diagnostics.skipped_nested_repositories.lock().unwrap(); + assert_eq!(summary.total, 24); + assert_eq!(summary.names, expected); + drop(summary); + let note = diagnostics.skipped_nested_repositories_note().unwrap(); + assert_eq!( + note, + "Skipped nested repositories outside parent index: \ + \"repo-00\", \"repo-01\", \"repo-02\", \"repo-03\", \ + \"repo-04\", \"repo-05\", \"repo-06\", \"repo-07\" (and 16 more)" + ); + } +} + +#[test] +fn repository_diagnostics_follow_policy_without_crossing_boundaries() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "keep.rs", "pub fn parent() {}\n"); + for directory in ["app", "excluded", "node_modules"] { + write(root.path(), &format!("{directory}/.git"), "gitdir: foreign\n"); + write(root.path(), &format!("{directory}/child.rs"), "child source\n"); + } + for (include, expected_note) in [ + ( + vec![], + "Skipped nested repositories outside parent index: \"app\"", + ), + ( + vec!["excluded/child.rs".to_owned()], + "Skipped nested repositories outside parent index: \"app\", \"excluded\"", + ), + ] { + let policy = IndexPathPolicyV1::new( + vec!["excluded/**".to_owned(), "node_modules/**".to_owned()], + include, + ) + .unwrap(); + let (walk, diagnostics) = source_walk_with_diagnostics(root.path(), None, &policy) + .expect("source walk"); + let files = walk + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| entry.path().strip_prefix(root.path()).unwrap().to_path_buf()) + .collect::>(); + assert_eq!(files, vec![PathBuf::from("keep.rs")]); + assert_eq!( + diagnostics.skipped_nested_repositories_note().as_deref(), + Some(expected_note) + ); + } +} + +#[test] +fn audit_bounds_repository_summary_and_preserves_unsupported_caveat() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "main.go", "package main\n"); + for index in (0..12).rev() { + write( + root.path(), + &format!("repo-{index:02}/.git/HEAD"), + "ref: refs/heads/main\n", + ); + write( + root.path(), + &format!("repo-{index:02}/child.go"), + "package child\n", + ); + } + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + let audit = audit_project(root.path(), &policy).expect("audit"); + let go = audit + .ecosystems + .iter() + .find(|ecosystem| ecosystem.ecosystem == "go") + .expect("parent Go source was counted"); + assert_eq!(go.scanned_file_count, 1); + assert_eq!(go.status.as_str(), "unsupported"); + assert_eq!( + go.note.as_deref(), + Some( + "1 go source file(s) are present and were not audited, \ + this report cannot say whether any of them is unreachable \ + Skipped nested repositories outside parent index: \ + \"repo-00\", \"repo-01\", \"repo-02\", \"repo-03\", \ + \"repo-04\", \"repo-05\", \"repo-06\", \"repo-07\" (and 4 more)" + ) + ); +} + +#[test] +fn repository_diagnostic_paths_are_clipped_on_character_boundaries() { + let diagnostics = SourceWalkDiagnostics::default(); + diagnostics.record_skipped_nested_repository("é".repeat(161)); + assert_eq!( + diagnostics.skipped_nested_repositories_note().unwrap(), + format!( + "Skipped nested repositories outside parent index: \"{}\"…", + "é".repeat(160) + ) + ); +} + +#[cfg(unix)] +#[test] +fn audit_quotes_control_characters_in_repository_names() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "keep.py", "print('parent')\n"); + write(root.path(), "line\nbreak/.git", "gitdir: foreign\n"); + write(root.path(), "line\nbreak/child.py", "print('child')\n"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + let audit = audit_project(root.path(), &policy).expect("audit"); + let python = audit + .ecosystems + .iter() + .find(|ecosystem| ecosystem.ecosystem == "python") + .expect("parent Python source was counted"); + assert_eq!(python.scanned_file_count, 1); + let note = python.note.as_deref().expect("scope diagnostic"); + assert!(note.contains("\"line\\nbreak\""), "{note}"); + assert!(!note.contains('\n'), "a path must not inject report lines"); + assert!(root.path().join("line\nbreak/child.py").is_file()); +} + +#[test] +fn missing_source_root_is_an_error_not_a_clean_audit() { + let root = tempfile::tempdir().unwrap(); + let missing = root.path().join("missing"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + match audit_project(&missing, &policy) { + Err(TraceDecayError::File { path, message }) => { + assert_eq!(path, missing.display().to_string()); + assert!(message.contains("source walk did not complete")); + } + Err(error) => panic!("unexpected error: {error}"), + Ok(_) => panic!("a failed walk must not report an empty successful audit"), + } +} diff --git a/crates/tracedecay-code-index/src/unmounted_files.rs b/crates/tracedecay-code-index/src/unmounted_files.rs index 5e5ee2e690d..31cf2d6f594 100644 --- a/crates/tracedecay-code-index/src/unmounted_files.rs +++ b/crates/tracedecay-code-index/src/unmounted_files.rs @@ -152,7 +152,7 @@ pub struct ProjectAudit { pub(super) struct ProjectFiles { root: PathBuf, files: Vec, - skipped_nested_repositories: Vec, + nested_repository_note: Option, } impl ProjectFiles { @@ -164,7 +164,10 @@ impl ProjectFiles { })?; let mut files = Vec::new(); for entry in walk { - let Ok(entry) = entry else { continue }; + let entry = entry.map_err(|error| TraceDecayError::File { + message: format!("source walk did not complete: {error}"), + path: project_root.display().to_string(), + })?; if !entry.file_type().is_some_and(|kind| kind.is_file()) { continue; } @@ -174,7 +177,7 @@ impl ProjectFiles { Ok(Self { root: project_root.to_path_buf(), files, - skipped_nested_repositories: diagnostics.skipped_nested_repositories(), + nested_repository_note: diagnostics.skipped_nested_repositories_note(), }) } @@ -259,25 +262,7 @@ pub fn audit_project(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Re }; let mut ecosystems = vec![rust, typescript]; ecosystems.extend(unmodelled_ecosystems(&files)); - if !files.skipped_nested_repositories.is_empty() { - // Preserve existing ecosystem caveats and bound the diagnostic text. - // Skipped child repositories are not indexed as parent source. - let named = files - .skipped_nested_repositories - .iter() - .take(8) - .cloned() - .collect::>(); - let remaining = files.skipped_nested_repositories.len() - named.len(); - let summary = format!( - "Skipped nested repositories outside parent index: {}{}", - named.join(", "), - if remaining == 0 { - String::new() - } else { - format!(" (and {remaining} more)") - } - ); + if let Some(summary) = &files.nested_repository_note { for ecosystem in &mut ecosystems { ecosystem.note = Some(match ecosystem.note.take() { Some(existing) => format!("{existing} {summary}"), From 1b49877815c81451007e66a4f2d54e4a92a06f6d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:39:59 -0800 Subject: [PATCH 04/79] style(code-index): format nested repository regressions Apply the exact formatter changes reported by the hosted Bazel rustfmt_check_2024 failure on PR #4801. Refs #4593 --- .../source_walk/nested_repository_tests.rs | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs index 5242c8305fa..0b77991f58e 100644 --- a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs @@ -49,8 +49,16 @@ fn repository_diagnostics_follow_policy_without_crossing_boundaries() { let root = tempfile::tempdir().unwrap(); write(root.path(), "keep.rs", "pub fn parent() {}\n"); for directory in ["app", "excluded", "node_modules"] { - write(root.path(), &format!("{directory}/.git"), "gitdir: foreign\n"); - write(root.path(), &format!("{directory}/child.rs"), "child source\n"); + write( + root.path(), + &format!("{directory}/.git"), + "gitdir: foreign\n", + ); + write( + root.path(), + &format!("{directory}/child.rs"), + "child source\n", + ); } for (include, expected_note) in [ ( @@ -67,12 +75,18 @@ fn repository_diagnostics_follow_policy_without_crossing_boundaries() { include, ) .unwrap(); - let (walk, diagnostics) = source_walk_with_diagnostics(root.path(), None, &policy) - .expect("source walk"); + let (walk, diagnostics) = + source_walk_with_diagnostics(root.path(), None, &policy).expect("source walk"); let files = walk .map(|entry| entry.expect("source entry")) .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) - .map(|entry| entry.path().strip_prefix(root.path()).unwrap().to_path_buf()) + .map(|entry| { + entry + .path() + .strip_prefix(root.path()) + .unwrap() + .to_path_buf() + }) .collect::>(); assert_eq!(files, vec![PathBuf::from("keep.rs")]); assert_eq!( From 624cc8338a1cfe8486b0638ba88fbbae0df9fa33 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:05:56 -0800 Subject: [PATCH 05/79] fix(code-index): align nested checkout capture with source walks Use one source boundary for interactive walks, Git candidate classification, worktree capture (including ignored-file admissions), and freshness. Parent tracked files hidden by a new nested .git marker become tombstones, invalidate clean HEAD-tree reuse, and return when the marker disappears. Reject source paths passing through symlinked directories or lexical escapes. Add real Git classification and filesystem regressions for nested directory, gitdir-file, symlink, path traversal and removal behavior. Refs #4593 --- .../code_index_scheduler/classification.rs | 134 +++++++++++++++++- .../code_index_scheduler/freshness_witness.rs | 14 +- .../src/code_index_scheduler/reconcile.rs | 8 ++ .../tracedecay-code-index/src/source_walk.rs | 107 +++++++++++++- 4 files changed, 251 insertions(+), 12 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index 8f057e94a28..ab168be9965 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -14,10 +14,11 @@ //! consumes the derived candidate and changed sets to build an incremental //! batch. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use gix::bstr::ByteSlice; +use tracedecay_code_index::source_walk::has_nested_repository_ancestor; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -25,6 +26,8 @@ use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREA pub enum ClassificationErrorV1 { #[error("code-index classification: {0}")] Git(String), + #[error("code-index source boundary: {0}")] + SourceBoundary(#[from] std::io::Error), } /// The truthful disposition of one repository-relative path. @@ -91,6 +94,9 @@ pub struct WorktreeChangeClassificationV1 { /// Present paths whose Git bytes are not UTF-8, so no `String` above /// names them. non_utf8_paths: BTreeSet>, + /// Files still tracked by the parent but hidden behind a nested checkout + /// or symlinked ancestor. These are tombstones, not indexable candidates. + nested_repository_exclusions: BTreeSet, } impl WorktreeChangeClassificationV1 { @@ -202,11 +208,40 @@ impl WorktreeChangeClassificationV1 { } } + // A nested .git marker can appear under paths that are still in + // the parent's index without changing Git status. The live walker + // refuses those files; classification must also exclude them and + // invalidate clean-tree and active-generation reuse. Resolve each + // parent directory once rather than probing per source file. + let mut nested_repository_exclusions = BTreeSet::new(); + if let Some(project_root) = repository.workdir() { + let mut checked_parents = BTreeMap::new(); + for path in committed_baseline + .iter() + .chain(changes.iter().map(|change| &change.path)) + { + let relative = Path::new(path); + let parent = relative.parent().unwrap_or_else(|| Path::new("")); + let excluded = match checked_parents.get(parent) { + Some(excluded) => *excluded, + None => { + let excluded = has_nested_repository_ancestor(project_root, relative)?; + checked_parents.insert(parent.to_path_buf(), excluded); + excluded + } + }; + if excluded { + nested_repository_exclusions.insert(path.clone()); + } + } + } + Ok(Self { committed_baseline, changes, non_utf8_changes, non_utf8_paths, + nested_repository_exclusions, }) } @@ -241,13 +276,16 @@ impl WorktreeChangeClassificationV1 { candidates.remove(&change.path); } } + candidates.retain(|path| !self.nested_repository_exclusions.contains(path)); candidates } /// Whether the worktree carries any change relative to the committed /// baseline, UTF-8-named or not. pub fn has_changes(&self) -> bool { - !(self.changes().is_empty() && self.non_utf8_changes().is_empty()) + !(self.changes().is_empty() + && self.non_utf8_changes().is_empty() + && self.nested_repository_exclusions.is_empty()) } /// Whether any classified change is a merge conflict. @@ -264,10 +302,13 @@ impl WorktreeChangeClassificationV1 { /// tombstones flow through). This is a hint to narrow work, not the identity /// authority; the generation planner still compares content digests. pub fn changed_paths(&self) -> BTreeSet { - self.changes + let mut changed = self + .changes .iter() .map(|change| change.path.clone()) - .collect() + .collect::>(); + changed.extend(self.nested_repository_exclusions.iter().cloned()); + changed } /// Paths removed from the present snapshot (staged or unstaged deletions). @@ -345,3 +386,88 @@ fn classify_item(item: &gix::status::Item) -> Option { }, }) } + +#[cfg(test)] +mod nested_repository_tests { + use std::fs; + use std::path::Path; + use std::process::Command; + + use super::WorktreeChangeClassificationV1; + + fn fixture_git(root: &Path, args: &[&str]) { + let program = + tracedecay_runtime_core::git::try_git_program().expect("fixture Git program"); + let mut command = Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("fixture Git command"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn nested_checkout_marker_tombstones_parent_tracked_sources() { + let root = tempfile::tempdir().unwrap(); + fixture_git(root.path(), &["init", "-q"]); + for path in ["src/keep.rs", "nested/foreign.rs", "linked/foreign.rs"] { + let full = root.path().join(path); + fs::create_dir_all(full.parent().unwrap()).unwrap(); + fs::write(full, "pub fn source() {}\n").unwrap(); + } + fixture_git(root.path(), &["add", "."]); + fixture_git(root.path(), &["commit", "-qm", "parent tracked files"]); + + let repository = tracedecay_runtime_core::git_open::open(root.path()).unwrap(); + let before = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!(!before.has_changes()); + assert!(before.candidate_paths().contains("nested/foreign.rs")); + + fs::create_dir(root.path().join("nested/.git")).unwrap(); + fs::write( + root.path().join("nested/.git/HEAD"), + "ref: refs/heads/main\n", + ) + .unwrap(); + fs::write( + root.path().join("linked/.git"), + "gitdir: /untrusted/worktrees/linked\n", + ) + .unwrap(); + let hidden = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + let candidates = hidden.candidate_paths(); + assert!(candidates.contains("src/keep.rs")); + assert!(!candidates.contains("nested/foreign.rs")); + assert!(!candidates.contains("linked/foreign.rs")); + assert!(hidden.has_changes(), "a clean HEAD-tree capture is now unsafe"); + assert!(hidden.changed_paths().contains("nested/foreign.rs")); + assert!(hidden.changed_paths().contains("linked/foreign.rs")); + + fs::remove_dir_all(root.path().join("nested/.git")).unwrap(); + fs::remove_file(root.path().join("linked/.git")).unwrap(); + let restored = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!(!restored.has_changes()); + assert!(restored.candidate_paths().contains("nested/foreign.rs")); + assert!(restored.candidate_paths().contains("linked/foreign.rs")); + } +} diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 8392b549551..48639d5bebf 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -129,7 +129,19 @@ fn source_candidates( .map(|admission| admission.logical_path.as_str()) .collect::>(); let mut candidate_paths = classification.candidate_paths(); - candidate_paths.extend(admitted_paths.iter().map(|path| (*path).to_owned())); + let project_root = repository.workdir().ok_or_else(|| { + CodeIndexSchedulerErrorV1::Git("source freshness requires a worktree".to_owned()) + })?; + for path in &admitted_paths { + // Explicit ignored-file enrollment lifts Git ignore rules, not the + // nested repository boundary shared with capture and live search. + if !tracedecay_code_index::source_walk::has_nested_repository_ancestor( + project_root, + Path::new(path), + )? { + candidate_paths.insert((*path).to_owned()); + } + } let mut unrepresentable = classification .non_utf8_paths() .iter() diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 9ca91320c7e..e2d51bfeb61 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -3974,6 +3974,14 @@ impl CodeIndexWorktreeSchedulerV1 { control: Option<&dyn CodeIndexExecutionControlV1>, explicitly_admitted: bool, ) -> Result { + // Explicit ignored-source admission cannot cross a nested checkout + // any more than an ordinary Git candidate or interactive source walk. + if tracedecay_code_index::source_walk::has_nested_repository_ancestor( + &self.project_root, + std::path::Path::new(logical_path), + )? { + return Ok(CapturedFileOutcomeV1::Absent); + } let absolute = self.project_root.join(logical_path); match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index e360e4e6807..6a363aed0d2 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -10,7 +10,7 @@ //! crate reuses this policy instead of restating it. use std::collections::BTreeSet; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; use ignore::overrides::{Override, OverrideBuilder}; @@ -107,6 +107,52 @@ impl PathGlobScope { const MAX_REPOSITORY_NAMES: usize = 8; const MAX_REPOSITORY_PATH_CHARS: usize = 160; +fn nested_repository_marker(directory: &Path) -> std::io::Result { + match std::fs::symlink_metadata(directory.join(".git")) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +/// Reject a source beneath another checkout or a symlinked directory. +/// The root's own Git metadata is exempt. This boundary is shared by +/// interactive source walks and the indexed worktree candidate roster, +/// so neither may admit bytes the other refuses. +pub fn has_nested_repository_ancestor( + project_root: &Path, + relative_path: &Path, +) -> std::io::Result { + if relative_path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "source path must be repository-relative", + )); + } + + let mut parent = relative_path.parent(); + while let Some(relative_dir) = parent { + if relative_dir.as_os_str().is_empty() { + break; + } + let directory = project_root.join(relative_dir); + match std::fs::symlink_metadata(&directory) { + Ok(metadata) if !metadata.is_dir() => return Ok(true), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + if nested_repository_marker(&directory)? { + return Ok(true); + } + parent = relative_dir.parent(); + } + Ok(false) +} + /// Diagnostics emitted by the same source walker that enforces the nested /// repository boundary. Only directories within the walk's policy are named. #[derive(Clone, Default)] @@ -272,13 +318,18 @@ fn source_walk_inner( // freshness, so even registered submodules remain separate // sources. Linked worktrees and nested clones share this boundary. // The project root is deliberately exempt at depth zero above. - if std::fs::symlink_metadata(entry.path().join(".git")).is_ok() { - if let Some(diagnostics) = &diagnostics { - diagnostics.record_skipped_nested_repository(relative); + match nested_repository_marker(entry.path()) { + Ok(true) => { + if let Some(diagnostics) = &diagnostics { + diagnostics.record_skipped_nested_repository(relative); + } + false } - return false; + Ok(false) => true, + // The walk cannot prove that an unreadable Git marker is + // absent. Never admit files across that boundary. + Err(_) => false, } - true }); if let Some(overrides) = overrides { builder.overrides(overrides); @@ -328,7 +379,49 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::{source_walk, source_walk_with_diagnostics}; + use super::{ + has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics, + }; + + #[test] + fn nested_repository_boundary_excludes_parent_markers_and_path_escape() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("parent/nested/.git")).unwrap(); + fs::write(root.path().join("parent/nested/.git/HEAD"), "ref: refs/heads/main\n").unwrap(); + assert!( + !has_nested_repository_ancestor(root.path(), std::path::Path::new("parent/source.rs")) + .unwrap() + ); + assert!( + has_nested_repository_ancestor( + root.path(), + std::path::Path::new("parent/nested/source.rs") + ) + .unwrap() + ); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("../escape.rs")) + .is_err() + ); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("/absolute.rs")) + .is_err() + ); + } + + #[cfg(unix)] + #[test] + fn nested_repository_boundary_excludes_symlinked_ancestors() { + use std::os::unix::fs::symlink; + + let root = TempDir::new().unwrap(); + let external = TempDir::new().unwrap(); + symlink(external.path(), root.path().join("foreign")).unwrap(); + assert!( + has_nested_repository_ancestor(root.path(), std::path::Path::new("foreign/source.rs")) + .unwrap() + ); + } fn walked_files(root: &std::path::Path, path_policy: &IndexPathPolicyV1) -> Vec { let mut files = source_walk(root, None, path_policy) From 9beb7b89f6ae6ca0c03b0713e561d5d792745334 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:39:23 -0800 Subject: [PATCH 06/79] style(code-index): format nested repository boundary tests Apply four exact replacements from hosted Bazel rustfmt_check_2024 at PR #4801 head 624cc833; no behavioral source change. Refs #4593 --- .../src/code_index_scheduler/classification.rs | 8 +++++--- crates/tracedecay-code-index/src/source_walk.rs | 10 ++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index ab168be9965..f8ef32be303 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -396,8 +396,7 @@ mod nested_repository_tests { use super::WorktreeChangeClassificationV1; fn fixture_git(root: &Path, args: &[&str]) { - let program = - tracedecay_runtime_core::git::try_git_program().expect("fixture Git program"); + let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git program"); let mut command = Command::new(program); command.current_dir(root); for (key, _) in std::env::vars_os() { @@ -459,7 +458,10 @@ mod nested_repository_tests { assert!(candidates.contains("src/keep.rs")); assert!(!candidates.contains("nested/foreign.rs")); assert!(!candidates.contains("linked/foreign.rs")); - assert!(hidden.has_changes(), "a clean HEAD-tree capture is now unsafe"); + assert!( + hidden.has_changes(), + "a clean HEAD-tree capture is now unsafe" + ); assert!(hidden.changed_paths().contains("nested/foreign.rs")); assert!(hidden.changed_paths().contains("linked/foreign.rs")); diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 6a363aed0d2..515a679fce2 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -379,15 +379,17 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::{ - has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics, - }; + use super::{has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics}; #[test] fn nested_repository_boundary_excludes_parent_markers_and_path_escape() { let root = TempDir::new().unwrap(); fs::create_dir_all(root.path().join("parent/nested/.git")).unwrap(); - fs::write(root.path().join("parent/nested/.git/HEAD"), "ref: refs/heads/main\n").unwrap(); + fs::write( + root.path().join("parent/nested/.git/HEAD"), + "ref: refs/heads/main\n", + ) + .unwrap(); assert!( !has_nested_repository_ancestor(root.path(), std::path::Path::new("parent/source.rs")) .unwrap() From 2386440dfa50f47393d1f380f0d4f43c707e95c9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:43:28 -0800 Subject: [PATCH 07/79] fix(code-index): invalidate cached rosters on nested Git markers Sample .git marker evidence for tracked and explicitly admitted source ancestors even when Git dirwalk never recurses into tracked-only folders. Test marker creation and removal against a real Git parent index. Refs #4593 --- .../code_index_scheduler/freshness_witness.rs | 132 +++++++++++++++++- 1 file changed, 131 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 48639d5bebf..1e5e03e8601 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -490,6 +490,7 @@ fn roster_evidence( repository: &gix::Repository, project_root: &Path, sampled_at: ChangeClockReading, + ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], ) -> Option)>> { let mut recorder = DirectoryEvidenceV1 { root: project_root.to_path_buf(), @@ -513,6 +514,37 @@ fn roster_evidence( } recorder.record_directory(project_root.to_path_buf()); let index = repository.index_or_empty().ok()?; + // Git's directory walk may not descend into tracked-only directories. + // Their nested .git markers still change source admission without a + // parent index or HEAD update, so key each tracked and explicitly + // admitted source ancestor independently of dirwalk traversal. + let mut source_ancestors = BTreeSet::new(); + let mut collect_ancestors = |relative: &Path| { + let mut current = relative.parent(); + while let Some(parent) = current { + if parent.as_os_str().is_empty() { + break; + } + source_ancestors.insert(parent.to_path_buf()); + current = parent.parent(); + } + }; + for entry in index.entries() { + collect_ancestors(gix::path::from_bstr(entry.path(&index)).ok()?); + } + for admission in ignored_source_admissions { + if validate_code_logical_path(&admission.logical_path).is_err() { + return None; + } + collect_ancestors(Path::new(&admission.logical_path)); + } + for relative in source_ancestors { + let directory = project_root.join(relative); + recorder.record_directory(directory.clone()); + // A missing marker is evidence too: a newly created gitdir file or + // directory must invalidate the roster even if Git ignores it. + recorder.record(directory.join(".git"), false); + } let options = repository .dirwalk_options() .ok()? @@ -597,7 +629,12 @@ impl SourceSweepCacheV1 { let Ok(repository) = tracedecay_runtime_core::git_open::open(project_root) else { return (false, stats); }; - let evidence = roster_evidence(&repository, project_root, sampled_at); + let evidence = roster_evidence( + &repository, + project_root, + sampled_at, + ignored_source_admissions, + ); let Ok(SourceCandidatesV1 { candidates, unrepresentable, @@ -1029,3 +1066,96 @@ mod tests { assert_eq!(first, sample(&path, sampled_at).unwrap().unwrap()); } } + +#[cfg(all(test, unix))] +mod tracked_parent_boundary_tests { + use std::path::Path; + use std::sync::Arc; + use std::time::{Duration, Instant}; + + use super::{CachedCandidateRosterV1, ChangeClockReading, roster_evidence}; + + fn git(root: &Path, args: &[&str]) { + let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git"); + let mut command = std::process::Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run fixture Git"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn cached_tracked_directory_roster_invalidates_on_nested_git_marker() { + let root = tempfile::tempdir().expect("parent checkout"); + let tracked = root.path().join("src/nested/tracked.rs"); + std::fs::create_dir_all(tracked.parent().unwrap()).unwrap(); + std::fs::write(&tracked, "pub fn tracked() {}\n").unwrap(); + git(root.path(), &["init", "-q"]); + git(root.path(), &["add", "src/nested/tracked.rs"]); + git(root.path(), &["commit", "-qm", "parent source"]); + + let repository = + tracedecay_runtime_core::git_open::open(root.path()).expect("open parent Git"); + let deadline = Instant::now() + Duration::from_secs(5); + let evidence = loop { + if let Some(evidence) = + roster_evidence(&repository, root.path(), ChangeClockReading::now(), &[]) + { + break evidence; + } + assert!( + Instant::now() < deadline, + "tracked parent directories never became settleable" + ); + std::thread::sleep(Duration::from_millis(20)); + }; + let marker = root.path().join("src/nested/.git"); + assert!( + evidence + .iter() + .any(|(path, key)| path == &marker && key.is_none()), + "missing nested marker must be sampled independently of gix dirwalk" + ); + + let cached = CachedCandidateRosterV1 { + git_metadata_signature: "unchanged-parent-git".to_owned(), + admitted_paths: Vec::new(), + evidence, + candidates: Arc::new(Vec::new()), + unrepresentable: Arc::new(Vec::new()), + }; + assert!(cached.holds("unchanged-parent-git", &[])); + std::fs::write(&marker, "gitdir: ../../.git/modules/nested\n") + .expect("create nested gitdir marker"); + assert!( + !cached.holds("unchanged-parent-git", &[]), + "a new nested repository marker must invalidate the cached roster even when parent Git metadata is unchanged" + ); + std::fs::remove_file(&marker).unwrap(); + assert!( + cached.holds("unchanged-parent-git", &[]), + "the pre-marker evidence should be valid again after removing it" + ); + } +} From 0119cf9ff865c41a92c4a9ecb1cd09f15ea0322d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:44:27 -0800 Subject: [PATCH 08/79] test(code-index): assert roster invalidation across marker removal Directory metadata changes when the marker is removed, so the original cached roster must not be treated as restored merely by marker absence. Refs #4593 --- .../src/code_index_scheduler/freshness_witness.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 1e5e03e8601..8eb1ba4fe0d 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -1154,8 +1154,8 @@ mod tracked_parent_boundary_tests { ); std::fs::remove_file(&marker).unwrap(); assert!( - cached.holds("unchanged-parent-git", &[]), - "the pre-marker evidence should be valid again after removing it" + !cached.holds("unchanged-parent-git", &[]), + "removing the marker also changes the tracked ancestor directory evidence" ); } } From 704a9aac135046fe1d720242a482561a61eb5274 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:16:52 -0800 Subject: [PATCH 09/79] fix(code-index): refuse markerless parent gitlink directories A registered gitlink remains an untrusted nested source boundary even if its child .git marker is absent. Keep live grep and audit in default-deny parity with the parent index; include/glob cannot lift repository trust. Exercise a real registered Git submodule with its marker removed. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 515a679fce2..9e3f4e973b1 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -115,6 +115,52 @@ fn nested_repository_marker(directory: &Path) -> std::io::Result { } } +/// Default-deny all gitlinks in the parent's Git index even when a child +/// checkout's .git marker has been removed. A gitlink is a repository boundary, +/// not an ordinary directory the live source walker can authenticate. +/// This is deny-only; it never enables submodule admission. +fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { + let marker = project_root.join(".git"); + let repository = match tracedecay_runtime_core::git_open::open(project_root) { + Ok(repository) => repository, + Err(error) => { + // Non-Git projects still support source walks. An initialized + // checkout whose Git authority is unreadable must fail closed. + if marker.is_file() || marker.join("HEAD").is_file() { + return Err(SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git boundaries: {error}"), + }); + } + return Ok(BTreeSet::new()); + } + }; + if repository.workdir().and_then(|workdir| workdir.canonicalize().ok()) + != project_root.canonicalize().ok() + { + return Err(SourceWalkError { + glob: String::new(), + message: "source walk root is not the opened Git worktree".to_owned(), + }); + } + let index = repository.index_or_empty().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git index: {error}"), + })?; + let mut paths = BTreeSet::new(); + for entry in index.entries() { + if entry.mode.is_commit() { + let path: &[u8] = entry.path(&index).as_ref(); + let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { + glob: String::new(), + message: "non-UTF-8 parent gitlink cannot be safely scoped".to_owned(), + })?; + paths.insert(relative.to_owned()); + } + } + Ok(paths) +} + /// Reject a source beneath another checkout or a symlinked directory. /// The root's own Git metadata is exempt. This boundary is shared by /// interactive source walks and the indexed worktree candidate roster, @@ -260,6 +306,7 @@ fn source_walk_inner( let path_glob_scope = path_glob.and_then(PathGlobScope::from_path_glob); let filter_root = project_root.to_path_buf(); let path_policy = path_policy.clone(); + let parent_gitlinks = parent_gitlink_paths(project_root)?; let mut builder = WalkBuilder::new(project_root); builder @@ -318,6 +365,12 @@ fn source_walk_inner( // freshness, so even registered submodules remain separate // sources. Linked worktrees and nested clones share this boundary. // The project root is deliberately exempt at depth zero above. + if parent_gitlinks.contains(&relative) { + if let Some(diagnostics) = &diagnostics { + diagnostics.record_skipped_nested_repository(relative); + } + return false; + } match nested_repository_marker(entry.path()) { Ok(true) => { if let Some(diagnostics) = &diagnostics { @@ -668,6 +721,33 @@ mod tests { "{rule}: an explicit scope cannot cross repositories" ); } + + // A missing child .git marker must not turn the parent's stage-0 + // gitlink into ordinary live source under an explicit include. + fs::remove_file(root.join("app/.git")).expect("remove child gitdir marker"); + assert!(root.join("app/src/feature.rs").is_file()); + let policy = IndexPathPolicyV1::new(vec![], vec!["app/**".into()]).unwrap(); + let (walk, diagnostics) = + source_walk_with_diagnostics(&root, None, &policy).expect("source walk"); + let files = walk + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| entry.into_path()) + .collect::>(); + assert!( + files.iter().all(|path| !path.starts_with(root.join("app"))), + "a registered gitlink stays excluded without a child marker: {files:?}" + ); + assert_eq!( + diagnostics.skipped_nested_repositories_note().as_deref(), + Some("Skipped nested repositories outside parent index: \"app\"") + ); + let explicitly_scoped = source_walk(&root, Some("app/**"), &policy) + .expect("explicit gitlink scope") + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .count(); + assert_eq!(explicitly_scoped, 0); } #[test] From 1899d261c9c6486f572a0cbdf619d07315b65378 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:50:36 -0800 Subject: [PATCH 10/79] style(code-index): satisfy Bazel formatting for gitlink guard Apply the two exact rustfmt_check_2024 hunks from hosted run 42466514013. No source behavior changes; authenticated submodule indexing remains pending. Refs #4593 --- crates/tracedecay-code-index/src/source_walk.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 9e3f4e973b1..86056966076 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -135,7 +135,9 @@ fn parent_gitlink_paths(project_root: &Path) -> Result, SourceW return Ok(BTreeSet::new()); } }; - if repository.workdir().and_then(|workdir| workdir.canonicalize().ok()) + if repository + .workdir() + .and_then(|workdir| workdir.canonicalize().ok()) != project_root.canonicalize().ok() { return Err(SourceWalkError { @@ -143,10 +145,12 @@ fn parent_gitlink_paths(project_root: &Path) -> Result, SourceW message: "source walk root is not the opened Git worktree".to_owned(), }); } - let index = repository.index_or_empty().map_err(|error| SourceWalkError { - glob: String::new(), - message: format!("cannot inspect parent Git index: {error}"), - })?; + let index = repository + .index_or_empty() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git index: {error}"), + })?; let mut paths = BTreeSet::new(); for entry in index.entries() { if entry.mode.is_commit() { From 77332ede0f9b9cb557bd41185203eba9500c3f0a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 04:54:15 -0800 Subject: [PATCH 11/79] fix(code-index): deny ignored admission beneath Git gitlinks A parent stage-0 gitlink remains tracked repository source even when its child checkout loses the .git marker. Check every indexed ancestor before an ignored dependency is admitted or revalidated, preserving the existing proof for ordinary ignored sources. Add a real local Git submodule regression that removes the child marker and checks both the admission proof and production source-read refusal. Refs #4593 --- .../ignored_dependencies.rs | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index a5475fc5906..089b3f80d12 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -861,6 +861,20 @@ fn prove_gix_ignored(root: &Path, logical_path: &str) -> Result<(), CodeIndexSch { return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); } + // A registered parent gitlink is still tracked source when the child + // checkout loses its .git marker. Ignore rules may match its descendants, + // but that cannot authorize indexing those descendants as dependencies. + let mut ancestor = Path::new(logical_path).parent(); + while let Some(relative) = ancestor { + if relative + .to_str() + .and_then(|path| index.entry_by_path(path.as_bytes().as_bstr())) + .is_some_and(|entry| entry.mode.is_commit()) + { + return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); + } + ancestor = relative.parent(); + } let mut excludes = repository .excludes( &index, @@ -933,3 +947,111 @@ fn map_reconcile_interruption_result( ) -> Result { result.map_err(map_reconcile_interruption) } + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::Path; + use std::process::{Command, Output}; + + use super::{ + CodeIndexIgnoredDependencyRefusalV1, CodeIndexSchedulerErrorV1, prove_gix_ignored, + read_bounded_admitted_source, + }; + + fn fixture_git(root: &Path, args: &[&str]) -> Output { + let program = tracedecay_runtime_core::git::try_git_program().expect("Git program"); + let mut command = Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run fixture Git"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + output + } + + #[test] + fn parent_gitlink_cannot_be_admitted_as_ignored_dependency_without_gitdir_marker() { + let fixture = tempfile::tempdir().expect("repository fixture"); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + + fixture_git(&child, &["init", "-q"]); + fs::write(child.join("index.ts"), "export type Child = number;\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + + fixture_git(&parent, &["init", "-q"]); + fs::write(parent.join(".gitignore"), "node_modules/\n").unwrap(); + fixture_git(&parent, &["add", ".gitignore"]); + fixture_git(&parent, &["commit", "-qm", "parent"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-f", + "-q", + "../child", + "node_modules/pkg", + ], + ); + + let staged = fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]); + assert!(staged.stdout.starts_with(b"160000 ")); + assert!(staged.stdout.ends_with(b" 0\tnode_modules/pkg\n")); + assert!(parent.join(".gitmodules").is_file()); + + fs::remove_file(parent.join("node_modules/pkg/.git")).unwrap(); + assert!(parent.join("node_modules/pkg/index.ts").is_file()); + let denied = prove_gix_ignored(&parent, "node_modules/pkg/index.ts"); + assert!(matches!( + denied, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + let admitted = read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); + assert!(matches!( + admitted, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + + // Ordinary ignored sources are still eligible; only gitlink ancestry + // withdraws the source trust that the ignored-dependency proof needs. + fs::create_dir_all(parent.join("node_modules/loose")).unwrap(); + fs::write( + parent.join("node_modules/loose/index.ts"), + "export type Loose = string;\n", + ) + .unwrap(); + prove_gix_ignored(&parent, "node_modules/loose/index.ts") + .expect("non-repository ignored source remains admissible"); + } +} From 2b230e8232f6ffd23675223c261fcfbd0a26e80a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:25:50 -0800 Subject: [PATCH 12/79] fix(code-index): preserve HEAD gitlink boundary on staged removal Refs #4593. Full authenticated child indexing still incomplete. --- .../tracedecay-code-index/src/source_walk.rs | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 86056966076..d0d3c932ab4 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -162,6 +162,47 @@ fn parent_gitlink_paths(project_root: &Path) -> Result, SourceW paths.insert(relative.to_owned()); } } + // A staged gitlink removal does not erase the checkout boundary from + // HEAD. Without this second authority, a child whose .git marker was + // removed becomes ordinary live source until the parent commits. + let head = repository.head().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git HEAD: {error}"), + })?; + if head.id().is_some() { + let commit = repository.head_commit().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot resolve parent Git HEAD: {error}"), + })?; + let tree = commit.tree().map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot read parent Git HEAD tree: {error}"), + })?; + let entries = tree + .traverse() + .breadthfirst + .files() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot walk parent Git HEAD tree: {error}"), + })?; + for entry in entries { + if !entry.mode.is_commit() { + continue; + } + let path: &[u8] = entry.filepath.as_ref(); + let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { + glob: String::new(), + message: "non-UTF-8 HEAD gitlink cannot be safely scoped".to_owned(), + })?; + paths.insert(relative.to_owned()); + } + } else if head.referent_name().is_none() { + return Err(SourceWalkError { + glob: String::new(), + message: "parent Git HEAD has neither a commit nor an unborn branch".to_owned(), + }); + } Ok(paths) } @@ -752,6 +793,36 @@ mod tests { .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) .count(); assert_eq!(explicitly_scoped, 0); + + // Staging removal of the gitlink also removes it from the parent + // index, but HEAD still owns that repository boundary. Neither an + // explicit include nor a path glob may adopt its live child bytes. + fixture_git(&root, &["add", ".gitmodules", "src/parent.rs"]); + fixture_git(&root, &["commit", "-qm", "parent with submodule"]); + fixture_git(&root, &["rm", "--cached", "-f", "app"]); + assert!( + fixture_git(&root, &["ls-files", "--stage", "--", "app"]) + .stdout + .is_empty() + ); + assert!( + fixture_git(&root, &["ls-tree", "HEAD", "--", "app"]) + .stdout + .starts_with(b"160000 commit ") + ); + assert!(root.join("app/src/feature.rs").is_file()); + + let files = walked_files(&root, &policy); + assert!( + files.iter().all(|path| !path.starts_with("app")), + "HEAD gitlink is still an excluded source boundary: {files:?}" + ); + let scoped = source_walk(&root, Some("app/**"), &policy) + .expect("scope cannot cross HEAD gitlink") + .map(|entry| entry.expect("source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .count(); + assert_eq!(scoped, 0); } #[test] From f925fe659de81b368a6bd6c2cb40835cb472f179 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:07:45 -0800 Subject: [PATCH 13/79] fix(code-index): deny staged-removal gitlink in source admission Refs #4593. Reuse the parent's index-plus-HEAD boundary for ignored sources and cover the real staged-removal checkout scenario. The authenticated opt-in indexing feature remains incomplete. --- .../ignored_dependencies.rs | 42 ++++++++++++++++--- .../tracedecay-code-index/src/source_walk.rs | 2 +- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 089b3f80d12..709e72145e6 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -14,6 +14,7 @@ use tracedecay_code_index::production::{ CodeIndexInterruptionV1, CodeIndexProductionErrorV1, MAX_IGNORED_DEPENDENCY_ENTRYPOINT_BYTES_V1, }; +use tracedecay_code_index::source_walk::parent_gitlink_paths; use tracedecay_contracts::ResolvedScope; use tracedecay_contracts::code_index_freshness::CodeIndexConvergenceParkedV1; use tracedecay_domain::{CodeGenerationId, SanitizerDispositionV1}; @@ -861,15 +862,15 @@ fn prove_gix_ignored(root: &Path, logical_path: &str) -> Result<(), CodeIndexSch { return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); } - // A registered parent gitlink is still tracked source when the child - // checkout loses its .git marker. Ignore rules may match its descendants, - // but that cannot authorize indexing those descendants as dependencies. + // Admission and live source walks share the union of staged and HEAD + // gitlinks: staging a removal cannot grant trust over the child checkout. + let parent_gitlinks = parent_gitlink_paths(root) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; let mut ancestor = Path::new(logical_path).parent(); while let Some(relative) = ancestor { if relative .to_str() - .and_then(|path| index.entry_by_path(path.as_bytes().as_bstr())) - .is_some_and(|entry| entry.mode.is_commit()) + .is_some_and(|path| parent_gitlinks.contains(path)) { return Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored.into()); } @@ -1043,6 +1044,37 @@ mod tests { )) )); + // Removing the staged gitlink does not authorize the child while + // the parent HEAD still owns the repository boundary. + fixture_git(&parent, &["add", ".gitmodules", ".gitignore"]); + fixture_git(&parent, &["commit", "-qm", "parent with submodule"]); + fixture_git(&parent, &["rm", "--cached", "-f", "node_modules/pkg"]); + assert!( + fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]) + .stdout + .is_empty() + ); + assert!( + fixture_git(&parent, &["ls-tree", "HEAD", "--", "node_modules/pkg"]) + .stdout + .starts_with(b"160000 commit ") + ); + let denied = prove_gix_ignored(&parent, "node_modules/pkg/index.ts"); + assert!(matches!( + denied, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + let denied_read = + read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); + assert!(matches!( + denied_read, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + )); + // Ordinary ignored sources are still eligible; only gitlink ancestry // withdraws the source trust that the ignored-dependency proof needs. fs::create_dir_all(parent.join("node_modules/loose")).unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index d0d3c932ab4..8d49b8dc925 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -119,7 +119,7 @@ fn nested_repository_marker(directory: &Path) -> std::io::Result { /// checkout's .git marker has been removed. A gitlink is a repository boundary, /// not an ordinary directory the live source walker can authenticate. /// This is deny-only; it never enables submodule admission. -fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { +pub fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { let marker = project_root.join(".git"); let repository = match tracedecay_runtime_core::git_open::open(project_root) { Ok(repository) => repository, From bcd2b46a77824ebeb7dd0fbe6806336c1f21fa13 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:52:59 -0800 Subject: [PATCH 14/79] style(code-index): fix ignored gitlink regression formatting Refs #4593 --- .../src/code_index_scheduler/ignored_dependencies.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 709e72145e6..cfae3260dc3 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -1066,8 +1066,7 @@ mod tests { CodeIndexIgnoredDependencyRefusalV1::NotIgnored )) )); - let denied_read = - read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); + let denied_read = read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); assert!(matches!( denied_read, Err(CodeIndexSchedulerErrorV1::IgnoredDependency( From 5edd6c6cb4915da74f5f7a392e340b364064710f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:56:52 -0800 Subject: [PATCH 15/79] fix(code-index): preserve HEAD gitlink boundaries in classification Refs #4593 --- .../code_index_scheduler/classification.rs | 84 +++++++++++++++++-- 1 file changed, 77 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index f8ef32be303..c6c110eba04 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -18,7 +18,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use gix::bstr::ByteSlice; -use tracedecay_code_index::source_walk::has_nested_repository_ancestor; +use tracedecay_code_index::source_walk::{has_nested_repository_ancestor, parent_gitlink_paths}; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -215,6 +215,11 @@ impl WorktreeChangeClassificationV1 { // parent directory once rather than probing per source file. let mut nested_repository_exclusions = BTreeSet::new(); if let Some(project_root) = repository.workdir() { + // The live walker also denies HEAD gitlinks staged for removal. + // Without this check, Git reports their markerless child sources + // as untracked, and worktree capture can re-admit those bytes. + let parent_gitlinks = parent_gitlink_paths(project_root) + .map_err(|error| ClassificationErrorV1::Git(error.message))?; let mut checked_parents = BTreeMap::new(); for path in committed_baseline .iter() @@ -222,12 +227,20 @@ impl WorktreeChangeClassificationV1 { { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - let excluded = match checked_parents.get(parent) { - Some(excluded) => *excluded, - None => { - let excluded = has_nested_repository_ancestor(project_root, relative)?; - checked_parents.insert(parent.to_path_buf(), excluded); - excluded + let excluded = if relative + .ancestors() + .filter_map(Path::to_str) + .any(|ancestor| parent_gitlinks.contains(ancestor)) + { + true + } else { + match checked_parents.get(parent) { + Some(excluded) => *excluded, + None => { + let excluded = has_nested_repository_ancestor(project_root, relative)?; + checked_parents.insert(parent.to_path_buf(), excluded); + excluded + } } }; if excluded { @@ -472,4 +485,61 @@ mod nested_repository_tests { assert!(restored.candidate_paths().contains("nested/foreign.rs")); assert!(restored.candidate_paths().contains("linked/foreign.rs")); } + + #[test] + fn staged_gitlink_removal_without_child_marker_does_not_admit_child_sources() { + let fixture = tempfile::tempdir().unwrap(); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + + fixture_git(&child, &["init", "-q"]); + fs::create_dir_all(child.join("src")).unwrap(); + fs::write(child.join("src/lib.rs"), "pub fn child() {}\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + + fixture_git(&parent, &["init", "-q"]); + fs::create_dir_all(parent.join("src")).unwrap(); + fs::write(parent.join("src/lib.rs"), "pub fn parent() {}\n").unwrap(); + fixture_git(&parent, &["add", "."]); + fixture_git(&parent, &["commit", "-qm", "parent"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + fixture_git(&parent, &["add", ".gitmodules", "app"]); + fixture_git(&parent, &["commit", "-qm", "register submodule"]); + + // Git still records app as a mode-160000 gitlink in HEAD, but + // removing the staged gitlink and child marker makes child files + // appear untracked to the parent. They must never enter its index. + fs::remove_file(parent.join("app/.git")).unwrap(); + fixture_git(&parent, &["rm", "-q", "--cached", "app"]); + assert!(parent.join("app/src/lib.rs").is_file()); + + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!( + classified + .changes() + .iter() + .any(|change| change.path == "app/src/lib.rs"), + "fixture must exercise a child file reported by parent Git status" + ); + assert!(classified.has_changes(), "clean HEAD reuse must be refused"); + assert!(!classified.candidate_paths().contains("app/src/lib.rs")); + assert!(classified.changed_paths().contains("app/src/lib.rs")); + assert!(classified.candidate_paths().contains("src/lib.rs")); + } + } From d31966d29f5630b479168dafad49ae162ba96c27 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:58:03 -0800 Subject: [PATCH 16/79] fix(code-index): preserve clean registered gitlink state Refs #4593 --- .../src/code_index_scheduler/classification.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index c6c110eba04..f5dee93d95c 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -229,6 +229,7 @@ impl WorktreeChangeClassificationV1 { let parent = relative.parent().unwrap_or_else(|| Path::new("")); let excluded = if relative .ancestors() + .skip(1) .filter_map(Path::to_str) .any(|ancestor| parent_gitlinks.contains(ancestor)) { @@ -520,6 +521,13 @@ mod nested_repository_tests { fixture_git(&parent, &["add", ".gitmodules", "app"]); fixture_git(&parent, &["commit", "-qm", "register submodule"]); + // A registered, clean gitlink is not itself a source file or a + // dirty nested descendant. It must not defeat clean HEAD reuse. + let clean_repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let clean = WorktreeChangeClassificationV1::classify(&clean_repository).unwrap(); + assert!(!clean.has_changes()); + assert!(clean.candidate_paths().contains("src/lib.rs")); + // Git still records app as a mode-160000 gitlink in HEAD, but // removing the staged gitlink and child marker makes child files // appear untracked to the parent. They must never enter its index. @@ -541,5 +549,4 @@ mod nested_repository_tests { assert!(classified.changed_paths().contains("app/src/lib.rs")); assert!(classified.candidate_paths().contains("src/lib.rs")); } - } From a04503a06d1d98b458e86d8499ba9f7d557ca60d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:53:08 -0800 Subject: [PATCH 17/79] fix(code-index): deny gitlink children in freshness roster Refs #4593. Do not merge until integrated submodule admission and Bazel regression acceptance. --- .../code_index_scheduler/freshness_witness.rs | 98 +++++++++++++++++-- 1 file changed, 90 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 8eb1ba4fe0d..28f60ca5431 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -132,15 +132,27 @@ fn source_candidates( let project_root = repository.workdir().ok_or_else(|| { CodeIndexSchedulerErrorV1::Git("source freshness requires a worktree".to_owned()) })?; + // Explicit ignored-file enrollment lifts Git ignore rules, never the + // parent's index/HEAD gitlink boundary. A staged gitlink removal and + // missing child .git marker must not re-admit its files through freshness. + let parent_gitlinks = tracedecay_code_index::source_walk::parent_gitlink_paths(project_root) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; for path in &admitted_paths { - // Explicit ignored-file enrollment lifts Git ignore rules, not the - // nested repository boundary shared with capture and live search. - if !tracedecay_code_index::source_walk::has_nested_repository_ancestor( - project_root, - Path::new(path), - )? { - candidate_paths.insert((*path).to_owned()); + let relative = Path::new(path); + let beneath_gitlink = relative + .ancestors() + .skip(1) + .filter_map(Path::to_str) + .any(|ancestor| parent_gitlinks.contains(ancestor)); + if beneath_gitlink + || tracedecay_code_index::source_walk::has_nested_repository_ancestor( + project_root, + relative, + )? + { + continue; } + candidate_paths.insert((*path).to_owned()); } let mut unrepresentable = classification .non_utf8_paths() @@ -1073,7 +1085,10 @@ mod tracked_parent_boundary_tests { use std::sync::Arc; use std::time::{Duration, Instant}; - use super::{CachedCandidateRosterV1, ChangeClockReading, roster_evidence}; + use super::{ + CachedCandidateRosterV1, ChangeClockReading, CodeIndexIgnoredSourceAdmissionV1, + roster_evidence, source_candidates, + }; fn git(root: &Path, args: &[&str]) { let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git"); @@ -1105,6 +1120,73 @@ mod tracked_parent_boundary_tests { ); } + #[test] + fn ignored_source_cannot_reenter_markerless_parent_gitlink_freshness() { + let fixture = tempfile::tempdir().expect("Git fixture"); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + std::fs::create_dir_all(&parent).unwrap(); + std::fs::create_dir_all(&child).unwrap(); + + git(&child, &["init", "-q"]); + std::fs::write(child.join("child.rs"), "pub fn child() {}\n").unwrap(); + git(&child, &["add", "child.rs"]); + git(&child, &["commit", "-qm", "child"]); + + git(&parent, &["init", "-q"]); + std::fs::create_dir_all(parent.join("ignored")).unwrap(); + std::fs::write(parent.join(".gitignore"), "ignored/\n").unwrap(); + std::fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); + std::fs::write( + parent.join("ignored/ordinary.rs"), + "pub fn ordinary() {}\n", + ) + .unwrap(); + git(&parent, &["add", ".gitignore", "parent.rs"]); + git(&parent, &["commit", "-qm", "parent"]); + git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + git(&parent, &["add", ".gitmodules", "app"]); + git(&parent, &["commit", "-qm", "registered submodule"]); + + // Git still records the child gitlink in HEAD after staging its + // removal. Its missing marker makes child.rs appear untracked, but + // an old explicit ignored admission must not revive it in freshness. + std::fs::remove_file(parent.join("app/.git")).unwrap(); + git(&parent, &["rm", "-q", "--cached", "app"]); + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let admitted = [ + CodeIndexIgnoredSourceAdmissionV1 { + logical_path: "app/child.rs".to_owned(), + }, + CodeIndexIgnoredSourceAdmissionV1 { + logical_path: "ignored/ordinary.rs".to_owned(), + }, + ]; + let candidates = source_candidates(&repository, &admitted).unwrap(); + let paths = candidates + .candidates + .iter() + .map(|candidate| candidate.logical_path.as_str()) + .collect::>(); + assert!(paths.contains("parent.rs")); + assert!(paths.contains("ignored/ordinary.rs")); + assert!( + !paths.contains("app/child.rs"), + "ignored admission must not cross a HEAD gitlink boundary" + ); + } + #[test] fn cached_tracked_directory_roster_invalidates_on_nested_git_marker() { let root = tempfile::tempdir().expect("parent checkout"); From d6e0edfe128b74abab74a255185729ca827129d6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:20:49 -0800 Subject: [PATCH 18/79] style(code-index): format nested repository fixture --- .../src/code_index_scheduler/freshness_witness.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 28f60ca5431..61606433316 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -1137,11 +1137,7 @@ mod tracked_parent_boundary_tests { std::fs::create_dir_all(parent.join("ignored")).unwrap(); std::fs::write(parent.join(".gitignore"), "ignored/\n").unwrap(); std::fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); - std::fs::write( - parent.join("ignored/ordinary.rs"), - "pub fn ordinary() {}\n", - ) - .unwrap(); + std::fs::write(parent.join("ignored/ordinary.rs"), "pub fn ordinary() {}\n").unwrap(); git(&parent, &["add", ".gitignore", "parent.rs"]); git(&parent, &["commit", "-qm", "parent"]); git( From 1bbf847d0340238b91df20188278caec6d8fa6ea Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:26:06 -0800 Subject: [PATCH 19/79] fix(code-index): reuse verified Git boundaries during freshness --- .../code_index_scheduler/classification.rs | 44 +++++++++++-------- .../code_index_scheduler/freshness_witness.rs | 9 +--- .../tracedecay-code-index/src/source_walk.rs | 19 ++++---- .../source_walk/nested_repository_tests.rs | 14 ++++++ 4 files changed, 52 insertions(+), 34 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index f5dee93d95c..b69b8c16d46 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -97,6 +97,7 @@ pub struct WorktreeChangeClassificationV1 { /// Files still tracked by the parent but hidden behind a nested checkout /// or symlinked ancestor. These are tombstones, not indexable candidates. nested_repository_exclusions: BTreeSet, + parent_gitlinks: BTreeSet, } impl WorktreeChangeClassificationV1 { @@ -213,26 +214,28 @@ impl WorktreeChangeClassificationV1 { // refuses those files; classification must also exclude them and // invalidate clean-tree and active-generation reuse. Resolve each // parent directory once rather than probing per source file. - let mut nested_repository_exclusions = BTreeSet::new(); + let mut classification = Self { + committed_baseline, + changes, + non_utf8_changes, + non_utf8_paths, + ..Self::default() + }; if let Some(project_root) = repository.workdir() { // The live walker also denies HEAD gitlinks staged for removal. // Without this check, Git reports their markerless child sources // as untracked, and worktree capture can re-admit those bytes. - let parent_gitlinks = parent_gitlink_paths(project_root) + classification.parent_gitlinks = parent_gitlink_paths(project_root) .map_err(|error| ClassificationErrorV1::Git(error.message))?; let mut checked_parents = BTreeMap::new(); - for path in committed_baseline + for path in classification + .committed_baseline .iter() - .chain(changes.iter().map(|change| &change.path)) + .chain(classification.changes.iter().map(|change| &change.path)) { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - let excluded = if relative - .ancestors() - .skip(1) - .filter_map(Path::to_str) - .any(|ancestor| parent_gitlinks.contains(ancestor)) - { + let excluded = if classification.beneath_gitlink(relative) { true } else { match checked_parents.get(parent) { @@ -245,18 +248,23 @@ impl WorktreeChangeClassificationV1 { } }; if excluded { - nested_repository_exclusions.insert(path.clone()); + classification + .nested_repository_exclusions + .insert(path.clone()); } } } - Ok(Self { - committed_baseline, - changes, - non_utf8_changes, - non_utf8_paths, - nested_repository_exclusions, - }) + Ok(classification) + } + + /// Explicitly enrolled files must respect the same Git boundary snapshot + /// as ordinary candidates, including a gitlink staged for removal. + pub(super) fn beneath_gitlink(&self, path: &Path) -> bool { + path.ancestors() + .skip(1) + .filter_map(Path::to_str) + .any(|ancestor| self.parent_gitlinks.contains(ancestor)) } /// Paths whose Git bytes are not UTF-8, present or not. diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 61606433316..c0edd6eb308 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -135,16 +135,9 @@ fn source_candidates( // Explicit ignored-file enrollment lifts Git ignore rules, never the // parent's index/HEAD gitlink boundary. A staged gitlink removal and // missing child .git marker must not re-admit its files through freshness. - let parent_gitlinks = tracedecay_code_index::source_walk::parent_gitlink_paths(project_root) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; for path in &admitted_paths { let relative = Path::new(path); - let beneath_gitlink = relative - .ancestors() - .skip(1) - .filter_map(Path::to_str) - .any(|ancestor| parent_gitlinks.contains(ancestor)); - if beneath_gitlink + if classification.beneath_gitlink(relative) || tracedecay_code_index::source_walk::has_nested_repository_ancestor( project_root, relative, diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 8d49b8dc925..bc41a8beb7d 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -120,19 +120,22 @@ fn nested_repository_marker(directory: &Path) -> std::io::Result { /// not an ordinary directory the live source walker can authenticate. /// This is deny-only; it never enables submodule admission. pub fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { - let marker = project_root.join(".git"); let repository = match tracedecay_runtime_core::git_open::open(project_root) { Ok(repository) => repository, Err(error) => { // Non-Git projects still support source walks. An initialized // checkout whose Git authority is unreadable must fail closed. - if marker.is_file() || marker.join("HEAD").is_file() { - return Err(SourceWalkError { + return match nested_repository_marker(project_root) { + Ok(false) => Ok(BTreeSet::new()), + Ok(true) => Err(SourceWalkError { glob: String::new(), message: format!("cannot inspect parent Git boundaries: {error}"), - }); - } - return Ok(BTreeSet::new()); + }), + Err(marker_error) => Err(SourceWalkError { + glob: String::new(), + message: format!("cannot inspect parent Git marker: {marker_error}"), + }), + }; } }; if repository @@ -642,7 +645,7 @@ mod tests { #[test] fn nested_repository_is_not_part_of_the_parent_source_tree() { let root = TempDir::new().expect("project root"); - fs::create_dir(root.path().join(".git")).expect("parent git directory"); + fixture_git(root.path(), &["init", "-q"]); fs::create_dir_all(root.path().join("src")).expect("parent source directory"); fs::write(root.path().join("src/lib.rs"), "pub fn parent() {}\n").expect("parent source"); fs::create_dir_all(root.path().join("plain")).expect("plain directory"); @@ -828,7 +831,7 @@ mod tests { #[test] fn submodule_metadata_cannot_break_an_unrelated_source_include() { let root = TempDir::new().expect("project root"); - fs::create_dir_all(root.path().join(".git")).unwrap(); + fixture_git(root.path(), &["init", "-q"]); fs::create_dir_all(root.path().join("src")).unwrap(); fs::create_dir_all(root.path().join("app/.git")).unwrap(); fs::write(root.path().join("src/keep.rs"), "pub fn keep() {}\n").unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs index 0b77991f58e..406671e045d 100644 --- a/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/nested_repository_tests.rs @@ -181,3 +181,17 @@ fn missing_source_root_is_an_error_not_a_clean_audit() { Ok(_) => panic!("a failed walk must not report an empty successful audit"), } } + +#[test] +fn incomplete_git_root_is_refused_before_walking_sources() { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "src/lib.rs", "pub fn source() {}"); + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + assert!(audit_project(root.path(), &policy).is_ok()); + + std::fs::create_dir(root.path().join(".git")).unwrap(); + assert!(matches!( + audit_project(root.path(), &policy), + Err(TraceDecayError::Config { message }) if message.contains("parent Git boundaries") + )); +} From dd1ddca5b0fdcb5220e81ac7fa56187410a95b4e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:49:52 -0800 Subject: [PATCH 20/79] fix(code-index): recognize submodule modes in the Git index --- crates/tracedecay-code-index/src/source_walk.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index bc41a8beb7d..d8e5ca885fc 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -156,7 +156,7 @@ pub fn parent_gitlink_paths(project_root: &Path) -> Result, Sou })?; let mut paths = BTreeSet::new(); for entry in index.entries() { - if entry.mode.is_commit() { + if entry.mode.is_submodule() { let path: &[u8] = entry.path(&index).as_ref(); let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { glob: String::new(), From 1f7b37aa41da122818ed3ea67b75657bc9266f00 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:55:55 -0800 Subject: [PATCH 21/79] fix(code-index): borrow Git paths for roster freshness checks --- .../src/code_index_scheduler/freshness_witness.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index c0edd6eb308..721491ddc45 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -535,7 +535,7 @@ fn roster_evidence( } }; for entry in index.entries() { - collect_ancestors(gix::path::from_bstr(entry.path(&index)).ok()?); + collect_ancestors(gix::path::from_bstr(entry.path(&index)).ok()?.as_ref()); } for admission in ignored_source_admissions { if validate_code_logical_path(&admission.logical_path).is_err() { From 5eaa4993902f1f9dd545eb92b4aed304737f2a36 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:59:40 -0800 Subject: [PATCH 22/79] test(code-index): bind ignored reads to the canonical fixture root --- .../ignored_dependencies.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index cfae3260dc3..2819eb78153 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -955,6 +955,8 @@ mod tests { use std::path::Path; use std::process::{Command, Output}; + use tracedecay_runtime_core::path_safety::canonical_existing_identity; + use super::{ CodeIndexIgnoredDependencyRefusalV1, CodeIndexSchedulerErrorV1, prove_gix_ignored, read_bounded_admitted_source, @@ -998,6 +1000,7 @@ mod tests { let child = fixture.path().join("child"); fs::create_dir_all(&parent).unwrap(); fs::create_dir_all(&child).unwrap(); + let parent = canonical_existing_identity(&parent).expect("canonical project root"); fixture_git(&child, &["init", "-q"]); fs::write(child.join("index.ts"), "export type Child = number;\n").unwrap(); @@ -1037,12 +1040,15 @@ mod tests { )) )); let admitted = read_bounded_admitted_source(&parent, "node_modules/pkg/index.ts", None); - assert!(matches!( - admitted, - Err(CodeIndexSchedulerErrorV1::IgnoredDependency( - CodeIndexIgnoredDependencyRefusalV1::NotIgnored - )) - )); + assert!( + matches!( + admitted, + Err(CodeIndexSchedulerErrorV1::IgnoredDependency( + CodeIndexIgnoredDependencyRefusalV1::NotIgnored + )) + ), + "unexpected source admission: {admitted:?}" + ); // Removing the staged gitlink does not authorize the child while // the parent HEAD still owns the repository boundary. From bc7c1fc9edbee03318afd4d408374b0c3cb5adca Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:55:19 -0800 Subject: [PATCH 23/79] fix(code-index): refuse symlinked source leaf capture Refs #4593. Preserve live-walk exclusion through classification, worktree capture, and freshness even when a tracked symlink resolves into a registered child repository. Adds real Git submodule and symlink regressions. Partial boundary repair; keep PR draft. --- .../code_index_scheduler/classification.rs | 63 ++++++++++++++++++- .../code_index_scheduler/freshness_witness.rs | 13 ++++ .../src/code_index_scheduler/reconcile.rs | 6 +- .../tracedecay-code-index/src/source_walk.rs | 40 +++++++++++- 4 files changed, 118 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs index b69b8c16d46..0550c4a843c 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs @@ -18,7 +18,9 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use gix::bstr::ByteSlice; -use tracedecay_code_index::source_walk::{has_nested_repository_ancestor, parent_gitlink_paths}; +use tracedecay_code_index::source_walk::{ + has_nested_repository_ancestor, is_source_symlink, parent_gitlink_paths, +}; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -235,7 +237,13 @@ impl WorktreeChangeClassificationV1 { { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - let excluded = if classification.beneath_gitlink(relative) { + // The live walker never follows a symlink file, even when + // the lexical parent is ordinary and the target is inside a + // registered submodule. Do not capture its target through a + // clean-tree or worktree path. + let excluded = if classification.beneath_gitlink(relative) + || is_source_symlink(project_root, relative)? + { true } else { match checked_parents.get(parent) { @@ -495,6 +503,57 @@ mod nested_repository_tests { assert!(restored.candidate_paths().contains("linked/foreign.rs")); } + #[cfg(unix)] + #[test] + fn tracked_symlink_into_submodule_is_not_parent_source() { + use std::os::unix::fs::symlink; + + let fixture = tempfile::tempdir().unwrap(); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(&child).unwrap(); + fixture_git(&child, &["init", "-q"]); + fs::create_dir_all(child.join("src")).unwrap(); + fs::write(child.join("src/lib.rs"), "pub fn child() {}\n").unwrap(); + fixture_git(&child, &["add", "."]); + fixture_git(&child, &["commit", "-qm", "child"]); + fixture_git(&parent, &["init", "-q"]); + fixture_git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + symlink("app/src/lib.rs", parent.join("bridge.rs")).unwrap(); + fixture_git(&parent, &["add", ".gitmodules", "app", "bridge.rs"]); + fixture_git(&parent, &["commit", "-qm", "registered child and source alias"]); + assert_eq!( + fs::read_to_string(parent.join("bridge.rs")).unwrap(), + "pub fn child() {}\n", + "fixture must prove the parent symlink actually reaches child bytes" + ); + assert!( + fs::symlink_metadata(parent.join("bridge.rs")) + .unwrap() + .file_type() + .is_symlink() + ); + + let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); + let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); + assert!(!classified.candidate_paths().contains("bridge.rs")); + assert!(classified.changed_paths().contains("bridge.rs")); + assert!(classified.has_changes(), "do not reuse HEAD tree source bytes"); + assert!(!classified.candidate_paths().contains("app/src/lib.rs")); + } + #[test] fn staged_gitlink_removal_without_child_marker_does_not_admit_child_sources() { let fixture = tempfile::tempdir().unwrap(); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 721491ddc45..d81c425e9e3 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -142,6 +142,7 @@ fn source_candidates( project_root, relative, )? + || tracedecay_code_index::source_walk::is_source_symlink(project_root, relative)? { continue; } @@ -243,6 +244,18 @@ fn read_candidate( project_root: &Path, candidate: &StatCandidateV1, ) -> Result, CodeIndexSchedulerErrorV1> { + // A link installed after candidate classification must not turn a + // freshness proof into a read of a different repository's source bytes. + if tracedecay_code_index::source_walk::is_source_symlink( + project_root, + Path::new(&candidate.logical_path), + )? { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "source symlink is outside the parent source inventory", + ) + .into()); + } if candidate.explicitly_admitted { ignored_dependencies::read_explicitly_admitted_source( project_root, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index e2d51bfeb61..5724543efff 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -3976,9 +3976,13 @@ impl CodeIndexWorktreeSchedulerV1 { ) -> Result { // Explicit ignored-source admission cannot cross a nested checkout // any more than an ordinary Git candidate or interactive source walk. + let relative = std::path::Path::new(logical_path); if tracedecay_code_index::source_walk::has_nested_repository_ancestor( &self.project_root, - std::path::Path::new(logical_path), + relative, + )? || tracedecay_code_index::source_walk::is_source_symlink( + &self.project_root, + relative, )? { return Ok(CapturedFileOutcomeV1::Absent); } diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index d8e5ca885fc..a72a87b77ea 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -247,6 +247,28 @@ pub fn has_nested_repository_ancestor( Ok(false) } +/// A worktree source entry must be a real file, not a link to bytes outside +/// the parent's Git authority. The shared live walker does not follow leaf +/// symlinks; classification, capture, and freshness must make the same choice. +/// Missing paths remain possible tombstones and are not symlink exclusions. +pub fn is_source_symlink(project_root: &Path, relative_path: &Path) -> std::io::Result { + if relative_path.as_os_str().is_empty() + || relative_path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "source path must be repository-relative", + )); + } + match std::fs::symlink_metadata(project_root.join(relative_path)) { + Ok(metadata) => Ok(metadata.file_type().is_symlink()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + /// Diagnostics emitted by the same source walker that enforces the nested /// repository boundary. Only directories within the walk's policy are named. #[derive(Clone, Default)] @@ -480,7 +502,9 @@ mod tests { use tempfile::TempDir; use tracedecay_domain::IndexPathPolicyV1; - use super::{has_nested_repository_ancestor, source_walk, source_walk_with_diagnostics}; + use super::{ + has_nested_repository_ancestor, is_source_symlink, source_walk, source_walk_with_diagnostics, + }; #[test] fn nested_repository_boundary_excludes_parent_markers_and_path_escape() { @@ -526,6 +550,20 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn readable_symlink_leaf_is_not_an_admissible_source() { + use std::os::unix::fs::symlink; + + let root = TempDir::new().unwrap(); + fs::write(root.path().join("real.rs"), "pub fn real() {}\n").unwrap(); + symlink("real.rs", root.path().join("alias.rs")).unwrap(); + assert!(is_source_symlink(root.path(), std::path::Path::new("alias.rs")).unwrap()); + assert!(!is_source_symlink(root.path(), std::path::Path::new("real.rs")).unwrap()); + assert!(!is_source_symlink(root.path(), std::path::Path::new("missing.rs")).unwrap()); + assert!(is_source_symlink(root.path(), std::path::Path::new("../escape.rs")).is_err()); + } + fn walked_files(root: &std::path::Path, path_policy: &IndexPathPolicyV1) -> Vec { let mut files = source_walk(root, None, path_policy) .expect("source walk") From 632729906cbf0fe46b858bd556e716550db062b2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:52:34 -0800 Subject: [PATCH 24/79] test(automation): bind scheduler fixtures to backend identities Repair three hosted automation_runner_test failures by supplying the configured executable identity required by the production prefilter. Use the same identity for the prefilter and retained runner, preserving manual-schedule, exact-ledger, live-lock and lock-race assertions. Add a production-prefilter regression for unconfigured refusal, deduplication and same-occurrence recovery after binding, with zero backend calls and no effectful terminal. Do not relax backend admission, change timeouts, or alter the graph publication changes. Refs #4532 --- .../tests/automation_runner_test/jobs.rs | 124 ++++++++++++++++-- 1 file changed, 110 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay/tests/automation_runner_test/jobs.rs b/crates/tracedecay/tests/automation_runner_test/jobs.rs index 740283d56e8..7342786381b 100644 --- a/crates/tracedecay/tests/automation_runner_test/jobs.rs +++ b/crates/tracedecay/tests/automation_runner_test/jobs.rs @@ -86,6 +86,7 @@ fn enabled_job_config() -> AutomationSettingsV1 { struct ContentBackend { calls: AtomicUsize, content: &'static str, + executable: Option, } impl ContentBackend { @@ -93,9 +94,15 @@ impl ContentBackend { Self { calls: AtomicUsize::new(0), content, + executable: None, } } + fn with_executable(mut self, executable: &Path) -> Self { + self.executable = Some(executable.to_path_buf()); + self + } + fn calls(&self) -> usize { self.calls.load(Ordering::SeqCst) } @@ -126,7 +133,7 @@ impl AgentTaskBackend for ContentBackend { } fn executable(&self) -> Option<&std::path::Path> { - None + self.executable.as_deref() } } @@ -988,6 +995,10 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let mut job = sample_job("sched-job"); job.schedule = Some("manual".to_string()); let config = enabled_job_config(); + // Reach the schedule gate with a bound identity, not the earlier + // backend-unconfigured refusal. This gate never invokes the backend. + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); // Manual schedule: every scheduler tick of the same occurrence reaches // the prefilter, which records the skip once. @@ -996,7 +1007,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let run = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1019,7 +1030,7 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { let repeated = evaluate_and_record_scheduler_skip( &dashboard_root, &config, - None, + backend.executable(), &job, "sched-occurrence", None, @@ -1031,6 +1042,76 @@ async fn scheduler_trigger_skips_repeat_and_respects_lock_discipline() { repeated.ledger_record, records[0], "an immediate repeat skip must return the exact durable prior row" ); + assert_eq!(backend.calls(), 0); +} + +#[tokio::test] +async fn scheduler_prefilter_rejects_unconfigured_backend_and_recovers_without_effect() { + let temp = tempdir().unwrap(); + let dashboard_root = temp.path().join("dashboard"); + let job = sample_job("backend-recovery"); + let config = enabled_job_config(); + let occurrence = "backend-recovery-occurrence"; + let backend = ContentBackend::new("unused"); + + let first = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("a due job still requires a configured backend"); + assert_eq!(first.report["reason"], json!("backend_unconfigured")); + assert_eq!(first.ledger_record.status, AutomationRunStatus::Skipped); + assert_ne!(first.run_id, occurrence); + let repeated = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(repeated.ledger_record, first.ledger_record); + + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = backend.with_executable(executable.path()); + assert!( + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), + "binding the backend must admit the same due occurrence without cooldown" + ); + assert_eq!(backend.calls(), 0); + assert_eq!( + load_run_records(&dashboard_root, 10).await.unwrap(), + vec![first.ledger_record], + "advisory admission must not append an effectful terminal" + ); + assert!( + tracedecay_automation_runtime::automation::run_ledger::find_run_record_exact_bounded( + &dashboard_root, + occurrence, + ) + .await + .unwrap() + .is_none() + ); } #[tokio::test] @@ -1206,15 +1287,22 @@ async fn scheduler_prefilter_live_lock_wins_over_not_due_summary() { let config = enabled_job_config(); let occurrence = "locked-prefilter-occurrence"; - let prefilter = - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .expect("the live job lock must produce a diagnostic"); + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); + let prefilter = evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .expect("the live job lock must produce a diagnostic"); assert_eq!(prefilter.report["reason"], json!("scheduler_lock_active")); assert_ne!(prefilter.run_id, occurrence); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, @@ -1262,12 +1350,21 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { job.schedule = Some("every 1h".to_owned()); let config = enabled_job_config(); let occurrence = "prefilter-gap-occurrence"; + let executable = tempfile::NamedTempFile::new().unwrap(); + let backend = ContentBackend::new("unused").with_executable(executable.path()); assert!( - evaluate_and_record_scheduler_skip(&dashboard_root, &config, None, &job, occurrence, None) - .await - .unwrap() - .is_none(), + evaluate_and_record_scheduler_skip( + &dashboard_root, + &config, + backend.executable(), + &job, + occurrence, + None, + ) + .await + .unwrap() + .is_none(), "a new interval job is advisory-due before effect admission" ); let _raced_in = @@ -1276,7 +1373,6 @@ async fn retained_scheduler_runner_reacquires_after_due_prefilter() { .unwrap() .expect("the advisory prefilter must release its reservation lock"); - let backend = ContentBackend::new("unused"); let admitted_job = tracedecay_automation_runtime::automation::jobs::admit_job(&job).unwrap(); let retained = run_user_job_with_backend_for_retained_settlement( &dashboard_root, From 3fdb50ddbe2b9fe19f80f3bb94b96d2c738c66b0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:03:19 -0800 Subject: [PATCH 25/79] feat(code-index): index authenticated direct submodules --- Cargo.lock | 1 + .../src/code_index_scheduler.rs | 2 +- .../code_index_scheduler/freshness_witness.rs | 128 +++++++-- .../code_index_scheduler/git_tree_capture.rs | 119 ++++++-- .../src/code_index_scheduler/identity.rs | 60 +++- .../ignored_dependencies.rs | 2 +- .../src/code_index_scheduler/reconcile.rs | 124 +++++++-- .../tests/index_path_policy_tests.rs | 213 ++++++++++++++ .../tests/code_index_scheduler/reconcile.rs | 26 +- crates/tracedecay-code-index/Cargo.toml | 1 + .../benches/partitioned_codec.rs | 2 + .../benches/sealed_storage.rs | 2 + .../src/production/ignored_sources.rs | 3 + .../src/production/lexical_page_source.rs | 1 + .../production/lexical_page_source_tests.rs | 2 + .../src/production/mod.rs | 20 ++ .../src/production/sparse_increment_tests.rs | 1 + .../src/production/worker_tests.rs | 2 + .../tracedecay-code-index/src/source_walk.rs | 262 +++++++++++++----- .../src/source_walk}/classification.rs | 115 +++++++- .../src/source_walk/submodules.rs | 188 +++++++++++++ .../production_orchestration.rs | 5 + .../parallel_equivalence.rs | 1 + .../tests/resident_accounting.rs | 2 + .../src/config/mod.rs | 34 ++- crates/tracedecay-domain/src/configuration.rs | 2 + crates/tracedecay-domain/src/lib.rs | 7 +- .../src/source_path_policy.rs | 18 +- .../src/configuration/registry.rs | 20 +- .../src/configuration/store.rs | 11 +- .../src/configuration/store/tests/mod.rs | 45 +-- .../src/handlers/analysis/field_sites.rs | 7 +- .../src/handlers/analysis/mod.rs | 8 +- .../src/handlers/graph/rename_preview.rs | 16 ++ .../benches/code_lexical_catchup.rs | 2 + .../src/bin/tracedecay_index_bench.rs | 2 + .../src/bin/tracedecay_search_bench.rs | 2 + .../candidate_producers.rs | 2 + .../src/git_repository.rs | 2 +- .../src/git_repository/native_integration.rs | 47 +++- .../native_integration_authority.rs | 3 + .../src/candidate_output.rs | 2 + .../src/candidate_output/cancellation.rs | 2 + docs/USER-GUIDE.md | 16 ++ 44 files changed, 1321 insertions(+), 209 deletions(-) rename crates/{tracedecay-code-index-runtime/src/code_index_scheduler => tracedecay-code-index/src/source_walk}/classification.rs (85%) create mode 100644 crates/tracedecay-code-index/src/source_walk/submodules.rs diff --git a/Cargo.lock b/Cargo.lock index 860f7e0e1d2..4238a98de78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6430,6 +6430,7 @@ dependencies = [ "base64 0.23.1", "criterion", "flate2", + "gix", "hex", "ignore", "miniz_oxide", diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs index a325e598138..d3ea24dc19c 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs @@ -201,7 +201,7 @@ mod activation; pub mod branch_generations; pub mod branch_publication; mod cadence; -mod classification; +use tracedecay_code_index::source_walk::classification; mod demand_admission; mod freshness_witness; pub use freshness_witness::sealed_source_bytes; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index d81c425e9e3..b98634731bb 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -76,10 +76,12 @@ pub struct WorktreeStatSweepV1 { pub fn worktree_stat_sweep( project_root: &Path, ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], + include_submodules: bool, ) -> Result { let repository = tracedecay_runtime_core::git_open::open(project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - let candidate_roster = source_candidates(&repository, ignored_source_admissions)?; + let candidate_roster = + source_candidates(&repository, ignored_source_admissions, include_submodules)?; // One sweep span covers the O(candidates) stat walk. Never emit one // trace event per file. let mut buf = Vec::new(); @@ -120,9 +122,11 @@ struct SourceCandidatesV1 { fn source_candidates( repository: &gix::Repository, ignored_source_admissions: &[CodeIndexIgnoredSourceAdmissionV1], + include_submodules: bool, ) -> Result { - let classification = classification::WorktreeChangeClassificationV1::classify(repository) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = + classification::WorktreeChangeClassificationV1::for_policy(repository, include_submodules) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; let registry = StaticLanguageRegistry::new(); let admitted_paths = ignored_source_admissions .iter() @@ -647,17 +651,27 @@ impl SourceSweepCacheV1 { let Ok(repository) = tracedecay_runtime_core::git_open::open(project_root) else { return (false, stats); }; - let evidence = roster_evidence( - &repository, - project_root, - sampled_at, - ignored_source_admissions, - ); + // Parent-only directory evidence cannot vouch for child Git indexes, + // refs or untracked entries. Enumerate both Git rosters on every child proof. + let evidence = (!self.path_policy.includes_submodules()) + .then(|| { + roster_evidence( + &repository, + project_root, + sampled_at, + ignored_source_admissions, + ) + }) + .flatten(); let Ok(SourceCandidatesV1 { candidates, unrepresentable, .. - }) = source_candidates(&repository, ignored_source_admissions) + }) = source_candidates( + &repository, + ignored_source_admissions, + self.path_policy.includes_submodules(), + ) else { return (false, stats); }; @@ -735,12 +749,14 @@ impl SourceSweepCacheV1 { &mut self, project_root: &Path, manifest: &SourceContentManifestV1, + candidate_paths: &BTreeSet, shutting_down: &AtomicBool, ) -> Result, CodeIndexSchedulerErrorV1> { let registry = StaticLanguageRegistry::new(); let candidates = manifest .files .keys() + .filter(|path| candidate_paths.contains(*path)) .filter_map(|logical_path| source_candidate(®istry, logical_path.clone(), false)) .collect::>(); let present = present_candidates( @@ -838,7 +854,12 @@ impl SourceSweepCacheV1 { if disputed.is_empty() { return Ok(disputed); } - Ok(clean_filter_mismatches(project_root, disputed, manifest)) + Ok(clean_filter_mismatches( + project_root, + disputed, + manifest, + self.path_policy.includes_submodules(), + )) } } @@ -882,10 +903,56 @@ fn clean_filter_mismatches<'a>( project_root: &Path, disputed: Vec<&'a StatCandidateV1>, manifest: &SourceContentManifestV1, + include_submodules: bool, ) -> Vec<&'a StatCandidateV1> { let Ok(repository) = tracedecay_runtime_core::git_open::open(project_root) else { return disputed; }; + let mut remaining = disputed; + let mut mismatches = Vec::new(); + if include_submodules { + let Ok(children) = + tracedecay_code_index::source_walk::submodules::live_submodules(&repository) + else { + return remaining; + }; + for (prefix, binding) in children { + let (local, other): (Vec<_>, Vec<_>) = remaining + .into_iter() + .partition(|candidate| Path::new(&candidate.logical_path).starts_with(&prefix)); + remaining = other; + if local.is_empty() { + continue; + } + match tracedecay_runtime_core::git_open::open(&binding.git_dir) { + Ok(child) => mismatches.extend(filter_mismatches_in_repository( + project_root, + &child, + Path::new(&prefix), + local, + manifest, + )), + Err(_) => mismatches.extend(local), + } + } + } + mismatches.extend(filter_mismatches_in_repository( + project_root, + &repository, + Path::new(""), + remaining, + manifest, + )); + mismatches +} + +fn filter_mismatches_in_repository<'a>( + project_root: &Path, + repository: &gix::Repository, + prefix: &Path, + disputed: Vec<&'a StatCandidateV1>, + manifest: &SourceContentManifestV1, +) -> Vec<&'a StatCandidateV1> { let Ok((mut pipeline, index)) = repository.filter_pipeline(None) else { return disputed; }; @@ -895,15 +962,17 @@ fn clean_filter_mismatches<'a>( let Some(expected) = manifest.files.get(&candidate.logical_path) else { return true; }; + let Ok(rela_path) = Path::new(&candidate.logical_path).strip_prefix(prefix) else { + return true; + }; if candidate.explicitly_admitted || index - .entry_by_path(BStr::new(candidate.logical_path.as_str())) + .entry_by_path(BStr::new(rela_path.as_os_str().as_encoded_bytes())) .is_none() { return true; } - let rela_path = Path::new(&candidate.logical_path); - let Ok(file) = std::fs::File::open(project_root.join(rela_path)) else { + let Ok(file) = std::fs::File::open(project_root.join(&candidate.logical_path)) else { return true; }; let Ok(mut converted) = pipeline.convert_to_git(file, rela_path, &index) else { @@ -930,8 +999,13 @@ pub fn sealed_source_bytes( project_root: &Path, logical_path: &str, language: &LanguageId, + include_submodules: bool, sealed: &ContentDigest, ) -> Result>, CodeIndexSchedulerErrorV1> { + if tracedecay_code_index::source_walk::is_source_symlink(project_root, Path::new(logical_path))? + { + return Ok(None); + } let raw = ignored_dependencies::read_bounded_snapshot_source(&project_root.join(logical_path), None)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &raw)?; @@ -942,15 +1016,35 @@ pub fn sealed_source_bytes( // hoist them into a caller-held reader if a caller ever needs hundreds. let repository = tracedecay_runtime_core::git_open::open(project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let mut repository = repository; + let mut relative = Path::new(logical_path); + if include_submodules { + let children = tracedecay_code_index::source_walk::submodules::live_submodules(&repository) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; + if let Some((prefix, binding)) = children + .iter() + .find(|(prefix, _)| relative.starts_with(prefix)) + { + let child_path = relative + .strip_prefix(prefix) + .map_err(|_| CodeIndexSchedulerErrorV1::Git("invalid child filter path".into()))?; + repository = tracedecay_runtime_core::git_open::open(&binding.git_dir) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + relative = child_path; + } + } let (mut pipeline, index) = repository .filter_pipeline(None) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - if index.entry_by_path(BStr::new(logical_path)).is_none() { + if index + .entry_by_path(BStr::new(relative.as_os_str().as_encoded_bytes())) + .is_none() + { return Ok(None); } let mut filtered = Vec::new(); pipeline - .convert_to_git(raw.as_slice(), Path::new(logical_path), &index) + .convert_to_git(raw.as_slice(), relative, &index) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? .read_to_end(&mut filtered)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &filtered)?; @@ -1175,7 +1269,7 @@ mod tracked_parent_boundary_tests { logical_path: "ignored/ordinary.rs".to_owned(), }, ]; - let candidates = source_candidates(&repository, &admitted).unwrap(); + let candidates = source_candidates(&repository, &admitted, false).unwrap(); let paths = candidates .candidates .iter() diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs index 7db65b9c882..922d0c889e9 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs @@ -16,7 +16,9 @@ use tracedecay_domain::{ }; use tracedecay_privacy::CODE_SOURCE_SANITIZER_VERSION_V1; use tracedecay_query::code_search::CodeIndexSearchUnavailableReasonV1; -use tracedecay_runtime_core::git_repository::resolve_tree_link; +use tracedecay_runtime_core::git_repository::{ + GitNativeCandidateTreeVisitError, GitTreeSubmodulePins, resolve_tree_link, +}; use tracedecay_runtime_core::resident_memory::ResidentMemoryReservationV1; use super::{ @@ -161,7 +163,7 @@ pub(super) fn representable_logical_path(git_path: &[u8]) -> Option<&str> { /// Whether capture would parse `git_path` as source. A committed symlink at /// such a path is read through to its target, as a checkout reads it, so its /// link text never becomes source. -fn is_source_path(registry: &StaticLanguageRegistry, git_path: &[u8]) -> bool { +pub(super) fn is_source_path(registry: &StaticLanguageRegistry, git_path: &[u8]) -> bool { representable_logical_path(git_path) .and_then(|logical_path| Path::new(logical_path).extension()) .and_then(|extension| extension.to_str()) @@ -396,9 +398,8 @@ impl CodeIndexWorktreeSchedulerV1 { } /// HEAD's tree standing in for a clean checkout. Git stores a symlink as - /// its target path, while worktree capture and every source sweep read - /// through the link, so a tree holding a source symlink would seal bytes - /// the next sweep disproves. That tree is `GenerationUnavailable` here and + /// its target path, while the live roster excludes the link. A tree holding + /// a source symlink would seal bytes the next sweep disproves. That tree is `GenerationUnavailable` here and /// the checkout is captured from disk instead. pub(super) fn capture_clean_head_tree_snapshot( &self, @@ -458,8 +459,15 @@ impl CodeIndexWorktreeSchedulerV1 { source.tree.clone(), control, |visitor| { + let mut gitlinks = Vec::new(); for entry in entries { control.termination().map_or(Ok(()), Err)?; + if self.path_policy.includes_submodules() && entry.mode.is_commit() { + let path = entry.filepath.to_str().map_err(|_| { + CodeIndexSearchUnavailableReasonV1::GenerationUnavailable + })?; + gitlinks.push((path.to_owned(), entry.oid)); + } if entry.mode.is_tree() || entry.mode.is_commit() { continue; } @@ -480,11 +488,61 @@ impl CodeIndexWorktreeSchedulerV1 { } visitor(entry.filepath.as_ref(), &blob.data)?; } - Ok(()) + let pins = GitTreeSubmodulePins::from_tree(&tree, gitlinks) + .map_err(|_| CodeIndexSearchUnavailableReasonV1::GenerationUnavailable)?; + self.visit_pinned_submodule_blobs(&repository, &pins, control, visitor) }, ) } + fn visit_pinned_submodule_blobs( + &self, + parent: &gix::Repository, + pins: &GitTreeSubmodulePins, + control: &branch_generations::BranchGenerationReadControlV1, + visitor: &mut dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1>, + ) -> Result<(), CodeIndexSearchUnavailableReasonV1> { + let unavailable = CodeIndexSearchUnavailableReasonV1::GenerationUnavailable; + let children = + tracedecay_code_index::source_walk::submodules::pinned_submodules(parent, pins) + .map_err(|_| unavailable)?; + let registry = StaticLanguageRegistry::new(); + for (prefix, (repo, pin)) in children { + control.termination().map_or(Ok(()), Err)?; + let tree = repo + .find_commit(pin) + .and_then(|commit| commit.tree()) + .map_err(|_| unavailable)?; + for entry in tree + .traverse() + .breadthfirst + .files() + .map_err(|_| unavailable)? + { + control.termination().map_or(Ok(()), Err)?; + if !entry.mode.is_blob() { + continue; + } + let path = [prefix.as_bytes(), b"/", entry.filepath.as_slice()].concat(); + let Some(logical_path) = representable_logical_path(&path) else { + // Capture records the unrepresentable name without consulting its blob. + visitor(&path, &[])?; + continue; + }; + if !is_source_path(®istry, &path) + || !tracedecay_code_index::source_walk::submodules::is_source_location( + Path::new(logical_path), + ) + { + continue; + } + let blob = repo.find_blob(entry.oid).map_err(|_| unavailable)?; + visitor(&path, &blob.data)?; + } + } + Ok(()) + } + #[tracing::instrument( name = "daemon.code_index.capture.native_candidate_tree", level = "trace", @@ -510,14 +568,24 @@ impl CodeIndexWorktreeSchedulerV1 { expected_tree.clone(), control, |visitor| { - candidate.visit_blobs(|path| is_source_path(®istry, path), visitor).map_err(|error| match error { - tracedecay_runtime_core::git_repository::GitNativeCandidateTreeVisitError::Repository(_) => { - control + let pins = candidate + .visit_blobs( + self.path_policy.includes_submodules(), + |path| is_source_path(®istry, path), + &mut *visitor, + ) + .map_err(|error| match error { + GitNativeCandidateTreeVisitError::Repository(_) => control .termination() - .unwrap_or(CodeIndexSearchUnavailableReasonV1::GenerationUnavailable) - } - tracedecay_runtime_core::git_repository::GitNativeCandidateTreeVisitError::Visitor(reason) => reason, - }) + .unwrap_or(CodeIndexSearchUnavailableReasonV1::GenerationUnavailable), + GitNativeCandidateTreeVisitError::Visitor(reason) => reason, + })?; + if pins.gitlinks.is_empty() { + return Ok(()); + } + let parent = tracedecay_runtime_core::git_open::open(&self.project_root) + .map_err(|_| CodeIndexSearchUnavailableReasonV1::GenerationUnavailable)?; + self.visit_pinned_submodule_blobs(&parent, &pins, control, visitor) }, ) } @@ -601,6 +669,7 @@ impl CodeIndexWorktreeSchedulerV1 { snapshot_content_identity(&files, &sanitization_receipts, &omitted_sources); Ok(CapturedSnapshotV1 { repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: self.path_policy.includes_submodules(), tree: Some(tree), dirty: tracedecay_domain::RepositoryDirtyStateV1::Clean, }, @@ -1326,16 +1395,24 @@ mod tests { ], "only source links that name a file in the tree are captured" ); + let live = present( + &scheduler + .capture_authoritative_snapshot_without_active_generation_reuse(None) + .expect("capture the checkout from disk") + .snapshot, + ); assert_eq!( - branch, - present( - &scheduler - .capture_authoritative_snapshot_without_active_generation_reuse(None) - .expect("capture the checkout from disk") - .snapshot - ), - "a branch capture must index the bytes a checkout reads through each link" + live.iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + ["AGENTS.md", "src/real.rs"] ); + for regular in live { + assert!( + branch.contains(®ular), + "historical and live regular files must carry the same source digest" + ); + } let notes = captured .snapshot .files diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 88ca93c0a42..728e42d7b45 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -154,6 +154,11 @@ pub struct GitMetadataFingerprintV1 { /// branch switch changes this even when mtime resolution is too coarse to /// observe, so content is compared alongside mtimes. head_contents: Option, + // Include the registered child pin, bound checkout and resolved HEAD in the same + // read-admission proof as the parent. A failed probe cannot renew that proof. + submodule_refs: Option, + submodule_indexes: Option, + submodule_probe_failed: bool, } impl GitMetadataFingerprintV1 { @@ -176,19 +181,60 @@ impl GitMetadataFingerprintV1 { packed_refs: mtime(&common_dir.join("packed-refs")), refs_heads: refs_heads_signature(&common_dir.join("refs").join("heads")), head_contents: std::fs::read_to_string(&head_path).ok(), + ..Self::default() } } + pub(super) fn for_policy(project_root: &Path, include_submodules: bool) -> Self { + let mut fingerprint = Self::capture(project_root); + if include_submodules { + let probes = (|| { + let parent = tracedecay_runtime_core::git_open::open(project_root).ok()?; + let children = + tracedecay_code_index::source_walk::submodules::live_submodules(&parent) + .ok()?; + let mut refs = Vec::new(); + let mut indexes = Vec::new(); + for (path, binding) in children { + let child = tracedecay_runtime_core::git_open::open(&binding.git_dir).ok()?; + let head = child.head_id().ok()?.detach(); + refs.push(( + path.clone(), + binding.git_dir, + binding.pin.to_string(), + head.to_string(), + )); + indexes.push((path, mtime(&child.index_path()))); + } + Some(( + tracedecay_domain::canonical_sha256(&refs).ok()?, + tracedecay_domain::canonical_sha256(&indexes).ok()?, + )) + })(); + match probes { + Some((refs, indexes)) => { + fingerprint.submodule_refs = Some(refs.as_str().to_owned()); + fingerprint.submodule_indexes = Some(indexes.as_str().to_owned()); + } + None => fingerprint.submodule_probe_failed = true, + } + } + fingerprint + } + /// Whether the sampled metadata differs from a previously captured value. pub fn differs_from(&self, other: &GitMetadataFingerprintV1) -> bool { - self != other + self.submodule_probe_failed || other.submodule_probe_failed || self != other } /// Whether HEAD and every branch ref are unchanged. The index is excluded: /// opening the repository may refresh `.git/index` on its own, while a /// commit, checkout, or ref update always moves HEAD or a ref. pub fn same_refs_as(&self, other: &GitMetadataFingerprintV1) -> bool { - self.head == other.head + !self.submodule_probe_failed + && !other.submodule_probe_failed + && self.submodule_refs == other.submodule_refs + && self.head == other.head && self.packed_refs == other.packed_refs && self.refs_heads == other.refs_heads && self.head_contents == other.head_contents @@ -206,7 +252,7 @@ impl GitMetadataFingerprintV1 { time.and_then(|time| time.duration_since(UNIX_EPOCH).ok()) .map_or_else(|| "-".to_owned(), |elapsed| elapsed.as_nanos().to_string()) } - format!( + let mut signature = format!( "head={}|index={}|packed_refs={}|refs_heads={}|head_contents={}", render_time(self.head), render_time(self.index), @@ -216,7 +262,13 @@ impl GitMetadataFingerprintV1 { || "-".to_owned(), |value| super::sha256_hex(value.as_bytes()) ), - ) + ); + if let (Some(refs), Some(indexes)) = (&self.submodule_refs, &self.submodule_indexes) { + signature.push_str(&format!( + "|submodule_refs={refs}|submodule_indexes={indexes}" + )); + } + signature } } diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 2819eb78153..23dacd76f56 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -260,7 +260,7 @@ impl CodeIndexWorktreeSchedulerV1 { ) -> Result { let source_control = self.reconcile_control(); let began_under = source_control.expected_epoch(); - let captured_under = super::identity::GitMetadataFingerprintV1::capture(&self.project_root); + let captured_under = self.freshness_fence().git_metadata(&self.project_root); let _workers = self.ensure_worker_plan()?; let _worker_memory = match self.reserve_worker_memory() { Ok(reservation) => reservation, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 5724543efff..4113a4ff273 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -567,6 +567,7 @@ pub(crate) struct SourceFreshnessFenceV1 { source_epoch: Arc, /// Held only by source sweeps, never across a scheduler pass. sweep_cache: Arc>, + include_submodules: bool, pending_source: Option>, } @@ -590,6 +591,10 @@ pub(super) struct SourceFreshnessFenceStateV1 { } impl SourceFreshnessFenceV1 { + pub(super) fn git_metadata(&self, root: &Path) -> identity::GitMetadataFingerprintV1 { + identity::GitMetadataFingerprintV1::for_policy(root, self.include_submodules) + } + fn unverified(source_epoch: Arc, path_policy: IndexPathPolicyV1) -> Self { Self { state: Arc::new(Mutex::new(SourceFreshnessFenceStateV1 { @@ -604,6 +609,7 @@ impl SourceFreshnessFenceV1 { })), last_reconciled_at_micros: Arc::new(AtomicI64::new(0)), source_epoch, + include_submodules: path_policy.includes_submodules(), sweep_cache: Arc::new(Mutex::new(SourceSweepCacheV1::new(path_policy))), pending_source: None, } @@ -723,7 +729,8 @@ impl SourceFreshnessFenceV1 { pub(super) fn verified_git_metadata_moved(&self, project_root: &Path) -> bool { let state = self.snapshot(); state.verified_against_source - && identity::GitMetadataFingerprintV1::capture(project_root) + && self + .git_metadata(project_root) .differs_from(&state.git_metadata) } @@ -760,7 +767,8 @@ impl SourceFreshnessFenceV1 { } state.verified_against_source && self.source_epoch.load(Ordering::Acquire) == state.reconciled_source_epoch - && !identity::GitMetadataFingerprintV1::capture(project_root) + && !self + .git_metadata(project_root) .differs_from(&state.git_metadata) } @@ -893,12 +901,13 @@ impl SourceFreshnessFenceV1 { &self, project_root: &Path, manifest: &SourceContentManifestV1, + candidate_paths: &BTreeSet, shutting_down: &AtomicBool, ) -> Result, CodeIndexSchedulerErrorV1> { self.sweep_cache .lock() .unwrap_or_else(PoisonError::into_inner) - .disputed_sources(project_root, manifest, shutting_down) + .disputed_sources(project_root, manifest, candidate_paths, shutting_down) } /// Sweep the current witness and report what the sweep had to read. @@ -916,7 +925,7 @@ impl SourceFreshnessFenceV1 { self.sweep_sources( project_root, &freshness.source_roster, - &identity::GitMetadataFingerprintV1::capture(project_root), + &self.git_metadata(project_root), &witness.content_manifest, shutting_down, ) @@ -937,7 +946,7 @@ impl SourceFreshnessFenceV1 { if !freshness.verified_against_source { return FreshnessProbeVerdictV1::Unverified; } - let git_metadata = identity::GitMetadataFingerprintV1::capture(project_root); + let git_metadata = self.git_metadata(project_root); if git_metadata.differs_from(&freshness.git_metadata) { return FreshnessProbeVerdictV1::Moved; } @@ -1336,6 +1345,7 @@ impl CodeIndexWorktreeSchedulerV1 { .with_shutdown_signal(Arc::clone(&shutting_down)) .with_decoded_content(byte_pool.decoded_content.clone()); let production_config = CodeIndexProductionConfigV1 { + include_submodules: path_policy.includes_submodules(), project_id: project_id.clone(), repository: repository_id.clone(), sanitizer_revision, @@ -1996,7 +2006,7 @@ impl CodeIndexWorktreeSchedulerV1 { { return Ok(None); } - let metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let metadata = self.freshness_fence.git_metadata(&self.project_root); if witness.git_metadata_signature != metadata.stable_signature() { return Ok(None); } @@ -2052,7 +2062,7 @@ impl CodeIndexWorktreeSchedulerV1 { if witness.generation_id != pointer.generation_id { return false; } - let metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let metadata = self.freshness_fence.git_metadata(&self.project_root); if witness.git_metadata_signature != metadata.stable_signature() { return false; } @@ -2234,7 +2244,7 @@ impl CodeIndexWorktreeSchedulerV1 { )); } self.identity = resolved; - let sampled_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let sampled_metadata = self.freshness_fence.git_metadata(&self.project_root); let Some(sampled_sweep) = self.worktree_stat_sweep().ok() else { return Ok(None); }; @@ -2343,7 +2353,7 @@ impl CodeIndexWorktreeSchedulerV1 { let captured = self .capture_authoritative_snapshot_without_active_generation_reuse(Some(&control)) .map_err(|error| prefer_resident_memory_refusal(&control, error))?; - let git_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let git_metadata = self.freshness_fence.git_metadata(&self.project_root); let stat_signature = self.worktree_stat_signature().ok(); let drained_hints = { let mut hints = self @@ -2927,7 +2937,7 @@ impl CodeIndexWorktreeSchedulerV1 { // bound to; a HEAD move under the same worktree is allowed and simply // records a new source revision, so the served generation is never // mis-attributed across identities. - let captured_under = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let captured_under = self.freshness_fence.git_metadata(&self.project_root); let resolved = identity::IndexingIdentityV1::resolve(&self.project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Identity(error.to_string()))?; if !resolved.authorizes_reuse_of(&self.identity) { @@ -3199,7 +3209,7 @@ impl CodeIndexWorktreeSchedulerV1 { source_epoch: u64, captured_under: &identity::GitMetadataFingerprintV1, ) { - let sampled = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let sampled = self.freshness_fence.git_metadata(&self.project_root); let metadata = if sampled.same_refs_as(captured_under) { sampled } else { @@ -3318,7 +3328,9 @@ impl CodeIndexWorktreeSchedulerV1 { } let freshness = self.freshness_fence.snapshot(); if !freshness.verified_against_source - || identity::GitMetadataFingerprintV1::capture(&self.project_root) + || self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata) || freshness.last_reconciled_at.elapsed() >= self.policy.staleness_threshold { @@ -3338,7 +3350,9 @@ impl CodeIndexWorktreeSchedulerV1 { } let freshness = self.freshness_fence.snapshot(); if !freshness.verified_against_source - || identity::GitMetadataFingerprintV1::capture(&self.project_root) + || self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata) { self.request_background_reconcile(); @@ -3364,7 +3378,7 @@ impl CodeIndexWorktreeSchedulerV1 { .sweep_sources( &self.project_root, &self.ignored_source_admissions, - &identity::GitMetadataFingerprintV1::capture(&self.project_root), + &self.freshness_fence.git_metadata(&self.project_root), manifest, &self.shutting_down, ) @@ -3376,7 +3390,7 @@ impl CodeIndexWorktreeSchedulerV1 { fn source_witness_matches_worktree(&self, freshness: &SourceFreshnessFenceStateV1) -> bool { self.freshness_fence.source_witness_matches( freshness, - &identity::GitMetadataFingerprintV1::capture(&self.project_root), + &self.freshness_fence.git_metadata(&self.project_root), &self.project_root, &self.shutting_down, ) @@ -3432,7 +3446,7 @@ impl CodeIndexWorktreeSchedulerV1 { // Sample after the walk. `git_open::open` inside the digest comparison can // move index metadata; storing the post-walk sample is what keeps the // next probe from calling that side effect a new generation. - let git_metadata = identity::GitMetadataFingerprintV1::capture(&self.project_root); + let git_metadata = self.freshness_fence.git_metadata(&self.project_root); self.freshness_fence.rebind_admission_clock(git_metadata); self.freshness_fence .source_currency_witness_for(generation_id, snapshot_content_identity) @@ -3450,7 +3464,11 @@ impl CodeIndexWorktreeSchedulerV1 { fn worktree_stat_sweep( &self, ) -> Result { - freshness_witness::worktree_stat_sweep(&self.project_root, &self.ignored_source_admissions) + freshness_witness::worktree_stat_sweep( + &self.project_root, + &self.ignored_source_admissions, + self.path_policy.includes_submodules(), + ) } /// Deliver debounced hook hints (exact touched paths) into the incremental @@ -3495,7 +3513,9 @@ impl CodeIndexWorktreeSchedulerV1 { // suppressing cadence on open-time clocks is not. return Ok(Some(self.reconcile_now()?)); } - let git_changed = identity::GitMetadataFingerprintV1::capture(&self.project_root) + let git_changed = self + .freshness_fence + .git_metadata(&self.project_root) .differs_from(&freshness.git_metadata); if git_changed { // Tier 1: a git-mediated mutation is authoritative evidence; reconcile. @@ -3957,7 +3977,28 @@ impl CodeIndexWorktreeSchedulerV1 { .ignored_source_admissions .iter() .any(|admission| admission.logical_path == logical_path); - self.capture_admitted_candidate(registry, logical_path, control, explicitly_admitted) + let child_root = if self.path_policy.includes_submodules() && !explicitly_admitted { + let repository = tracedecay_runtime_core::git_open::open(&self.project_root) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = + classification::WorktreeChangeClassificationV1::for_policy(&repository, true) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + if classification.beneath_gitlink(Path::new(logical_path)) + && classification.child_root(logical_path).is_none() + { + return Ok(CapturedFileOutcomeV1::Absent); + } + classification.child_root(logical_path).map(str::to_owned) + } else { + None + }; + self.capture_admitted_candidate( + registry, + logical_path, + control, + explicitly_admitted, + child_root.as_deref(), + ) } fn ignored_admission_paths(&self) -> BTreeSet<&str> { @@ -3973,20 +4014,41 @@ impl CodeIndexWorktreeSchedulerV1 { logical_path: &str, control: Option<&dyn CodeIndexExecutionControlV1>, explicitly_admitted: bool, + child_root: Option<&str>, ) -> Result { // Explicit ignored-source admission cannot cross a nested checkout // any more than an ordinary Git candidate or interactive source walk. - let relative = std::path::Path::new(logical_path); + let relative = Path::new(logical_path); + let absolute = self.project_root.join(relative); + let (source_root, source_relative) = match child_root { + Some(prefix) if !explicitly_admitted => { + if !git_tree_capture::is_source_path(registry, logical_path.as_bytes()) { + return Ok(CapturedFileOutcomeV1::Absent); + } + let root = self.project_root.join(prefix); + if root.canonicalize().ok().as_ref() != Some(&root) + || !absolute + .symlink_metadata() + .is_ok_and(|metadata| metadata.is_file()) + { + return Ok(CapturedFileOutcomeV1::Absent); + } + let relative = relative.strip_prefix(prefix).map_err(|_| { + CodeIndexSchedulerErrorV1::Git("invalid bound submodule path".into()) + })?; + (root, relative) + } + _ => (self.project_root.clone(), relative), + }; if tracedecay_code_index::source_walk::has_nested_repository_ancestor( - &self.project_root, - relative, + &source_root, + source_relative, )? || tracedecay_code_index::source_walk::is_source_symlink( - &self.project_root, - relative, + &source_root, + source_relative, )? { return Ok(CapturedFileOutcomeV1::Absent); } - let absolute = self.project_root.join(logical_path); match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), Ok(_) => {} @@ -4048,8 +4110,11 @@ impl CodeIndexWorktreeSchedulerV1 { // Classify committed/staged/unstaged/untracked/deleted/renamed paths // truthfully from gix. Deletions drop out of the present candidate set; // their tombstones flow through `changed_paths`. - let classification = classification::WorktreeChangeClassificationV1::classify(&repository) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + let classification = classification::WorktreeChangeClassificationV1::for_policy( + &repository, + self.path_policy.includes_submodules(), + ) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; if self.shutting_down.load(Ordering::Acquire) { return Err(cancelled_code_index_reconcile()); } @@ -4126,6 +4191,7 @@ impl CodeIndexWorktreeSchedulerV1 { // assume-unchanged. A sealed row is reused only while its bytes still // carry its digest; every row they disprove beyond the visible commit // and status deltas is a worktree change stat could not see. + let mut candidate_paths = classification.candidate_paths(); let stat_blind_paths = match reusable_active.as_ref() { Some(active) => { let visible = classification.changed_paths(); @@ -4133,6 +4199,7 @@ impl CodeIndexWorktreeSchedulerV1 { .disputed_sources( &self.project_root, &SourceContentManifestV1::for_snapshot(active.snapshot()), + &candidate_paths, &self.shutting_down, )? .into_iter() @@ -4193,7 +4260,6 @@ impl CodeIndexWorktreeSchedulerV1 { let source_revision = (self.ignored_source_admissions.is_empty() && !has_changes) .then(|| self.identity.head_commit().cloned()) .flatten(); - let mut candidate_paths = classification.candidate_paths(); let mut changed_paths = classification.changed_paths(); changed_paths.extend(tree_delta); changed_paths.extend(stat_blind_paths); @@ -4274,6 +4340,7 @@ impl CodeIndexWorktreeSchedulerV1 { logical_path, control, admitted_paths.contains(logical_path.as_str()), + classification.child_root(logical_path), ) }) }) @@ -4364,6 +4431,7 @@ impl CodeIndexWorktreeSchedulerV1 { snapshot_content_identity(&files, &sanitization_receipts, &omitted_sources); let captured = CapturedSnapshotV1 { repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: self.path_policy.includes_submodules(), tree: self.identity.head_tree().cloned(), dirty, }, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index 06cf0bd3f33..f90c046bd00 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -191,3 +191,216 @@ async fn hook_hints_for_excluded_paths_queue_no_work() { ); registry.shutdown().await; } + +#[test] +fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() { + use super::{git_stdout, run_git_in}; + use crate::code_index_scheduler::{ + branch_generations::BranchGenerationReadControlV1, git_tree_capture::ExactGitTreeSourceV1, + }; + use tracedecay_code_index::grep_search::{GrepSearchQuery, search_tree_with_cancel}; + use tracedecay_domain::{CommitId, RefId, RelationEdgeKindV1, RepositoryDirtyStateV1, TreeId}; + + let parent = GitFixture::new(&[( + "dev/launch.ts", + "import { feature } from '../app/lib';\nexport function launch() { return feature(); }\n", + )]); + let child = GitFixture::new(&[ + ( + "lib.ts", + "export function feature() { return 'pinned_symbol'; }\n", + ), + ("gone.ts", "export function gone() {}\n"), + ("media.bin", "local binary fixture"), + (".gitignore", "*.private.ts\n"), + ]); + run_git_in( + parent.path(), + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + child.path().to_str().unwrap(), + "app", + ], + ); + run_git_in(parent.path(), &["commit", "-qm", "bind application"]); + run_git_in(parent.path(), &["branch", "pinned"]); + let pinned = ExactGitTreeSourceV1 { + reference: RefId::new("refs/heads/pinned").unwrap(), + revision: CommitId::new(git_stdout(parent.path(), &["rev-parse", "HEAD"])).unwrap(), + tree: TreeId::new(git_stdout(parent.path(), &["rev-parse", "HEAD^{tree}"])).unwrap(), + }; + let store = TempDir::new().unwrap(); + let enabled = policy(&[], &[]).with_submodules(true); + let grep = |needle: &str, policy: &IndexPathPolicyV1| { + search_tree_with_cancel( + parent.path(), + &GrepSearchQuery { + pattern: needle.to_owned(), + fixed_strings: true, + case_sensitive: true, + path_glob: Some("app/**".into()), + scope_prefix: None, + context_lines: 0, + max_results: 8, + }, + policy, + || false, + ) + .unwrap() + .hits + .into_iter() + .map(|hit| hit.file.to_string()) + .collect::>() + }; + let mut off = start(&parent, store.path(), policy(&[], &[])); + off.reconcile_now().unwrap(); + assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); + assert!(grep("pinned_symbol", &policy(&[], &[])).is_empty()); + drop(off); + + let mut on = start(&parent, store.path(), enabled.clone()); + on.reconcile_now().unwrap(); + assert_eq!( + indexed_paths(&on), + ["app/gone.ts", "app/lib.ts", "dev/launch.ts"] + ); + assert_eq!(grep("pinned_symbol", &enabled), ["app/lib.ts"]); + let assert_edge = + |generation: &tracedecay_code_index::production::CodeIndexPublishedGenerationV1| { + let symbol = |name: &str| { + generation + .symbols() + .symbols + .iter() + .find(|item| item.simple_name == name) + .unwrap() + .occurrence + .clone() + }; + assert!( + generation + .edges() + .iter() + .any(|edge| edge.kind == RelationEdgeKindV1::Calls + && edge.from_occurrence == symbol("launch") + && edge.to_occurrence == symbol("feature")) + ); + }; + assert_edge(on.latest_complete().unwrap().generation()); + let checkout = parent.path().join("app"); + std::fs::write( + checkout.join("ignored.private.ts"), + "const private_marker = 1;", + ) + .unwrap(); + assert!(grep("private_marker", &enabled).is_empty()); + std::fs::write( + checkout.join("lib.ts"), + "export function feature() { return 'successor_symbol'; }\n", + ) + .unwrap(); + std::fs::remove_file(checkout.join("gone.ts")).unwrap(); + run_git_in(&checkout, &["add", "-A"]); + run_git_in( + &checkout, + &["commit", "-qm", "move child HEAD independently"], + ); + std::fs::write( + checkout.join("pending.ts"), + "export function pending() {}\n", + ) + .unwrap(); + assert!(on.ensure_fresh_for_query().unwrap().is_some()); + assert_eq!( + indexed_paths(&on), + ["app/lib.ts", "app/pending.ts", "dev/launch.ts"] + ); + assert_eq!( + on.latest_complete() + .unwrap() + .generation() + .repository_parse_identity() + .dirty, + RepositoryDirtyStateV1::Dirty + ); + assert_eq!(grep("successor_symbol", &enabled), ["app/lib.ts"]); + run_git_in(parent.path(), &["submodule", "deinit", "-f", "app"]); + assert!(!checkout.join(".git").exists()); + let historical = on + .publish_exact_git_tree_generation( + &pinned, + &BranchGenerationReadControlV1 { + deadline: None, + cancellation: None, + }, + ) + .unwrap(); + let historical = historical.generation(); + assert_edge(historical); + assert!( + !historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/media.bin") + ); + assert!( + historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/gone.ts") + ); + assert!( + !historical + .snapshot() + .files + .iter() + .any(|file| file.logical_path == "app/pending.ts") + ); + let pinned_file = historical + .snapshot() + .files + .iter() + .find(|file| file.logical_path == "app/lib.ts") + .unwrap(); + assert_eq!( + pinned_file.content_digest, + tracedecay_code_index::chunks::content_digest( + b"export function feature() { return 'pinned_symbol'; }\n" + ) + ); + drop(on); + let mut off = start(&parent, store.path(), policy(&[], &[])); + off.reconcile_now().unwrap(); + assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); +} + +#[test] +fn parent_filtered_source_does_not_require_submodule_registration_when_disabled() { + use super::run_git_in; + use tracedecay_code_index::chunks::content_digest; + use tracedecay_domain::LanguageId; + + let source = "pub fn parent() {}\n"; + let parent = GitFixture::new(&[ + ("lib.rs", source), + (".gitmodules", "[submodule \"child\"]\npath = child\n"), + ]); + run_git_in(parent.path(), &["config", "core.autocrlf", "true"]); + std::fs::write(parent.path().join("lib.rs"), source.replace('\n', "\r\n")).unwrap(); + std::fs::write(parent.path().join(".gitmodules"), "[unfinished").unwrap(); + let sealed = super::super::sealed_source_bytes( + parent.path(), + "lib.rs", + &LanguageId::new("rust").unwrap(), + false, + &content_digest(source.as_bytes()), + ) + .unwrap(); + assert_eq!(sealed.as_deref(), Some(source.as_bytes())); +} diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs index 35ca13c8c1d..5aa18b0df2a 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs @@ -825,13 +825,11 @@ fn unfollowable_committed_source_links_are_absent_from_disk_capture() { ); } -/// Git stores a committed source symlink as its target path, while the -/// worktree reads through it. Sealing the link text from HEAD's tree left the -/// first post-publication sweep disproving an unchanged checkout, so every -/// mount rebuilt once and served the rebuild as dirty. +/// A readable live symlink stays outside the worktree roster. The remaining +/// files can still establish a current generation without naming the full HEAD tree. #[cfg(unix)] #[test] -fn committed_source_symlink_publishes_a_current_clean_generation() { +fn committed_source_symlink_publishes_current_regular_files_only() { let fixture = GitFixture::new(&[ ("src/lib.rs", "pub mod pricing;\n"), ("src/pricing.rs", "pub fn total() -> u64 { 1 }\n"), @@ -851,16 +849,16 @@ fn committed_source_symlink_publishes_a_current_clean_generation() { !scheduler.request_fresh_now_background(), "the sweep after publication must find the unchanged checkout current" ); + let latest = scheduler.latest_complete().expect("published generation"); + let snapshot = latest.generation.snapshot(); + assert!(snapshot.source_revision.is_none()); assert_eq!( - scheduler - .latest_complete() - .expect("published generation") - .generation - .snapshot() - .source_revision - .as_ref(), - scheduler.identity.head_commit(), - "a clean checkout's generation must carry HEAD's revision" + snapshot + .files + .iter() + .map(|file| file.logical_path.as_str()) + .collect::>(), + ["src/lib.rs", "src/pricing.rs"] ); } diff --git a/crates/tracedecay-code-index/Cargo.toml b/crates/tracedecay-code-index/Cargo.toml index 5404891c52a..cba8706cb2c 100644 --- a/crates/tracedecay-code-index/Cargo.toml +++ b/crates/tracedecay-code-index/Cargo.toml @@ -66,6 +66,7 @@ lang-lean = ["tracedecay-code-extraction/lang-lean"] [dependencies] ast-grep-core = "0.50" flate2.workspace = true +gix.workspace = true miniz_oxide.workspace = true base64.workspace = true hex.workspace = true diff --git a/crates/tracedecay-code-index/benches/partitioned_codec.rs b/crates/tracedecay-code-index/benches/partitioned_codec.rs index 12a962fddf0..c9330ca623b 100644 --- a/crates/tracedecay-code-index/benches/partitioned_codec.rs +++ b/crates/tracedecay-code-index/benches/partitioned_codec.rs @@ -286,6 +286,7 @@ fn build_generation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -315,6 +316,7 @@ fn build_generation( fn production_config() -> Result> { Ok(CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.partitioned-codec")?, repository: id::("repository.partitioned-codec")?, sanitizer_revision: id::("sanitizer.partitioned-codec.v1")?, diff --git a/crates/tracedecay-code-index/benches/sealed_storage.rs b/crates/tracedecay-code-index/benches/sealed_storage.rs index f3cc2da231d..7e927b88c23 100644 --- a/crates/tracedecay-code-index/benches/sealed_storage.rs +++ b/crates/tracedecay-code-index/benches/sealed_storage.rs @@ -468,6 +468,7 @@ fn request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(id::(tree)?), dirty: RepositoryDirtyStateV1::Clean, }, @@ -482,6 +483,7 @@ fn request( fn config() -> Result> { Ok(CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.sealed-storage")?, repository: id::("repository.sealed-storage")?, sanitizer_revision: id::("sanitizer.sealed-storage.v1")?, diff --git a/crates/tracedecay-code-index/src/production/ignored_sources.rs b/crates/tracedecay-code-index/src/production/ignored_sources.rs index 2a103a5303c..beafc93a382 100644 --- a/crates/tracedecay-code-index/src/production/ignored_sources.rs +++ b/crates/tracedecay-code-index/src/production/ignored_sources.rs @@ -53,6 +53,9 @@ pub struct CodeIndexBuildRequestV1 { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct CodeIndexRepositoryParseIdentityV1 { + /// Older sealed generations used the default parent-only roster. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub include_submodules: bool, pub tree: Option, pub dirty: RepositoryDirtyStateV1, } diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source.rs b/crates/tracedecay-code-index/src/production/lexical_page_source.rs index 76497c639a9..df469bda139 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source.rs @@ -1332,6 +1332,7 @@ impl VerifiedSealedTextGenerationMetadataV1 { &self.snapshot, config, ); + compatibility.observe_submodule_policy(self.repository_parse_identity(), config); self.chunk_policy.observe(config, &mut compatibility); compatibility } diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs index 6ce31293960..6b2276f9b9e 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs @@ -611,6 +611,7 @@ fn fixture_for_source_files( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -624,6 +625,7 @@ fn fixture_for_source_files( }; let mut owner = CodeIndexProductionOwnerV1::new( CodeIndexProductionConfigV1 { + include_submodules: false, project_id: ProjectId::new("project.lexical-page-batch").expect("fixture project ID"), repository: RepositoryId::new("repository.lexical-page-batch") .expect("fixture repository ID"), diff --git a/crates/tracedecay-code-index/src/production/mod.rs b/crates/tracedecay-code-index/src/production/mod.rs index b4b48db5acb..af485652be0 100644 --- a/crates/tracedecay-code-index/src/production/mod.rs +++ b/crates/tracedecay-code-index/src/production/mod.rs @@ -151,6 +151,7 @@ pub const DAEMON_CODE_INDEX_CHUNKER_REVISION: &str = "chunker.daemon.v5"; /// Immutable configuration retained by one production index owner. #[derive(Clone, Debug)] pub struct CodeIndexProductionConfigV1 { + pub include_submodules: bool, pub project_id: ProjectId, pub repository: RepositoryId, pub sanitizer_revision: SanitizerRevision, @@ -192,6 +193,7 @@ pub enum CodeIndexGenerationIncompatibilityV1 { SanitizerRevision, PolicyRevision, MixedPolicyRevisions, + SubmodulePolicy, ChunkerRevision, PrivacyDomain, PrivacyKeyEpoch, @@ -203,6 +205,7 @@ impl CodeIndexGenerationIncompatibilityV1 { Self::Project => "project", Self::LanguageRevisions => "language_revisions", Self::SanitizerRevision => "sanitizer_revision", + Self::SubmodulePolicy => "submodule_policy", Self::PolicyRevision => "policy_revision", Self::MixedPolicyRevisions => "mixed_policy_revisions", Self::ChunkerRevision => "chunker_revision", @@ -226,6 +229,17 @@ pub struct CodeIndexGenerationCompatibilityV1 { } impl CodeIndexGenerationCompatibilityV1 { + fn observe_submodule_policy( + &mut self, + identity: &CodeIndexRepositoryParseIdentityV1, + config: &CodeIndexProductionConfigV1, + ) { + if identity.include_submodules != config.include_submodules { + self.incompatibilities + .insert(CodeIndexGenerationIncompatibilityV1::SubmodulePolicy); + } + } + fn for_metadata( manifest: &CodeGenerationManifestV1, snapshot: &SanitizedCodeSnapshotV1, @@ -1456,6 +1470,7 @@ impl CodeIndexPublishedGenerationV1 { &self.snapshot, config, ); + compatibility.observe_submodule_policy(self.repository_parse_identity(), config); self.chunk_policy_summary() .observe(config, &mut compatibility); compatibility @@ -2364,6 +2379,11 @@ where crate::observe::record_generation_state("building"); crate::observe::record_rebuild_state("unknown"); lexical_page_source::checkpoint(control)?; + if request.repository_parse_identity.include_submodules != self.config.include_submodules { + return Err(CodeIndexProductionErrorV1::Contract( + "snapshot submodule policy differs from owner".to_owned(), + )); + } let ignored_source_roster = IgnoredSourceRosterV1::admit( &request.snapshot, &request.repository_parse_identity, diff --git a/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs b/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs index 01eb1dcb764..66d8af7915c 100644 --- a/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs +++ b/crates/tracedecay-code-index/src/production/sparse_increment_tests.rs @@ -100,6 +100,7 @@ pub(super) fn request(files: &[(&str, &str, &str)], sealed_at: i64) -> CodeIndex invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: tracedecay_domain::RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/src/production/worker_tests.rs b/crates/tracedecay-code-index/src/production/worker_tests.rs index 7f210388b60..71b9eedfccc 100644 --- a/crates/tracedecay-code-index/src/production/worker_tests.rs +++ b/crates/tracedecay-code-index/src/production/worker_tests.rs @@ -39,6 +39,7 @@ where pub(super) fn worker_config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: worker_id("project.worker"), repository: worker_id("repository.worker"), sanitizer_revision: worker_id::("sanitizer.v1"), @@ -87,6 +88,7 @@ pub(super) fn worker_request_with_source( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: tracedecay_domain::RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index a72a87b77ea..78c682e2971 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -9,6 +9,9 @@ //! walk is therefore public rather than crate-private, the audit in the root //! crate reuses this policy instead of restating it. +pub mod classification; +pub mod submodules; + use std::collections::BTreeSet; use std::path::{Component, Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; @@ -17,6 +20,8 @@ use ignore::overrides::{Override, OverrideBuilder}; use ignore::{Walk, WalkBuilder}; use tracedecay_domain::{IndexPathPolicyV1, forward_slash_path}; +pub type SourceWalk = std::iter::Flatten>; + #[derive(Debug)] pub struct SourceWalkError { pub glob: String, @@ -25,6 +30,7 @@ pub struct SourceWalkError { /// Conservative directory reachability for a positive override. The override /// matcher remains authoritative for files, including directory inheritance. +#[derive(Clone)] struct PathGlobScope { literal_prefix: PathBuf, may_match_descendants: bool, @@ -340,7 +346,7 @@ pub fn source_walk( project_root: &Path, path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, -) -> Result { +) -> Result { source_walk_inner(project_root, path_glob, path_policy, None) } @@ -351,7 +357,7 @@ pub(crate) fn source_walk_with_diagnostics( project_root: &Path, path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, -) -> Result<(Walk, SourceWalkDiagnostics), SourceWalkError> { +) -> Result<(SourceWalk, SourceWalkDiagnostics), SourceWalkError> { let diagnostics = SourceWalkDiagnostics::default(); let walk = source_walk_inner( project_root, @@ -367,7 +373,7 @@ fn source_walk_inner( path_glob: Option<&str>, path_policy: &IndexPathPolicyV1, diagnostics: Option, -) -> Result { +) -> Result { let overrides = build_overrides(project_root, path_glob)?; let has_positive_override = overrides .as_ref() @@ -376,88 +382,163 @@ fn source_walk_inner( let path_glob_scope = path_glob.and_then(PathGlobScope::from_path_glob); let filter_root = project_root.to_path_buf(); let path_policy = path_policy.clone(); - let parent_gitlinks = parent_gitlink_paths(project_root)?; - - let mut builder = WalkBuilder::new(project_root); - builder - .follow_links(false) - .hidden(false) - .git_ignore(true) - .git_global(true) - .git_exclude(true) - .add_custom_ignore_filename(".gitignore") - .filter_entry(move |entry| { - if entry.depth() == 0 { - return true; - } - let segment = entry.file_name().to_string_lossy(); - if segment == ".git" || segment == ".tracedecay" { - return false; - } - let Ok(relative) = entry.path().strip_prefix(&filter_root) else { - return false; - }; - let relative = forward_slash_path(relative); - let is_dir = entry.file_type().is_some_and(|kind| kind.is_dir()); - if is_dir - && path_glob_scope + let scoped_policy = { + let root = filter_root.clone(); + let scope = path_glob_scope.clone(); + move |path: &Path, is_dir: bool| { + has_positive_override + && (generated_dir_overrides .as_ref() - .is_some_and(|scope| !scope.may_contain_match(&filter_root, entry.path())) - { - return false; + .is_some_and(|overrides| overrides.matched(path, is_dir).is_whitelist()) + || (is_dir + && scope + .as_ref() + .is_some_and(|scope| scope.allows(&root, path)))) + } + }; + let mut parent_gitlinks = parent_gitlink_paths(project_root)?; + let mut authenticated = BTreeSet::new(); + let mut child_files = BTreeSet::new(); + if path_policy.includes_submodules() && !parent_gitlinks.is_empty() { + let parent = tracedecay_runtime_core::git_open::open(project_root).map_err(|error| { + SourceWalkError { + glob: String::new(), + message: error.to_string(), } - // An entry the glob names, or a directory it must pass through, - // is judged without the generated-directory defaults: the scope - // is the operator asking for that noise. Every other exclusion - // still applies; only `index.include.v1` lifts those. - let explicitly_requested = has_positive_override - && (generated_dir_overrides.as_ref().is_some_and(|overrides| { - overrides.matched(entry.path(), is_dir).is_whitelist() - }) || (is_dir - && path_glob_scope - .as_ref() - .is_some_and(|scope| scope.allows(&filter_root, entry.path())))); - let path_policy = if explicitly_requested { + })?; + for (path, child) in submodules::live_submodules(&parent)? { + let absolute = project_root.join(&path); + let policy = if scoped_policy(&absolute, true) { path_policy.without_generated_dir_defaults() } else { &path_policy }; - if !is_dir { - return !path_policy.excludes(&relative); + if policy.excludes_directory(&path) + || path_glob_scope + .as_ref() + .is_some_and(|scope| !scope.may_contain_match(project_root, &absolute)) + { + continue; } - // Excluded directories are outside this audit, not missing source - // caused by the repository boundary. Do not probe or name them. - if path_policy.excludes_directory(&relative) { - return false; + let repo = + tracedecay_runtime_core::git_open::open(&child.git_dir).map_err(|error| { + SourceWalkError { + glob: String::new(), + message: error.to_string(), + } + })?; + let inventory = classification::WorktreeChangeClassificationV1::classify(&repo) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: error.to_string(), + })?; + let files = inventory + .regular_child_files(&child.worktree) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: error.to_string(), + })?; + child_files.extend(files.into_iter().map(|file| format!("{path}/{file}"))); + // Only direct registrations belong to this parent. + for nested in inventory.gitlink_paths() { + parent_gitlinks.insert(format!("{path}/{nested}")); + } + authenticated.insert(path); + } + } + + let roots = authenticated + .iter() + .map(|path| project_root.join(path)) + .collect::>(); + let child_dirs = child_files + .iter() + .flat_map(|path| Path::new(path).ancestors().skip(1)) + .map(Path::to_owned) + .collect::>(); + + let accept = Arc::new(move |entry: &ignore::DirEntry, in_child: bool| { + let segment = entry.file_name().to_string_lossy(); + if segment == ".git" || segment == ".tracedecay" { + return false; + } + let Ok(relative) = entry.path().strip_prefix(&filter_root) else { + return false; + }; + let relative = forward_slash_path(relative); + let is_dir = entry.file_type().is_some_and(|kind| kind.is_dir()); + if is_dir + && path_glob_scope + .as_ref() + .is_some_and(|scope| !scope.may_contain_match(&filter_root, entry.path())) + { + return false; + } + // An entry the glob names, or a directory it must pass through, + // is judged without the generated-directory defaults: the scope + // is the operator asking for that noise. Every other exclusion + // still applies; only `index.include.v1` lifts those. + let path_policy = if scoped_policy(entry.path(), is_dir) { + path_policy.without_generated_dir_defaults() + } else { + &path_policy + }; + if !is_dir { + return !path_policy.excludes(&relative) + && (!in_child || child_files.contains(&relative)); + } + // Excluded directories are outside this audit, not missing source + // caused by the repository boundary. Do not probe or name them. + if path_policy.excludes_directory(&relative) { + return false; + } + // Only an independently bound direct gitlink may cross a checkout boundary. + if authenticated.contains(&relative) { + return in_child; + } + if parent_gitlinks.contains(&relative) { + if let Some(diagnostics) = &diagnostics { + diagnostics.record_skipped_nested_repository(relative); } - // Includes select paths, not repository authority. A parent - // gitlink does not capture the child's files or prove their - // freshness, so even registered submodules remain separate - // sources. Linked worktrees and nested clones share this boundary. - // The project root is deliberately exempt at depth zero above. - if parent_gitlinks.contains(&relative) { + return false; + } + match nested_repository_marker(entry.path()) { + Ok(true) => { if let Some(diagnostics) = &diagnostics { diagnostics.record_skipped_nested_repository(relative); } - return false; + false } - match nested_repository_marker(entry.path()) { - Ok(true) => { - if let Some(diagnostics) = &diagnostics { - diagnostics.record_skipped_nested_repository(relative); - } - false - } - Ok(false) => true, - // The walk cannot prove that an unreadable Git marker is - // absent. Never admit files across that boundary. - Err(_) => false, + Ok(false) => !in_child || child_dirs.contains(Path::new(&relative)), + // The walk cannot prove that an unreadable Git marker is + // absent. Never admit files across that boundary. + Err(_) => false, + } + }); + let mut builders = vec![(WalkBuilder::new(project_root), false)]; + builders.extend(roots.into_iter().map(|root| (WalkBuilder::new(root), true))); + let walks = builders + .into_iter() + .map(|(mut builder, in_child)| { + // The child Git roster already applied that checkout's ignores, including + // tracked exceptions. Running parent ignore rules over it would hide those files. + if in_child { + builder.standard_filters(false); + } else { + builder.add_custom_ignore_filename(".gitignore"); } - }); - if let Some(overrides) = overrides { - builder.overrides(overrides); - } - Ok(builder.build()) + let accept = Arc::clone(&accept); + builder + .follow_links(false) + .hidden(false) + .filter_entry(move |entry| accept(entry, in_child)); + if let Some(overrides) = &overrides { + builder.overrides(overrides.clone()); + } + builder.build() + }) + .collect::>(); + Ok(walks.into_iter().flatten()) } /// Project-relative path rendered with forward slashes. @@ -503,7 +584,8 @@ mod tests { use tracedecay_domain::IndexPathPolicyV1; use super::{ - has_nested_repository_ancestor, is_source_symlink, source_walk, source_walk_with_diagnostics, + has_nested_repository_ancestor, is_source_symlink, source_walk, + source_walk_with_diagnostics, }; #[test] @@ -808,11 +890,45 @@ mod tests { ); } + let enabled = IndexPathPolicyV1::new(vec![], vec![]) + .unwrap() + .with_submodules(true); + fixture_git( + &child, + &[ + "worktree", + "add", + "-q", + "--detach", + root.join("other-worktree").to_str().unwrap(), + "HEAD", + ], + ); + fixture_git( + &root, + &["clone", "-q", child.to_str().unwrap(), "other-clone"], + ); + let files = walked_files(&root, &enabled); + assert!(files.contains(&PathBuf::from("app/src/feature.rs"))); + assert!(files.iter().all(|path|!path.starts_with("other-worktree") && !path.starts_with("other-clone"))); + fixture_git( + &child, + &[ + "worktree", + "remove", + root.join("other-worktree").to_str().unwrap(), + ], + ); + fs::remove_dir_all(root.join("other-clone")).unwrap(); + fs::remove_file(root.join(".gitmodules")).unwrap(); + assert!(!walked_files(&root, &enabled).contains(&PathBuf::from("app/src/feature.rs"))); + fixture_git(&root, &["checkout", "--", ".gitmodules"]); // A missing child .git marker must not turn the parent's stage-0 // gitlink into ordinary live source under an explicit include. fs::remove_file(root.join("app/.git")).expect("remove child gitdir marker"); assert!(root.join("app/src/feature.rs").is_file()); let policy = IndexPathPolicyV1::new(vec![], vec!["app/**".into()]).unwrap(); + assert!(!walked_files(&root, &enabled).contains(&PathBuf::from("app/src/feature.rs"))); let (walk, diagnostics) = source_walk_with_diagnostics(&root, None, &policy).expect("source walk"); let files = walk diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs b/crates/tracedecay-code-index/src/source_walk/classification.rs similarity index 85% rename from crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs rename to crates/tracedecay-code-index/src/source_walk/classification.rs index 0550c4a843c..931bdf5329e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/classification.rs +++ b/crates/tracedecay-code-index/src/source_walk/classification.rs @@ -17,10 +17,11 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; -use gix::bstr::ByteSlice; -use tracedecay_code_index::source_walk::{ +use super::{ has_nested_repository_ancestor, is_source_symlink, parent_gitlink_paths, + submodules::live_submodules, }; +use gix::bstr::ByteSlice; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; /// Failure to classify worktree changes through gix. @@ -100,6 +101,9 @@ pub struct WorktreeChangeClassificationV1 { /// or symlinked ancestor. These are tombstones, not indexable candidates. nested_repository_exclusions: BTreeSet, parent_gitlinks: BTreeSet, + /// Direct parent-relative child paths admitted from that checkout's own Git roster. + child_sources: BTreeMap, + child_revision_changed: bool, } impl WorktreeChangeClassificationV1 { @@ -266,9 +270,93 @@ impl WorktreeChangeClassificationV1 { Ok(classification) } - /// Explicitly enrolled files must respect the same Git boundary snapshot - /// as ordinary candidates, including a gitlink staged for removal. - pub(super) fn beneath_gitlink(&self, path: &Path) -> bool { + pub fn for_policy( + repository: &gix::Repository, + include_submodules: bool, + ) -> Result { + let mut parent = Self::classify(repository)?; + if !include_submodules || parent.parent_gitlinks.is_empty() { + return Ok(parent); + } + let children = live_submodules(repository) + .map_err(|error| ClassificationErrorV1::Git(error.message))?; + // A missing checkout must not let the clean parent-tree shortcut import children. + parent.child_revision_changed = parent + .parent_gitlinks + .iter() + .any(|path| !children.contains_key(path)); + for (prefix, binding) in children { + let child = tracedecay_runtime_core::git_open::open(&binding.git_dir) + .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; + let child_head = child + .head_id() + .map_err(|error| ClassificationErrorV1::Git(error.to_string()))?; + parent.child_revision_changed |= child_head.detach() != binding.pin; + let local = Self::classify(&child)?; + for path in local.regular_child_files(&binding.worktree)? { + let logical = format!("{prefix}/{path}"); + parent.committed_baseline.insert(logical.clone()); + parent.child_sources.insert(logical, prefix.clone()); + } + for change in local.changes { + let path = format!("{prefix}/{}", change.path); + if !change.class.presents_content() || parent.child_sources.contains_key(&path) { + parent.changes.push(ClassifiedChangeV1 { + path, + class: change.class, + }); + } + } + parent.nested_repository_exclusions.extend( + local + .nested_repository_exclusions + .into_iter() + .map(|path| format!("{prefix}/{path}")), + ); + let qualify = |path: &[u8]| [prefix.as_bytes(), b"/", path].concat(); + parent + .non_utf8_paths + .extend(local.non_utf8_paths.into_iter().map(|path| qualify(&path))); + parent + .non_utf8_changes + .extend(local.non_utf8_changes.into_iter().map(|change| { + NonUtf8ClassifiedChangeV1 { + git_path: qualify(&change.git_path), + class: change.class, + } + })); + } + Ok(parent) + } + + pub(super) fn regular_child_files(&self, root: &Path) -> std::io::Result> { + let mut files = BTreeSet::new(); + for path in self.candidate_paths() { + if !super::submodules::is_source_location(Path::new(&path)) { + continue; + } + match root.join(&path).symlink_metadata() { + Ok(metadata) if metadata.is_file() => { + files.insert(path); + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(files) + } + + pub fn child_root(&self, logical_path: &str) -> Option<&str> { + self.child_sources.get(logical_path).map(String::as_str) + } + + pub(super) fn gitlink_paths(&self) -> &BTreeSet { + &self.parent_gitlinks + } + + /// Explicit enrollment cannot cross a parent gitlink staged for removal. + pub fn beneath_gitlink(&self, path: &Path) -> bool { path.ancestors() .skip(1) .filter_map(Path::to_str) @@ -315,7 +403,8 @@ impl WorktreeChangeClassificationV1 { pub fn has_changes(&self) -> bool { !(self.changes().is_empty() && self.non_utf8_changes().is_empty() - && self.nested_repository_exclusions.is_empty()) + && self.nested_repository_exclusions.is_empty() + && !self.child_revision_changed) } /// Whether any classified change is a merge conflict. @@ -342,7 +431,7 @@ impl WorktreeChangeClassificationV1 { } /// Paths removed from the present snapshot (staged or unstaged deletions). - #[cfg(test)] + #[cfg(any(test, feature = "test-helpers"))] pub fn deleted_paths(&self) -> BTreeSet { self.changes .iter() @@ -364,7 +453,7 @@ impl WorktreeChangeClassificationV1 { } /// The class recorded for `path`, if any change touched it. - #[cfg(test)] + #[cfg(any(test, feature = "test-helpers"))] pub fn class_of(&self, path: &str) -> Option { self.changes .iter() @@ -533,7 +622,10 @@ mod nested_repository_tests { ); symlink("app/src/lib.rs", parent.join("bridge.rs")).unwrap(); fixture_git(&parent, &["add", ".gitmodules", "app", "bridge.rs"]); - fixture_git(&parent, &["commit", "-qm", "registered child and source alias"]); + fixture_git( + &parent, + &["commit", "-qm", "registered child and source alias"], + ); assert_eq!( fs::read_to_string(parent.join("bridge.rs")).unwrap(), "pub fn child() {}\n", @@ -550,7 +642,10 @@ mod nested_repository_tests { let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); assert!(!classified.candidate_paths().contains("bridge.rs")); assert!(classified.changed_paths().contains("bridge.rs")); - assert!(classified.has_changes(), "do not reuse HEAD tree source bytes"); + assert!( + classified.has_changes(), + "do not reuse HEAD tree source bytes" + ); assert!(!classified.candidate_paths().contains("app/src/lib.rs")); } diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs new file mode 100644 index 00000000000..5c08ebbcd1a --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -0,0 +1,188 @@ +//! A parent gitlink names a commit, not the worktree that may supply its bytes. +//! Bind the registration, object store, and checkout before crossing that boundary. + +use std::collections::BTreeMap; +use std::path::{Component, Path, PathBuf}; + +use gix::bstr::ByteSlice; + +use super::{SourceWalkError, has_nested_repository_ancestor}; + +fn failure(error: impl std::fmt::Display) -> SourceWalkError { + SourceWalkError { + glob: String::new(), + message: format!("cannot inspect registered submodule: {error}"), + } +} + +#[derive(Clone, Debug)] +pub struct AuthenticatedSubmodule { + pub worktree: PathBuf, + pub git_dir: PathBuf, + pub pin: gix::ObjectId, +} + +fn normal_relative(path: &Path) -> bool { + !path.as_os_str().is_empty() + && path + .components() + .all(|component| matches!(component, Component::Normal(_))) +} + +/// Administrative data is outside every parent source surface, even with an explicit glob. +pub fn is_source_location(relative: &Path) -> bool { + normal_relative(relative) + && !relative.components().any(|component| { + component.as_os_str() == ".git" || component.as_os_str() == ".tracedecay" + }) +} + +fn registrations(bytes: &[u8]) -> Result, SourceWalkError> { + let modules = gix::submodule::File::from_bytes(bytes, None, &gix::config::File::default()) + .map_err(failure)?; + let mut registrations = BTreeMap::new(); + for name in modules.names() { + let name = gix::validate::submodule::name(name).map_err(failure)?; + let path = modules.path(name).map_err(failure)?; + let path = path.to_str().map_err(failure)?; + if !is_source_location(Path::new(path)) { + continue; + } + let name = PathBuf::from(name.to_str().map_err(failure)?); + if !normal_relative(&name) + || !normal_relative(Path::new(path)) + || registrations.insert(path.to_owned(), name).is_some() + { + return Err(failure("ambiguous or invalid .gitmodules path")); + } + } + Ok(registrations) +} + +fn module_stores<'a>( + parent: &'a gix::Repository, + name: &'a Path, +) -> impl Iterator + 'a { + [parent.git_dir(), parent.common_dir()] + .into_iter() + .filter_map(move |directory| { + let expected = directory.canonicalize().ok()?.join("modules").join(name); + (expected.canonicalize().ok()? == expected).then_some(expected) + }) +} + +fn open_pinned_store(expected: &Path, pin: gix::ObjectId) -> Option { + let child = tracedecay_runtime_core::git_open::open(expected).ok()?; + if child.git_dir().canonicalize().ok()? != expected + || child.common_dir().canonicalize().ok()? != expected + { + return None; + } + child.find_commit(pin).ok()?; + Some(child) +} + +fn bound_checkout( + parent: &gix::Repository, + path: &str, + name: &Path, + pin: gix::ObjectId, +) -> Option { + let root = parent.workdir()?.canonicalize().ok()?; + let path = Path::new(path); + if has_nested_repository_ancestor(&root, path).ok()? { + return None; + } + let worktree = root.join(path); + let marker = worktree.join(".git"); + if worktree.canonicalize().ok()? != worktree || !marker.symlink_metadata().ok()?.is_file() { + return None; + } + let git_dir = gix::discover::path::from_gitdir_file(&marker) + .ok()? + .canonicalize() + .ok()?; + // Git initializes children of a linked parent under that worktree's + // administrative directory. Check the name-derived, canonical location + // before opening any repository named by the checkout's marker. + let expected = module_stores(parent, name).find(|candidate| *candidate == git_dir)?; + let child = open_pinned_store(&expected, pin)?; + if child.workdir()?.canonicalize().ok()? != worktree { + return None; + } + Some(AuthenticatedSubmodule { + worktree, + git_dir: expected, + pin, + }) +} + +pub fn live_submodules( + parent: &gix::Repository, +) -> Result, SourceWalkError> { + let mut children = BTreeMap::new(); + let index = parent.index_or_empty().map_err(failure)?; + let stage = gix::index::entry::Stage::Unconflicted; + let Some(modules) = index.entry_by_path_and_stage(".gitmodules".into(), stage) else { + return Ok(children); + }; + if !modules + .mode + .to_tree_entry_mode() + .is_some_and(|mode| mode.is_blob()) + { + return Ok(children); + } + let blob = parent.find_blob(modules.id).map_err(failure)?; + let registered = registrations(&blob.data)?; + // An unstaged deletion or replacement of the registration cannot authorize a checkout. + let Some(live_registration) = parent.open_modules_file().map_err(failure)? else { + return Ok(children); + }; + for (path, name) in registered { + let Some(entry) = index.entry_by_path_and_stage(path.as_bytes().as_bstr(), stage) else { + continue; + }; + if !entry.mode.is_submodule() { + continue; + } + if !live_registration + .path(name.as_os_str().as_encoded_bytes().as_bstr()) + .is_ok_and(|current| current.as_slice() == path.as_bytes()) + { + continue; + } + if let Some(child) = bound_checkout(parent, &path, &name, entry.id) { + children.insert(path, child); + } + } + Ok(children) +} + +/// Only the selected parent tree may supply historical registrations and pins. +pub fn pinned_submodules( + parent: &gix::Repository, + pins: &tracedecay_runtime_core::git_repository::GitTreeSubmodulePins, +) -> Result, SourceWalkError> { + let mut children = BTreeMap::new(); + if pins.gitlinks.is_empty() { + return Ok(children); + } + let registrations = registrations(&pins.registration)?; + for (path, pin) in &pins.gitlinks { + let Some(name) = registrations.get(path) else { + continue; + }; + // The checkout can move or be deinitialized after the parent revision. + // Its current bytes and .git marker are never historical authorities. + let repo = module_stores(parent, name) + .find_map(|directory| open_pinned_store(&directory, *pin)) + .ok_or_else(|| { + failure(format!( + "registered object store or pinned commit unavailable for {path}" + )) + })?; + children.insert(path.clone(), (repo, *pin)); + } + Ok(children) +} diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index 1e1558def46..5ef6dd9e3d2 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -277,6 +277,7 @@ impl CodeIndexExecutionControlV1 for MutableCancellationControl { pub(super) fn config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.production"), repository: id::("repository.production"), sanitizer_revision: id::("sanitizer.v1"), @@ -319,6 +320,7 @@ fn request_in_scope( request.snapshot.worktree = worktree.map(id::); request.snapshot.source_revision = Some(id::(source_revision)); request.repository_parse_identity = CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(id::(&format!("tree.{source_revision}"))), dirty: RepositoryDirtyStateV1::Dirty, }; @@ -365,6 +367,7 @@ fn request_at_path( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -1033,6 +1036,7 @@ fn resumed_parse_quanta_publish_the_same_complete_generation() { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -3003,6 +3007,7 @@ fn partitioned_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs index a23f502b509..7f70e4b9f82 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration/parallel_equivalence.rs @@ -92,6 +92,7 @@ fn multi_file_request(file_count: usize, sealed_at: i64) -> CodeIndexBuildReques invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, diff --git a/crates/tracedecay-code-index/tests/resident_accounting.rs b/crates/tracedecay-code-index/tests/resident_accounting.rs index 18a6afcf497..05fd76256c1 100644 --- a/crates/tracedecay-code-index/tests/resident_accounting.rs +++ b/crates/tracedecay-code-index/tests/resident_accounting.rs @@ -154,6 +154,7 @@ fn request(files: usize) -> CodeIndexBuildRequestV1 { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -168,6 +169,7 @@ fn request(files: usize) -> CodeIndexBuildRequestV1 { fn config() -> CodeIndexProductionConfigV1 { CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.accounting"), repository: id::("repository.accounting"), sanitizer_revision: id::("sanitizer.v1"), diff --git a/crates/tracedecay-configuration/src/config/mod.rs b/crates/tracedecay-configuration/src/config/mod.rs index 12628536d73..52d0ca57254 100644 --- a/crates/tracedecay-configuration/src/config/mod.rs +++ b/crates/tracedecay-configuration/src/config/mod.rs @@ -22,9 +22,9 @@ use tracedecay_domain::configuration::{ AUTOMATION_SETTINGS_SETTING_KEY, AutomationSettingsV1, ConfigurationRevisionId, ConfigurationSnapshotV1, ConfigurationValueV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, INDEX_INCLUDE_SETTING_KEY, - INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, - INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, - LcmSummarizerExecutablesV1, SYNC_AUTO_INIT_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, + INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutablesV1, SYNC_AUTO_INIT_SETTING_KEY, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, SYNC_AUTO_WATCH_SETTING_KEY, SYNC_BACKSTOP_INTERVAL_MINS_SETTING_KEY, SYNC_BRANCH_GC_DAYS_SETTING_KEY, SYNC_READ_COOLDOWN_SECS_SETTING_KEY, @@ -277,7 +277,11 @@ fn runtime_config_from_snapshot( required_string_list(snapshot, INDEX_EXCLUDE_SETTING_KEY)?, required_string_list(snapshot, INDEX_INCLUDE_SETTING_KEY)?, ) - .map_err(|error| config_error(format!("resolved index path policy is invalid: {error}")))?, + .map_err(|error| config_error(format!("resolved index path policy is invalid: {error}")))? + .with_submodules(required_bool( + snapshot, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + )?), max_file_size: required_unsigned(snapshot, INDEX_MAX_FILE_SIZE_SETTING_KEY)?, extract_docstrings: required_bool(snapshot, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY)?, track_call_sites: required_bool(snapshot, INDEX_TRACK_CALL_SITES_SETTING_KEY)?, @@ -460,6 +464,28 @@ mod tests { } } + #[test] + fn submodule_policy_is_default_off_and_materializes_the_project_override() { + let default = + PinnedRuntimeConfiguration::new(target(), revision(), resolved(BTreeMap::new())) + .unwrap(); + assert!(!default.config().index_paths.includes_submodules()); + let entries = BTreeMap::from([( + SettingKey::new(super::INDEX_INCLUDE_SUBMODULES_SETTING_KEY).unwrap(), + ConfigurationValueV1::Boolean(true), + )]); + let configured = + PinnedRuntimeConfiguration::new(target(), revision(), resolved(entries)).unwrap(); + assert!(configured.config().index_paths.includes_submodules()); + assert!( + configured + .config() + .index_paths + .without_generated_dir_defaults() + .includes_submodules() + ); + } + #[test] fn pin_rejects_a_snapshot_missing_a_required_setting() { let complete = resolved(BTreeMap::new()); diff --git a/crates/tracedecay-domain/src/configuration.rs b/crates/tracedecay-domain/src/configuration.rs index 601a3eb328f..35d41cb26cd 100644 --- a/crates/tracedecay-domain/src/configuration.rs +++ b/crates/tracedecay-domain/src/configuration.rs @@ -68,6 +68,7 @@ pub fn is_profile_setting_key(key: &SettingKey) -> bool { /// Canonical project-scoped runtime settings. pub const INDEX_EXCLUDE_SETTING_KEY: &str = "index.exclude.v1"; +pub const INDEX_INCLUDE_SUBMODULES_SETTING_KEY: &str = "index.include_submodules.v1"; pub const INDEX_INCLUDE_SETTING_KEY: &str = "index.include.v1"; pub const INDEX_MAX_FILE_SIZE_SETTING_KEY: &str = "index.max_file_size.v1"; pub const INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY: &str = "index.extract_docstrings.v1"; @@ -110,6 +111,7 @@ pub const CONFIGURATION_SETTING_KEYS_V1: &[&str] = &[ USER_WORK_EXPERTISE_CONSENT_SETTING_KEY, INDEX_EXCLUDE_SETTING_KEY, INDEX_INCLUDE_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, diff --git a/crates/tracedecay-domain/src/lib.rs b/crates/tracedecay-domain/src/lib.rs index ffbef3e2d16..764d8cd2317 100644 --- a/crates/tracedecay-domain/src/lib.rs +++ b/crates/tracedecay-domain/src/lib.rs @@ -104,9 +104,10 @@ pub use configuration::{ ContextScoutConfigurationStateV1, ContextScoutConfiguredModelPathV1, ContextScoutSettingsV1, CredentialReferenceId, CrossMergeModeV1, CrossMergePolicyV1, GitHubStackedPullRequestPolicyV1, HistoryRewritePolicyV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, - INDEX_INCLUDE_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, - LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1, + INDEX_INCLUDE_SETTING_KEY, INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, + LcmSummarizerExecutableV1, LcmSummarizerExecutablesV1, MAX_WORK_EXPERTISE_CONSENT_LIFETIME_MICROS_V1, PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY, ProtectedApplyRequest, ProtectedChange, ProtectedChangePlan, ProtectedChangeSnapshotError, ProtectedRefDispositionV1, ProtectedRefRuleV1, ProtectedRefSelectorV1, QueryCollectionId, diff --git a/crates/tracedecay-domain/src/source_path_policy.rs b/crates/tracedecay-domain/src/source_path_policy.rs index 9871687d7e4..9a921d6fb85 100644 --- a/crates/tracedecay-domain/src/source_path_policy.rs +++ b/crates/tracedecay-domain/src/source_path_policy.rs @@ -60,6 +60,7 @@ pub struct IndexPathPatternError { /// files. #[derive(Clone, Debug)] pub struct IndexPathPolicyV1 { + include_submodules: bool, exclude: Vec, include: Vec, exclude_set: GlobSet, @@ -78,7 +79,9 @@ pub struct IndexPathPolicyV1 { impl PartialEq for IndexPathPolicyV1 { fn eq(&self, other: &Self) -> bool { - self.exclude == other.exclude && self.include == other.include + self.include_submodules == other.include_submodules + && self.exclude == other.exclude + && self.include == other.include } } @@ -109,6 +112,7 @@ impl IndexPathPolicyV1 { .then(|| Self::new(configured, include.clone()).map(Box::new)) .transpose()?; Ok(Self { + include_submodules: false, exclude, include, exclude_set, @@ -132,6 +136,18 @@ impl IndexPathPolicyV1 { .unwrap_or(self) } + pub fn with_submodules(mut self, enabled: bool) -> Self { + self.include_submodules = enabled; + if let Some(scoped) = self.without_generated_dir_defaults.as_mut() { + scoped.include_submodules = enabled; + } + self + } + + pub fn includes_submodules(&self) -> bool { + self.include_submodules + } + pub fn exclude_patterns(&self) -> &[String] { &self.exclude } diff --git a/crates/tracedecay-global-db/src/configuration/registry.rs b/crates/tracedecay-global-db/src/configuration/registry.rs index 97d964bf4af..02c3b283cb9 100644 --- a/crates/tracedecay-global-db/src/configuration/registry.rs +++ b/crates/tracedecay-global-db/src/configuration/registry.rs @@ -8,9 +8,10 @@ use tracedecay_domain::configuration::{ AnalyzerSettingsV1, CONFIGURATION_SETTING_KEYS_V1, CONTEXT_SCOUT_SETTINGS_SETTING_KEY, CodeIndexWorkerSelectionV1, ConfigurationValueKindV1, ConfigurationValueV1, ContextScoutSettingsV1, INDEX_EXCLUDE_SETTING_KEY, INDEX_EXTRACT_DOCSTRINGS_SETTING_KEY, - INDEX_INCLUDE_SETTING_KEY, INDEX_MAX_FILE_SIZE_SETTING_KEY, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, INDEX_TRACK_CALL_SITES_SETTING_KEY, - LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, LcmSummarizerExecutablesV1, PROFILE_SETTING_KEYS_V1, + INDEX_INCLUDE_SETTING_KEY, INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + INDEX_MAX_FILE_SIZE_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + INDEX_TRACK_CALL_SITES_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, + LcmSummarizerExecutablesV1, PROFILE_SETTING_KEYS_V1, PROJECT_WORK_EXPERTISE_CONSENT_SETTING_KEY, RestartRequirementV1, SOURCE_BINDINGS_SETTING_KEY, SYNC_AUTO_INIT_SETTING_KEY, SYNC_AUTO_TRACK_PR_BRANCHES_SETTING_KEY, SYNC_AUTO_TRACK_PR_POLL_SECS_SETTING_KEY, SYNC_AUTO_WATCH_SETTING_KEY, @@ -35,7 +36,7 @@ pub const MAX_PROXIMITY_RISK_THRESHOLD_BASIS_POINTS_V1: u64 = 10_000; /// Registry schema revision. Increment only when setting-definition semantics /// change, not when a setting value changes. -pub const CONFIGURATION_REGISTRY_SCHEMA_REVISION: u16 = 8; +pub const CONFIGURATION_REGISTRY_SCHEMA_REVISION: u16 = 9; #[derive(Debug, Error)] pub enum ConfigurationRegistryError { @@ -523,6 +524,12 @@ fn register_project_settings( SettingSensitivityV1::Sensitive, RestartRequirementV1::DaemonRestart, ), + ( + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + ConfigurationValueV1::Boolean(false), + SettingSensitivityV1::Public, + RestartRequirementV1::DaemonRestart, + ), ( INDEX_MAX_FILE_SIZE_SETTING_KEY, ConfigurationValueV1::Unsigned(defaults.max_file_size), @@ -779,7 +786,10 @@ mod user_profile_settings_tests { ); let definition = registry.definition(&key).expect("definition"); - assert_eq!(definition.schema_revision, 8); + assert_eq!( + definition.schema_revision, + CONFIGURATION_REGISTRY_SCHEMA_REVISION + ); assert_eq!(definition.scope, SettingScopeV1::UserProfile); assert_eq!( definition.value_kind, diff --git a/crates/tracedecay-global-db/src/configuration/store.rs b/crates/tracedecay-global-db/src/configuration/store.rs index f20a0c2bf64..ff751c349a5 100644 --- a/crates/tracedecay-global-db/src/configuration/store.rs +++ b/crates/tracedecay-global-db/src/configuration/store.rs @@ -16,11 +16,11 @@ use tracedecay_domain::configuration::{ ConfigurationAuditEvent, ConfigurationAuditEventId, ConfigurationAuditEventKindV1, ConfigurationCandidateV1, ConfigurationIdempotencyKey, ConfigurationLayerIdV1, ConfigurationReceiptId, ConfigurationRevisionId, ConfigurationSnapshotV1, ConfigurationValueV1, - INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, - ProtectedChange, ProtectedChangePlan, ProtectedChangeSnapshotError, - RedactedConfigurationChangeV1, RollbackModeV1, SOURCE_BINDINGS_SETTING_KEY, - SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ScopeControlOperationV1, SettingKey, - USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, + LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, ProtectedChange, ProtectedChangePlan, + ProtectedChangeSnapshotError, RedactedConfigurationChangeV1, RollbackModeV1, + SOURCE_BINDINGS_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, ScopeControlOperationV1, + SettingKey, USER_CODE_INDEX_WORKERS_SETTING_KEY, USER_EXTRACTION_TIMEOUT_SECS_SETTING_KEY, USER_REPEATED_QUERY_HINTS_SETTING_KEY, USER_WATCHER_DEBOUNCE_MS_SETTING_KEY, UserProfileId, WORK_TOPOLOGY_POLICY_SETTING_KEY, }; @@ -349,6 +349,7 @@ impl<'db> GlobalDbConfigurationControlStore<'db> { occurred_at, ®istry, &[ + INDEX_INCLUDE_SUBMODULES_SETTING_KEY, INDEX_NATIVE_GRAPH_ACTIVATION_SETTING_KEY, LCM_SUMMARIZER_EXECUTABLES_SETTING_KEY, SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, diff --git a/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs b/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs index 5e1b7f7d9cb..8c6b4ab1288 100644 --- a/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs +++ b/crates/tracedecay-global-db/src/configuration/store/tests/mod.rs @@ -50,13 +50,19 @@ fn incomplete_snapshot_requires_reset_instead_of_default_repair() { } #[asupersync::test] -async fn linked_worktree_default_converges_into_existing_snapshot() { +async fn linked_worktree_and_submodule_defaults_converge_into_existing_snapshot() { let (_directory, runtime, root) = Box::pin(global_setup()).await; - let key = SettingKey::new(SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY).unwrap(); + let keys = [ + SYNC_WATCH_LINKED_WORKTREES_SETTING_KEY, + tracedecay_domain::configuration::INDEX_INCLUDE_SUBMODULES_SETTING_KEY, + ] + .map(|name| SettingKey::new(name).unwrap()); let mut effective_values = root.snapshot.effective_values.clone(); let mut provenance = root.snapshot.provenance.clone(); - effective_values.remove(&key); - provenance.remove(&key); + for key in &keys { + effective_values.remove(key); + provenance.remove(key); + } let historical_snapshot = ConfigurationSnapshotV1::new(effective_values, provenance).unwrap(); let db = runtime .registered_database(HostAdmissionScope::Project) @@ -69,13 +75,18 @@ async fn linked_worktree_default_converges_into_existing_snapshot() { ) .await .unwrap(); - transaction - .execute( - "DELETE FROM configuration_entries WHERE revision_id = ?1 AND key = ?2", - tracedecay_runtime_core::db::engine::params![root.revision_id.as_str(), key.as_str()], - ) - .await - .unwrap(); + for key in &keys { + transaction + .execute( + "DELETE FROM configuration_entries WHERE revision_id = ?1 AND key = ?2", + tracedecay_runtime_core::db::engine::params![ + root.revision_id.as_str(), + key.as_str() + ], + ) + .await + .unwrap(); + } point_revision_at_snapshot(&transaction, &root.revision_id, &historical_snapshot).await; transaction.commit().await.unwrap(); let store = GlobalDbConfigurationControlStore::new_registered(db); @@ -86,11 +97,13 @@ async fn linked_worktree_default_converges_into_existing_snapshot() { .unwrap(); assert_ne!(converged.revision_id, root.revision_id); - assert_eq!( - converged.snapshot.effective_values.get(&key), - Some(&ConfigurationValueV1::Boolean(false)) - ); - assert!(converged.snapshot.provenance.contains_key(&key)); + for key in &keys { + assert_eq!( + converged.snapshot.effective_values.get(key), + Some(&ConfigurationValueV1::Boolean(false)) + ); + assert!(converged.snapshot.provenance.contains_key(key)); + } validate_snapshot_registry_completeness(&converged.snapshot).unwrap(); } diff --git a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs index 75f161b692a..d8e2a8c5c1e 100644 --- a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs +++ b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs @@ -13,6 +13,7 @@ pub(super) async fn compute_field_sites( graph: &tracedecay_graph_query::VerifiedGraphQuery, args: Value, scope_prefix: Option<&str>, + include_submodules: bool, ) -> Result { let request: FieldSitesSurfaceRequestV1 = decode_primitive_request(&args, "tracedecay_field_sites")?; @@ -63,6 +64,7 @@ pub(super) async fn compute_field_sites( let needle = format!(".{field_name}"); 'outer: for file in &files { + if tracedecay_code_index::source_walk::is_source_symlink(&project_root,Path::new(file))? { continue; } let abs = project_root.join(file); let Ok(disk_source) = tracedecay_runtime_core::sync::read_source_file(&abs) else { continue; @@ -76,7 +78,7 @@ pub(super) async fn compute_field_sites( if !disk_source.contains(&needle) { continue; } - let source = sealed_field_source(&project_root, file, sealed_files.get(file))?; + let source = sealed_field_source(&project_root, file, sealed_files.get(file), include_submodules)?; let masked = path_is_rust(file).then(|| { tracedecay_code_extraction::source_mask::masked_rust_source_with( &source, @@ -198,6 +200,7 @@ fn sealed_field_source( project_root: &Path, file: &str, sealed: Option<&(LanguageId, ContentDigest)>, + include_submodules: bool, ) -> Result { let Some((language, digest)) = sealed else { return Err(TraceDecayError::project_route( @@ -208,7 +211,7 @@ fn sealed_field_source( ), )); }; - let bytes = sealed_source_bytes(project_root, file, language, digest) + let bytes = sealed_source_bytes(project_root, file, language, include_submodules, digest) .map_err(|error| { TraceDecayError::project_route( "verified-field-source-unavailable", diff --git a/crates/tracedecay-mcp/src/handlers/analysis/mod.rs b/crates/tracedecay-mcp/src/handlers/analysis/mod.rs index a6008e9cfdb..33a5babdad6 100644 --- a/crates/tracedecay-mcp/src/handlers/analysis/mod.rs +++ b/crates/tracedecay-mcp/src/handlers/analysis/mod.rs @@ -100,7 +100,13 @@ pub async fn compute_analysis_report( compute_constructors(&graph, args, scope_prefix).await } ApplicationSurfaceOperation::FieldSites => { - compute_field_sites(&graph, args, scope_prefix).await + compute_field_sites( + &graph, + args, + scope_prefix, + ctx.index_path_policy().includes_submodules(), + ) + .await } operation => Err(unknown_tool_error(operation.mcp_tool_name())), } diff --git a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs index 35f13c19289..8822df7de21 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs @@ -175,6 +175,7 @@ struct RenameReferenceSiteInput { } struct PreviewFileWalk { + include_submodules: bool, project_root: PathBuf, declaration_line: u32, result: RenamePreviewPrimitiveResultV1, @@ -202,6 +203,19 @@ impl PreviewFileWalk { let has_references = reference_inputs .peek() .is_some_and(|input| input.file == file); + if tracedecay_code_index::source_walk::is_source_symlink( + &self.project_root, + std::path::Path::new(&file), + )? { + if has_references { + return Err(TraceDecayError::project_route( + "verified-reference-source-unavailable", + true, + format!("a source symlink replaced '{file}'"), + )); + } + continue; + } let text = match tracedecay_runtime_core::sync::read_source_file( &self.project_root.join(&file), ) { @@ -220,6 +234,7 @@ impl PreviewFileWalk { &self.project_root, &file, language, + self.include_submodules, digest, ) .map_err(|error| { @@ -427,6 +442,7 @@ pub async fn compute_rename_preview( freshness: None, }; let walk = PreviewFileWalk { + include_submodules: ctx.index_path_policy().includes_submodules(), project_root: ctx.project_root().to_path_buf(), declaration_line, result, diff --git a/crates/tracedecay-query/benches/code_lexical_catchup.rs b/crates/tracedecay-query/benches/code_lexical_catchup.rs index 434a75740a9..6b45e53cd60 100644 --- a/crates/tracedecay-query/benches/code_lexical_catchup.rs +++ b/crates/tracedecay-query/benches/code_lexical_catchup.rs @@ -184,6 +184,7 @@ fn build_fixture() -> Fixture { invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -195,6 +196,7 @@ fn build_fixture() -> Fixture { }, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.catchup-benchmark"), repository: repository.clone(), sanitizer_revision, diff --git a/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs b/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs index 64681d78557..78419426251 100644 --- a/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs +++ b/crates/tracedecay-query/src/bin/tracedecay_index_bench.rs @@ -561,6 +561,7 @@ fn run(options: &Options) -> Result { let repository = identity::("repository.index-bench"); let sanitizer_revision = identity::("sanitizer.index-bench.v1"); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: identity::("project.index-bench"), repository: repository.clone(), sanitizer_revision: sanitizer_revision.clone(), @@ -736,6 +737,7 @@ fn build_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(identity::(tree)), dirty: RepositoryDirtyStateV1::Clean, }, diff --git a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs index d1d6a515c8e..9b9cd38fde5 100644 --- a/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs +++ b/crates/tracedecay-query/src/bin/tracedecay_search_bench.rs @@ -329,6 +329,7 @@ fn run(options: &Options) -> Result { let repository = identity::("repository.search-bench"); let sanitizer_revision = identity::("sanitizer.search-bench.v1"); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: identity::("project.search-bench"), repository: repository.clone(), sanitizer_revision: sanitizer_revision.clone(), @@ -726,6 +727,7 @@ fn build_request( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: Some(identity::("tree.search-bench.clean")), dirty: RepositoryDirtyStateV1::Clean, }, diff --git a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs index f186edc00ec..aa1def7ed41 100644 --- a/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs +++ b/crates/tracedecay-query/tests/search_quality_suite/candidate_producers.rs @@ -425,6 +425,7 @@ pub(crate) fn real_lexical_source_fixture_from_sources( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -436,6 +437,7 @@ pub(crate) fn real_lexical_source_fixture_from_sources( }, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.artifact"), repository: repository.clone(), sanitizer_revision, diff --git a/crates/tracedecay-runtime-core/src/git_repository.rs b/crates/tracedecay-runtime-core/src/git_repository.rs index 13c8dadc6d8..bfb65d12122 100644 --- a/crates/tracedecay-runtime-core/src/git_repository.rs +++ b/crates/tracedecay-runtime-core/src/git_repository.rs @@ -33,7 +33,7 @@ pub use history::{ pub use native_integration::{ GitNativeApplyOutcome, GitNativeCandidateTreeV1, GitNativeCandidateTreeVisitError, GitNativeIntegrationMode, GitNativePreflight, GitNativePreflightCaptureError, - GitNativePreflightDisposition, GitNativeUnsupportedReason, + GitNativePreflightDisposition, GitNativeUnsupportedReason, GitTreeSubmodulePins, }; pub use tree_links::resolve_tree_link; diff --git a/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs b/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs index e09324c149f..ada2a0215a1 100644 --- a/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs +++ b/crates/tracedecay-runtime-core/src/git_repository/native_integration.rs @@ -67,6 +67,40 @@ pub struct GitNativeCandidateTreeV1<'repository> { cancellation: &'repository CancellationToken, } +/// Registration and child commits taken from one immutable parent tree. +#[derive(Debug, Default)] +pub struct GitTreeSubmodulePins { + pub registration: Vec, + pub gitlinks: Vec<(String, gix::ObjectId)>, +} + +impl GitTreeSubmodulePins { + pub fn from_tree( + tree: &gix::Tree<'_>, + gitlinks: Vec<(String, gix::ObjectId)>, + ) -> Result { + if gitlinks.is_empty() { + return Ok(Self::default()); + } + let registration = match tree + .find_entry(".gitmodules") + .filter(|entry| entry.mode().is_blob()) + { + Some(entry) => tree + .repo + .find_blob(entry.object_id()) + .map_err(|error| operation("read pinned .gitmodules", error))? + .data + .clone(), + None => Vec::new(), + }; + Ok(Self { + registration, + gitlinks, + }) + } +} + impl GitNativeCandidateTreeV1<'_> { pub fn tree(&self) -> Result { oid(self.tree) @@ -78,9 +112,10 @@ impl GitNativeCandidateTreeV1<'_> { /// when it names none; every other symlink is visited with its link text. pub fn visit_blobs( &self, + include_submodules: bool, read_through: impl Fn(&[u8]) -> bool, mut visitor: impl FnMut(&[u8], &[u8]) -> Result<(), E>, - ) -> Result<(), GitNativeCandidateTreeVisitError> { + ) -> Result> { let tree = self .repository .find_tree(self.tree) @@ -90,6 +125,7 @@ impl GitNativeCandidateTreeV1<'_> { repository_visit_error("native candidate tree traversal", error) })?; entries.sort_by(|left, right| left.filepath.cmp(&right.filepath)); + let mut gitlinks = Vec::new(); for entry in entries { if self.cancellation.is_cancelled() { return Err(GitNativeCandidateTreeVisitError::Repository( @@ -99,6 +135,12 @@ impl GitNativeCandidateTreeV1<'_> { }, )); } + if include_submodules && entry.mode.is_commit() { + let path = entry.filepath.to_str().map_err(|error| { + repository_visit_error("native candidate gitlink path", error) + })?; + gitlinks.push((path.to_owned(), entry.oid)); + } if entry.mode.is_tree() || entry.mode.is_commit() { continue; } @@ -120,7 +162,8 @@ impl GitNativeCandidateTreeV1<'_> { visitor(entry.filepath.as_ref(), &blob.data) .map_err(GitNativeCandidateTreeVisitError::Visitor)?; } - Ok(()) + GitTreeSubmodulePins::from_tree(&tree, gitlinks) + .map_err(GitNativeCandidateTreeVisitError::Repository) } } diff --git a/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs b/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs index a1dc0be714e..2f8d6c85f09 100644 --- a/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs +++ b/crates/tracedecay-runtime-core/tests/runtime_core_suite/native_integration_authority.rs @@ -268,6 +268,7 @@ fn synthetic_candidate_tree_is_streamed_in_order_before_object_memory_drops() { let mut paths = Vec::new(); candidate .visit_blobs( + false, |_| false, |path, _bytes| { paths.push(String::from_utf8(path.to_vec()).expect("UTF-8 test path")); @@ -306,6 +307,7 @@ fn synthetic_candidate_tree_is_streamed_in_order_before_object_memory_drops() { |_, candidate| { candidate .visit_blobs( + false, |_| false, |_, _| { cancelled.cancel(); @@ -358,6 +360,7 @@ fn candidate_tree_reads_accepted_symlinks_through_to_their_blobs() { let mut visited = Vec::new(); candidate .visit_blobs( + false, |path| path.ends_with(b".rs"), |path, bytes| { visited.push(( diff --git a/crates/tracedecay-search-eval/src/candidate_output.rs b/crates/tracedecay-search-eval/src/candidate_output.rs index dcb3c2bb70b..4bb11bb8d56 100644 --- a/crates/tracedecay-search-eval/src/candidate_output.rs +++ b/crates/tracedecay-search-eval/src/candidate_output.rs @@ -1324,6 +1324,7 @@ fn publish_scope_generation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -1331,6 +1332,7 @@ fn publish_scope_generation( target_projection_key, }; let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.candidate.fixture")?, repository: id::("repository.candidate.fixture")?, sanitizer_revision: id::("sanitizer.candidate.v1")?, diff --git a/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs b/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs index eff54a990f0..b258849ab29 100644 --- a/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs +++ b/crates/tracedecay-search-eval/src/candidate_output/cancellation.rs @@ -72,6 +72,7 @@ pub(super) fn prove_cancellation( invalidations: BTreeSet::new(), ignored_source_admissions: Vec::new(), repository_parse_identity: CodeIndexRepositoryParseIdentityV1 { + include_submodules: false, tree: None, dirty: RepositoryDirtyStateV1::Dirty, }, @@ -84,6 +85,7 @@ pub(super) fn prove_cancellation( }; let generation_scope = CodeIndexGenerationScopeV1::for_snapshot(&request.snapshot); let config = CodeIndexProductionConfigV1 { + include_submodules: false, project_id: id::("project.candidate.cancel")?, repository: id::("repository.candidate.cancel")?, sanitizer_revision: id::("sanitizer.candidate.v1")?, diff --git a/docs/USER-GUIDE.md b/docs/USER-GUIDE.md index d7eb98fc29d..15a3ee90140 100644 --- a/docs/USER-GUIDE.md +++ b/docs/USER-GUIDE.md @@ -209,6 +209,22 @@ The value replaces the whole list, so start from the current effective value malformed pattern is refused before anything is written. There is no per-run folder exclusion; `init` and `sync` take no folder flags. +Direct Git submodules are excluded by default. To index their checked-out +files under parent-relative paths, set the project’s Boolean +`index.include_submodules.v1` setting and restart the daemon. + +```bash +tracedecay tool configuration_set --args '{"layer":{"kind":"project","project_id":""},"key":"index.include_submodules.v1","value":{"kind":"boolean","value":true},"expected_revision":"","idempotency_key":""}' +``` + +The parent index and `.gitmodules` must identify the child, and its checkout +must use the matching repository under the parent's Git modules directory. +Live indexing includes child worktree changes. Historical indexing reads the +child commit recorded by the selected parent revision from the local object +store. Initialize the submodule locally before requesting that history. +Linked worktrees, unrelated clones, and deeper nested repositories stay +outside the parent index. + ### Seeing what changed Use the daemon's status/coverage result to see the selected generation, exact From b96724957d79e242838c22b14b48c9d7c17e64b1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:42:02 -0800 Subject: [PATCH 26/79] fix(graph): bind foreground reads to current checkout sources --- .../code_index_scheduler/freshness_witness.rs | 32 +---- .../tests/index_path_policy_tests.rs | 20 +++- .../tracedecay-code-index/src/source_walk.rs | 74 +++++------- .../src/source_walk/submodules.rs | 111 +++++++++++++++++- .../src/handlers/analysis/field_sites.rs | 18 ++- .../src/handlers/graph/rename_preview.rs | 27 +++-- 6 files changed, 188 insertions(+), 94 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index b98634731bb..00228326aab 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -996,43 +996,21 @@ fn filter_mismatches_in_repository<'a>( /// disk first, then through the repository's own filter pipeline, the same /// two readings [`clean_filter_mismatches`] accepts as current. pub fn sealed_source_bytes( - project_root: &Path, + boundaries: &tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries, logical_path: &str, language: &LanguageId, - include_submodules: bool, sealed: &ContentDigest, ) -> Result>, CodeIndexSchedulerErrorV1> { - if tracedecay_code_index::source_walk::is_source_symlink(project_root, Path::new(logical_path))? - { + let Some((root, relative)) = boundaries.resolve(Path::new(logical_path))? else { return Ok(None); - } - let raw = - ignored_dependencies::read_bounded_snapshot_source(&project_root.join(logical_path), None)?; + }; + let raw = ignored_dependencies::read_bounded_snapshot_source(&root.join(relative), None)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &raw)?; if content_digest(&sanitized) == *sealed { return Ok(Some(sanitized)); } - // ponytail: one repository open and filter pipeline per filtered file; - // hoist them into a caller-held reader if a caller ever needs hundreds. - let repository = tracedecay_runtime_core::git_open::open(project_root) + let repository = tracedecay_runtime_core::git_open::open(root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - let mut repository = repository; - let mut relative = Path::new(logical_path); - if include_submodules { - let children = tracedecay_code_index::source_walk::submodules::live_submodules(&repository) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.message))?; - if let Some((prefix, binding)) = children - .iter() - .find(|(prefix, _)| relative.starts_with(prefix)) - { - let child_path = relative - .strip_prefix(prefix) - .map_err(|_| CodeIndexSchedulerErrorV1::Git("invalid child filter path".into()))?; - repository = tracedecay_runtime_core::git_open::open(&binding.git_dir) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - relative = child_path; - } - } let (mut pipeline, index) = repository .filter_pipeline(None) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index f90c046bd00..c0e00751f89 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -256,6 +256,17 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() .map(|hit| hit.file.to_string()) .collect::>() }; + let child_path = std::path::Path::new("app/lib.ts"); + let direct_reads = |enabled| { + tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries::open( + parent.path(), + enabled, + ) + .unwrap() + }; + assert!(direct_reads(false).resolve(child_path).unwrap().is_none()); + let active_reads = direct_reads(true); + assert!(active_reads.resolve(child_path).unwrap().is_some()); let mut off = start(&parent, store.path(), policy(&[], &[])); off.reconcile_now().unwrap(); assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); @@ -330,6 +341,7 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() assert_eq!(grep("successor_symbol", &enabled), ["app/lib.ts"]); run_git_in(parent.path(), &["submodule", "deinit", "-f", "app"]); assert!(!checkout.join(".git").exists()); + assert!(active_reads.resolve(child_path).unwrap().is_none()); let historical = on .publish_exact_git_tree_generation( &pinned, @@ -394,11 +406,15 @@ fn parent_filtered_source_does_not_require_submodule_registration_when_disabled( run_git_in(parent.path(), &["config", "core.autocrlf", "true"]); std::fs::write(parent.path().join("lib.rs"), source.replace('\n', "\r\n")).unwrap(); std::fs::write(parent.path().join(".gitmodules"), "[unfinished").unwrap(); - let sealed = super::super::sealed_source_bytes( + let boundaries = tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries::open( parent.path(), + false, + ) + .unwrap(); + let sealed = super::super::sealed_source_bytes( + &boundaries, "lib.rs", &LanguageId::new("rust").unwrap(), - false, &content_digest(source.as_bytes()), ) .unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 78c682e2971..3701db9d5f8 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -396,55 +396,37 @@ fn source_walk_inner( .is_some_and(|scope| scope.allows(&root, path)))) } }; - let mut parent_gitlinks = parent_gitlink_paths(project_root)?; + let boundaries = + submodules::CheckoutBoundaries::open(project_root, path_policy.includes_submodules())?; + let mut parent_gitlinks = boundaries.gitlinks; let mut authenticated = BTreeSet::new(); let mut child_files = BTreeSet::new(); - if path_policy.includes_submodules() && !parent_gitlinks.is_empty() { - let parent = tracedecay_runtime_core::git_open::open(project_root).map_err(|error| { - SourceWalkError { - glob: String::new(), - message: error.to_string(), - } - })?; - for (path, child) in submodules::live_submodules(&parent)? { - let absolute = project_root.join(&path); - let policy = if scoped_policy(&absolute, true) { - path_policy.without_generated_dir_defaults() - } else { - &path_policy - }; - if policy.excludes_directory(&path) - || path_glob_scope - .as_ref() - .is_some_and(|scope| !scope.may_contain_match(project_root, &absolute)) - { - continue; - } - let repo = - tracedecay_runtime_core::git_open::open(&child.git_dir).map_err(|error| { - SourceWalkError { - glob: String::new(), - message: error.to_string(), - } - })?; - let inventory = classification::WorktreeChangeClassificationV1::classify(&repo) - .map_err(|error| SourceWalkError { - glob: String::new(), - message: error.to_string(), - })?; - let files = inventory - .regular_child_files(&child.worktree) - .map_err(|error| SourceWalkError { - glob: String::new(), - message: error.to_string(), - })?; - child_files.extend(files.into_iter().map(|file| format!("{path}/{file}"))); - // Only direct registrations belong to this parent. - for nested in inventory.gitlink_paths() { - parent_gitlinks.insert(format!("{path}/{nested}")); - } - authenticated.insert(path); + for (path, child) in boundaries.children { + let absolute = project_root.join(&path); + let policy = if scoped_policy(&absolute, true) { + path_policy.without_generated_dir_defaults() + } else { + &path_policy + }; + if policy.excludes_directory(&path) + || path_glob_scope + .as_ref() + .is_some_and(|scope| !scope.may_contain_match(project_root, &absolute)) + { + continue; + } + let inventory = child.sources()?; + child_files.extend( + inventory + .files + .into_iter() + .map(|file| format!("{path}/{file}")), + ); + // Only direct registrations belong to this parent. + for nested in inventory.gitlinks { + parent_gitlinks.insert(format!("{path}/{nested}")); } + authenticated.insert(path); } let roots = authenticated diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 5c08ebbcd1a..461f79aff41 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -1,7 +1,7 @@ //! A parent gitlink names a commit, not the worktree that may supply its bytes. //! Bind the registration, object store, and checkout before crossing that boundary. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::{Component, Path, PathBuf}; use gix::bstr::ByteSlice; @@ -22,6 +22,44 @@ pub struct AuthenticatedSubmodule { pub pin: gix::ObjectId, } +pub(super) struct ChildSources { + pub files: BTreeSet, + pub gitlinks: BTreeSet, +} + +impl AuthenticatedSubmodule { + pub(super) fn sources(&self) -> Result { + let repository = tracedecay_runtime_core::git_open::open(&self.git_dir).map_err(failure)?; + let inventory = + super::classification::WorktreeChangeClassificationV1::classify(&repository) + .map_err(failure)?; + Ok(ChildSources { + files: inventory + .regular_child_files(&self.worktree) + .map_err(failure)?, + gitlinks: inventory.gitlink_paths().clone(), + }) + } +} + +pub(super) struct CheckoutBoundaries { + pub gitlinks: BTreeSet, + pub children: BTreeMap, +} + +impl CheckoutBoundaries { + pub(super) fn open(root: &Path, include_submodules: bool) -> Result { + let gitlinks = super::parent_gitlink_paths(root)?; + let children = if include_submodules && !gitlinks.is_empty() { + let parent = tracedecay_runtime_core::git_open::open(root).map_err(failure)?; + live_submodules(&parent)? + } else { + BTreeMap::new() + }; + Ok(Self { gitlinks, children }) + } +} + fn normal_relative(path: &Path) -> bool { !path.as_os_str().is_empty() && path @@ -159,6 +197,77 @@ pub fn live_submodules( Ok(children) } +/// Batch the same Git inventory for foreground graph readers. A graph path can +/// outlive its checkout binding, even when a replacement file has equal bytes. +pub struct LiveSourceBoundaries { + project_root: PathBuf, + gitlinks: BTreeSet, + children: BTreeMap)>, +} + +impl LiveSourceBoundaries { + pub fn open(root: &Path, include_submodules: bool) -> Result { + let boundaries = CheckoutBoundaries::open(root, include_submodules)?; + let mut children = BTreeMap::new(); + for (prefix, child) in boundaries.children { + let files = child.sources()?.files; + children.insert(prefix, (child, files)); + } + Ok(Self { + project_root: root.to_owned(), + gitlinks: boundaries.gitlinks, + children, + }) + } + + /// The owning checkout and its relative file path, after checking the + /// current filesystem boundary before reading any bytes. + pub fn resolve<'a, 'p>( + &'a self, + path: &'p Path, + ) -> std::io::Result> { + if !is_source_location(path) { + return Ok(None); + } + let mut root = self.project_root.as_path(); + let mut relative = path; + for ancestor in path.ancestors().skip(1).filter_map(Path::to_str) { + if !self.gitlinks.contains(ancestor) { + continue; + } + let Some((child, files)) = self.children.get(ancestor) else { + return Ok(None); + }; + relative = path.strip_prefix(ancestor).map_err(std::io::Error::other)?; + if !relative.to_str().is_some_and(|file| files.contains(file)) + || child.worktree.canonicalize().ok().as_ref() != Some(&child.worktree) + { + return Ok(None); + } + let marker = child.worktree.join(".git"); + if !marker + .symlink_metadata() + .is_ok_and(|metadata| metadata.is_file()) + || gix::discover::path::from_gitdir_file(&marker) + .ok() + .and_then(|target| target.canonicalize().ok()) + .as_ref() + != Some(&child.git_dir) + { + return Ok(None); + } + root = &child.worktree; + break; + } + if has_nested_repository_ancestor(root, relative)? + || super::is_source_symlink(root, relative)? + { + return Ok(None); + } + Ok(Some((root, relative))) + } +} + /// Only the selected parent tree may supply historical registrations and pins. pub fn pinned_submodules( parent: &gix::Repository, diff --git a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs index d8e2a8c5c1e..d3df5f2f09e 100644 --- a/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs +++ b/crates/tracedecay-mcp/src/handlers/analysis/field_sites.rs @@ -1,5 +1,6 @@ //! `tracedecay_field_sites`, read and write references to a named field. +use tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries; use tracedecay_code_index_runtime::code_index_scheduler::sealed_source_bytes; use tracedecay_contracts::retrieval::{ FieldSiteV1, FieldSitesResultV1, FieldSitesSurfaceRequestV1, @@ -62,10 +63,16 @@ pub(super) async fn compute_field_sites( let mut reads: Vec = Vec::new(); let mut touched: Vec = Vec::new(); + let boundaries = LiveSourceBoundaries::open(&project_root, include_submodules) + .map_err(|error| TraceDecayError::project_route( + "verified-field-source-unavailable", true, error.message, + ))?; let needle = format!(".{field_name}"); 'outer: for file in &files { - if tracedecay_code_index::source_walk::is_source_symlink(&project_root,Path::new(file))? { continue; } - let abs = project_root.join(file); + let Some((root, relative)) = boundaries.resolve(Path::new(file))? else { + continue; + }; + let abs = root.join(relative); let Ok(disk_source) = tracedecay_runtime_core::sync::read_source_file(&abs) else { continue; }; @@ -78,7 +85,7 @@ pub(super) async fn compute_field_sites( if !disk_source.contains(&needle) { continue; } - let source = sealed_field_source(&project_root, file, sealed_files.get(file), include_submodules)?; + let source = sealed_field_source(&boundaries, file, sealed_files.get(file))?; let masked = path_is_rust(file).then(|| { tracedecay_code_extraction::source_mask::masked_rust_source_with( &source, @@ -197,10 +204,9 @@ pub(super) async fn compute_field_sites( /// the generation sealed, which differ from the checkout when git's clean /// filters rewrite it (a CRLF checkout under `core.autocrlf=true`). fn sealed_field_source( - project_root: &Path, + boundaries: &LiveSourceBoundaries, file: &str, sealed: Option<&(LanguageId, ContentDigest)>, - include_submodules: bool, ) -> Result { let Some((language, digest)) = sealed else { return Err(TraceDecayError::project_route( @@ -211,7 +217,7 @@ fn sealed_field_source( ), )); }; - let bytes = sealed_source_bytes(project_root, file, language, include_submodules, digest) + let bytes = sealed_source_bytes(boundaries, file, language, digest) .map_err(|error| { TraceDecayError::project_route( "verified-field-source-unavailable", diff --git a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs index 8822df7de21..f7ab389acf1 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/rename_preview.rs @@ -1,12 +1,13 @@ //! `tracedecay_rename_preview`: what a rename of one symbol would touch. use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use serde_json::Value; +use tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries; use tracedecay_code_index_runtime::code_index_scheduler::sealed_source_bytes; use tracedecay_contracts::graph_tool::{GraphToolCompletionV1, GraphToolResultV1}; use tracedecay_contracts::retrieval::{ @@ -198,27 +199,30 @@ impl PreviewFileWalk { let mut text_only_matches = HashMap::new(); // Only the current file's decoded text and line index are resident. // Ordinals retain the graph's reference order across the grouped walk. + let boundaries = LiveSourceBoundaries::open(&self.project_root, self.include_submodules) + .map_err(|error| { + TraceDecayError::project_route( + "verified-reference-source-unavailable", + true, + error.message, + ) + })?; for file in files { self.control.check()?; let has_references = reference_inputs .peek() .is_some_and(|input| input.file == file); - if tracedecay_code_index::source_walk::is_source_symlink( - &self.project_root, - std::path::Path::new(&file), - )? { + let Some((root, relative)) = boundaries.resolve(Path::new(&file))? else { if has_references { return Err(TraceDecayError::project_route( "verified-reference-source-unavailable", true, - format!("a source symlink replaced '{file}'"), + format!("'{file}' is outside the current checkout source boundary"), )); } continue; - } - let text = match tracedecay_runtime_core::sync::read_source_file( - &self.project_root.join(&file), - ) { + }; + let text = match tracedecay_runtime_core::sync::read_source_file(&root.join(relative)) { Ok(text) => text, Err(error) if has_references => return Err(error.into()), // Declaration snippets are optional; reference evidence is not. @@ -231,10 +235,9 @@ impl PreviewFileWalk { // after the seal only its current text remains to map against. let sealed = match self.sealed_files.get(&file) { Some((language, digest)) => sealed_source_bytes( - &self.project_root, + &boundaries, &file, language, - self.include_submodules, digest, ) .map_err(|error| { From b25c3f54da4482046eb9e173757234b14d590044 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:24:38 -0800 Subject: [PATCH 27/79] fix(code-index): reject failed submodule freshness witnesses Refs #4593. Encode a failed authenticated probe in the persisted metadata signature, and never treat repeated failed probes as retained-generation freshness evidence. Add positive and negative behavioral regressions. --- .../src/code_index_scheduler/identity.rs | 64 +++++++++++++++++-- .../src/code_index_scheduler/reconcile.rs | 9 ++- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 728e42d7b45..39f0813488b 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -240,12 +240,18 @@ impl GitMetadataFingerprintV1 { && self.head_contents == other.head_contents } + /// Whether an authenticated submodule probe can establish a freshness witness. + pub(super) fn can_prove_freshness(&self) -> bool { + !self.submodule_probe_failed + } + /// A stable, order-independent textual signature of this fingerprint. /// /// Used to persist a restore-time freshness witness: two fingerprints share - /// a signature iff they are structurally equal (`PartialEq`), so comparing - /// the recomputed signature against a witness recorded at seal time is a - /// faithful stand-in for `differs_from`. `SystemTime` mtimes are rendered as + /// a signature iff they are structurally equal (`PartialEq`). A failed + /// submodule probe is explicitly encoded, but even matching failed probes + /// cannot prove freshness: callers must reject them, as `differs_from` does. + /// `SystemTime` mtimes are rendered as /// nanos-since-epoch so the encoding is deterministic across processes. pub fn stable_signature(&self) -> String { fn render_time(time: Option) -> String { @@ -263,10 +269,14 @@ impl GitMetadataFingerprintV1 { |value| super::sha256_hex(value.as_bytes()) ), ); - if let (Some(refs), Some(indexes)) = (&self.submodule_refs, &self.submodule_indexes) { - signature.push_str(&format!( - "|submodule_refs={refs}|submodule_indexes={indexes}" - )); + if let Some(refs) = &self.submodule_refs { + signature.push_str(&format!("|submodule_refs={refs}")); + } + if let Some(indexes) = &self.submodule_indexes { + signature.push_str(&format!("|submodule_indexes={indexes}")); + } + if self.submodule_probe_failed { + signature.push_str("|submodule_probe_failed=true"); } signature } @@ -496,6 +506,46 @@ mod tests { ); } + #[test] + fn failed_submodule_probe_cannot_reuse_a_persisted_parent_only_witness() { + let root = TempDir::new().expect("non-repository root"); + let parent_only = GitMetadataFingerprintV1::capture(root.path()); + let failed = GitMetadataFingerprintV1::for_policy(root.path(), true); + + assert!(failed.submodule_probe_failed, "opt-in probe must fail closed"); + assert!(!failed.can_prove_freshness()); + assert!(failed.differs_from(&parent_only)); + assert_ne!( + failed.stable_signature(), + parent_only.stable_signature(), + "restore cannot mistake failed authentication for parent-only freshness" + ); + assert_eq!( + GitMetadataFingerprintV1::for_policy(root.path(), false).stable_signature(), + parent_only.stable_signature(), + "the default-off persisted witness remains stable" + ); + let failed_again = GitMetadataFingerprintV1::for_policy(root.path(), true); + assert_eq!(failed.stable_signature(), failed_again.stable_signature()); + assert!( + !failed_again.can_prove_freshness(), + "repeated matching failures cannot authorize retained reuse" + ); + } + + #[test] + fn successful_submodule_probe_has_a_distinct_stable_witness() { + let repo = init_repo(&[("src/lib.rs", "pub fn example() {}\n")]); + let disabled = GitMetadataFingerprintV1::for_policy(repo.path(), false); + let enabled = GitMetadataFingerprintV1::for_policy(repo.path(), true); + + assert!(enabled.can_prove_freshness()); + assert!(!disabled.submodule_probe_failed); + assert!(enabled.submodule_refs.is_some()); + assert!(enabled.submodule_indexes.is_some()); + assert_ne!(enabled.stable_signature(), disabled.stable_signature()); + } + #[test] fn git_metadata_fingerprint_detects_a_commit() { let repo = init_repo(&[("src/lib.rs", "pub fn a() {}\n")]); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 4113a4ff273..c9c21b66661 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -2007,7 +2007,9 @@ impl CodeIndexWorktreeSchedulerV1 { return Ok(None); } let metadata = self.freshness_fence.git_metadata(&self.project_root); - if witness.git_metadata_signature != metadata.stable_signature() { + if !metadata.can_prove_freshness() + || witness.git_metadata_signature != metadata.stable_signature() + { return Ok(None); } // Re-swept under the adopted ignored-source roster. Equal metadata is @@ -2063,7 +2065,9 @@ impl CodeIndexWorktreeSchedulerV1 { return false; } let metadata = self.freshness_fence.git_metadata(&self.project_root); - if witness.git_metadata_signature != metadata.stable_signature() { + if !metadata.can_prove_freshness() + || witness.git_metadata_signature != metadata.stable_signature() + { return false; } self.worktree_stat_sweep() @@ -2266,6 +2270,7 @@ impl CodeIndexWorktreeSchedulerV1 { hook(); } let quiet_witness = sealed_bytes_match + && sampled_metadata.can_prove_freshness() && witness.as_ref().is_some_and(|witness| { witness.git_metadata_signature == sampled_metadata.stable_signature() && witness.stat_signature == sampled_sweep.signature From ebd52fe3e494b0cfb03a1bd488c1db280a407dcf Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:36:57 -0800 Subject: [PATCH 28/79] fix(code-index): require valid probes for retained graph reads --- .../src/code_index_scheduler/reconcile.rs | 15 +++-- .../tests/index_path_policy_tests.rs | 65 +++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index c9c21b66661..a820832fb4f 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -663,7 +663,7 @@ impl SourceFreshnessFenceV1 { state.source_witness = source_witness; state.source_roster = source_roster.to_vec(); state.last_reconciled_at = Instant::now(); - state.verified_against_source = true; + state.verified_against_source = state.git_metadata.can_prove_freshness(); state.reconciled_without_generation = reconciled_without_generation; state.reconciled_source_epoch = source_epoch; state.successor_build_in_flight = false; @@ -845,7 +845,7 @@ impl SourceFreshnessFenceV1 { let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); state.git_metadata = git_metadata; state.last_reconciled_at = Instant::now(); - state.verified_against_source = true; + state.verified_against_source = state.git_metadata.can_prove_freshness(); self.last_reconciled_at_micros .store(micros, Ordering::Release); } @@ -883,6 +883,10 @@ impl SourceFreshnessFenceV1 { manifest: &SourceContentManifestV1, shutting_down: &AtomicBool, ) -> (bool, freshness_witness::SourceSweepStatsV1) { + // Equal source bytes cannot authenticate a previously selected child checkout. + if !git_metadata.can_prove_freshness() { + return (false, freshness_witness::SourceSweepStatsV1::default()); + } self.sweep_cache .lock() .unwrap_or_else(PoisonError::into_inner) @@ -2263,14 +2267,15 @@ impl CodeIndexWorktreeSchedulerV1 { // generation's sealed file digests; its matching stat signature is // the negative cache that lets a moved tree skip the byte comparison. let source_manifest = SourceContentManifestV1::for_snapshot(metadata.snapshot()); - let sealed_bytes_match = - retained_is_reusable && !has_hints && self.sources_match_manifest(&source_manifest); + let sealed_bytes_match = retained_is_reusable + && !has_hints + && sampled_metadata.can_prove_freshness() + && self.sources_match_manifest(&source_manifest); #[cfg(test)] if let Some(hook) = self.after_retained_source_check.clone() { hook(); } let quiet_witness = sealed_bytes_match - && sampled_metadata.can_prove_freshness() && witness.as_ref().is_some_and(|witness| { witness.git_metadata_signature == sampled_metadata.stable_signature() && witness.stat_signature == sampled_sweep.signature diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index c0e00751f89..57e30b6b5e0 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -420,3 +420,68 @@ fn parent_filtered_source_does_not_require_submodule_registration_when_disabled( .unwrap(); assert_eq!(sealed.as_deref(), Some(source.as_bytes())); } + +#[test] +fn retained_graph_checks_require_a_successful_submodule_probe_each_time() { + let registration = "[submodule \"child\"]\npath = child\n"; + let parent = GitFixture::new(&[ + ("lib.rs", "pub fn first() {}\n"), + (".gitmodules", registration), + ]); + let store = TempDir::new().unwrap(); + // Keep the sealed attribution worktree-relative across the configuration edit. + std::fs::write(parent.path().join("lib.rs"), "pub fn latest() {}\n").unwrap(); + let mut scheduler = start( + &parent, + store.path(), + policy(&[], &[]).with_submodules(true), + ); + published(scheduler.reconcile_now().unwrap()); + let metadata = scheduler + .publication + .load_active_manifest() + .unwrap() + .unwrap() + .metadata; + // A partially staged edit leaves the worktree and its source bytes readable, + // while the index registration still cannot establish submodule identity. + let registration_path = parent.path().join(".gitmodules"); + std::fs::write(®istration_path, "[unfinished").unwrap(); + super::run_git_in(parent.path(), &["add", ".gitmodules"]); + std::fs::write(®istration_path, registration).unwrap(); + for _ in 0..2 { + assert!( + !scheduler + .freshness_fence() + .git_metadata(parent.path()) + .can_prove_freshness() + ); + let serving_witness = scheduler.currency_witness_for_sealed_snapshot( + &metadata.manifest().generation_id, + &metadata.snapshot().content_identity, + ); + let retained = scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .unwrap(); + assert_eq!( + (serving_witness.is_some(), retained.is_some()), + (false, false), + "unchanged source bytes cannot replace a failed submodule probe" + ); + } + super::run_git_in(parent.path(), &["add", ".gitmodules"]); + assert!(matches!( + scheduler + .reconcile_retained_text_generation_with(&metadata, false) + .unwrap(), + Some(CodeIndexReconcileOutcomeV1::Noop(_)) + )); + assert!( + scheduler + .currency_witness_for_sealed_snapshot( + &metadata.manifest().generation_id, + &metadata.snapshot().content_identity, + ) + .is_some() + ); +} From 478da430ede75cb22cb1f7ffc97a5f14364a0f46 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:37:44 -0800 Subject: [PATCH 29/79] style(code-index): format submodule witness assertion --- .../src/code_index_scheduler/identity.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 39f0813488b..71d215b6b83 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -512,7 +512,10 @@ mod tests { let parent_only = GitMetadataFingerprintV1::capture(root.path()); let failed = GitMetadataFingerprintV1::for_policy(root.path(), true); - assert!(failed.submodule_probe_failed, "opt-in probe must fail closed"); + assert!( + failed.submodule_probe_failed, + "opt-in probe must fail closed" + ); assert!(!failed.can_prove_freshness()); assert!(failed.differs_from(&parent_only)); assert_ne!( From 2b752c786481b9d4e4fa3d6d837459325b7f526f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:31:12 -0800 Subject: [PATCH 30/79] fix(code-index): clear submodule indexing Clippy diagnostics --- .../src/code_index_scheduler/git_tree_capture.rs | 9 +++++---- .../src/code_index_scheduler/identity.rs | 6 ++++-- .../src/code_index_scheduler/ignored_dependencies.rs | 7 +++---- .../tests/index_path_policy_tests.rs | 7 +++++-- crates/tracedecay-code-index/src/source_walk.rs | 4 +++- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs index 922d0c889e9..26b798f880b 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs @@ -35,6 +35,9 @@ use tracedecay_code_index::production::{ CodeIndexRepositoryParseIdentityV1, }; +type TreeBlobVisitor<'a> = + dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1> + 'a; + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ExactGitTreeSourceV1 { pub reference: tracedecay_domain::RefId, @@ -500,7 +503,7 @@ impl CodeIndexWorktreeSchedulerV1 { parent: &gix::Repository, pins: &GitTreeSubmodulePins, control: &branch_generations::BranchGenerationReadControlV1, - visitor: &mut dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1>, + visitor: &mut TreeBlobVisitor<'_>, ) -> Result<(), CodeIndexSearchUnavailableReasonV1> { let unavailable = CodeIndexSearchUnavailableReasonV1::GenerationUnavailable; let children = @@ -596,9 +599,7 @@ impl CodeIndexWorktreeSchedulerV1 { source_revision: Option, tree: tracedecay_domain::TreeId, control: &branch_generations::BranchGenerationReadControlV1, - visit: impl FnOnce( - &mut dyn FnMut(&[u8], &[u8]) -> Result<(), CodeIndexSearchUnavailableReasonV1>, - ) -> Result<(), CodeIndexSearchUnavailableReasonV1>, + visit: impl FnOnce(&mut TreeBlobVisitor<'_>) -> Result<(), CodeIndexSearchUnavailableReasonV1>, ) -> Result { let registry = StaticLanguageRegistry::new(); let mut roster = CapturedFileRosterV1::default(); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs index 71d215b6b83..ed1378ee977 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/identity.rs @@ -270,10 +270,12 @@ impl GitMetadataFingerprintV1 { ), ); if let Some(refs) = &self.submodule_refs { - signature.push_str(&format!("|submodule_refs={refs}")); + signature.push_str("|submodule_refs="); + signature.push_str(refs); } if let Some(indexes) = &self.submodule_indexes { - signature.push_str(&format!("|submodule_indexes={indexes}")); + signature.push_str("|submodule_indexes="); + signature.push_str(indexes); } if self.submodule_probe_failed { signature.push_str("|submodule_probe_failed=true"); diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs index 23dacd76f56..f0d6c56fa00 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/ignored_dependencies.rs @@ -1055,10 +1055,9 @@ mod tests { fixture_git(&parent, &["add", ".gitmodules", ".gitignore"]); fixture_git(&parent, &["commit", "-qm", "parent with submodule"]); fixture_git(&parent, &["rm", "--cached", "-f", "node_modules/pkg"]); - assert!( - fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]) - .stdout - .is_empty() + assert_eq!( + fixture_git(&parent, &["ls-files", "--stage", "--", "node_modules/pkg"]).stdout, + b"" ); assert!( fixture_git(&parent, &["ls-tree", "HEAD", "--", "node_modules/pkg"]) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index 57e30b6b5e0..9ec5037f423 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -270,7 +270,10 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() let mut off = start(&parent, store.path(), policy(&[], &[])); off.reconcile_now().unwrap(); assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); - assert!(grep("pinned_symbol", &policy(&[], &[])).is_empty()); + assert_eq!( + grep("pinned_symbol", &policy(&[], &[])), + Vec::::new() + ); drop(off); let mut on = start(&parent, store.path(), enabled.clone()); @@ -308,7 +311,7 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() "const private_marker = 1;", ) .unwrap(); - assert!(grep("private_marker", &enabled).is_empty()); + assert_eq!(grep("private_marker", &enabled), Vec::::new()); std::fs::write( checkout.join("lib.ts"), "export function feature() { return 'successor_symbol'; }\n", diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 3701db9d5f8..e227f3032d4 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -847,7 +847,9 @@ mod tests { &IndexPathPolicyV1::new(vec![], vec![]).unwrap(), ) .expect("source walk with nested repository diagnostics"); - let _visited = walk.map(|entry| entry.expect("source entry")).count(); + for entry in walk { + entry.expect("source entry"); + } assert_eq!( diagnostics.skipped_nested_repositories_note().as_deref(), Some("Skipped nested repositories outside parent index: \"app\"") From 26d7171bf87eebd8cd9a75abc24d9e3107670f44 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:51:17 -0800 Subject: [PATCH 31/79] fix(code-index): preserve retained source boundary refusals --- .../src/code_index_scheduler/reconcile.rs | 19 ++++++++++++++++--- crates/tracedecay-mcp/src/handlers/grep.rs | 16 +++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index a820832fb4f..6e0a7feb15e 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -4050,14 +4050,27 @@ impl CodeIndexWorktreeSchedulerV1 { } _ => (self.project_root.clone(), relative), }; - if tracedecay_code_index::source_walk::has_nested_repository_ancestor( + let refusal = if tracedecay_code_index::source_walk::has_nested_repository_ancestor( &source_root, source_relative, - )? || tracedecay_code_index::source_walk::is_source_symlink( + )? { + Some(CodeIndexIgnoredDependencyRefusalV1::NotIgnored) + } else if tracedecay_code_index::source_walk::is_source_symlink( &source_root, source_relative, )? { - return Ok(CapturedFileOutcomeV1::Absent); + Some(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) + } else { + None + }; + if let Some(reason) = refusal { + // A retained admission names required bytes. Omitting them silently + // would turn a source refusal into a broken snapshot contract. + return if explicitly_admitted { + Err(reason.into()) + } else { + Ok(CapturedFileOutcomeV1::Absent) + }; } match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), diff --git a/crates/tracedecay-mcp/src/handlers/grep.rs b/crates/tracedecay-mcp/src/handlers/grep.rs index 1cb09655889..1d09dc452ee 100644 --- a/crates/tracedecay-mcp/src/handlers/grep.rs +++ b/crates/tracedecay-mcp/src/handlers/grep.rs @@ -477,11 +477,17 @@ mod tests { "// UNIQUE_GENERATED_DIR_TOKEN\n", ) .expect("source fixture"); - std::fs::write( - project.path().join(".git"), - "gitdir: UNIQUE_GENERATED_DIR_TOKEN\n", - ) - .expect("linked-worktree git file fixture"); + let administration = tempfile::tempdir().expect("isolated Git administration"); + let git = tracedecay_runtime_core::git::try_git_program().expect("Git program"); + let initialized = std::process::Command::new(git) + .args(["init", "--quiet", "--template=", "--separate-git-dir"]) + .arg(administration.path().join("UNIQUE_GENERATED_DIR_TOKEN")) + .arg(project.path()) + .env("HOME", administration.path()) + .env("GIT_CONFIG_NOSYSTEM", "1") + .status() + .expect("initialize checkout with a Git pointer file"); + assert!(initialized.success()); std::fs::write( project.path().join(".tracedecay/internal.txt"), "UNIQUE_GENERATED_DIR_TOKEN\n", From da69c12c2a3b511cdf33d02c2377e8ff5dc0b294 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:18:56 -0800 Subject: [PATCH 32/79] fix(code-index): retain source aliases within the parent checkout --- .../code_index_scheduler/freshness_witness.rs | 41 ++++---- .../code_index_scheduler/git_tree_capture.rs | 31 ++++-- .../src/code_index_scheduler/reconcile.rs | 35 +++---- .../tracedecay-code-index/src/source_walk.rs | 8 +- .../src/source_walk/classification.rs | 60 +++++++++--- .../src/source_walk/submodules.rs | 15 ++- .../src/source_walk/tracked_links.rs | 94 +++++++++++++++++++ .../src/git_repository.rs | 2 +- .../src/git_repository/tree_links.rs | 67 +++++++++---- docs/USER-GUIDE.md | 4 +- 10 files changed, 269 insertions(+), 88 deletions(-) create mode 100644 crates/tracedecay-code-index/src/source_walk/tracked_links.rs diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs index 00228326aab..fe936ae0327 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/freshness_witness.rs @@ -248,18 +248,6 @@ fn read_candidate( project_root: &Path, candidate: &StatCandidateV1, ) -> Result, CodeIndexSchedulerErrorV1> { - // A link installed after candidate classification must not turn a - // freshness proof into a read of a different repository's source bytes. - if tracedecay_code_index::source_walk::is_source_symlink( - project_root, - Path::new(&candidate.logical_path), - )? { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "source symlink is outside the parent source inventory", - ) - .into()); - } if candidate.explicitly_admitted { ignored_dependencies::read_explicitly_admitted_source( project_root, @@ -267,13 +255,22 @@ fn read_candidate( None, ) } else { - ignored_dependencies::read_bounded_snapshot_source( - &project_root.join(&candidate.logical_path), - None, - ) + let path = resolved_candidate_path(project_root, &candidate.logical_path)?; + ignored_dependencies::read_bounded_snapshot_source(&path, None) } } +fn resolved_candidate_path(root: &Path, logical_path: &str) -> std::io::Result { + tracedecay_code_index::source_walk::resolve_parent_source(root, Path::new(logical_path))? + .map(|relative| root.join(relative)) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "source alias is outside the parent Git inventory", + ) + }) +} + /// The canonical content digest capture would seal for `raw`: the digest of /// the sanitized bytes. fn sanitized_digest( @@ -956,13 +953,17 @@ fn filter_mismatches_in_repository<'a>( let Ok((mut pipeline, index)) = repository.filter_pipeline(None) else { return disputed; }; + let repository_root = project_root.join(prefix); disputed .into_iter() .filter(|candidate| { let Some(expected) = manifest.files.get(&candidate.logical_path) else { return true; }; - let Ok(rela_path) = Path::new(&candidate.logical_path).strip_prefix(prefix) else { + let Ok(path) = resolved_candidate_path(project_root, &candidate.logical_path) else { + return true; + }; + let Ok(rela_path) = path.strip_prefix(&repository_root) else { return true; }; if candidate.explicitly_admitted @@ -972,7 +973,7 @@ fn filter_mismatches_in_repository<'a>( { return true; } - let Ok(file) = std::fs::File::open(project_root.join(&candidate.logical_path)) else { + let Ok(file) = std::fs::File::open(&path) else { return true; }; let Ok(mut converted) = pipeline.convert_to_git(file, rela_path, &index) else { @@ -1004,7 +1005,7 @@ pub fn sealed_source_bytes( let Some((root, relative)) = boundaries.resolve(Path::new(logical_path))? else { return Ok(None); }; - let raw = ignored_dependencies::read_bounded_snapshot_source(&root.join(relative), None)?; + let raw = ignored_dependencies::read_bounded_snapshot_source(&root.join(&relative), None)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &raw)?; if content_digest(&sanitized) == *sealed { return Ok(Some(sanitized)); @@ -1022,7 +1023,7 @@ pub fn sealed_source_bytes( } let mut filtered = Vec::new(); pipeline - .convert_to_git(raw.as_slice(), relative, &index) + .convert_to_git(raw.as_slice(), relative.as_ref(), &index) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? .read_to_end(&mut filtered)?; let (sanitized, _, _) = privacy::sanitize_code_file(language, &filtered)?; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs index 26b798f880b..ef4a82f1335 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/git_tree_capture.rs @@ -1403,15 +1403,32 @@ mod tests { .snapshot, ); assert_eq!( - live.iter() - .map(|(path, _)| path.as_str()) - .collect::>(), - ["AGENTS.md", "src/real.rs"] + branch, live, + "same-parent source aliases resolve to the live checkout's bytes" ); - for regular in live { + let changed = b"pub fn linked_target() -> u32 { 19 }\n"; + std::fs::write(root.join("src/real.rs"), changed).expect("edit only the alias target"); + let edited = present( + &scheduler + .capture_authoritative_snapshot_without_active_generation_reuse(None) + .expect("capture unstaged target through each tracked alias") + .snapshot, + ); + let expected = tracedecay_code_index::chunks::content_digest(changed); + for alias in [ + "src/alias.rs", + "src/nested/chained.rs", + "src/through_directory.rs", + ] { + assert!( + edited + .iter() + .any(|(path, digest)| path == alias && *digest == expected) + ); assert!( - branch.contains(®ular), - "historical and live regular files must carry the same source digest" + branch + .iter() + .any(|(path, digest)| path == alias && *digest != expected) ); } let notes = captured diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 6e0a7feb15e..56e5ea5a844 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -4050,28 +4050,29 @@ impl CodeIndexWorktreeSchedulerV1 { } _ => (self.project_root.clone(), relative), }; - let refusal = if tracedecay_code_index::source_walk::has_nested_repository_ancestor( + let resolved = if tracedecay_code_index::source_walk::has_nested_repository_ancestor( &source_root, source_relative, )? { - Some(CodeIndexIgnoredDependencyRefusalV1::NotIgnored) - } else if tracedecay_code_index::source_walk::is_source_symlink( - &source_root, - source_relative, - )? { - Some(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) + Err(CodeIndexIgnoredDependencyRefusalV1::NotIgnored) + } else if explicitly_admitted + && tracedecay_code_index::source_walk::is_source_symlink(&source_root, source_relative)? + { + Err(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) } else { - None + tracedecay_code_index::source_walk::resolve_parent_source( + &source_root, + source_relative, + )? + .ok_or(CodeIndexIgnoredDependencyRefusalV1::SymlinkEscape) }; - if let Some(reason) = refusal { - // A retained admission names required bytes. Omitting them silently - // would turn a source refusal into a broken snapshot contract. - return if explicitly_admitted { - Err(reason.into()) - } else { - Ok(CapturedFileOutcomeV1::Absent) - }; - } + let resolved = match resolved { + Ok(path) => path, + // Retained admissions name required bytes. A refusal must remain visible. + Err(reason) if explicitly_admitted => return Err(reason.into()), + Err(_) => return Ok(CapturedFileOutcomeV1::Absent), + }; + let absolute = source_root.join(resolved); match absolute.metadata() { Ok(metadata) if !metadata.is_file() => return Ok(CapturedFileOutcomeV1::Absent), Ok(_) => {} diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index e227f3032d4..93b6b3847e3 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -11,6 +11,8 @@ pub mod classification; pub mod submodules; +mod tracked_links; +pub use tracked_links::resolve_parent_source; use std::collections::BTreeSet; use std::path::{Component, Path, PathBuf}; @@ -253,9 +255,7 @@ pub fn has_nested_repository_ancestor( Ok(false) } -/// A worktree source entry must be a real file, not a link to bytes outside -/// the parent's Git authority. The shared live walker does not follow leaf -/// symlinks; classification, capture, and freshness must make the same choice. +/// Distinguish leaf links before selecting their source authority. /// Missing paths remain possible tombstones and are not symlink exclusions. pub fn is_source_symlink(project_root: &Path, relative_path: &Path) -> std::io::Result { if relative_path.as_os_str().is_empty() @@ -616,7 +616,7 @@ mod tests { #[cfg(unix)] #[test] - fn readable_symlink_leaf_is_not_an_admissible_source() { + fn leaf_link_detection_preserves_missing_paths() { use std::os::unix::fs::symlink; let root = TempDir::new().unwrap(); diff --git a/crates/tracedecay-code-index/src/source_walk/classification.rs b/crates/tracedecay-code-index/src/source_walk/classification.rs index 931bdf5329e..2042361ca32 100644 --- a/crates/tracedecay-code-index/src/source_walk/classification.rs +++ b/crates/tracedecay-code-index/src/source_walk/classification.rs @@ -18,7 +18,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use super::{ - has_nested_repository_ancestor, is_source_symlink, parent_gitlink_paths, + has_nested_repository_ancestor, parent_gitlink_paths, resolve_parent_source, submodules::live_submodules, }; use gix::bstr::ByteSlice; @@ -241,12 +241,8 @@ impl WorktreeChangeClassificationV1 { { let relative = Path::new(path); let parent = relative.parent().unwrap_or_else(|| Path::new("")); - // The live walker never follows a symlink file, even when - // the lexical parent is ordinary and the target is inside a - // registered submodule. Do not capture its target through a - // clean-tree or worktree path. let excluded = if classification.beneath_gitlink(relative) - || is_source_symlink(project_root, relative)? + || resolve_parent_source(project_root, relative)?.is_none() { true } else { @@ -621,7 +617,19 @@ mod nested_repository_tests { ], ); symlink("app/src/lib.rs", parent.join("bridge.rs")).unwrap(); - fixture_git(&parent, &["add", ".gitmodules", "app", "bridge.rs"]); + fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); + symlink("parent.rs", parent.join("local.rs")).unwrap(); + fixture_git( + &parent, + &[ + "add", + ".gitmodules", + "app", + "bridge.rs", + "parent.rs", + "local.rs", + ], + ); fixture_git( &parent, &["commit", "-qm", "registered child and source alias"], @@ -639,14 +647,40 @@ mod nested_repository_tests { ); let repository = tracedecay_runtime_core::git_open::open(&parent).unwrap(); - let classified = WorktreeChangeClassificationV1::classify(&repository).unwrap(); - assert!(!classified.candidate_paths().contains("bridge.rs")); - assert!(classified.changed_paths().contains("bridge.rs")); + for include_submodules in [false, true] { + let classified = + WorktreeChangeClassificationV1::for_policy(&repository, include_submodules) + .unwrap(); + let candidates = classified.candidate_paths(); + let boundaries = crate::source_walk::submodules::LiveSourceBoundaries::open( + &parent, + include_submodules, + ) + .unwrap(); + let (_, relative) = boundaries.resolve(Path::new("local.rs")).unwrap().unwrap(); + assert_eq!(relative.as_ref(), Path::new("parent.rs")); + assert!( + boundaries + .resolve(Path::new("bridge.rs")) + .unwrap() + .is_none() + ); + assert!(candidates.contains("local.rs")); + assert!(!candidates.contains("bridge.rs")); + assert!(classified.changed_paths().contains("bridge.rs")); + assert!( + classified.has_changes(), + "do not reuse HEAD tree source bytes" + ); + assert_eq!(candidates.contains("app/src/lib.rs"), include_submodules); + } + symlink("parent.rs", parent.join("untracked_alias.rs")).unwrap(); + let classification = WorktreeChangeClassificationV1::classify(&repository).unwrap(); assert!( - classified.has_changes(), - "do not reuse HEAD tree source bytes" + !classification + .candidate_paths() + .contains("untracked_alias.rs") ); - assert!(!classified.candidate_paths().contains("app/src/lib.rs")); } #[test] diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 461f79aff41..0c068e8d328 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -1,6 +1,7 @@ //! A parent gitlink names a commit, not the worktree that may supply its bytes. //! Bind the registration, object store, and checkout before crossing that boundary. +use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Component, Path, PathBuf}; @@ -225,7 +226,7 @@ impl LiveSourceBoundaries { pub fn resolve<'a, 'p>( &'a self, path: &'p Path, - ) -> std::io::Result> { + ) -> std::io::Result)>> { if !is_source_location(path) { return Ok(None); } @@ -259,12 +260,16 @@ impl LiveSourceBoundaries { root = &child.worktree; break; } - if has_nested_repository_ancestor(root, relative)? - || super::is_source_symlink(root, relative)? - { + if has_nested_repository_ancestor(root, relative)? { return Ok(None); } - Ok(Some((root, relative))) + if root == self.project_root { + Ok(super::resolve_parent_source(root, relative)?.map(|path| (root, path))) + } else if super::is_source_symlink(root, relative)? { + Ok(None) + } else { + Ok(Some((root, Cow::Borrowed(relative)))) + } } } diff --git a/crates/tracedecay-code-index/src/source_walk/tracked_links.rs b/crates/tracedecay-code-index/src/source_walk/tracked_links.rs new file mode 100644 index 00000000000..35d47fa9149 --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/tracked_links.rs @@ -0,0 +1,94 @@ +//! Resolve a tracked source alias using the parent Git index and live worktree. + +use std::borrow::Cow; +use std::io; +use std::path::{Component, Path}; + +use gix::bstr::ByteSlice; +use gix::index::entry::{Mode, Stage}; +use tracedecay_runtime_core::git_repository::{RepositoryLinkEntry, resolve_repository_link}; + +use super::{is_source_symlink, nested_repository_marker, submodules::is_source_location}; + +/// Ordinary files keep their original path. A live parent alias may only read +/// a tracked regular file through tracked links and parent-owned directories. +pub fn resolve_parent_source<'a>( + root: &Path, + relative: &'a Path, +) -> io::Result>> { + if !is_source_symlink(root, relative)? { + return Ok(Some(Cow::Borrowed(relative))); + } + if !nested_repository_marker(root)? { + return Ok(None); + } + let repo = tracedecay_runtime_core::git_open::open(root).map_err(io::Error::other)?; + let index = repo.index_or_empty().map_err(io::Error::other)?; + let head = repo.head().map_err(io::Error::other)?; + let tree = if head.id().is_some() { + Some(repo.head_tree().map_err(io::Error::other)?) + } else { + None + }; + let alias = gix::path::into_bstr(relative).map_err(io::Error::other)?; + let alias = gix::path::to_unix_separators_on_windows(alias); + if !index + .entry_by_path_and_stage(&alias, Stage::Unconflicted) + .is_some_and(|entry| entry.mode == Mode::SYMLINK) + { + return Ok(None); + } + resolve_repository_link(b"", &alias, |components| { + let bytes = components.join(&b'/'); + let path = gix::path::from_byte_slice(&bytes).map_err(io::Error::other)?; + if !is_source_location(path) { + return Ok(None); + } + let absolute = root.join(path); + let metadata = match absolute.symlink_metadata() { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let indexed = index.entry_by_path_and_stage(bytes.as_bstr(), Stage::Unconflicted); + if metadata.is_dir() { + // A gitlink staged for removal still bounds the old checkout even + // after its marker disappears. Check HEAD before descending. + let head_entry = tree + .as_ref() + .map(|tree| tree.lookup_entry(components.iter().map(Vec::as_slice))) + .transpose() + .map_err(io::Error::other)? + .flatten(); + if index.entry_by_path(bytes.as_bstr()).is_some() + || head_entry.is_some_and(|entry| entry.mode().is_commit()) + || nested_repository_marker(&absolute)? + { + return Ok(None); + } + return Ok(Some(RepositoryLinkEntry::Directory)); + } + let Some(entry) = indexed else { + return Ok(None); + }; + if metadata.file_type().is_symlink() && entry.mode == Mode::SYMLINK { + let target = std::fs::read_link(&absolute)?; + if target + .components() + .any(|component| matches!(component, Component::Prefix(_) | Component::RootDir)) + { + return Ok(None); + } + let target = gix::path::into_bstr(target).map_err(io::Error::other)?; + return Ok(Some(RepositoryLinkEntry::Link( + gix::path::to_unix_separators_on_windows(target).to_vec(), + ))); + } + Ok((metadata.is_file() + && entry + .mode + .to_tree_entry_mode() + .is_some_and(|mode| mode.is_blob())) + .then(|| RepositoryLinkEntry::File(Cow::Owned(path.to_owned())))) + }) +} diff --git a/crates/tracedecay-runtime-core/src/git_repository.rs b/crates/tracedecay-runtime-core/src/git_repository.rs index bfb65d12122..e978a2cce04 100644 --- a/crates/tracedecay-runtime-core/src/git_repository.rs +++ b/crates/tracedecay-runtime-core/src/git_repository.rs @@ -35,7 +35,7 @@ pub use native_integration::{ GitNativeIntegrationMode, GitNativePreflight, GitNativePreflightCaptureError, GitNativePreflightDisposition, GitNativeUnsupportedReason, GitTreeSubmodulePins, }; -pub use tree_links::resolve_tree_link; +pub use tree_links::{RepositoryLinkEntry, resolve_repository_link, resolve_tree_link}; /// A typed failure from the in-process Git repository authority. #[derive(Clone, Debug, thiserror::Error)] diff --git a/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs b/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs index ccee3e049f4..31e81fe6525 100644 --- a/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs +++ b/crates/tracedecay-runtime-core/src/git_repository/tree_links.rs @@ -15,6 +15,42 @@ pub fn resolve_tree_link( link_path: &[u8], target: &[u8], ) -> Result, GitRepositoryError> { + resolve_repository_link(link_path, target, |path| { + let Some(entry) = tree + .lookup_entry(path.iter().map(Vec::as_slice)) + .map_err(|error| operation("tree link lookup", error))? + else { + return Ok(None); + }; + let mode = entry.mode(); + Ok(if mode.is_tree() { + Some(RepositoryLinkEntry::Directory) + } else if mode.is_link() { + let link = entry + .object() + .map_err(|error| operation("tree link target", error))?; + Some(RepositoryLinkEntry::Link(link.data.clone())) + } else if mode.is_blob() { + Some(RepositoryLinkEntry::File(entry.object_id())) + } else { + None + }) + }) +} + +pub enum RepositoryLinkEntry { + Directory, + Link(Vec), + File(T), +} + +/// Walk relative link components with the caller's repository authority. Each +/// directory is checked before descending, including one followed by `..`. +pub fn resolve_repository_link( + link_path: &[u8], + target: &[u8], + mut lookup: impl FnMut(&[Vec]) -> Result>, E>, +) -> Result, E> { let mut resolved = components(link_path).collect::>(); resolved.pop(); let mut pending = Vec::new(); @@ -33,26 +69,17 @@ pub fn resolve_tree_link( } _ => resolved.push(component), } - let Some(entry) = tree - .lookup_entry(resolved.iter().map(Vec::as_slice)) - .map_err(|error| operation("tree link lookup", error))? - else { - return Ok(None); - }; - let mode = entry.mode(); - if mode.is_tree() { - continue; - } - if !mode.is_link() { - return Ok((pending.is_empty() && mode.is_blob()).then(|| entry.object_id())); - } - hops += 1; - resolved.pop(); - let link = entry - .object() - .map_err(|error| operation("tree link target", error))?; - if hops > MAX_LINK_HOPS || !push_target(&mut pending, &link.data) { - return Ok(None); + match lookup(&resolved)? { + Some(RepositoryLinkEntry::Directory) => {} + Some(RepositoryLinkEntry::Link(target)) => { + hops += 1; + resolved.pop(); + if hops > MAX_LINK_HOPS || !push_target(&mut pending, &target) { + return Ok(None); + } + } + Some(RepositoryLinkEntry::File(file)) if pending.is_empty() => return Ok(Some(file)), + _ => return Ok(None), } } Ok(None) diff --git a/docs/USER-GUIDE.md b/docs/USER-GUIDE.md index 15a3ee90140..5ef9797bc9d 100644 --- a/docs/USER-GUIDE.md +++ b/docs/USER-GUIDE.md @@ -219,7 +219,9 @@ tracedecay tool configuration_set --args '{"layer":{"kind":"project","project_id The parent index and `.gitmodules` must identify the child, and its checkout must use the matching repository under the parent's Git modules directory. -Live indexing includes child worktree changes. Historical indexing reads the +Live indexing includes child worktree changes. The parent graph retains tracked +source aliases that resolve through tracked links to a tracked parent file. A +parent alias cannot cross into a child checkout. Historical indexing reads the child commit recorded by the selected parent revision from the local object store. Initialize the submodule locally before requesting that history. Linked worktrees, unrelated clones, and deeper nested repositories stay From f36fbb3841056efbef51057d6e6fdc9573bc81b2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:38:44 -0800 Subject: [PATCH 33/79] test(code-index): require clean revisions for tracked aliases --- .../tests/code_index_scheduler/reconcile.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs index 5aa18b0df2a..318dc166f93 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs @@ -825,11 +825,11 @@ fn unfollowable_committed_source_links_are_absent_from_disk_capture() { ); } -/// A readable live symlink stays outside the worktree roster. The remaining -/// files can still establish a current generation without naming the full HEAD tree. +/// Publishing the link target bytes under both tracked paths must preserve +/// the clean HEAD revision and avoid a redundant post-publication rebuild. #[cfg(unix)] #[test] -fn committed_source_symlink_publishes_current_regular_files_only() { +fn committed_source_symlink_publishes_a_current_clean_generation() { let fixture = GitFixture::new(&[ ("src/lib.rs", "pub mod pricing;\n"), ("src/pricing.rs", "pub fn total() -> u64 { 1 }\n"), @@ -851,14 +851,17 @@ fn committed_source_symlink_publishes_current_regular_files_only() { ); let latest = scheduler.latest_complete().expect("published generation"); let snapshot = latest.generation.snapshot(); - assert!(snapshot.source_revision.is_none()); + assert_eq!( + snapshot.source_revision.as_ref(), + scheduler.identity.head_commit() + ); assert_eq!( snapshot .files .iter() .map(|file| file.logical_path.as_str()) .collect::>(), - ["src/lib.rs", "src/pricing.rs"] + ["src/lib.rs", "src/pricing.rs", "src/pricing_alias.rs"] ); } From 88ca6c1dee8bb9cea75aaea30370834226d4f04d Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:46:37 -0800 Subject: [PATCH 34/79] fix(code-index): preserve authenticated child capture on Windows Refs #4593 Normalize the existing child-root canonical identity before comparing it to the scheduler's plain path. Windows canonicalize returns a verbatim disk path even for the same checkout; the raw comparison silently dropped live child candidates after a child HEAD advance. Keep the strict existing-root/no-symlink boundary. The already-failing native Windows submodule graph regression exercises this path; run it on the new head with Bazel before accepting. --- .../src/code_index_scheduler/reconcile.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 56e5ea5a844..c2028cd1243 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -4036,7 +4036,14 @@ impl CodeIndexWorktreeSchedulerV1 { return Ok(CapturedFileOutcomeV1::Absent); } let root = self.project_root.join(prefix); - if root.canonicalize().ok().as_ref() != Some(&root) + // The scheduler stores a plain canonical path. Windows + // canonicalize() returns a verbatim \\?\\ disk path, so compare + // through the same existing-root identity without relaxing + // the no-symlink/no-escape boundary. + if tracedecay_runtime_core::path_safety::canonical_existing_identity(&root) + .ok() + .as_ref() + != Some(&root) || !absolute .symlink_metadata() .is_ok_and(|metadata| metadata.is_file()) From f0874897e1d49b9542fe12d2c3735b32e68b53d6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:27:12 -0800 Subject: [PATCH 35/79] fix(code-index): revalidate lazy submodule source walks Retain each authenticated child binding while its source iterator is alive. Reuse the graph reader's checkout/marker liveness proof before advancing the child walk, including the depth-zero root. Return a typed walk error and permanently stop that child if its binding changes. Add real-Git regressions for marker removal, retargeting, mid-iteration binding loss and a replaced root, with positive parent/child controls. Rust regression execution and native verification remain required. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 23 ++- .../src/source_walk/submodules.rs | 76 +++++-- .../src/source_walk/submodules/tests.rs | 194 ++++++++++++++++++ 3 files changed, 270 insertions(+), 23 deletions(-) create mode 100644 crates/tracedecay-code-index/src/source_walk/submodules/tests.rs diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index 93b6b3847e3..aa71768a395 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -18,11 +18,11 @@ use std::collections::BTreeSet; use std::path::{Component, Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; +use ignore::WalkBuilder; use ignore::overrides::{Override, OverrideBuilder}; -use ignore::{Walk, WalkBuilder}; use tracedecay_domain::{IndexPathPolicyV1, forward_slash_path}; -pub type SourceWalk = std::iter::Flatten>; +pub type SourceWalk = std::iter::Flatten>; #[derive(Debug)] pub struct SourceWalkError { @@ -401,6 +401,7 @@ fn source_walk_inner( let mut parent_gitlinks = boundaries.gitlinks; let mut authenticated = BTreeSet::new(); let mut child_files = BTreeSet::new(); + let mut roots = Vec::new(); for (path, child) in boundaries.children { let absolute = project_root.join(&path); let policy = if scoped_policy(&absolute, true) { @@ -427,12 +428,9 @@ fn source_walk_inner( parent_gitlinks.insert(format!("{path}/{nested}")); } authenticated.insert(path); + roots.push((absolute, child)); } - let roots = authenticated - .iter() - .map(|path| project_root.join(path)) - .collect::>(); let child_dirs = child_files .iter() .flat_map(|path| Path::new(path).ancestors().skip(1)) @@ -497,11 +495,16 @@ fn source_walk_inner( Err(_) => false, } }); - let mut builders = vec![(WalkBuilder::new(project_root), false)]; - builders.extend(roots.into_iter().map(|root| (WalkBuilder::new(root), true))); + let mut builders = vec![(WalkBuilder::new(project_root), None)]; + builders.extend( + roots + .into_iter() + .map(|(root, child)| (WalkBuilder::new(root), Some(child))), + ); let walks = builders .into_iter() - .map(|(mut builder, in_child)| { + .map(|(mut builder, child)| { + let in_child = child.is_some(); // The child Git roster already applied that checkout's ignores, including // tracked exceptions. Running parent ignore rules over it would hide those files. if in_child { @@ -517,7 +520,7 @@ fn source_walk_inner( if let Some(overrides) = &overrides { builder.overrides(overrides.clone()); } - builder.build() + submodules::CheckoutWalk::new(builder.build(), child) }) .collect::>(); Ok(walks.into_iter().flatten()) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 0c068e8d328..e7811a6dabc 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -3,9 +3,11 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; +use std::io; use std::path::{Component, Path, PathBuf}; use gix::bstr::ByteSlice; +use ignore::{DirEntry, Error as WalkError, Walk}; use super::{SourceWalkError, has_nested_repository_ancestor}; @@ -29,6 +31,19 @@ pub(super) struct ChildSources { } impl AuthenticatedSubmodule { + fn checkout_is_live(&self) -> bool { + let marker = self.worktree.join(".git"); + self.worktree.canonicalize().ok().as_ref() == Some(&self.worktree) + && marker + .symlink_metadata() + .is_ok_and(|metadata| metadata.is_file()) + && gix::discover::path::from_gitdir_file(&marker) + .ok() + .and_then(|target| target.canonicalize().ok()) + .as_ref() + == Some(&self.git_dir) + } + pub(super) fn sources(&self) -> Result { let repository = tracedecay_runtime_core::git_open::open(&self.git_dir).map_err(failure)?; let inventory = @@ -43,6 +58,50 @@ impl AuthenticatedSubmodule { } } +/// A lazy walk that cannot outlive its authenticated child checkout binding. +/// The ordinary parent walk has no child binding and retains its own policy. +pub struct CheckoutWalk { + walk: Option, + child: Option, +} + +impl CheckoutWalk { + pub(super) fn new(walk: Walk, child: Option) -> Self { + Self { + walk: Some(walk), + child, + } + } +} + +impl Iterator for CheckoutWalk { + type Item = Result; + + fn next(&mut self) -> Option { + self.walk.as_ref()?; + if let Some(child) = &self.child + && !child.checkout_is_live() + { + // Check before polling, including the root entry: ignore does not + // apply filter_entry to a walk's root. Stop permanently after a + // binding change, even if the original marker is later restored. + self.walk = None; + return Some(Err(WalkError::WithPath { + path: child.worktree.clone(), + err: Box::new(WalkError::Io(io::Error::new( + io::ErrorKind::InvalidData, + "authenticated submodule checkout binding changed during source walk", + ))), + })); + } + let next = self.walk.as_mut()?.next(); + if next.is_none() { + self.walk = None; + } + next + } +} + pub(super) struct CheckoutBoundaries { pub gitlinks: BTreeSet, pub children: BTreeMap, @@ -241,19 +300,7 @@ impl LiveSourceBoundaries { }; relative = path.strip_prefix(ancestor).map_err(std::io::Error::other)?; if !relative.to_str().is_some_and(|file| files.contains(file)) - || child.worktree.canonicalize().ok().as_ref() != Some(&child.worktree) - { - return Ok(None); - } - let marker = child.worktree.join(".git"); - if !marker - .symlink_metadata() - .is_ok_and(|metadata| metadata.is_file()) - || gix::discover::path::from_gitdir_file(&marker) - .ok() - .and_then(|target| target.canonicalize().ok()) - .as_ref() - != Some(&child.git_dir) + || !child.checkout_is_live() { return Ok(None); } @@ -300,3 +347,6 @@ pub fn pinned_submodules( } Ok(children) } + +#[cfg(test)] +mod tests; diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs new file mode 100644 index 00000000000..2766aeb4913 --- /dev/null +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -0,0 +1,194 @@ +use std::collections::BTreeSet; +use std::fs; +use std::io; +use std::path::Path; +use std::process::Command; + +use ignore::Error as WalkError; +use tempfile::TempDir; +use tracedecay_domain::IndexPathPolicyV1; + +use super::LiveSourceBoundaries; +use crate::source_walk::{SourceWalk, source_walk}; + +fn git(root: &Path, args: &[&str]) { + let program = tracedecay_runtime_core::git::try_git_program().expect("fixture Git"); + let mut command = Command::new(program); + command.current_dir(root); + for (key, _) in std::env::vars_os() { + if key.to_string_lossy().starts_with("GIT_") { + command.env_remove(key); + } + } + let output = command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", root.join(".fixture-gitconfig")) + .args([ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run fixture Git"); + assert!( + output.status.success(), + "Git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +fn fixture() -> TempDir { + let fixture = TempDir::new().unwrap(); + let parent = fixture.path().join("parent"); + let child = fixture.path().join("child"); + fs::create_dir_all(&parent).unwrap(); + fs::create_dir_all(child.join("nested")).unwrap(); + git(&parent, &["init", "-q"]); + git(&child, &["init", "-q"]); + fs::write(parent.join("parent.rs"), "pub fn parent() {}\n").unwrap(); + fs::write(child.join("first.rs"), "pub fn first() {}\n").unwrap(); + fs::write(child.join("nested/second.rs"), "pub fn second() {}\n").unwrap(); + git(&child, &["add", "."]); + git(&child, &["commit", "-qm", "child source"]); + git( + &parent, + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + "../child", + "app", + ], + ); + git(&parent, &["add", "."]); + git(&parent, &["commit", "-qm", "parent with child"]); + fixture +} + +fn walk(root: &Path) -> SourceWalk { + let policy = IndexPathPolicyV1::new(vec![], vec![]) + .unwrap() + .with_submodules(true); + source_walk(root, None, &policy).expect("authenticated source walk") +} + +fn assert_live_control(root: &Path) { + let files = walk(root) + .map(|entry| entry.expect("unchanged checkout")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| entry.path().strip_prefix(root).unwrap().to_path_buf()) + .collect::>(); + for source in ["parent.rs", "app/first.rs", "app/nested/second.rs"] { + assert!(files.contains(Path::new(source)), "{source}: {files:?}"); + } +} + +fn assert_binding_error(error: &WalkError) { + assert_eq!( + error.io_error().expect("typed binding refusal").kind(), + io::ErrorKind::InvalidData + ); + assert!( + error.to_string().contains("checkout binding changed"), + "{error}" + ); +} + +fn assert_refused_walk(root: &Path, walk: SourceWalk) { + let mut files = BTreeSet::new(); + let mut errors = 0; + for entry in walk { + match entry { + Ok(entry) if entry.file_type().is_some_and(|kind| kind.is_file()) => { + files.insert(entry.path().strip_prefix(root).unwrap().to_path_buf()); + } + Ok(_) => {} + Err(error) => { + assert_binding_error(&error); + errors += 1; + } + } + } + assert_eq!(errors, 1, "binding loss must not be a successful empty walk"); + assert!(files.contains(Path::new("parent.rs"))); + assert!(files.iter().all(|path| !path.starts_with("app"))); +} + +fn marker_change_before_iteration(replacement: Option<&str>) { + let fixture = fixture(); + let root = fixture.path().join("parent"); + assert_live_control(&root); + let boundaries = LiveSourceBoundaries::open(&root, true).unwrap(); + let path = Path::new("app/first.rs"); + assert!(boundaries.resolve(path).unwrap().is_some()); + let pending = walk(&root); + let marker = root.join("app/.git"); + match replacement { + Some(contents) => fs::write(&marker, contents).unwrap(), + None => fs::remove_file(&marker).unwrap(), + } + assert!(root.join(path).is_file(), "the child bytes are still present"); + assert!(boundaries.resolve(path).unwrap().is_none()); + assert_refused_walk(&root, pending); +} + +#[test] +fn lazy_submodule_walk_refuses_removed_marker() { + marker_change_before_iteration(None); +} + +#[test] +fn lazy_submodule_walk_refuses_retargeted_marker() { + // The source repository has identical tracked files but is not the + // parent's previously authenticated module store. + marker_change_before_iteration(Some("gitdir: ../../child/.git\n")); +} + +#[test] +fn lazy_submodule_walk_stops_after_mid_iteration_binding_loss() { + let fixture = fixture(); + let root = fixture.path().join("parent"); + assert_live_control(&root); + let mut pending = walk(&root); + let first = pending + .by_ref() + .map(|entry| entry.expect("unchanged checkout")) + .find(|entry| { + entry.file_type().is_some_and(|kind| kind.is_file()) + && entry.path().starts_with(root.join("app")) + }) + .expect("at least one child source was yielded"); + assert!(first.path().is_file()); + let marker = root.join("app/.git"); + let original = fs::read(&marker).unwrap(); + fs::remove_file(&marker).unwrap(); + let error = pending + .next() + .expect("binding refusal after a yielded child file") + .expect_err("must revalidate before advancing the child walk"); + assert_binding_error(&error); + fs::write(&marker, original).unwrap(); + assert!(pending.next().is_none(), "a failed walk cannot resume"); +} + +#[cfg(unix)] +#[test] +fn lazy_submodule_walk_refuses_replaced_root_before_root_entry() { + let fixture = fixture(); + let root = fixture.path().join("parent"); + assert_live_control(&root); + let pending = walk(&root); + let app = root.join("app"); + fs::rename(&app, fixture.path().join("detached-child")).unwrap(); + std::os::unix::fs::symlink(fixture.path().join("child"), &app).unwrap(); + assert!(app.join("first.rs").is_file()); + // ignore does not filter its depth-zero entry. The guard must run + // before polling this separate child root, not only in filter_entry. + assert_refused_walk(&root, pending); +} From 7473407babeb56d5d2eb7916e98c190b17f871ca Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:32:26 -0800 Subject: [PATCH 36/79] fix(code-index): require explicit sealed submodule policy Refs #4593 Reject older generation parse identities without an include_submodules field rather than silently treating them as default-off. Seal both true and false explicitly, with a regression for round-trip and omitted-field refusal. Apply the two exact Bazel rustfmt corrections reported for the new authenticated submodule source-walk regressions. Native Bazel behavioral tests and independent review remain required. --- .../src/production/ignored_sources.rs | 43 ++++++++++++++++++- .../src/source_walk/submodules/tests.rs | 10 ++++- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay-code-index/src/production/ignored_sources.rs b/crates/tracedecay-code-index/src/production/ignored_sources.rs index beafc93a382..438ec1c1d72 100644 --- a/crates/tracedecay-code-index/src/production/ignored_sources.rs +++ b/crates/tracedecay-code-index/src/production/ignored_sources.rs @@ -53,8 +53,8 @@ pub struct CodeIndexBuildRequestV1 { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct CodeIndexRepositoryParseIdentityV1 { - /// Older sealed generations used the default parent-only roster. - #[serde(default, skip_serializing_if = "std::ops::Not::not")] + /// Every sealed generation records its admission policy explicitly. + /// Missing policy evidence is an incompatible persisted shape, not default-off. pub include_submodules: bool, pub tree: Option, pub dirty: RepositoryDirtyStateV1, @@ -185,3 +185,42 @@ impl CodeIndexPublishedGenerationV1 { self.ignored_source_roster.digest() } } + +#[cfg(test)] +mod parse_identity_tests { + use super::CodeIndexRepositoryParseIdentityV1; + use tracedecay_domain::RepositoryDirtyStateV1; + + #[test] + fn sealed_parse_identity_requires_an_explicit_submodule_policy() { + for include_submodules in [false, true] { + let identity = CodeIndexRepositoryParseIdentityV1 { + include_submodules, + tree: None, + dirty: RepositoryDirtyStateV1::Clean, + }; + let serialized = serde_json::to_value(&identity).expect("serialize parse identity"); + assert_eq!( + serialized + .get("include_submodules") + .and_then(serde_json::Value::as_bool), + Some(include_submodules) + ); + let restored: CodeIndexRepositoryParseIdentityV1 = + serde_json::from_value(serialized.clone()).expect("explicit policy round trip"); + assert_eq!(restored, identity); + + let mut missing_policy = serialized; + missing_policy + .as_object_mut() + .expect("parse identity object") + .remove("include_submodules"); + let refused: Result = + serde_json::from_value(missing_policy); + assert!( + refused.unwrap_err().to_string().contains("include_submodules"), + "missing policy must be rejected" + ); + } + } +} diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 2766aeb4913..41f3cf7e74f 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -115,7 +115,10 @@ fn assert_refused_walk(root: &Path, walk: SourceWalk) { } } } - assert_eq!(errors, 1, "binding loss must not be a successful empty walk"); + assert_eq!( + errors, 1, + "binding loss must not be a successful empty walk" + ); assert!(files.contains(Path::new("parent.rs"))); assert!(files.iter().all(|path| !path.starts_with("app"))); } @@ -133,7 +136,10 @@ fn marker_change_before_iteration(replacement: Option<&str>) { Some(contents) => fs::write(&marker, contents).unwrap(), None => fs::remove_file(&marker).unwrap(), } - assert!(root.join(path).is_file(), "the child bytes are still present"); + assert!( + root.join(path).is_file(), + "the child bytes are still present" + ); assert!(boundaries.resolve(path).unwrap().is_none()); assert_refused_walk(&root, pending); } From 71602073d01bfdda3971da98c6ceef79fa73d8bc Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:02:07 -0800 Subject: [PATCH 37/79] perf(code-index): restrict scoped Git boundary traversal Avoid scanning every parent index entry and HEAD-tree object before a literal-prefix source search. Probe index ancestors and the sorted prefix range, and resolve only HEAD ancestors and the matching subtree. Broad or basename scopes keep the complete fail-closed boundary inventory. Add a real-Git regression where an unrelated committed subtree is unreadable: its absence must not block an exact-file scope, while a full source walk must still refuse the incomplete HEAD authority. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 199 ++++++++++++++++-- .../src/source_walk/submodules.rs | 10 +- 2 files changed, 190 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index aa71768a395..f62ae0d1ede 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -128,6 +128,21 @@ fn nested_repository_marker(directory: &Path) -> std::io::Result { /// not an ordinary directory the live source walker can authenticate. /// This is deny-only; it never enables submodule admission. pub fn parent_gitlink_paths(project_root: &Path) -> Result, SourceWalkError> { + parent_gitlink_paths_scoped(project_root, None) +} + +/// A positive source glob with a literal prefix cannot inspect paths outside +/// that prefix. Keep ancestor gitlinks and any gitlinks below a named directory. +pub(super) fn parent_gitlink_paths_scoped( + project_root: &Path, + scope: Option<&Path>, +) -> Result, SourceWalkError> { + let scope = scope.filter(|scope| { + !scope.as_os_str().is_empty() + && scope + .components() + .all(|component| matches!(component, Component::Normal(_))) + }); let repository = match tracedecay_runtime_core::git_open::open(project_root) { Ok(repository) => repository, Err(error) => { @@ -163,14 +178,56 @@ pub fn parent_gitlink_paths(project_root: &Path) -> Result, Sou message: format!("cannot inspect parent Git index: {error}"), })?; let mut paths = BTreeSet::new(); - for entry in index.entries() { - if entry.mode.is_submodule() { - let path: &[u8] = entry.path(&index).as_ref(); + { + let mut record = |path: &[u8]| -> Result<(), SourceWalkError> { let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { glob: String::new(), message: "non-UTF-8 parent gitlink cannot be safely scoped".to_owned(), })?; paths.insert(relative.to_owned()); + Ok(()) + }; + let entries = index.entries(); + if let Some(scope) = scope { + // Git index entries are sorted by byte-exact path. Probe every + // ancestor, including unmerged stages, then only the prefix range. + for ancestor in scope.ancestors().filter(|path| !path.as_os_str().is_empty()) { + let name = forward_slash_path(ancestor); + let bytes = name.as_bytes(); + let start = entries.partition_point(|entry| { + let path: &[u8] = entry.path(&index).as_ref(); + path < bytes + }); + for entry in entries[start..].iter().take_while(|entry| { + let path: &[u8] = entry.path(&index).as_ref(); + path == bytes + }) { + if entry.mode.is_submodule() { + record(bytes)?; + } + } + } + let prefix = format!("{}/", forward_slash_path(scope)); + let start = entries.partition_point(|entry| { + let path: &[u8] = entry.path(&index).as_ref(); + path < prefix.as_bytes() + }); + for entry in entries[start..].iter().take_while(|entry| { + let path: &[u8] = entry.path(&index).as_ref(); + path.starts_with(prefix.as_bytes()) + }) { + if entry.mode.is_submodule() { + let path: &[u8] = entry.path(&index).as_ref(); + record(path)?; + } + } + } else { + for entry in entries { + if entry.mode.is_submodule() { + let path: &[u8] = entry.path(&index).as_ref(); + record(path)?; + } + } } } // A staged gitlink removal does not erase the checkout boundary from @@ -189,24 +246,78 @@ pub fn parent_gitlink_paths(project_root: &Path) -> Result, Sou glob: String::new(), message: format!("cannot read parent Git HEAD tree: {error}"), })?; - let entries = tree - .traverse() - .breadthfirst - .files() - .map_err(|error| SourceWalkError { - glob: String::new(), - message: format!("cannot walk parent Git HEAD tree: {error}"), - })?; - for entry in entries { - if !entry.mode.is_commit() { - continue; - } - let path: &[u8] = entry.filepath.as_ref(); + let mut record = |path: &[u8]| -> Result<(), SourceWalkError> { let relative = std::str::from_utf8(path).map_err(|_| SourceWalkError { glob: String::new(), message: "non-UTF-8 HEAD gitlink cannot be safely scoped".to_owned(), })?; paths.insert(relative.to_owned()); + Ok(()) + }; + if let Some(scope) = scope { + // A staged gitlink removal survives in HEAD even without a child + // marker. Follow only the scope's ancestors and committed subtree. + let ancestors = scope + .ancestors() + .filter(|path| !path.as_os_str().is_empty()) + .collect::>(); + for ancestor in ancestors.into_iter().rev() { + let Some(entry) = tree + .lookup_entry_by_path(ancestor) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect scoped parent HEAD tree: {error}"), + })? + else { + break; + }; + if entry.mode().is_commit() { + record(forward_slash_path(ancestor).as_bytes())?; + break; + } + if !entry.mode().is_tree() { + break; + } + if ancestor == scope { + let subtree = repository + .find_tree(entry.object_id()) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot read scoped parent HEAD tree: {error}"), + })?; + let entries = subtree + .traverse() + .breadthfirst + .files() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot walk scoped parent HEAD tree: {error}"), + })?; + let prefix = forward_slash_path(scope); + for entry in entries { + if entry.mode.is_commit() { + let mut path = prefix.as_bytes().to_vec(); + path.push(b'/'); + path.extend_from_slice(entry.filepath.as_ref()); + record(&path)?; + } + } + } + } + } else { + let entries = tree + .traverse() + .breadthfirst + .files() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot walk parent Git HEAD tree: {error}"), + })?; + for entry in entries { + if entry.mode.is_commit() { + record(entry.filepath.as_ref())?; + } + } } } else if head.referent_name().is_none() { return Err(SourceWalkError { @@ -396,8 +507,14 @@ fn source_walk_inner( .is_some_and(|scope| scope.allows(&root, path)))) } }; - let boundaries = - submodules::CheckoutBoundaries::open(project_root, path_policy.includes_submodules())?; + let literal_scope = path_glob_scope.as_ref().and_then(|scope| { + (!scope.literal_prefix.as_os_str().is_empty()).then_some(scope.literal_prefix.as_path()) + }); + let boundaries = submodules::CheckoutBoundaries::open_scoped( + project_root, + path_policy.includes_submodules(), + literal_scope, + )?; let mut parent_gitlinks = boundaries.gitlinks; let mut authenticated = BTreeSet::new(); let mut child_files = BTreeSet::new(); @@ -814,6 +931,52 @@ mod tests { output } + #[test] + fn a_literal_scope_does_not_read_unrelated_committed_trees() { + let root = TempDir::new().expect("parent checkout"); + fixture_git(root.path(), &["init", "-q"]); + for (name, contents) in [ + ("src/keep.rs", "pub fn kept() {}\n"), + ("unrelated/deep/noise.rs", "pub fn noise() {}\n"), + ] { + let path = root.path().join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, contents).unwrap(); + } + fixture_git(root.path(), &["add", "."]); + fixture_git(root.path(), &["commit", "-qm", "two independent trees"]); + let output = fixture_git(root.path(), &["rev-parse", "HEAD:unrelated"]); + let oid = String::from_utf8(output.stdout).unwrap(); + let oid = oid.trim(); + assert!(oid.len() > 2 && oid.bytes().all(|byte| byte.is_ascii_hexdigit())); + let object = root + .path() + .join(".git/objects") + .join(&oid[..2]) + .join(&oid[2..]); + assert!(object.is_file(), "fixture must have a loose unrelated tree"); + fs::remove_file(&object).unwrap(); + + let policy = IndexPathPolicyV1::new(vec![], vec![]).unwrap(); + let files = source_walk(root.path(), Some("src/keep.rs"), &policy) + .expect("literal scope must not read unrelated HEAD tree") + .map(|entry| entry.expect("scoped source entry")) + .filter(|entry| entry.file_type().is_some_and(|kind| kind.is_file())) + .map(|entry| { + entry + .into_path() + .strip_prefix(root.path()) + .unwrap() + .to_path_buf() + }) + .collect::>(); + assert_eq!(files, vec![PathBuf::from("src/keep.rs")]); + assert!( + source_walk(root.path(), None, &policy).is_err(), + "an unscoped walk must still refuse unreadable HEAD tree authority" + ); + } + #[test] fn includes_cannot_admit_registered_submodule_files_as_parent_source() { let fixture = TempDir::new().expect("repository fixture"); diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index e7811a6dabc..57c49b81635 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -109,7 +109,15 @@ pub(super) struct CheckoutBoundaries { impl CheckoutBoundaries { pub(super) fn open(root: &Path, include_submodules: bool) -> Result { - let gitlinks = super::parent_gitlink_paths(root)?; + Self::open_scoped(root, include_submodules, None) + } + + pub(super) fn open_scoped( + root: &Path, + include_submodules: bool, + scope: Option<&Path>, + ) -> Result { + let gitlinks = super::parent_gitlink_paths_scoped(root, scope)?; let children = if include_submodules && !gitlinks.is_empty() { let parent = tracedecay_runtime_core::git_open::open(root).map_err(failure)?; live_submodules(&parent)? From 967d9d19d0081703d92a25f9abc392e1744f1aeb Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:14:33 -0800 Subject: [PATCH 38/79] style(code-index): apply Bazel rustfmt to scoped Git boundaries Apply the five precise formatting corrections from repository-gates run 42740034779 for source_walk.rs. No behavior or policy changes. Refs #4593 --- .../tracedecay-code-index/src/source_walk.rs | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk.rs b/crates/tracedecay-code-index/src/source_walk.rs index f62ae0d1ede..a2bb37cadaa 100644 --- a/crates/tracedecay-code-index/src/source_walk.rs +++ b/crates/tracedecay-code-index/src/source_walk.rs @@ -191,7 +191,10 @@ pub(super) fn parent_gitlink_paths_scoped( if let Some(scope) = scope { // Git index entries are sorted by byte-exact path. Probe every // ancestor, including unmerged stages, then only the prefix range. - for ancestor in scope.ancestors().filter(|path| !path.as_os_str().is_empty()) { + for ancestor in scope + .ancestors() + .filter(|path| !path.as_os_str().is_empty()) + { let name = forward_slash_path(ancestor); let bytes = name.as_bytes(); let start = entries.partition_point(|entry| { @@ -262,12 +265,12 @@ pub(super) fn parent_gitlink_paths_scoped( .filter(|path| !path.as_os_str().is_empty()) .collect::>(); for ancestor in ancestors.into_iter().rev() { - let Some(entry) = tree - .lookup_entry_by_path(ancestor) - .map_err(|error| SourceWalkError { - glob: String::new(), - message: format!("cannot inspect scoped parent HEAD tree: {error}"), - })? + let Some(entry) = + tree.lookup_entry_by_path(ancestor) + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot inspect scoped parent HEAD tree: {error}"), + })? else { break; }; @@ -279,20 +282,18 @@ pub(super) fn parent_gitlink_paths_scoped( break; } if ancestor == scope { - let subtree = repository - .find_tree(entry.object_id()) - .map_err(|error| SourceWalkError { + let subtree = repository.find_tree(entry.object_id()).map_err(|error| { + SourceWalkError { glob: String::new(), message: format!("cannot read scoped parent HEAD tree: {error}"), - })?; - let entries = subtree - .traverse() - .breadthfirst - .files() - .map_err(|error| SourceWalkError { + } + })?; + let entries = subtree.traverse().breadthfirst.files().map_err(|error| { + SourceWalkError { glob: String::new(), message: format!("cannot walk scoped parent HEAD tree: {error}"), - })?; + } + })?; let prefix = forward_slash_path(scope); for entry in entries { if entry.mode.is_commit() { @@ -305,14 +306,14 @@ pub(super) fn parent_gitlink_paths_scoped( } } } else { - let entries = tree - .traverse() - .breadthfirst - .files() - .map_err(|error| SourceWalkError { - glob: String::new(), - message: format!("cannot walk parent Git HEAD tree: {error}"), - })?; + let entries = + tree.traverse() + .breadthfirst + .files() + .map_err(|error| SourceWalkError { + glob: String::new(), + message: format!("cannot walk parent Git HEAD tree: {error}"), + })?; for entry in entries { if entry.mode.is_commit() { record(entry.filepath.as_ref())?; From 2647906c2ac4de4a8fea89e11f928f0858f80116 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:22:02 -0800 Subject: [PATCH 39/79] style(code-index): format explicit submodule policy rejection --- .../tracedecay-code-index/src/production/ignored_sources.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/production/ignored_sources.rs b/crates/tracedecay-code-index/src/production/ignored_sources.rs index 438ec1c1d72..fba65f7ed18 100644 --- a/crates/tracedecay-code-index/src/production/ignored_sources.rs +++ b/crates/tracedecay-code-index/src/production/ignored_sources.rs @@ -218,7 +218,10 @@ mod parse_identity_tests { let refused: Result = serde_json::from_value(missing_policy); assert!( - refused.unwrap_err().to_string().contains("include_submodules"), + refused + .unwrap_err() + .to_string() + .contains("include_submodules"), "missing policy must be rejected" ); } From bfe14a24e51106c061dbc163b20e2817ac6adfd7 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:07:32 -0800 Subject: [PATCH 40/79] fix(code-index): revalidate parent submodule authority during reads Refs #4593 Recheck the parent's exact staged mode-160000 gitlink and pin, staged .gitmodules identity, live registration and owned module store on existing child source and graph reads. The child marker alone is not sufficient authority. Add real-Git regressions for working and staged registration removal, staged gitlink removal and pin replacement after reader construction. These new Rust tests still require exact-head nonzero Bazel validation. --- .../src/source_walk/submodules.rs | 64 ++++++++++++++++++- .../src/source_walk/submodules/tests.rs | 47 ++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 57c49b81635..cd633f2b32f 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -23,6 +23,10 @@ pub struct AuthenticatedSubmodule { pub worktree: PathBuf, pub git_dir: PathBuf, pub pin: gix::ObjectId, + parent_root: PathBuf, + module_path: String, + module_name: PathBuf, + staged_modules_id: gix::ObjectId, } pub(super) struct ChildSources { @@ -31,6 +35,57 @@ pub(super) struct ChildSources { } impl AuthenticatedSubmodule { + fn parent_registration_is_live(&self) -> bool { + // An already-open iterator or graph reader cannot continue to use a + // child after the parent revokes its staged gitlink or registration. + // Inspect exact index entries, not the entire committed Git tree. + let Ok(parent) = tracedecay_runtime_core::git_open::open(&self.parent_root) else { + return false; + }; + if parent + .workdir() + .and_then(|root| root.canonicalize().ok()) + .as_ref() + != Some(&self.parent_root) + { + return false; + } + // A replaced parent Git directory cannot inherit the old child's + // authorization merely by advertising the same staged paths. + if !module_stores(&parent, &self.module_name).any(|store| store == self.git_dir) { + return false; + } + let Ok(index) = parent.index_or_empty() else { + return false; + }; + let stage = gix::index::entry::Stage::Unconflicted; + let Some(modules) = index.entry_by_path_and_stage(".gitmodules".into(), stage) else { + return false; + }; + if modules.id != self.staged_modules_id + || !modules + .mode + .to_tree_entry_mode() + .is_some_and(|mode| mode.is_blob()) + { + return false; + } + let Some(gitlink) = + index.entry_by_path_and_stage(self.module_path.as_bytes().as_bstr(), stage) + else { + return false; + }; + if !gitlink.mode.is_submodule() || gitlink.id != self.pin { + return false; + } + let Ok(Some(working)) = parent.open_modules_file() else { + return false; + }; + working + .path(self.module_name.as_os_str().as_encoded_bytes().as_bstr()) + .is_ok_and(|path| path.as_slice() == self.module_path.as_bytes()) + } + fn checkout_is_live(&self) -> bool { let marker = self.worktree.join(".git"); self.worktree.canonicalize().ok().as_ref() == Some(&self.worktree) @@ -42,6 +97,7 @@ impl AuthenticatedSubmodule { .and_then(|target| target.canonicalize().ok()) .as_ref() == Some(&self.git_dir) + && self.parent_registration_is_live() } pub(super) fn sources(&self) -> Result { @@ -193,8 +249,10 @@ fn bound_checkout( path: &str, name: &Path, pin: gix::ObjectId, + staged_modules_id: gix::ObjectId, ) -> Option { let root = parent.workdir()?.canonicalize().ok()?; + let module_path = path.to_owned(); let path = Path::new(path); if has_nested_repository_ancestor(&root, path).ok()? { return None; @@ -220,6 +278,10 @@ fn bound_checkout( worktree, git_dir: expected, pin, + parent_root: root, + module_path, + module_name: name.to_path_buf(), + staged_modules_id, }) } @@ -258,7 +320,7 @@ pub fn live_submodules( { continue; } - if let Some(child) = bound_checkout(parent, &path, &name, entry.id) { + if let Some(child) = bound_checkout(parent, &path, &name, entry.id, modules.id) { children.insert(path, child); } } diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 41f3cf7e74f..bc13ac8cd38 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -144,6 +144,53 @@ fn marker_change_before_iteration(replacement: Option<&str>) { assert_refused_walk(&root, pending); } +fn parent_authority_change_before_iteration(change: impl FnOnce(&Path)) { + let fixture = fixture(); + let root = fixture.path().join("parent"); + assert_live_control(&root); + let boundaries = LiveSourceBoundaries::open(&root, true).unwrap(); + let path = Path::new("app/first.rs"); + assert!(boundaries.resolve(path).unwrap().is_some()); + let pending = walk(&root); + change(&root); + assert!(root.join(path).is_file(), "the same child bytes still exist"); + assert!(root.join("app/.git").is_file(), "child marker is unchanged"); + assert!(boundaries.resolve(path).unwrap().is_none()); + assert_refused_walk(&root, pending); +} + +#[test] +fn lazy_submodule_walk_refuses_removed_worktree_registration() { + parent_authority_change_before_iteration(|root| { + fs::remove_file(root.join(".gitmodules")).unwrap(); + }); +} + +#[test] +fn lazy_submodule_walk_refuses_removed_staged_gitlink() { + parent_authority_change_before_iteration(|root| { + git(root, &["rm", "-q", "--cached", "app"]); + }); +} + +#[test] +fn lazy_submodule_walk_refuses_removed_staged_registration() { + parent_authority_change_before_iteration(|root| { + git(root, &["rm", "-q", "--cached", ".gitmodules"]); + }); +} + +#[test] +fn lazy_submodule_walk_refuses_updated_staged_child_pin() { + parent_authority_change_before_iteration(|root| { + let child = root.join("app"); + fs::write(child.join("first.rs"), "pub fn updated() {}\n").unwrap(); + git(&child, &["add", "first.rs"]); + git(&child, &["commit", "-qm", "updated child"]); + git(root, &["add", "app"]); + }); +} + #[test] fn lazy_submodule_walk_refuses_removed_marker() { marker_change_before_iteration(None); From 5f2d9f307ad0b83724702ba56c84dd6a3b715a89 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 22:31:07 -0800 Subject: [PATCH 41/79] style(code-index): apply Bazel rustfmt to submodule authority test Refs #4593 Apply the single exact formatter hunk reported by the current-head hosted Bazel rustfmt_check_2024 failure (run 42761466450). Retain all four new behavioral regression assertions unchanged. --- .../src/source_walk/submodules/tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index bc13ac8cd38..e2f479a5fb8 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -153,7 +153,10 @@ fn parent_authority_change_before_iteration(change: impl FnOnce(&Path)) { assert!(boundaries.resolve(path).unwrap().is_some()); let pending = walk(&root); change(&root); - assert!(root.join(path).is_file(), "the same child bytes still exist"); + assert!( + root.join(path).is_file(), + "the same child bytes still exist" + ); assert!(root.join("app/.git").is_file(), "child marker is unchanged"); assert!(boundaries.resolve(path).unwrap().is_none()); assert_refused_walk(&root, pending); From d245259e78419ca1c401d6b606465c5f37c60f4f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Mon, 9 Nov 2026 23:56:47 -0800 Subject: [PATCH 42/79] fix(code-index): recheck child repository worktree binding Refs #4593 Reject existing child source walks and graph reads when the child Git administration changes core.worktree without changing the .git marker. Reuse the initial pinned-store topology proof on every bound read. Add a real-Git regression that changes only the child repository worktree configuration after source and graph readers were created. The native Git fixture demonstrates the mismatch; Rust acceptance still needs nonzero Bazel tests on this exact commit. --- .../src/source_walk/submodules.rs | 7 +++++ .../src/source_walk/submodules/tests.rs | 28 +++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index cd633f2b32f..f2fe2a256c4 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -98,6 +98,13 @@ impl AuthenticatedSubmodule { .as_ref() == Some(&self.git_dir) && self.parent_registration_is_live() + // The child gitdir marker is unchanged if its repository config + // redirects core.worktree. A fresh binding would refuse that store; + // an existing graph reader or lazy walk must do the same. + && open_pinned_store(&self.git_dir, self.pin) + .and_then(|child| child.workdir()?.canonicalize().ok()) + .as_ref() + == Some(&self.worktree) } pub(super) fn sources(&self) -> Result { diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index e2f479a5fb8..0152a5cb35a 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -144,7 +144,7 @@ fn marker_change_before_iteration(replacement: Option<&str>) { assert_refused_walk(&root, pending); } -fn parent_authority_change_before_iteration(change: impl FnOnce(&Path)) { +fn binding_change_before_iteration(change: impl FnOnce(&Path)) { let fixture = fixture(); let root = fixture.path().join("parent"); assert_live_control(&root); @@ -164,28 +164,28 @@ fn parent_authority_change_before_iteration(change: impl FnOnce(&Path)) { #[test] fn lazy_submodule_walk_refuses_removed_worktree_registration() { - parent_authority_change_before_iteration(|root| { + binding_change_before_iteration(|root| { fs::remove_file(root.join(".gitmodules")).unwrap(); }); } #[test] fn lazy_submodule_walk_refuses_removed_staged_gitlink() { - parent_authority_change_before_iteration(|root| { + binding_change_before_iteration(|root| { git(root, &["rm", "-q", "--cached", "app"]); }); } #[test] fn lazy_submodule_walk_refuses_removed_staged_registration() { - parent_authority_change_before_iteration(|root| { + binding_change_before_iteration(|root| { git(root, &["rm", "-q", "--cached", ".gitmodules"]); }); } #[test] fn lazy_submodule_walk_refuses_updated_staged_child_pin() { - parent_authority_change_before_iteration(|root| { + binding_change_before_iteration(|root| { let child = root.join("app"); fs::write(child.join("first.rs"), "pub fn updated() {}\n").unwrap(); git(&child, &["add", "first.rs"]); @@ -194,6 +194,24 @@ fn lazy_submodule_walk_refuses_updated_staged_child_pin() { }); } +#[test] +fn lazy_submodule_walk_refuses_changed_child_repository_worktree() { + binding_change_before_iteration(|root| { + // The marker stays at app/.git and the source bytes remain in app, + // but the parent-owned child Git admin now advertises another worktree. + let other = root.parent().unwrap().join("child"); + git( + &root.join("app"), + &["config", "core.worktree", other.to_str().unwrap()], + ); + let parent = tracedecay_runtime_core::git_open::open(root).unwrap(); + assert!( + !super::live_submodules(&parent).unwrap().contains_key("app"), + "fresh admission must reject a retargeted child repository worktree" + ); + }); +} + #[test] fn lazy_submodule_walk_refuses_removed_marker() { marker_change_before_iteration(None); From f1e2f5c29c3e5d91ba452aa3ffc10dfb1020d3e2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:19:22 -0800 Subject: [PATCH 43/79] fix(code-index): revalidate bound submodules during capture Refs #4593 Retain canonical submodule checkout authority alongside classified source paths. Reject stale parent or child binding before/after each child file read and around snapshot reuse, including the clean-HEAD shortcut. Add real-Git staged-gitlink revocation/recovery regression coverage. Native exact-head Bazel verification and independent review remain pending. --- .../src/code_index_scheduler/reconcile.rs | 57 ++++++++++++++----- .../src/source_walk/classification.rs | 20 ++++++- .../src/source_walk/submodules.rs | 2 +- .../src/source_walk/submodules/tests.rs | 31 ++++++++++ 4 files changed, 91 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index c2028cd1243..1b03cba13a1 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -18,6 +18,7 @@ use gix::{ use thiserror::Error; use tracedecay_application::code_index::DaemonCodeIndexControlV1; use tracedecay_code_index::production::CodeIndexProductionOwnerV1; +use tracedecay_code_index::source_walk::submodules::AuthenticatedSubmodule; use tracedecay_code_index_retention::code_index_generations::{ CodeIndexScopeStoreResetV1, DurablePublicationPointerV1, DurableSealedCodeGenerationIdentityV1, }; @@ -3987,27 +3988,30 @@ impl CodeIndexWorktreeSchedulerV1 { .ignored_source_admissions .iter() .any(|admission| admission.logical_path == logical_path); - let child_root = if self.path_policy.includes_submodules() && !explicitly_admitted { + let classification = if self.path_policy.includes_submodules() && !explicitly_admitted { let repository = tracedecay_runtime_core::git_open::open(&self.project_root) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - let classification = + Some( classification::WorktreeChangeClassificationV1::for_policy(&repository, true) - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - if classification.beneath_gitlink(Path::new(logical_path)) - && classification.child_root(logical_path).is_none() - { - return Ok(CapturedFileOutcomeV1::Absent); - } - classification.child_root(logical_path).map(str::to_owned) + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?, + ) } else { None }; + if classification.as_ref().is_some_and(|classified| { + classified.beneath_gitlink(Path::new(logical_path)) + && classified.bound_child(logical_path).is_none() + }) { + return Ok(CapturedFileOutcomeV1::Absent); + } self.capture_admitted_candidate( registry, logical_path, control, explicitly_admitted, - child_root.as_deref(), + classification + .as_ref() + .and_then(|classified| classified.bound_child(logical_path)), ) } @@ -4024,15 +4028,17 @@ impl CodeIndexWorktreeSchedulerV1 { logical_path: &str, control: Option<&dyn CodeIndexExecutionControlV1>, explicitly_admitted: bool, - child_root: Option<&str>, + child_binding: Option<(&str, &AuthenticatedSubmodule)>, ) -> Result { // Explicit ignored-source admission cannot cross a nested checkout // any more than an ordinary Git candidate or interactive source walk. let relative = Path::new(logical_path); let absolute = self.project_root.join(relative); - let (source_root, source_relative) = match child_root { - Some(prefix) if !explicitly_admitted => { - if !git_tree_capture::is_source_path(registry, logical_path.as_bytes()) { + let (source_root, source_relative) = match child_binding { + Some((prefix, binding)) if !explicitly_admitted => { + if !binding.checkout_is_live() + || !git_tree_capture::is_source_path(registry, logical_path.as_bytes()) + { return Ok(CapturedFileOutcomeV1::Absent); } let root = self.project_root.join(prefix); @@ -4100,6 +4106,11 @@ impl CodeIndexWorktreeSchedulerV1 { ignored_dependencies::read_bounded_snapshot_source(&absolute, control)? }; ignored_dependencies::checkpoint_if_present(control)?; + // A staged gitlink or registration may be revoked while a file is + // streaming. Do not admit its bytes after the original binding expires. + if child_binding.is_some_and(|(_, binding)| !binding.checkout_is_live()) { + return Ok(CapturedFileOutcomeV1::Absent); + } self.capture_candidate_bytes( registry, logical_path.as_bytes(), @@ -4146,6 +4157,11 @@ impl CodeIndexWorktreeSchedulerV1 { self.path_policy.includes_submodules(), ) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; + if !classification.child_bindings_live() { + return Err(CodeIndexSchedulerErrorV1::Git( + "authenticated child checkout binding changed during classification".into(), + )); + } if self.shutting_down.load(Ordering::Acquire) { return Err(cancelled_code_index_reconcile()); } @@ -4262,6 +4278,12 @@ impl CodeIndexWorktreeSchedulerV1 { }, ) { Ok(captured) => { + if !classification.child_bindings_live() { + return Err(CodeIndexSchedulerErrorV1::Git( + "authenticated child checkout binding changed during tree capture" + .into(), + )); + } *self .active_snapshot_changed_paths .lock() @@ -4371,7 +4393,7 @@ impl CodeIndexWorktreeSchedulerV1 { logical_path, control, admitted_paths.contains(logical_path.as_str()), - classification.child_root(logical_path), + classification.bound_child(logical_path), ) }) }) @@ -4483,6 +4505,11 @@ impl CodeIndexWorktreeSchedulerV1 { retained_bytes, retained_reservations, }; + if !classification.child_bindings_live() { + return Err(CodeIndexSchedulerErrorV1::Git( + "authenticated child checkout binding changed during snapshot capture".into(), + )); + } *self .active_snapshot_changed_paths .lock() diff --git a/crates/tracedecay-code-index/src/source_walk/classification.rs b/crates/tracedecay-code-index/src/source_walk/classification.rs index 2042361ca32..628fef52f6e 100644 --- a/crates/tracedecay-code-index/src/source_walk/classification.rs +++ b/crates/tracedecay-code-index/src/source_walk/classification.rs @@ -19,7 +19,7 @@ use std::path::Path; use super::{ has_nested_repository_ancestor, parent_gitlink_paths, resolve_parent_source, - submodules::live_submodules, + submodules::{AuthenticatedSubmodule, live_submodules}, }; use gix::bstr::ByteSlice; use tracedecay_runtime_core::git_repository::GIT_STATUS_MODIFICATION_CHECK_THREADS; @@ -103,6 +103,8 @@ pub struct WorktreeChangeClassificationV1 { parent_gitlinks: BTreeSet, /// Direct parent-relative child paths admitted from that checkout's own Git roster. child_sources: BTreeMap, + /// Retained parent-owned authority for every admitted direct child. + child_bindings: BTreeMap, child_revision_changed: bool, } @@ -321,6 +323,7 @@ impl WorktreeChangeClassificationV1 { class: change.class, } })); + parent.child_bindings.insert(prefix, binding); } Ok(parent) } @@ -343,8 +346,19 @@ impl WorktreeChangeClassificationV1 { Ok(files) } - pub fn child_root(&self, logical_path: &str) -> Option<&str> { - self.child_sources.get(logical_path).map(String::as_str) + /// Keep the original child authority with its admitted logical path; + /// a filesystem root alone cannot survive a staged parent revocation. + pub fn bound_child(&self, logical_path: &str) -> Option<(&str, &AuthenticatedSubmodule)> { + let prefix = self.child_sources.get(logical_path)?.as_str(); + self.child_bindings.get(prefix).map(|binding| (prefix, binding)) + } + + /// A snapshot may reuse unchanged child rows without reading them again. + /// That reuse must not outlive any captured parent/child authorization. + pub fn child_bindings_live(&self) -> bool { + self.child_bindings + .values() + .all(AuthenticatedSubmodule::checkout_is_live) } pub(super) fn gitlink_paths(&self) -> &BTreeSet { diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index f2fe2a256c4..015bceaa0d2 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -86,7 +86,7 @@ impl AuthenticatedSubmodule { .is_ok_and(|path| path.as_slice() == self.module_path.as_bytes()) } - fn checkout_is_live(&self) -> bool { + pub fn checkout_is_live(&self) -> bool { let marker = self.worktree.join(".git"); self.worktree.canonicalize().ok().as_ref() == Some(&self.worktree) && marker diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 0152a5cb35a..4535c62b4a1 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -266,3 +266,34 @@ fn lazy_submodule_walk_refuses_replaced_root_before_root_entry() { // before polling this separate child root, not only in filter_entry. assert_refused_walk(&root, pending); } + +#[test] +fn indexed_child_inventory_refuses_revoked_gitlink_before_capture() { + let fixture = fixture(); + let root = fixture.path().join("parent"); + let repository = tracedecay_runtime_core::git_open::open(&root).unwrap(); + let classified = crate::source_walk::classification::WorktreeChangeClassificationV1::for_policy( + &repository, + true, + ) + .unwrap(); + let (_, child) = classified + .bound_child("app/first.rs") + .expect("registered child source"); + assert!(child.checkout_is_live()); + assert!(classified.child_bindings_live()); + + git(&root, &["rm", "-q", "--cached", "app"]); + assert!(root.join("app/.git").is_file()); + assert!(root.join("app/first.rs").is_file()); + assert!(classified.bound_child("app/first.rs").is_some()); + assert!(!child.checkout_is_live()); + assert!( + !classified.child_bindings_live(), + "retained capture authority must expire when the staged gitlink disappears" + ); + + git(&root, &["add", "app"]); + assert!(child.checkout_is_live()); + assert!(classified.child_bindings_live()); +} From eed23c361c9f38f287618633d6600e8a0937de35 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 13:54:56 -0800 Subject: [PATCH 44/79] style(code-index): apply Bazel rustfmt to child binding tests Apply the two exact formatting hunks from CI 42904015316 without changing source semantics or weakening the staged-revocation regression. Refs #4593 --- .../src/source_walk/classification.rs | 4 +++- .../src/source_walk/submodules/tests.rs | 11 ++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk/classification.rs b/crates/tracedecay-code-index/src/source_walk/classification.rs index 628fef52f6e..4fc432cb68e 100644 --- a/crates/tracedecay-code-index/src/source_walk/classification.rs +++ b/crates/tracedecay-code-index/src/source_walk/classification.rs @@ -350,7 +350,9 @@ impl WorktreeChangeClassificationV1 { /// a filesystem root alone cannot survive a staged parent revocation. pub fn bound_child(&self, logical_path: &str) -> Option<(&str, &AuthenticatedSubmodule)> { let prefix = self.child_sources.get(logical_path)?.as_str(); - self.child_bindings.get(prefix).map(|binding| (prefix, binding)) + self.child_bindings + .get(prefix) + .map(|binding| (prefix, binding)) } /// A snapshot may reuse unchanged child rows without reading them again. diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 4535c62b4a1..cc443ff3684 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -272,11 +272,12 @@ fn indexed_child_inventory_refuses_revoked_gitlink_before_capture() { let fixture = fixture(); let root = fixture.path().join("parent"); let repository = tracedecay_runtime_core::git_open::open(&root).unwrap(); - let classified = crate::source_walk::classification::WorktreeChangeClassificationV1::for_policy( - &repository, - true, - ) - .unwrap(); + let classified = + crate::source_walk::classification::WorktreeChangeClassificationV1::for_policy( + &repository, + true, + ) + .unwrap(); let (_, child) = classified .bound_child("app/first.rs") .expect("registered child source"); From 49fcebc99047530a0b38ba73b153e00e66e0bed0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:18:51 -0800 Subject: [PATCH 45/79] fix(submodules): report parent Git failures during source reads Refs #4593 Keep unreadable parent index and malformed registration errors distinct from revoked child identities. Propagate through live graph, lazy source walk, and indexed capture. Add a real-Git regression for index corruption. --- .../src/code_index_scheduler/reconcile.rs | 28 +++-- .../src/source_walk/classification.rs | 11 +- .../src/source_walk/submodules.rs | 102 +++++++++--------- .../src/source_walk/submodules/tests.rs | 38 +++++-- 4 files changed, 115 insertions(+), 64 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 1b03cba13a1..badb6c58b5b 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -4036,7 +4036,9 @@ impl CodeIndexWorktreeSchedulerV1 { let absolute = self.project_root.join(relative); let (source_root, source_relative) = match child_binding { Some((prefix, binding)) if !explicitly_admitted => { - if !binding.checkout_is_live() + if !binding + .checkout_is_live() + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? || !git_tree_capture::is_source_path(registry, logical_path.as_bytes()) { return Ok(CapturedFileOutcomeV1::Absent); @@ -4108,8 +4110,13 @@ impl CodeIndexWorktreeSchedulerV1 { ignored_dependencies::checkpoint_if_present(control)?; // A staged gitlink or registration may be revoked while a file is // streaming. Do not admit its bytes after the original binding expires. - if child_binding.is_some_and(|(_, binding)| !binding.checkout_is_live()) { - return Ok(CapturedFileOutcomeV1::Absent); + if let Some((_, binding)) = child_binding { + if !binding + .checkout_is_live() + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? + { + return Ok(CapturedFileOutcomeV1::Absent); + } } self.capture_candidate_bytes( registry, @@ -4157,7 +4164,10 @@ impl CodeIndexWorktreeSchedulerV1 { self.path_policy.includes_submodules(), ) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; - if !classification.child_bindings_live() { + if !classification + .child_bindings_live() + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? + { return Err(CodeIndexSchedulerErrorV1::Git( "authenticated child checkout binding changed during classification".into(), )); @@ -4278,7 +4288,10 @@ impl CodeIndexWorktreeSchedulerV1 { }, ) { Ok(captured) => { - if !classification.child_bindings_live() { + if !classification + .child_bindings_live() + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? + { return Err(CodeIndexSchedulerErrorV1::Git( "authenticated child checkout binding changed during tree capture" .into(), @@ -4505,7 +4518,10 @@ impl CodeIndexWorktreeSchedulerV1 { retained_bytes, retained_reservations, }; - if !classification.child_bindings_live() { + if !classification + .child_bindings_live() + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? + { return Err(CodeIndexSchedulerErrorV1::Git( "authenticated child checkout binding changed during snapshot capture".into(), )); diff --git a/crates/tracedecay-code-index/src/source_walk/classification.rs b/crates/tracedecay-code-index/src/source_walk/classification.rs index 4fc432cb68e..938d49d2d63 100644 --- a/crates/tracedecay-code-index/src/source_walk/classification.rs +++ b/crates/tracedecay-code-index/src/source_walk/classification.rs @@ -357,10 +357,13 @@ impl WorktreeChangeClassificationV1 { /// A snapshot may reuse unchanged child rows without reading them again. /// That reuse must not outlive any captured parent/child authorization. - pub fn child_bindings_live(&self) -> bool { - self.child_bindings - .values() - .all(AuthenticatedSubmodule::checkout_is_live) + pub fn child_bindings_live(&self) -> std::io::Result { + for child in self.child_bindings.values() { + if !child.checkout_is_live()? { + return Ok(false); + } + } + Ok(true) } pub(super) fn gitlink_paths(&self) -> &BTreeSet { diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 015bceaa0d2..f174109a15f 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -35,32 +35,28 @@ pub(super) struct ChildSources { } impl AuthenticatedSubmodule { - fn parent_registration_is_live(&self) -> bool { - // An already-open iterator or graph reader cannot continue to use a - // child after the parent revokes its staged gitlink or registration. - // Inspect exact index entries, not the entire committed Git tree. - let Ok(parent) = tracedecay_runtime_core::git_open::open(&self.parent_root) else { - return false; - }; + fn parent_registration_is_live(&self) -> io::Result { + // A retained child must preserve the parent's exact staged authority. + // An unreadable Git authority is not a revoked or absent source. + let parent = tracedecay_runtime_core::git_open::open(&self.parent_root) + .map_err(|error| io::Error::other(error.to_string()))?; if parent .workdir() .and_then(|root| root.canonicalize().ok()) .as_ref() != Some(&self.parent_root) { - return false; + return Ok(false); } - // A replaced parent Git directory cannot inherit the old child's - // authorization merely by advertising the same staged paths. if !module_stores(&parent, &self.module_name).any(|store| store == self.git_dir) { - return false; + return Ok(false); } - let Ok(index) = parent.index_or_empty() else { - return false; - }; + let index = parent + .index_or_empty() + .map_err(|error| io::Error::other(error.to_string()))?; let stage = gix::index::entry::Stage::Unconflicted; let Some(modules) = index.entry_by_path_and_stage(".gitmodules".into(), stage) else { - return false; + return Ok(false); }; if modules.id != self.staged_modules_id || !modules @@ -68,27 +64,32 @@ impl AuthenticatedSubmodule { .to_tree_entry_mode() .is_some_and(|mode| mode.is_blob()) { - return false; + return Ok(false); } let Some(gitlink) = index.entry_by_path_and_stage(self.module_path.as_bytes().as_bstr(), stage) else { - return false; + return Ok(false); }; if !gitlink.mode.is_submodule() || gitlink.id != self.pin { - return false; + return Ok(false); } - let Ok(Some(working)) = parent.open_modules_file() else { - return false; + let Some(working) = parent + .open_modules_file() + .map_err(|error| io::Error::other(error.to_string()))? + else { + return Ok(false); }; - working + let registered_path = working .path(self.module_name.as_os_str().as_encoded_bytes().as_bstr()) - .is_ok_and(|path| path.as_slice() == self.module_path.as_bytes()) + .map_err(|error| io::Error::other(error.to_string()))?; + Ok(registered_path.as_slice() == self.module_path.as_bytes()) } - pub fn checkout_is_live(&self) -> bool { + pub fn checkout_is_live(&self) -> io::Result { let marker = self.worktree.join(".git"); - self.worktree.canonicalize().ok().as_ref() == Some(&self.worktree) + let checkout_matches = self.worktree.canonicalize().ok().as_ref() + == Some(&self.worktree) && marker .symlink_metadata() .is_ok_and(|metadata| metadata.is_file()) @@ -96,15 +97,15 @@ impl AuthenticatedSubmodule { .ok() .and_then(|target| target.canonicalize().ok()) .as_ref() - == Some(&self.git_dir) - && self.parent_registration_is_live() - // The child gitdir marker is unchanged if its repository config - // redirects core.worktree. A fresh binding would refuse that store; - // an existing graph reader or lazy walk must do the same. - && open_pinned_store(&self.git_dir, self.pin) - .and_then(|child| child.workdir()?.canonicalize().ok()) - .as_ref() - == Some(&self.worktree) + == Some(&self.git_dir); + if !checkout_matches || !self.parent_registration_is_live()? { + return Ok(false); + } + // Rechecking the pinned child also rejects core.worktree redirection. + Ok(open_pinned_store(&self.git_dir, self.pin) + .and_then(|child| child.workdir()?.canonicalize().ok()) + .as_ref() + == Some(&self.worktree)) } pub(super) fn sources(&self) -> Result { @@ -142,21 +143,25 @@ impl Iterator for CheckoutWalk { fn next(&mut self) -> Option { self.walk.as_ref()?; - if let Some(child) = &self.child - && !child.checkout_is_live() - { - // Check before polling, including the root entry: ignore does not - // apply filter_entry to a walk's root. Stop permanently after a - // binding change, even if the original marker is later restored. - self.walk = None; - return Some(Err(WalkError::WithPath { - path: child.worktree.clone(), - err: Box::new(WalkError::Io(io::Error::new( + if let Some(child) = &self.child { + let refusal = match child.checkout_is_live() { + Ok(true) => None, + Ok(false) => Some(io::Error::new( io::ErrorKind::InvalidData, "authenticated submodule checkout binding changed during source walk", - ))), - })); + )), + Err(error) => Some(error), + }; + if let Some(error) = refusal { + // A failed Git read must not become a successful empty walk. + self.walk = None; + return Some(Err(WalkError::WithPath { + path: child.worktree.clone(), + err: Box::new(WalkError::Io(error)), + })); + } } + let next = self.walk.as_mut()?.next(); if next.is_none() { self.walk = None; @@ -376,9 +381,10 @@ impl LiveSourceBoundaries { return Ok(None); }; relative = path.strip_prefix(ancestor).map_err(std::io::Error::other)?; - if !relative.to_str().is_some_and(|file| files.contains(file)) - || !child.checkout_is_live() - { + if !relative.to_str().is_some_and(|file| files.contains(file)) { + return Ok(None); + } + if !child.checkout_is_live()? { return Ok(None); } root = &child.worktree; diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index cc443ff3684..698bd4f10fe 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -281,20 +281,46 @@ fn indexed_child_inventory_refuses_revoked_gitlink_before_capture() { let (_, child) = classified .bound_child("app/first.rs") .expect("registered child source"); - assert!(child.checkout_is_live()); - assert!(classified.child_bindings_live()); + assert!(child.checkout_is_live().unwrap()); + assert!(classified.child_bindings_live().unwrap()); git(&root, &["rm", "-q", "--cached", "app"]); assert!(root.join("app/.git").is_file()); assert!(root.join("app/first.rs").is_file()); assert!(classified.bound_child("app/first.rs").is_some()); - assert!(!child.checkout_is_live()); + assert!(!child.checkout_is_live().unwrap()); assert!( - !classified.child_bindings_live(), + !classified.child_bindings_live().unwrap(), "retained capture authority must expire when the staged gitlink disappears" ); git(&root, &["add", "app"]); - assert!(child.checkout_is_live()); - assert!(classified.child_bindings_live()); + assert!(child.checkout_is_live().unwrap()); + assert!(classified.child_bindings_live().unwrap()); +} + +#[test] +fn unavailable_parent_index_refuses_retained_graph_and_lazy_walk() { + let fixture = fixture(); + let root = fixture.path().join("parent"); + assert_live_control(&root); + let boundaries = LiveSourceBoundaries::open(&root, true).unwrap(); + let path = Path::new("app/first.rs"); + assert!(boundaries.resolve(path).unwrap().is_some()); + let pending = walk(&root); + + // The marker and child bytes are intact, but parent authority is corrupt. + // Returning Ok(None) here would make unavailable data look absent. + fs::write(root.join(".git/index"), b"not a valid Git index").unwrap(); + assert!(root.join(path).is_file()); + let error = boundaries + .resolve(path) + .expect_err("unreadable parent index must not hide an admitted child"); + assert_eq!(error.kind(), io::ErrorKind::Other); + + let error = pending + .filter_map(Result::err) + .next() + .expect("the child walk must report unavailable parent authority"); + assert_eq!(error.io_error().unwrap().kind(), io::ErrorKind::Other); } From 9875a9e2f2ccb1802d309b760d70179c079d5f71 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:49:38 -0800 Subject: [PATCH 46/79] style(code-index): format submodule binding checks Apply the pinned formatter's line wrapping and indentation to the submodule checkout and scheduler checks flagged by repository gates. --- .../src/code_index_scheduler/reconcile.rs | 12 ++++++------ .../src/source_walk/submodules.rs | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index badb6c58b5b..da750460f0c 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -4165,9 +4165,9 @@ impl CodeIndexWorktreeSchedulerV1 { ) .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))?; if !classification - .child_bindings_live() - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? - { + .child_bindings_live() + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? + { return Err(CodeIndexSchedulerErrorV1::Git( "authenticated child checkout binding changed during classification".into(), )); @@ -4519,9 +4519,9 @@ impl CodeIndexWorktreeSchedulerV1 { retained_reservations, }; if !classification - .child_bindings_live() - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? - { + .child_bindings_live() + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? + { return Err(CodeIndexSchedulerErrorV1::Git( "authenticated child checkout binding changed during snapshot capture".into(), )); diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index f174109a15f..0c081cecae2 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -88,8 +88,7 @@ impl AuthenticatedSubmodule { pub fn checkout_is_live(&self) -> io::Result { let marker = self.worktree.join(".git"); - let checkout_matches = self.worktree.canonicalize().ok().as_ref() - == Some(&self.worktree) + let checkout_matches = self.worktree.canonicalize().ok().as_ref() == Some(&self.worktree) && marker .symlink_metadata() .is_ok_and(|metadata| metadata.is_file()) From 5c4810978a655fea6cbedb78ecca2f3175b34808 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:19:28 -0800 Subject: [PATCH 47/79] perf(code-index): reuse stable submodule binding witnesses Refs #4593 Avoid reloading the parent Git index and reopening the child store for all entries of an unchanged authenticated checkout on Unix. Compare native inode, size and change-time identity on the parent, registration, child administration and object pack paths before every source read. Revalidate through the exact Git authority on mutation, with no cache for alternates or uncertain filesystem evidence. Windows retains full validation. Exercise 128 repeated real child reads, staged revocation and same-length working registration changes without weakening existing source refusals. --- .../src/source_walk/submodules.rs | 169 ++++++++++++++++++ .../src/source_walk/submodules/tests.rs | 54 ++++++ 2 files changed, 223 insertions(+) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 0c081cecae2..415d73d4f3f 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -4,7 +4,11 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; use std::io; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; use std::path::{Component, Path, PathBuf}; +#[cfg(unix)] +use std::sync::{Arc, Mutex, PoisonError}; use gix::bstr::ByteSlice; use ignore::{DirEntry, Error as WalkError, Walk}; @@ -27,6 +31,76 @@ pub struct AuthenticatedSubmodule { module_path: String, module_name: PathBuf, staged_modules_id: gix::ObjectId, + #[cfg(unix)] + parent_git_dir: PathBuf, + #[cfg(unix)] + parent_common_dir: PathBuf, + #[cfg(unix)] + verified: Arc>>, +} + +#[cfg(unix)] +#[derive(Clone, Debug, PartialEq, Eq)] +struct FileWitness { + named: (u64, u64, u32, u64, i64, i64, i64, i64), + target: Option<(u64, u64, u32, u64, i64, i64, i64, i64)>, +} + +#[cfg(unix)] +impl FileWitness { + fn identity(metadata: &std::fs::Metadata) -> (u64, u64, u32, u64, i64, i64, i64, i64) { + ( + metadata.dev(), + metadata.ino(), + metadata.mode(), + metadata.len(), + metadata.mtime(), + metadata.mtime_nsec(), + metadata.ctime(), + metadata.ctime_nsec(), + ) + } + + fn at(path: &Path) -> io::Result> { + let metadata = match std::fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let target = if metadata.file_type().is_symlink() { + Some(Self::identity(&std::fs::metadata(path)?)) + } else { + None + }; + Ok(Some(Self { + named: Self::identity(&metadata), + target, + })) + } +} + +#[cfg(unix)] +#[derive(Clone, Debug, PartialEq, Eq)] +struct BindingWitness { + paths: BTreeMap>, +} + +#[cfg(unix)] +impl BindingWitness { + fn unchanged(&self) -> io::Result { + for (path, witness) in &self.paths { + if FileWitness::at(path)? != *witness { + return Ok(false); + } + } + Ok(true) + } +} + +#[cfg(test)] +thread_local! { + static FULL_BINDING_VALIDATIONS: std::cell::Cell = + const { std::cell::Cell::new(0) }; } pub(super) struct ChildSources { @@ -35,6 +109,57 @@ pub(super) struct ChildSources { } impl AuthenticatedSubmodule { + #[cfg(unix)] + fn binding_witness(&self) -> io::Result> { + let objects = self.git_dir.join("objects"); + // Object alternates may change outside this parent-owned checkout. + // Unobservable external authority cannot be covered by a local stamp. + match std::fs::symlink_metadata(objects.join("info/alternates")) { + Ok(_) => return Ok(None), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let pin = self.pin.to_string(); + let mut paths = vec![ + self.parent_root.clone(), + self.parent_root.join(".git"), + self.parent_root.join(".gitmodules"), + self.parent_git_dir.clone(), + self.parent_git_dir.join("index"), + self.parent_git_dir.join("config"), + self.parent_git_dir.join("commondir"), + self.parent_common_dir.clone(), + self.parent_common_dir.join("config"), + self.parent_common_dir.join("modules"), + self.git_dir.clone(), + self.git_dir.join("config"), + self.git_dir.join("config.worktree"), + objects.clone(), + objects.join("pack"), + objects.join("info/alternates"), + objects.join(&pin[..2]), + objects.join(&pin[..2]).join(&pin[2..]), + self.worktree.clone(), + self.worktree.join(".git"), + ]; + // The pack directory stamp detects additions and deletions. Track + // existing pack members as well to reject an in-place replacement. + match std::fs::read_dir(objects.join("pack")) { + Ok(entries) => { + for entry in entries { + paths.push(entry?.path()); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let mut observed = BTreeMap::new(); + for path in paths { + observed.insert(path.clone(), FileWitness::at(&path)?); + } + Ok(Some(BindingWitness { paths: observed })) + } + fn parent_registration_is_live(&self) -> io::Result { // A retained child must preserve the parent's exact staged authority. // An unreadable Git authority is not a revoked or absent source. @@ -87,6 +212,44 @@ impl AuthenticatedSubmodule { } pub fn checkout_is_live(&self) -> io::Result { + #[cfg(unix)] + { + let cached = self + .verified + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + if let Some(witness) = cached + && witness.unchanged()? + { + return Ok(true); + } + // Invalidate BEFORE touching Git. A read failure must never retain + // a previous successful authority for a later caller. + *self + .verified + .lock() + .unwrap_or_else(PoisonError::into_inner) = None; + } + #[cfg(unix)] + let before = self.binding_witness().ok().flatten(); + let valid = self.verify_checkout_binding()?; + #[cfg(unix)] + if valid { + let after = self.binding_witness().ok().flatten(); + if before.is_some() && before == after { + *self + .verified + .lock() + .unwrap_or_else(PoisonError::into_inner) = after; + } + } + Ok(valid) + } + + fn verify_checkout_binding(&self) -> io::Result { + #[cfg(test)] + FULL_BINDING_VALIDATIONS.with(|count| count.set(count.get() + 1)); let marker = self.worktree.join(".git"); let checkout_matches = self.worktree.canonicalize().ok().as_ref() == Some(&self.worktree) && marker @@ -293,6 +456,12 @@ fn bound_checkout( module_path, module_name: name.to_path_buf(), staged_modules_id, + #[cfg(unix)] + parent_git_dir: parent.git_dir().to_path_buf(), + #[cfg(unix)] + parent_common_dir: parent.common_dir().to_path_buf(), + #[cfg(unix)] + verified: Arc::new(Mutex::new(None)), }) } diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 698bd4f10fe..a47b624eafb 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -324,3 +324,57 @@ fn unavailable_parent_index_refuses_retained_graph_and_lazy_walk() { .expect("the child walk must report unavailable parent authority"); assert_eq!(error.io_error().unwrap().kind(), io::ErrorKind::Other); } + +#[cfg(unix)] +#[test] +fn repeated_child_reads_do_not_reparse_stable_parent_git_authority() { + let fixture = fixture(); + let root = fixture.path().join("parent"); + let boundaries = LiveSourceBoundaries::open(&root, true).unwrap(); + let source = Path::new("app/first.rs"); + super::FULL_BINDING_VALIDATIONS.with(|count| count.set(0)); + + for _ in 0..128 { + assert!(boundaries.resolve(source).unwrap().is_some()); + } + super::FULL_BINDING_VALIDATIONS.with(|count| { + assert_eq!( + count.get(), + 1, + "128 stable source reads must share one full Git binding validation" + ); + }); + git(&root, &["rm", "-q", "--cached", "app"]); + assert!( + boundaries.resolve(source).unwrap().is_none(), + "a staged gitlink removal must revoke an otherwise unchanged child" + ); + super::FULL_BINDING_VALIDATIONS.with(|count| { + assert_eq!(count.get(), 2, "revocation requires a fresh Git validation"); + }); + git(&root, &["add", "app"]); + assert!(boundaries.resolve(source).unwrap().is_some()); +} + +#[cfg(unix)] +#[test] +fn cached_child_binding_refuses_equal_length_working_registration_change() { + let fixture = fixture(); + let root = fixture.path().join("parent"); + let boundaries = LiveSourceBoundaries::open(&root, true).unwrap(); + let source = Path::new("app/first.rs"); + assert!(boundaries.resolve(source).unwrap().is_some()); + + let path = root.join(".gitmodules"); + let before = fs::read_to_string(&path).unwrap(); + let changed = before.replacen("path = app", "path = bad", 1); + assert_ne!(changed, before); + assert_eq!(changed.len(), before.len()); + fs::write(&path, changed).unwrap(); + assert!( + boundaries.resolve(source).unwrap().is_none(), + "a same-length working registration change must not reuse authority" + ); + fs::write(&path, before).unwrap(); + assert!(boundaries.resolve(source).unwrap().is_some()); +} From 1bd73d4b1d684c22c1c00e3b97136ee6e2a991af Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:51:36 -0800 Subject: [PATCH 48/79] style(code-index): format cached binding assignments --- .../src/source_walk/submodules.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 415d73d4f3f..ecf999fe007 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -226,10 +226,7 @@ impl AuthenticatedSubmodule { } // Invalidate BEFORE touching Git. A read failure must never retain // a previous successful authority for a later caller. - *self - .verified - .lock() - .unwrap_or_else(PoisonError::into_inner) = None; + *self.verified.lock().unwrap_or_else(PoisonError::into_inner) = None; } #[cfg(unix)] let before = self.binding_witness().ok().flatten(); @@ -238,10 +235,7 @@ impl AuthenticatedSubmodule { if valid { let after = self.binding_witness().ok().flatten(); if before.is_some() && before == after { - *self - .verified - .lock() - .unwrap_or_else(PoisonError::into_inner) = after; + *self.verified.lock().unwrap_or_else(PoisonError::into_inner) = after; } } Ok(valid) From 21ece5e67d46d1f62775be0ab4efd65127d01e35 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:17:09 -0800 Subject: [PATCH 49/79] fix(code-index): witness split-index Git authority Refs #4593 Track secondary shared-index file identities in the per-entry submodule binding witness. Main index and Git-dir stamps alone cannot detect an in-place corrupted/replaced shared index, allowing stale authenticated sources to be reused. Add a real Git split-index mutation regression. --- .../src/source_walk/submodules.rs | 12 ++++++ .../src/source_walk/submodules/tests.rs | 39 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index ecf999fe007..a6ec7e65492 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -142,6 +142,18 @@ impl AuthenticatedSubmodule { self.worktree.clone(), self.worktree.join(".git"), ]; + // Git split indexes carry staged gitlinks in sharedindex.* files. + // Their contents can change without changing the main index or its + // parent directory. Witness each file in both administrative roots; + // the directory stamps above detect newly added or removed files. + for admin in [&self.parent_git_dir, &self.parent_common_dir] { + for entry in std::fs::read_dir(admin)? { + let entry = entry?; + if entry.file_name().to_string_lossy().starts_with("sharedindex.") { + paths.push(entry.path()); + } + } + } // The pack directory stamp detects additions and deletions. Track // existing pack members as well to reject an in-place replacement. match std::fs::read_dir(objects.join("pack")) { diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index a47b624eafb..5ee8cb847b7 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -378,3 +378,42 @@ fn cached_child_binding_refuses_equal_length_working_registration_change() { fs::write(&path, before).unwrap(); assert!(boundaries.resolve(source).unwrap().is_some()); } + +#[cfg(unix)] +#[test] +fn changing_a_split_index_invalidates_the_child_binding_witness() { + let fixture = fixture(); + let root = fixture.path().join("parent"); + let boundaries = LiveSourceBoundaries::open(&root, true).unwrap(); + let (child, _) = boundaries.children.get("app").expect("authenticated child"); + git(&root, &["update-index", "--split-index"]); + + let shared = fs::read_dir(root.join(".git")) + .unwrap() + .map(|entry| entry.unwrap()) + .find(|entry| entry.file_name().to_string_lossy().starts_with("sharedindex.")) + .expect("real Git split index") + .path(); + let witness = child.binding_witness().unwrap().expect("local Git authority"); + assert!( + witness.paths.contains_key(&shared), + "the secondary index is part of staged gitlink authority" + ); + assert!(witness.unchanged().unwrap()); + + // Mutate only the secondary index's bytes, not the main index or its + // parent directory. Watching only .git/index would reuse stale authority. + let index = root.join(".git/index"); + let original_index = super::FileWitness::at(&index).unwrap(); + let original_directory = super::FileWitness::at(&root.join(".git")).unwrap(); + fs::write(&shared, b"corrupt shared index").unwrap(); + assert_eq!(super::FileWitness::at(&index).unwrap(), original_index); + assert_eq!( + super::FileWitness::at(&root.join(".git")).unwrap(), + original_directory + ); + assert!( + !witness.unchanged().unwrap(), + "a corrupt shared-index inode must invalidate the cached Git authority" + ); +} From e24cb03c687123b77a89ea75576ced0e74ce9d80 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:21:52 -0800 Subject: [PATCH 50/79] style(code-index): format split-index witness checks --- .../src/source_walk/submodules.rs | 6 +++++- .../src/source_walk/submodules/tests.rs | 12 ++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index a6ec7e65492..d7f7154c8a6 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -149,7 +149,11 @@ impl AuthenticatedSubmodule { for admin in [&self.parent_git_dir, &self.parent_common_dir] { for entry in std::fs::read_dir(admin)? { let entry = entry?; - if entry.file_name().to_string_lossy().starts_with("sharedindex.") { + if entry + .file_name() + .to_string_lossy() + .starts_with("sharedindex.") + { paths.push(entry.path()); } } diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 5ee8cb847b7..5163be042d7 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -391,10 +391,18 @@ fn changing_a_split_index_invalidates_the_child_binding_witness() { let shared = fs::read_dir(root.join(".git")) .unwrap() .map(|entry| entry.unwrap()) - .find(|entry| entry.file_name().to_string_lossy().starts_with("sharedindex.")) + .find(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with("sharedindex.") + }) .expect("real Git split index") .path(); - let witness = child.binding_witness().unwrap().expect("local Git authority"); + let witness = child + .binding_witness() + .unwrap() + .expect("local Git authority"); assert!( witness.paths.contains_key(&shared), "the secondary index is part of staged gitlink authority" From f3d0f38535ce21005ae6381d92d02efdc63f4055 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:46:59 -0800 Subject: [PATCH 51/79] fix(code-index): watch parent worktree config in child bindings Refs #4593 A retained child can outlive a change to the parent's effective core.worktree in .git/config.worktree. The old cache watched the shared config and index but not the per-worktree configuration: an in-place edit leaves those stamps unchanged. Include both parent administrative config.worktree witnesses and regress a real Git parent/child checkout with an unchanged index/directory. Keep the original full Git validation and typed revocation, without granting any new source identity. Exact-head nonzero Bazel validation remains owed. --- .../src/source_walk/submodules.rs | 4 ++ .../src/source_walk/submodules/tests.rs | 41 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index d7f7154c8a6..858687043b1 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -127,9 +127,11 @@ impl AuthenticatedSubmodule { self.parent_git_dir.clone(), self.parent_git_dir.join("index"), self.parent_git_dir.join("config"), + self.parent_git_dir.join("config.worktree"), self.parent_git_dir.join("commondir"), self.parent_common_dir.clone(), self.parent_common_dir.join("config"), + self.parent_common_dir.join("config.worktree"), self.parent_common_dir.join("modules"), self.git_dir.clone(), self.git_dir.join("config"), @@ -142,6 +144,8 @@ impl AuthenticatedSubmodule { self.worktree.clone(), self.worktree.join(".git"), ]; + // Git's per-worktree config can redirect core.worktree without + // modifying the shared config, index, or Git directory itself. // Git split indexes carry staged gitlinks in sharedindex.* files. // Their contents can change without changing the main index or its // parent directory. Witness each file in both administrative roots; diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 5163be042d7..a6010c2e3c0 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -379,6 +379,47 @@ fn cached_child_binding_refuses_equal_length_working_registration_change() { assert!(boundaries.resolve(source).unwrap().is_some()); } +#[cfg(unix)] +#[test] +fn parent_worktree_config_change_revokes_cached_child_authority() { + let fixture = fixture(); + let root = fixture.path().join("parent"); + git(&root, &["config", "extensions.worktreeConfig", "true"]); + let config = root.join(".git/config.worktree"); + let original = format!("[core]\\n\\tworktree = {}\\n", root.display()); + fs::write(&config, &original).unwrap(); + + let boundaries = LiveSourceBoundaries::open(&root, true).unwrap(); + let source = Path::new("app/first.rs"); + assert!(boundaries.resolve(source).unwrap().is_some()); + // This change is entirely inside an existing file: neither the parent + // index nor Git admin directory changes, even though Git's effective + // worktree authority is redirected to a different checkout. + let git_directory = root.join(".git"); + let directory_before = super::FileWitness::at(&git_directory).unwrap(); + let index_before = super::FileWitness::at(&git_directory.join("index")).unwrap(); + let foreign = fixture.path().join("foreign"); + fs::create_dir_all(&foreign).unwrap(); + fs::write( + &config, + format!("[core]\\n\\tworktree = {}\\n", foreign.display()), + ) + .unwrap(); + assert_eq!(super::FileWitness::at(&git_directory).unwrap(), directory_before); + assert_eq!( + super::FileWitness::at(&git_directory.join("index")).unwrap(), + index_before + ); + assert!(root.join(source).is_file(), "child content still exists"); + assert!( + boundaries.resolve(source).unwrap().is_none(), + "cached child authority must expire on a worktree-local parent config change" + ); + + fs::write(&config, original).unwrap(); + assert!(boundaries.resolve(source).unwrap().is_some()); +} + #[cfg(unix)] #[test] fn changing_a_split_index_invalidates_the_child_binding_witness() { From 72f91dad89a613da3e7d99dfd0faba6dea7cd71e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:47:48 -0800 Subject: [PATCH 52/79] test(code-index): emit actual parent worktree Git configuration Refs #4593 Use real newline and tab escapes in the new on-disk config.worktree fixture. The parent worktree redirection must be parsed by Git, not written as literal escape sequences. Preserve the same revocation assertions. --- .../src/source_walk/submodules/tests.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index a6010c2e3c0..b649210c6ac 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -386,7 +386,7 @@ fn parent_worktree_config_change_revokes_cached_child_authority() { let root = fixture.path().join("parent"); git(&root, &["config", "extensions.worktreeConfig", "true"]); let config = root.join(".git/config.worktree"); - let original = format!("[core]\\n\\tworktree = {}\\n", root.display()); + let original = format!("[core]\n\tworktree = {}\n", root.display()); fs::write(&config, &original).unwrap(); let boundaries = LiveSourceBoundaries::open(&root, true).unwrap(); @@ -400,11 +400,8 @@ fn parent_worktree_config_change_revokes_cached_child_authority() { let index_before = super::FileWitness::at(&git_directory.join("index")).unwrap(); let foreign = fixture.path().join("foreign"); fs::create_dir_all(&foreign).unwrap(); - fs::write( - &config, - format!("[core]\\n\\tworktree = {}\\n", foreign.display()), - ) - .unwrap(); + let alternate = format!("[core]\n\tworktree = {}\n", foreign.display()); + fs::write(&config, alternate).unwrap(); assert_eq!(super::FileWitness::at(&git_directory).unwrap(), directory_before); assert_eq!( super::FileWitness::at(&git_directory.join("index")).unwrap(), From 26693a7554e019f61876e48b7439fa2a7c0eb8d1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:20:46 -0800 Subject: [PATCH 53/79] style(code-index): format parent binding witness assertion Apply the pinned formatter layout reported by repository gates. Validation: owning Bazel formatter target passed; independent review completed. --- .../src/source_walk/submodules/tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index b649210c6ac..72bff8960b8 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -402,7 +402,10 @@ fn parent_worktree_config_change_revokes_cached_child_authority() { fs::create_dir_all(&foreign).unwrap(); let alternate = format!("[core]\n\tworktree = {}\n", foreign.display()); fs::write(&config, alternate).unwrap(); - assert_eq!(super::FileWitness::at(&git_directory).unwrap(), directory_before); + assert_eq!( + super::FileWitness::at(&git_directory).unwrap(), + directory_before + ); assert_eq!( super::FileWitness::at(&git_directory.join("index")).unwrap(), index_before From 6ca67c5b3669d72a60a1e6dd1379e765e355f6a2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:12:23 -0800 Subject: [PATCH 54/79] fix(code-index): verify retained submodule authority refusals Fix strict Clippy type complexity and nested binding checks, and format the reconciled tree-capture guard. Exercise split-index corruption through retained readers and revoke a real staged gitlink during snapshot capture, asserting refusal, preserved publication, and recovery. --- .../src/code_index_scheduler/reconcile.rs | 15 ++- .../tests/index_path_policy_tests.rs | 92 ++++++++++++++++++- .../src/source_walk/submodules.rs | 9 +- .../src/source_walk/submodules/tests.rs | 14 ++- 4 files changed, 116 insertions(+), 14 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs index 19c8e750e1f..713455d8e59 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/reconcile.rs @@ -4157,13 +4157,12 @@ impl CodeIndexWorktreeSchedulerV1 { ignored_dependencies::checkpoint_if_present(control)?; // A staged gitlink or registration may be revoked while a file is // streaming. Do not admit its bytes after the original binding expires. - if let Some((_, binding)) = child_binding { - if !binding + if let Some((_, binding)) = child_binding + && !binding .checkout_is_live() .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? - { - return Ok(CapturedFileOutcomeV1::Absent); - } + { + return Ok(CapturedFileOutcomeV1::Absent); } self.capture_candidate_bytes( registry, @@ -4337,9 +4336,9 @@ impl CodeIndexWorktreeSchedulerV1 { ) { Ok(captured) => { if !classification - .child_bindings_live() - .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? - { + .child_bindings_live() + .map_err(|error| CodeIndexSchedulerErrorV1::Git(error.to_string()))? + { return Err(CodeIndexSchedulerErrorV1::Git( "authenticated child checkout binding changed during tree capture" .into(), diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs index 9ec5037f423..1b29895dc67 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/index_path_policy_tests.rs @@ -4,17 +4,19 @@ use std::path::Path; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use tempfile::TempDir; +use tracedecay_code_index::production::{CodeIndexExecutionControlV1, CodeIndexPresealWorkV1}; use tracedecay_domain::{IndexPathPolicyV1, SnapshotFileDispositionV1}; use super::{ - GitFixture, SharedCodeIndexBytePoolV1, published, test_project_id, + GitFixture, SharedCodeIndexBytePoolV1, published, run_git_in, test_project_id, wait_for_live_complete_generation, }; use crate::code_index_scheduler::{ CodeIndexDemandAdmissionV1, CodeIndexHintPolicyV1, CodeIndexReconcileOutcomeV1, - CodeIndexSchedulerRegistryV1, CodeIndexWorktreeSchedulerV1, + CodeIndexSchedulerErrorV1, CodeIndexSchedulerRegistryV1, CodeIndexWorktreeSchedulerV1, }; use crate::config::registry_default_index_path_policy; @@ -395,6 +397,92 @@ fn submodule_policy_binds_live_graph_and_historical_child_pins_across_restart() assert_eq!(indexed_paths(&off), ["dev/launch.ts"]); } +#[test] +fn submodule_revocation_during_capture_refuses_snapshot_and_preserves_publication() { + let parent = GitFixture::new(&[("parent.rs", "pub fn parent() {}\n")]); + let child = GitFixture::new(&[("lib.rs", "pub fn original() {}\n")]); + run_git_in( + parent.path(), + &[ + "-c", + "protocol.file.allow=always", + "submodule", + "add", + "-q", + child.path().to_str().unwrap(), + "app", + ], + ); + run_git_in(parent.path(), &["commit", "-qm", "bind child"]); + let store = TempDir::new().unwrap(); + let mut scheduler = start( + &parent, + store.path(), + policy(&[], &[]).with_submodules(true), + ); + let original = published(scheduler.reconcile_now().unwrap()).generation_id; + assert_eq!(indexed_paths(&scheduler), ["app/lib.rs", "parent.rs"]); + std::fs::write(parent.path().join("app/lib.rs"), "pub fn edited() {}\n").unwrap(); + + struct RevokeAfterCapture<'a> { + root: &'a Path, + revoked: AtomicBool, + } + + impl CodeIndexExecutionControlV1 for RevokeAfterCapture<'_> { + fn is_cancelled(&self) -> bool { + false + } + + fn is_deadline_exceeded(&self) -> bool { + false + } + + fn observed_preseal_work(&self, _work: CodeIndexPresealWorkV1) { + if !self.revoked.swap(true, Ordering::AcqRel) { + run_git_in(self.root, &["rm", "-q", "--cached", "app"]); + } + } + } + + let control = RevokeAfterCapture { + root: parent.path(), + revoked: AtomicBool::new(false), + }; + let captured = + scheduler.capture_authoritative_snapshot_without_active_generation_reuse(Some(&control)); + assert!(control.revoked.load(Ordering::Acquire)); + assert!( + matches!(captured, Err(CodeIndexSchedulerErrorV1::Git(ref message)) + if message.contains("checkout binding changed")), + "revoked child bytes must not become an authoritative snapshot" + ); + assert_eq!( + scheduler + .latest_complete() + .unwrap() + .generation() + .manifest() + .generation_id, + original, + "a refused capture must not replace the published generation" + ); + + run_git_in(parent.path(), &["add", "app"]); + published(scheduler.reconcile_now().unwrap()); + assert_eq!(indexed_paths(&scheduler), ["app/lib.rs", "parent.rs"]); + assert_ne!( + scheduler + .latest_complete() + .unwrap() + .generation() + .manifest() + .generation_id, + original, + "restoring authorization allows the edited child to publish" + ); +} + #[test] fn parent_filtered_source_does_not_require_submodule_registration_when_disabled() { use super::run_git_in; diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 858687043b1..dd2bc160a43 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -39,16 +39,19 @@ pub struct AuthenticatedSubmodule { verified: Arc>>, } +#[cfg(unix)] +type FileIdentity = (u64, u64, u32, u64, i64, i64, i64, i64); + #[cfg(unix)] #[derive(Clone, Debug, PartialEq, Eq)] struct FileWitness { - named: (u64, u64, u32, u64, i64, i64, i64, i64), - target: Option<(u64, u64, u32, u64, i64, i64, i64, i64)>, + named: FileIdentity, + target: Option, } #[cfg(unix)] impl FileWitness { - fn identity(metadata: &std::fs::Metadata) -> (u64, u64, u32, u64, i64, i64, i64, i64) { + fn identity(metadata: &std::fs::Metadata) -> FileIdentity { ( metadata.dev(), metadata.ino(), diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 72bff8960b8..1398cba6547 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -425,9 +425,12 @@ fn parent_worktree_config_change_revokes_cached_child_authority() { fn changing_a_split_index_invalidates_the_child_binding_witness() { let fixture = fixture(); let root = fixture.path().join("parent"); + git(&root, &["update-index", "--split-index"]); let boundaries = LiveSourceBoundaries::open(&root, true).unwrap(); + let source = Path::new("app/first.rs"); + assert!(boundaries.resolve(source).unwrap().is_some()); + let pending = walk(&root); let (child, _) = boundaries.children.get("app").expect("authenticated child"); - git(&root, &["update-index", "--split-index"]); let shared = fs::read_dir(root.join(".git")) .unwrap() @@ -465,4 +468,13 @@ fn changing_a_split_index_invalidates_the_child_binding_witness() { !witness.unchanged().unwrap(), "a corrupt shared-index inode must invalidate the cached Git authority" ); + assert!( + boundaries.resolve(source).is_err(), + "a retained graph reader must refuse unreadable split-index authority" + ); + let error = pending + .filter_map(Result::err) + .next() + .expect("a retained child walk must report unreadable split-index authority"); + assert_eq!(error.io_error().unwrap().kind(), io::ErrorKind::Other); } From bc2ea36c53ab9151e00a8e23aab51c27e7a16f68 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:16:01 -0800 Subject: [PATCH 55/79] test(index): align sealed policy bytes and batch overlap setup Refresh the existing canonical manifest expectation for the required include_submodules policy field. Preserve all segment-byte and round-trip assertions. Also apply the measured canonical batch setup repair for the 10,001-row global-db overlap fixture without changing its acceptance or timeout. Exact-head CI verification remains required. --- .../code_index_suite/production_orchestration.rs | 2 +- .../src/observation_projection/rebuild.rs | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index 519b25d1109..3d502532aac 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -3176,7 +3176,7 @@ fn partitioned_codec_fixture() -> ( } const PARTITIONED_FORMAT_STATE_DIGEST: &str = - "sha256:08e91a89b2dd82615897c5df08b74740fc483551f8f8cef1ef57cf96a93c837f"; + "sha256:b3566559dfef68d8cb147e461ec928920367bae8c314d703375da29fd16f2037"; const PARTITIONED_FORMAT_SEGMENTS: &[(&str, u64)] = &[ ( "sha256:9b175e6aa1ddf6db4138181c0dd1cd522f37381898a58d46e26b1165b613727b", diff --git a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs index beb861a9d9a..af96c31baf2 100644 --- a/crates/tracedecay-global-db/src/observation_projection/rebuild.rs +++ b/crates/tracedecay-global-db/src/observation_projection/rebuild.rs @@ -2693,18 +2693,16 @@ mod activation_tests { let harness = RegisteredGlobalDbHarness::open("session-overlap-paging").await; let mut staged_rows = Vec::with_capacity(OVERLAPS_PAST_TRANSPORT_ROW_CAP as usize); for index in 0..OVERLAPS_PAST_TRANSPORT_ROW_CAP { - let session_id = format!("session.{index:05}"); - let active = SessionRecord { - session_id: session_id.clone(), - ..session(None, None) - }; - assert!(harness.registered.upsert_session(&active).await); staged_rows.push(SessionRecord { - session_id, - title: Some(format!("composer {index:05}")), + session_id: format!("session.{index:05}"), ..session(None, None) }); } + // Fixture setup needs one durable batch, not a commit per overlap. + assert!(harness.registered.upsert_sessions(&staged_rows).await); + for (index, staged) in staged_rows.iter_mut().enumerate() { + staged.title = Some(format!("composer {index:05}")); + } let transaction = harness.registered.begin_write_transaction().await.unwrap(); transaction .execute( From 600f7e8bcebf41b8a2093cde97037887f6f15a1a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:09:27 -0800 Subject: [PATCH 56/79] test(maintenance): await the admitted retention writer --- crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs b/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs index 5407b7d71a4..2787ba27dba 100644 --- a/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs +++ b/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs @@ -439,6 +439,12 @@ fn terminal_backlog_does_not_block_an_unrelated_store_writer() { held.rollback().await.expect("release A"); let outcome = native_runtime::timeout(Duration::from_secs(10), tick.wait()).await.expect("terminal tick settled"); assert!(matches!(outcome, native_runtime::NativeTaskOutcome::Completed(_))); + // A queued maintenance admission is supervised independently of the + // census tick. Join its fair store turn before reading the committed + // page, without admitting a second maintenance page. + native_runtime::timeout(Duration::from_secs(10), + daemon.engine.store_administration.with_writer_in(a_scope, || async {}) + ).await.expect("terminal store writer settled"); assert!(progressed.is_ok(), "B must write while A is suspended with a terminal backlog"); assert_eq!(retired_signals(&shard).await, 16, "one admission runs one bounded terminal page"); assert_eq!(retirement_frontiers(&shard).await, 1); From 2d3fc9970a5175ca19c8b52aa7af76948986e81c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:38:36 -0800 Subject: [PATCH 57/79] test(lsp): trace late delivery receipt admission Port the reviewed delivery admission phase events and full daemon failure log to the persistent submodule branch. The two source files exactly match the diagnostic's original preimage. Keep receipt durability and the existing ACK deadline unchanged. These events distinguish submission, blocking-owner start, and admission result; they do not change the delivery protocol or establish a timeout cause. --- .../src/invocation/lsp_delivery.rs | 20 ++++++++++++++++++- .../runtime_surface_acceptance.rs | 13 ++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs b/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs index 83be58f6b98..fc4e5bfd6b8 100644 --- a/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs +++ b/crates/tracedecay-daemon-service/src/invocation/lsp_delivery.rs @@ -114,10 +114,28 @@ impl LspDeliverySettlementClaim { self, project_lease: ProjectRuntimeRequestLeaseV1, ) -> impl Future + Send { + let owner_event_id = self.settlement.attempt.owner_event_id.clone(); + tracing::trace!( + %owner_event_id, + phase = "submit", + "LSP delivery receipt admission" + ); let admission = tracedecay_runtime_core::native_runtime::on_blocking_pool( async move { let _project_lease = project_lease; - self.admit_inline() + tracing::trace!( + %owner_event_id, + phase = "started", + "LSP delivery receipt admission" + ); + let outcome = self.admit_inline(); + tracing::trace!( + %owner_event_id, + phase = "completed", + ?outcome, + "LSP delivery receipt admission" + ); + outcome } .with_current_subscriber(), ); diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs index 8d0be3c8460..3737ab21b4f 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs @@ -136,9 +136,13 @@ impl RuntimeFixture { } } - fn daemon_log_tail(&self) -> String { + fn daemon_log(&self) -> String { std::fs::read_to_string(self._environment.scratch().join("runtime-daemon.log")) .expect("read isolated runtime daemon log") + } + + fn daemon_log_tail(&self) -> String { + self.daemon_log() .lines() .rev() .take(160) @@ -224,7 +228,8 @@ async fn lsp_runtime_fixture() -> RuntimeFixture { command .env( "RUST_LOG", - "info,tracedecay_daemon_service::invocation=trace", + "info,tracedecay_daemon_service::invocation=trace,\ + tracedecay_private_fs::framed_log=trace", ) .env("TRACEDECAY_SPAN_TIMINGS", "1") .stderr(daemon_log); @@ -439,10 +444,10 @@ async fn poll_lsp_response( .unwrap_or_else(|error| { panic!( "acknowledge daemon LSP frame while awaiting response {response_id}: \ - {error:?}; frame id={:?}, method={:?}; daemon log tail:\n{}", + {error:?}; frame id={:?}, method={:?}; daemon log:\n{}", value.get("id"), value.get("method"), - fixture.daemon_log_tail(), + fixture.daemon_log(), ) }); if value.get("id").and_then(Value::as_u64) == Some(response_id) { From c39c51a5e92d9939fb6f45ff64eea337a3f7feb0 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:45:33 -0800 Subject: [PATCH 58/79] test(daemon): wait for completed terminal retention pages Use the reviewed maintenance fixture from the bounded-retention branch. Retry means the admitted page is still pending, so drive it within the native owner and require its StoreRetention completion before checking the durable page. Open store B before holding store A's writer, preserving the independent two-second writer check. Drive the mounted-store tail through its typed continuation states under a ten-second bound. Keep the first-window, retired-row, frontier, and every-shard durability assertions unchanged. --- .../src/daemon/tests/terminal_housekeeping.rs | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs b/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs index 2787ba27dba..54dd4624338 100644 --- a/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs +++ b/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs @@ -412,11 +412,20 @@ fn terminal_backlog_does_not_block_an_unrelated_store_writer() { for ordinal in 0..33 { seed_stack_signal(&shard, &format!("signal.{ordinal:03}"), 100 + ordinal, true).await; } + let other = tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness::open("terminal-writer-b").await; let held = shard.begin_write_transaction().await.expect("hold A SQLite writer"); let owner = daemon.coordinator.clone(); let administration = daemon.engine.store_administration.clone(); let tick = NativeTask::spawn(async move { - terminal_tick(&owner, &administration, None).await + // A queued store job or busy background CPU returns Retry before + // its page finishes. Drive that state as the cadence owner does. + loop { + let outcome = terminal_tick(&owner, &administration, None).await; + if outcome != MaintenanceTickOutcome::Retry { + return outcome; + } + tokio::task::yield_now().await; + } }).expect("native terminal tick"); let a_scope = store_writer_scope(shard.db_path().parent().expect("A store root"), StoreWriterClass::Content); native_runtime::timeout(Duration::from_secs(10), async { @@ -425,7 +434,6 @@ fn terminal_backlog_does_not_block_an_unrelated_store_writer() { tokio::task::yield_now().await; } }).await.expect("maintenance admitted A"); - let other = tracedecay_global_db::tests::harness::RegisteredGlobalDbHarness::open("terminal-writer-b").await; let b_scope = store_writer_scope(other.registered.db_path().parent().expect("B store root"), StoreWriterClass::Content); let progressed = native_runtime::timeout(Duration::from_secs(2), daemon.engine.store_administration.with_writer_in(b_scope, || async { @@ -438,13 +446,9 @@ fn terminal_backlog_does_not_block_an_unrelated_store_writer() { // settles its admitted owner and releases every fixture lease. held.rollback().await.expect("release A"); let outcome = native_runtime::timeout(Duration::from_secs(10), tick.wait()).await.expect("terminal tick settled"); - assert!(matches!(outcome, native_runtime::NativeTaskOutcome::Completed(_))); - // A queued maintenance admission is supervised independently of the - // census tick. Join its fair store turn before reading the committed - // page, without admitting a second maintenance page. - native_runtime::timeout(Duration::from_secs(10), - daemon.engine.store_administration.with_writer_in(a_scope, || async {}) - ).await.expect("terminal store writer settled"); + assert!(matches!(outcome, native_runtime::NativeTaskOutcome::Completed( + MaintenanceTickOutcome::Continue(MaintenanceContinuation::StoreRetention) + )), "the terminal page must finish before its durable result is asserted: {outcome:?}"); assert!(progressed.is_ok(), "B must write while A is suspended with a terminal backlog"); assert_eq!(retired_signals(&shard).await, 16, "one admission runs one bounded terminal page"); assert_eq!(retirement_frontiers(&shard).await, 1); @@ -481,7 +485,16 @@ fn terminal_housekeeping_bounds_a_mounted_store_window_and_reaches_its_tail() { retired += retired_signals(database).await; } assert_eq!(retired, 8, "one tick admits at most eight store pages"); - assert_eq!(daemon.tick().await, MaintenanceTickOutcome::Complete); + // A queued writer or busy background slot returns Retry before the + // tail is durable. Follow the owner's continuation outcomes while + // preserving the first window's exact page bound above. + native_runtime::timeout(Duration::from_secs(10), async { + while daemon.tick().await != MaintenanceTickOutcome::Complete { + tokio::task::yield_now().await; + } + }) + .await + .expect("the mounted-store tail settles"); for database in &databases { assert_eq!( retired_signals(database).await, From e2800f7559bf782db91254e0bfdbf8c6ebd58bf1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:02:54 -0800 Subject: [PATCH 59/79] fix(diagnostics): appoint compiler before runtime publication Register the producer's pending state under the primitive publication lock and hand it to supervision immediately. Public reads no longer observe a false unsupported result before the compiler task starts. Retain the paused-registration regression with its Clippy-compliant predicate. --- .../src/invocation/primitive.rs | 39 ++++-- .../src/daemon/project_open_owners.rs | 48 ++++--- .../compiler_diagnostics_producer.rs | 52 ++++---- .../project_open_owners/primitive_runtime.rs | 15 ++- .../src/daemon/tests/compiler_status.rs | 125 +++++++++++++++++- 5 files changed, 225 insertions(+), 54 deletions(-) diff --git a/crates/tracedecay-daemon-service/src/invocation/primitive.rs b/crates/tracedecay-daemon-service/src/invocation/primitive.rs index 00631809958..6416b91ea1a 100644 --- a/crates/tracedecay-daemon-service/src/invocation/primitive.rs +++ b/crates/tracedecay-daemon-service/src/invocation/primitive.rs @@ -7,7 +7,7 @@ use tracedecay_agent_hosts::agents::context_scout::{ ContextScoutMutationResultV1, ContextScoutMutationSettlementOutcomeV1, ContextScoutPublicMutationV1, ContextScoutRecentStateV1, }; -use tracedecay_application::diagnostics_producer::TypeScriptProducerStatusHandleV1; +use tracedecay_application::diagnostics_producer::TypeScriptProducerStateV1; use tracedecay_application::primitives::{ ProductionPrimitiveOpenRequestV1, open_production_primitive_runtime, }; @@ -19,6 +19,7 @@ use tracedecay_contracts::context_scout::{ ContextScoutSuggestionProjectionV1, ContextScoutSurfaceRequestV1, }; use tracedecay_domain::sha256_hex_suffix; +use tracedecay_lsp::TypeScriptCompilerAdmission; use tracedecay_runtime_core::cancellation::CancellationToken; use tracedecay_tool_catalog::ApplicationSurfaceOperation; @@ -1380,9 +1381,10 @@ impl DaemonPrimitiveRuntimeRegistrar { &self, project_root: PathBuf, request: ProductionPrimitiveOpenRequestV1, - ) -> Result { + mut compiler_admission: Option, + ) -> Result, DaemonPrimitiveRuntimeRegistrationError> { request.validate_registration_root(&project_root)?; - let retained = self + let (retained, compiler_producer, already_current) = self .service .project_runtimes .register_or_reconcile_with( @@ -1398,11 +1400,28 @@ impl DaemonPrimitiveRuntimeRegistrar { .await .map_err(DaemonPrimitiveRuntimeRegistrationError::from) }, - PrimitiveProjectRuntime::clone, + |runtime| { + // The registry projects under its publication lock. Appoint + // the producer before a read can observe this runtime, so + // discovery is pending rather than falsely unsupported. + let already_current = request.validate_retained(runtime)?; + let producer = if already_current { + compiler_admission.take().map(|admission| { + TypeScriptProducerStateV1::new(runtime.compiler_status(), admission) + }) + } else { + None + }; + Ok::<_, DaemonPrimitiveRuntimeRegistrationError>(( + runtime.clone(), + producer, + already_current, + )) + }, ) - .await?; - if request.validate_retained(&retained)? { - return Ok(retained.compiler_status()); + .await??; + if already_current { + return Ok(compiler_producer); } // A sibling can retain this root's store owner after route eviction. @@ -1419,7 +1438,11 @@ impl DaemonPrimitiveRuntimeRegistrar { .map_err(Into::into) }, || async { Ok(replacement.clone()) }, - PrimitiveProjectRuntime::compiler_status, + |runtime| { + compiler_admission.map(|admission| { + TypeScriptProducerStateV1::new(runtime.compiler_status(), admission) + }) + }, ) .await } diff --git a/crates/tracedecay/src/daemon/project_open_owners.rs b/crates/tracedecay/src/daemon/project_open_owners.rs index 478ce9f4614..494bc5f3cd3 100644 --- a/crates/tracedecay/src/daemon/project_open_owners.rs +++ b/crates/tracedecay/src/daemon/project_open_owners.rs @@ -23,7 +23,7 @@ use tracedecay_agent_hosts::native_integration::{ DaemonNativeIntegrationAnalysisV1, DaemonNativeIntegrationServiceRegistry, NativeIntegrationTargetV1, }; -use tracedecay_application::diagnostics_producer::TypeScriptProducerStatusHandleV1; +use tracedecay_application::diagnostics_producer::TypeScriptProducerStateV1; use tracedecay_application::lsp_runtime::DaemonLspSessionFactory; use tracedecay_application::source_authorization::ProjectSourceAccessSnapshot; use tracedecay_code_index_runtime::git_transactions::DaemonGitIndexTransactionServiceRegistry; @@ -37,6 +37,7 @@ use tracedecay_daemon_service::{ }, }; use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_lsp::TypeScriptCompilerAdmission; use tracedecay_lsp::analyzer::broker::AdmittedLspProvider; use tracedecay_lsp::analyzer::client::LspRefreshTimeouts; use tracedecay_session_runtime::session_retrieval::DaemonSessionLookupPrimitiveV1; @@ -169,8 +170,12 @@ pub(crate) async fn install_project_open_source_edit_owners_for_test( /// Registers the owners that serve code reads on an admitted route: the /// callable-code authorization and the primitive runtime. Returns the -/// admitted root URI and the compiler status retained by their runtime. +/// admitted root URI and the compiler producer appointed by their runtime. #[tracing::instrument(name = "daemon.project.owners.code_reads", level = "trace", skip_all)] +#[expect( + clippy::too_many_arguments, + reason = "Code-read registration carries the admitted compiler with its route authorities." +)] async fn register_project_code_read_owners( invocation: &DaemonInvocationState, project_root: &Path, @@ -179,7 +184,8 @@ async fn register_project_code_read_owners( access: &ProjectSourceAccessSnapshot, session_db: tracedecay_global_db::RegisteredGlobalDbLeaseV1, temporal: Arc, -) -> Result<(String, TypeScriptProducerStatusHandleV1)> { + compiler_admission: Option, +) -> Result<(String, Option)> { // Primitive reads are part of the admitted core route. Publish their // runtime and callable-code authorization before the slower mutation, // delivery, native-integration, and Work owners finish mounting. @@ -221,6 +227,7 @@ async fn register_project_code_read_owners( session_db, temporal, access.clone(), + compiler_admission, ) .await } @@ -273,6 +280,7 @@ pub(super) async fn register_reset_required_route_code_read_owners( &access, session_store, Arc::new(primitive_runtime::ResetRequiredSessionLookupV1), + None, )) .await .map(drop) @@ -382,7 +390,12 @@ pub(super) async fn register_project_open_production_owners( let temporal = Arc::new(DaemonSessionLookupPrimitiveV1::new( server.project_session_application_retrieval_service(&access.scope)?, )); - let (admitted_root_uri, compiler_status) = Box::pin(register_project_code_read_owners( + let compiler_admission = compiler_diagnostics_producer::compiler_admission_for_project( + invocation, + project_root, + &scope, + ); + let (admitted_root_uri, compiler_producer) = Box::pin(register_project_code_read_owners( invocation, project_root, server, @@ -390,8 +403,21 @@ pub(super) async fn register_project_open_production_owners( &access, session_db.clone(), temporal, + compiler_admission, )) .await?; + // The producer needs only the admitted graph, scheduler, and compiler + // authority. Hand the appointed state to supervision before any later + // owner can fail or yield. + if let Some(compiler_producer) = compiler_producer { + let _typescript_producer_admitted = compiler_diagnostics_producer::spawn_typescript_diagnostics_producer( + server, + invocation.clone(), + project_root.to_path_buf(), + Arc::clone(&graph), + compiler_producer, + ); + } owner_phase("code_read_owners_registered"); // One worktree discovery serves both the Git transaction owner here and @@ -783,20 +809,6 @@ pub(super) async fn register_project_open_production_owners( .await; }); - // A TypeScript project with its own compiler gets an automatic diagnostics - // producer: bounded background work after each complete generation, so - // `tracedecay_diagnostics` has a publication to read without a caller - // pasting compiler output first. - let _typescript_producer_admitted = - compiler_diagnostics_producer::spawn_typescript_diagnostics_producer( - server, - invocation.clone(), - project_root.to_path_buf(), - &scope, - Arc::clone(&graph), - compiler_status, - ); - tracing::info!( event = "project_open_owner_phase", project = %project_root.display(), diff --git a/crates/tracedecay/src/daemon/project_open_owners/compiler_diagnostics_producer.rs b/crates/tracedecay/src/daemon/project_open_owners/compiler_diagnostics_producer.rs index 9ed180b9d4a..af53aeac498 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/compiler_diagnostics_producer.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/compiler_diagnostics_producer.rs @@ -15,14 +15,17 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use tracedecay_application::diagnostics_producer::{ - CompilerProducerRunV1, TypeScriptProducerStateV1, TypeScriptProducerStatusHandleV1, - publish_typescript_discovery_v1, run_typescript_producer_v1, + CompilerProducerRunV1, TypeScriptProducerStateV1, publish_typescript_discovery_v1, + run_typescript_producer_v1, }; use tracedecay_application::diagnostics_store::DiagnosticsStore; use tracedecay_code_index_runtime::code_index_scheduler::CodeIndexOwnerSignalsV1; use tracedecay_contracts::now_micros; use tracedecay_domain::CodeGenerationId; -use tracedecay_lsp::{TypeScriptProject, TypeScriptProjectDiscoveryError, typescript_projects}; +use tracedecay_lsp::{ + TypeScriptCompilerAdmission, TypeScriptProject, TypeScriptProjectDiscoveryError, + typescript_projects, +}; use tracedecay_runtime_core::logging::log_daemon_event; use crate::daemon::invocation_state::DaemonInvocationState; @@ -31,34 +34,37 @@ use crate::daemon::invocation_state::DaemonInvocationState; /// per checked tsconfig; it leaves with the store. const BUILD_INFO_DIR: &str = "typescript-build-info"; -/// Spawns the TypeScript producer for `project_root`. Returns `false` when -/// nothing was spawned: a route that indexes no code by contract, so there is -/// never a generation to publish against. The spawned task publishes its -/// discovery and ends at once when no TypeScript project under the root has a -/// compiler; reads then describe the missing compiler from that record. -pub(super) fn spawn_typescript_diagnostics_producer( - owner: &crate::mcp::McpServer, - invocation: DaemonInvocationState, - project_root: PathBuf, +/// The compiler authority for a route that can publish code generations. +/// Routes that never index code must not appoint a producer that cannot run. +pub(super) fn compiler_admission_for_project( + invocation: &DaemonInvocationState, + project_root: &Path, scope: &tracedecay_contracts::ResolvedScope, - graph: Arc, - status: TypeScriptProducerStatusHandleV1, -) -> bool { +) -> Option { if tracedecay_code_index_runtime::project_reads::code_index_disabled_for_scope( &invocation.code_index_schedulers, scope, ) { - log_producer_event(&project_root, "code_index_disabled"); - return false; + log_producer_event(project_root, "code_index_disabled"); + return None; } let Some(admission) = invocation.compiler_admission.get().cloned() else { - log_producer_event(&project_root, "compiler_admission_unavailable"); - return false; + log_producer_event(project_root, "compiler_admission_unavailable"); + return None; }; - // Reopen can retain the primitive runtime while the prior server's - // compiler task is still retiring. Fence it before this task can yield - // in discovery; scheduling order must not choose the current producer. - let mut state = TypeScriptProducerStateV1::new(status, admission); + Some(admission) +} + +/// Runs the producer already appointed when its primitive runtime was +/// registered. Discovery ends the task when no TypeScript project has a +/// compiler; reads then describe the missing compiler from that record. +pub(super) fn spawn_typescript_diagnostics_producer( + owner: &crate::mcp::McpServer, + invocation: DaemonInvocationState, + project_root: PathBuf, + graph: Arc, + mut state: TypeScriptProducerStateV1, +) -> bool { owner.spawn_background_task(tracing::Instrument::instrument( async move { let schedulers = invocation.code_index_schedulers.clone(); diff --git a/crates/tracedecay/src/daemon/project_open_owners/primitive_runtime.rs b/crates/tracedecay/src/daemon/project_open_owners/primitive_runtime.rs index 9627a0f449c..0b399324edf 100644 --- a/crates/tracedecay/src/daemon/project_open_owners/primitive_runtime.rs +++ b/crates/tracedecay/src/daemon/project_open_owners/primitive_runtime.rs @@ -3,7 +3,7 @@ use std::path::Path; use std::sync::Arc; -use tracedecay_application::diagnostics_producer::TypeScriptProducerStatusHandleV1; +use tracedecay_application::diagnostics_producer::TypeScriptProducerStateV1; use tracedecay_application::primitives::{ ProductionPrimitiveCodeAuthoritiesV1, ProductionPrimitiveOpenRequestV1, admitted_root_uri_for_project, @@ -19,6 +19,7 @@ use tracedecay_contracts::retrieval::{ use tracedecay_daemon_service::DaemonPrimitiveRuntimeRegistrationError; use tracedecay_domain::errors::{Result, TraceDecayError}; use tracedecay_graph_query::SourceReadContext; +use tracedecay_lsp::TypeScriptCompilerAdmission; /// Session lookups on a route whose session store is held in its typed /// reset-required state: every lookup answers that refusal. @@ -35,6 +36,10 @@ impl TemporalRetrievalPort for ResetRequiredSessionLookupV1 { } #[tracing::instrument(name = "daemon.project.owners.primitive", level = "trace", skip_all)] +#[expect( + clippy::too_many_arguments, + reason = "Primitive registration carries the admitted compiler with its route authorities." +)] pub(super) async fn open_and_register_project_primitive_runtime( invocation: &DaemonInvocationState, project_root: &Path, @@ -43,7 +48,8 @@ pub(super) async fn open_and_register_project_primitive_runtime( session_db: tracedecay_global_db::RegisteredGlobalDbLeaseV1, temporal: Arc, access: ProjectSourceAccessSnapshot, -) -> Result<(String, TypeScriptProducerStatusHandleV1)> { + compiler_admission: Option, +) -> Result<(String, Option)> { let admitted_root_uri = admitted_root_uri_for_project(project_root).map_err(|error| TraceDecayError::Config { message: format!("project-open admitted root URI denied: {error}"), @@ -60,7 +66,7 @@ pub(super) async fn open_and_register_project_primitive_runtime( message: "project-open primitive runtime requires the mounted ignored-dependency admission authority" .to_owned(), })?; - let compiler_status = invocation + let compiler_producer = invocation .primitive_runtime_registrar() .open_and_register( project_root.to_path_buf(), @@ -78,10 +84,11 @@ pub(super) async fn open_and_register_project_primitive_runtime( access, admitted_root_uri.clone(), ), + compiler_admission, ) .await .map_err(primitive_runtime_registration_error)?; - Ok((admitted_root_uri, compiler_status)) + Ok((admitted_root_uri, compiler_producer)) } fn primitive_runtime_registration_error( diff --git a/crates/tracedecay/src/daemon/tests/compiler_status.rs b/crates/tracedecay/src/daemon/tests/compiler_status.rs index 91439f7645e..9fb0da71571 100644 --- a/crates/tracedecay/src/daemon/tests/compiler_status.rs +++ b/crates/tracedecay/src/daemon/tests/compiler_status.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; -use std::sync::Arc; +use std::sync::{Arc, Mutex, mpsc as blocking_mpsc}; use std::time::Duration; use tracedecay_daemon_protocol::{ DaemonHandshake, DaemonInvocationOutcome, DaemonInvocationRequest, @@ -52,6 +52,32 @@ where } } +struct PrimitiveRegistrationPaused { + registered: mpsc::UnboundedSender<()>, + resume: Mutex>>, +} + +impl Layer for PrimitiveRegistrationPaused +where + S: tracing::Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_close(&self, id: tracing::span::Id, context: Context<'_, S>) { + if context + .span(&id) + .is_none_or(|span| span.name() != "daemon.project.owners.primitive") + { + return; + } + let resume = self.resume.lock().unwrap().take(); + if let Some(resume) = resume { + self.registered.send(()).expect("registration observer"); + resume + .recv_timeout(Duration::from_mins(1)) + .expect("release primitive registration after the public read"); + } + } +} + fn install_project_compiler(project: &Path, compiler: &Path, missing: &str) { std::fs::create_dir_all(project.join("node_modules/.bin")).unwrap(); std::os::unix::fs::symlink(compiler, project.join("node_modules/.bin/tsc")).unwrap(); @@ -126,6 +152,103 @@ fn compiler_failure(outcome: &DaemonInvocationOutcome) -> String { diagnostic.message.clone() } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn published_primitive_reads_pending_before_the_compiler_task_starts() { + const CHILD: &str = "TRACEDECAY_TEST_COMPILER_REGISTRATION_CHILD"; + if std::env::var_os(CHILD).is_none() { + let name = format!( + "{}::published_primitive_reads_pending_before_the_compiler_task_starts", + module_path!().split_once("::").unwrap().1 + ); + let home = TempDir::new().unwrap(); + let output = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &name, "--test-threads=1", "--nocapture"]) + .env(CHILD, "1") + .env("HOME", home.path()) + .env("XDG_CONFIG_HOME", home.path().join("config")) + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_NOSYSTEM", "1") + .output() + .expect("isolated compiler-registration journey"); + assert!( + output.status.success() + && String::from_utf8_lossy(&output.stdout).contains("test result: ok. 1 passed"), + "compiler-registration journey failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + return; + } + + let temp = TempDir::new().unwrap(); + let project = temp.path().join("project"); + let profile = temp.path().join("profile"); + std::fs::create_dir_all(project.join("packages/app/src")).unwrap(); + std::fs::write( + project.join("packages/app/src/index.ts"), + "export const value = 1;\n", + ) + .unwrap(); + std::fs::write( + project.join("packages/app/tsconfig.json"), + r#"{"include":["src/**/*.ts"]}"#, + ) + .unwrap(); + git(&project, &["init", "--quiet", "--initial-branch=main"]); + git(&project, &["add", "--all"]); + git( + &project, + &["commit", "--quiet", "-m", "test: compiler registration"], + ); + let client_identity = test_client_identity_for(profile.clone()); + initialize_test_project(&project, &client_identity).await; + let project = project.canonicalize().unwrap(); + let handshake = DaemonHandshake { + project_path: Some(project), + client_identity, + ..test_handshake_defaults() + }; + let _database_scope = enter_test_daemon_database_scope(&profile, "compiler-registration"); + let engine = test_daemon_engine_for_profile(&profile); + let (registered, mut registrations) = mpsc::unbounded_channel(); + let (resume, paused) = blocking_mpsc::channel(); + tracing::subscriber::set_global_default(tracing_subscriber::registry().with( + PrimitiveRegistrationPaused { + registered, + resume: Mutex::new(Some(paused)), + }, + )) + .unwrap(); + let opening_engine = engine.clone(); + let opening_handshake = handshake.clone(); + let opening = + tokio::spawn(async move { opening_engine.open_project_server(&opening_handshake).await }); + tokio::time::timeout(Duration::from_mins(1), registrations.recv()) + .await + .expect("the real primitive runtime must register") + .expect("registration observer stays alive"); + + // Registration has published the callable runtime, but the remaining + // owners and the compiler task are still held. Exercise the public read + // at precisely the point that formerly claimed no producer would run. + let outcome = diagnostics(&engine, &handshake).await; + resume.send(()).expect("resume the owner mount"); + let (_, _, server, _) = opening.await.unwrap().expect("project owner admission"); + drop(server); + let shutdown = engine.shutdown_all().await; + assert!(shutdown.project_servers.is_clean(), "{shutdown:?}"); + assert!(shutdown.background.unfinished().is_empty(), "{shutdown:?}"); + + let DaemonInvocationOutcome::ApplicationProblem { problem } = outcome else { + panic!("an appointed producer must report its pending state: {outcome:?}"); + }; + assert_eq!( + problem.diagnostic().expect("pending diagnostic").code, + "application.diagnostics.pending", + "the public runtime must not claim that its appointed producer is unsupported" + ); +} + async fn register_work_for_configuration( engine: &DaemonEngine, target_root: &Path, From 51a896e8259b740619c6b303e9803532b0ca27c8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:06:28 -0800 Subject: [PATCH 60/79] test(lsp): restore the baseline fixture trace filter --- .../runtime_acceptance_suite/runtime_surface_acceptance.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs index 3737ab21b4f..1fea2c8882a 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/runtime_surface_acceptance.rs @@ -228,8 +228,7 @@ async fn lsp_runtime_fixture() -> RuntimeFixture { command .env( "RUST_LOG", - "info,tracedecay_daemon_service::invocation=trace,\ - tracedecay_private_fs::framed_log=trace", + "info,tracedecay_daemon_service::invocation=trace", ) .env("TRACEDECAY_SPAN_TIMINGS", "1") .stderr(daemon_log); From 83f8572ac26d1289f6419482f1749c2ad2f126c6 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:10:08 -0800 Subject: [PATCH 61/79] fix(code-index): validate retained submodule search reads --- .../src/ast_grep_search.rs | 13 ++- .../tracedecay-code-index/src/grep_search.rs | 8 +- .../src/source_walk/submodules.rs | 96 ++++++++++++++----- .../src/source_walk/submodules/tests.rs | 27 ++++++ 4 files changed, 113 insertions(+), 31 deletions(-) diff --git a/crates/tracedecay-code-index/src/ast_grep_search.rs b/crates/tracedecay-code-index/src/ast_grep_search.rs index adb9d448b57..5e7e676a5d6 100644 --- a/crates/tracedecay-code-index/src/ast_grep_search.rs +++ b/crates/tracedecay-code-index/src/ast_grep_search.rs @@ -210,6 +210,8 @@ pub enum AstGrepSearchError { InvalidPattern { lang: String, message: String }, /// The `path_glob` was not a valid glob. InvalidGlob(String), + /// The walk or a source read lost its current filesystem authority. + SourceUnavailable(String), } impl std::fmt::Display for AstGrepSearchError { @@ -225,6 +227,9 @@ impl std::fmt::Display for AstGrepSearchError { write!(f, "invalid {lang} pattern: {message}") } AstGrepSearchError::InvalidGlob(glob) => write!(f, "invalid path_glob '{glob}'"), + AstGrepSearchError::SourceUnavailable(message) => { + write!(f, "source is unavailable: {message}") + } } } } @@ -324,7 +329,7 @@ where result.cancelled = true; return Ok(result); } - let Ok(entry) = entry else { continue }; + let entry = entry.map_err(|error| AstGrepSearchError::SourceUnavailable(error.to_string()))?; if !entry.file_type().is_some_and(|ft| ft.is_file()) { continue; } @@ -364,9 +369,9 @@ where result.cancelled = true; return Ok(result); } - let Ok(bytes) = std::fs::read(path) else { - continue; - }; + let bytes = entry + .read() + .map_err(|error| AstGrepSearchError::SourceUnavailable(error.to_string()))?; if looks_binary(&bytes) { continue; } diff --git a/crates/tracedecay-code-index/src/grep_search.rs b/crates/tracedecay-code-index/src/grep_search.rs index e707cb5d53a..b174387dcb6 100644 --- a/crates/tracedecay-code-index/src/grep_search.rs +++ b/crates/tracedecay-code-index/src/grep_search.rs @@ -2,12 +2,12 @@ use std::collections::VecDeque; use std::path::Path; use std::sync::Arc; -use ignore::DirEntry; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use regex::{Regex, RegexBuilder}; use tracedecay_domain::{IndexPathPolicyV1, forward_slash_path}; use crate::parallelism::{CodeIndexParallelismErrorV1, install_interactive}; +use crate::source_walk::submodules::SourceWalkEntry; use crate::source_walk::{forward_slash_relative, source_walk}; const MAX_HITS_PER_FILE: usize = 20; @@ -170,7 +170,7 @@ pub fn search_tree_with_cancel( .map_err(GrepSearchError::Parallelism) } -fn in_scope(entry: &DirEntry, project_root: &Path, scope_prefix: Option<&str>) -> bool { +fn in_scope(entry: &SourceWalkEntry, project_root: &Path, scope_prefix: Option<&str>) -> bool { scope_prefix.is_none_or(|scope| { entry .path() @@ -182,7 +182,7 @@ fn in_scope(entry: &DirEntry, project_root: &Path, scope_prefix: Option<&str>) - } fn scan_grep_file( - entry: &DirEntry, + entry: &SourceWalkEntry, project_root: &Path, matcher: &Regex, query: &GrepSearchQuery, @@ -210,7 +210,7 @@ fn scan_grep_file( result.cancelled = true; return result; } - let Ok(bytes) = std::fs::read(path) else { + let Ok(bytes) = entry.read() else { result.omissions.unavailable_sources += 1; return result; }; diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index dd2bc160a43..880d0d09a61 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -7,8 +7,9 @@ use std::io; #[cfg(unix)] use std::os::unix::fs::MetadataExt; use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; #[cfg(unix)] -use std::sync::{Arc, Mutex, PoisonError}; +use std::sync::{Mutex, PoisonError}; use gix::bstr::ByteSlice; use ignore::{DirEntry, Error as WalkError, Walk}; @@ -264,6 +265,17 @@ impl AuthenticatedSubmodule { Ok(valid) } + fn require_live_checkout(&self) -> io::Result<()> { + if self.checkout_is_live()? { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidData, + "authenticated submodule checkout binding changed during source walk", + )) + } + } + fn verify_checkout_binding(&self) -> io::Result { #[cfg(test)] FULL_BINDING_VALIDATIONS.with(|count| count.set(count.get() + 1)); @@ -301,51 +313,89 @@ impl AuthenticatedSubmodule { } } -/// A lazy walk that cannot outlive its authenticated child checkout binding. +/// A walked path retains the checkout authority used to admit it. Search +/// workers may queue entries after the walk advances, so the read itself must +/// validate that authority before accepting bytes. +#[derive(Clone, Debug)] +pub struct SourceWalkEntry { + entry: DirEntry, + child: Option>, +} + +impl SourceWalkEntry { + pub fn path(&self) -> &Path { + self.entry.path() + } + + pub fn into_path(self) -> PathBuf { + self.entry.into_path() + } + + pub fn file_type(&self) -> Option { + self.entry.file_type() + } + + pub fn metadata(&self) -> Result { + self.entry.metadata() + } + + /// Return bytes only while the original child binding still holds. The + /// second check covers revocation during the filesystem read, matching + /// the source-capture boundary used by graph publication. + pub fn read(&self) -> io::Result> { + if let Some(child) = &self.child { + child.require_live_checkout()?; + } + let bytes = std::fs::read(self.path())?; + if let Some(child) = &self.child { + child.require_live_checkout()?; + } + Ok(bytes) + } +} + +/// A lazy walk and its yielded entries retain the authenticated child binding. /// The ordinary parent walk has no child binding and retains its own policy. pub struct CheckoutWalk { walk: Option, - child: Option, + child: Option>, } impl CheckoutWalk { pub(super) fn new(walk: Walk, child: Option) -> Self { Self { walk: Some(walk), - child, + child: child.map(Arc::new), } } } impl Iterator for CheckoutWalk { - type Item = Result; + type Item = Result; fn next(&mut self) -> Option { self.walk.as_ref()?; - if let Some(child) = &self.child { - let refusal = match child.checkout_is_live() { - Ok(true) => None, - Ok(false) => Some(io::Error::new( - io::ErrorKind::InvalidData, - "authenticated submodule checkout binding changed during source walk", - )), - Err(error) => Some(error), - }; - if let Some(error) = refusal { - // A failed Git read must not become a successful empty walk. - self.walk = None; - return Some(Err(WalkError::WithPath { - path: child.worktree.clone(), - err: Box::new(WalkError::Io(error)), - })); - } + if let Some(child) = &self.child + && let Err(error) = child.require_live_checkout() + { + // A failed Git read must not become a successful empty walk. + self.walk = None; + return Some(Err(WalkError::WithPath { + path: child.worktree.clone(), + err: Box::new(WalkError::Io(error)), + })); } let next = self.walk.as_mut()?.next(); if next.is_none() { self.walk = None; } - next + next.map(|entry| { + entry.map(|entry| SourceWalkEntry { + entry, + child: self.child.clone(), + }) + }) } } diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 1398cba6547..00d62cebe7b 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -162,6 +162,33 @@ fn binding_change_before_iteration(change: impl FnOnce(&Path)) { assert_refused_walk(&root, pending); } +#[test] +fn queued_submodule_entry_keeps_its_read_authority_after_the_walk_drops() { + let fixture = fixture(); + let root = fixture.path().join("parent"); + let entries = walk(&root) + .map(|entry| entry.expect("unchanged source walk")) + .collect::>(); + let child = entries + .iter() + .find(|entry| entry.path() == root.join("app/first.rs")) + .expect("queued child source"); + let parent = entries + .iter() + .find(|entry| entry.path() == root.join("parent.rs")) + .expect("queued parent source"); + assert_eq!(child.read().unwrap(), b"pub fn first() {}\n"); + + fs::remove_file(root.join(".gitmodules")).unwrap(); + + let error = child + .read() + .expect_err("revoked child binding must refuse the read"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert!(error.to_string().contains("checkout binding changed")); + assert_eq!(parent.read().unwrap(), b"pub fn parent() {}\n"); +} + #[test] fn lazy_submodule_walk_refuses_removed_worktree_registration() { binding_change_before_iteration(|root| { From 78f859437daac059e54d9d0414b4ae218c71b164 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:10:11 -0800 Subject: [PATCH 62/79] refactor(code-index): share retained checkout read guards --- .../src/source_walk/submodules.rs | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 880d0d09a61..7700565c70e 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -265,7 +265,7 @@ impl AuthenticatedSubmodule { Ok(valid) } - fn require_live_checkout(&self) -> io::Result<()> { + pub(crate) fn require_live_checkout(&self) -> io::Result<()> { if self.checkout_is_live()? { Ok(()) } else { @@ -276,6 +276,16 @@ impl AuthenticatedSubmodule { } } + pub(crate) fn with_live_checkout( + &self, + read: impl FnOnce() -> io::Result, + ) -> io::Result { + self.require_live_checkout()?; + let value = read()?; + self.require_live_checkout()?; + Ok(value) + } + fn verify_checkout_binding(&self) -> io::Result { #[cfg(test)] FULL_BINDING_VALIDATIONS.with(|count| count.set(count.get() + 1)); @@ -323,6 +333,10 @@ pub struct SourceWalkEntry { } impl SourceWalkEntry { + pub(crate) fn child_binding(&self) -> Option<&Arc> { + self.child.as_ref() + } + pub fn path(&self) -> &Path { self.entry.path() } @@ -343,14 +357,11 @@ impl SourceWalkEntry { /// second check covers revocation during the filesystem read, matching /// the source-capture boundary used by graph publication. pub fn read(&self) -> io::Result> { - if let Some(child) = &self.child { - child.require_live_checkout()?; - } - let bytes = std::fs::read(self.path())?; - if let Some(child) = &self.child { - child.require_live_checkout()?; + let read = || std::fs::read(self.path()); + match &self.child { + Some(child) => child.with_live_checkout(read), + None => read(), } - Ok(bytes) } } From b8270cf7f9640d68c756644a4ad9816062635abf Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:10:14 -0800 Subject: [PATCH 63/79] test(code-index): cover revocation during retained reads --- .../src/source_walk/submodules/tests.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index 00d62cebe7b..f01a99f6621 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -179,7 +179,16 @@ fn queued_submodule_entry_keeps_its_read_authority_after_the_walk_drops() { .expect("queued parent source"); assert_eq!(child.read().unwrap(), b"pub fn first() {}\n"); - fs::remove_file(root.join(".gitmodules")).unwrap(); + let error = child + .child_binding() + .expect("queued child binding") + .with_live_checkout(|| { + let bytes = fs::read(child.path())?; + fs::remove_file(root.join(".gitmodules"))?; + Ok(bytes) + }) + .expect_err("revocation during the read must discard its bytes"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); let error = child .read() From aec4f21c3eb311f40ab334aef5f542c888cf33dc Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:25:36 -0800 Subject: [PATCH 64/79] fix(code-index): retain checkout authority through module audits --- .../src/source_walk/submodules.rs | 2 +- .../src/source_walk/submodules/tests.rs | 2 +- .../src/unmounted_files.rs | 159 +++++++++++++++++- .../src/unmounted_files/rust.rs | 157 ++++++++++------- .../src/unmounted_files/typescript.rs | 4 +- 5 files changed, 259 insertions(+), 65 deletions(-) diff --git a/crates/tracedecay-code-index/src/source_walk/submodules.rs b/crates/tracedecay-code-index/src/source_walk/submodules.rs index 7700565c70e..7a11eb7585e 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules.rs @@ -678,4 +678,4 @@ pub fn pinned_submodules( } #[cfg(test)] -mod tests; +pub(crate) mod tests; diff --git a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs index f01a99f6621..54b3c7dbb1e 100644 --- a/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs +++ b/crates/tracedecay-code-index/src/source_walk/submodules/tests.rs @@ -41,7 +41,7 @@ fn git(root: &Path, args: &[&str]) { ); } -fn fixture() -> TempDir { +pub(crate) fn fixture() -> TempDir { let fixture = TempDir::new().unwrap(); let parent = fixture.path().join("parent"); let child = fixture.path().join("child"); diff --git a/crates/tracedecay-code-index/src/unmounted_files.rs b/crates/tracedecay-code-index/src/unmounted_files.rs index 31cf2d6f594..5e87bf604f4 100644 --- a/crates/tracedecay-code-index/src/unmounted_files.rs +++ b/crates/tracedecay-code-index/src/unmounted_files.rs @@ -48,11 +48,15 @@ mod rust; mod typescript; use std::collections::BTreeMap; +use std::io; use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; use tracedecay_domain::IndexPathPolicyV1; use tracedecay_domain::errors::{Result, TraceDecayError}; +use crate::source_walk::submodules::AuthenticatedSubmodule; + /// One orphaned file and, where the ecosystem admits one, the smallest repair /// that would mount it. pub struct UnmountedFile { @@ -152,6 +156,8 @@ pub struct ProjectAudit { pub(super) struct ProjectFiles { root: PathBuf, files: Vec, + canonical_root: Option, + checkouts: BTreeMap>, nested_repository_note: Option, } @@ -163,22 +169,104 @@ impl ProjectFiles { message: format!("source walk rejected its own scope: {}", error.message), })?; let mut files = Vec::new(); + let mut checkouts = BTreeMap::new(); for entry in walk { let entry = entry.map_err(|error| TraceDecayError::File { message: format!("source walk did not complete: {error}"), path: project_root.display().to_string(), })?; + if let Some(child) = entry.child_binding() { + checkouts + .entry(child.worktree.clone()) + .or_insert_with(|| Arc::clone(child)); + } if !entry.file_type().is_some_and(|kind| kind.is_file()) { continue; } files.push(entry.into_path()); } files.sort(); - Ok(Self { + let canonical_root = if checkouts.is_empty() { + None + } else { + Some( + project_root + .canonicalize() + .map_err(|error| input_error(project_root, error))?, + ) + }; + let files = Self { root: project_root.to_path_buf(), files, + canonical_root, + checkouts, nested_repository_note: diagnostics.skipped_nested_repositories_note(), + }; + files.require_live_checkouts()?; + Ok(files) + } + + /// A declared Rust target or module may be outside the walked inventory. + /// Keep that reachability, while retaining the checkout authority for any + /// later read beneath an admitted child. Map the caller's root spelling to + /// its original canonical identity without resolving a possibly replaced + /// child directory during the lookup. + fn with_live_checkout( + &self, + path: &Path, + read: impl FnOnce() -> io::Result, + ) -> io::Result { + if self.checkouts.is_empty() { + return read(); + } + let path = normalized(path); + let identity = self + .canonical_root + .as_ref() + .and_then(|root| { + path.strip_prefix(normalized(&self.root)) + .ok() + .map(|relative| root.join(relative)) + }) + .unwrap_or(path); + let child = identity + .ancestors() + .find_map(|root| self.checkouts.get(root)); + match child { + Some(child) => child.with_live_checkout(read), + None => read(), + } + } + + fn read_to_string(&self, path: &Path) -> io::Result { + self.with_live_checkout(path, || std::fs::read_to_string(path)) + } + + fn metadata(&self, path: &Path) -> Result> { + self.with_live_checkout(path, || match std::fs::metadata(path) { + Ok(metadata) => Ok(Some(metadata)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), }) + .map_err(|error| input_error(path, error)) + } + + fn read_dir(&self, path: &Path) -> Result> { + self.with_live_checkout(path, || match std::fs::read_dir(path) { + Ok(entries) => entries.collect(), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(Vec::new()), + Err(error) => Err(error), + }) + .map_err(|error| input_error(path, error)) + } + + fn require_live_checkouts(&self) -> Result<()> { + for (path, child) in &self.checkouts { + child + .require_live_checkout() + .map_err(|error| input_error(path, error))?; + } + Ok(()) } pub(super) fn root(&self) -> &Path { @@ -218,6 +306,13 @@ impl ProjectFiles { } } +fn input_error(path: &Path, error: io::Error) -> TraceDecayError { + TraceDecayError::File { + message: format!("unmounted-file audit input is unavailable: {error}"), + path: path.display().to_string(), + } +} + /// Project-relative, forward-slashed rendering of `path`. pub(super) fn relative_display(root: &Path, path: &Path) -> String { tracedecay_domain::forward_slash_path(path.strip_prefix(root).unwrap_or(path)) @@ -270,6 +365,7 @@ pub fn audit_project(project_root: &Path, path_policy: &IndexPathPolicyV1) -> Re }); } } + files.require_live_checkouts()?; Ok(ProjectAudit { ecosystems }) } @@ -384,6 +480,67 @@ mod tests { ); } + #[test] + fn retained_audit_inputs_require_the_original_submodule_binding() { + let fixture = crate::source_walk::submodules::tests::fixture(); + let root = fixture.path().join("parent"); + write( + &root, + "app/Cargo.toml", + "[package]\nname = \"child\"\nversion = \"0.0.0\"\n", + ); + write( + &root, + "app/src/lib.rs", + "#[path = \"../excluded/bridge.rs\"] mod bridge;\n", + ); + write( + &root, + "app/excluded/bridge.rs", + "#[path = \"../src/mounted.rs\"] mod mounted;\n", + ); + write(&root, "app/src/mounted.rs", "pub fn mounted() {}\n"); + write(&root, "app/src/orphan.rs", "pub fn orphan() {}\n"); + write( + &root, + "app/package.json", + r#"{"name":"child","main":"web/index.js"}"#, + ); + write(&root, "app/web/index.js", "import './mounted';\n"); + write(&root, "app/web/mounted.js", "export const mounted = 1;\n"); + write(&root, "app/web/orphan.js", "export const orphan = 1;\n"); + let policy = IndexPathPolicyV1::new(vec!["app/excluded/**".to_owned()], Vec::new()) + .expect("excluded bridge policy") + .with_submodules(true); + let files = ProjectFiles::collect(&root, &policy).expect("authenticated inventory"); + assert_eq!(files.checkouts.len(), 1); + assert!(!files.files.contains(&root.join("app/excluded/bridge.rs"))); + + let rust = super::rust::audit(&files).expect("declared paths retain reachability"); + assert_eq!(rust.mounted_file_count, 3); + assert_eq!(rust.unmounted.len(), 1); + assert_eq!(rust.unmounted[0].file, "app/src/orphan.rs"); + let typescript = super::typescript::audit(&files).expect("live TypeScript audit"); + assert_eq!(typescript.status.as_str(), "audited"); + assert_eq!(typescript.unmounted.len(), 1); + assert_eq!(typescript.unmounted[0].file, "app/web/orphan.js"); + + std::fs::remove_file(root.join(".gitmodules")).expect("retire child registration"); + let denied = files + .read_to_string(&root.join("app/excluded/bridge.rs")) + .expect_err("declared paths retain the retired checkout binding"); + assert_eq!(denied.kind(), std::io::ErrorKind::InvalidData); + assert!(matches!( + super::rust::audit(&files), + Err(tracedecay_domain::errors::TraceDecayError::File { .. }) + )); + let typescript = super::typescript::audit(&files).expect("incomplete TypeScript audit"); + assert_eq!(typescript.status.as_str(), "partial"); + assert!(typescript.unmounted.is_empty()); + assert!(files.require_live_checkouts().is_err()); + assert!(files.read_to_string(&root.join("parent.rs")).is_ok()); + } + /// A language with no reachability model is named with its file count, so /// "no findings" can never be mistaken for "nothing was looked at". #[test] diff --git a/crates/tracedecay-code-index/src/unmounted_files/rust.rs b/crates/tracedecay-code-index/src/unmounted_files/rust.rs index 67eb426dcd2..dc657bd49df 100644 --- a/crates/tracedecay-code-index/src/unmounted_files/rust.rs +++ b/crates/tracedecay-code-index/src/unmounted_files/rust.rs @@ -16,6 +16,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::ffi::{OsStr, OsString}; +use std::io; use std::path::{Path, PathBuf}; use ignore::overrides::OverrideBuilder; @@ -23,7 +24,8 @@ use tracedecay_domain::errors::{Result, TraceDecayError}; use tree_sitter::{Node, Parser}; use super::{ - EcosystemAudit, EcosystemStatus, ProjectFiles, UnmountedFile, normalized, relative_display, + EcosystemAudit, EcosystemStatus, ProjectFiles, UnmountedFile, input_error, normalized, + relative_display, }; /// True for the bytes that may appear inside a Rust identifier, so `path` @@ -144,7 +146,7 @@ pub(super) fn audit(files: &ProjectFiles) -> Result { return Ok(EcosystemAudit::not_present("rust", RUST_VERDICT)); } - let (excluded_globs, workspace_excluded_dirs) = workspace_declarations(project_root)?; + let (excluded_globs, workspace_excluded_dirs) = workspace_declarations(files)?; let manifests = manifest_paths .iter() .filter_map(|manifest| manifest.parent()) @@ -155,10 +157,10 @@ pub(super) fn audit(files: &ProjectFiles) -> Result { .any(|excluded| dir.starts_with(excluded)) }) .map(|dir| { - let package = cargo_package(project_root, &dir); - ManifestEntry { dir, package } + let package = cargo_package(files, &dir)?; + Ok(ManifestEntry { dir, package }) }) - .collect::>(); + .collect::>>()?; // One mounted set across all packages: a file is mounted if *any* target in // the tree reaches it, and asking per package would report a file @@ -170,7 +172,13 @@ pub(super) fn audit(files: &ProjectFiles) -> Result { let mut entry_point_count = 0usize; for package in manifests.iter().filter_map(|entry| entry.package.as_ref()) { entry_point_count += package.roots.len(); - walk_mounted_files(&package.roots, &mut mounted, &mut visited, &mut directories); + walk_mounted_files( + files, + &package.roots, + &mut mounted, + &mut visited, + &mut directories, + )?; } let excluded = build_excluded_matcher(project_root, &excluded_globs); @@ -256,11 +264,17 @@ pub(super) fn audit(files: &ProjectFiles) -> Result { level = "trace", skip_all )] -fn workspace_declarations(project_root: &Path) -> Result<(Vec, Vec)> { - let Ok(text) = std::fs::read_to_string(project_root.join("Cargo.toml")) else { - // Not a cargo project at the root. Packages may still exist deeper in - // the tree, so this is a missing declaration, not a missing audit. - return Ok((Vec::new(), Vec::new())); +fn workspace_declarations(files: &ProjectFiles) -> Result<(Vec, Vec)> { + let project_root = files.root(); + let path = project_root.join("Cargo.toml"); + let text = match files.read_to_string(&path) { + Ok(text) => text, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + // Not a cargo project at the root. Packages may still exist deeper + // in the tree, so this is a missing declaration, not a missing audit. + return Ok((Vec::new(), Vec::new())); + } + Err(error) => return Err(input_error(&path, error)), }; let Ok(manifest) = toml::from_str::(&text) else { return Err(TraceDecayError::Config { @@ -320,11 +334,18 @@ fn build_excluded_matcher( /// audit walks. Returns `None` for a manifest with no `[package]` (a virtual /// workspace root owns no targets) and for one whose declared targets do not /// exist on disk, both are claim boundaries with nothing to audit. -fn cargo_package(project_root: &Path, dir: &Path) -> Option { +fn cargo_package(files: &ProjectFiles, dir: &Path) -> Result> { + let project_root = files.root(); let manifest_path = dir.join("Cargo.toml"); - let manifest_text = std::fs::read_to_string(&manifest_path).ok()?; - let manifest = toml::from_str::(&manifest_text).ok()?; - let package = manifest.get("package")?; + let manifest_text = files + .read_to_string(&manifest_path) + .map_err(|error| input_error(&manifest_path, error))?; + let Ok(manifest) = toml::from_str::(&manifest_text) else { + return Ok(None); + }; + let Some(package) = manifest.get("package") else { + return Ok(None); + }; let name = package .get("name") .and_then(toml::Value::as_str) @@ -380,7 +401,7 @@ fn cargo_package(project_root: &Path, dir: &Path) -> Option { .and_then(toml::Value::as_bool) .unwrap_or(true); if auto_discovers { - roots.extend(auto_discovered_roots(&dir.join(directory))); + roots.extend(auto_discovered_roots(files, &dir.join(directory))?); } } @@ -388,7 +409,11 @@ fn cargo_package(project_root: &Path, dir: &Path) -> Option { let mut seen: HashSet = HashSet::new(); for root in roots { let root = normalized(&root); - if !root.is_file() || !seen.insert(root.clone()) { + if !files + .metadata(&root)? + .is_some_and(|metadata| metadata.is_file()) + || !seen.insert(root.clone()) + { continue; } // A crate root's own `mod` declarations resolve against its parent @@ -399,22 +424,26 @@ fn cargo_package(project_root: &Path, dir: &Path) -> Option { deduped.push((root, parent)); } if deduped.is_empty() { - return None; + return Ok(None); } - let source_dirs = CARGO_SOURCE_DIRS - .iter() - .map(|name| dir.join(name)) - .filter(|candidate| candidate.is_dir()) - .map(|candidate| normalized(&candidate)) - .collect::>(); + let mut source_dirs = Vec::new(); + for name in CARGO_SOURCE_DIRS { + let candidate = dir.join(name); + if files + .metadata(&candidate)? + .is_some_and(|metadata| metadata.is_dir()) + { + source_dirs.push(normalized(&candidate)); + } + } - Some(CratePackage { + Ok(Some(CratePackage { name, manifest: relative_display(project_root, &manifest_path), roots: deduped, source_dirs, - }) + })) } /// Cargo's convention for a target directory: every `/*.rs`, plus every @@ -424,16 +453,19 @@ fn cargo_package(project_root: &Path, dir: &Path) -> Option { /// a root; every other file under `tests/mcp_suite/` is reachable only through /// that root's own `mod` declarations, so a file added to the directory without /// a matching `mod` line compiles nowhere and is seen by no one. -fn auto_discovered_roots(directory: &Path) -> Vec { - let Ok(entries) = std::fs::read_dir(directory) else { - return Vec::new(); - }; +fn auto_discovered_roots(files: &ProjectFiles, directory: &Path) -> Result> { let mut roots = Vec::new(); - for entry in entries.flatten() { + for entry in files.read_dir(directory)? { let path = entry.path(); - if path.is_dir() { + if files + .metadata(&path)? + .is_some_and(|metadata| metadata.is_dir()) + { let main = path.join("main.rs"); - if main.is_file() { + if files + .metadata(&main)? + .is_some_and(|metadata| metadata.is_file()) + { roots.push(main); } } else if path.extension().is_some_and(|ext| ext == "rs") { @@ -441,7 +473,7 @@ fn auto_discovered_roots(directory: &Path) -> Vec { } } roots.sort(); - roots + Ok(roots) } /// Directory listings, cached, so a resolved candidate carries the file's real @@ -458,36 +490,41 @@ struct DirIndex { } impl DirIndex { - fn resolve(&mut self, candidate: &Path) -> Option { - if !candidate.is_file() { - return None; + fn resolve(&mut self, files: &ProjectFiles, candidate: &Path) -> Result> { + if !files + .metadata(candidate)? + .is_some_and(|metadata| metadata.is_file()) + { + return Ok(None); } let candidate = normalized(candidate); let (Some(parent), Some(name)) = (candidate.parent(), candidate.file_name()) else { - return Some(candidate.clone()); + return Ok(Some(candidate.clone())); }; - let listing = self - .entries - .entry(parent.to_path_buf()) - .or_insert_with(|| read_dir_index(parent)); - match on_disk_spelling(listing, name) { - Some(actual) if actual != name => Some(parent.join(actual)), - _ => Some(candidate.clone()), - } + let listing = match self.entries.entry(parent.to_path_buf()) { + std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(read_dir_index(files, parent)?) + } + }; + Ok(Some(match on_disk_spelling(listing, name) { + Some(actual) if actual != name => parent.join(actual), + _ => candidate.clone(), + })) } } -fn read_dir_index(directory: &Path) -> HashMap> { +fn read_dir_index( + files: &ProjectFiles, + directory: &Path, +) -> Result>> { let mut index: HashMap> = HashMap::new(); - let Ok(entries) = std::fs::read_dir(directory) else { - return index; - }; - for entry in entries.flatten() { + for entry in files.read_dir(directory)? { let name = entry.file_name(); let lowered = name.to_string_lossy().to_ascii_lowercase(); index.entry(lowered).or_default().push(name); } - index + Ok(index) } /// The on-disk name for `wanted`: itself when the directory really holds that @@ -519,11 +556,12 @@ fn on_disk_spelling(listing: &HashMap>, wanted: &OsStr) -> skip_all )] fn walk_mounted_files( + files: &ProjectFiles, roots: &[(PathBuf, PathBuf)], mounted: &mut HashSet, visited: &mut HashSet<(PathBuf, PathBuf)>, directories: &mut DirIndex, -) { +) -> Result<()> { let mut queue: VecDeque<(PathBuf, PathBuf)> = VecDeque::new(); for (root, directory) in roots { mounted.insert(root.clone()); @@ -532,9 +570,9 @@ fn walk_mounted_files( } } while let Some((file, directory)) = queue.pop_front() { - let Ok(source) = std::fs::read_to_string(&file) else { - continue; - }; + let source = files + .read_to_string(&file) + .map_err(|error| input_error(&file, error))?; let Some(tree) = parse_rust(&source) else { continue; }; @@ -563,12 +601,12 @@ fn walk_mounted_files( // rather than looking in a phantom `automation/tests/` directory. let mut reached = Vec::new(); for candidate in inclusive { - if let Some(actual) = directories.resolve(&candidate) { + if let Some(actual) = directories.resolve(files, &candidate)? { reached.push((actual, true)); } } for candidate in exclusive { - if let Some(actual) = directories.resolve(&candidate) { + if let Some(actual) = directories.resolve(files, &candidate)? { reached.push((actual, false)); break; } @@ -592,7 +630,7 @@ fn walk_mounted_files( let mut includes = Vec::new(); collect_include_paths(&source, tree.root_node(), &file_directory, &mut includes); for include in includes { - let Some(actual) = directories.resolve(&include) else { + let Some(actual) = directories.resolve(files, &include)? else { continue; }; mounted.insert(actual.clone()); @@ -601,6 +639,7 @@ fn walk_mounted_files( } } } + Ok(()) } #[tracing::instrument( diff --git a/crates/tracedecay-code-index/src/unmounted_files/typescript.rs b/crates/tracedecay-code-index/src/unmounted_files/typescript.rs index 3789b0c41b7..969cd3911e4 100644 --- a/crates/tracedecay-code-index/src/unmounted_files/typescript.rs +++ b/crates/tracedecay-code-index/src/unmounted_files/typescript.rs @@ -173,9 +173,7 @@ impl AliasRule { /// Walks every discovered npm package's import graph and reports the source /// files no entry point reaches. pub(super) fn audit(files: &ProjectFiles) -> Result { - Ok(audit_with_reader(files, &|path| { - std::fs::read_to_string(path) - })) + Ok(audit_with_reader(files, &|path| files.read_to_string(path))) } fn audit_with_reader( From 08b3ea8afa55432555b89961ff57b5c8c4d2b902 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:09:33 -0800 Subject: [PATCH 65/79] style(code-index): apply hosted search formatting --- crates/tracedecay-code-index/src/ast_grep_search.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/src/ast_grep_search.rs b/crates/tracedecay-code-index/src/ast_grep_search.rs index 5e7e676a5d6..32ef5b76a03 100644 --- a/crates/tracedecay-code-index/src/ast_grep_search.rs +++ b/crates/tracedecay-code-index/src/ast_grep_search.rs @@ -329,7 +329,8 @@ where result.cancelled = true; return Ok(result); } - let entry = entry.map_err(|error| AstGrepSearchError::SourceUnavailable(error.to_string()))?; + let entry = + entry.map_err(|error| AstGrepSearchError::SourceUnavailable(error.to_string()))?; if !entry.file_type().is_some_and(|ft| ft.is_file()) { continue; } From 70795820f6eac2cf81e42e1212fa4443b7a153f9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:43:03 -0800 Subject: [PATCH 66/79] test(sessions): reuse registered owner for admission digests --- .../src/session/tests/temporal_application.rs | 89 ++++++++++--------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/crates/tracedecay-session-memory/src/session/tests/temporal_application.rs b/crates/tracedecay-session-memory/src/session/tests/temporal_application.rs index 39a4692d58d..60c24c53a68 100644 --- a/crates/tracedecay-session-memory/src/session/tests/temporal_application.rs +++ b/crates/tracedecay-session-memory/src/session/tests/temporal_application.rs @@ -498,16 +498,16 @@ fn rendered(outcome: &SessionRetrievalOutcome) -> &str { } /// `(request digest, access digest)` the production admission step binds for -/// this request, read before any execution is constructed. -async fn admitted_digests( +/// this request, read without executing against the registered store. +fn admitted_digests( + harness: &RegisteredTemporalHarness, authorizer: A, context: TestRequestContext, query: SessionTemporalQuery, estimator: Words, configuration: SessionRetrievalConfiguration, ) -> (String, String) { - let harness = RegisteredTemporalHarness::open("temporal-admission-digest").await; - let service = service(&harness, authorizer, estimator, configuration); + let service = service(harness, authorizer, estimator, configuration); let admitted = service .admit_execution(&context.request, context.binding(), &query) .unwrap(); @@ -518,16 +518,23 @@ async fn admitted_digests( ) } -async fn admitted_digest( +fn admitted_digest( + harness: &RegisteredTemporalHarness, authorizer: A, context: TestRequestContext, query: SessionTemporalQuery, estimator: Words, configuration: SessionRetrievalConfiguration, ) -> String { - admitted_digests(authorizer, context, query, estimator, configuration) - .await - .0 + admitted_digests( + harness, + authorizer, + context, + query, + estimator, + configuration, + ) + .0 } const BASELINE_GRANT: GrantAuthorizer = GrantAuthorizer { @@ -552,29 +559,29 @@ async fn canonical_request_digest_drifts_for_query_and_root_changes() { )); } let first = admitted_digest( + &harness, AllowAuthorizer, context("root.one", policy), query("alpha"), Words("words-v1"), configuration(), - ) - .await; + ); let second = admitted_digest( + &harness, AllowAuthorizer, context("root.one", policy), query("beta"), Words("words-v1"), configuration(), - ) - .await; + ); let third = admitted_digest( + &harness, AllowAuthorizer, context("root.two", policy), query("alpha"), Words("words-v1"), configuration(), - ) - .await; + ); assert_ne!(first, second); assert_ne!(first, third); } @@ -660,36 +667,39 @@ async fn application_binds_and_freezes_root_wide_retrieval_scope() { #[tokio::test] async fn canonical_digest_binds_every_semantic_input_and_excludes_resume_ephemera() { + // Admission binds inputs without executing a store query. Open its real + // registered owner once, before any request deadline starts. + let harness = RegisteredTemporalHarness::open("temporal-admission-digest").await; let baseline_context = || ContextSpec::new("root.one", DIGEST); let (baseline, baseline_access) = admitted_digests( + &harness, BASELINE_GRANT, baseline_context().build(), QuerySpec::default().build(), Words("words-v1"), configuration(), - ) - .await; - let digest_for = |context: ContextSpec, spec: QuerySpec| async move { + ); + let digest_for = |context: ContextSpec, spec: QuerySpec| { let estimator = Words(if spec.context_budget.estimator_version == "words-v2" { "words-v2" } else { "words-v1" }); admitted_digests( + &harness, BASELINE_GRANT, context.build(), spec.build(), estimator, configuration(), ) - .await }; let root_wide = QuerySpec { retrieval_scope: Some(SessionRetrievalScope::AllSessionsInAuthorizedRoot), ..QuerySpec::default() }; - let (root_wide_digest, _) = digest_for(baseline_context(), root_wide.clone()).await; + let (root_wide_digest, _) = digest_for(baseline_context(), root_wide.clone()); assert_ne!(baseline, root_wide_digest); assert_eq!( root_wide_digest, @@ -700,7 +710,6 @@ async fn canonical_digest_binds_every_semantic_input_and_excludes_resume_ephemer ..root_wide }, ) - .await .0 ); @@ -715,13 +724,13 @@ async fn canonical_digest_binds_every_semantic_input_and_excludes_resume_ephemer }, ] { let (request_digest, access_digest) = admitted_digests( + &harness, authorizer, baseline_context().build(), QuerySpec::default().build(), Words("words-v1"), configuration(), - ) - .await; + ); assert_ne!(baseline, request_digest); assert_eq!(baseline_access, access_digest); } @@ -825,7 +834,7 @@ async fn canonical_digest_binds_every_semantic_input_and_excludes_resume_ephemer }); } for spec in semantic_variants { - assert_ne!(baseline, digest_for(baseline_context(), spec).await.0); + assert_ne!(baseline, digest_for(baseline_context(), spec).0); } for (budgets, configuration_digest) in [ @@ -849,7 +858,7 @@ async fn canonical_digest_binds_every_semantic_input_and_excludes_resume_ephemer configuration: configuration_digest, ..baseline_context() }; - assert_ne!(baseline, digest_for(context, QuerySpec::default()).await.0); + assert_ne!(baseline, digest_for(context, QuerySpec::default()).0); } for (capability, policy) in [([0x5b; 32], DIGEST), (DIGEST, [0x5b; 32])] { let context = ContextSpec { @@ -858,7 +867,7 @@ async fn canonical_digest_binds_every_semantic_input_and_excludes_resume_ephemer policy, ..baseline_context() }; - let (request_digest, access_digest) = digest_for(context, QuerySpec::default()).await; + let (request_digest, access_digest) = digest_for(context, QuerySpec::default()); assert_ne!(baseline, request_digest); if policy == DIGEST { assert_eq!(baseline_access, access_digest); @@ -873,20 +882,18 @@ async fn canonical_digest_binds_every_semantic_input_and_excludes_resume_ephemer assert_ne!( baseline, admitted_digest( + &harness, BASELINE_GRANT, baseline_context().build(), QuerySpec::default().build(), Words("words-v1"), configuration, ) - .await ); } assert_ne!( baseline, - digest_for(ContextSpec::new("root.two", DIGEST), QuerySpec::default()) - .await - .0 + digest_for(ContextSpec::new("root.two", DIGEST), QuerySpec::default()).0 ); let route = |repository: &str, worktree: &str, branch: &str| { @@ -969,7 +976,7 @@ async fn canonical_digest_binds_every_semantic_input_and_excludes_resume_ephemer identity, ..baseline_context() }; - assert_ne!(baseline, digest_for(context, QuerySpec::default()).await.0); + assert_ne!(baseline, digest_for(context, QuerySpec::default()).0); } let (other_actor_request, other_actor_access) = digest_for( @@ -979,8 +986,7 @@ async fn canonical_digest_binds_every_semantic_input_and_excludes_resume_ephemer ..baseline_context() }, QuerySpec::default(), - ) - .await; + ); assert_ne!(baseline, other_actor_request); assert_eq!(baseline_access, other_actor_access); @@ -996,7 +1002,6 @@ async fn canonical_digest_binds_every_semantic_input_and_excludes_resume_ephemer ..QuerySpec::default() }, ) - .await .0 ); } @@ -1157,7 +1162,9 @@ async fn request_budget_preflight_rejects_before_execution() { #[tokio::test] async fn mode_cutoff_is_bound_and_a_forged_cursor_is_refused_by_the_cursor_authority() { + let harness = RegisteredTemporalHarness::open("temporal-forged-cursor").await; let as_of = admitted_digest( + &harness, AllowAuthorizer, context("root.one", DIGEST), QuerySpec { @@ -1169,9 +1176,9 @@ async fn mode_cutoff_is_bound_and_a_forged_cursor_is_refused_by_the_cursor_autho .build(), Words("words-v1"), configuration(), - ) - .await; + ); let evolution = admitted_digest( + &harness, AllowAuthorizer, context("root.one", DIGEST), QuerySpec { @@ -1181,11 +1188,9 @@ async fn mode_cutoff_is_bound_and_a_forged_cursor_is_refused_by_the_cursor_autho .build(), Words("words-v1"), configuration(), - ) - .await; + ); assert_ne!(as_of, evolution); - let harness = RegisteredTemporalHarness::open("temporal-forged-cursor").await; let policy = harness.seed_application_fixture().await; let service = allow(&harness); assert!( @@ -1205,21 +1210,21 @@ async fn mode_cutoff_is_bound_and_a_forged_cursor_is_refused_by_the_cursor_autho }; assert_eq!( admitted_digest( + &harness, AllowAuthorizer, context("root.one", policy), forged.clone().build(), Words("words-v1"), configuration(), - ) - .await, + ), admitted_digest( + &harness, AllowAuthorizer, context("root.one", policy), inline_query().build(), Words("words-v1"), configuration(), - ) - .await, + ), "the cursor is resume ephemera, not a request input" ); assert_eq!( From 91e02d447c5626c003801e716e977831323807f5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:45:14 -0800 Subject: [PATCH 67/79] test(code-index): retire obsolete busy wake expectation --- .../tests/code_index_scheduler/reconcile.rs | 87 ------------------- 1 file changed, 87 deletions(-) diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs index 1644ebf7f66..4a166a7a7a9 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/reconcile.rs @@ -13254,93 +13254,6 @@ async fn graph_decode_does_not_block_text_freshness() { registry.shutdown().await; } -/// Busy admission preserves the prior generation and schedules a follow-up wake -/// so serve-during-refresh cannot leave the index stale indefinitely. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn busy_admission_schedules_follow_up_cadence_wake() { - let fixture = GitFixture::new(ALPHA_LIB_V1); - let store = TempDir::new().expect("store root"); - let registry = CodeIndexSchedulerRegistryV1::new(1); - registry - .mount_worktree( - test_project_id(), - fixture.path(), - store.path().to_path_buf(), - ) - .await - .expect("mount"); - let expected = wait_for_live_complete_generation(®istry, fixture.path()) - .await - .generation - .manifest() - .generation_id - .clone(); - // The mount pass records its receipt after it releases its in-progress - // guard, so wait for the receipt itself before taking the baseline. - wait_for_quiescent_owner_pass(®istry, fixture.path()).await; - wait_for_event_to_ready(®istry).await; - let before_receipts = registry.event_to_ready_receipts().len(); - // A new branch ref moves Git metadata past the seated proof without - // changing HEAD or any source, so the follow-up pass is an unchanged - // reconcile. A busy read must leave that one follow-up wake rather than - // trust the expired proof. - git(fixture.path(), &["branch", "proof-moved"]); - - let scheduler = registry - .scheduler_handle(fixture.path()) - .await - .expect("scheduler handle"); - let (held_tx, held_rx) = std::sync::mpsc::channel(); - let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); - let lock_thread = std::thread::spawn(move || { - let _guard = scheduler - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - held_tx.send(()).expect("signal held"); - let _ = release_rx.recv(); - }); - held_rx.recv().expect("lock acquired"); - // Busy means a pass owns the worktree: wake one and let it park on the - // held scheduler before the read. - let _ = registry.notify_hook_overflow(fixture.path()).await; - wait_for_owner_pass(®istry, fixture.path()).await; - - // The scheduler stays held until after this read, so a read that waited - // on it could never return. - let latest = tokio::time::timeout( - SERVING_SEAT_FAILURE_CEILING, - registry.latest_complete_fresh(fixture.path()), - ) - .await - .expect("must not wait on busy lock") - .expect("prior generation served"); - assert_eq!(latest.generation.manifest().generation_id, expected); - - release_tx.send(()).expect("release"); - lock_thread.join().expect("join"); - - // The follow-up wake must produce its own cadence receipt after the lock - // frees. - let mut signals = OwnerSignals::subscribe(®istry, fixture.path()).await; - let deadline = std::time::Instant::now() + SERVING_SEAT_FAILURE_CEILING; - loop { - let receipts = registry.event_to_ready_receipts(); - if receipts - .iter() - .skip(before_receipts) - .any(|receipt| receipt.trigger == CodeIndexCadenceTriggerV1::BusyFollowUp) - { - break; - } - assert!( - std::time::Instant::now() <= deadline, - "busy follow-up wake did not produce a cadence receipt" - ); - signals.changed_before(deadline).await; - } - registry.shutdown().await; -} - /// A successful reconcile is product state; optional lifecycle telemetry may /// not keep its in-progress guard or freshness response open while the /// observability store is busy. From a4751e1bc55e62b5c9142dd84f4a9a900430d5a1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:24:34 -0800 Subject: [PATCH 68/79] test(daemon): run disabled scout journey on native runtime --- .../advisory_cycle_language_journey_test.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs index 9bd9ef19436..828c7cf604d 100644 --- a/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs @@ -446,8 +446,15 @@ async fn saved_edit_of_a_language_less_file_completes_its_hook_work() { harness.shutdown().await; } -#[tokio::test(flavor = "multi_thread")] -async fn disabled_context_scout_ends_the_hook_cycle_after_its_advisory_half() { +#[test] +fn disabled_context_scout_ends_the_hook_cycle_after_its_advisory_half() { + tracedecay_runtime_core::native_runtime::default_builder() + .build() + .expect("native test runtime") + .block_on(verify_disabled_context_scout_hook_completion()); +} + +async fn verify_disabled_context_scout_hook_completion() { let isolation = tempfile::TempDir::new().expect("production harness isolation"); let (harness, project) = settled_checkout_with_language_less_file(isolation.path()).await; From ae1384ffcea1a5027516350ed6afef51356631af Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:19:30 -0800 Subject: [PATCH 69/79] test(daemon): run language refusal on native runtime --- .../advisory_cycle_language_journey_test.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs index 828c7cf604d..dbba2fdf634 100644 --- a/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/advisory_cycle_language_journey_test.rs @@ -323,8 +323,15 @@ async fn settled_checkout_with_language_less_file( (harness, project) } -#[tokio::test(flavor = "multi_thread")] -async fn language_less_document_is_a_terminal_advisory_request() { +#[test] +fn language_less_document_is_a_terminal_advisory_request() { + tracedecay_runtime_core::native_runtime::default_builder() + .build() + .expect("native test runtime") + .block_on(verify_language_less_document_terminal_advisory_request()); +} + +async fn verify_language_less_document_terminal_advisory_request() { let isolation = tempfile::TempDir::new().expect("production harness isolation"); let (harness, project) = settled_checkout_with_language_less_file(isolation.path()).await; From 0caca64c2ec51e81d904165202ecf87e39643763 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:19:32 -0800 Subject: [PATCH 70/79] test(global-db): reuse the published cancellation fixture --- .../src/registered/workflow_schema_tests.rs | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs index 01e87f7f53c..2973d16e469 100644 --- a/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/workflow_schema_tests.rs @@ -1074,10 +1074,36 @@ mod published_artifact_ceiling { let stride = (total_polls / 12).max(1); let mut cancelled_published = 0; + // Each convergence joins its exact runtime retirement. Reuse the + // canonically initialized store, restoring the upgrade's inputs and + // proving the entire workflow state before admitting a fresh runtime. + let store = &reference.store; for cancel_at in (1..total_polls).step_by(stride) { - let daemon_store = DaemonPublishedStore::create().await; - let store = &daemon_store.store; - store.run(BULK_ROWS); + { + let mut connection = Connection::open(&store.path).unwrap(); + let transaction = connection.transaction().unwrap(); + republish(&transaction, &before.table_sql, &before.digest); + { + let mut insert = transaction + .prepare( + "INSERT INTO workflow_artifact_payloads + (rowid, payload_digest, byte_length, payload) + VALUES (?1, ?2, ?3, ?4)", + ) + .unwrap(); + for (rowid, digest, byte_length, payload) in &before.artifacts { + insert + .execute(rusqlite::params![rowid, digest, byte_length, payload]) + .unwrap(); + } + } + transaction.commit().unwrap(); + } + assert_eq!( + store.state(), + before, + "poll {cancel_at} must start from the published workflow state" + ); let (outcome, _) = converge_cancelled_at(&store.path, cancel_at).await; let state = store.state(); match outcome { From bbf12146cb68407ee7cad0369fb49be3ba06bcc5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:19:48 -0800 Subject: [PATCH 71/79] test(mcp): measure initial publication phases --- .../tests/mcp_suite/analytics_test.rs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/crates/tracedecay/tests/mcp_suite/analytics_test.rs b/crates/tracedecay/tests/mcp_suite/analytics_test.rs index 291061b4a11..db9c4513164 100644 --- a/crates/tracedecay/tests/mcp_suite/analytics_test.rs +++ b/crates/tracedecay/tests/mcp_suite/analytics_test.rs @@ -17,6 +17,41 @@ use tracedecay_global_db::{AnalyticsEventInsert, RegisteredGlobalDb}; #[cfg(feature = "test-transport")] #[cfg(feature = "test-transport")] use tracedecay_runtime_core::tracedecay::current_timestamp; +#[cfg(feature = "test-transport")] +use tracing_subscriber::{ + Layer, + filter::{LevelFilter, filter_fn}, + fmt::format::FmtSpan, + layer::SubscriberExt, + util::SubscriberInitExt, +}; + +#[cfg(feature = "test-transport")] +fn install_publication_phase_logging() { + static INIT: std::sync::Once = std::sync::Once::new(); + INIT.call_once(|| { + let filter = filter_fn(|metadata| { + *metadata.level() <= tracing::Level::WARN + || matches!( + metadata.target(), + "tracedecay_code_index::production" + | "tracedecay_code_index_runtime::code_index_scheduler::publication_store" + ) + }) + .with_max_level_hint(LevelFilter::TRACE); + let _ = tracing_subscriber::registry() + .with( + tracing_subscriber::fmt::layer() + .with_ansi(false) + .with_writer(std::io::stderr) + .with_thread_ids(true) + .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) + .compact() + .with_filter(filter), + ) + .try_init(); + }); +} #[cfg(feature = "test-transport")] fn assert_invalid_request(response: &Value, message: &str) { @@ -99,6 +134,7 @@ fn ledger_row(run_id: &str, task: &str, status: &str, started_at: i64) -> String #[cfg(feature = "test-transport")] #[tokio::test] async fn analytics_automation_reports_a_ledger_that_exactly_fills_the_page_as_complete() { + install_publication_phase_logging(); // The newest-rows page is bounded. A ledger holding exactly that many rows // is the whole ledger, not a truncated window: nothing was left behind. let fixture = production_composition_fixture().await; @@ -860,6 +896,7 @@ async fn analytics_reconciles_public_catalog_with_alias_internal_and_unknown_or_ #[cfg(feature = "test-transport")] #[tokio::test] async fn analytics_counts_a_rewrite_call_as_an_available_tool() { + install_publication_phase_logging(); let fixture = production_composition_fixture().await; let project_id = RegisteredGlobalDb::canonical_project_key(&fixture.project_root); let timestamp = current_timestamp() - 60; @@ -904,6 +941,7 @@ async fn analytics_counts_a_rewrite_call_as_an_available_tool() { #[cfg(feature = "test-transport")] #[tokio::test] async fn analytics_aggregates_sections_before_any_event_sample_cap() { + install_publication_phase_logging(); let fixture = production_composition_fixture().await; let project_id = RegisteredGlobalDb::canonical_project_key(&fixture.project_root); let timestamp = current_timestamp() - 60; @@ -1119,6 +1157,7 @@ async fn analytics_scope_all_counts_foreign_project_events_and_keeps_project_fac #[cfg(feature = "test-transport")] #[tokio::test] async fn analytics_applies_the_requested_window_and_refuses_one_out_of_range() { + install_publication_phase_logging(); let fixture = production_composition_fixture().await; let project_id = RegisteredGlobalDb::canonical_project_key(&fixture.project_root); let now = current_timestamp(); From 52de465afc97c0cf8fde3691eb6d53cbbd8d41a8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:19:49 -0800 Subject: [PATCH 72/79] test(mcp): remove temporary publication tracing --- .../tests/mcp_suite/analytics_test.rs | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/crates/tracedecay/tests/mcp_suite/analytics_test.rs b/crates/tracedecay/tests/mcp_suite/analytics_test.rs index db9c4513164..291061b4a11 100644 --- a/crates/tracedecay/tests/mcp_suite/analytics_test.rs +++ b/crates/tracedecay/tests/mcp_suite/analytics_test.rs @@ -17,41 +17,6 @@ use tracedecay_global_db::{AnalyticsEventInsert, RegisteredGlobalDb}; #[cfg(feature = "test-transport")] #[cfg(feature = "test-transport")] use tracedecay_runtime_core::tracedecay::current_timestamp; -#[cfg(feature = "test-transport")] -use tracing_subscriber::{ - Layer, - filter::{LevelFilter, filter_fn}, - fmt::format::FmtSpan, - layer::SubscriberExt, - util::SubscriberInitExt, -}; - -#[cfg(feature = "test-transport")] -fn install_publication_phase_logging() { - static INIT: std::sync::Once = std::sync::Once::new(); - INIT.call_once(|| { - let filter = filter_fn(|metadata| { - *metadata.level() <= tracing::Level::WARN - || matches!( - metadata.target(), - "tracedecay_code_index::production" - | "tracedecay_code_index_runtime::code_index_scheduler::publication_store" - ) - }) - .with_max_level_hint(LevelFilter::TRACE); - let _ = tracing_subscriber::registry() - .with( - tracing_subscriber::fmt::layer() - .with_ansi(false) - .with_writer(std::io::stderr) - .with_thread_ids(true) - .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) - .compact() - .with_filter(filter), - ) - .try_init(); - }); -} #[cfg(feature = "test-transport")] fn assert_invalid_request(response: &Value, message: &str) { @@ -134,7 +99,6 @@ fn ledger_row(run_id: &str, task: &str, status: &str, started_at: i64) -> String #[cfg(feature = "test-transport")] #[tokio::test] async fn analytics_automation_reports_a_ledger_that_exactly_fills_the_page_as_complete() { - install_publication_phase_logging(); // The newest-rows page is bounded. A ledger holding exactly that many rows // is the whole ledger, not a truncated window: nothing was left behind. let fixture = production_composition_fixture().await; @@ -896,7 +860,6 @@ async fn analytics_reconciles_public_catalog_with_alias_internal_and_unknown_or_ #[cfg(feature = "test-transport")] #[tokio::test] async fn analytics_counts_a_rewrite_call_as_an_available_tool() { - install_publication_phase_logging(); let fixture = production_composition_fixture().await; let project_id = RegisteredGlobalDb::canonical_project_key(&fixture.project_root); let timestamp = current_timestamp() - 60; @@ -941,7 +904,6 @@ async fn analytics_counts_a_rewrite_call_as_an_available_tool() { #[cfg(feature = "test-transport")] #[tokio::test] async fn analytics_aggregates_sections_before_any_event_sample_cap() { - install_publication_phase_logging(); let fixture = production_composition_fixture().await; let project_id = RegisteredGlobalDb::canonical_project_key(&fixture.project_root); let timestamp = current_timestamp() - 60; @@ -1157,7 +1119,6 @@ async fn analytics_scope_all_counts_foreign_project_events_and_keeps_project_fac #[cfg(feature = "test-transport")] #[tokio::test] async fn analytics_applies_the_requested_window_and_refuses_one_out_of_range() { - install_publication_phase_logging(); let fixture = production_composition_fixture().await; let project_id = RegisteredGlobalDb::canonical_project_key(&fixture.project_root); let now = current_timestamp(); From 79ccd2ef8ca4b5b7b84a0ad2eaa95378aa11c234 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 02:59:57 -0800 Subject: [PATCH 73/79] test(dashboard): measure slow-curator ledger read phases --- .../tests/dashboard_api_test/automation.rs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/crates/tracedecay/tests/dashboard_api_test/automation.rs b/crates/tracedecay/tests/dashboard_api_test/automation.rs index 7759e4537fb..3592af1a0e1 100644 --- a/crates/tracedecay/tests/dashboard_api_test/automation.rs +++ b/crates/tracedecay/tests/dashboard_api_test/automation.rs @@ -1,5 +1,41 @@ +use std::sync::Once; + use crate::dashboard_api_support::*; use tracedecay_automation_runtime::automation::backend::AgentTaskRetryAttempt; +use tracing::level_filters::LevelFilter; +use tracing_subscriber::Layer as _; +use tracing_subscriber::filter::Targets; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::layer::SubscriberExt as _; +use tracing_subscriber::util::SubscriberInitExt as _; + +fn measure_automation_run_reads() { + static START: Once = Once::new(); + START.call_once(|| { + // The HTTP server and ledger reads run on separate runtime threads. + // These phase spans identify threads, not individual fixture requests. + let layer = tracing_subscriber::fmt::layer() + .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) + .with_thread_ids(true) + .with_ansi(false) + .with_writer(std::io::stderr) + .with_filter( + Targets::new() + .with_default(LevelFilter::WARN) + .with_target( + "tracedecay_dashboard_api::automation_run_api", + LevelFilter::TRACE, + ) + .with_target( + "tracedecay_automation_runtime::automation::run_ledger", + LevelFilter::TRACE, + ), + ); + if let Err(error) = tracing_subscriber::registry().with(layer).try_init() { + eprintln!("automation run measurement subscriber was not installed: {error}"); + } + }); +} #[test] fn fact_store_curate_is_the_only_public_manual_automation_launcher() { @@ -415,6 +451,7 @@ fn final_self_improvement_smoke_covers_autonomous_curation_and_skill_deployment( /// admission with the run's receipt, and the run settles on the daemon. #[test] fn fact_store_curate_answers_with_its_receipt_while_a_slow_curator_runs() { + measure_automation_run_reads(); let runtime = create_runtime(); runtime.block_on(async { let tmp = tempdir_or_panic(); From f4f60cb172f60fc153c8a013b017731c4b2852c9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:26:39 -0800 Subject: [PATCH 74/79] test(dashboard): capture fact-add request phases --- .../tests/dashboard_api_test/automation.rs | 27 ++++++++++++++----- .../dashboard_api_test/memory_curation.rs | 13 +++++++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/crates/tracedecay/tests/dashboard_api_test/automation.rs b/crates/tracedecay/tests/dashboard_api_test/automation.rs index 3592af1a0e1..ebfd83cef2a 100644 --- a/crates/tracedecay/tests/dashboard_api_test/automation.rs +++ b/crates/tracedecay/tests/dashboard_api_test/automation.rs @@ -9,11 +9,12 @@ use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::layer::SubscriberExt as _; use tracing_subscriber::util::SubscriberInitExt as _; -fn measure_automation_run_reads() { +pub(crate) fn measure_dashboard_request_phases() { static START: Once = Once::new(); START.call_once(|| { - // The HTTP server and ledger reads run on separate runtime threads. - // These phase spans identify threads, not individual fixture requests. + // One process-wide subscriber observes both request families. Server + // and native-worker spans can include concurrent fixtures; thread ids + // do not establish a causal link to a caller's HTTP request. let layer = tracing_subscriber::fmt::layer() .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) .with_thread_ids(true) @@ -29,10 +30,24 @@ fn measure_automation_run_reads() { .with_target( "tracedecay_automation_runtime::automation::run_ledger", LevelFilter::TRACE, - ), + ) + .with_target("dashboard_api_test::memory_curation", LevelFilter::TRACE) + .with_target( + "tracedecay_daemon_service::application_surface", + LevelFilter::TRACE, + ) + .with_target( + "tracedecay_store_runtime::retained_memory", + LevelFilter::TRACE, + ) + .with_target( + "tracedecay_session_memory::memory::project_memory::add", + LevelFilter::TRACE, + ) + .with_target("tracedecay_session_memory::fact_store", LevelFilter::TRACE), ); if let Err(error) = tracing_subscriber::registry().with(layer).try_init() { - eprintln!("automation run measurement subscriber was not installed: {error}"); + eprintln!("dashboard request measurement subscriber was not installed: {error}"); } }); } @@ -451,7 +466,7 @@ fn final_self_improvement_smoke_covers_autonomous_curation_and_skill_deployment( /// admission with the run's receipt, and the run settles on the daemon. #[test] fn fact_store_curate_answers_with_its_receipt_while_a_slow_curator_runs() { - measure_automation_run_reads(); + measure_dashboard_request_phases(); let runtime = create_runtime(); runtime.block_on(async { let tmp = tempdir_or_panic(); diff --git a/crates/tracedecay/tests/dashboard_api_test/memory_curation.rs b/crates/tracedecay/tests/dashboard_api_test/memory_curation.rs index fa9b596d1de..fe42abfea2f 100644 --- a/crates/tracedecay/tests/dashboard_api_test/memory_curation.rs +++ b/crates/tracedecay/tests/dashboard_api_test/memory_curation.rs @@ -212,6 +212,7 @@ fn retained_admin_journey_commits_add_update_feedback_and_remove() { #[test] fn fact_detail_keeps_a_superseded_fact_readable_by_id() { + crate::automation::measure_dashboard_request_phases(); let runtime = create_runtime(); runtime.block_on(async { let fixture = start_dashboard_configuration_fixture().await; @@ -228,8 +229,16 @@ fn fact_detail_keeps_a_superseded_fact_readable_by_id() { .unwrap_or_else(|| panic!("add must return a fact id: {added}")) .to_owned() }; - let retired = add("Dashboard detail reads the retired wording"); - let successor = add("Dashboard detail reads the corrected wording"); + let retired = { + let _span = + tracing::trace_span!("dashboard.fixture.fact_add", phase = "retired").entered(); + add("Dashboard detail reads the retired wording") + }; + let successor = { + let _span = + tracing::trace_span!("dashboard.fixture.fact_add", phase = "successor").entered(); + add("Dashboard detail reads the corrected wording") + }; let (status, superseded) = post_json_body( &agent, &retained_url(&fixture, "fact_store_supersede"), From 34e92220289929dd8f9b83080356272d13f8ce37 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:26:55 -0800 Subject: [PATCH 75/79] test(global-db): avoid unused schema fixture actor startup --- .../src/tests/lcm_schema/mod.rs | 7 +++- .../src/tests/lcm_schema/temporal_catalog.rs | 18 ++++------ .../lcm_schema/temporal_catalog/admission.rs | 33 +++++++------------ .../tests/lcm_schema/temporal_constraints.rs | 15 +++------ .../src/tests/lcm_schema/temporal_cursor.rs | 15 +++------ 5 files changed, 33 insertions(+), 55 deletions(-) diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs index 22a7d3611c0..fbfe8b5f1f8 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs @@ -15,7 +15,7 @@ use tracedecay_runtime_core::db::{ use crate::tests::harness::open_registered_test_database_fixture; -async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result { +async fn initialize_global_db(db_path: &Path) -> tracedecay_domain::errors::Result<()> { if let Some(parent) = db_path.parent() { std::fs::create_dir_all(parent)?; } @@ -23,6 +23,11 @@ async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result tracedecay_domain::errors::Result { + initialize_global_db(db_path).await?; Ok(TestConnection::open(db_path)) } diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs index 4dcc84fa725..1d2cb2f65d2 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs @@ -63,10 +63,9 @@ async fn fresh_temporal_schema_stores_and_searches_a_summary_node() { async fn temporal_payload_manifest_schema_is_payload_global() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -175,10 +174,9 @@ mod admission; async fn temporal_schema_query_indexes_cover_exact_lookup_shapes() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -305,10 +303,9 @@ async fn temporal_schema_root_retrieval_indexes_cover_catalog_and_large_query_sh let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -608,10 +605,9 @@ async fn temporal_schema_root_retrieval_indexes_cover_catalog_and_large_query_sh async fn temporal_schema_rejects_unexpected_indexes_without_dropping_them() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -719,10 +715,9 @@ async fn temporal_schema_refuses_markerless_malformed_fts_without_mutation() { async fn temporal_schema_rejects_missing_fts_without_rebuilding_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -791,10 +786,9 @@ async fn temporal_schema_rejects_missing_fts_without_rebuilding_it() { async fn temporal_schema_rejects_missing_graph_publication_authority_without_rebuilding_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs index a5ba897a0e1..c1ec89d7d12 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs @@ -285,10 +285,9 @@ async fn temporal_schema_replaces_an_empty_earlier_authority_with_the_final_cont async fn temporal_schema_rejects_transition_storage_without_mutating_it() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh store should receive the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -332,10 +331,9 @@ async fn temporal_schema_rejects_transition_storage_without_mutating_it() { async fn temporal_schema_rejects_foreign_summary_namespace_tables_without_mutation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -383,10 +381,9 @@ async fn temporal_schema_rejects_foreign_summary_namespace_tables_without_mutati async fn final_store_missing_temporal_marker_requires_reset_without_mutation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh store should receive every final authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -556,10 +553,9 @@ async fn writer_ledger_does_not_hide_a_nonempty_unmarked_store() { async fn final_schema_admission_rejects_extra_temporal_column_metadata() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -598,10 +594,9 @@ async fn final_schema_admission_rejects_extra_temporal_column_metadata() { async fn final_schema_admission_rejects_missing_temporal_occurrence_time_index() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal indexes"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -635,10 +630,9 @@ async fn final_schema_admission_rejects_missing_temporal_occurrence_time_index() async fn final_schema_admission_rejects_missing_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -670,10 +664,9 @@ async fn final_schema_admission_rejects_missing_graph_publication_table() { async fn final_schema_admission_rejects_incompatible_graph_publication_column_metadata() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -712,10 +705,9 @@ async fn final_schema_admission_rejects_incompatible_graph_publication_column_me async fn final_schema_admission_rejects_extra_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -753,10 +745,9 @@ async fn final_schema_admission_rejects_extra_graph_publication_table() { async fn final_schema_admission_rejects_mixed_case_extra_temporal_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install the final temporal schema"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -794,10 +785,9 @@ async fn final_schema_admission_rejects_mixed_case_extra_temporal_table() { async fn final_schema_admission_rejects_mixed_case_extra_graph_publication_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -835,10 +825,9 @@ async fn final_schema_admission_rejects_mixed_case_extra_graph_publication_table async fn final_schema_admission_rejects_extra_index_on_canonical_graph_table() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("fresh initialization should install graph publication authority"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs index b9c1c52d30f..17c54837976 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs @@ -71,10 +71,9 @@ async fn temporal_schema_rejects_cross_session_and_generation_rows() { async fn temporal_schema_rejects_invalid_current_assertion_and_valid_time_rows() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -258,10 +257,9 @@ async fn temporal_schema_rejects_invalid_current_assertion_and_valid_time_rows() async fn temporal_schema_uses_explicit_projection_receipt_progress_counts() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -365,10 +363,9 @@ async fn temporal_schema_uses_explicit_projection_receipt_progress_counts() { async fn temporal_schema_enforces_refresh_progress_and_terminal_receipts() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -1160,10 +1157,9 @@ async fn temporal_schema_enforces_refresh_progress_and_terminal_receipts() { async fn temporal_schema_enforces_generation_state_machine_and_durability() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -1271,10 +1267,9 @@ async fn temporal_schema_enforces_generation_state_machine_and_durability() { async fn temporal_schema_keeps_append_only_authority_immutable() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs index a1f6398e1b2..ad69571ce89 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs @@ -87,10 +87,9 @@ async fn temporal_schema_persists_cursor_keys_without_read_creation() { async fn temporal_schema_rejects_direct_cursor_retirement() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -121,10 +120,9 @@ async fn temporal_schema_rejects_direct_cursor_retirement() { async fn temporal_schema_rotates_cursor_keys_atomically_and_survives_restart() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -250,10 +248,9 @@ async fn temporal_schema_rotates_cursor_keys_atomically_and_survives_restart() { async fn temporal_schema_cursor_audit_rejects_nonmax_active_key() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -304,10 +301,9 @@ async fn temporal_schema_cursor_audit_rejects_skipped_successor_chain() { .join(fixture) .join(".tracedecay") .join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); @@ -362,10 +358,9 @@ async fn temporal_schema_cursor_audit_accepts_valid_successor_chains() { .join(fixture) .join(".tracedecay") .join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); From e7de66c4537d27283c2f08154fd7aeae333a790b Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:26:56 -0800 Subject: [PATCH 76/79] chore(dashboard): remove temporary request phase capture --- .../tests/dashboard_api_test/automation.rs | 52 ------------------- .../dashboard_api_test/memory_curation.rs | 13 +---- 2 files changed, 2 insertions(+), 63 deletions(-) diff --git a/crates/tracedecay/tests/dashboard_api_test/automation.rs b/crates/tracedecay/tests/dashboard_api_test/automation.rs index ebfd83cef2a..7759e4537fb 100644 --- a/crates/tracedecay/tests/dashboard_api_test/automation.rs +++ b/crates/tracedecay/tests/dashboard_api_test/automation.rs @@ -1,56 +1,5 @@ -use std::sync::Once; - use crate::dashboard_api_support::*; use tracedecay_automation_runtime::automation::backend::AgentTaskRetryAttempt; -use tracing::level_filters::LevelFilter; -use tracing_subscriber::Layer as _; -use tracing_subscriber::filter::Targets; -use tracing_subscriber::fmt::format::FmtSpan; -use tracing_subscriber::layer::SubscriberExt as _; -use tracing_subscriber::util::SubscriberInitExt as _; - -pub(crate) fn measure_dashboard_request_phases() { - static START: Once = Once::new(); - START.call_once(|| { - // One process-wide subscriber observes both request families. Server - // and native-worker spans can include concurrent fixtures; thread ids - // do not establish a causal link to a caller's HTTP request. - let layer = tracing_subscriber::fmt::layer() - .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) - .with_thread_ids(true) - .with_ansi(false) - .with_writer(std::io::stderr) - .with_filter( - Targets::new() - .with_default(LevelFilter::WARN) - .with_target( - "tracedecay_dashboard_api::automation_run_api", - LevelFilter::TRACE, - ) - .with_target( - "tracedecay_automation_runtime::automation::run_ledger", - LevelFilter::TRACE, - ) - .with_target("dashboard_api_test::memory_curation", LevelFilter::TRACE) - .with_target( - "tracedecay_daemon_service::application_surface", - LevelFilter::TRACE, - ) - .with_target( - "tracedecay_store_runtime::retained_memory", - LevelFilter::TRACE, - ) - .with_target( - "tracedecay_session_memory::memory::project_memory::add", - LevelFilter::TRACE, - ) - .with_target("tracedecay_session_memory::fact_store", LevelFilter::TRACE), - ); - if let Err(error) = tracing_subscriber::registry().with(layer).try_init() { - eprintln!("dashboard request measurement subscriber was not installed: {error}"); - } - }); -} #[test] fn fact_store_curate_is_the_only_public_manual_automation_launcher() { @@ -466,7 +415,6 @@ fn final_self_improvement_smoke_covers_autonomous_curation_and_skill_deployment( /// admission with the run's receipt, and the run settles on the daemon. #[test] fn fact_store_curate_answers_with_its_receipt_while_a_slow_curator_runs() { - measure_dashboard_request_phases(); let runtime = create_runtime(); runtime.block_on(async { let tmp = tempdir_or_panic(); diff --git a/crates/tracedecay/tests/dashboard_api_test/memory_curation.rs b/crates/tracedecay/tests/dashboard_api_test/memory_curation.rs index fe42abfea2f..fa9b596d1de 100644 --- a/crates/tracedecay/tests/dashboard_api_test/memory_curation.rs +++ b/crates/tracedecay/tests/dashboard_api_test/memory_curation.rs @@ -212,7 +212,6 @@ fn retained_admin_journey_commits_add_update_feedback_and_remove() { #[test] fn fact_detail_keeps_a_superseded_fact_readable_by_id() { - crate::automation::measure_dashboard_request_phases(); let runtime = create_runtime(); runtime.block_on(async { let fixture = start_dashboard_configuration_fixture().await; @@ -229,16 +228,8 @@ fn fact_detail_keeps_a_superseded_fact_readable_by_id() { .unwrap_or_else(|| panic!("add must return a fact id: {added}")) .to_owned() }; - let retired = { - let _span = - tracing::trace_span!("dashboard.fixture.fact_add", phase = "retired").entered(); - add("Dashboard detail reads the retired wording") - }; - let successor = { - let _span = - tracing::trace_span!("dashboard.fixture.fact_add", phase = "successor").entered(); - add("Dashboard detail reads the corrected wording") - }; + let retired = add("Dashboard detail reads the retired wording"); + let successor = add("Dashboard detail reads the corrected wording"); let (status, superseded) = post_json_body( &agent, &retained_url(&fixture, "fact_store_supersede"), From f538ecf6d7cd5e2fd1c3e08d6888f25ddbd32ee4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:10:30 -0800 Subject: [PATCH 77/79] fix(mcp): bind call-path reads to live source boundaries --- .../src/handlers/graph/context_path.rs | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/crates/tracedecay-mcp/src/handlers/graph/context_path.rs b/crates/tracedecay-mcp/src/handlers/graph/context_path.rs index 607c43aabb5..4356a59d2ab 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/context_path.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/context_path.rs @@ -5,11 +5,11 @@ mod tests; use std::collections::{BTreeMap, HashSet}; use std::future::Future; -use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use tracedecay_code_index::graph_projection::CodeGraphSymbolSummaryV1; +use tracedecay_code_index::source_walk::submodules::LiveSourceBoundaries; use tracedecay_code_index_runtime::code_index_scheduler::sealed_source_bytes; use tracedecay_contracts::code_index_freshness::CodeIndexFreshnessPayloadV1; use tracedecay_contracts::retrieval::{ @@ -157,7 +157,14 @@ pub(super) async fn context_call_path( let code = if let Some(max_code_blocks) = code_block_limit && let ContextCallPathOutcomeV1::Found { nodes, .. } = &mut result.outcome { - match path_source(graph, symbols, max_code_blocks).await { + match path_source( + graph, + symbols, + max_code_blocks, + ctx.index_path_policy().includes_submodules(), + ) + .await + { Ok((sources, code)) => { for (node, source) in nodes.iter_mut().zip(sources) { node.source = source; @@ -355,6 +362,7 @@ async fn path_source( graph: &VerifiedGraphQuery, symbols: Vec, max_code_blocks: usize, + include_submodules: bool, ) -> Result<(Vec, Vec)> { let project_root = graph.project_root()?.to_path_buf(); let mut files = BTreeMap::new(); @@ -372,8 +380,17 @@ async fn path_source( let worker_control = Arc::clone(&control); await_path_source(control, async move { tracedecay_runtime_core::native_runtime::spawn_blocking(move || { + worker_control.check()?; + let boundaries = LiveSourceBoundaries::open(&project_root, include_submodules) + .map_err(|error| { + TraceDecayError::project_route( + "context-call-path-source-unavailable", + true, + error.message, + ) + })?; read_path_source( - &project_root, + &boundaries, &symbols, &files, max_code_blocks, @@ -393,7 +410,7 @@ async fn path_source( } fn read_path_source( - project_root: &Path, + boundaries: &LiveSourceBoundaries, symbols: &[CodeGraphSymbolSummaryV1], files: &BTreeMap>, max_code_blocks: usize, @@ -423,7 +440,7 @@ fn read_path_source( continue; } let source = source_by_file.entry(path.to_owned()).or_insert_with(|| { - sealed_path_source(project_root, path, files.get(path).and_then(Option::as_ref)) + sealed_path_source(boundaries, path, files.get(path).and_then(Option::as_ref)) .map_err(|error| unavailable_evidence(&error)) }); let block = match source { @@ -444,7 +461,7 @@ fn read_path_source( } fn sealed_path_source( - project_root: &Path, + boundaries: &LiveSourceBoundaries, path: &str, file: Option<&SanitizedCodeFileV1>, ) -> Result { @@ -458,7 +475,7 @@ fn sealed_path_source( let language = file.language.as_ref().ok_or_else(|| { graph_symbol_corrupt(format!("call-path file '{path}' has no indexed language")) })?; - let source = sealed_source_bytes(project_root, path, language, &file.content_digest) + let source = sealed_source_bytes(boundaries, path, language, &file.content_digest) .map_err(|error| { TraceDecayError::project_route( "context-call-path-source-unavailable", From b77eb94340f8319a40de0d76bceda1a54597002c Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:36:17 -0800 Subject: [PATCH 78/79] test(code-index): pin the observed combined manifest digest --- .../tests/code_index_suite/production_orchestration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs index 5a8b4a6ec82..05cef899c39 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/production_orchestration.rs @@ -3177,7 +3177,7 @@ fn partitioned_codec_fixture() -> ( // The sealed generation revision is part of the authenticated manifest payload. const PARTITIONED_FORMAT_STATE_DIGEST: &str = - "sha256:b3566559dfef68d8cb147e461ec928920367bae8c314d703375da29fd16f2037"; + "sha256:b8669b6269e0a571d9f6fa58fb9f9f7e254fe47c03c1dc231a7d71623abcfac5"; const PARTITIONED_FORMAT_SEGMENTS: &[(&str, u64)] = &[ ( "sha256:49ac89476728e3983d94d2fa74f2844144dd800e6df4a82fe24484c5a7369518", From f78313be78e80b700397805d941f06c8ee2b7e5f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:13:00 -0800 Subject: [PATCH 79/79] test(sessions): report retained snapshot scan evidence --- .../src/runtime/hosts/opencode_tests.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/tracedecay-sessions/src/runtime/hosts/opencode_tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/opencode_tests.rs index bc22ece14f7..4542ee07bba 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/opencode_tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/opencode_tests.rs @@ -1227,14 +1227,21 @@ async fn retained_read_snapshot_keeps_reference_scope_and_payload_together() { COMMIT;", ) .unwrap(); - let (page, _) = materialize_reference_page( + let (page, budget) = materialize_reference_page( reader.reader.connection(), &source, references.references, budget, ) .unwrap(); - assert_eq!(page.records.len(), 1); + assert_eq!( + page.records.len(), + 1, + "snapshot hydration: fully_processed={}, input_bytes={}, evidence={:?}", + page.fully_processed, + page.input_bytes, + budget.evidence() + ); let payload = String::from_utf8(page.records[0].payload.clone()).unwrap(); let transcript = serde_json::to_string(&source.source_path).unwrap(); let expected =