From eeb6205ae946ca46aebc26d3c80c3c8e2fd20c02 Mon Sep 17 00:00:00 2001 From: kogekiplay <46434871+kogekiplay@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:22:31 +0800 Subject: [PATCH 1/3] refactor(search): stage native search and foreground scheduling proposal --- docs/workspace-search-design.md | 109 ++ src-tauri/src/commands/folders.rs | 54 +- src-tauri/src/commands/mod.rs | 1 + .../src/commands/workspace_file_search.rs | 1168 +++++++++++++++++ src-tauri/src/lib.rs | 3 + src-tauri/src/web/handlers/folders.rs | 46 + src-tauri/src/web/router.rs | 12 + src/components/canvas/add-node-menu.tsx | 10 +- .../suggestion/suggestion-popup.test.tsx | 143 +- .../composer/suggestion/suggestion-popup.tsx | 95 +- .../chat/composer/suggestion/types.ts | 11 +- .../composer/use-reference-search.test.ts | 134 ++ .../chat/composer/use-reference-search.ts | 258 ++-- .../conversations/search-command-dialog.tsx | 7 +- src/hooks/use-file-tree.test.ts | 163 +++ src/hooks/use-file-tree.ts | 77 +- src/lib/api-workspace-search.test.ts | 71 + src/lib/api.ts | 57 +- 18 files changed, 2236 insertions(+), 183 deletions(-) create mode 100644 docs/workspace-search-design.md create mode 100644 src-tauri/src/commands/workspace_file_search.rs create mode 100644 src/hooks/use-file-tree.test.ts create mode 100644 src/lib/api-workspace-search.test.ts diff --git a/docs/workspace-search-design.md b/docs/workspace-search-design.md new file mode 100644 index 00000000000..436035f9fc9 --- /dev/null +++ b/docs/workspace-search-design.md @@ -0,0 +1,109 @@ +# Workspace search: scheduling and cache revision (proposal) + +This is the fourth, **draft** split of #1097. The current compact native index, +query cancellation, server/desktop APIs, and bounded result transfer remain here +for review. This branch is not ready to merge or install. The single-root cache +and traversal under a global mutex are still present; the proposed replacement +below must land and pass the acceptance tests before this PR becomes ready. +The WebKit, Tauri callback, and highlighting splits are independent of this work. + +## Foreground work must not wait for background traversal + +Separate the short-held cache/registry mutex from traversal and scoring. The +registry holds per-canonical-root state and shared immutable snapshots; a worker +never retains its mutex while touching the filesystem. Canonical roots include +the authorized linked-root/ignore configuration in their cache identity. + +Use a dispatcher with at most two discovery workers: one foreground-reserved +slot and one shared slot. Admit at most one background prewarm, and preempt its +cancellation token when foreground discovery needs capacity. Empty queries only +read and return a bounded top-level listing; they do not wait for a full index. +Scoring an already built snapshot does not wait for discovery admission. +Coalesce discovery for the same root. Queries attach to that root's discovery, +while cancellation removes only that consumer; it cancels the traversal when +no consumer or explicit retained refresh remains. A small-root foreground query +can proceed even while a large root is being prepared. + +Preemption can discard an incomplete snapshot. Only successful, complete walks +are published. Worker completion must verify the root generation before updating +the registry, so a late cancelled build cannot replace a newer snapshot. Keep +cancellation/deadline checks in discovery and scoring, and cancel queued jobs +before acquiring a worker permit. + +## Prewarm belongs to a visible consumer + +Give prepare requests the same consumer ID/generation contract as queries, plus +a dedicated prepare cancellation endpoint for desktop and server modes. The +file picker and mention panel own an AbortController for their prepare lifecycle; +close, disable, unmount, root change, or a new generation detaches it. A consumer +query can promote an existing prewarm to foreground rather than repeat its walk. +Use bounded registry tombstones to reject a cancelled request that arrives late. +Prewarm is an optimization and must never outlive all of its consumers. + +## Reuse roots within one global budget + +Replace the single cache slot with an LRU of snapshots keyed by canonical root. +Retain at most 128 MiB across **all** root snapshots and at most eight roots; do +not assign 128 MiB to each root. Include deduplicated strings, node arrays, and +metadata in accounting. Queries hold an Arc lease; eviction removes its cache +ownership, while active leases continue counting against the process-wide +budget until released. Budget reservations therefore precede construction, not +only publication. Cap simultaneous discovery allocations globally at 192 MiB +and bound foreground scoring scratch space by result limit. + +A -> B -> A within the TTL should reuse both complete snapshots if they fit the +total budget. Refresh timestamps change only after a real successful discovery, +not after fallback queries. Root/config invalidation creates a new generation; +stale refresh results cannot overwrite it. Disk and memory expiry are explicit. + +## Oversized roots reuse a disk snapshot + +While walking, incrementally write a temporary SQLite snapshot (rusqlite is +already a dependency). Keep a compact memory index only while its reservation +permits. When the memory builder reaches its share of the global build budget, +drop that builder and continue the same discovery into the disk snapshot. This +produces complete results without rescanning the filesystem on each keystroke. +Publish the disk snapshot atomically only on successful completion. Bind its +temporary-file lifetime to the snapshot lease and remove abandoned files on +startup. Use a bounded global SQLite page cache and a disk quota/LRU as well; +never convert a disk result into a full renderer-side file list. + +Repeated oversized-root queries scan the cached rows in bounded pages and keep +only top-k scores. They remain cancellable between pages. This still has O(N) +scoring cost, which should be measured; it removes repeated filesystem discovery +and bounds retained RAM. If disk quota/storage prevents a complete snapshot, +return a retriable index-unavailable error with backoff, rather than marking a +partial tree complete or silently restarting a full walk for every keypress. +Incremental updates and a trigram index can be evaluated later if measurements +justify them. + +## UI regressions fixed in this draft + +Keyboard and pointer navigation pin the active suggestion group. Selection +tracks reference identity within that group across progressive updates, with a +valid fallback when an item is removed. Delayed file arrival cannot reinterpret +a selected session as a file or invalidate Enter. Window focus invalidates the +lazy session/commit caches for the next query while keeping the valid current +request alive. The ES2020 test helper no longer uses Array.at. + +## Required acceptance before marking ready + +- Block a large root's prewarm inside discovery; a small-root foreground and an + empty listing must complete before that block is released. +- Close the only consumer during queued and active prewarm. Assert cancellation, + no partial snapshot publication, no stale event delivery, and released permits. + With two consumers, closing one must preserve the other's work. +- Query A, B, A; count one discovery per root, including root aliases/config + changes and LRU eviction. Assert the *total* retained and active-lease budgets. +- Force the memory budget small; issue several oversized-root queries and assert + one filesystem walk, complete late-file matches, bounded SQLite/cache memory, + cancellation during scoring, and temporary-file cleanup on failure/eviction. +- Race cancellation, root change, foreground promotion, and TTL refresh; verify + registry generations and no starvation. Instrument queue time separately from + discovery/scoring time for a realistic large workspace. +- Keep the keyboard progressive-result and window-focus regressions green. + Run full TypeScript including tests, ESLint, Vitest, production build, and + desktop/server Rust checks/tests/clippy against the final implementation. + +The remaining backend design/implementation is a merge blocker **only for this +draft**. The other three split PRs can be reviewed and released independently. diff --git a/src-tauri/src/commands/folders.rs b/src-tauri/src/commands/folders.rs index dd08d2a2a6e..af4f101dc5d 100644 --- a/src-tauri/src/commands/folders.rs +++ b/src-tauri/src/commands/folders.rs @@ -5174,16 +5174,24 @@ fn build_file_tree( } /// Flat, gitignore-aware listing of every file and directory under `path`, for -/// fuzzy file search. Unlike [`get_file_tree`], ignored directories -/// (`node_modules`, `target`, `dist`, …) are pruned *during* the walk, so no -/// depth cap is needed: deep files stay reachable while the heavy trees are -/// never descended and the payload stays small. Gitignore handling that used to -/// run client-side now happens here at native speed in a single pass. +/// compatibility with clients that need enumeration. Search surfaces use +/// `workspace_file_search` so an unbounded listing never reaches the renderer. #[cfg_attr(feature = "tauri-runtime", tauri::command)] pub async fn list_workspace_files( path: String, ) -> Result, AppCommandError> { - let root = PathBuf::from(&path); + let mut entries = Vec::new(); + visit_workspace_files(&path, |entry| { + entries.push(entry); + true + }); + Ok(entries) +} + +/// Visit the search-visible entries without collecting the whole workspace. +/// Returning false stops both the main walk and any authorized linked roots. +pub(super) fn visit_workspace_files(path: &str, mut visit: impl FnMut(WorkspaceFileEntry) -> bool) { + let root = PathBuf::from(path); // Linked directories are walked separately below, rooted at the link so // their own `.gitignore` applies. Excluded from the main pass because the // walker reports a symlink as a leaf file. @@ -5220,7 +5228,6 @@ pub async fn list_workspace_files( }) .build(); - let mut entries: Vec = Vec::new(); for result in walker { // Skip unreadable entries (permission errors, transient races) rather // than failing the whole search. @@ -5244,7 +5251,7 @@ pub async fn list_workspace_files( .to_string_lossy() .replace('\\', "/"); - entries.push(WorkspaceFileEntry { + if !visit(WorkspaceFileEntry { name, path: rel_path, kind: if is_dir { @@ -5252,28 +5259,36 @@ pub async fn list_workspace_files( } else { WorkspaceEntryKind::File }, - }); + }) { + return; + } } for (link_name, target) in linked { - entries.push(WorkspaceFileEntry { + if !visit(WorkspaceFileEntry { name: link_name.clone(), path: link_name.clone(), kind: WorkspaceEntryKind::Dir, - }); + }) { + return; + } // Rooted at the resolved target so the linked project's own ignore // files apply, then re-prefixed with the link name so every path stays // relative to the workspace root and resolves back through the link. - entries.extend(list_files_under(&target, &link_name)); + if !visit_files_under(&target, &link_name, &mut visit) { + return; + } } - - Ok(entries) } /// Flat listing of `root`, with every path prefixed by `prefix/`. Shares the /// ignore configuration of [`list_workspace_files`]; nested symlinks are not /// followed, so this cannot recurse. -fn list_files_under(root: &Path, prefix: &str) -> Vec { +fn visit_files_under( + root: &Path, + prefix: &str, + visit: &mut impl FnMut(WorkspaceFileEntry) -> bool, +) -> bool { let walker = WalkBuilder::new(root) .hidden(false) .parents(false) @@ -5295,7 +5310,6 @@ fn list_files_under(root: &Path, prefix: &str) -> Vec { }) .build(); - let mut entries = Vec::new(); for result in walker { let Ok(entry) = result else { continue }; let entry_path = entry.path(); @@ -5308,7 +5322,7 @@ fn list_files_under(root: &Path, prefix: &str) -> Vec { let is_dir = entry .file_type() .is_some_and(|t| entry_presents_as_dir(&t, entry_path)); - entries.push(WorkspaceFileEntry { + if !visit(WorkspaceFileEntry { name: entry.file_name().to_string_lossy().to_string(), path: format!("{prefix}/{}", rel.to_string_lossy().replace('\\', "/")), kind: if is_dir { @@ -5316,9 +5330,11 @@ fn list_files_under(root: &Path, prefix: &str) -> Vec { } else { WorkspaceEntryKind::File }, - }); + }) { + return false; + } } - entries + true } #[cfg_attr(feature = "tauri-runtime", tauri::command)] diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index c3bdf783993..885bdb9e869 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -83,6 +83,7 @@ pub mod version_control; #[cfg(feature = "tauri-runtime")] pub mod windows; pub mod work_task; +pub mod workspace_file_search; pub mod workspace_state; #[cfg(feature = "tauri-runtime")] pub mod workspace_windows; diff --git a/src-tauri/src/commands/workspace_file_search.rs b/src-tauri/src/commands/workspace_file_search.rs new file mode 100644 index 00000000000..c6f548f84cf --- /dev/null +++ b/src-tauri/src/commands/workspace_file_search.rs @@ -0,0 +1,1168 @@ +//! Keep workspace discovery and ranking out of the renderer. A workspace can +//! contain millions of non-ignored files: sending a flat list to each composer +//! multiplies path strings, lowercase copies and JS objects across webviews. + +use std::cmp::{Ordering, Reverse}; +use std::collections::{BinaryHeap, HashMap}; +use std::mem::size_of; +use std::sync::{Arc, LazyLock, Mutex as StdMutex}; +use std::time::{Duration, Instant}; + +use serde::Deserialize; +use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; + +use super::folders::{visit_workspace_files, WorkspaceEntryKind, WorkspaceFileEntry}; +use crate::app_error::AppCommandError; + +const MAX_RESULTS: usize = 200; +const INDEX_BYTES: usize = 128 * 1024 * 1024; +const BUILD_BYTES: usize = 192 * 1024 * 1024; +const CACHE_TTL: Duration = Duration::from_secs(300); +const SEARCH_TIMEOUT: Duration = Duration::from_secs(300); +const MAX_CONSUMERS: usize = 128; +const NO_PARENT: u32 = u32::MAX; + +#[derive(Clone, Copy)] +struct IndexBudget { + build: usize, + retained: usize, +} + +const INDEX_BUDGET: IndexBudget = IndexBudget { + build: BUILD_BYTES, + retained: INDEX_BYTES, +}; + +// One workspace cache and one builder/search worker for the whole process, +// shared by every composer, dialog, desktop window and web client. Switching +// roots drops the previous index before building the next one. +static CACHE: LazyLock>> = LazyLock::new(|| Mutex::new(None)); +static SEARCHES: LazyLock> = + LazyLock::new(|| StdMutex::new(Searches::default())); + +#[derive(Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchRequest { + pub consumer_id: String, + pub generation: u64, +} + +struct Consumer { + generation: u64, + cancel: CancellationToken, + touched: Instant, + active: bool, +} + +#[derive(Default)] +struct Searches(HashMap); + +impl Searches { + fn register( + &mut self, + request: &SearchRequest, + cancel: bool, + ) -> Result, AppCommandError> { + if request.consumer_id.is_empty() || request.consumer_id.len() > 128 { + return Err(AppCommandError::invalid_input("Invalid search consumer")); + } + self.0 + .retain(|_, c| c.active || c.touched.elapsed() < SEARCH_TIMEOUT); + if let Some(previous) = self.0.get(&request.consumer_id) { + if request.generation <= previous.generation { + if cancel && request.generation == previous.generation { + previous.cancel.cancel(); + } + return Ok(None); + } + previous.cancel.cancel(); + } else if self.0.len() >= MAX_CONSUMERS { + // Retain active consumers; bound completed generation tombstones + // too. Tombstones handle cancellation arriving before its request. + let oldest = self + .0 + .iter() + .filter(|(_, c)| !c.active) + .min_by_key(|(_, c)| c.touched) + .map(|(key, _)| key.clone()); + if let Some(oldest) = oldest { + self.0.remove(&oldest); + } else { + return Err(AppCommandError::task_execution_failed( + "Too many workspace file searches", + )); + } + } + let token = CancellationToken::new(); + if cancel { + token.cancel(); + } + self.0.insert( + request.consumer_id.clone(), + Consumer { + generation: request.generation, + cancel: token.clone(), + touched: Instant::now(), + active: !cancel, + }, + ); + Ok(Some(token)) + } +} + +struct SearchControl { + cancel: CancellationToken, + deadline: Instant, +} + +impl Default for SearchControl { + fn default() -> Self { + Self { + cancel: CancellationToken::new(), + deadline: Instant::now() + SEARCH_TIMEOUT, + } + } +} + +impl SearchControl { + fn stopped(&self) -> bool { + self.cancel.is_cancelled() || Instant::now() >= self.deadline + } +} + +struct ActiveSearch { + request: SearchRequest, + control: SearchControl, +} + +impl Drop for ActiveSearch { + fn drop(&mut self) { + self.control.cancel.cancel(); + if let Some(consumer) = SEARCHES + .lock() + .unwrap() + .0 + .get_mut(&self.request.consumer_id) + { + if consumer.generation == self.request.generation { + consumer.active = false; + consumer.touched = Instant::now(); + } + } + } +} + +struct CachedIndex { + root: String, + built_at: Instant, + index: Option, +} + +// Store each basename once and each parent as a small integer. In a workspace +// containing many checkouts/archives this avoids retaining the same long path +// prefix (and basename) millions of times. +#[derive(Clone, Copy)] +struct Node { + parent: u32, + name: u32, + is_dir: bool, +} + +struct IndexedName { + text: Arc, + lower: String, + utf16_len: usize, + byte_mask: u64, +} + +fn byte_mask(text: &str) -> u64 { + text.bytes().fold(0, |mask, byte| mask | (1 << (byte % 64))) +} + +#[derive(Default)] +struct CompactIndex { + nodes: Vec, + names: Vec, + string_bytes: usize, +} + +impl CompactIndex { + fn bytes(&self) -> usize { + self.nodes.capacity() * size_of::() + + self.names.capacity() * size_of::() + + self.string_bytes + } + + fn search( + &self, + query: &str, + limit: usize, + files_only: bool, + control: &SearchControl, + ) -> Vec { + let mut matches = Matches::new(query, limit, files_only); + // Repeated basenames share normalization and name-tier scoring too. + // A full set of name matches outranks every path/subsequence match. + let name_scores: Vec<_> = self + .names + .iter() + .map(|name| score_name(&matches.query, &name.lower, name.utf16_len)) + .collect(); + let query_mask = matches.query.is_ascii().then(|| byte_mask(&matches.query)); + let mut ancestors: Vec<(u32, usize, usize, u64)> = Vec::new(); + let mut path = String::new(); + let mut lower_path = String::new(); + for (id, node) in self.nodes.iter().enumerate() { + if control.stopped() { + return Vec::new(); + } + while ancestors.last().map(|(id, ..)| *id).unwrap_or(NO_PARENT) != node.parent { + ancestors.pop(); + path.truncate(ancestors.last().map(|(_, len, ..)| *len).unwrap_or(0)); + lower_path.truncate(ancestors.last().map(|(_, _, len, _)| *len).unwrap_or(0)); + } + let parent_len = path.len(); + let lower_parent_len = lower_path.len(); + let name = &self.names[node.name as usize]; + let mask = ancestors.last().map(|(_, _, _, mask)| *mask).unwrap_or(0) + | name.byte_mask + | if parent_len > 0 { byte_mask("/") } else { 0 }; + let name_score = name_scores[node.name as usize]; + let may_match = (!files_only || !node.is_dir) + && match name_score { + Some(score) => matches.accepts(score, id), + None => { + matches.needs_path_matches() + && query_mask.is_none_or(|query| mask & query == query) + } + }; + // Directories still maintain the prefix for descendants; rejected + // files need no full path, lowercase allocation or UTF-16 scan. + if !node.is_dir && !may_match { + continue; + } + if !path.is_empty() { + path.push('/'); + lower_path.push('/'); + } + path.push_str(&name.text); + lower_path.push_str(&name.lower); + if may_match { + let score = name_score.or_else(|| { + score_path(&matches.query, &name.lower, name.utf16_len, &lower_path) + }); + if let Some(score) = score { + matches.offer_ranked(&name.text, &path, node.is_dir, id, score); + } + } + if node.is_dir { + ancestors.push((id as u32, path.len(), lower_path.len(), mask)); + } else { + path.truncate(parent_len); + lower_path.truncate(lower_parent_len); + } + } + matches.finish() + } +} + +#[derive(Default)] +struct Builder { + index: CompactIndex, + names: HashMap, u32>, + parents: Vec, +} + +impl Builder { + fn push(&mut self, entry: &WorkspaceFileEntry, budget: usize) -> bool { + // Account for capacity growth before reserving it. The interning table + // exists only during the build; it is dropped before caching the index. + let new_name = !self.names.contains_key(entry.name.as_str()); + let extra_nodes = if self.index.nodes.len() == self.index.nodes.capacity() { + 4096 * size_of::() + } else { + 0 + }; + let extra_names = if new_name && self.index.names.len() == self.index.names.capacity() { + 1024 * size_of::() + } else { + 0 + }; + let table_capacity = if new_name && self.names.len() == self.names.capacity() { + (self.names.capacity().max(4) * 2).max(self.names.len() + 1) + } else { + self.names.capacity() + }; + // HashMap capacity excludes empty buckets/control bytes. Round up and + // include Arc's allocation header/alignment for each interned string. + let table_bytes = table_capacity * (size_of::<(Arc, u32)>() + 1) * 8 / 7 + 32; + let lower = new_name.then(|| entry.name.to_lowercase()); + let string_bytes = lower + .as_ref() + .map(|lower| entry.name.len() + 32 + lower.capacity() + 16) + .unwrap_or(0); + if self.index.bytes() + extra_nodes + extra_names + table_bytes + string_bytes > budget + || self.index.nodes.len() >= u32::MAX as usize + { + return false; + } + if extra_nodes != 0 { + self.index.nodes.reserve_exact(4096); + } + if extra_names != 0 { + self.index.names.reserve_exact(1024); + } + let name = match self.names.get(entry.name.as_str()) { + Some(id) => *id, + None => { + let id = self.index.names.len() as u32; + let name: Arc = Arc::from(entry.name.as_str()); + self.names.insert(Arc::clone(&name), id); + let lower = lower.unwrap(); + self.index.names.push(IndexedName { + utf16_len: lower.encode_utf16().count(), + byte_mask: byte_mask(&lower), + text: name, + lower, + }); + self.index.string_bytes += string_bytes; + id + } + }; + let depth = entry.path.bytes().filter(|c| *c == b'/').count(); + self.parents.truncate(depth); + if self.parents.len() != depth { + return false; + } + let id = self.index.nodes.len() as u32; + self.index.nodes.push(Node { + parent: self.parents.last().copied().unwrap_or(NO_PARENT), + name, + is_dir: entry.kind == WorkspaceEntryKind::Dir, + }); + if entry.kind == WorkspaceEntryKind::Dir { + self.parents.push(id); + } + true + } +} + +struct Ranked { + score: i64, + order: usize, + entry: WorkspaceFileEntry, +} + +impl PartialEq for Ranked { + fn eq(&self, other: &Self) -> bool { + self.score == other.score && self.order == other.order + } +} +impl Eq for Ranked {} +impl PartialOrd for Ranked { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for Ranked { + fn cmp(&self, other: &Self) -> Ordering { + self.score + .cmp(&other.score) + .then_with(|| other.order.cmp(&self.order)) + } +} + +struct Matches { + query: String, + limit: usize, + files_only: bool, + best: BinaryHeap>, +} + +impl Matches { + fn new(query: &str, limit: usize, files_only: bool) -> Self { + Self { + query: query.trim().to_lowercase(), + limit, + files_only, + best: BinaryHeap::new(), + } + } + + fn offer(&mut self, name: &str, path: &str, is_dir: bool, order: usize) { + if self.limit == 0 || (self.files_only && is_dir) { + return; + } + let Some(score) = score_file(&self.query, name, path) else { + return; + }; + self.offer_ranked(name, path, is_dir, order, score); + } + + fn accepts(&self, score: i64, order: usize) -> bool { + self.limit != 0 + && (self.best.len() < self.limit + || self.best.peek().is_some_and(|worst| { + score > worst.0.score || (score == worst.0.score && order < worst.0.order) + })) + } + + fn needs_path_matches(&self) -> bool { + self.limit != 0 + && (self.best.len() < self.limit + || self + .best + .peek() + .is_some_and(|worst| worst.0.score < 300_000_000)) + } + + fn offer_ranked(&mut self, name: &str, path: &str, is_dir: bool, order: usize, score: i64) { + if !self.accepts(score, order) { + return; + } + self.best.push(Reverse(Ranked { + score, + order, + entry: WorkspaceFileEntry { + name: name.to_owned(), + path: path.to_owned(), + kind: if is_dir { + WorkspaceEntryKind::Dir + } else { + WorkspaceEntryKind::File + }, + }, + })); + if self.best.len() > self.limit { + self.best.pop(); + } + } + + fn finish(self) -> Vec { + let mut best = self.best.into_iter().map(|r| r.0).collect::>(); + best.sort_by(|a, b| b.cmp(a)); + best.into_iter().map(|r| r.entry).collect() + } +} + +fn tier_score(tier: i64, position: usize, length: usize) -> i64 { + tier * 100_000_000 - position.min(9999) as i64 * 1000 - length.min(999) as i64 +} + +fn subsequence(query: &str, text: &str) -> Option { + let mut query = query.encode_utf16().peekable(); + let mut first = None; + for (i, ch) in text.encode_utf16().enumerate() { + if query.peek() == Some(&ch) { + first.get_or_insert(i); + query.next(); + if query.peek().is_none() { + return first; + } + } + } + None +} + +// Same tiers and UTF-16 positions/lengths as lib/file-search-match.ts. +fn score_file(query: &str, name: &str, path: &str) -> Option { + let name = name.to_lowercase(); + let name_len = name.encode_utf16().count(); + score_name(query, &name, name_len) + .or_else(|| score_path(query, &name, name_len, &path.to_lowercase())) +} + +fn score_name(query: &str, name: &str, name_len: usize) -> Option { + if query.is_empty() { + return Some(0); + } + if name == query { + return Some(tier_score(6, 0, name_len)); + } + if name.starts_with(query) { + return Some(tier_score(5, 0, name_len)); + } + if let Some(i) = name.find(query) { + return Some(tier_score(4, name[..i].encode_utf16().count(), name_len)); + } + None +} + +fn score_path(query: &str, name: &str, name_len: usize, path: &str) -> Option { + let path_len = path.encode_utf16().count(); + if let Some(i) = path.find(query) { + return Some(tier_score(3, path[..i].encode_utf16().count(), path_len)); + } + if let Some(i) = subsequence(query, name) { + return Some(tier_score(2, i, name_len)); + } + subsequence(query, path).map(|i| tier_score(1, i, path_len)) +} + +#[derive(Clone, Copy)] +struct SearchOptions<'a> { + query: &'a str, + limit: usize, + files_only: bool, + refresh: bool, +} + +fn search_or_build( + cache: &mut Option, + root: String, + options: SearchOptions<'_>, + budget: IndexBudget, + control: &SearchControl, +) -> Vec { + let SearchOptions { + query, + limit, + files_only, + refresh, + } = options; + if control.stopped() { + return Vec::new(); + } + let fresh = cache + .as_ref() + .is_some_and(|c| c.root == root && c.built_at.elapsed() < CACHE_TTL); + if fresh && !refresh { + if limit == 0 { + return Vec::new(); + } + if let Some(index) = cache.as_ref().and_then(|c| c.index.as_ref()) { + return index.search(query, limit, files_only, control); + } + } + let should_build = !fresh || refresh; + if should_build { + *cache = None; + } + let mut builder = should_build.then(Builder::default); + let mut matches = Matches::new(query, limit, files_only); + let mut order = 0; + visit_workspace_files(&root, |entry| { + if control.stopped() { + return false; + } + matches.offer( + &entry.name, + &entry.path, + entry.kind == WorkspaceEntryKind::Dir, + order, + ); + order += 1; + if builder + .as_mut() + .is_some_and(|b| !b.push(&entry, budget.build)) + { + builder = None; + } + // A warmup can stop once the budget is exhausted. A real search keeps + // walking and ranking so a late/deep exact match is never omitted. + builder.is_some() || limit != 0 + }); + if control.stopped() { + // A partial traversal must never become a fresh, incomplete index. + return Vec::new(); + } + if should_build { + let index = builder + .map(|b| b.index) + .filter(|i| i.bytes() <= budget.retained); + *cache = Some(CachedIndex { + root, + built_at: Instant::now(), + index, + }); + } + matches.finish() +} + +#[cfg_attr(feature = "tauri-runtime", tauri::command)] +pub async fn search_workspace_files( + path: String, + query: String, + limit: Option, + files_only: Option, + refresh: Option, + request: Option, +) -> Result, AppCommandError> { + let requested_at = Instant::now(); + if path.trim().is_empty() || query.len() > 2048 { + return Err(AppCommandError::invalid_input( + "Invalid workspace file search", + )); + } + let limit = limit.unwrap_or(100).clamp(1, MAX_RESULTS); + let files_only = files_only.unwrap_or(false); + // Register before admission: a newer generation cancels both queued and + // running work for this consumer, without affecting another picker/client. + let request = request.unwrap_or_else(|| SearchRequest { + consumer_id: uuid::Uuid::new_v4().to_string(), + generation: 0, + }); + let cancel = match SEARCHES.lock().unwrap().register(&request, false)? { + Some(token) => token, + None => return Ok(Vec::new()), + }; + let search = ActiveSearch { + request, + control: SearchControl { + cancel, + deadline: requested_at + SEARCH_TIMEOUT, + }, + }; + let mut cache = tokio::select! { + biased; + _ = search.control.cancel.cancelled() => return Ok(Vec::new()), + _ = tokio::time::sleep_until(search.control.deadline.into()) => { + return Err(AppCommandError::task_execution_failed("Workspace file search timed out")); + } + cache = CACHE.lock() => cache, + }; + // A warmup or another refresh may have rebuilt the index while this + // request waited. Reuse that newer snapshot instead of scanning twice. + let refresh = + refresh.unwrap_or(false) && cache.as_ref().is_none_or(|c| c.built_at < requested_at); + tokio::task::spawn_blocking(move || { + let control = &search.control; + let results = if query.trim().is_empty() { + // Even files-only listings can walk an entire tree of directories. + // They share the worker, cancellation and deadline with searches. + let mut entries = Vec::new(); + visit_workspace_files(&path, |entry| { + if control.stopped() { + return false; + } + if !files_only || entry.kind == WorkspaceEntryKind::File { + entries.push(entry); + } + entries.len() < limit + }); + entries + } else { + search_or_build( + &mut cache, + path, + SearchOptions { + query: &query, + limit, + files_only, + refresh, + }, + INDEX_BUDGET, + control, + ) + }; + if Instant::now() >= control.deadline { + Err(AppCommandError::task_execution_failed( + "Workspace file search timed out", + )) + } else if control.cancel.is_cancelled() { + Ok(Vec::new()) + } else { + Ok(results) + } + }) + .await + .map_err(|e| AppCommandError::task_execution_failed(e.to_string()))? +} + +#[cfg_attr(feature = "tauri-runtime", tauri::command)] +pub async fn cancel_workspace_file_search(request: SearchRequest) -> Result<(), AppCommandError> { + SEARCHES.lock().unwrap().register(&request, true)?; + Ok(()) +} + +/// Prewarm only native, budgeted storage. Return no file list to the renderer. +#[cfg_attr(feature = "tauri-runtime", tauri::command)] +pub async fn prepare_workspace_file_search(path: String) { + if path.trim().is_empty() { + return; + } + let Ok(mut cache) = CACHE.try_lock() else { + return; + }; + tokio::task::spawn_blocking(move || { + search_or_build( + &mut cache, + path, + SearchOptions { + query: "", + limit: 0, + files_only: false, + refresh: false, + }, + INDEX_BUDGET, + &SearchControl::default(), + ); + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run_search( + cache: &mut Option, + root: String, + query: &str, + limit: usize, + files_only: bool, + refresh: bool, + budget: IndexBudget, + ) -> Vec { + search_or_build( + cache, + root, + SearchOptions { + query, + limit, + files_only, + refresh, + }, + budget, + &SearchControl::default(), + ) + } + + fn file(path: &str, is_dir: bool) -> WorkspaceFileEntry { + WorkspaceFileEntry { + name: path.rsplit('/').next().unwrap().into(), + path: path.into(), + kind: if is_dir { + WorkspaceEntryKind::Dir + } else { + WorkspaceEntryKind::File + }, + } + } + + #[test] + fn compact_index_preserves_paths_and_rank_order() { + let entries = [ + file("a", true), + file("a/deep", true), + file("a/deep/target.ts", false), + file("a/target-extra.ts", false), + file("b", true), + file("b/target.ts", false), + ]; + let mut builder = Builder::default(); + let mut matches = Matches::new("target", 3, false); + for (i, e) in entries.iter().enumerate() { + assert!(builder.push(e, BUILD_BYTES)); + matches.offer(&e.name, &e.path, e.kind == WorkspaceEntryKind::Dir, i); + } + assert_eq!( + builder + .index + .search("target", 3, false, &SearchControl::default()), + matches.finish() + ); + assert_eq!( + builder + .index + .search("", 100, false, &SearchControl::default()), + entries + ); + } + + #[test] + fn scores_match_frontend_tiers_and_utf16_offsets() { + assert_eq!(score_file("foo", "FOO", "a/FOO"), Some(599_999_997)); + assert_eq!(score_file("foo", "foo-long", "foo-long"), Some(499_999_992)); + assert_eq!(score_file("foo", "😀foo", "😀foo"), Some(399_997_995)); + assert!(score_file("fbr", "FooBar.rs", "src/FooBar.rs").is_some()); + assert_eq!(score_file("xyz", "abc", "src/abc"), None); + } + + #[test] + fn compact_search_pruning_preserves_path_unicode_and_subsequence_matches() { + let entries = [ + file("foo", true), + file("foo/deep", true), + file("foo/deep/😀FooBar.rs", false), + file("foo/deep/README.md", false), + file("foo/README.md", false), + file("src", true), + file("src/ab", true), + file("src/ab/README.md", false), + file("src/ab/İstanbul.ts", false), + file("src/FooBar.rs", false), + file("src/éclair.ts", false), + ]; + let mut builder = Builder::default(); + for entry in &entries { + assert!(builder.push(entry, BUILD_BYTES)); + } + for query in ["", "README.md", "src/ab", "/", "fbr", "😀", "é", "İ"] { + for limit in [1, 3, 100] { + for files_only in [false, true] { + let mut expected = Matches::new(query, limit, files_only); + for (i, entry) in entries.iter().enumerate() { + expected.offer( + &entry.name, + &entry.path, + entry.kind == WorkspaceEntryKind::Dir, + i, + ); + } + assert_eq!( + builder + .index + .search(query, limit, files_only, &SearchControl::default()), + expected.finish(), + "query={query:?}, limit={limit}, files_only={files_only}" + ); + } + } + } + } + + #[test] + fn result_budget_keeps_late_exact_match_and_stable_ties() { + let mut matches = Matches::new("needle", 2, false); + for i in 0..1000 { + matches.offer("needle-extra", &format!("{i}/needle-extra"), false, i); + } + matches.offer("needle", "very/deep/needle", false, 1000); + assert_eq!(matches.best.len(), 2); + let best = matches.finish(); + assert_eq!(best[0].path, "very/deep/needle"); + assert_eq!(best[1].path, "0/needle-extra"); + } + + #[test] + fn budget_fallback_still_searches_the_entire_workspace() { + let dir = tempfile::tempdir().unwrap(); + for i in 0..100 { + std::fs::write(dir.path().join(format!("a{i:03}.txt")), "x").unwrap(); + } + std::fs::create_dir_all(dir.path().join("z/deep")).unwrap(); + std::fs::write(dir.path().join("z/deep/needle.txt"), "x").unwrap(); + let mut cache = None; + let root = dir.path().to_string_lossy().into_owned(); + let budget = IndexBudget { + build: 1024, + retained: 1024, + }; + let best = run_search(&mut cache, root.clone(), "needle", 1, true, false, budget); + assert_eq!(best[0].path, "z/deep/needle.txt"); + assert!(cache.as_ref().unwrap().index.is_none()); + assert_eq!( + run_search(&mut cache, root, "needle", 1, true, false, budget), + best + ); + } + + #[test] + fn budget_fallback_preserves_failed_build_ttl_and_eventually_retries() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("needle.txt"), "x").unwrap(); + let root = dir.path().to_string_lossy().into_owned(); + let mut cache = None; + run_search( + &mut cache, + root.clone(), + "needle", + 1, + true, + false, + IndexBudget { + build: 1024, + retained: 1024, + }, + ); + let marker = Instant::now() - Duration::from_secs(290); + cache.as_mut().unwrap().built_at = marker; + for _ in 0..3 { + let result = run_search( + &mut cache, + root.clone(), + "needle", + 1, + true, + false, + INDEX_BUDGET, + ); + assert_eq!(result[0].path, "needle.txt"); + assert_eq!(cache.as_ref().unwrap().built_at, marker); + assert!(cache.as_ref().unwrap().index.is_none()); + } + cache.as_mut().unwrap().built_at = Instant::now() - CACHE_TTL; + run_search(&mut cache, root, "needle", 1, true, false, INDEX_BUDGET); + assert!(cache.as_ref().unwrap().index.is_some()); + } + + fn request(consumer: &str, generation: u64) -> SearchRequest { + SearchRequest { + consumer_id: consumer.into(), + generation, + } + } + + #[test] + fn superseding_requests_cancel_only_their_own_consumer_and_reject_old_generations() { + let mut searches = Searches::default(); + let first = searches + .register(&request("picker", 1), false) + .unwrap() + .unwrap(); + let other = searches + .register(&request("other-client", 1), false) + .unwrap() + .unwrap(); + let latest = searches + .register(&request("picker", 3), false) + .unwrap() + .unwrap(); + assert!(first.is_cancelled()); + assert!(!other.is_cancelled()); + assert!(searches + .register(&request("picker", 2), false) + .unwrap() + .is_none()); + searches.register(&request("picker", 1), true).unwrap(); + assert!(!latest.is_cancelled()); + searches.register(&request("picker", 3), true).unwrap(); + assert!(latest.is_cancelled()); + searches + .register(&request("before-arrival", 2), true) + .unwrap(); + assert!(searches + .register(&request("before-arrival", 2), false) + .unwrap() + .is_none()); + } + + #[test] + fn search_admission_and_completed_consumer_markers_are_bounded() { + let mut searches = Searches::default(); + for i in 0..MAX_CONSUMERS { + searches + .register(&request(&i.to_string(), 1), false) + .unwrap(); + } + assert!(searches.register(&request("overflow", 1), false).is_err()); + searches.0.get_mut("0").unwrap().active = false; + searches.register(&request("overflow", 1), false).unwrap(); + assert_eq!(searches.0.len(), MAX_CONSUMERS); + } + + #[test] + fn canceled_or_expired_searches_do_not_replace_a_cached_index() { + let mut cache = Some(CachedIndex { + root: "existing".into(), + built_at: Instant::now(), + index: None, + }); + let canceled = SearchControl::default(); + canceled.cancel.cancel(); + let expired = SearchControl { + deadline: Instant::now() - Duration::from_secs(1), + ..SearchControl::default() + }; + for control in [canceled, expired] { + assert!(search_or_build( + &mut cache, + "replacement".into(), + SearchOptions { + query: "needle", + limit: 1, + files_only: true, + refresh: true + }, + INDEX_BUDGET, + &control, + ) + .is_empty()); + assert_eq!(cache.as_ref().unwrap().root, "existing"); + } + } + + #[tokio::test] + async fn empty_query_shares_admission_and_can_cancel_while_waiting() { + let cache = CACHE.lock().await; + let request = request(&uuid::Uuid::new_v4().to_string(), 1); + let search_request = request.clone(); + let task = tokio::spawn(search_workspace_files( + "/unused-because-canceled".into(), + "".into(), + Some(200), + Some(true), + None, + Some(search_request), + )); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if SEARCHES + .lock() + .unwrap() + .0 + .contains_key(&request.consumer_id) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!( + !task.is_finished(), + "empty queries must wait for the shared worker" + ); + cancel_workspace_file_search(request).await.unwrap(); + let result = tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(result.is_empty()); + drop(cache); + } + + #[tokio::test] + async fn newer_generation_coalesces_a_queued_search_before_it_can_traverse() { + let cache = CACHE.lock().await; + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("latest.txt"), "x").unwrap(); + let root = dir.path().to_string_lossy().into_owned(); + let first = request(&uuid::Uuid::new_v4().to_string(), 1); + let latest = request(&first.consumer_id, 2); + let old_task = tokio::spawn(search_workspace_files( + root.clone(), + "obsolete".into(), + None, + None, + None, + Some(first.clone()), + )); + tokio::time::timeout(Duration::from_secs(1), async { + while !SEARCHES.lock().unwrap().0.contains_key(&first.consumer_id) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let latest_task = tokio::spawn(search_workspace_files( + root, + "latest".into(), + None, + None, + None, + Some(latest), + )); + // The old request completes while the worker is still held: it never + // starts a superseded traversal or waits for the newer one's scan. + assert!(tokio::time::timeout(Duration::from_secs(1), old_task) + .await + .unwrap() + .unwrap() + .unwrap() + .is_empty()); + assert!(!latest_task.is_finished()); + drop(cache); + let results = latest_task.await.unwrap().unwrap(); + assert_eq!(results[0].path, "latest.txt"); + } + + #[test] + fn refresh_and_root_changes_replace_cached_entries() { + let first = tempfile::tempdir().unwrap(); + let second = tempfile::tempdir().unwrap(); + std::fs::write(first.path().join("old.ts"), "x").unwrap(); + std::fs::write(second.path().join("other.ts"), "x").unwrap(); + let mut cache = None; + let root = first.path().to_string_lossy().into_owned(); + run_search(&mut cache, root.clone(), "", 0, false, false, INDEX_BUDGET); + std::fs::write(first.path().join("new.ts"), "x").unwrap(); + assert!(run_search( + &mut cache, + root.clone(), + "new", + 10, + true, + false, + INDEX_BUDGET + ) + .is_empty()); + assert_eq!( + run_search(&mut cache, root, "new", 10, true, true, INDEX_BUDGET)[0].path, + "new.ts" + ); + let result = run_search( + &mut cache, + second.path().to_string_lossy().into_owned(), + "ts", + 10, + true, + false, + INDEX_BUDGET, + ); + assert_eq!(result.len(), 1); + assert_eq!(result[0].path, "other.ts"); + assert!(cache.as_ref().unwrap().index.as_ref().unwrap().bytes() <= INDEX_BYTES); + } + + #[tokio::test] + async fn api_caps_empty_query_and_keeps_files_only() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("a-directory")).unwrap(); + for i in 0..300 { + std::fs::write(dir.path().join(format!("f{i:03}.txt")), "x").unwrap(); + } + let results = search_workspace_files( + dir.path().to_string_lossy().into_owned(), + "".into(), + Some(usize::MAX), + Some(true), + None, + None, + ) + .await + .unwrap(); + assert_eq!(results.len(), MAX_RESULTS); + assert!(results.iter().all(|e| e.kind == WorkspaceEntryKind::File)); + } + + /// Manual end-to-end probe for unusually large workspaces. The path stays + /// local; output contains counts/budgets/timings rather than filenames. + #[test] + #[ignore = "set CODEG_MEMORY_WORKSPACE to profile a real workspace"] + fn large_workspace_memory_probe() { + let root = std::env::var("CODEG_MEMORY_WORKSPACE").expect("CODEG_MEMORY_WORKSPACE"); + let started = Instant::now(); + let mut cache = None; + println!("probe_pid={}", std::process::id()); + run_search(&mut cache, root.clone(), "", 0, false, false, INDEX_BUDGET); + let warm_seconds = started.elapsed().as_secs_f64(); + let stats = cache.as_ref().and_then(|c| c.index.as_ref()).map(|i| { + serde_json::json!({"entries":i.nodes.len(),"uniqueNames":i.names.len(),"retainedBytes":i.bytes()}) + }); + println!( + "{}", + serde_json::json!({"warmSeconds":warm_seconds,"index":stats}) + ); + for query in ["README.md", "index.ts", "workspace_file_search"] { + let started = Instant::now(); + let results = run_search( + &mut cache, + root.clone(), + query, + 100, + true, + false, + INDEX_BUDGET, + ); + assert!(results.len() <= 100); + println!( + "{}", + serde_json::json!({"query":query,"results":results.len(),"responseBytes":serde_json::to_vec(&results).unwrap().len(),"seconds":started.elapsed().as_secs_f64()}) + ); + } + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index e5900e5aef0..3b629f27a3d 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1981,6 +1981,9 @@ mod tauri_app { folders::list_directory_with_files, folders::get_file_tree, folders::list_workspace_files, + crate::commands::workspace_file_search::search_workspace_files, + crate::commands::workspace_file_search::cancel_workspace_file_search, + crate::commands::workspace_file_search::prepare_workspace_file_search, folders::read_file_base64, folders::read_workspace_file_base64, folders::read_file_preview, diff --git a/src-tauri/src/web/handlers/folders.rs b/src-tauri/src/web/handlers/folders.rs index 9ade38657d8..a0d27722618 100644 --- a/src-tauri/src/web/handlers/folders.rs +++ b/src-tauri/src/web/handlers/folders.rs @@ -410,6 +410,52 @@ pub async fn list_workspace_files( Ok(Json(result)) } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchWorkspaceFilesParams { + pub path: String, + pub query: String, + pub limit: Option, + pub files_only: Option, + pub refresh: Option, + pub request: Option, +} + +pub async fn search_workspace_files( + Json(params): Json, +) -> Result>, AppCommandError> { + Ok(Json( + crate::commands::workspace_file_search::search_workspace_files( + params.path, + params.query, + params.limit, + params.files_only, + params.refresh, + params.request, + ) + .await?, + )) +} + +#[derive(Deserialize)] +pub struct CancelWorkspaceFileSearchParams { + pub request: crate::commands::workspace_file_search::SearchRequest, +} + +pub async fn cancel_workspace_file_search( + Json(params): Json, +) -> Result, AppCommandError> { + crate::commands::workspace_file_search::cancel_workspace_file_search(params.request).await?; + Ok(Json(())) +} + +pub async fn prepare_workspace_file_search( + Json(params): Json, +) -> Json<()> { + crate::commands::workspace_file_search::prepare_workspace_file_search(params.path).await; + Json(()) +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct OpenSettingsWindowParams { diff --git a/src-tauri/src/web/router.rs b/src-tauri/src/web/router.rs index e2008192d32..d388c55df31 100644 --- a/src-tauri/src/web/router.rs +++ b/src-tauri/src/web/router.rs @@ -529,6 +529,18 @@ pub fn build_router_with_proxy_auth( "/list_workspace_files", post(handlers::folders::list_workspace_files), ) + .route( + "/search_workspace_files", + post(handlers::folders::search_workspace_files), + ) + .route( + "/cancel_workspace_file_search", + post(handlers::folders::cancel_workspace_file_search), + ) + .route( + "/prepare_workspace_file_search", + post(handlers::folders::prepare_workspace_file_search), + ) .route( "/start_workspace_state_stream", post(handlers::workspace_state::start_workspace_state_stream), diff --git a/src/components/canvas/add-node-menu.tsx b/src/components/canvas/add-node-menu.tsx index 2fa323a68cd..19473a83b0a 100644 --- a/src/components/canvas/add-node-menu.tsx +++ b/src/components/canvas/add-node-menu.tsx @@ -371,9 +371,8 @@ const FILE_RESULT_LIMIT = 20 * search dialog and the composer's `@` picker use. "Browse…" covers everything * outside a workspace folder. * - * The walk is lazy: `useFileTree` only fetches once `enabled` goes true, which - * happens when the submenu opens — a board that never adds a file never pays - * for the listing. + * Workspace discovery starts only when the user types a query; the initial + * open-files listing needs no backend traversal. */ function AddFileSubmenu({ onPick }: { onPick: (absPath: string) => void }) { const t = useTranslations("Canvas") @@ -393,7 +392,10 @@ function AddFileSubmenu({ onPick }: { onPick: (absPath: string) => void }) { const { allFiles, loading } = useFileTree({ folderPath: searchFolder?.path, - enabled: open && searchFolder != null, + enabled: open && searchFolder != null && query.trim().length > 0, + query, + filesOnly: true, + limit: FILE_RESULT_LIMIT, }) const results = useMemo(() => { diff --git a/src/components/chat/composer/suggestion/suggestion-popup.test.tsx b/src/components/chat/composer/suggestion/suggestion-popup.test.tsx index cc711990cd9..e71a2a1fc29 100644 --- a/src/components/chat/composer/suggestion/suggestion-popup.test.tsx +++ b/src/components/chat/composer/suggestion/suggestion-popup.test.tsx @@ -709,10 +709,134 @@ describe("SuggestionPopup with a chosen tab order", () => { const spy = vi.fn(() => groups) mountPopup({ search: spy, tabs: ["file", "agent"] }) await screen.findByText("alpha.md") - expect(spy).toHaveBeenCalledWith(state.query, expect.any(AbortSignal), [ - "file", - "agent", - ]) + expect(spy).toHaveBeenCalledWith( + state.query, + expect.any(AbortSignal), + ["file", "agent"], + expect.any(Function) + ) + }) + + it("shows ready agents while files load and preserves selection when file results arrive", async () => { + let finish!: (groups: SuggestionGroup[]) => void + const progressive: ReferenceSearch = (_query, _signal, _kinds, update) => { + update?.( + groups.map((group) => + group.kind === "file" ? { ...group, items: [], loading: true } : group + ) + ) + return new Promise((resolve) => { + finish = resolve + }) + } + const { ref, onSelect } = mountPopup({ + search: progressive, + tabs: ["agent", "file"], + }) + await screen.findByText("Codex Helper") + fireEvent.click(screen.getByRole("tab", { name: /^Files/ })) + expect(screen.getByRole("status")).toHaveTextContent("Searching…") + fireEvent.click(screen.getByRole("tab", { name: /^Agents/ })) + act(() => ref.current?.onKeyDown(key("ArrowDown"))) + await act(async () => finish(groups)) + act(() => ref.current?.onKeyDown(key("Enter"))) + expect(onSelect).toHaveBeenCalledWith(agentRef2, state.range) + }) + + it.each([1, 2])( + "keeps the keyboard-selected session when %i delayed files arrive", + async (fileCount) => { + const sessions = [1, 2].map((id) => ({ + reference: { + refType: "session" as const, + id: String(id), + label: `Ready session ${id}`, + uri: `codeg://session/${id}`, + meta: null, + }, + })) + const ready: SuggestionGroup[] = [ + { kind: "file", label: "Files", items: [], loading: true }, + { kind: "session", label: "Sessions", items: sessions }, + ] + let finish!: (value: SuggestionGroup[]) => void + const progressive: ReferenceSearch = ( + _query, + _signal, + _kinds, + update + ) => { + update?.(ready) + return new Promise((resolve) => { + finish = resolve + }) + } + const { ref, onSelect } = mountPopup({ + search: progressive, + tabs: ["file", "session"], + }) + await screen.findByText("Ready session 1") + act(() => ref.current?.onKeyDown(key("ArrowDown"))) + await act(async () => + finish([ + { + kind: "file", + label: "Files", + items: Array.from({ length: fileCount }, (_, i) => ({ + reference: { + ...fileRef, + id: `late${i}.ts`, + label: `late${i}.ts`, + }, + })), + }, + ready[1], + ]) + ) + expect(screen.getByRole("tab", { name: /^Sessions/ })).toHaveAttribute( + "aria-selected", + "true" + ) + act(() => ref.current?.onKeyDown(key("Enter"))) + expect(onSelect).toHaveBeenCalledWith(sessions[1].reference, state.range) + } + ) + + it("preserves the selected reference across a progressive reorder and clamps a removed row", async () => { + let update!: NonNullable[3]> + const progressive: ReferenceSearch = (_query, _signal, _kinds, publish) => { + update = publish! + return groups + } + const { ref, onSelect } = mountPopup({ search: progressive }) + await screen.findByText("Codex Helper") + act(() => ref.current?.onKeyDown(key("ArrowDown"))) + act(() => + update( + groups.map((group) => + group.kind === "agent" + ? { ...group, items: [...group.items].reverse() } + : group + ) + ) + ) + act(() => ref.current?.onKeyDown(key("Enter"))) + expect(onSelect).toHaveBeenLastCalledWith(agentRef2, state.range) + act(() => + update( + groups.map((group) => + group.kind === "agent" + ? { ...group, items: [{ reference: agentRef }] } + : group + ) + ) + ) + const active = screen + .getAllByRole("option") + .filter((row) => row.getAttribute("aria-selected") === "true") + expect(active).toHaveLength(1) + act(() => ref.current?.onKeyDown(key("Enter"))) + expect(onSelect).toHaveBeenLastCalledWith(agentRef, state.range) }) it("re-queries when the shown tabs change while it is open", async () => { @@ -739,11 +863,12 @@ describe("SuggestionPopup with a chosen tab order", () => { // keystroke. rerender(view(["file", "agent", "session"])) await waitFor(() => expect(spy).toHaveBeenCalledTimes(2)) - expect(spy).toHaveBeenLastCalledWith(state.query, expect.any(AbortSignal), [ - "file", - "agent", - "session", - ]) + expect(spy).toHaveBeenLastCalledWith( + state.query, + expect.any(AbortSignal), + ["file", "agent", "session"], + expect.any(Function) + ) expect(screen.getAllByRole("tab")).toHaveLength(3) }) diff --git a/src/components/chat/composer/suggestion/suggestion-popup.tsx b/src/components/chat/composer/suggestion/suggestion-popup.tsx index b886670194f..ec36a3c50f9 100644 --- a/src/components/chat/composer/suggestion/suggestion-popup.tsx +++ b/src/components/chat/composer/suggestion/suggestion-popup.tsx @@ -2,6 +2,7 @@ import { forwardRef, + useCallback, useEffect, useImperativeHandle, useLayoutEffect, @@ -179,8 +180,13 @@ export const SuggestionPopup = forwardRef< kinds: readonly ReferenceKind[] groups: SuggestionGroup[] }>({ query: null, kinds: [], groups: [] }) - const [selectedIndex, setSelectedIndex] = useState(0) - // The tab the user explicitly chose (via Tab/click), or null to auto-follow + const [selection, setSelection] = useState<{ + query: string + kind: ReferenceKind + id: string + index: number + } | null>(null) + // The tab the user explicitly chose or navigated, or null to auto-follow // the first non-empty tab. Pinning survives subsequent keystrokes within this // open session; reopening the panel remounts and resets it to null. const [pinnedTab, setPinnedTab] = useState(null) @@ -207,18 +213,33 @@ export const SuggestionPopup = forwardRef< useEffect(() => { const abort = new AbortController() let active = true + let published = false + const publish = (groups: SuggestionGroup[]) => { + if (!active || abort.signal.aborted) return + const kinds = tabOrder.filter( + (kind) => !groups.some((group) => group.kind === kind && group.loading) + ) + setResult({ query: state.query, kinds, groups }) + // A late file result must not reset navigation within a ready group. + const firstPublication = !published + setSelection((current) => { + if (firstPublication || current?.query !== state.query) return null + const group = groups.find(({ kind }) => kind === current.kind) + if (!group || group.loading) return current + const found = group.items.findIndex( + ({ reference }) => reference.id === current.id + ) + const index = + found >= 0 ? found : Math.min(current.index, group.items.length - 1) + const chosen = group.items[index] + return chosen ? { ...current, index, id: chosen.reference.id } : null + }) + published = true + } const timer = setTimeout(() => { - Promise.resolve(search(state.query, abort.signal, tabOrder)) - .then((groups) => { - if (!active || abort.signal.aborted) return - setResult({ query: state.query, kinds: tabOrder, groups }) - setSelectedIndex(0) - }) - .catch(() => { - if (!active || abort.signal.aborted) return - setResult({ query: state.query, kinds: tabOrder, groups: [] }) - setSelectedIndex(0) - }) + Promise.resolve(search(state.query, abort.signal, tabOrder, publish)) + .then(publish) + .catch(() => publish([])) }, FETCH_DEBOUNCE_MS) return () => { active = false @@ -257,6 +278,31 @@ export const SuggestionPopup = forwardRef< () => (stale || !activeGroup ? [] : activeGroup.items), [stale, activeGroup] ) + const currentSelection = + selection?.query === state.query && selection.kind === activeTab + ? selection + : null + const foundIndex = currentSelection + ? flat.findIndex(({ reference }) => reference.id === currentSelection.id) + : -1 + const selectedIndex = + foundIndex >= 0 + ? foundIndex + : Math.min(currentSelection?.index ?? 0, Math.max(0, flat.length - 1)) + const selectIndex = useCallback( + (index: number) => { + const item = flat[index] + if (!item) return + setPinnedTab(activeTab) + setSelection({ + query: state.query, + kind: activeTab, + id: item.reference.id, + index, + }) + }, + [flat, activeTab, state.query] + ) // Scroll the active option into view (scoped to options so it never targets // the active tab button, which also carries an active marker via class only). @@ -422,14 +468,12 @@ export const SuggestionPopup = forwardRef< switch (event.key) { case "ArrowDown": if (flat.length > 0) { - setSelectedIndex((index) => (index + 1) % flat.length) + selectIndex((selectedIndex + 1) % flat.length) } return true case "ArrowUp": if (flat.length > 0) { - setSelectedIndex( - (index) => (index - 1 + flat.length) % flat.length - ) + selectIndex((selectedIndex - 1 + flat.length) % flat.length) } return true case "Tab": { @@ -440,7 +484,7 @@ export const SuggestionPopup = forwardRef< setPinnedTab( tabOrder[(at + dir + tabOrder.length) % tabOrder.length] ) - setSelectedIndex(0) + setSelection(null) return true } case "Enter": { @@ -468,7 +512,16 @@ export const SuggestionPopup = forwardRef< } }, }), - [flat, selectedIndex, activeTab, tabOrder, onSelect, onClose, state.range] + [ + flat, + selectedIndex, + selectIndex, + activeTab, + tabOrder, + onSelect, + onClose, + state.range, + ] ) const activeLabel = tabLabels[activeTab] @@ -552,7 +605,7 @@ export const SuggestionPopup = forwardRef< onMouseDown={(event) => event.preventDefault()} onClick={() => { setPinnedTab(kind) - setSelectedIndex(0) + setSelection(null) }} className={cn( "flex shrink-0 items-center gap-1 rounded-md px-2 py-1 text-xs font-medium", @@ -617,7 +670,7 @@ export const SuggestionPopup = forwardRef< event.preventDefault() onSelect(item.reference, state.range) }} - onMouseEnter={() => setSelectedIndex(index)} + onMouseEnter={() => selectIndex(index)} > {/* Label at its content width (shrinking + truncating only diff --git a/src/components/chat/composer/suggestion/types.ts b/src/components/chat/composer/suggestion/types.ts index 0e37b803b0e..85d410c3020 100644 --- a/src/components/chat/composer/suggestion/types.ts +++ b/src/components/chat/composer/suggestion/types.ts @@ -22,6 +22,8 @@ export interface SuggestionGroup { * the overflow. */ truncated?: boolean + /** This source is still loading; other groups may already be selectable. */ + loading?: boolean } /** @@ -50,14 +52,15 @@ export interface MentionUiLabels { * outside it is never displayed, so an implementation may skip loading its * source and return it empty; omitted means every kind is wanted. * - * Phase 2 ships the panel against this interface; Phase 3 supplies the real - * implementation (wired to the live data hooks) when the composer replaces the - * textarea in message-input. + * `onUpdate` publishes partial snapshots as sources become ready. Groups still + * waiting for their source set `loading`; the returned value is the final + * snapshot. Consumers that omit the callback can await all groups as before. */ export type ReferenceSearch = ( query: string, signal?: AbortSignal, - kinds?: readonly ReferenceKind[] + kinds?: readonly ReferenceKind[], + onUpdate?: (groups: SuggestionGroup[]) => void ) => SuggestionGroup[] | Promise /** State the suggestion plugin pushes to React while the `@` panel is open. */ diff --git a/src/components/chat/composer/use-reference-search.test.ts b/src/components/chat/composer/use-reference-search.test.ts index 13f71358bab..795b3353092 100644 --- a/src/components/chat/composer/use-reference-search.test.ts +++ b/src/components/chat/composer/use-reference-search.test.ts @@ -415,6 +415,8 @@ const mocks = vi.hoisted(() => ({ files: { allFiles: [] as FlatFileEntry[], loaded: false }, listAllConversations: vi.fn(), gitLog: vi.fn(), + searchWorkspaceFiles: vi.fn(), + prepareWorkspaceFileSearch: vi.fn(), })) vi.mock("@/hooks/use-file-tree", () => ({ @@ -432,6 +434,10 @@ vi.mock("@/lib/api", () => ({ listAllConversations: (...args: unknown[]) => mocks.listAllConversations(...args), gitLog: (...args: unknown[]) => mocks.gitLog(...args), + searchWorkspaceFiles: (...args: unknown[]) => + mocks.searchWorkspaceFiles(...args), + prepareWorkspaceFileSearch: (...args: unknown[]) => + mocks.prepareWorkspaceFileSearch(...args), })) describe("useReferenceSearch", () => { @@ -439,12 +445,49 @@ describe("useReferenceSearch", () => { resetAssistantsStore() mocks.agents = [] mocks.files = { allFiles: [], loaded: false } + mocks.prepareWorkspaceFileSearch.mockReset().mockResolvedValue(undefined) + mocks.searchWorkspaceFiles.mockReset().mockImplementation(async () => + mocks.files.loaded + ? mocks.files.allFiles.map((f) => ({ + name: f.name, + path: f.relativePath, + kind: f.kind, + })) + : [] + ) mocks.listAllConversations.mockReset().mockResolvedValue([]) mocks.gitLog .mockReset() .mockResolvedValue({ entries: [], has_upstream: false }) }) + it("prepares native storage without transferring files until the file group is requested", async () => { + const { result } = renderHook(() => + useReferenceSearch({ defaultPath: "/repo" }) + ) + expect(mocks.prepareWorkspaceFileSearch).toHaveBeenCalledWith("/repo") + expect(mocks.searchWorkspaceFiles).not.toHaveBeenCalled() + await act(async () => { + await result.current("a", undefined, ["agent"]) + }) + expect(mocks.searchWorkspaceFiles).not.toHaveBeenCalled() + await act(async () => { + await result.current("a", undefined, ["file"]) + }) + expect(mocks.searchWorkspaceFiles).toHaveBeenCalledExactlyOnceWith( + "/repo", + "a", + { + limit: 51, + request: { + consumerId: expect.any(String), + generation: expect.any(Number), + }, + signal: expect.any(AbortSignal), + } + ) + }) + it("returns a referentially stable search across data-source updates (R7)", async () => { mocks.agents = [makeAgent("codex", { name: "Codex" })] const { result, rerender } = renderHook( @@ -730,4 +773,95 @@ describe("useReferenceSearch", () => { // …which succeeds and populates the group. expect(itemsOf(second, "session")).toHaveLength(1) }) + + it("publishes ready groups while file discovery is pending and merges each source independently", async () => { + mocks.agents = [makeAgent("codex")] + useAssistantsStore.getState().merge([makeAssistant(3, "Reviewer")]) + let filesReady!: ( + files: { name: string; path: string; kind: "file" }[] + ) => void + mocks.searchWorkspaceFiles.mockReturnValue( + new Promise((resolve) => { + filesReady = resolve + }) + ) + let sessionsReady!: (sessions: DbConversationSummary[]) => void + mocks.listAllConversations.mockReturnValue( + new Promise((resolve) => { + sessionsReady = resolve + }) + ) + const { result } = renderHook(() => + useReferenceSearch({ defaultPath: "/repo" }) + ) + const update = vi.fn() + const pending = result.current("", undefined, undefined, update) + const ready = update.mock.lastCall![0] as SuggestionGroup[] + expect(itemsOf(ready, "agent")).toHaveLength(1) + expect(itemsOf(ready, "assistant")).toHaveLength(1) + expect(ready.find((g) => g.kind === "file")?.loading).toBe(true) + sessionsReady([makeConversation(7, "Session")]) + await vi.waitFor(() => { + expect(itemsOf(update.mock.lastCall![0], "session")).toHaveLength(1) + }) + expect( + update.mock.lastCall![0].find((g: SuggestionGroup) => g.kind === "file") + ?.loading + ).toBe(true) + filesReady([{ name: "a.ts", path: "a.ts", kind: "file" }]) + expect(itemsOf(await pending, "file")).toHaveLength(1) + expect(itemsOf(update.mock.lastCall![0], "file")).toHaveLength(1) + }) + + it("keeps an in-flight query valid when the window regains focus", async () => { + let filesReady!: ( + files: { name: string; path: string; kind: "file" }[] + ) => void + mocks.searchWorkspaceFiles.mockReturnValueOnce( + new Promise((resolve) => { + filesReady = resolve + }) + ) + const { result } = renderHook(() => + useReferenceSearch({ defaultPath: "/repo" }) + ) + const update = vi.fn() + const pending = result.current("focus", undefined, ["file"], update) + const options = mocks.searchWorkspaceFiles.mock.calls[0][2] + act(() => window.dispatchEvent(new Event("focus"))) + expect(options.signal.aborted).toBe(false) + filesReady([{ name: "focus.ts", path: "focus.ts", kind: "file" }]) + const groups = await pending + expect( + itemsOf(groups, "file").map(({ reference }) => reference.id) + ).toEqual(["focus.ts"]) + expect(itemsOf(update.mock.lastCall![0], "file")).toHaveLength(1) + expect(mocks.searchWorkspaceFiles).toHaveBeenCalledTimes(1) + }) + + it("cancels superseded file searches and suppresses stale partial updates", async () => { + let resolveOld!: (files: []) => void + mocks.searchWorkspaceFiles.mockReturnValueOnce( + new Promise((resolve) => { + resolveOld = resolve + }) + ) + const { result, unmount } = renderHook(() => + useReferenceSearch({ defaultPath: "/repo" }) + ) + const oldUpdate = vi.fn() + const old = result.current("a", undefined, ["file"], oldUpdate) + const first = mocks.searchWorkspaceFiles.mock.calls[0][2] + const newer = result.current("ab", undefined, ["file"]) + const second = mocks.searchWorkspaceFiles.mock.calls[1][2] + expect(first.signal.aborted).toBe(true) + expect(second.request.consumerId).toBe(first.request.consumerId) + expect(second.request.generation).toBeGreaterThan(first.request.generation) + await newer + oldUpdate.mockClear() + resolveOld([]) + expect(await old).toEqual([]) + expect(oldUpdate).not.toHaveBeenCalled() + unmount() + }) }) diff --git a/src/components/chat/composer/use-reference-search.ts b/src/components/chat/composer/use-reference-search.ts index fecf3aee4d8..c034af92e0c 100644 --- a/src/components/chat/composer/use-reference-search.ts +++ b/src/components/chat/composer/use-reference-search.ts @@ -3,8 +3,13 @@ import { useCallback, useEffect, useLayoutEffect, useRef } from "react" import { useAcpAgents } from "@/hooks/use-acp-agents" -import { useFileTree, type FlatFileEntry } from "@/hooks/use-file-tree" -import { gitLog, listAllConversations } from "@/lib/api" +import type { FlatFileEntry } from "@/hooks/use-file-tree" +import { + gitLog, + listAllConversations, + prepareWorkspaceFileSearch, + searchWorkspaceFiles, +} from "@/lib/api" import type { AcpAgentInfo, AssistantInfo, @@ -12,6 +17,7 @@ import type { GitLogEntry, } from "@/lib/types" import { useAssistants } from "@/stores/assistants-store" +import { randomUUID } from "@/lib/utils" import { agentToSuggestion, @@ -108,9 +114,8 @@ export function buildReferenceGroups( ): SuggestionGroup[] { const q = query.trim().toLowerCase() - // Files: filter the (potentially large) list on its pre-lowered fields before - // paying to adapt the survivors. `truncated` is a cheap boolean — set when a - // match is found past the cap — so we never scan the whole list for a count. + // Native search supplies at most MAX_PER_GROUP + 1 ranked matches. Keep the + // mention panel's substring semantics and use the extra row for truncation. const fileItems: SuggestionItem[] = [] let fileTruncated = false const root = sources.workspaceRoot @@ -239,10 +244,11 @@ export interface UseReferenceSearchOptions { * on window focus) updates the refs but leaves `search` identity untouched — the * open panel keeps its results and the user's selection (R7). * - * Files and agents are hook-loaded (and pre-warmed via `enabled`); assistants - * come from the assistants store, which the workspace layout keeps live. Sessions and - * the git log are fetched lazily on the first `@`, key-cached in a ref, and - * awaited by `search` so the first open is populated without an extra keystroke; + * File discovery is pre-warmed in shared, budgeted native storage; only the + * current query's matches are transferred on `@`. Agents are hook-loaded and + * assistants come from the live store. Sessions and the git log are fetched + * lazily on the first `@`, key-cached in a ref, and published independently + * so the first open is populated without an extra keystroke; * window focus busts those caches so they stay fresh. A tab the user hid (the * `kinds` the panel passes leave it out) costs nothing: its source isn't * fetched and its group comes back empty. @@ -254,20 +260,18 @@ export function useReferenceSearch({ }: UseReferenceSearchOptions): ReferenceSearch { const path = defaultPath || null - const { allFiles, loaded } = useFileTree({ - folderPath: path ?? undefined, - enabled, - }) + useEffect(() => { + if (enabled && path) void prepareWorkspaceFileSearch(path).catch(() => {}) + }, [enabled, path]) const { agents } = useAcpAgents() const assistants = useAssistants() // Mirror every changing source into a ref so `search` can stay identity-stable // (see the doc comment). Initialized from the first render so the refs are // sane even before the sync effect below runs. - const filesRef = useRef<{ root: string | null; files: FlatFileEntry[] }>({ - root: null, - files: [], - }) + const fileSearchRef = useRef(null) + const consumerRef = useRef(undefined) + const generationRef = useRef(0) const agentsRef = useRef(agents) const assistantsRef = useRef(assistants) const pathRef = useRef(path) @@ -283,20 +287,17 @@ export function useReferenceSearch({ useIsomorphicLayoutEffect(() => { pathRef.current = path enabledRef.current = enabled + return () => { + generationRef.current += 1 + fileSearchRef.current?.abort() + } }, [path, enabled]) useEffect(() => { - // Only expose files once the tree has loaded for the *current* path, so the - // search never joins the current workspace root onto a previous folder's - // relative paths during a folder switch. - filesRef.current = - loaded && path - ? { root: path, files: allFiles } - : { root: null, files: [] } agentsRef.current = agents assistantsRef.current = assistants labelsRef.current = labels - }, [allFiles, loaded, path, agents, assistants, labels]) + }, [agents, assistants, labels]) // Lazily-fetched network sources, key-cached so repeat searches reuse the // in-flight/resolved promise while a folder switch refetches. @@ -321,86 +322,147 @@ export function useReferenceSearch({ return () => window.removeEventListener("focus", onFocus) }, []) - return useCallback(async (query, signal, kinds) => { - if (!enabledRef.current) return [] + return useCallback( + async (query, signal, kinds, onUpdate) => { + if (!enabledRef.current || signal?.aborted) return [] - const path = pathRef.current - const wants = (kind: ReferenceKind) => !kinds || kinds.includes(kind) + const path = pathRef.current + const wants = (kind: ReferenceKind) => !kinds || kinds.includes(kind) + const generation = ++generationRef.current + fileSearchRef.current?.abort() + const abort = new AbortController() + fileSearchRef.current = abort + const onAbort = () => abort.abort() + signal?.addEventListener("abort", onAbort, { once: true }) + const pendingKinds = new Set() - // Lazy session fetch. On rejection the cache entry is cleared (not cached as - // an empty result) so the next `@` retries instead of wedging on `[]`. - let sessionsPromise = EMPTY_SESSIONS - if (wants("session")) { - const sessionsKey = "all" - let sessionsEntry = sessionsRef.current - if (sessionsEntry?.key !== sessionsKey) { - const created: NonNullable = { - key: sessionsKey, - promise: listAllConversations().catch(() => { - if (sessionsRef.current === created) sessionsRef.current = null - return [] as DbConversationSummary[] - }), - } - sessionsRef.current = created - sessionsEntry = created + // Retain only this query's matches, never the workspace's full file index. + // Ask for one extra row so the existing per-group truncation cue survives. + let filesPromise: Promise = Promise.resolve([]) + if (path && wants("file")) { + pendingKinds.add("file") + filesPromise = searchWorkspaceFiles(path, query, { + limit: MAX_PER_GROUP + 1, + request: { + consumerId: (consumerRef.current ??= randomUUID()), + generation, + }, + signal: abort.signal, + }) + .then((files) => + files.map((file) => ({ + name: file.name, + relativePath: file.path, + kind: file.kind, + lowerName: file.name.toLowerCase(), + lowerPath: file.path.toLowerCase(), + })) + ) + .catch(() => [] as FlatFileEntry[]) } - sessionsPromise = sessionsEntry.promise - } - // Lazy git-log fetch, keyed by path with the same retry-on-rejection policy. - // A hidden commit tab leaves the cache alone: it is still keyed by path, so - // switching the tab back on reuses it or refetches as usual. - let commitsPromise = EMPTY_COMMITS - if (path && wants("commit")) { - let commitsEntry = commitsRef.current - if (commitsEntry?.key !== path) { - const created: NonNullable = { - key: path, - promise: gitLog(path, GIT_LOG_LIMIT) - .then((result) => result.entries) - .catch(() => { - if (commitsRef.current === created) commitsRef.current = null - return [] as GitLogEntry[] + // Lazy session fetch. On rejection the cache entry is cleared (not cached as + // an empty result) so the next `@` retries instead of wedging on `[]`. + let sessionsPromise = EMPTY_SESSIONS + if (wants("session")) { + pendingKinds.add("session") + const sessionsKey = "all" + let sessionsEntry = sessionsRef.current + if (sessionsEntry?.key !== sessionsKey) { + const created: NonNullable = { + key: sessionsKey, + promise: listAllConversations().catch(() => { + if (sessionsRef.current === created) sessionsRef.current = null + return [] as DbConversationSummary[] }), + } + sessionsRef.current = created + sessionsEntry = created } - commitsRef.current = created - commitsEntry = created + sessionsPromise = sessionsEntry.promise } - commitsPromise = commitsEntry.promise - } else if (!path) { - commitsRef.current = null - } - const [sessions, commits] = await Promise.all([ - sessionsPromise, - commitsPromise, - ]) - // Discard this result if it can no longer be trusted for the live panel: a - // newer query aborted us, the composer was disabled, or the workspace folder - // changed while the network fetch was in flight (the popup only aborts on a - // query change, so a folder switch would otherwise leak the old repo's - // commits — built against `path` — into the new folder's panel). The next - // keystroke re-runs the search against the current folder. - if (signal?.aborted || !enabledRef.current || pathRef.current !== path) { - return [] - } + // Lazy git-log fetch, keyed by path with the same retry-on-rejection policy. + // A hidden commit tab leaves the cache alone: it is still keyed by path, so + // switching the tab back on reuses it or refetches as usual. + let commitsPromise = EMPTY_COMMITS + if (path && wants("commit")) { + pendingKinds.add("commit") + let commitsEntry = commitsRef.current + if (commitsEntry?.key !== path) { + const created: NonNullable = { + key: path, + promise: gitLog(path, GIT_LOG_LIMIT) + .then((result) => result.entries) + .catch(() => { + if (commitsRef.current === created) commitsRef.current = null + return [] as GitLogEntry[] + }), + } + commitsRef.current = created + commitsEntry = created + } + commitsPromise = commitsEntry.promise + } else if (!path) { + commitsRef.current = null + } - const fileState = filesRef.current - return buildReferenceGroups( - query, - { - // Already loaded by their hooks, but a hidden tab's rows would only be - // matched and adapted to be thrown away. - files: wants("file") ? fileState.files : [], - workspaceRoot: fileState.root, - agents: wants("agent") ? agentsRef.current : [], - assistants: wants("assistant") ? assistantsRef.current : [], - assistantAgents: wants("assistant") ? agentsRef.current : [], - sessions, - commits, - repoKey: path, - }, - labelsRef.current ?? DEFAULT_GROUP_LABELS - ) - }, []) + let files: FlatFileEntry[] = [] + let sessions: DbConversationSummary[] = [] + let commits: GitLogEntry[] = [] + const current = () => + !signal?.aborted && + !abort.signal.aborted && + enabledRef.current && + pathRef.current === path && + generationRef.current === generation + const groups = () => + buildReferenceGroups( + query, + { + files, + workspaceRoot: path, + agents: wants("agent") ? agentsRef.current : [], + assistants: wants("assistant") ? assistantsRef.current : [], + assistantAgents: wants("assistant") ? agentsRef.current : [], + sessions, + commits, + repoKey: path, + }, + labelsRef.current ?? DEFAULT_GROUP_LABELS + ).map((group) => + pendingKinds.has(group.kind) ? { ...group, loading: true } : group + ) + const publish = () => { + if (current()) onUpdate?.(groups()) + } + // Loaded agents/assistants are useful immediately, even if native file + // discovery takes a minute. Each lazy source then publishes independently. + publish() + try { + await Promise.all([ + filesPromise.then((value) => { + files = value + pendingKinds.delete("file") + publish() + }), + sessionsPromise.then((value) => { + sessions = value + pendingKinds.delete("session") + publish() + }), + commitsPromise.then((value) => { + commits = value + pendingKinds.delete("commit") + publish() + }), + ]) + return current() ? groups() : [] + } finally { + signal?.removeEventListener("abort", onAbort) + if (fileSearchRef.current === abort) fileSearchRef.current = null + } + }, + [] + ) } diff --git a/src/components/conversations/search-command-dialog.tsx b/src/components/conversations/search-command-dialog.tsx index 6dbcd551b77..6726c1cfd70 100644 --- a/src/components/conversations/search-command-dialog.tsx +++ b/src/components/conversations/search-command-dialog.tsx @@ -88,7 +88,8 @@ export function SearchCommandDialog({ reset: resetFileTree, } = useFileTree({ folderPath: folderPath || undefined, - enabled: activeTab === "files", + enabled: open && activeTab === "files", + query, }) // Compute which agent types exist in current folder @@ -96,8 +97,8 @@ export function SearchCommandDialog({ new Set(conversations.map((c) => c.agent_type)) ).sort(compareAgentType) - // Rank files by relevance (name/path tiers + fuzzy subsequence), scanning the - // full list so a deeply nested match isn't crowded out by shallower ones. + // The backend ranks the whole workspace before capping the response, so a + // deeply nested match remains reachable without retaining the full index. const filteredFiles = useMemo( () => rankFileMatches(query, allFiles, 100), [allFiles, query] diff --git a/src/hooks/use-file-tree.test.ts b/src/hooks/use-file-tree.test.ts new file mode 100644 index 00000000000..611c25be531 --- /dev/null +++ b/src/hooks/use-file-tree.test.ts @@ -0,0 +1,163 @@ +import { act, renderHook } from "@testing-library/react" +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import type { WorkspaceFileEntry } from "@/lib/types" +import { useFileTree } from "./use-file-tree" + +const api = vi.hoisted(() => ({ prepare: vi.fn(), search: vi.fn() })) +vi.mock("@/lib/api", () => ({ + prepareWorkspaceFileSearch: api.prepare, + searchWorkspaceFiles: api.search, +})) + +function file(path: string): WorkspaceFileEntry { + const parts = path.split("/") + return { name: parts[parts.length - 1], path, kind: "file" } +} + +function deferred() { + let resolve!: (files: WorkspaceFileEntry[]) => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +describe("bounded workspace file search", () => { + beforeEach(() => { + vi.useFakeTimers() + api.prepare.mockReset().mockResolvedValue(undefined) + api.search.mockReset().mockResolvedValue([]) + }) + afterEach(() => { + vi.useRealTimers() + }) + + it("does no discovery while the picker is disabled", async () => { + renderHook(() => + useFileTree({ folderPath: "/repo", enabled: false, query: "index" }) + ) + await act(async () => { + await vi.advanceTimersByTimeAsync(1000) + }) + expect(api.prepare).not.toHaveBeenCalled() + expect(api.search).not.toHaveBeenCalled() + }) + + it("debounces typing and requests bounded matches for only the latest query", async () => { + const { rerender } = renderHook( + (query: string) => + useFileTree({ folderPath: "/repo", enabled: true, query }), + { initialProps: "a" } + ) + rerender("auth") + await act(async () => { + await vi.advanceTimersByTimeAsync(150) + }) + expect(api.prepare).toHaveBeenCalledOnce() + expect(api.search).toHaveBeenCalledExactlyOnceWith("/repo", "auth", { + limit: 100, + filesOnly: false, + refresh: false, + request: { + consumerId: expect.any(String), + generation: expect.any(Number), + }, + signal: expect.any(AbortSignal), + }) + }) + + it("drops a response from the previous workspace even when it resolves last", async () => { + const old = deferred() + api.search.mockImplementation((path: string) => + path === "/old" ? old.promise : Promise.resolve([file("new.ts")]) + ) + const { result, rerender } = renderHook( + (folderPath: string) => + useFileTree({ folderPath, enabled: true, query: "ts" }), + { initialProps: "/old" } + ) + await act(async () => { + await vi.advanceTimersByTimeAsync(150) + }) + rerender("/new") + await act(async () => { + await vi.advanceTimersByTimeAsync(150) + }) + await act(async () => { + old.resolve([file("old.ts")]) + }) + expect(result.current.allFiles.map((f) => f.relativePath)).toEqual([ + "new.ts", + ]) + expect(result.current.loading).toBe(false) + }) + + it("reset invalidates in-flight results and requests a fresh native search", async () => { + const old = deferred() + api.search + .mockReturnValueOnce(old.promise) + .mockResolvedValue([file("fresh.ts")]) + const { result } = renderHook(() => + useFileTree({ folderPath: "/repo", enabled: true, query: "ts" }) + ) + await act(async () => { + await vi.advanceTimersByTimeAsync(150) + }) + act(() => result.current.reset(true)) + await act(async () => { + await vi.advanceTimersByTimeAsync(150) + }) + await act(async () => { + old.resolve([file("obsolete.ts")]) + }) + expect(api.search).toHaveBeenLastCalledWith( + "/repo", + "ts", + expect.objectContaining({ refresh: true }) + ) + expect(result.current.allFiles.map((f) => f.relativePath)).toEqual([ + "fresh.ts", + ]) + }) + + it("closing and reopening clears local results without invalidating a warm index", async () => { + const { result, rerender } = renderHook( + (enabled: boolean) => + useFileTree({ folderPath: "/repo", enabled, query: "ts" }), + { initialProps: false } + ) + act(() => result.current.reset()) + rerender(true) + await act(async () => { + await vi.advanceTimersByTimeAsync(150) + }) + expect(api.search).toHaveBeenLastCalledWith( + "/repo", + "ts", + expect.objectContaining({ refresh: false }) + ) + }) + + it("cancels backend work for the old query and keeps a per-picker consumer", async () => { + api.search.mockReturnValue(new Promise(() => {})) + const { rerender, unmount } = renderHook( + (query: string) => + useFileTree({ folderPath: "/repo", enabled: true, query }), + { initialProps: "a" } + ) + await act(async () => { + await vi.advanceTimersByTimeAsync(150) + }) + const first = api.search.mock.calls[0][2] + rerender("ab") + expect(first.signal.aborted).toBe(true) + await act(async () => { + await vi.advanceTimersByTimeAsync(150) + }) + const second = api.search.mock.calls[1][2] + expect(second.request.consumerId).toBe(first.request.consumerId) + expect(second.request.generation).toBeGreaterThan(first.request.generation) + unmount() + expect(second.signal.aborted).toBe(true) + }) +}) diff --git a/src/hooks/use-file-tree.ts b/src/hooks/use-file-tree.ts index e1d9a9ba35b..57011876fa2 100644 --- a/src/hooks/use-file-tree.ts +++ b/src/hooks/use-file-tree.ts @@ -1,8 +1,9 @@ "use client" import { useState, useEffect, useRef, useCallback } from "react" -import { listWorkspaceFiles } from "@/lib/api" +import { prepareWorkspaceFileSearch, searchWorkspaceFiles } from "@/lib/api" import type { WorkspaceFileEntry } from "@/lib/types" +import { randomUUID } from "@/lib/utils" export interface FlatFileEntry { name: string @@ -18,47 +19,74 @@ export interface FlatFileEntry { interface UseFileTreeOptions { folderPath: string | undefined enabled: boolean + query: string + limit?: number + filesOnly?: boolean } interface UseFileTreeResult { allFiles: FlatFileEntry[] loading: boolean loaded: boolean - /** Clear cached data so the next `enabled=true` triggers a fresh load. */ - reset: () => void + /** Discard local results; an explicit refresh also invalidates native data. */ + reset: (refreshIndex?: boolean) => void } /** - * Loads a flat, gitignore-aware listing of every file/dir under `folderPath` - * (lazily, when `enabled`) for in-memory file search — shared by the search - * dialog and the composer `@`-mention picker. - * - * Discovery and gitignore filtering run on the backend (`list_workspace_files`), - * which prunes ignored directories *during* the walk and applies no depth cap, - * so deeply nested files are reachable while `node_modules`/`target`/… are never - * descended. The result is cached per folder path; a folder switch keeps showing - * the previous list until the new one loads (`loaded` gates that transition). + * Keep only the current query's bounded matches in the renderer. Discovery, + * indexing and ranking are shared in the backend, with a native cache budget. */ export function useFileTree({ folderPath, enabled, + query, + limit = 100, + filesOnly = false, }: UseFileTreeOptions): UseFileTreeResult { const [allFiles, setAllFiles] = useState([]) const [loading, setLoading] = useState(false) const loadedForPathRef = useRef(null) + const [revision, setRevision] = useState(0) + const generation = useRef(0) + const refresh = useRef(false) + const consumer = useRef(undefined) useEffect(() => { - if (!enabled || !folderPath) return - if (loadedForPathRef.current === folderPath) return + if (enabled && folderPath) { + void prepareWorkspaceFileSearch(folderPath).catch(() => {}) + } + }, [enabled, folderPath]) + useEffect(() => { + const current = ++generation.current + loadedForPathRef.current = null + setAllFiles([]) + if (!enabled || !folderPath) { + setLoading(false) + return + } let canceled = false + const abort = new AbortController() setLoading(true) async function load() { try { - const files: WorkspaceFileEntry[] = await listWorkspaceFiles( - folderPath! + const files: WorkspaceFileEntry[] = await searchWorkspaceFiles( + folderPath!, + query, + { + limit, + filesOnly, + refresh: refresh.current, + request: { + consumerId: (consumer.current ??= randomUUID()), + generation: current, + }, + signal: abort.signal, + } ) + if (canceled || generation.current !== current) return + if (query.trim()) refresh.current = false const flat: FlatFileEntry[] = files.map((f) => ({ name: f.name, relativePath: f.path, @@ -67,26 +95,31 @@ export function useFileTree({ lowerName: f.name.toLowerCase(), })) - if (!canceled) { + if (!canceled && generation.current === current) { setAllFiles(flat) loadedForPathRef.current = folderPath! } } catch { - if (!canceled) setAllFiles([]) + if (!canceled && generation.current === current) setAllFiles([]) } finally { - if (!canceled) setLoading(false) + if (!canceled && generation.current === current) setLoading(false) } } - void load() + const timer = setTimeout(() => void load(), query.trim() ? 150 : 0) return () => { canceled = true + abort.abort() + clearTimeout(timer) } - }, [enabled, folderPath]) + }, [enabled, folderPath, query, limit, filesOnly, revision]) - const reset = useCallback(() => { + const reset = useCallback((refreshIndex = false) => { loadedForPathRef.current = null + generation.current += 1 + refresh.current ||= refreshIndex setAllFiles([]) + setRevision((value) => value + 1) }, []) return { diff --git a/src/lib/api-workspace-search.test.ts b/src/lib/api-workspace-search.test.ts new file mode 100644 index 00000000000..09cf6b295d3 --- /dev/null +++ b/src/lib/api-workspace-search.test.ts @@ -0,0 +1,71 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { searchWorkspaceFiles } from "./api" + +const { call } = vi.hoisted(() => ({ call: vi.fn() })) +vi.mock("@/lib/transport", () => ({ + getTransport: () => ({ call }), +})) + +describe("workspace search transport", () => { + beforeEach(() => call.mockReset().mockResolvedValue([])) + afterEach(() => vi.unstubAllGlobals()) + + it("generates consumer IDs on plain HTTP origins without crypto.randomUUID", async () => { + vi.stubGlobal("crypto", { + getRandomValues: crypto.getRandomValues.bind(crypto), + }) + await searchWorkspaceFiles("/repo", "readme") + expect(call.mock.calls[0][1].request.consumerId).toMatch( + /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/ + ) + }) + + it("covers native queue and traversal deadlines in web and remote desktop calls", async () => { + await searchWorkspaceFiles("/repo", "readme", { limit: 51 }) + expect(call).toHaveBeenCalledExactlyOnceWith( + "search_workspace_files", + { + path: "/repo", + query: "readme", + limit: 51, + request: { consumerId: expect.any(String), generation: 0 }, + }, + { timeoutMs: 330_000 } + ) + }) + + it("propagates abort to the captured backend without serializing the signal", async () => { + let resolve!: (entries: []) => void + call.mockReturnValueOnce( + new Promise((done) => { + resolve = done + }) + ) + const abort = new AbortController() + const request = { consumerId: "picker", generation: 2 } + const pending = searchWorkspaceFiles("/repo", "readme", { + request, + signal: abort.signal, + }) + abort.abort() + expect(call).toHaveBeenLastCalledWith("cancel_workspace_file_search", { + request, + }) + expect(call.mock.calls[0][1]).not.toHaveProperty("signal") + resolve([]) + await pending + }) + + it("skips an already obsolete query and removes abort listeners after completion", async () => { + const old = new AbortController() + old.abort() + expect( + await searchWorkspaceFiles("/repo", "old", { signal: old.signal }) + ).toEqual([]) + expect(call).not.toHaveBeenCalled() + const finished = new AbortController() + await searchWorkspaceFiles("/repo", "new", { signal: finished.signal }) + finished.abort() + expect(call).toHaveBeenCalledOnce() + }) +}) diff --git a/src/lib/api.ts b/src/lib/api.ts index c7250ff793d..7b9803618d1 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -12,6 +12,7 @@ import { unauthorizedSource, } from "./transport/web-auth" import { notifyWebUnauthorized } from "./transport/web-connection-store" +import { randomUUID } from "./utils" import { getCurrentEffectiveAppLocale } from "./i18n" import { DEFAULT_FORGE_COMMENT_PAGE_SIZE, @@ -5028,9 +5029,9 @@ export async function getFileTree( } /** - * Flat, gitignore-aware listing of every file/dir under `path`. Ignored - * directories are pruned during the backend walk (no depth cap), so deeply - * nested files are reachable while the payload stays small. Used by file search. + * Enumerate every non-ignored file/dir under `path`, without a depth cap. + * This can return millions of entries; search surfaces use searchWorkspaceFiles + * to keep the renderer's payload and retained state bounded. */ export async function listWorkspaceFiles( path: string @@ -5038,6 +5039,56 @@ export async function listWorkspaceFiles( return getTransport().call("list_workspace_files", { path }) } +/** Warm a shared, budgeted native index without sending its entries to JS. */ +export async function prepareWorkspaceFileSearch(path: string): Promise { + return getTransport().call("prepare_workspace_file_search", { path }) +} + +/** Rank across the workspace and transfer only a bounded set of matches. */ +export async function searchWorkspaceFiles( + path: string, + query: string, + options: { + limit?: number + filesOnly?: boolean + refresh?: boolean + request?: { consumerId: string; generation: number } + signal?: AbortSignal + } = {} +): Promise { + const { + signal, + request = { consumerId: randomUUID(), generation: 0 }, + ...params + } = options + if (signal?.aborted) return [] + const transport = getTransport() + const cancel = () => { + void transport + .call("cancel_workspace_file_search", { request }) + .catch(() => {}) + } + signal?.addEventListener("abort", cancel, { once: true }) + try { + return await transport.call( + "search_workspace_files", + { + path, + query, + ...params, + request, + }, + { + // The backend bounds queue wait + traversal to five minutes. Give both + // HTTP transports time to receive that result, including a cold build. + timeoutMs: 330_000, + } + ) + } finally { + signal?.removeEventListener("abort", cancel) + } +} + export async function startWorkspaceStateStream( rootPath: string, wantsTreeGit = true From 5467b82e4146105a1688e960a245cc85bf6c1169 Mon Sep 17 00:00:00 2001 From: kogekiplay <46434871+kogekiplay@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:13:18 +0800 Subject: [PATCH 2/3] fix(search): defer workspace discovery until a foreground query --- docs/workspace-search-design.md | 20 +++++++++++++++ .../src/commands/workspace_file_search.rs | 25 +++---------------- .../composer/use-reference-search.test.ts | 4 +-- .../chat/composer/use-reference-search.ts | 10 +------- src/hooks/use-file-tree.test.ts | 2 +- src/hooks/use-file-tree.ts | 8 +----- 6 files changed, 29 insertions(+), 40 deletions(-) diff --git a/docs/workspace-search-design.md b/docs/workspace-search-design.md index 436035f9fc9..aebb13f789f 100644 --- a/docs/workspace-search-design.md +++ b/docs/workspace-search-design.md @@ -7,6 +7,26 @@ and traversal under a global mutex are still present; the proposed replacement below must land and pass the acceptance tests before this PR becomes ready. The WebKit, Tauri callback, and highlighting splits are independent of this work. +## Current experiment: no eager discovery + +The 0.42.2 experiment initially included only the other three splits. It still +issued full `list_workspace_files` requests when composers mounted. An isolated +read-only client issued eight of those requests during startup; a previous +measurement of the user's workspace counted 5,308,089 entries and a 970,417,831 +byte response. That path is sufficient to multiply gigabytes of path strings +and objects across composers. + +The experimental integration now uses this draft's bounded native queries. +Mounting a composer or picker never schedules a prewarm. The legacy prepare +endpoint is a no-op for compatibility; only an actual query owns a discovery +and cancellation token. Closed search dialogs do not request results. + +This removes eager full-workspace payloads from the renderer. It does not +resolve this draft's remaining single-root cache, global query admission, or +repeated over-budget traversal issues. Those remain merge blockers. Reintroduce +background prewarming only after the lifecycle/scheduling design below is +implemented and validated. + ## Foreground work must not wait for background traversal Separate the short-held cache/registry mutex from traversal and scoring. The diff --git a/src-tauri/src/commands/workspace_file_search.rs b/src-tauri/src/commands/workspace_file_search.rs index c6f548f84cf..9430b716853 100644 --- a/src-tauri/src/commands/workspace_file_search.rs +++ b/src-tauri/src/commands/workspace_file_search.rs @@ -676,29 +676,12 @@ pub async fn cancel_workspace_file_search(request: SearchRequest) -> Result<(), Ok(()) } -/// Prewarm only native, budgeted storage. Return no file list to the renderer. +/// Compatibility endpoint for older clients. Discovery is now owned by an +/// actual foreground query and its cancellation token; mounting a composer +/// must never start an uncancellable full-workspace traversal. #[cfg_attr(feature = "tauri-runtime", tauri::command)] pub async fn prepare_workspace_file_search(path: String) { - if path.trim().is_empty() { - return; - } - let Ok(mut cache) = CACHE.try_lock() else { - return; - }; - tokio::task::spawn_blocking(move || { - search_or_build( - &mut cache, - path, - SearchOptions { - query: "", - limit: 0, - files_only: false, - refresh: false, - }, - INDEX_BUDGET, - &SearchControl::default(), - ); - }); + drop(path); } #[cfg(test)] diff --git a/src/components/chat/composer/use-reference-search.test.ts b/src/components/chat/composer/use-reference-search.test.ts index 795b3353092..f87e45bce11 100644 --- a/src/components/chat/composer/use-reference-search.test.ts +++ b/src/components/chat/composer/use-reference-search.test.ts @@ -461,11 +461,11 @@ describe("useReferenceSearch", () => { .mockResolvedValue({ entries: [], has_upstream: false }) }) - it("prepares native storage without transferring files until the file group is requested", async () => { + it("starts no discovery or prewarm until the file group is requested", async () => { const { result } = renderHook(() => useReferenceSearch({ defaultPath: "/repo" }) ) - expect(mocks.prepareWorkspaceFileSearch).toHaveBeenCalledWith("/repo") + expect(mocks.prepareWorkspaceFileSearch).not.toHaveBeenCalled() expect(mocks.searchWorkspaceFiles).not.toHaveBeenCalled() await act(async () => { await result.current("a", undefined, ["agent"]) diff --git a/src/components/chat/composer/use-reference-search.ts b/src/components/chat/composer/use-reference-search.ts index c034af92e0c..aa0678f2b5e 100644 --- a/src/components/chat/composer/use-reference-search.ts +++ b/src/components/chat/composer/use-reference-search.ts @@ -4,12 +4,7 @@ import { useCallback, useEffect, useLayoutEffect, useRef } from "react" import { useAcpAgents } from "@/hooks/use-acp-agents" import type { FlatFileEntry } from "@/hooks/use-file-tree" -import { - gitLog, - listAllConversations, - prepareWorkspaceFileSearch, - searchWorkspaceFiles, -} from "@/lib/api" +import { gitLog, listAllConversations, searchWorkspaceFiles } from "@/lib/api" import type { AcpAgentInfo, AssistantInfo, @@ -260,9 +255,6 @@ export function useReferenceSearch({ }: UseReferenceSearchOptions): ReferenceSearch { const path = defaultPath || null - useEffect(() => { - if (enabled && path) void prepareWorkspaceFileSearch(path).catch(() => {}) - }, [enabled, path]) const { agents } = useAcpAgents() const assistants = useAssistants() diff --git a/src/hooks/use-file-tree.test.ts b/src/hooks/use-file-tree.test.ts index 611c25be531..d19ac19d99c 100644 --- a/src/hooks/use-file-tree.test.ts +++ b/src/hooks/use-file-tree.test.ts @@ -53,7 +53,7 @@ describe("bounded workspace file search", () => { await act(async () => { await vi.advanceTimersByTimeAsync(150) }) - expect(api.prepare).toHaveBeenCalledOnce() + expect(api.prepare).not.toHaveBeenCalled() expect(api.search).toHaveBeenCalledExactlyOnceWith("/repo", "auth", { limit: 100, filesOnly: false, diff --git a/src/hooks/use-file-tree.ts b/src/hooks/use-file-tree.ts index 57011876fa2..9aab95fc915 100644 --- a/src/hooks/use-file-tree.ts +++ b/src/hooks/use-file-tree.ts @@ -1,7 +1,7 @@ "use client" import { useState, useEffect, useRef, useCallback } from "react" -import { prepareWorkspaceFileSearch, searchWorkspaceFiles } from "@/lib/api" +import { searchWorkspaceFiles } from "@/lib/api" import type { WorkspaceFileEntry } from "@/lib/types" import { randomUUID } from "@/lib/utils" @@ -51,12 +51,6 @@ export function useFileTree({ const refresh = useRef(false) const consumer = useRef(undefined) - useEffect(() => { - if (enabled && folderPath) { - void prepareWorkspaceFileSearch(folderPath).catch(() => {}) - } - }, [enabled, folderPath]) - useEffect(() => { const current = ++generation.current loadedForPathRef.current = null From c4c095c8b8697f0c34e34db40df44ac280b9d4ca Mon Sep 17 00:00:00 2001 From: kogekiplay <46434871+kogekiplay@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:46:32 +0800 Subject: [PATCH 3/3] fix(search): schedule queries independently and reuse bounded root snapshots --- docs/workspace-search-design.md | 254 +++---- src-tauri/src/commands/folders.rs | 2 +- .../src/commands/workspace_file_search.rs | 515 ++----------- .../workspace_file_search/index_cache.rs | 619 +++++++++++++++ .../index_cache/tests.rs | 710 ++++++++++++++++++ src/lib/api.ts | 2 +- 6 files changed, 1534 insertions(+), 568 deletions(-) create mode 100644 src-tauri/src/commands/workspace_file_search/index_cache.rs create mode 100644 src-tauri/src/commands/workspace_file_search/index_cache/tests.rs diff --git a/docs/workspace-search-design.md b/docs/workspace-search-design.md index aebb13f789f..4303a721745 100644 --- a/docs/workspace-search-design.md +++ b/docs/workspace-search-design.md @@ -1,129 +1,125 @@ -# Workspace search: scheduling and cache revision (proposal) - -This is the fourth, **draft** split of #1097. The current compact native index, -query cancellation, server/desktop APIs, and bounded result transfer remain here -for review. This branch is not ready to merge or install. The single-root cache -and traversal under a global mutex are still present; the proposed replacement -below must land and pass the acceptance tests before this PR becomes ready. -The WebKit, Tauri callback, and highlighting splits are independent of this work. - -## Current experiment: no eager discovery - -The 0.42.2 experiment initially included only the other three splits. It still -issued full `list_workspace_files` requests when composers mounted. An isolated -read-only client issued eight of those requests during startup; a previous -measurement of the user's workspace counted 5,308,089 entries and a 970,417,831 -byte response. That path is sufficient to multiply gigabytes of path strings -and objects across composers. - -The experimental integration now uses this draft's bounded native queries. -Mounting a composer or picker never schedules a prewarm. The legacy prepare -endpoint is a no-op for compatibility; only an actual query owns a discovery -and cancellation token. Closed search dialogs do not request results. - -This removes eager full-workspace payloads from the renderer. It does not -resolve this draft's remaining single-root cache, global query admission, or -repeated over-budget traversal issues. Those remain merge blockers. Reintroduce -background prewarming only after the lifecycle/scheduling design below is -implemented and validated. - -## Foreground work must not wait for background traversal - -Separate the short-held cache/registry mutex from traversal and scoring. The -registry holds per-canonical-root state and shared immutable snapshots; a worker -never retains its mutex while touching the filesystem. Canonical roots include -the authorized linked-root/ignore configuration in their cache identity. - -Use a dispatcher with at most two discovery workers: one foreground-reserved -slot and one shared slot. Admit at most one background prewarm, and preempt its -cancellation token when foreground discovery needs capacity. Empty queries only -read and return a bounded top-level listing; they do not wait for a full index. -Scoring an already built snapshot does not wait for discovery admission. -Coalesce discovery for the same root. Queries attach to that root's discovery, -while cancellation removes only that consumer; it cancels the traversal when -no consumer or explicit retained refresh remains. A small-root foreground query -can proceed even while a large root is being prepared. - -Preemption can discard an incomplete snapshot. Only successful, complete walks -are published. Worker completion must verify the root generation before updating -the registry, so a late cancelled build cannot replace a newer snapshot. Keep -cancellation/deadline checks in discovery and scoring, and cancel queued jobs -before acquiring a worker permit. - -## Prewarm belongs to a visible consumer - -Give prepare requests the same consumer ID/generation contract as queries, plus -a dedicated prepare cancellation endpoint for desktop and server modes. The -file picker and mention panel own an AbortController for their prepare lifecycle; -close, disable, unmount, root change, or a new generation detaches it. A consumer -query can promote an existing prewarm to foreground rather than repeat its walk. -Use bounded registry tombstones to reject a cancelled request that arrives late. -Prewarm is an optimization and must never outlive all of its consumers. - -## Reuse roots within one global budget - -Replace the single cache slot with an LRU of snapshots keyed by canonical root. -Retain at most 128 MiB across **all** root snapshots and at most eight roots; do -not assign 128 MiB to each root. Include deduplicated strings, node arrays, and -metadata in accounting. Queries hold an Arc lease; eviction removes its cache -ownership, while active leases continue counting against the process-wide -budget until released. Budget reservations therefore precede construction, not -only publication. Cap simultaneous discovery allocations globally at 192 MiB -and bound foreground scoring scratch space by result limit. - -A -> B -> A within the TTL should reuse both complete snapshots if they fit the -total budget. Refresh timestamps change only after a real successful discovery, -not after fallback queries. Root/config invalidation creates a new generation; -stale refresh results cannot overwrite it. Disk and memory expiry are explicit. - -## Oversized roots reuse a disk snapshot - -While walking, incrementally write a temporary SQLite snapshot (rusqlite is -already a dependency). Keep a compact memory index only while its reservation -permits. When the memory builder reaches its share of the global build budget, -drop that builder and continue the same discovery into the disk snapshot. This -produces complete results without rescanning the filesystem on each keystroke. -Publish the disk snapshot atomically only on successful completion. Bind its -temporary-file lifetime to the snapshot lease and remove abandoned files on -startup. Use a bounded global SQLite page cache and a disk quota/LRU as well; -never convert a disk result into a full renderer-side file list. - -Repeated oversized-root queries scan the cached rows in bounded pages and keep -only top-k scores. They remain cancellable between pages. This still has O(N) -scoring cost, which should be measured; it removes repeated filesystem discovery -and bounds retained RAM. If disk quota/storage prevents a complete snapshot, -return a retriable index-unavailable error with backoff, rather than marking a -partial tree complete or silently restarting a full walk for every keypress. -Incremental updates and a trigram index can be evaluated later if measurements -justify them. - -## UI regressions fixed in this draft - -Keyboard and pointer navigation pin the active suggestion group. Selection -tracks reference identity within that group across progressive updates, with a -valid fallback when an item is removed. Delayed file arrival cannot reinterpret -a selected session as a file or invalidate Enter. Window focus invalidates the -lazy session/commit caches for the next query while keeping the valid current -request alive. The ES2020 test helper no longer uses Array.at. - -## Required acceptance before marking ready - -- Block a large root's prewarm inside discovery; a small-root foreground and an - empty listing must complete before that block is released. -- Close the only consumer during queued and active prewarm. Assert cancellation, - no partial snapshot publication, no stale event delivery, and released permits. - With two consumers, closing one must preserve the other's work. -- Query A, B, A; count one discovery per root, including root aliases/config - changes and LRU eviction. Assert the *total* retained and active-lease budgets. -- Force the memory budget small; issue several oversized-root queries and assert - one filesystem walk, complete late-file matches, bounded SQLite/cache memory, - cancellation during scoring, and temporary-file cleanup on failure/eviction. -- Race cancellation, root change, foreground promotion, and TTL refresh; verify - registry generations and no starvation. Instrument queue time separately from - discovery/scoring time for a realistic large workspace. -- Keep the keyboard progressive-result and window-focus regressions green. - Run full TypeScript including tests, ESLint, Vitest, production build, and - desktop/server Rust checks/tests/clippy against the final implementation. - -The remaining backend design/implementation is a merge blocker **only for this -draft**. The other three split PRs can be reviewed and released independently. +# Native workspace search + +Search surfaces never enumerate an entire workspace in the renderer. The +desktop and HTTP commands rank natively and return at most 200 entries. The +ranking tiers, UTF-16 offsets, deterministic tie order, ignore rules and +authorized linked-directory behavior match the previous file search. + +## Discovery belongs to a query + +Mounting a composer or picker starts no discovery. The legacy prepare command +is a compatibility no-op. Empty queries return a bounded traversal prefix using +two listing permits, independently of index discovery and nonempty scoring. + +Nonempty queries share a process-wide `IndexService`. A short-held registry +mutex protects an LRU of complete snapshots, in-progress discovery jobs, and +temporary failures. Traversal, file I/O and scoring run on blocking workers +outside this lock. Discovery has two permits, scoring four, and empty listings +two: one large discovery cannot block a small-root discovery or an empty +listing. Permit acquisition is cancellable and bounded by the query deadline. +There are no background jobs competing with foreground requests. + +Queries for the same root attach to one discovery job. Each waiting query owns +a lease. Closing one consumer removes that lease without interrupting another +consumer; when the last leaves, the discovery cancellation token is signalled. +Queued work exits without acquiring a permit. Running work checks cancellation +between entries and never publishes a partial snapshot. A cancelled worker +cannot publish over a newer job for the same root: publication checks the job +identity under the registry lock. Dropping an async search also drops its lease. + +Consumer IDs and generations reject stale requests and handle cancellation that +arrives before its query. The consumer registry and failure backoff map are +bounded to 128 entries. A five-minute cooperative deadline covers admission, +discovery and scoring; the transport allows 330 seconds for the response. + +## Multiple roots share total budgets + +The LRU holds at most eight complete roots. Keys use the canonical workspace +path, current authorized link names/targets, and root/linked-root ignore-file +metadata. Aliases share a snapshot. Authorization and root-ignore changes select +a different snapshot immediately; a configuration change during discovery +invalidates that build. Other filesystem changes become visible after a +five-minute TTL or explicit refresh. Refresh publishes only after a complete +successful discovery and does not move the timestamp on each query. + +Compact in-memory snapshots deduplicate basenames and store integer parent +references. All roots share a 128 MiB retained-index budget. Each of the two +discovery workers can use at most half the 192 MiB compact-builder budget, +including its interning table. The cache reserves space before publication. +Evicted snapshots still held by active queries remain charged until their final +lease drops, so eviction cannot admit more than the total budget. When memory +admission is unavailable, the complete disk snapshot remains usable instead. + +Scoring retains only top-k candidates. The compact scorer's basename memoization +is limited to 4,096 entries per worker, rather than allocating one score for +every unique name in every concurrent query. The limits describe index-owned +allocations; filesystem walkers, OS page cache, thread stacks and the app's other +subsystems have separate overhead and are not represented as a total RSS limit. + +## Oversized roots reuse a complete disk snapshot + +Discovery writes a length-prefixed binary snapshot in traversal order while +optionally building the compact memory index. Once the compact budget is +exhausted, it drops that builder and continues the same walk to disk. Every +entry remains searchable, including late/deep exact matches. There is no +full-workspace payload or fallback list allocated in the renderer. + +The disk format contains UTF-8 basename/path lengths, directory kind and their +bytes. A query reads records in 64 KiB buffered pages and keeps only the best +matches. Independent positional-read cursors allow concurrent queries on one +snapshot without shared-seek races. No database dependency, migration, persistent +index directory or full-path array is needed. + +All roots and active disk builds share a 2 GiB disk quota, with up to 1 GiB +reserved for each discovery. Successful snapshots shrink their reservation to +their actual bytes. LRU eviction releases ownership; active leases continue to +count until close. A record is capped at 1 MiB. Quota or I/O failure yields an +error and a five-second retry backoff. It never publishes an incomplete index or +silently rescans the filesystem for each keystroke after a failed build. + +Snapshots use `tempfile::tempfile`: unlinked at creation on Unix and +delete-on-close on Windows. Cancellation, construction errors, eviction and +process exit close them automatically, including abrupt termination without an +application-managed stale-file sweep. Successful compact-index publication +closes the temporary disk copy as well. + +Disk scoring remains O(N) in snapshot records; initial discovery still walks +the filesystem. This change removes repeated filesystem discovery within the +TTL and bounds retained data, rather than claiming constant-time search in +multi-million-entry workspaces. Incremental updates and a secondary text index +can be evaluated separately using measured latency. + +## UI lifecycle + +Keyboard and pointer navigation pin the suggestion group. Selection follows +reference identity across progressive updates, with a valid fallback after +deletion. Delayed file results cannot turn a selected session into a file or +invalidate Enter. Focus clears lazy session/commit caches for the next query +while keeping the current in-flight search valid. + +Picker closure, root changes, newer typing and hook unmount abort queries. +Frontend state retains only the current bounded results. There is no +Array.at dependency beyond the project's ES2020 target. + +## Regression coverage + +`workspace_file_search/index_cache/tests.rs` uses independent services and +controlled workers, including: + +- A blocked large-root discovery while a small-root query and empty listing + complete, plus empty listing with both discovery permits held. +- Same-root coalescing, one of two consumers closing, last-consumer and queued + cancellation, dropped queries, deadlines, and cancelled old-generation + completion racing its successor. +- A -> B -> A reuse, canonical aliases, authorization changes, ignore metadata, + TTL, refresh and LRU eviction. +- Tiny forced memory budgets with complete late matches from disk, one walk + across repeated queries, and concurrent positional reads. +- Memory and disk quotas with active evicted leases, disk overflow cleanup, + retry backoff, cancelled disk scoring, Unicode/path ranking and stable ties. + +The frontend tests cover bounded/cancelled transport requests, no eager +prewarm, picker closure, delayed groups, selection reorder/deletion and focus. +An ignored `large_workspace_memory_probe` can be run with +`CODEG_MEMORY_WORKSPACE` to report counts, response bytes, memory/disk accounting, +walk counts and query timings without printing workspace filenames. diff --git a/src-tauri/src/commands/folders.rs b/src-tauri/src/commands/folders.rs index af4f101dc5d..bd3cad16282 100644 --- a/src-tauri/src/commands/folders.rs +++ b/src-tauri/src/commands/folders.rs @@ -4872,7 +4872,7 @@ pub async fn list_directory_with_files( /// Only *authorized* links are returned (see [`crate::folder_links`]): a /// symlink that merely happens to sit in the tree — the `secrets -> ~/.ssh` a /// cloned repo might ship — is not one of them and stays unfollowed. -fn authorized_links_in(root: &Path) -> Vec<(String, PathBuf)> { +pub(super) fn authorized_links_in(root: &Path) -> Vec<(String, PathBuf)> { let Ok(canonical_root) = std::fs::canonicalize(root) else { return Vec::new(); }; diff --git a/src-tauri/src/commands/workspace_file_search.rs b/src-tauri/src/commands/workspace_file_search.rs index 9430b716853..776b2e8e392 100644 --- a/src-tauri/src/commands/workspace_file_search.rs +++ b/src-tauri/src/commands/workspace_file_search.rs @@ -9,7 +9,6 @@ use std::sync::{Arc, LazyLock, Mutex as StdMutex}; use std::time::{Duration, Instant}; use serde::Deserialize; -use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; use super::folders::{visit_workspace_files, WorkspaceEntryKind, WorkspaceFileEntry}; @@ -23,21 +22,10 @@ const SEARCH_TIMEOUT: Duration = Duration::from_secs(300); const MAX_CONSUMERS: usize = 128; const NO_PARENT: u32 = u32::MAX; -#[derive(Clone, Copy)] -struct IndexBudget { - build: usize, - retained: usize, -} +mod index_cache; +use index_cache::{IndexService, RootKey}; -const INDEX_BUDGET: IndexBudget = IndexBudget { - build: BUILD_BYTES, - retained: INDEX_BYTES, -}; - -// One workspace cache and one builder/search worker for the whole process, -// shared by every composer, dialog, desktop window and web client. Switching -// roots drops the previous index before building the next one. -static CACHE: LazyLock>> = LazyLock::new(|| Mutex::new(None)); +static INDEXES: LazyLock> = LazyLock::new(IndexService::production); static SEARCHES: LazyLock> = LazyLock::new(|| StdMutex::new(Searches::default())); @@ -133,7 +121,7 @@ impl SearchControl { struct ActiveSearch { request: SearchRequest, - control: SearchControl, + control: Arc, } impl Drop for ActiveSearch { @@ -153,12 +141,6 @@ impl Drop for ActiveSearch { } } -struct CachedIndex { - root: String, - built_at: Instant, - index: Option, -} - // Store each basename once and each parent as a small integer. In a workspace // containing many checkouts/archives this avoids retaining the same long path // prefix (and basename) millions of times. @@ -204,11 +186,9 @@ impl CompactIndex { let mut matches = Matches::new(query, limit, files_only); // Repeated basenames share normalization and name-tier scoring too. // A full set of name matches outranks every path/subsequence match. - let name_scores: Vec<_> = self - .names - .iter() - .map(|name| score_name(&matches.query, &name.lower, name.utf16_len)) - .collect(); + // A cache sized by all unique names would multiply scratch memory by + // each concurrent query. Cache only a fixed number of basename scores. + let mut name_scores = HashMap::new(); let query_mask = matches.query.is_ascii().then(|| byte_mask(&matches.query)); let mut ancestors: Vec<(u32, usize, usize, u64)> = Vec::new(); let mut path = String::new(); @@ -228,7 +208,12 @@ impl CompactIndex { let mask = ancestors.last().map(|(_, _, _, mask)| *mask).unwrap_or(0) | name.byte_mask | if parent_len > 0 { byte_mask("/") } else { 0 }; - let name_score = name_scores[node.name as usize]; + let name_score = *name_scores + .entry(node.name) + .or_insert_with(|| score_name(&matches.query, &name.lower, name.utf16_len)); + if name_scores.len() >= 4096 { + name_scores.clear(); + } let may_match = (!files_only || !node.is_dir) && match name_score { Some(score) => matches.accepts(score, id), @@ -302,7 +287,15 @@ impl Builder { .as_ref() .map(|lower| entry.name.len() + 32 + lower.capacity() + 16) .unwrap_or(0); - if self.index.bytes() + extra_nodes + extra_names + table_bytes + string_bytes > budget + let depth = entry.path.bytes().filter(|c| *c == b'/').count(); + let parents_capacity = self.parents.capacity().max(depth + 1); + if self.index.bytes() + + extra_nodes + + extra_names + + table_bytes + + string_bytes + + parents_capacity * size_of::() + > budget || self.index.nodes.len() >= u32::MAX as usize { return false; @@ -330,7 +323,6 @@ impl Builder { id } }; - let depth = entry.path.bytes().filter(|c| *c == b'/').count(); self.parents.truncate(depth); if self.parents.len() != depth { return false; @@ -342,6 +334,9 @@ impl Builder { is_dir: entry.kind == WorkspaceEntryKind::Dir, }); if entry.kind == WorkspaceEntryKind::Dir { + if self.parents.len() == self.parents.capacity() { + self.parents.reserve_exact(1); + } self.parents.push(id); } true @@ -394,8 +389,22 @@ impl Matches { if self.limit == 0 || (self.files_only && is_dir) { return; } - let Some(score) = score_file(&self.query, name, path) else { - return; + let lower_name = name.to_lowercase(); + let name_len = lower_name.encode_utf16().count(); + let score = if let Some(score) = score_name(&self.query, &lower_name, name_len) { + score + } else { + // Once top-k contains only basename hits, path and subsequence + // tiers cannot enter it. In disk snapshots this avoids allocating + // and lowercasing millions of irrelevant long ancestor paths. + if !self.needs_path_matches() { + return; + } + let Some(score) = score_path(&self.query, &lower_name, name_len, &path.to_lowercase()) + else { + return; + }; + score }; self.offer_ranked(name, path, is_dir, order, score); } @@ -466,6 +475,7 @@ fn subsequence(query: &str, text: &str) -> Option { } // Same tiers and UTF-16 positions/lengths as lib/file-search-match.ts. +#[cfg(test)] fn score_file(query: &str, name: &str, path: &str) -> Option { let name = name.to_lowercase(); let name_len = name.encode_utf16().count(); @@ -508,78 +518,6 @@ struct SearchOptions<'a> { refresh: bool, } -fn search_or_build( - cache: &mut Option, - root: String, - options: SearchOptions<'_>, - budget: IndexBudget, - control: &SearchControl, -) -> Vec { - let SearchOptions { - query, - limit, - files_only, - refresh, - } = options; - if control.stopped() { - return Vec::new(); - } - let fresh = cache - .as_ref() - .is_some_and(|c| c.root == root && c.built_at.elapsed() < CACHE_TTL); - if fresh && !refresh { - if limit == 0 { - return Vec::new(); - } - if let Some(index) = cache.as_ref().and_then(|c| c.index.as_ref()) { - return index.search(query, limit, files_only, control); - } - } - let should_build = !fresh || refresh; - if should_build { - *cache = None; - } - let mut builder = should_build.then(Builder::default); - let mut matches = Matches::new(query, limit, files_only); - let mut order = 0; - visit_workspace_files(&root, |entry| { - if control.stopped() { - return false; - } - matches.offer( - &entry.name, - &entry.path, - entry.kind == WorkspaceEntryKind::Dir, - order, - ); - order += 1; - if builder - .as_mut() - .is_some_and(|b| !b.push(&entry, budget.build)) - { - builder = None; - } - // A warmup can stop once the budget is exhausted. A real search keeps - // walking and ranking so a late/deep exact match is never omitted. - builder.is_some() || limit != 0 - }); - if control.stopped() { - // A partial traversal must never become a fresh, incomplete index. - return Vec::new(); - } - if should_build { - let index = builder - .map(|b| b.index) - .filter(|i| i.bytes() <= budget.retained); - *cache = Some(CachedIndex { - root, - built_at: Instant::now(), - index, - }); - } - matches.finish() -} - #[cfg_attr(feature = "tauri-runtime", tauri::command)] pub async fn search_workspace_files( path: String, @@ -595,10 +533,6 @@ pub async fn search_workspace_files( "Invalid workspace file search", )); } - let limit = limit.unwrap_or(100).clamp(1, MAX_RESULTS); - let files_only = files_only.unwrap_or(false); - // Register before admission: a newer generation cancels both queued and - // running work for this consumer, without affecting another picker/client. let request = request.unwrap_or_else(|| SearchRequest { consumer_id: uuid::Uuid::new_v4().to_string(), generation: 0, @@ -609,65 +543,40 @@ pub async fn search_workspace_files( }; let search = ActiveSearch { request, - control: SearchControl { + control: Arc::new(SearchControl { cancel, deadline: requested_at + SEARCH_TIMEOUT, - }, + }), }; - let mut cache = tokio::select! { - biased; - _ = search.control.cancel.cancelled() => return Ok(Vec::new()), - _ = tokio::time::sleep_until(search.control.deadline.into()) => { - return Err(AppCommandError::task_execution_failed("Workspace file search timed out")); - } - cache = CACHE.lock() => cache, - }; - // A warmup or another refresh may have rebuilt the index while this - // request waited. Reuse that newer snapshot instead of scanning twice. - let refresh = - refresh.unwrap_or(false) && cache.as_ref().is_none_or(|c| c.built_at < requested_at); - tokio::task::spawn_blocking(move || { - let control = &search.control; - let results = if query.trim().is_empty() { - // Even files-only listings can walk an entire tree of directories. - // They share the worker, cancellation and deadline with searches. - let mut entries = Vec::new(); - visit_workspace_files(&path, |entry| { - if control.stopped() { - return false; - } - if !files_only || entry.kind == WorkspaceEntryKind::File { - entries.push(entry); - } - entries.len() < limit - }); - entries - } else { - search_or_build( - &mut cache, - path, - SearchOptions { - query: &query, - limit, - files_only, - refresh, - }, - INDEX_BUDGET, - control, - ) - }; - if Instant::now() >= control.deadline { - Err(AppCommandError::task_execution_failed( - "Workspace file search timed out", - )) - } else if control.cancel.is_cancelled() { - Ok(Vec::new()) - } else { - Ok(results) - } - }) - .await - .map_err(|e| AppCommandError::task_execution_failed(e.to_string()))? + // Canonicalization and linked-root authorization can touch the filesystem. + // They run off the async executor, before looking up the short-held registry. + let key = tokio::task::spawn_blocking(move || RootKey::resolve(&path)) + .await + .map_err(|e| AppCommandError::task_execution_failed(e.to_string()))? + .map_err(AppCommandError::task_execution_failed)?; + let results = INDEXES + .search( + key, + SearchOptions { + query: &query, + limit: limit.unwrap_or(100).clamp(1, MAX_RESULTS), + files_only: files_only.unwrap_or(false), + refresh: refresh.unwrap_or(false), + }, + requested_at, + Arc::clone(&search.control), + ) + .await + .map_err(AppCommandError::task_execution_failed)?; + if Instant::now() >= search.control.deadline { + Err(AppCommandError::task_execution_failed( + "Workspace file search timed out", + )) + } else if search.control.cancel.is_cancelled() { + Ok(Vec::new()) + } else { + Ok(results) + } } #[cfg_attr(feature = "tauri-runtime", tauri::command)] @@ -688,29 +597,6 @@ pub async fn prepare_workspace_file_search(path: String) { mod tests { use super::*; - fn run_search( - cache: &mut Option, - root: String, - query: &str, - limit: usize, - files_only: bool, - refresh: bool, - budget: IndexBudget, - ) -> Vec { - search_or_build( - cache, - root, - SearchOptions { - query, - limit, - files_only, - refresh, - }, - budget, - &SearchControl::default(), - ) - } - fn file(path: &str, is_dir: bool) -> WorkspaceFileEntry { WorkspaceFileEntry { name: path.rsplit('/').next().unwrap().into(), @@ -818,68 +704,6 @@ mod tests { assert_eq!(best[1].path, "0/needle-extra"); } - #[test] - fn budget_fallback_still_searches_the_entire_workspace() { - let dir = tempfile::tempdir().unwrap(); - for i in 0..100 { - std::fs::write(dir.path().join(format!("a{i:03}.txt")), "x").unwrap(); - } - std::fs::create_dir_all(dir.path().join("z/deep")).unwrap(); - std::fs::write(dir.path().join("z/deep/needle.txt"), "x").unwrap(); - let mut cache = None; - let root = dir.path().to_string_lossy().into_owned(); - let budget = IndexBudget { - build: 1024, - retained: 1024, - }; - let best = run_search(&mut cache, root.clone(), "needle", 1, true, false, budget); - assert_eq!(best[0].path, "z/deep/needle.txt"); - assert!(cache.as_ref().unwrap().index.is_none()); - assert_eq!( - run_search(&mut cache, root, "needle", 1, true, false, budget), - best - ); - } - - #[test] - fn budget_fallback_preserves_failed_build_ttl_and_eventually_retries() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("needle.txt"), "x").unwrap(); - let root = dir.path().to_string_lossy().into_owned(); - let mut cache = None; - run_search( - &mut cache, - root.clone(), - "needle", - 1, - true, - false, - IndexBudget { - build: 1024, - retained: 1024, - }, - ); - let marker = Instant::now() - Duration::from_secs(290); - cache.as_mut().unwrap().built_at = marker; - for _ in 0..3 { - let result = run_search( - &mut cache, - root.clone(), - "needle", - 1, - true, - false, - INDEX_BUDGET, - ); - assert_eq!(result[0].path, "needle.txt"); - assert_eq!(cache.as_ref().unwrap().built_at, marker); - assert!(cache.as_ref().unwrap().index.is_none()); - } - cache.as_mut().unwrap().built_at = Instant::now() - CACHE_TTL; - run_search(&mut cache, root, "needle", 1, true, false, INDEX_BUDGET); - assert!(cache.as_ref().unwrap().index.is_some()); - } - fn request(consumer: &str, generation: u64) -> SearchRequest { SearchRequest { consumer_id: consumer.into(), @@ -935,160 +759,14 @@ mod tests { assert_eq!(searches.0.len(), MAX_CONSUMERS); } - #[test] - fn canceled_or_expired_searches_do_not_replace_a_cached_index() { - let mut cache = Some(CachedIndex { - root: "existing".into(), - built_at: Instant::now(), - index: None, - }); - let canceled = SearchControl::default(); - canceled.cancel.cancel(); - let expired = SearchControl { - deadline: Instant::now() - Duration::from_secs(1), - ..SearchControl::default() - }; - for control in [canceled, expired] { - assert!(search_or_build( - &mut cache, - "replacement".into(), - SearchOptions { - query: "needle", - limit: 1, - files_only: true, - refresh: true - }, - INDEX_BUDGET, - &control, - ) - .is_empty()); - assert_eq!(cache.as_ref().unwrap().root, "existing"); - } - } - #[tokio::test] - async fn empty_query_shares_admission_and_can_cancel_while_waiting() { - let cache = CACHE.lock().await; - let request = request(&uuid::Uuid::new_v4().to_string(), 1); - let search_request = request.clone(); - let task = tokio::spawn(search_workspace_files( - "/unused-because-canceled".into(), - "".into(), - Some(200), - Some(true), - None, - Some(search_request), - )); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if SEARCHES - .lock() - .unwrap() - .0 - .contains_key(&request.consumer_id) - { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - assert!( - !task.is_finished(), - "empty queries must wait for the shared worker" - ); - cancel_workspace_file_search(request).await.unwrap(); - let result = tokio::time::timeout(Duration::from_secs(1), task) - .await - .unwrap() - .unwrap() - .unwrap(); - assert!(result.is_empty()); - drop(cache); - } - - #[tokio::test] - async fn newer_generation_coalesces_a_queued_search_before_it_can_traverse() { - let cache = CACHE.lock().await; - let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("latest.txt"), "x").unwrap(); - let root = dir.path().to_string_lossy().into_owned(); - let first = request(&uuid::Uuid::new_v4().to_string(), 1); - let latest = request(&first.consumer_id, 2); - let old_task = tokio::spawn(search_workspace_files( - root.clone(), - "obsolete".into(), - None, - None, - None, - Some(first.clone()), - )); - tokio::time::timeout(Duration::from_secs(1), async { - while !SEARCHES.lock().unwrap().0.contains_key(&first.consumer_id) { - tokio::task::yield_now().await; - } - }) + async fn legacy_prepare_does_no_background_work() { + tokio::time::timeout( + Duration::from_secs(1), + prepare_workspace_file_search("/not-a-workspace-and-must-not-be-touched".into()), + ) .await .unwrap(); - let latest_task = tokio::spawn(search_workspace_files( - root, - "latest".into(), - None, - None, - None, - Some(latest), - )); - // The old request completes while the worker is still held: it never - // starts a superseded traversal or waits for the newer one's scan. - assert!(tokio::time::timeout(Duration::from_secs(1), old_task) - .await - .unwrap() - .unwrap() - .unwrap() - .is_empty()); - assert!(!latest_task.is_finished()); - drop(cache); - let results = latest_task.await.unwrap().unwrap(); - assert_eq!(results[0].path, "latest.txt"); - } - - #[test] - fn refresh_and_root_changes_replace_cached_entries() { - let first = tempfile::tempdir().unwrap(); - let second = tempfile::tempdir().unwrap(); - std::fs::write(first.path().join("old.ts"), "x").unwrap(); - std::fs::write(second.path().join("other.ts"), "x").unwrap(); - let mut cache = None; - let root = first.path().to_string_lossy().into_owned(); - run_search(&mut cache, root.clone(), "", 0, false, false, INDEX_BUDGET); - std::fs::write(first.path().join("new.ts"), "x").unwrap(); - assert!(run_search( - &mut cache, - root.clone(), - "new", - 10, - true, - false, - INDEX_BUDGET - ) - .is_empty()); - assert_eq!( - run_search(&mut cache, root, "new", 10, true, true, INDEX_BUDGET)[0].path, - "new.ts" - ); - let result = run_search( - &mut cache, - second.path().to_string_lossy().into_owned(), - "ts", - 10, - true, - false, - INDEX_BUDGET, - ); - assert_eq!(result.len(), 1); - assert_eq!(result[0].path, "other.ts"); - assert!(cache.as_ref().unwrap().index.as_ref().unwrap().bytes() <= INDEX_BYTES); } #[tokio::test] @@ -1111,41 +789,4 @@ mod tests { assert_eq!(results.len(), MAX_RESULTS); assert!(results.iter().all(|e| e.kind == WorkspaceEntryKind::File)); } - - /// Manual end-to-end probe for unusually large workspaces. The path stays - /// local; output contains counts/budgets/timings rather than filenames. - #[test] - #[ignore = "set CODEG_MEMORY_WORKSPACE to profile a real workspace"] - fn large_workspace_memory_probe() { - let root = std::env::var("CODEG_MEMORY_WORKSPACE").expect("CODEG_MEMORY_WORKSPACE"); - let started = Instant::now(); - let mut cache = None; - println!("probe_pid={}", std::process::id()); - run_search(&mut cache, root.clone(), "", 0, false, false, INDEX_BUDGET); - let warm_seconds = started.elapsed().as_secs_f64(); - let stats = cache.as_ref().and_then(|c| c.index.as_ref()).map(|i| { - serde_json::json!({"entries":i.nodes.len(),"uniqueNames":i.names.len(),"retainedBytes":i.bytes()}) - }); - println!( - "{}", - serde_json::json!({"warmSeconds":warm_seconds,"index":stats}) - ); - for query in ["README.md", "index.ts", "workspace_file_search"] { - let started = Instant::now(); - let results = run_search( - &mut cache, - root.clone(), - query, - 100, - true, - false, - INDEX_BUDGET, - ); - assert!(results.len() <= 100); - println!( - "{}", - serde_json::json!({"query":query,"results":results.len(),"responseBytes":serde_json::to_vec(&results).unwrap().len(),"seconds":started.elapsed().as_secs_f64()}) - ); - } - } } diff --git a/src-tauri/src/commands/workspace_file_search/index_cache.rs b/src-tauri/src/commands/workspace_file_search/index_cache.rs new file mode 100644 index 00000000000..83dd8e97e31 --- /dev/null +++ b/src-tauri/src/commands/workspace_file_search/index_cache.rs @@ -0,0 +1,619 @@ +//! Short-lived registry locks, coalesced query-owned discovery, and a global +//! LRU budget. Oversized roots use an unnamed temporary file: subsequent +//! searches read this complete snapshot, never walk the filesystem again. + +use std::collections::{HashMap, VecDeque}; +use std::fs::File; +use std::io::{self, BufReader, BufWriter, Read, Write}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, SystemTime}; + +use tokio::sync::{watch, Semaphore}; + +use super::super::folders::authorized_links_in; +use super::{ + visit_workspace_files, Builder, CompactIndex, Matches, SearchControl, SearchOptions, + WorkspaceEntryKind, WorkspaceFileEntry, BUILD_BYTES, CACHE_TTL, INDEX_BYTES, MAX_CONSUMERS, +}; + +const ROOT_LIMIT: usize = 8; +const WORKERS: usize = 2; +const QUERY_WORKERS: usize = 4; +const DISK_BYTES: usize = 2 * 1024 * 1024 * 1024; +const SNAPSHOT_DISK_BYTES: usize = 1024 * 1024 * 1024; +const RECORD_BYTES: usize = 1024 * 1024; +const ERROR_BACKOFF: Duration = Duration::from_secs(5); + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub(super) struct RootKey { + path: PathBuf, + // Linked-root authorization changes must take effect before the TTL. + links: Vec<(String, PathBuf)>, + ignore_files: Vec>, +} + +impl RootKey { + pub(super) fn resolve(path: &str) -> Result { + let path = std::fs::canonicalize(path).map_err(|e| e.to_string())?; + if path.to_str().is_none() { + return Err("Workspace search path is not UTF-8".into()); + } + if !path.is_dir() { + return Err("Workspace search root is not a directory".into()); + } + let links = authorized_links_in(&path); + let mut ignore_files = Vec::new(); + for root in std::iter::once(&path).chain(links.iter().map(|(_, path)| path)) { + for file in [".gitignore", ".ignore", ".git/info/exclude"] { + ignore_files.push( + std::fs::metadata(root.join(file)) + .ok() + .and_then(|m| Some((m.modified().ok()?, m.len()))), + ); + } + } + Ok(Self { + path, + links, + ignore_files, + }) + } + + fn path(&self) -> &str { + // Input paths already cross the String-based desktop/HTTP boundary. + self.path.to_str().expect("canonical UTF-8 workspace path") + } +} + +#[derive(Clone, Copy)] +struct Limits { + memory: usize, + build: usize, + disk: usize, + disk_per_root: usize, + roots: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { + memory: INDEX_BYTES, + build: BUILD_BYTES, + disk: DISK_BYTES, + disk_per_root: SNAPSHOT_DISK_BYTES, + roots: ROOT_LIMIT, + } + } +} + +struct Quota { + limit: usize, + used: AtomicUsize, +} + +impl Quota { + fn new(limit: usize) -> Arc { + Arc::new(Self { + limit, + used: AtomicUsize::new(0), + }) + } + + fn reserve(self: &Arc, bytes: usize) -> Option { + let mut used = self.used.load(Ordering::Acquire); + loop { + let next = used.checked_add(bytes).filter(|next| *next <= self.limit)?; + match self + .used + .compare_exchange_weak(used, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => break, + Err(current) => used = current, + } + } + Some(Reservation { + quota: Arc::clone(self), + bytes, + }) + } +} + +struct Reservation { + quota: Arc, + bytes: usize, +} + +impl Reservation { + fn shrink(&mut self, bytes: usize) { + assert!(bytes <= self.bytes); + self.quota + .used + .fetch_sub(self.bytes - bytes, Ordering::AcqRel); + self.bytes = bytes; + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + self.quota.used.fetch_sub(self.bytes, Ordering::AcqRel); + } +} + +enum Storage { + Memory { + index: CompactIndex, + _charge: Reservation, + }, + Disk { + file: Arc, + bytes: usize, + _charge: Reservation, + }, +} + +pub(super) struct Snapshot { + built_at: Instant, + storage: Storage, +} + +impl Snapshot { + fn search( + &self, + query: &str, + limit: usize, + files_only: bool, + control: &SearchControl, + ) -> Result, String> { + match &self.storage { + Storage::Memory { index, .. } => Ok(index.search(query, limit, files_only, control)), + Storage::Disk { file, bytes, .. } => { + let mut reader = BufReader::with_capacity( + 64 * 1024, + PositionedReader { + file: Arc::clone(file), + offset: 0, + }, + ); + let mut matches = Matches::new(query, limit, files_only); + let mut read = 0; + let mut order = 0; + // Positional reads give every query an independent cursor; + // File::try_clone + seek would share offsets on Unix. + while read < *bytes { + if control.stopped() { + return Ok(Vec::new()); + } + let mut header = [0u8; 9]; + reader.read_exact(&mut header).map_err(|e| e.to_string())?; + let name_len = u32::from_le_bytes(header[..4].try_into().unwrap()) as usize; + let path_len = u32::from_le_bytes(header[4..8].try_into().unwrap()) as usize; + let len = name_len + .checked_add(path_len) + .filter(|len| *len <= RECORD_BYTES) + .ok_or("Invalid workspace snapshot record")?; + if len + 9 > bytes - read { + return Err("Truncated workspace snapshot".into()); + } + let mut text = vec![0; len]; + reader.read_exact(&mut text).map_err(|e| e.to_string())?; + let name = std::str::from_utf8(&text[..name_len]).map_err(|e| e.to_string())?; + let path = std::str::from_utf8(&text[name_len..]).map_err(|e| e.to_string())?; + matches.offer(name, path, header[8] != 0, order); + read += 9 + len; + order += 1; + } + Ok(matches.finish()) + } + } + } +} + +struct PositionedReader { + file: Arc, + offset: u64, +} + +impl Read for PositionedReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + #[cfg(unix)] + let bytes = std::os::unix::fs::FileExt::read_at(&*self.file, buf, self.offset)?; + #[cfg(windows)] + let bytes = std::os::windows::fs::FileExt::seek_read(&*self.file, buf, self.offset)?; + self.offset += bytes as u64; + Ok(bytes) + } +} + +#[derive(Clone)] +enum BuildError { + Cancelled, + Failed(Arc), +} + +impl From for BuildError { + fn from(error: String) -> Self { + Self::Failed(error.into()) + } +} + +type BuildResult = Result, BuildError>; +type Walker = dyn Fn(&str, &mut dyn FnMut(WorkspaceFileEntry) -> bool) + Send + Sync; + +struct BuildJob { + control: SearchControl, + users: AtomicUsize, + result: watch::Sender>, +} + +struct BuildLease(Arc); + +impl Drop for BuildLease { + fn drop(&mut self) { + if self.0.users.fetch_sub(1, Ordering::AcqRel) == 1 { + self.0.control.cancel.cancel(); + } + } +} + +#[derive(Default)] +struct Registry { + // Front is least recently used. Only complete snapshots are published. + snapshots: VecDeque<(RootKey, Arc)>, + jobs: HashMap>, + failures: HashMap)>, +} + +pub(super) struct IndexService { + limits: Limits, + registry: Mutex, + memory: Arc, + disk: Arc, + discovery: Arc, + scoring: Arc, + listing: Arc, + walk: Arc, +} + +impl IndexService { + pub(super) fn production() -> Arc { + Self::new( + Limits::default(), + Arc::new(|root, visit| visit_workspace_files(root, visit)), + ) + } + + fn new(limits: Limits, walk: Arc) -> Arc { + Arc::new(Self { + limits, + registry: Mutex::new(Registry::default()), + memory: Quota::new(limits.memory), + disk: Quota::new(limits.disk), + discovery: Arc::new(Semaphore::new(WORKERS)), + scoring: Arc::new(Semaphore::new(QUERY_WORKERS)), + listing: Arc::new(Semaphore::new(WORKERS)), + walk, + }) + } + + fn reserve(&self, quota: &Arc, bytes: usize) -> Option { + if bytes > quota.limit { + return None; + } + let mut registry = self.registry.lock().unwrap(); + loop { + if let Some(charge) = quota.reserve(bytes) { + return Some(charge); + } + // An in-flight query can still own an evicted snapshot. Its lease + // keeps counting until it actually drops; eviction never lies + // about the process-wide total to admit a new allocation. + registry.snapshots.pop_front()?; + } + } + + async fn snapshot( + self: &Arc, + key: RootKey, + refresh: bool, + requested_at: Instant, + control: &SearchControl, + ) -> Result>, String> { + loop { + if control.stopped() { + return Ok(None); + } + let lease = + { + let mut registry = self.registry.lock().unwrap(); + registry + .snapshots + .retain(|(_, s)| s.built_at.elapsed() < CACHE_TTL); + registry + .failures + .retain(|_, (at, _)| at.elapsed() < ERROR_BACKOFF); + if let Some(index) = registry.snapshots.iter().position(|(root, s)| { + root == &key && (!refresh || s.built_at >= requested_at) + }) { + let item = registry.snapshots.remove(index).unwrap(); + let snapshot = Arc::clone(&item.1); + registry.snapshots.push_back(item); + return Ok(Some(snapshot)); + } + if let Some((_, error)) = registry.failures.get(&key) { + return Err(error.to_string()); + } + let job = registry + .jobs + .get(&key) + .filter(|job| !job.control.stopped()) + .cloned(); + let job = job.unwrap_or_else(|| { + let (result, _) = watch::channel(None); + let job = Arc::new(BuildJob { + control: SearchControl::default(), + users: AtomicUsize::new(0), + result, + }); + registry.jobs.insert(key.clone(), Arc::clone(&job)); + self.start_build(key.clone(), Arc::clone(&job)); + job + }); + job.users.fetch_add(1, Ordering::AcqRel); + BuildLease(job) + }; + let mut result = lease.0.result.subscribe(); + let completed = loop { + if let Some(done) = result.borrow().clone() { + break done; + } + tokio::select! { + biased; + _ = control.cancel.cancelled() => return Ok(None), + _ = tokio::time::sleep_until(control.deadline.into()) => return Ok(None), + changed = result.changed() => { + if changed.is_err() { return Err("Workspace discovery worker stopped".into()); } + } + } + }; + match completed { + Ok(snapshot) => return Ok(Some(snapshot)), + Err(BuildError::Failed(error)) => return Err(error.to_string()), + // A last-consumer cancellation racing a new query must not + // strand that new query on the cancelled job's result. + Err(BuildError::Cancelled) => continue, + } + } + } + + fn start_build(self: &Arc, key: RootKey, job: Arc) { + let service = Arc::clone(self); + tokio::spawn(async move { + let permit = tokio::select! { + biased; + _ = job.control.cancel.cancelled() => None, + _ = tokio::time::sleep_until(job.control.deadline.into()) => None, + permit = service.discovery.clone().acquire_owned() => permit.ok(), + }; + let result = if let Some(permit) = permit { + let worker = Arc::clone(&service); + let worker_key = key.clone(); + let worker_job = Arc::clone(&job); + match tokio::task::spawn_blocking(move || { + let _permit = permit; + worker.build(&worker_key, &worker_job.control) + }) + .await + { + Ok(result) => result, + Err(error) => Err(BuildError::Failed(error.to_string().into())), + } + } else { + Err(BuildError::Cancelled) + }; + { + let mut registry = service.registry.lock().unwrap(); + // A cancelled worker may finish after its successor. It owns + // neither the successor's registry entry nor its generation. + if registry + .jobs + .get(&key) + .is_some_and(|current| Arc::ptr_eq(current, &job)) + { + registry.jobs.remove(&key); + if !job.control.stopped() { + match &result { + Ok(snapshot) => { + registry.snapshots.retain(|(root, _)| root != &key); + registry + .snapshots + .push_back((key.clone(), Arc::clone(snapshot))); + while registry.snapshots.len() > service.limits.roots { + registry.snapshots.pop_front(); + } + } + Err(BuildError::Failed(error)) => { + registry + .failures + .retain(|_, (at, _)| at.elapsed() < ERROR_BACKOFF); + if registry.failures.len() >= MAX_CONSUMERS { + if let Some(oldest) = registry + .failures + .iter() + .min_by_key(|(_, (at, _))| *at) + .map(|(root, _)| root.clone()) + { + registry.failures.remove(&oldest); + } + } + registry + .failures + .insert(key, (Instant::now(), Arc::clone(error))); + } + Err(BuildError::Cancelled) => {} + } + } + } + } + job.result.send_replace(Some(result)); + }); + } + + fn build(&self, key: &RootKey, control: &SearchControl) -> BuildResult { + if control.stopped() { + return Err(BuildError::Cancelled); + } + // Reserve disk before writing, counting active snapshots and builds. + // If active leases prevent admission, fail with backoff; never silently + // restart an uncached full traversal for every keystroke. + let mut disk = self + .reserve(&self.disk, self.limits.disk_per_root) + .ok_or_else(|| { + BuildError::Failed("Workspace snapshot disk budget is in use; retry shortly".into()) + })?; + // tempfile() is unlinked immediately on Unix and delete-on-close on + // Windows. Cancellation, errors, eviction, and process exit all close + // it without leaving named files to accumulate after crashes. + let file = tempfile::tempfile().map_err(|e| BuildError::from(e.to_string()))?; + let mut writer = BufWriter::with_capacity(64 * 1024, file); + let mut bytes = 0usize; + let mut builder = Some(Builder::default()); + let mut error = None; + // Two discovery permits, each at most half the global build budget. + let build_budget = self.limits.build / WORKERS; + (self.walk)(key.path(), &mut |entry| { + if control.stopped() { + return false; + } + let len = entry.name.len().saturating_add(entry.path.len()); + if len > RECORD_BYTES + || len.saturating_add(9) > self.limits.disk_per_root.saturating_sub(bytes) + { + error = Some("Workspace snapshot exceeded its disk budget".to_owned()); + return false; + } + let write = (|| -> io::Result<()> { + writer.write_all(&(entry.name.len() as u32).to_le_bytes())?; + writer.write_all(&(entry.path.len() as u32).to_le_bytes())?; + writer.write_all(&[u8::from(entry.kind == WorkspaceEntryKind::Dir)])?; + writer.write_all(entry.name.as_bytes())?; + writer.write_all(entry.path.as_bytes()) + })(); + if let Err(e) = write { + error = Some(e.to_string()); + return false; + } + bytes += len + 9; + if builder + .as_mut() + .is_some_and(|b| !b.push(&entry, build_budget)) + { + builder = None; + } + true + }); + if control.stopped() { + return Err(BuildError::Cancelled); + } + if let Some(error) = error { + return Err(error.into()); + } + // Authorization/root ignore changes during discovery invalidate the + // whole result instead of publishing a mixture of configurations. + if RootKey::resolve(key.path()).map_err(BuildError::from)? != *key { + return Err(BuildError::Failed( + "Workspace configuration changed during discovery; retry".into(), + )); + } + writer + .flush() + .map_err(|e| BuildError::from(e.to_string()))?; + let storage = if let Some((index, charge)) = builder.and_then(|b| { + let bytes = b.index.bytes(); + self.reserve(&self.memory, bytes) + .map(|charge| (b.index, charge)) + }) { + Storage::Memory { + index, + _charge: charge, + } + } else { + disk.shrink(bytes); + Storage::Disk { + file: Arc::new( + writer + .into_inner() + .map_err(|e| BuildError::from(e.to_string()))?, + ), + bytes, + _charge: disk, + } + }; + Ok(Arc::new(Snapshot { + built_at: Instant::now(), + storage, + })) + } + + pub(super) async fn search( + self: &Arc, + key: RootKey, + options: SearchOptions<'_>, + requested_at: Instant, + control: Arc, + ) -> Result, String> { + let SearchOptions { + query, + limit, + files_only, + refresh, + } = options; + let empty = query.trim().is_empty(); + let query = query.to_owned(); + let snapshot = if empty { + None + } else { + self.snapshot(key.clone(), refresh, requested_at, &control) + .await? + }; + if control.stopped() { + return Ok(Vec::new()); + } + let pool = if empty { &self.listing } else { &self.scoring }; + let permit = tokio::select! { + biased; + _ = control.cancel.cancelled() => return Ok(Vec::new()), + _ = tokio::time::sleep_until(control.deadline.into()) => return Ok(Vec::new()), + permit = pool.clone().acquire_owned() => permit.map_err(|e| e.to_string())?, + }; + let service = Arc::clone(self); + tokio::task::spawn_blocking(move || { + let _permit = permit; + if let Some(snapshot) = snapshot { + snapshot.search(&query, limit, files_only, &control) + } else { + let mut entries = Vec::new(); + (service.walk)(key.path(), &mut |entry| { + if control.stopped() { + return false; + } + if !files_only || entry.kind == WorkspaceEntryKind::File { + entries.push(entry); + } + entries.len() < limit + }); + Ok(if control.stopped() { + Vec::new() + } else { + entries + }) + } + }) + .await + .map_err(|e| e.to_string())? + } +} + +#[cfg(test)] +mod tests; diff --git a/src-tauri/src/commands/workspace_file_search/index_cache/tests.rs b/src-tauri/src/commands/workspace_file_search/index_cache/tests.rs new file mode 100644 index 00000000000..391f1263d95 --- /dev/null +++ b/src-tauri/src/commands/workspace_file_search/index_cache/tests.rs @@ -0,0 +1,710 @@ +use super::*; +use std::path::Path; +use std::sync::atomic::AtomicBool; +use tokio::sync::Notify; + +fn entry(path: &str, dir: bool) -> WorkspaceFileEntry { + WorkspaceFileEntry { + name: path.rsplit('/').next().unwrap().into(), + path: path.into(), + kind: if dir { + WorkspaceEntryKind::Dir + } else { + WorkspaceEntryKind::File + }, + } +} + +async fn search(service: &Arc, key: RootKey, query: &str) -> Vec { + service + .search( + key, + SearchOptions { + query, + limit: 10, + files_only: true, + refresh: false, + }, + Instant::now(), + Arc::new(SearchControl::default()), + ) + .await + .unwrap() +} + +fn counted(limits: Limits) -> (Arc, Arc) { + let visits = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&visits); + let service = IndexService::new( + limits, + Arc::new(move |path, visit| { + count.fetch_add(1, Ordering::SeqCst); + visit_workspace_files(path, visit) + }), + ); + (service, visits) +} + +struct Gate { + closed: AtomicBool, + started: Notify, +} + +impl Gate { + fn new() -> Arc { + Arc::new(Self { + closed: AtomicBool::new(true), + started: Notify::new(), + }) + } + fn wait(&self) { + self.started.notify_one(); + let start = Instant::now(); + while self.closed.load(Ordering::Acquire) && start.elapsed() < Duration::from_secs(5) { + std::thread::sleep(Duration::from_millis(2)); + } + } + fn release(&self) { + self.closed.store(false, Ordering::Release); + } +} + +#[tokio::test] +async fn large_discovery_does_not_block_small_root_or_empty_listing() { + let large = tempfile::tempdir().unwrap(); + let small = tempfile::tempdir().unwrap(); + std::fs::write(small.path().join("needle.ts"), "x").unwrap(); + let blocked = std::fs::canonicalize(large.path()).unwrap(); + let gate = Gate::new(); + let worker_gate = Arc::clone(&gate); + let service = IndexService::new( + Limits::default(), + Arc::new(move |path, visit| { + if Path::new(path) == blocked { + worker_gate.wait(); + } + visit_workspace_files(path, visit); + }), + ); + let worker = Arc::clone(&service); + let large_key = RootKey::resolve(large.path().to_str().unwrap()).unwrap(); + let pending = tokio::spawn(async move { search(&worker, large_key, "needle").await }); + tokio::time::timeout(Duration::from_secs(2), gate.started.notified()) + .await + .unwrap(); + let small_key = RootKey::resolve(small.path().to_str().unwrap()).unwrap(); + let result = tokio::time::timeout( + Duration::from_secs(1), + search(&service, small_key.clone(), "needle"), + ) + .await + .unwrap(); + assert_eq!(result[0].path, "needle.ts"); + let empty = tokio::time::timeout(Duration::from_secs(1), search(&service, small_key, "")) + .await + .unwrap(); + assert_eq!(empty[0].path, "needle.ts"); + assert!(!pending.is_finished()); + gate.release(); + pending.await.unwrap(); +} + +#[tokio::test] +async fn empty_listing_proceeds_with_both_discovery_permits_held() { + let (service, _) = counted(Limits::default()); + let one = service.discovery.clone().acquire_owned().await.unwrap(); + let two = service.discovery.clone().acquire_owned().await.unwrap(); + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("a.ts"), "x").unwrap(); + let key = RootKey::resolve(root.path().to_str().unwrap()).unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), search(&service, key, "")) + .await + .unwrap() + .len(), + 1 + ); + drop((one, two)); +} + +#[tokio::test] +async fn roots_a_b_a_and_canonical_aliases_reuse_complete_indexes() { + let (service, visits) = counted(Limits::default()); + let a = tempfile::tempdir().unwrap(); + let b = tempfile::tempdir().unwrap(); + for root in [&a, &b] { + std::fs::write(root.path().join("needle"), "x").unwrap(); + } + for root in [&a, &b, &a] { + let result = search( + &service, + RootKey::resolve(root.path().to_str().unwrap()).unwrap(), + "needle", + ) + .await; + assert_eq!(result[0].path, "needle"); + } + let alias = format!("{}/.", a.path().display()); + search(&service, RootKey::resolve(&alias).unwrap(), "needle").await; + assert_eq!(visits.load(Ordering::SeqCst), 2); + assert_eq!(service.registry.lock().unwrap().snapshots.len(), 2); + assert!(service.memory.used.load(Ordering::Acquire) <= INDEX_BYTES); +} + +#[tokio::test] +async fn oversized_roots_are_complete_and_reused_without_another_walk() { + let limits = Limits { + memory: 1024, + build: 2048, + disk: 4 * 1024 * 1024, + disk_per_root: 2 * 1024 * 1024, + ..Limits::default() + }; + let (service, visits) = counted(limits); + let root = tempfile::tempdir().unwrap(); + for i in 0..300 { + std::fs::write(root.path().join(format!("a{i:03}")), "x").unwrap(); + } + std::fs::create_dir_all(root.path().join("z/deep")).unwrap(); + std::fs::write(root.path().join("z/deep/needle.ts"), "x").unwrap(); + let key = RootKey::resolve(root.path().to_str().unwrap()).unwrap(); + for query in ["needle", "needle.ts", "ndle", "z/deep"] { + let result = search(&service, key.clone(), query).await; + assert_eq!(result[0].path, "z/deep/needle.ts"); + } + assert_eq!(visits.load(Ordering::SeqCst), 1); + assert!(matches!( + service.registry.lock().unwrap().snapshots[0].1.storage, + Storage::Disk { .. } + )); + assert_eq!(service.memory.used.load(Ordering::Acquire), 0); + assert!(service.disk.used.load(Ordering::Acquire) > 0); + assert!(service.disk.used.load(Ordering::Acquire) < limits.disk_per_root); + + let mut concurrent = Vec::new(); + for _ in 0..8 { + let worker = Arc::clone(&service); + let key = key.clone(); + concurrent.push(tokio::spawn( + async move { search(&worker, key, "needle").await }, + )); + } + for task in concurrent { + assert_eq!(task.await.unwrap()[0].path, "z/deep/needle.ts"); + } + assert_eq!(visits.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn same_root_discovery_coalesces_and_one_closed_consumer_does_not_cancel_the_other() { + let root = tempfile::tempdir().unwrap(); + let key = RootKey::resolve(root.path().to_str().unwrap()).unwrap(); + let gate = Gate::new(); + let worker_gate = Arc::clone(&gate); + let visits = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&visits); + let service = IndexService::new( + Limits::default(), + Arc::new(move |_, visit| { + count.fetch_add(1, Ordering::SeqCst); + worker_gate.wait(); + visit(entry("needle.ts", false)); + }), + ); + let first_control = Arc::new(SearchControl::default()); + let worker = Arc::clone(&service); + let first_key = key.clone(); + let control = Arc::clone(&first_control); + let first = tokio::spawn(async move { + worker + .search( + first_key, + SearchOptions { + query: "needle", + limit: 10, + files_only: true, + refresh: false, + }, + Instant::now(), + control, + ) + .await + .unwrap() + }); + tokio::time::timeout(Duration::from_secs(2), gate.started.notified()) + .await + .unwrap(); + let worker = Arc::clone(&service); + let second_key = key.clone(); + let second = tokio::spawn(async move { search(&worker, second_key, "needle").await }); + tokio::time::timeout(Duration::from_secs(1), async { + while service.registry.lock().unwrap().jobs[&key] + .users + .load(Ordering::Acquire) + < 2 + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + first_control.cancel.cancel(); + assert!(first.await.unwrap().is_empty()); + assert!(!service.registry.lock().unwrap().jobs[&key] + .control + .stopped()); + gate.release(); + assert_eq!(second.await.unwrap()[0].path, "needle.ts"); + assert_eq!(visits.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn closing_the_last_consumer_cancels_discovery_and_never_publishes_partial_data() { + let root = tempfile::tempdir().unwrap(); + let key = RootKey::resolve(root.path().to_str().unwrap()).unwrap(); + let gate = Gate::new(); + let worker_gate = Arc::clone(&gate); + let service = IndexService::new( + Limits::default(), + Arc::new(move |_, visit| { + visit(entry("prefix.ts", false)); + worker_gate.wait(); + visit(entry("late.ts", false)); + }), + ); + let control = Arc::new(SearchControl::default()); + let request_control = Arc::clone(&control); + let worker = Arc::clone(&service); + let task = tokio::spawn(async move { + worker + .search( + key, + SearchOptions { + query: "late", + limit: 1, + files_only: true, + refresh: false, + }, + Instant::now(), + request_control, + ) + .await + .unwrap() + }); + tokio::time::timeout(Duration::from_secs(2), gate.started.notified()) + .await + .unwrap(); + control.cancel.cancel(); + assert!(tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap() + .is_empty()); + gate.release(); + tokio::time::timeout(Duration::from_secs(1), async { + while service.discovery.available_permits() < WORKERS + || !service.registry.lock().unwrap().jobs.is_empty() + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(service.registry.lock().unwrap().snapshots.is_empty()); + assert_eq!(service.disk.used.load(Ordering::Acquire), 0); + assert_eq!(service.memory.used.load(Ordering::Acquire), 0); +} + +#[tokio::test] +async fn cancelled_queued_discovery_and_dropped_query_release_admission() { + let (service, visits) = counted(Limits::default()); + let permits = service + .discovery + .clone() + .acquire_many_owned(WORKERS as u32) + .await + .unwrap(); + let root = tempfile::tempdir().unwrap(); + let key = RootKey::resolve(root.path().to_str().unwrap()).unwrap(); + let worker = Arc::clone(&service); + let worker_key = key.clone(); + let task = tokio::spawn(async move { search(&worker, worker_key, "needle").await }); + tokio::time::timeout(Duration::from_secs(1), async { + while !service.registry.lock().unwrap().jobs.contains_key(&key) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + task.abort(); + let _ = task.await; + tokio::time::timeout(Duration::from_secs(1), async { + while !service.registry.lock().unwrap().jobs.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + drop(permits); + assert_eq!(visits.load(Ordering::SeqCst), 0); + assert!(service.registry.lock().unwrap().snapshots.is_empty()); +} + +#[tokio::test] +async fn a_cancelled_old_build_cannot_replace_the_new_same_root_generation() { + let root = tempfile::tempdir().unwrap(); + let key = RootKey::resolve(root.path().to_str().unwrap()).unwrap(); + let gate = Gate::new(); + let worker_gate = Arc::clone(&gate); + let visits = Arc::new(AtomicUsize::new(0)); + let count = Arc::clone(&visits); + let service = IndexService::new( + Limits::default(), + Arc::new(move |_, visit| { + if count.fetch_add(1, Ordering::SeqCst) == 0 { + worker_gate.wait(); + visit(entry("old.ts", false)); + } else { + visit(entry("new.ts", false)); + } + }), + ); + let worker = Arc::clone(&service); + let old_key = key.clone(); + let task = tokio::spawn(async move { search(&worker, old_key, ".ts").await }); + tokio::time::timeout(Duration::from_secs(2), gate.started.notified()) + .await + .unwrap(); + task.abort(); + let _ = task.await; + let result = search(&service, key.clone(), ".ts").await; + assert_eq!(result[0].path, "new.ts"); + gate.release(); + tokio::time::timeout(Duration::from_secs(1), async { + while service.discovery.available_permits() < WORKERS { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(search(&service, key, ".ts").await[0].path, "new.ts"); + assert_eq!(visits.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn active_snapshot_leases_remain_charged_after_lru_eviction() { + let one = tempfile::tempdir().unwrap(); + let two = tempfile::tempdir().unwrap(); + for root in [&one, &two] { + std::fs::write(root.path().join("needle"), "x").unwrap(); + } + let mut builder = Builder::default(); + assert!(builder.push(&entry("needle", false), BUILD_BYTES)); + let bytes = builder.index.bytes(); + let limits = Limits { + memory: bytes, + roots: 1, + ..Limits::default() + }; + let (service, _) = counted(limits); + let key = RootKey::resolve(one.path().to_str().unwrap()).unwrap(); + let lease = service + .snapshot(key, false, Instant::now(), &SearchControl::default()) + .await + .unwrap() + .unwrap(); + assert_eq!(service.memory.used.load(Ordering::Acquire), bytes); + search( + &service, + RootKey::resolve(two.path().to_str().unwrap()).unwrap(), + "needle", + ) + .await; + assert_eq!(service.memory.used.load(Ordering::Acquire), bytes); + assert!(matches!( + service.registry.lock().unwrap().snapshots[0].1.storage, + Storage::Disk { .. } + )); + drop(lease); + assert_eq!(service.memory.used.load(Ordering::Acquire), 0); +} + +#[tokio::test] +async fn total_disk_budget_includes_active_leases_and_failures_back_off() { + let limits = Limits { + memory: 0, + disk: 50, + disk_per_root: 50, + roots: 1, + ..Limits::default() + }; + let (service, visits) = counted(limits); + let a = tempfile::tempdir().unwrap(); + let b = tempfile::tempdir().unwrap(); + for root in [&a, &b] { + std::fs::write(root.path().join("needle"), "x").unwrap(); + } + let key_a = RootKey::resolve(a.path().to_str().unwrap()).unwrap(); + let key_b = RootKey::resolve(b.path().to_str().unwrap()).unwrap(); + let held = service + .snapshot(key_a, false, Instant::now(), &SearchControl::default()) + .await + .unwrap() + .unwrap(); + assert!(service.disk.used.load(Ordering::Acquire) > 0); + for _ in 0..3 { + let error = service + .snapshot( + key_b.clone(), + false, + Instant::now(), + &SearchControl::default(), + ) + .await + .err() + .unwrap(); + assert!(error.contains("budget is in use")); + } + assert_eq!(visits.load(Ordering::SeqCst), 1); + assert!(service.disk.used.load(Ordering::Acquire) <= limits.disk); + drop(held); + assert_eq!(service.disk.used.load(Ordering::Acquire), 0); +} + +#[tokio::test] +async fn disk_overflow_never_publishes_partial_results_or_rewalks_on_each_query() { + let (service, visits) = counted(Limits { + memory: 0, + disk: 32, + disk_per_root: 32, + ..Limits::default() + }); + let root = tempfile::tempdir().unwrap(); + for name in ["a.txt", "late-needle.txt"] { + std::fs::write(root.path().join(name), "x").unwrap(); + } + let key = RootKey::resolve(root.path().to_str().unwrap()).unwrap(); + for _ in 0..3 { + assert!(service + .snapshot( + key.clone(), + false, + Instant::now(), + &SearchControl::default() + ) + .await + .is_err()); + } + assert_eq!(visits.load(Ordering::SeqCst), 1); + assert!(service.registry.lock().unwrap().snapshots.is_empty()); + assert_eq!(service.disk.used.load(Ordering::Acquire), 0); +} + +#[tokio::test] +async fn refresh_expiry_root_configuration_and_lru_eviction_trigger_real_rebuilds() { + let (service, visits) = counted(Limits { + roots: 1, + ..Limits::default() + }); + let a = tempfile::tempdir().unwrap(); + let b = tempfile::tempdir().unwrap(); + for root in [&a, &b] { + std::fs::write(root.path().join("needle"), "x").unwrap(); + } + let key = RootKey::resolve(a.path().to_str().unwrap()).unwrap(); + search(&service, key.clone(), "needle").await; + service + .search( + key.clone(), + SearchOptions { + query: "needle", + limit: 1, + files_only: true, + refresh: true, + }, + Instant::now(), + Arc::new(SearchControl::default()), + ) + .await + .unwrap(); + assert_eq!(visits.load(Ordering::SeqCst), 2); + { + let mut registry = service.registry.lock().unwrap(); + Arc::get_mut(&mut registry.snapshots[0].1).unwrap().built_at = Instant::now() - CACHE_TTL; + } + search(&service, key, "needle").await; + assert_eq!(visits.load(Ordering::SeqCst), 3); + std::fs::write(a.path().join(".gitignore"), "needle\n").unwrap(); + assert!(search( + &service, + RootKey::resolve(a.path().to_str().unwrap()).unwrap(), + "needle" + ) + .await + .is_empty()); + assert_eq!(visits.load(Ordering::SeqCst), 4); + search( + &service, + RootKey::resolve(b.path().to_str().unwrap()).unwrap(), + "needle", + ) + .await; + search( + &service, + RootKey::resolve(a.path().to_str().unwrap()).unwrap(), + "needle", + ) + .await; + assert_eq!(visits.load(Ordering::SeqCst), 6); + assert_eq!(service.registry.lock().unwrap().snapshots.len(), 1); +} + +#[cfg(unix)] +#[tokio::test] +async fn linked_root_authorization_is_part_of_the_cache_identity() { + let root = tempfile::tempdir().unwrap(); + let linked = tempfile::tempdir().unwrap(); + std::fs::write(linked.path().join("needle.ts"), "x").unwrap(); + std::os::unix::fs::symlink(linked.path(), root.path().join("api")).unwrap(); + let target = std::fs::canonicalize(linked.path()).unwrap(); + let (service, visits) = counted(Limits::default()); + let path = root.path().to_str().unwrap(); + assert!(search(&service, RootKey::resolve(path).unwrap(), "needle") + .await + .is_empty()); + crate::folder_links::register(root.path(), &target); + let authorized = search(&service, RootKey::resolve(path).unwrap(), "needle").await; + crate::folder_links::unregister(root.path(), &target); + assert_eq!(authorized[0].path, "api/needle.ts"); + assert!(search(&service, RootKey::resolve(path).unwrap(), "needle") + .await + .is_empty()); + assert_eq!( + visits.load(Ordering::SeqCst), + 2, + "revocation can reuse the snapshot made before authorization" + ); +} + +#[tokio::test] +async fn disk_scoring_matches_memory_unicode_path_and_stable_tie_ranking() { + let root = tempfile::tempdir().unwrap(); + let key = RootKey::resolve(root.path().to_str().unwrap()).unwrap(); + let entries = vec![ + entry("foo", true), + entry("foo/😀FooBar.rs", false), + entry("foo/README.md", false), + entry("src", true), + entry("src/İstanbul.ts", false), + entry("src/éclair.ts", false), + entry("src/README.md", false), + ]; + let copied = entries.clone(); + let service = IndexService::new( + Limits { + memory: 0, + ..Limits::default() + }, + Arc::new(move |_, visit| { + for entry in copied.clone() { + if !visit(entry) { + break; + } + } + }), + ); + for query in ["README", "src/", "fbr", "😀", "é", "İ"] { + let mut expected = Matches::new(query, 10, true); + for (i, entry) in entries.iter().enumerate() { + expected.offer( + &entry.name, + &entry.path, + entry.kind == WorkspaceEntryKind::Dir, + i, + ); + } + assert_eq!( + search(&service, key.clone(), query).await, + expected.finish(), + "{query}" + ); + } + let snapshot = service + .snapshot(key, false, Instant::now(), &SearchControl::default()) + .await + .unwrap() + .unwrap(); + let cancelled = SearchControl::default(); + cancelled.cancel.cancel(); + assert!(snapshot + .search("README", 10, true, &cancelled) + .unwrap() + .is_empty()); +} + +#[tokio::test] +async fn expired_waiter_cancels_a_queued_job_and_leaves_no_worker_or_snapshot() { + let root = tempfile::tempdir().unwrap(); + let key = RootKey::resolve(root.path().to_str().unwrap()).unwrap(); + let (service, visits) = counted(Limits::default()); + let held = service + .discovery + .clone() + .acquire_many_owned(WORKERS as u32) + .await + .unwrap(); + let control = Arc::new(SearchControl { + deadline: Instant::now() + Duration::from_millis(10), + ..SearchControl::default() + }); + assert!(service + .search( + key, + SearchOptions { + query: "needle", + limit: 10, + files_only: true, + refresh: false + }, + Instant::now(), + control + ) + .await + .unwrap() + .is_empty()); + tokio::time::timeout(Duration::from_secs(1), async { + while !service.registry.lock().unwrap().jobs.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + drop(held); + assert_eq!(visits.load(Ordering::SeqCst), 0); + assert_eq!(service.disk.used.load(Ordering::Acquire), 0); +} + +/// Run explicitly to profile a real workspace. Outputs counts/bytes/timings, +/// never filenames. The temporary snapshot closes when the service drops. +#[tokio::test] +#[ignore = "set CODEG_MEMORY_WORKSPACE to profile a real workspace"] +async fn large_workspace_memory_probe() { + let path = std::env::var("CODEG_MEMORY_WORKSPACE").expect("CODEG_MEMORY_WORKSPACE"); + let key = RootKey::resolve(&path).unwrap(); + let (service, visits) = counted(Limits::default()); + println!("probe_pid={}", std::process::id()); + for query in ["README.md", "index.ts", "workspace_file_search"] { + let at = Instant::now(); + let result = search(&service, key.clone(), query).await; + println!( + "{}", + serde_json::json!({ + "results": result.len(), "responseBytes": serde_json::to_vec(&result).unwrap().len(), + "seconds": at.elapsed().as_secs_f64(), "walks": visits.load(Ordering::Acquire), + "retainedMemoryBytes": service.memory.used.load(Ordering::Acquire), + "snapshotDiskBytes": service.disk.used.load(Ordering::Acquire), + }) + ); + } + assert_eq!(visits.load(Ordering::Acquire), 1); +} diff --git a/src/lib/api.ts b/src/lib/api.ts index 7b9803618d1..39f1656be36 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -5039,7 +5039,7 @@ export async function listWorkspaceFiles( return getTransport().call("list_workspace_files", { path }) } -/** Warm a shared, budgeted native index without sending its entries to JS. */ +/** Compatibility no-op. Discovery starts only with a cancellable query. */ export async function prepareWorkspaceFileSearch(path: string): Promise { return getTransport().call("prepare_workspace_file_search", { path }) }