diff --git a/crates/tracedecay-application/src/feedback/concrete.rs b/crates/tracedecay-application/src/feedback/concrete.rs index 2a9f5a7d02a..a6fff6542b6 100644 --- a/crates/tracedecay-application/src/feedback/concrete.rs +++ b/crates/tracedecay-application/src/feedback/concrete.rs @@ -47,6 +47,7 @@ use tracedecay_domain::{ }; use tracedecay_store::DiagnosticStore; use tracedecay_tool_catalog::{CapabilityId, UseCaseId}; +use tracing::Instrument; use super::concrete_evidence::{complete, cursor_refused, interruption, unavailable}; use super::observations::{ @@ -294,11 +295,24 @@ impl ProjectFeedbackObservationSinkV1 { Ok(()) } - #[tracing::instrument(name = "usecases.feedback.close_drain", level = "trace", skip_all)] + #[tracing::instrument( + name = "usecases.feedback.close_drain", + level = "trace", + skip_all, + fields(producer_boot_id = %self.boot_id) + )] async fn join_worker(&self) -> Result<(), FeedbackRuntimeError> { - let mut worker = self.worker.lock().await; + let mut worker = self + .worker + .lock() + .instrument(tracing::trace_span!("usecases.feedback.close_drain.lock")) + .await; let outcome = match worker.as_ref() { - Some(task) => match task.wait().await { + Some(task) => match task + .wait() + .instrument(tracing::trace_span!("usecases.feedback.close_drain.worker")) + .await + { NativeTaskOutcome::Completed(Ok(())) => Ok(()), NativeTaskOutcome::Completed(Err(())) => Err(FeedbackRuntimeError::Store), NativeTaskOutcome::Cancelled | NativeTaskOutcome::Panicked(_) => { @@ -1783,7 +1797,11 @@ fn interruption_outcome(context: &RequestContext, observed_at: UtcMicros) -> Fee #[tracing::instrument( name = "usecases.feedback.persist_observation", level = "trace", - skip_all + skip_all, + fields( + producer_boot_id = ?envelope.producer_boot_id, + producer_sequence = ?envelope.producer_sequence + ) )] async fn persist_feedback_observation( database: &Database, @@ -1794,9 +1812,12 @@ async fn persist_feedback_observation( } let transaction = database .begin_write_transaction("record feedback observation") + .instrument(tracing::trace_span!("usecases.feedback.writer_admission")) .await .map_err(|_| FeedbackRuntimeError::Store)?; - let mut ledger = load_observation_ledger(&transaction).await?; + let mut ledger = load_observation_ledger(&transaction) + .instrument(tracing::trace_span!("usecases.feedback.load_ledger")) + .await?; if ledger .observations .iter() @@ -1850,18 +1871,27 @@ async fn persist_feedback_observation( } ledger.observations.push(envelope); ledger.schema_version = OBSERVATION_LEDGER_SCHEMA_VERSION; - let encoded = encode_bounded_observation_ledger( - &mut ledger, - MAX_STORED_OBSERVATIONS, - MAX_STORED_OBSERVATION_BOOTS, - MAX_OBSERVATION_LEDGER_BYTES, - )?; + let encoded = tracing::trace_span!( + "usecases.feedback.encode_ledger", + observations = ledger.observations.len(), + boots = ledger.producer_boots.len() + ) + .in_scope(|| { + encode_bounded_observation_ledger( + &mut ledger, + MAX_STORED_OBSERVATIONS, + MAX_STORED_OBSERVATION_BOOTS, + MAX_OBSERVATION_LEDGER_BYTES, + ) + })?; database .set_metadata_unguarded(&transaction, OBSERVATION_LEDGER_METADATA_KEY, &encoded) + .instrument(tracing::trace_span!("usecases.feedback.persist_ledger")) .await .map_err(|_| FeedbackRuntimeError::Store)?; transaction .commit() + .instrument(tracing::trace_span!("usecases.feedback.commit")) .await .map_err(|_| FeedbackRuntimeError::Store) } diff --git a/crates/tracedecay-daemon-service/tests/code_index_graph_activation.rs b/crates/tracedecay-daemon-service/tests/code_index_graph_activation.rs index ed51beb1bd1..0a9f3941425 100644 --- a/crates/tracedecay-daemon-service/tests/code_index_graph_activation.rs +++ b/crates/tracedecay-daemon-service/tests/code_index_graph_activation.rs @@ -470,8 +470,17 @@ async fn persistent_graph_activation_publishes_a_small_generation() { .expect("join graph reconciliation tasks"); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn persistent_callers_cursor_keeps_generation_a_without_repointing_generation_b() { +#[test] +fn persistent_callers_cursor_keeps_generation_a_without_repointing_generation_b() { + // Native tasks inherit this runtime, including the daemon's wake policy. + tracedecay_runtime_core::native_runtime::default_builder() + .worker_threads(2) + .build() + .expect("persistent graph cursor runtime") + .block_on(persistent_callers_cursor_across_generations()); +} + +async fn persistent_callers_cursor_across_generations() { let fixture = GitFixture::new(&[( "src/lib.rs", "pub fn hub() {}\npub fn caller_a() { hub(); }\npub fn caller_b() { hub(); }\n", diff --git a/crates/tracedecay-dashboard-api/src/server_tests.rs b/crates/tracedecay-dashboard-api/src/server_tests.rs index d4ba444ca69..b999feae9f3 100644 --- a/crates/tracedecay-dashboard-api/src/server_tests.rs +++ b/crates/tracedecay-dashboard-api/src/server_tests.rs @@ -81,7 +81,7 @@ fn loopback_server_drains_in_flight_request_after_shutdown() { /// Cancelling the listener must abort the connection tasks it owns, including /// a handler that cannot finish gracefully. -#[tokio::test] +#[tokio::test(flavor = "current_thread")] async fn cancelling_the_loopback_server_drops_its_open_connections() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -122,16 +122,30 @@ async fn cancelling_the_loopback_server_drops_its_open_connections() { .await .expect("the connection must admit the held handler"); + // Connect without yielding this current-thread runtime, so the socket + // remains in this listener's accept backlog when cancellation lands. + // Reconnecting after shutdown could reach another test that reused the + // ephemeral port and would not identify the listener being checked. + let queued_peer = TcpStream::connect_timeout(&address, bound).unwrap(); + queued_peer.set_nonblocking(true).unwrap(); + let mut queued_peer = tokio::net::TcpStream::from_std(queued_peer).unwrap(); server.abort(); assert!(server.await.unwrap_err().is_cancelled()); tokio::time::timeout(bound, dropped.notified()) .await .expect("listener cancellation must drop its active handler"); - // The joined task has dropped the listener it exclusively owned. Its - // ephemeral address may already belong to another parallel test, so a new - // connection cannot identify that listener. Observe the original accepted - // connection below instead. let mut byte = [0]; + let queued_received = tokio::time::timeout(bound, queued_peer.read(&mut byte)) + .await + .expect("listener cancellation must close its unaccepted socket"); + assert!( + matches!(queued_received, Ok(0)) + || matches!(queued_received, Err(ref error) if matches!( + error.kind(), + std::io::ErrorKind::ConnectionReset | std::io::ErrorKind::ConnectionAborted + )), + "the cancelled listener must close its backlog: {queued_received:?}" + ); let received = tokio::time::timeout(bound, peer.read(&mut byte)) .await .expect("listener cancellation must close its accepted socket"); diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs index 574f15286b1..192ab5fdf7c 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/lcm_schema_contract.rs @@ -7,7 +7,7 @@ async fn legacy_profile_requires_reset_without_carrying_session_content_forward( let db_path = tmp.path().join(".tracedecay").join("sessions.db"); create_legacy_sessions_db(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Err(error) => error, Ok(_) => panic!("legacy session content must require an explicit profile reset"), }; @@ -31,8 +31,9 @@ async fn stale_or_future_lcm_marker_requires_reset_without_rewriting_marker() { ] { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path).await.expect("fresh global db"); - drop(db); + initialize_global_db(&db_path) + .await + .expect("fresh global db"); set_migration_version(&db_path, found_version).await; let (lease, owner) = open_registered_test_database_fixture( diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs index fbfe8b5f1f8..88ef9206b40 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/mod.rs @@ -26,11 +26,6 @@ async fn initialize_global_db(db_path: &Path) -> tracedecay_domain::errors::Resu Ok(()) } -async fn open_global_db(db_path: &Path) -> tracedecay_domain::errors::Result { - initialize_global_db(db_path).await?; - Ok(TestConnection::open(db_path)) -} - async fn open_read_only_global_db( db_path: &Path, ) -> tracedecay_domain::errors::Result> { diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs index 1d2cb2f65d2..6606fbdb0bc 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog.rs @@ -5,10 +5,9 @@ async fn fresh_temporal_schema_stores_and_searches_a_summary_node() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); assert!( table_exists(&db_path, "lcm_raw_messages").await, "fresh initialization must compose the final temporal and LCM schemas" @@ -622,7 +621,7 @@ async fn temporal_schema_rejects_unexpected_indexes_without_dropping_them() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("unexpected temporal indexes must require reset"), Err(error) => error, }; @@ -675,7 +674,7 @@ async fn temporal_schema_refuses_markerless_malformed_fts_without_mutation() { drop(raw_db); let before_catalog = temporal_schema_object_catalog(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a markerless malformed FTS store must require reset"), Err(error) => error, }; @@ -743,7 +742,7 @@ async fn temporal_schema_rejects_missing_fts_without_rebuilding_it() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("missing temporal FTS objects must require reset"), Err(error) => error, }; @@ -798,7 +797,7 @@ async fn temporal_schema_rejects_missing_graph_publication_authority_without_reb drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("missing graph publication authority must require reset"), Err(error) => error, }; diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs index c1ec89d7d12..699e17c46a1 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_catalog/admission.rs @@ -74,7 +74,7 @@ async fn temporal_schema_accepts_only_fresh_or_exact_final_stores() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a partial temporal schema must require reset"), Err(error) => error, }; @@ -105,7 +105,7 @@ async fn temporal_schema_accepts_only_fresh_or_exact_final_stores() { // Removing an application table does not make a previously opened SQLite // namespace freshly owned. Existing sidecars remain unclaimed. - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("unpublished existing SQLite must not be reinitialized"), Err(error) => error, }; @@ -119,19 +119,17 @@ async fn temporal_schema_accepts_only_fresh_or_exact_final_stores() { ); let fresh_path = tmp.path().join(".tracedecay").join("fresh.db"); - let db = open_global_db(&fresh_path) + initialize_global_db(&fresh_path) .await .expect("a new prospective store should receive the final temporal schema"); - drop(db); let initial_catalog = temporal_schema_object_catalog(&fresh_path).await; let initial_version = temporal_schema_version(&fresh_path).await; let restart_path = tmp.path().join(".tracedecay").join("restart.db"); copy_database_for_temporal_restart(&fresh_path, &restart_path).await; - let reopened = open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .expect("idempotent temporal reopen should succeed"); - drop(reopened); assert_eq!( temporal_schema_version(&restart_path).await, initial_version @@ -163,7 +161,7 @@ async fn temporal_schema_malformed_marker_requires_reset_without_mutation() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a malformed temporal marker must require reset"), Err(error) => error, }; @@ -187,10 +185,9 @@ async fn temporal_schema_malformed_marker_requires_reset_without_mutation() { async fn temporal_schema_lower_marker_requires_reset_without_repairing_guards() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let previous = SESSION_TEMPORAL_SCHEMA_VERSION - 1; let raw_db = TestConnection::open(&db_path); @@ -226,21 +223,17 @@ async fn temporal_schema_lower_marker_requires_reset_without_repairing_guards() async fn temporal_schema_replaces_an_empty_earlier_authority_with_the_final_contract() { let tmp = TempDir::new().unwrap(); let fresh_path = tmp.path().join("fresh").join("sessions.db"); - drop( - open_global_db(&fresh_path) - .await - .expect("fresh store should receive the final temporal schema"), - ); + initialize_global_db(&fresh_path) + .await + .expect("fresh store should receive the final temporal schema"); let final_catalog = temporal_schema_object_catalog(&fresh_path).await; let final_guard = normalized_trigger_sql(&fresh_path, "session_refresh_progress_insert_guard_v1").await; let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - drop( - open_global_db(&db_path) - .await - .expect("temporal schema initialization should not error"), - ); + initialize_global_db(&db_path) + .await + .expect("temporal schema initialization should not error"); let raw_db = TestConnection::open(&db_path); let conn = (*raw_db).clone(); conn.execute_batch(&format!( @@ -262,11 +255,9 @@ async fn temporal_schema_replaces_an_empty_earlier_authority_with_the_final_cont "the earlier shape lacks the introduced-row index" ); - drop( - open_global_db(&db_path) - .await - .expect("an earlier authority holding no rows is replaced, not refused"), - ); + initialize_global_db(&db_path) + .await + .expect("an earlier authority holding no rows is replaced, not refused"); assert_eq!( temporal_schema_version(&db_path).await, SESSION_TEMPORAL_SCHEMA_VERSION @@ -303,7 +294,7 @@ async fn temporal_schema_rejects_transition_storage_without_mutating_it() { let before_catalog = temporal_schema_object_catalog(&db_path).await; let before_version = temporal_schema_version(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("branch-local transition storage must require reset"), Err(error) => error, }; @@ -348,7 +339,7 @@ async fn temporal_schema_rejects_foreign_summary_namespace_tables_without_mutati let before_catalog = temporal_schema_object_catalog(&db_path).await; let before_version = temporal_schema_version(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("foreign summary-namespace storage must require reset"), Err(error) => error, }; @@ -399,7 +390,7 @@ async fn final_store_missing_temporal_marker_requires_reset_without_mutation() { drop(raw_db); let before_catalog = temporal_schema_object_catalog(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a final LCM store without temporal identity must require reset"), Err(error) => error, }; @@ -452,7 +443,7 @@ async fn temporal_schema_is_not_installed_into_a_nonempty_store() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a nonempty store without temporal identity must require reset"), Err(error) => error, }; @@ -482,7 +473,7 @@ async fn unpublished_writer_ledger_is_preserved_and_fresh_publication_still_work "the seeded store must hold writer ledger tables" ); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("an unpublished writer-ledger database is not a fresh store"), Err(error) => error, }; @@ -500,10 +491,9 @@ async fn unpublished_writer_ledger_is_preserved_and_fresh_publication_still_work ); let fresh_path = tmp.path().join("fresh").join("sessions.db"); - let db = open_global_db(&fresh_path) + initialize_global_db(&fresh_path) .await .expect("new prospective publication must install the final schema and writer ledger"); - drop(db); assert!( table_exists(&fresh_path, "tracedecay_schema_version").await && table_exists(&fresh_path, "td_runtime_writer_idempotency_v2").await, @@ -525,7 +515,7 @@ async fn writer_ledger_does_not_hide_a_nonempty_unmarked_store() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("an application table without a temporal marker must require reset"), Err(error) => error, }; @@ -568,7 +558,7 @@ async fn final_schema_admission_rejects_extra_temporal_column_metadata() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("extra persisted temporal column metadata must require reset"), Err(error) => error, }; @@ -607,7 +597,7 @@ async fn final_schema_admission_rejects_missing_temporal_occurrence_time_index() drop(raw_db); let rejected_catalog = temporal_schema_object_catalog(&db_path).await; - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a missing required temporal query index must require reset"), Err(error) => error, }; @@ -642,7 +632,7 @@ async fn final_schema_admission_rejects_missing_graph_publication_table() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a missing graph publication table must require reset"), Err(error) => error, }; @@ -679,7 +669,7 @@ async fn final_schema_admission_rejects_incompatible_graph_publication_column_me drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("incompatible graph publication column metadata must require reset"), Err(error) => error, }; @@ -722,7 +712,7 @@ async fn final_schema_admission_rejects_extra_graph_publication_table() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("an extra graph publication table must require reset"), Err(error) => error, }; @@ -762,7 +752,7 @@ async fn final_schema_admission_rejects_mixed_case_extra_temporal_table() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a mixed-case extra temporal table must require reset"), Err(error) => error, }; @@ -802,7 +792,7 @@ async fn final_schema_admission_rejects_mixed_case_extra_graph_publication_table drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("a mixed-case extra graph publication table must require reset"), Err(error) => error, }; @@ -840,7 +830,7 @@ async fn final_schema_admission_rejects_extra_index_on_canonical_graph_table() { drop(conn); drop(raw_db); - let error = match open_global_db(&db_path).await { + let error = match initialize_global_db(&db_path).await { Ok(_) => panic!("an extra index on a canonical graph table must require reset"), Err(error) => error, }; @@ -862,10 +852,9 @@ async fn final_schema_admission_rejects_extra_index_on_canonical_graph_table() { async fn temporal_schema_refuses_future_version_without_mutation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); assert!( table_exists(&db_path, "tracedecay_schema_version").await, "the temporal schema must install a version marker before a future version is tested" diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs index 17c54837976..fec4a8e7215 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_constraints.rs @@ -4,10 +4,9 @@ use super::*; async fn temporal_schema_rejects_cross_session_and_generation_rows() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); assert!( table_exists(&db_path, "session_temporal_generations").await, "the temporal generation owner table must exist before ownership checks" diff --git a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs index ad69571ce89..848337f9235 100644 --- a/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs +++ b/crates/tracedecay-global-db/src/tests/lcm_schema/temporal_cursor.rs @@ -24,10 +24,9 @@ fn assert_cursor_rotation_verdict(error: TraceDecayError, context: &str) { async fn temporal_schema_persists_cursor_keys_without_read_creation() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); assert!( table_exists(&db_path, "session_query_cursor_keys").await, "the temporal schema must create the cursor-key authority table" @@ -50,10 +49,9 @@ async fn temporal_schema_persists_cursor_keys_without_read_creation() { let restart_path = tmp.path().join(".tracedecay").join("restart.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; - let reopened = open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .expect("writer reopen should preserve a persisted cursor key"); - drop(reopened); assert_eq!( row_count(&restart_path, "session_query_cursor_keys").await, 1 @@ -234,10 +232,9 @@ async fn temporal_schema_rotates_cursor_keys_atomically_and_survives_restart() { let restart_path = tmp.path().join(".tracedecay").join("cursor-restart.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; - let reopened = open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .expect("rotated cursor key authority must pass restart validation"); - drop(reopened); assert_eq!( row_count(&restart_path, "session_query_cursor_keys").await, 2 @@ -281,10 +278,9 @@ async fn temporal_schema_cursor_audit_rejects_nonmax_active_key() { let restart_path = tmp.path().join(".tracedecay").join("cursor-audit.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; assert_cursor_rotation_verdict( - open_global_db(&restart_path) + initialize_global_db(&restart_path) .await - .err() - .expect("restart audit refuses the database"), + .expect_err("restart audit refuses the database"), "restart audit must reject an active key that is not the monotonic maximum", ); } @@ -335,10 +331,9 @@ async fn temporal_schema_cursor_audit_rejects_skipped_successor_chain() { .join("restart.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; assert_cursor_rotation_verdict( - open_global_db(&restart_path) + initialize_global_db(&restart_path) .await - .err() - .expect("restart audit refuses the database"), + .expect_err("restart audit refuses the database"), &format!( "{fixture}: a later key must not satisfy a skipped immediate-successor retirement" ), @@ -385,10 +380,9 @@ async fn temporal_schema_cursor_audit_accepts_valid_successor_chains() { .join(".tracedecay") .join("valid-restart.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; - let reopened = open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .expect("valid immediate-successor chain must pass restart audit"); - drop(reopened); assert_valid_cursor_chain(&cursor_key_history(&restart_path).await); } } @@ -397,10 +391,9 @@ async fn temporal_schema_cursor_audit_accepts_valid_successor_chains() { async fn temporal_schema_concurrent_cursor_rotations_serialize_safely() { let tmp = TempDir::new().unwrap(); let db_path = tmp.path().join(".tracedecay").join("sessions.db"); - let db = open_global_db(&db_path) + initialize_global_db(&db_path) .await .expect("temporal schema initialization should not error"); - drop(db); let seed_db = TestConnection::open(&db_path); let initial = (*seed_db).clone(); @@ -540,9 +533,8 @@ async fn temporal_schema_concurrent_cursor_rotations_serialize_safely() { let restart_path = tmp.path().join(".tracedecay").join("concurrent-restart.db"); copy_database_for_temporal_restart(&db_path, &restart_path).await; - let reopened = open_global_db(&restart_path) + initialize_global_db(&restart_path) .await .expect("serialized concurrent rotations must pass restart audit"); - drop(reopened); assert_valid_cursor_chain(&cursor_key_history(&restart_path).await); } diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs index 0643c653f97..e0ac1069f13 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex.rs @@ -1037,11 +1037,12 @@ struct CodexRetainedScan { enum CodexScanPhase { Directories { queued: VecDeque<(PathBuf, u8, u8, u64)>, - current: Option>, + current: Option>, }, Files { next_directory: usize, current: Option>, + verification: Option>, deferred: Option>, resume_directories: Option, }, @@ -1049,12 +1050,25 @@ enum CodexScanPhase { struct CodexDirectoryResume { queued: VecDeque<(PathBuf, u8, u8, u64)>, - current: Option>, + current: Option>, +} + +/// Retain both the native iterator and its unfinished name witness across +/// budgeted discovery passes; neither can be reconstructed from an offset. +struct CodexDirectoryListing { + path: PathBuf, + depth: u8, + root_order: u8, + entries: std::fs::ReadDir, + opening_stat: [u8; 32], + fingerprint: Option, + directory_index: usize, } #[derive(Clone, Copy)] struct CodexDirectoryWitness { stat: [u8; 32], + stable_stat: [u8; 32], entries: Option<[u8; 32]>, } @@ -2076,6 +2090,10 @@ fn retained_scan_step( let directory_work_limit = work_limit.saturating_mul(usize::from(MAX_SCAN_DEPTH).saturating_add(2)); let mut directory_work = 0usize; + // Flat archived directories can contain thousands of non-transcript + // entries. Retain their ReadDir cursor across turns, just as for child + // directories, instead of enumerating the whole archive in one call. + let mut directory_entries = 0usize; let mut file_work = 0usize; let mut paths = Vec::new(); let mut selected_sources = Vec::new(); @@ -2100,6 +2118,7 @@ fn retained_scan_step( scan.phase = CodexScanPhase::Files { next_directory: 0, current: None, + verification: None, deferred: None, resume_directories: Some(resume_directories), }; @@ -2120,6 +2139,7 @@ fn retained_scan_step( scan.phase = CodexScanPhase::Files { next_directory: 0, current: None, + verification: None, deferred: None, resume_directories: None, }; @@ -2170,6 +2190,7 @@ fn retained_scan_step( scan.phase = CodexScanPhase::Files { next_directory: 0, current: None, + verification: None, deferred: None, resume_directories: Some(resume_directories), }; @@ -2182,15 +2203,23 @@ fn retained_scan_step( }, )?; } - let identity = codex_directory_witness(&path)?; + // Do not enumerate a potentially huge unsettled archive + // just to obtain its witness before the bounded iterator. + let stat = directory_scan_witness(&path)?; + let identity = stat.as_ref().map(|witness| CodexDirectoryWitness { + stat: witness.stat, + stable_stat: witness.stable_stat, + entries: None, + }); + let directory_index = scan.directories.len(); scan.directories.push(CodexDirectoryIdentity { path: path.clone(), root_order, identity, }); - if identity.is_none() { + let Some(opening_stat) = identity.map(|witness| witness.stable_stat) else { continue; - } + }; let listed = std::fs::read_dir(&path).map_err(|source| { TranscriptIngestError::ScanIo { operation: "enumerate Codex transcript directories", @@ -2198,19 +2227,33 @@ fn retained_scan_step( source, } })?; - *current = Some(Box::new((path, depth, root_order, listed))); + *current = Some(Box::new(CodexDirectoryListing { + path, + depth, + root_order, + entries: listed, + opening_stat, + fingerprint: stat + .filter(|witness| !witness.settled) + .map(|_| CodexDirectoryEntryFingerprint::default()), + directory_index, + })); continue; } - let (dir, depth, root_order, listed) = current.as_deref_mut().ok_or( + let directory = current.as_deref_mut().ok_or( TranscriptIngestError::InvalidCodexDiscoveryFrontier { detail: "retained Codex directory iterator disappeared", }, )?; - match listed.next() { + if directory_entries >= directory_work_limit { + break; + } + match directory.entries.next() { Some(entry) => { + directory_entries += 1; let entry = entry.map_err(|source| TranscriptIngestError::ScanIo { operation: "read Codex transcript directory entry", - path: dir.clone(), + path: directory.path.clone(), source, })?; let file_type = @@ -2221,13 +2264,16 @@ fn retained_scan_step( path: entry.path(), source, })?; + if let Some(fingerprint) = directory.fingerprint.as_mut() { + fingerprint.observe(&entry.file_name(), file_type)?; + } // Rollout files are not structural work. Charging them // here spends the pass on names that are not child // directories, so the newest sessions are not emitted // until every older day has been listed. if file_type.is_dir() && !file_type.is_symlink() { directory_work += 1; - if *depth >= MAX_SCAN_DEPTH { + if directory.depth >= MAX_SCAN_DEPTH { return Err(TranscriptIngestError::ScanIo { operation: "traverse Codex transcript directory depth", path: entry.path(), @@ -2269,12 +2315,12 @@ fn retained_scan_step( } queued.push_front(( child, - depth.checked_add(1).ok_or( + directory.depth.checked_add(1).ok_or( TranscriptIngestError::InvalidCodexDiscoveryFrontier { detail: "Codex transcript directory depth overflowed", }, )?, - *root_order, + directory.root_order, 0, )); let resume_directories = CodexDirectoryResume { @@ -2284,6 +2330,7 @@ fn retained_scan_step( scan.phase = CodexScanPhase::Files { next_directory: 0, current: None, + verification: None, deferred: None, resume_directories: Some(resume_directories), }; @@ -2292,23 +2339,59 @@ fn retained_scan_step( scan.directory_bytes = charged; queued.push_front(( child, - depth.checked_add(1).ok_or( + directory.depth.checked_add(1).ok_or( TranscriptIngestError::InvalidCodexDiscoveryFrontier { detail: "Codex transcript directory depth overflowed", }, )?, - *root_order, + directory.root_order, charge, )); } } } - None => *current = None, + None => { + // A budgeted ReadDir can span many scheduler turns. + // If the directory changed while we held it, the + // iterator may omit new entries. Never publish a + // complete corpus frontier from that mixed snapshot. + let closing_stat = directory_scan_witness(&directory.path)?; + if closing_stat.as_ref().map(|stat| stat.stable_stat) + != Some(directory.opening_stat) + { + scan.complete = false; + } + // Chunks can retire their directory identity while + // retaining the unfinished iterator. The opening + // witness still fences coverage in that case. + if let Some(identity) = scan + .directories + .get_mut(directory.directory_index) + .filter(|item| { + item.path == directory.path + && item.root_order == directory.root_order + }) + .and_then(|item| item.identity.as_mut()) + { + if let Some(fingerprint) = directory.fingerprint.take() { + identity.entries = Some(fingerprint.finish()); + } + // Commit the closing witness so the idle probe + // compares against the state this sweep actually + // certified, not a stale unsettled sample. + if let Some(closing) = closing_stat { + identity.stat = closing.stat; + identity.stable_stat = closing.stable_stat; + } + } + *current = None; + } } } CodexScanPhase::Files { next_directory, current, + verification, deferred, resume_directories, } => { @@ -2323,6 +2406,71 @@ fn retained_scan_step( Some(*candidate) } else { loop { + // An unsettled directory's native timestamp can alias + // a new entry while a file iterator is suspended. + // Verify eligible names through a fresh, bounded walk. + if let Some((directory_index, entries, fingerprint)) = + verification.as_deref_mut() + { + if file_work >= work_limit { + break None; + } + match entries.next() { + Some(entry) => { + file_work += 1; + let entry = + entry.map_err(|source| TranscriptIngestError::ScanIo { + operation: "verify Codex transcript directory entry", + path: scan.directories[*directory_index].path.clone(), + source, + })?; + let file_type = entry.file_type().map_err(|source| { + TranscriptIngestError::ScanIo { + operation: "verify Codex transcript entry type", + path: entry.path(), + source, + } + })?; + fingerprint.observe(&entry.file_name(), file_type)?; + continue; + } + None => { + let directory_index = *directory_index; + let closing = { + let directory = &scan.directories[directory_index]; + directory_scan_witness(&directory.path)? + }; + let identity = scan.directories[directory_index].identity; + // The verification iterator may itself span + // the coarse timestamp window. Its opening + // witness must not gain settled authority + // from a closing stat after that window. + if closing.as_ref().map(|stamp| stamp.stat) + != identity.map(|identity| identity.stat) + || identity.and_then(|identity| identity.entries) + != Some(std::mem::take(fingerprint).finish()) + { + scan.complete = false; + // Force the next idle probe to restart + // discovery; the failed name witness + // cannot certify the closing snapshot. + scan.directories[directory_index].identity = None; + } + // Commit the closing witness so the idle + // probe compares against the state this + // sweep actually certified. + if let Some(closing) = closing + && let Some(identity) = + scan.directories[directory_index].identity.as_mut() + { + identity.stat = closing.stat; + identity.stable_stat = closing.stable_stat; + } + *verification = None; + continue; + } + } + } if current.is_none() { let Some(directory) = scan.directories.get(*next_directory) else { break None; @@ -2374,12 +2522,71 @@ fn retained_scan_step( break Some((path, metadata)); } } - None => *current = None, + None => { + // This file iterator is retained across + // scheduler turns. A new directory entry can + // arrive after the structural walk but before + // file enumeration finishes. Recheck its native + // identity before granting full coverage: a + // stale ReadDir may have omitted that entry. + let directory_index = *directory_index; + let closing = { + let directory = &scan.directories[directory_index]; + directory_scan_witness(&directory.path)? + }; + let identity = scan.directories[directory_index].identity; + if closing.as_ref().map(|stat| stat.stable_stat) + != identity.map(|identity| identity.stable_stat) + { + scan.complete = false; + } else if identity.and_then(|identity| identity.entries).is_some() { + // This initially unsettled listing needs an + // independent current-name check, even if the + // filesystem clock settled during the walk. + let path = scan.directories[directory_index].path.clone(); + let reopened = std::fs::read_dir(&path).map_err(|source| { + TranscriptIngestError::ScanIo { + operation: "verify Codex transcript bucket", + path, + source, + } + })?; + *verification = Some(Box::new(( + directory_index, + reopened, + CodexDirectoryEntryFingerprint::default(), + ))); + } else if closing.as_ref().is_some_and(|stamp| !stamp.settled) + || closing.as_ref().map(|stamp| stamp.stat) + != identity.map(|identity| identity.stat) + { + // Neither an unsettled native stat nor a + // transition to settled authority can vouch + // for a retained listing without a name + // witness, even if structural work resumes. + scan.complete = false; + scan.directories[directory_index].identity = None; + } + // Commit the closing witness so the idle probe + // compares against the state this sweep actually + // certified, not a stale unsettled sample. + if let Some(closing) = closing + && let Some(identity) = + scan.directories[directory_index].identity.as_mut() + { + identity.stat = closing.stat; + identity.stable_stat = closing.stable_stat; + } + *current = None; + } } } }; let Some((path, metadata)) = candidate else { - if *next_directory >= scan.directories.len() && current.is_none() { + if *next_directory >= scan.directories.len() + && current.is_none() + && verification.is_none() + { if let Some(resume) = resume_directories.take() { scan.directories.clear(); scan.directory_bytes = 0; @@ -2573,6 +2780,16 @@ struct DirectoryStatWitness { settled: bool, } +/// A directory witness for a budgeted sweep. `stable_stat` drops the +/// settled/unsettled encoding so a directory that settles mid-sweep is not +/// mistaken for a changed one; its name witness still describes the same +/// entries. +struct DirectoryScanWitness { + stat: [u8; 32], + stable_stat: [u8; 32], + settled: bool, +} + fn codex_directory_changed( path: &Path, stored: Option, @@ -2610,12 +2827,14 @@ fn codex_directory_changed_since( Ok(directory_entry_fingerprint(path)? != entries) } +#[cfg(test)] fn codex_directory_witness(path: &Path) -> TranscriptIngestResult> { - let Some(current) = directory_stat_witness(path)? else { + let Some(current) = directory_scan_witness(path)? else { return Ok(None); }; Ok(Some(CodexDirectoryWitness { stat: current.stat, + stable_stat: current.stable_stat, // In the settled case, a later stat comparison proves stability. // Avoid eagerly listing and sorting all historical rollout names; // an entry-name witness is needed only for a coarse-clock window. @@ -2627,7 +2846,7 @@ fn codex_directory_witness(path: &Path) -> TranscriptIngestResult TranscriptIngestResult> { +fn stat_directory(path: &Path) -> TranscriptIngestResult> { let Some(metadata) = stat_if_present(path, "stat Codex transcript directory")? else { return Ok(None); }; @@ -2641,6 +2860,13 @@ fn directory_stat_witness(path: &Path) -> TranscriptIngestResult TranscriptIngestResult> { + let Some(metadata) = stat_directory(path)? else { + return Ok(None); + }; let changed = change_stamp(path, &metadata); Ok(Some(DirectoryStatWitness { stat: codex_directory_stat_identity(path, &metadata, changed)?, @@ -2648,6 +2874,31 @@ fn directory_stat_witness(path: &Path) -> TranscriptIngestResult TranscriptIngestResult> { + let Some(metadata) = stat_directory(path)? else { + return Ok(None); + }; + let changed = change_stamp(path, &metadata); + Ok(Some(DirectoryScanWitness { + stat: codex_directory_stat_identity(path, &metadata, changed)?, + stable_stat: codex_directory_stable_identity(path, &metadata)?, + settled: changed.is_settled(), + })) +} + +/// The directory identity without the settled/unsettled encoding. A budgeted +/// sweep compares this across slices: a directory that settles mid-sweep has +/// not changed, and its name witness still describes the same entries. +fn codex_directory_stable_identity( + path: &Path, + metadata: &std::fs::Metadata, +) -> TranscriptIngestResult<[u8; 32]> { + let native_change_time = RewriteWitness::NATIVE + .change_time_nanos(metadata) + .map_or(0, |time| u128::from_le_bytes(time.to_le_bytes())); + codex_corpus_identity(path, metadata, ChangeStamp::Unvouched(native_change_time)) +} + // An unsettled ChangeStamp has a different random sample token on *every* // stat, even when the directory has not changed. That token is appropriate // for file-content authority, but would make an unchanged hot Codex directory @@ -2672,13 +2923,66 @@ fn codex_directory_stat_identity( codex_corpus_identity(path, metadata, stat_change) } +/// Order-independent bounded-memory set witness. Directory entries are +/// unique names, so their separately domain-hashed names can be combined +/// without storing and sorting an unbounded filename list. +#[derive(Default)] +struct CodexDirectoryEntryFingerprint { + names: u64, + xor: [u8; 32], +} + +impl CodexDirectoryEntryFingerprint { + fn observe( + &mut self, + name: &std::ffi::OsStr, + file_type: std::fs::FileType, + ) -> TranscriptIngestResult<()> { + if !file_type.is_dir() + && Path::new(name).extension().and_then(|ext| ext.to_str()) != Some("jsonl") + { + return Ok(()); + } + let tag = if file_type.is_symlink() { + 2_u8 + } else if file_type.is_dir() { + 1 + } else { + 0 + }; + let mut hasher = Sha256::new(); + hasher.update(b"tracedecay-codex-directory-entry-v2"); + hasher.update([tag]); + hash_directory_name(&mut hasher, name); + let digest: [u8; 32] = hasher.finalize().into(); + for (accumulator, byte) in self.xor.iter_mut().zip(digest) { + *accumulator ^= byte; + } + self.names = self.names.checked_add(1).ok_or( + TranscriptIngestError::InvalidCodexDiscoveryFrontier { + detail: "Codex directory fingerprint entry count overflowed", + }, + )?; + Ok(()) + } + + fn finish(self) -> [u8; 32] { + let mut hasher = Sha256::new(); + // These witnesses are process-local, not durable corpus identities. + hasher.update(b"tracedecay-codex-directory-entries-v2"); + hasher.update(self.names.to_le_bytes()); + hasher.update(self.xor); + hasher.finalize().into() + } +} + fn directory_entry_fingerprint(path: &Path) -> TranscriptIngestResult<[u8; 32]> { let listed = std::fs::read_dir(path).map_err(|source| TranscriptIngestError::ScanIo { operation: "fingerprint Codex transcript directory", path: path.to_path_buf(), source, })?; - let mut names = Vec::new(); + let mut fingerprint = CodexDirectoryEntryFingerprint::default(); for entry in listed { let entry = entry.map_err(|source| TranscriptIngestError::ScanIo { operation: "fingerprint Codex transcript directory entry", @@ -2692,24 +2996,9 @@ fn directory_entry_fingerprint(path: &Path) -> TranscriptIngestResult<[u8; 32]> path: entry.path(), source, })?; - let tag = if file_type.is_symlink() { - 2_u8 - } else if file_type.is_dir() { - 1 - } else { - 0 - }; - names.push((entry.file_name(), tag)); - } - names.sort(); - let mut hasher = Sha256::new(); - hasher.update(b"tracedecay-codex-directory-entries-v1"); - for (name, tag) in names { - hasher.update([tag]); - hash_directory_name(&mut hasher, &name); - hasher.update([0xff]); + fingerprint.observe(&entry.file_name(), file_type)?; } - Ok(hasher.finalize().into()) + Ok(fingerprint.finish()) } fn hash_directory_name(hasher: &mut Sha256, name: &std::ffi::OsStr) { diff --git a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs index 4a2dfcaec7a..550da23b247 100644 --- a/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs +++ b/crates/tracedecay-sessions/src/runtime/hosts/codex/tests.rs @@ -1230,15 +1230,14 @@ mod recent_first_discovery_tests { use crate::runtime::hosts::codex::{ CodexCorpusEpoch, CodexDiscoveryDelivery, CodexDiscoveryFrontier, CodexDiscoveryHub, CodexDiscoverySourceKey, CodexDiscoveryState, CodexExactSessionPathAuthority, - CodexIndexedPath, CodexReplayIndex, EXACT_HOOK_DISCOVERY_UNITS_PER_CALL, - IDLE_FULL_VALIDATION_CYCLES, MAX_EXACT_HOOK_SESSION_REQUESTS, - MAX_EXACT_HOOK_SOURCE_AUTHORITIES, MAX_SCAN_DEPTH, codex_corpus_identity, - codex_directory_changed, codex_directory_stat_identity, codex_directory_witness, - directory_entry_fingerprint, directory_stat_witness, indexed_replay_pass, + CodexIndexedPath, CodexReplayIndex, CodexScanPhase, DirectoryStatWitness, + EXACT_HOOK_DISCOVERY_UNITS_PER_CALL, IDLE_FULL_VALIDATION_CYCLES, + MAX_EXACT_HOOK_SESSION_REQUESTS, MAX_EXACT_HOOK_SOURCE_AUTHORITIES, MAX_SCAN_DEPTH, + codex_corpus_identity, codex_directory_changed, codex_directory_changed_since, + codex_directory_stat_identity, codex_directory_witness, directory_entry_fingerprint, + directory_scan_witness, directory_stat_witness, indexed_replay_pass, replay_index_entries_visited_for_test, reset_replay_index_entries_visited_for_test, }; - #[cfg(unix)] - use crate::runtime::hosts::codex::{DirectoryStatWitness, codex_directory_changed_since}; use crate::runtime::ingest::quarantine::SourceSweep; use crate::runtime::jsonl_observation_admission::{ SHARED_JSONL_PIN_GATES, install_test_shared_jsonl_preparation_authority, @@ -2685,6 +2684,46 @@ mod recent_first_discovery_tests { assert!(found, "the changed hot bucket must deliver the new rollout"); } + #[test] + fn ignored_codex_archive_entries_do_not_change_candidate_fingerprint() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join("archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + std::fs::write(archive.join("rollout-existing.jsonl"), b"{}\n").unwrap(); + let initial = directory_entry_fingerprint(&archive).unwrap(); + + // Non-JSONL names are in the directory but not in the candidate + // set. The previous fingerprint changed for every ignored addition. + for index in 0..128 { + std::fs::write( + archive.join(format!("rollout-{index:04}.jsonl.zst")), + b"zst", + ) + .unwrap(); + } + assert_eq!(directory_entry_fingerprint(&archive).unwrap(), initial); + + std::fs::rename( + archive.join("rollout-0000.jsonl.zst"), + archive.join("rollout-renamed.jsonl.zst"), + ) + .unwrap(); + assert_eq!(directory_entry_fingerprint(&archive).unwrap(), initial); + + let eligible = archive.join("rollout-new.jsonl"); + std::fs::write(&eligible, b"{}\n").unwrap(); + assert_ne!(directory_entry_fingerprint(&archive).unwrap(), initial); + std::fs::remove_file(eligible).unwrap(); + assert_eq!(directory_entry_fingerprint(&archive).unwrap(), initial); + + std::fs::create_dir(archive.join("2026")).unwrap(); + assert_ne!( + directory_entry_fingerprint(&archive).unwrap(), + initial, + "child directories must remain part of traversal authority" + ); + } + #[test] #[cfg(unix)] fn unsettled_directory_stat_does_not_change_with_each_sample() { @@ -2855,6 +2894,571 @@ mod recent_first_discovery_tests { ); } + #[test] + fn archived_non_jsonl_entries_yield_directory_enumeration_across_windows() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + for index in 0..64 { + std::fs::write( + archive.join(format!("rollout-{index:03}.zst")), + b"compressed", + ) + .unwrap(); + } + let admitted = archive.join("rollout-admitted.jsonl"); + std::fs::write(&admitted, b"{}\n").unwrap(); + // Isolate bounded enumeration from the file's coarse-clock transition. + spin_until_jsonl_change_settled(&admitted); + + let source = CodexSource::with_home(temp.path()); + let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let first = retained_pass(&source, &mut state, bounds, frontier); + assert!(first.report.is_truncated()); + assert!(first.report.paths.is_empty()); + assert!( + matches!( + state.scan.as_ref().map(|scan| &scan.phase), + Some(CodexScanPhase::Directories { + current: Some(_), + .. + }) + ), + "non-transcript directory entries must not exhaust one discovery turn" + ); + frontier = first.next_frontier; + + let mut covered = BTreeSet::new(); + for _ in 0..256 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + covered.extend(pass.report.paths); + frontier = pass.next_frontier; + if frontier.is_complete() { + break; + } + } + assert_eq!(covered, BTreeSet::from([admitted])); + assert!(frontier.is_complete(), "bounded pages must converge"); + let idle = retained_pass(&source, &mut state, bounds, frontier); + assert!(idle.report.paths.is_empty()); + } + + #[test] + fn file_walk_detects_a_new_rollout_while_its_directory_cursor_is_retained() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + for index in 0..64 { + std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"ignored").unwrap(); + } + std::fs::write(archive.join("rollout-existing.jsonl"), b"{}\n").unwrap(); + + let source = CodexSource::with_home(temp.path()); + let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let mut active = false; + for _ in 0..256 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if let Some(scan) = state.scan.as_ref() + && let CodexScanPhase::Files { + current: Some(current), + .. + } = &scan.phase + { + let directory = &scan.directories[current.0]; + assert_eq!(directory.path, archive); + assert!(directory.identity.is_some()); + active = true; + break; + } + } + assert!(active, "the file-phase iterator must retain its position"); + let prior = directory_stat_witness(&archive).unwrap().unwrap().stat; + let arrived = archive.join("rollout-added-during-file-walk.jsonl"); + std::fs::write(&arrived, b"{}\n").unwrap(); + assert_ne!( + directory_stat_witness(&archive).unwrap().unwrap().stat, + prior, + "the mutation must change the native directory witness" + ); + let mut finished = false; + for _ in 0..1024 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if state.scan.is_none() { + finished = true; + assert!( + !frontier.is_complete(), + "a new file during the retained file walk must invalidate its coverage" + ); + break; + } + } + assert!( + finished, + "the bounded file traversal must reach its frontier" + ); + } + + #[test] + fn aliased_coarse_directory_stamp_cannot_certify_a_stale_file_walk() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + for index in 0..64 { + std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"ignored").unwrap(); + } + std::fs::write(archive.join("rollout-existing.jsonl"), b"{}\n").unwrap(); + let source = CodexSource::with_home(temp.path()); + let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let mut active = false; + for _ in 0..256 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if let Some(scan) = state.scan.as_ref() + && let CodexScanPhase::Files { + current: Some(current), + .. + } = &scan.phase + && scan.directories[current.0].path == archive + { + active = true; + break; + } + } + assert!(active, "the file reader must remain suspended across turns"); + let original_names = directory_entry_fingerprint(&archive).unwrap(); + let late = archive.join("rollout-after-file-reader-open.jsonl"); + std::fs::write(&late, b"{}\n").unwrap(); + // Model two writes within one filesystem timestamp quantum: the + // directory stat aliases the scan's authority despite a new file. + let witness = directory_scan_witness(&archive).unwrap().unwrap(); + let alias = witness.stat; + let alias_stable = witness.stable_stat; + let scan = state.scan.as_mut().unwrap(); + let CodexScanPhase::Files { + current: Some(current), + .. + } = &scan.phase + else { + panic!("the file-phase iterator must still be retained"); + }; + let directory_index = current.0; + let identity = scan.directories[directory_index].identity.as_mut().unwrap(); + identity.stat = alias; + identity.stable_stat = alias_stable; + identity.entries = Some(original_names); + + let mut finished = false; + for _ in 0..2048 { + // Preserve the intentionally aliased stat through an arbitrarily + // slow test run: elapsed wall time must not make this regression + // pass by turning the native clock into a separate detection. + let witness = directory_scan_witness(&archive).unwrap().unwrap(); + if let Some(scan) = state.scan.as_mut() { + let directory = scan + .directories + .iter_mut() + .find(|directory| directory.path == archive) + .expect("retained archive directory identity"); + let identity = directory.identity.as_mut().unwrap(); + identity.stat = witness.stat; + identity.stable_stat = witness.stable_stat; + } + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if state.scan.is_none() { + assert!( + !frontier.is_complete(), + "an aliased directory timestamp hid a new eligible rollout" + ); + finished = true; + break; + } + } + assert!(finished, "the bounded verification must terminate"); + } + + #[test] + #[cfg(target_os = "linux")] + fn chunked_archive_alias_cannot_complete_without_file_name_witness() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(archive.join("deferred")).unwrap(); + std::fs::write(archive.join("rollout-existing.jsonl"), b"{}\n").unwrap(); + let source = CodexSource::with_home(temp.path()); + let mut bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let metadata_charge = std::mem::size_of::() as u64; + let sessions = temp.path().join(".codex/sessions"); + // Retain both roots, but force the archive's child into a later + // structural chunk while the archive's file walk owns the iterator. + bounds.max_discovery_bytes = + crate::runtime::hosts::codex::candidate_charge(&sessions, metadata_charge).unwrap() + + crate::runtime::hosts::codex::candidate_charge(&archive, metadata_charge) + .unwrap(); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let mut suspended = false; + for _ in 0..256 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if let Some(scan) = state.scan.as_mut() + && let CodexScanPhase::Files { + current: Some(current), + resume_directories: Some(_), + .. + } = &mut scan.phase + && scan.directories[current.0].path == archive + { + let identity = scan.directories[current.0].identity.as_ref().unwrap(); + assert!( + identity.entries.is_none(), + "retention must interrupt the structural name witness" + ); + // A native ReadDir may buffer every old name before the late + // creation. Exhaust it now so omission is deterministic. + for entry in current.1.by_ref() { + entry.unwrap(); + } + suspended = true; + break; + } + } + assert!(suspended, "the chunked file reader must be retained"); + + let arrived = archive.join("rollout-after-chunked-file-reader-open.jsonl"); + std::fs::write(&arrived, b"{}\n").unwrap(); + assert!(spin_until_jsonl_change_settled(&archive)); + let metadata = std::fs::metadata(&archive).unwrap(); + let closing = directory_scan_witness(&archive).unwrap().unwrap(); + assert!(closing.settled); + let scan = state.scan.as_mut().unwrap(); + let CodexScanPhase::Files { + current: Some(current), + resume_directories: Some(resume), + .. + } = &mut scan.phase + else { + panic!("the chunked file reader must still be retained"); + }; + // Model an unsettled opening sample whose native timestamp aliases + // the new entry, then closes after settlement. Derive it from the real + // directory metadata so scheduler delay cannot choose the clock branch. + // The resumed structural walk cannot vouch for the stale file listing. + let identity = scan.directories[current.0].identity.as_mut().unwrap(); + identity.stat = + codex_directory_stat_identity(&archive, &metadata, ChangeStamp::Unvouched(0)).unwrap(); + identity.stable_stat = closing.stable_stat; + let resumed = resume.current.as_deref_mut().unwrap(); + assert_eq!(resumed.path, archive); + resumed.opening_stat = closing.stable_stat; + assert!(scan.complete); + + let mut observed = BTreeSet::new(); + let mut finished = false; + for _ in 0..2048 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + observed.extend(pass.report.paths); + frontier = pass.next_frontier; + if state.scan.is_none() { + assert!( + observed.contains(&arrived) || !frontier.is_complete(), + "the resumed structural walk cannot vouch for rollout files" + ); + finished = true; + break; + } + } + assert!(finished, "the chunked sweep must terminate"); + + for _ in 0..2048 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + observed.extend(pass.report.paths); + frontier = pass.next_frontier; + if frontier.is_complete() { + break; + } + } + assert!(frontier.is_complete(), "a fresh sweep must converge"); + assert!(observed.contains(&arrived)); + } + + #[test] + #[cfg(unix)] + fn verification_cannot_settle_over_an_aliased_new_rollout() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + for index in 0..64 { + std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"ignored").unwrap(); + } + let existing = archive.join("rollout-existing.jsonl"); + std::fs::write(&existing, b"{}\n").unwrap(); + let original_names = directory_entry_fingerprint(&archive).unwrap(); + let source = CodexSource::with_home(temp.path()); + let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let mut verifying = false; + for _ in 0..256 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + if let Some(scan) = state.scan.as_mut() + && let CodexScanPhase::Files { + current, + verification, + .. + } = &mut scan.phase + { + if let Some(current) = current + && scan.directories[current.0].path == archive + { + // Keep the coarse-clock name witness independent of how + // quickly this runner finishes the structural traversal. + scan.directories[current.0] + .identity + .as_mut() + .unwrap() + .entries = Some(original_names); + } + if let Some(verification) = verification + && scan.directories[verification.0].path == archive + { + // A retained ReadDir may buffer every old name before a + // concurrent creation. Exhaust that real listing now so + // omission of the later rollout is deterministic. + for entry in verification.1.by_ref() { + let entry = entry.unwrap(); + verification + .2 + .observe(&entry.file_name(), entry.file_type().unwrap()) + .unwrap(); + } + verifying = true; + break; + } + } + } + assert!( + verifying, + "the bounded verification must retain its iterator" + ); + let arrived = archive.join("rollout-after-verification-open.jsonl"); + std::fs::write(&arrived, b"{}\n").unwrap(); + assert!(spin_until_jsonl_change_settled(&archive)); + let metadata = std::fs::metadata(&archive).unwrap(); + let closing = directory_scan_witness(&archive).unwrap().unwrap(); + assert!(closing.settled); + let scan = state.scan.as_mut().unwrap(); + let CodexScanPhase::Files { + verification: Some(verification), + .. + } = &scan.phase + else { + panic!("the verification iterator must still be retained"); + }; + let identity = scan.directories[verification.0].identity.as_mut().unwrap(); + // Model a native timestamp that aliases the new entry, while keeping + // the verification-opening sample unsettled. The names and stable + // stat alone must not promote this stale listing to settled coverage. + identity.stat = + codex_directory_stat_identity(&archive, &metadata, ChangeStamp::Unvouched(0)).unwrap(); + identity.stable_stat = closing.stable_stat; + identity.entries = Some(original_names); + assert!(scan.complete); + + let pass = retained_pass(&source, &mut state, bounds, frontier); + frontier = pass.next_frontier; + assert!(state.scan.is_none(), "verification must finish this sweep"); + assert!(!frontier.is_complete(), "the new rollout was not observed"); + let idle = state.idle.as_ref().unwrap(); + let stored = idle + .directories + .iter() + .find(|item| item.path == archive) + .unwrap(); + assert!( + codex_directory_changed(&archive, stored.identity).unwrap(), + "the next idle probe must restart instead of waiting for periodic validation" + ); + + let mut observed = BTreeSet::new(); + for _ in 0..1024 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + observed.extend(pass.report.paths); + frontier = pass.next_frontier; + if frontier.is_complete() { + break; + } + } + assert!( + frontier.is_complete(), + "a fresh settled sweep must converge" + ); + assert!(observed.contains(&arrived)); + } + + #[test] + fn a_mutated_directory_cannot_complete_a_budgeted_codex_sweep() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + for index in 0..64 { + std::fs::write(archive.join(format!("rollout-{index:03}.zst")), b"ignored").unwrap(); + } + let source = CodexSource::with_home(temp.path()); + let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let first = retained_pass(&source, &mut state, bounds, frontier); + let initial_stat = { + let scan = state.scan.as_ref().expect("partial structural scan"); + let CodexScanPhase::Directories { + current: Some(current), + .. + } = &scan.phase + else { + panic!("a flat archive must leave its directory reader active"); + }; + assert_eq!(current.path, archive); + scan.directories[current.directory_index] + .identity + .expect("initial directory witness") + .stat + }; + frontier = first.next_frontier; + + // The initial iterator may or may not see the new entry. Regardless, + // its opening stat no longer proves a complete, coherent sweep. + let added = archive.join("rollout-arrived-during-scan.jsonl"); + std::fs::write(&added, b"{}\n").unwrap(); + assert_ne!( + directory_stat_witness(&archive).unwrap().unwrap().stat, + initial_stat, + "the fixture must cause a real directory authority transition" + ); + + let mut observed = BTreeSet::new(); + let mut ended_first_sweep = false; + for _ in 0..512 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + observed.extend(pass.report.paths); + frontier = pass.next_frontier; + if state.scan.is_none() { + assert!( + !frontier.is_complete(), + "a changed directory must not earn complete coverage from a stale iterator" + ); + ended_first_sweep = true; + break; + } + } + assert!(ended_first_sweep, "the first bounded sweep must terminate"); + + for _ in 0..1024 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + observed.extend(pass.report.paths); + frontier = pass.next_frontier; + if frontier.is_complete() { + break; + } + } + assert!( + frontier.is_complete(), + "a stable subsequent sweep must converge" + ); + assert!( + observed.contains(&added), + "the new source must be rediscovered before complete coverage" + ); + } + + #[test] + #[cfg(unix)] + fn unsettled_flat_archive_fingerprint_is_incremental_across_budgeted_passes() { + let temp = TempDir::new().unwrap(); + let archive = temp.path().join(".codex/archived_sessions"); + std::fs::create_dir_all(&archive).unwrap(); + for index in 0..128 { + std::fs::write( + archive.join(format!("rollout-{index:04}.jsonl.zst")), + b"compressed", + ) + .unwrap(); + } + let eligible = archive.join("rollout-eligible.jsonl"); + std::fs::write(&eligible, b"{}\n").unwrap(); + spin_until_jsonl_change_settled(&eligible); + let expected = directory_entry_fingerprint(&archive).unwrap(); + + let source = CodexSource::with_home(temp.path()); + let bounds = TranscriptDiscoveryBounds::from_discovered_units(1); + let mut state = CodexDiscoveryState::default(); + let mut frontier = CodexDiscoveryFrontier::initial(); + let first = retained_pass(&source, &mut state, bounds, frontier); + assert!(first.report.is_truncated()); + let scan = state.scan.as_mut().expect("unfinished structural scan"); + let CodexScanPhase::Directories { + current: Some(current), + .. + } = &mut scan.phase + else { + panic!("the archive must retain a live directory iterator"); + }; + assert_eq!(current.path, archive); + assert!( + scan.directories[current.directory_index] + .identity + .expect("directory stat witness") + .entries + .is_none(), + "no eager whole-archive fingerprint before bounded traversal" + ); + // Model an initially unsettled directory witness independently of the + // native clock. Restart its iterator with the empty fingerprint so + // entries consumed by the first pass cannot be omitted from the witness. + current.entries = std::fs::read_dir(&archive).unwrap(); + current.fingerprint = Some(Default::default()); + frontier = first.next_frontier; + + let mut covered = BTreeSet::new(); + for _ in 0..512 { + let pass = retained_pass(&source, &mut state, bounds, frontier); + covered.extend(pass.report.paths); + frontier = pass.next_frontier; + if frontier.is_complete() { + break; + } + } + assert!(frontier.is_complete(), "bounded traversal must converge"); + assert_eq!(covered, BTreeSet::from([eligible])); + let idle = state + .idle + .as_ref() + .expect("completed sweep retains witnesses"); + let archive_identity = idle + .directories + .iter() + .find(|directory| directory.path == archive) + .and_then(|directory| directory.identity) + .expect("archive directory witness"); + assert_eq!( + archive_identity.entries, + Some(expected), + "the same full candidate witness must be assembled exactly once" + ); + + let stable = retained_pass(&source, &mut state, bounds, frontier); + assert!(stable.report.paths.is_empty()); + assert!(stable.next_frontier.is_complete()); + } + /// Sweep-complete is store-durable: a fresh source reading only admission /// parse offsets (the production persist path) idles instead of restarting /// truncated-from-zero. MemoryHostAdmission is the same offset table a diff --git a/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs b/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs index 1feffacc327..602cbe959a5 100644 --- a/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs +++ b/crates/tracedecay/src/daemon/production_harness/project_server_capacity_journey_test.rs @@ -190,8 +190,16 @@ fn assert_generation_contains_probe(latest: &LatestCompleteCodeIndexV1, probe: & ); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn project_full_publication_precedes_registered_schema_convergence() { +#[test] +fn project_full_publication_precedes_registered_schema_convergence() { + tracedecay_runtime_core::native_runtime::default_builder() + .worker_threads(2) + .build() + .expect("native project composition runtime") + .block_on(verify_project_full_publication_precedes_registered_schema_convergence()); +} + +async fn verify_project_full_publication_precedes_registered_schema_convergence() { let isolation = TempDir::new().expect("production harness isolation"); let bootstrap_project = isolation.path().join("bootstrap-project"); let target_project = isolation.path().join("target-project"); @@ -265,8 +273,16 @@ async fn project_full_publication_precedes_registered_schema_convergence() { harness.shutdown().await; } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn twelve_project_journey_retires_idle_owners_without_empty_graphs() { +#[test] +fn twelve_project_journey_retires_idle_owners_without_empty_graphs() { + tracedecay_runtime_core::native_runtime::default_builder() + .worker_threads(2) + .build() + .expect("native project composition runtime") + .block_on(verify_twelve_project_journey_retires_idle_owners_without_empty_graphs()); +} + +async fn verify_twelve_project_journey_retires_idle_owners_without_empty_graphs() { let isolation = TempDir::new().expect("production harness isolation"); let mut projects = Vec::new(); for ordinal in 0..12 { diff --git a/crates/tracedecay/src/daemon/tests/rmcp_route.rs b/crates/tracedecay/src/daemon/tests/rmcp_route.rs index de5c8297ef2..bcddc8fa755 100644 --- a/crates/tracedecay/src/daemon/tests/rmcp_route.rs +++ b/crates/tracedecay/src/daemon/tests/rmcp_route.rs @@ -662,12 +662,16 @@ async fn unix_fire_and_forget_hook(fixture: &RmcpRouteFixture, event: DaemonHook } #[cfg(unix)] -async fn wait_for_new_generation( +async fn wait_for_new_text_generation( fixture: &RmcpRouteFixture, project: &std::path::Path, prior: Option<&CodeGenerationId>, + expected_paths: &[&str], step: &str, ) -> CodeGenerationId { + // Queryable text is the edit's reconciliation receipt. The complete graph + // slot can still name its predecessor while optional graph activation runs. + // Match the source paths too: an unrelated generation change proves no edit. // Well inside the 30 s query-admission backstop, so only this step's // own hook wake can publish in time. tokio::time::timeout(PHASE_TIMEOUT, async { @@ -676,17 +680,26 @@ async fn wait_for_new_generation( .engine .invocation .code_index_schedulers - .latest_generation_id(project) + .latest_text_serving_for_root(project) .await - && Some(¤t) != prior { - return current; + let metadata = current.metadata(); + if Some(&metadata.manifest().generation_id) != prior + && metadata + .snapshot() + .files + .iter() + .map(|file| file.logical_path.as_str()) + .eq(expected_paths.iter().copied()) + { + return metadata.manifest().generation_id.clone(); + } } tokio::time::sleep(Duration::from_millis(10)).await; } }) .await - .unwrap_or_else(|_| panic!("{step} hook did not wake a reconcile pass")) + .unwrap_or_else(|_| panic!("{step} hook did not reconcile source paths {expected_paths:?}")) } /// Save, rename, and delete each arrive as a fire-and-forget after-edit @@ -713,20 +726,43 @@ async fn fire_and_forget_edit_hooks_each_wake_a_reconcile() { ) }; - let initial = wait_for_new_generation(&fixture, &project, None, "initial mount").await; + let initial = + wait_for_new_text_generation(&fixture, &project, None, &["src/main.rs"], "initial mount") + .await; std::fs::write(project.join("src/saved.rs"), "pub fn saved() {}\n").expect("save source"); unix_fire_and_forget_hook(&fixture, edit(&["src/saved.rs"])).await; - let saved = wait_for_new_generation(&fixture, &project, Some(&initial), "save").await; + let saved = wait_for_new_text_generation( + &fixture, + &project, + Some(&initial), + &["src/main.rs", "src/saved.rs"], + "save", + ) + .await; std::fs::rename(project.join("src/saved.rs"), project.join("src/renamed.rs")) .expect("rename source"); unix_fire_and_forget_hook(&fixture, edit(&["src/saved.rs", "src/renamed.rs"])).await; - let renamed = wait_for_new_generation(&fixture, &project, Some(&saved), "rename").await; + let renamed = wait_for_new_text_generation( + &fixture, + &project, + Some(&saved), + &["src/main.rs", "src/renamed.rs"], + "rename", + ) + .await; std::fs::remove_file(project.join("src/renamed.rs")).expect("delete source"); unix_fire_and_forget_hook(&fixture, edit(&["src/renamed.rs"])).await; - wait_for_new_generation(&fixture, &project, Some(&renamed), "delete").await; + wait_for_new_text_generation( + &fixture, + &project, + Some(&renamed), + &["src/main.rs"], + "delete", + ) + .await; } async fn portable_one_request(fixture: &RmcpRouteFixture, request: &Value) -> Vec { diff --git a/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs b/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs index 949395a7ffd..11481f1eab2 100644 --- a/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs +++ b/crates/tracedecay/tests/mcp_suite/mcp_cli_serve_test.rs @@ -1092,13 +1092,16 @@ async fn no_explicit_path_prefers_initialize_roots_over_global_fallback() { register_global_project(home.path(), stale.path()).await; let _daemon = common::spawn_tracedecay_daemon(home.path()); - let output = run_serve_runtime_with_initialize_root( - home.path(), - cwd.path(), - None, - file_uri(active.path()), - "active", - ); + // The stdio session blocks while the initialized projects still have + // runtime-owned work, so keep the current-thread worker available. + let home_path = home.path().to_path_buf(); + let cwd_path = cwd.path().to_path_buf(); + let root_uri = file_uri(active.path()); + let output = tokio::task::spawn_blocking(move || { + run_serve_runtime_with_initialize_root(&home_path, &cwd_path, None, root_uri, "active") + }) + .await + .expect("serve runtime host session"); assert_eq!( canonical_path_string(Path::new(&runtime_project_root(&output.stdout, 2))), diff --git a/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs b/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs index 2f13fb6e77e..1c753f532ed 100644 --- a/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs +++ b/crates/tracedecay/tests/runtime_acceptance_suite/private_route_restart_acceptance.rs @@ -1,5 +1,6 @@ use crate::common; +use std::cell::RefCell; use std::path::Path; use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; @@ -175,6 +176,9 @@ async fn selected_project_source_route_survives_physical_daemon_restart() { pub(crate) fn await_published_code_index(home: &Path, project: &Path) { let project_arg = project.to_string_lossy().into_owned(); let deadline = Instant::now() + Duration::from_secs(110); + // Keep the last completed observation when the deadline falls between + // probes; the timeout itself must not issue another status request. + let last_status = RefCell::new(None::); // CLI admission spends part of the request budget before status runs, so // a nested 110-second wait can exceed the remaining dispatch deadline. // Sample the same typed predicate under one end-to-end fixture deadline. @@ -185,7 +189,12 @@ pub(crate) fn await_published_code_index(home: &Path, project: &Path) { let remaining_ms = deadline .saturating_duration_since(Instant::now()) .as_millis(); - assert!(remaining_ms > 0, "code-index publication deadline elapsed"); + assert!( + remaining_ms > 0, + "code-index publication deadline elapsed for {}; last status: {}", + project.display(), + last_status.borrow().as_ref().unwrap_or(&Value::Null), + ); let output = common::tracedecay_command_with_home(home) .current_dir(project) .env(TOOL_REQUEST_DEADLINE_ENV, remaining_ms.to_string()) @@ -225,6 +234,7 @@ pub(crate) fn await_published_code_index(home: &Path, project: &Path) { { // The canonical readiness wait allows a scheduler to mount // inside its budget. This status sample has no mount refusal. + *last_status.borrow_mut() = Some(status); return None; } StatusCodeIndexFreshnessV1::MountFailed { .. } @@ -234,7 +244,10 @@ pub(crate) fn await_published_code_index(home: &Path, project: &Path) { }; match worktree.readiness(CodeIndexReadinessTargetV1::GraphReady) { CodeIndexReadinessV1::Reached => Some(()), - CodeIndexReadinessV1::Pending => None, + CodeIndexReadinessV1::Pending => { + *last_status.borrow_mut() = Some(status); + None + } CodeIndexReadinessV1::Unreachable { reason } => { panic!("code-index readiness cannot be reached: {reason}; {status}") } @@ -242,8 +255,9 @@ pub(crate) fn await_published_code_index(home: &Path, project: &Path) { }, || { format!( - "daemon never published a serving generation for {}", - project.display() + "daemon never published a serving generation for {}; last status: {}", + project.display(), + last_status.borrow().as_ref().unwrap_or(&Value::Null), ) }, );