diff --git a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs index 64b90ebeb1e..46e055a7a75 100644 --- a/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs +++ b/crates/tracedecay-global-db/src/tests/schema_install_cancellation.rs @@ -96,7 +96,7 @@ impl RegisteredStore { /// Inspect a completed family with an ordinary independent read-only SQLite /// connection. Reopening the registered runtime for every observation would /// perform another full attachment lifecycle and obscure cancellation timing. - fn external_schema_objects(&self) -> Vec { + fn external_schema_objects(&self) -> Vec<(String, Option)> { let connection = rusqlite::Connection::open_with_flags( &self.path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, @@ -104,12 +104,12 @@ impl RegisteredStore { .unwrap(); let mut statement = connection .prepare( - "SELECT type || ':' || name FROM sqlite_schema + "SELECT type || ':' || name, sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name", ) .unwrap(); statement - .query_map([], |row| row.get::<_, String>(0)) + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) .unwrap() .map(Result::unwrap) .collect() @@ -185,12 +185,14 @@ async fn unpublished_existing_wal_is_refused_without_deleting_foreign_bytes() { #[tokio::test(flavor = "multi_thread")] async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { - let reference = RegisteredStore::create(); - let (outcome, total_polls) = reference.attach_cancelled_at(usize::MAX).await; + let store = RegisteredStore::create(); + let (outcome, total_polls) = store.attach_cancelled_at(usize::MAX).await; outcome.unwrap(); - let installed = reference.external_schema_objects(); + let installed = store.external_schema_objects(); assert!( - installed.contains(&"table:code_projects".to_owned()), + installed + .iter() + .any(|(object, _)| object == "table:code_projects"), "a completed attach installs the registered schema: {installed:?}" ); @@ -201,12 +203,11 @@ async fn cancelled_daemon_attach_preserves_published_schema_and_reopens() { .chain((2..total_polls).step_by(stride)) .chain([total_polls + 10]) { - let store = RegisteredStore::create(); - // Cancellation cannot modify the already-published schema or - // prevent the next admission from reopening the same family. - let (owner, runtime, retirement) = store.publish().await.into_parts(); - drop((owner, runtime)); - retire_runtime(retirement).await; + // The reference admission, then each successful reopen below, leave + // this family closed with the exact baseline schema. Creating another + // family would repeat initialization without exercising another + // cancellation point. Keep real runtime retirement and reopening + // around every sampled admission. let before_admission = store.external_schema_objects(); assert_eq!(before_admission, installed); let (outcome, _) = store.attach_cancelled_at(cancel_at).await; diff --git a/crates/tracedecay-graph-db/src/generation_runtime.rs b/crates/tracedecay-graph-db/src/generation_runtime.rs index cd9e1575b32..4bbcd45aa9f 100644 --- a/crates/tracedecay-graph-db/src/generation_runtime.rs +++ b/crates/tracedecay-graph-db/src/generation_runtime.rs @@ -31,8 +31,7 @@ use crate::schema::{NAMESPACE_PROPERTY, relation_kind_from_type, required_string use crate::sealed_store::{SealedReleaseEvidence, SealedStoreInstall}; use crate::state::{ EndpointIdentityCache, latest_projection, load_relation_by_edge_cached, - projection_entity_deletion_page_checked, projection_node_counts, - projection_relation_deletion_page_checked, + projection_deletion_page_checked, projection_node_counts, }; use crate::verified_marker::GenerationVerification; use crate::{ @@ -289,12 +288,6 @@ impl GenerationStageOutcome { } } -#[derive(Clone, Copy)] -enum GenerationRetirementPageKind { - Relations, - Entities, -} - impl GraphDb { #[cfg(any(feature = "test-helpers", feature = "eval-helpers"))] #[tracing::instrument( @@ -1966,57 +1959,39 @@ impl GraphDb { check: &dyn Fn() -> Result<(), GraphDbError>, ) -> Result { check()?; - let mut last_commit = None; - for kind in [ - GenerationRetirementPageKind::Relations, - GenerationRetirementPageKind::Entities, - ] { - loop { - let (commit, applied) = self.delete_projection_page_checked( - &namespace, - &projection, - &source_generation, - &watermark, - kind, - check, - )?; - last_commit = Some(commit); - if !applied { - break; - } - // A committed page stays durable. Cancellation at this exact - // boundary leaves the remaining rows for an idempotent retry. - check()?; + loop { + let (commit, applied) = self.delete_projection_page_checked( + &namespace, + &projection, + &source_generation, + &watermark, + check, + )?; + if !applied { + return Ok(commit); } + // A committed page stays durable. Cancellation at this exact + // boundary leaves the remaining rows for an idempotent retry. + check()?; } - last_commit.ok_or_else(|| GraphDbError::unavailable("graph projection disappeared")) } - #[allow(clippy::too_many_arguments)] + /// Commits one bounded page of the projection's row release: relations + /// first, then entities once no relation remains, all in one durable + /// transaction. `false` means nothing was left to delete. fn delete_projection_page_checked( &self, namespace: &GraphNamespace, projection: &crate::GraphProjectionId, source_generation: &crate::SourceGeneration, watermark: &crate::GraphWatermark, - kind: GenerationRetirementPageKind, check: &dyn Fn() -> Result<(), GraphDbError>, ) -> Result<(GraphCommit, bool), GraphDbError> { self.run_gated_batch( check, |database| { - let mutations = match kind { - GenerationRetirementPageKind::Relations => { - projection_relation_deletion_page_checked( - database, namespace, projection, check, - )? - } - GenerationRetirementPageKind::Entities => { - projection_entity_deletion_page_checked( - database, namespace, projection, check, - )? - } - }; + let mutations = + projection_deletion_page_checked(database, namespace, projection, check)?; if mutations.is_empty() { let commit = latest_projection(database, namespace, projection)? .ok_or_else(|| GraphDbError::unavailable("graph projection disappeared"))? @@ -3334,6 +3309,213 @@ mod tests { owner.close().unwrap(); } + /// A generation of `entities` entities and `relations` relations, each + /// relation joining the first two entities. + fn mixed_manifest( + namespace: &str, + entities: usize, + relations: usize, + ) -> GraphGenerationManifest { + let projection = GraphProjectionIdentity::new( + GraphNamespace::new(namespace).unwrap(), + GraphProjectionId::new("mixed").unwrap(), + ); + let endpoint = |index: usize| { + GraphEntityRef::new( + projection.clone(), + GraphEntityId::new(format!("entity:{index:05}")).unwrap(), + ) + }; + GraphGenerationManifest::new( + projection.clone(), + GraphGenerationId::new("generation-mixed").unwrap(), + SourceGeneration::new("source-mixed").unwrap(), + GraphWatermark::new("watermark-mixed").unwrap(), + vec![], + (0..entities) + .map(|index| { + GraphEntity::new( + GraphEntityId::new(format!("entity:{index:05}")).unwrap(), + BTreeSet::new(), + BTreeMap::new(), + ) + .unwrap() + }) + .collect(), + (0..relations) + .map(|index| { + GraphGenerationRelation::new( + GraphRelationId::new(format!("relation:{index:05}")).unwrap(), + endpoint(0), + endpoint(1), + GraphRelationKind::new("calls").unwrap(), + BTreeMap::new(), + ) + .unwrap() + }) + .collect(), + ) + .unwrap() + } + + fn staged_persistent_generation( + temp: &TempDir, + manifest: &GraphGenerationManifest, + ) -> (GraphDbOwner, crate::GraphDbLeaseV1) { + let (owner, database) = persistent_database(temp); + database + .apply_generation_unverified_with_digest_and_work( + arc_manifest(manifest), + &sealed_digest(manifest), + &|| Ok(()), + None, + ) + .unwrap(); + (owner, database) + } + + fn delete_one_release_page( + database: &crate::GraphDbLeaseV1, + manifest: &GraphGenerationManifest, + ) -> bool { + let (_, applied) = database + .delete_projection_page_checked( + &manifest.identity().physical_namespace().unwrap(), + &manifest.projection.projection, + &manifest.source_generation, + &manifest.watermark, + &|| Ok(()), + ) + .unwrap(); + applied + } + + /// A small generation's relations and entities fit one page, so its + /// staging rows are released by one durable transaction: the projection + /// commit sequence advances once. Planning relations and entities as + /// separate pages forced two fsync-backed commits for even three rows, + /// which is the release sweep's whole measured latency. + #[test] + fn small_mixed_generation_releases_its_rows_in_one_durable_commit() { + let temp = TempDir::new().unwrap(); + let manifest = mixed_manifest("one-commit-release", 2, 1); + let locator = + GenerationLocator::new(manifest.projection.clone(), manifest.generation.clone()); + let (owner, database) = staged_persistent_generation(&temp, &manifest); + assert_eq!( + database + .staging_generation_row_counts(&manifest.identity()) + .unwrap(), + (2, 1) + ); + let staged = database + .staging_generation_commit(&locator) + .unwrap() + .expect("staged generation commit"); + + assert!(matches!( + database.delete_staged_generation_rows(&locator, &|| Ok(())), + Ok(StagedGenerationRowsDeletion::Deleted { removed_rows: true }) + )); + let released = database + .staging_generation_commit(&locator) + .unwrap() + .expect("released projection keeps its commit"); + assert_eq!( + released.sequence, + staged.sequence + 1, + "relations and entities of a one-page generation must release in one commit" + ); + assert_eq!(released.source_generation, manifest.source_generation); + assert_eq!(released.watermark, manifest.watermark); + drop(database); + owner.close().unwrap(); + + // The single commit is the durable one: a physical reopen observes + // the same commit and no rows. + let (reopened_owner, reopened) = persistent_database(&temp); + assert_eq!( + reopened + .staging_generation_row_counts(&manifest.identity()) + .unwrap(), + (0, 0) + ); + assert_eq!( + reopened + .staging_generation_commit(&locator) + .unwrap() + .map(|commit| commit.sequence), + Some(released.sequence) + ); + drop(reopened); + reopened_owner.close().unwrap(); + } + + /// Relations and entities share one page budget: once every relation is + /// admitted, entities fill exactly the remainder of the same page. + #[test] + fn release_page_fills_its_shared_budget_with_entities_after_relations() { + let temp = TempDir::new().unwrap(); + let entities = MAX_VERIFIED_GENERATION_BATCH_MUTATIONS + 10; + let manifest = mixed_manifest("shared-release-budget", entities, 1); + let (owner, database) = staged_persistent_generation(&temp, &manifest); + + assert!(delete_one_release_page(&database, &manifest)); + assert_eq!( + database + .staging_generation_row_counts(&manifest.identity()) + .unwrap(), + (entities - (MAX_VERIFIED_GENERATION_BATCH_MUTATIONS - 1), 0), + "the first page must delete the relation and fill the rest of its budget with entities" + ); + assert!(delete_one_release_page(&database, &manifest)); + assert_eq!( + database + .staging_generation_row_counts(&manifest.identity()) + .unwrap(), + (0, 0) + ); + assert!(!delete_one_release_page(&database, &manifest)); + drop(database); + owner.close().unwrap(); + } + + /// An entity is never released by a page that leaves one of the + /// generation's relations behind: a page full of relations commits + /// relations only, and entities join the page that admits the last one. + #[test] + fn release_page_admits_entities_only_after_every_relation() { + let temp = TempDir::new().unwrap(); + let relations = MAX_VERIFIED_GENERATION_BATCH_MUTATIONS + 1; + let manifest = mixed_manifest("relations-before-entities", 2, relations); + let (owner, database) = staged_persistent_generation(&temp, &manifest); + + assert!(delete_one_release_page(&database, &manifest)); + assert_eq!( + database + .staging_generation_row_counts(&manifest.identity()) + .unwrap(), + (2, 1), + "a page filled by relations must leave every entity in place" + ); + drop(database); + owner.close().unwrap(); + + // The interrupted release resumes after a physical reopen, and the + // page that admits the last relation also releases the entities. + let (reopened_owner, reopened) = persistent_database(&temp); + assert!(delete_one_release_page(&reopened, &manifest)); + assert_eq!( + reopened + .staging_generation_row_counts(&manifest.identity()) + .unwrap(), + (0, 0) + ); + assert!(!delete_one_release_page(&reopened, &manifest)); + drop(reopened); + reopened_owner.close().unwrap(); + } + /// Retirement pages a generation out in `MAX_VERIFIED_GENERATION_BATCH_MUTATIONS` /// slices. Each slice must read only the rows it retires: reading the /// whole remaining projection to pick one page made a corpus release @@ -3506,7 +3688,6 @@ mod tests { &manifest.projection.projection, &manifest.source_generation, &manifest.watermark, - super::GenerationRetirementPageKind::Entities, &|| Ok(()), ) .unwrap(); diff --git a/crates/tracedecay-graph-db/src/state.rs b/crates/tracedecay-graph-db/src/state.rs index ba3d752a7c5..21369994fd1 100644 --- a/crates/tracedecay-graph-db/src/state.rs +++ b/crates/tracedecay-graph-db/src/state.rs @@ -866,44 +866,85 @@ pub(crate) fn projection_relations_checked( Ok(relations) } -pub(crate) fn projection_entity_deletion_page_checked( +/// One bounded durable page of a projection's row release. +/// +/// Relations come first and entities follow in the same page, under one +/// shared mutation-count and live-byte budget. Entities join only once every +/// remaining relation was admitted, so an entity is never deleted in a page +/// that leaves one of its relations behind, and a crash or cancellation +/// between pages leaves a prefix that the idempotent retry finishes. The +/// canonical mutation order (`DeleteRelation` before `DeleteEntity`) applies +/// the relation deletions first inside the one transaction. +/// +/// Planning the two kinds as separate pages forced every release into at +/// least two fsync-backed commits; a small mixed generation now releases in +/// one. +pub(crate) fn projection_deletion_page_checked( database: &GrafeoDB, namespace: &GraphNamespace, projection: &GraphProjectionId, check: &dyn Fn() -> Result<(), GraphDbError>, ) -> Result, GraphDbError> { - projection_identity_deletion_page_checked( + let mut budget = DeletionPageBudget::default(); + let (relations, relations_complete) = projection_identity_deletion_page_checked( + database, + &relation_projection_label(namespace, projection), + &DeletionRecordKind::RELATIONS, + &mut budget, + check, + )?; + let mut mutations = relations + .into_iter() + .map(|identity| GraphRelationId::new(identity).map(GraphMutation::DeleteRelation)) + .collect::, _>>()?; + if !relations_complete { + return Ok(mutations); + } + let (entities, _) = projection_identity_deletion_page_checked( database, &entity_projection_label(namespace, projection), - ENTITY_LABEL, - ENTITY_ID_PROPERTY, - MAX_VERIFIED_GENERATION_ENTITIES, - "entity", + &DeletionRecordKind::ENTITIES, + &mut budget, check, - )? - .into_iter() - .map(|identity| GraphEntityId::new(identity).map(GraphMutation::DeleteEntity)) - .collect() + )?; + for identity in entities { + mutations.push(GraphMutation::DeleteEntity(GraphEntityId::new(identity)?)); + } + Ok(mutations) } -pub(crate) fn projection_relation_deletion_page_checked( - database: &GrafeoDB, - namespace: &GraphNamespace, - projection: &GraphProjectionId, - check: &dyn Fn() -> Result<(), GraphDbError>, -) -> Result, GraphDbError> { - projection_identity_deletion_page_checked( - database, - &relation_projection_label(namespace, projection), - RELATION_LABEL, - crate::schema::RELATION_ID_PROPERTY, - MAX_VERIFIED_GENERATION_RELATIONS, - "relation", - check, - )? - .into_iter() - .map(|identity| GraphRelationId::new(identity).map(GraphMutation::DeleteRelation)) - .collect() +/// The mutation-count and live-byte budget one deletion page shares across +/// every record kind it admits. +#[derive(Default)] +struct DeletionPageBudget { + mutations: usize, + live_bytes: usize, +} + +/// How one record kind is recognized and bounded by a deletion page scan. +struct DeletionRecordKind { + record_label: &'static str, + identity_property: &'static str, + maximum_records: usize, + capacity: &'static str, + description: &'static str, +} + +impl DeletionRecordKind { + const RELATIONS: Self = Self { + record_label: RELATION_LABEL, + identity_property: crate::schema::RELATION_ID_PROPERTY, + maximum_records: MAX_VERIFIED_GENERATION_RELATIONS, + capacity: "relations", + description: "relation", + }; + const ENTITIES: Self = Self { + record_label: ENTITY_LABEL, + identity_property: ENTITY_ID_PROPERTY, + maximum_records: MAX_VERIFIED_GENERATION_ENTITIES, + capacity: "entities", + description: "entity", + }; } #[cfg(test)] @@ -923,7 +964,9 @@ pub(crate) fn retirement_page_record_reads() -> usize { RETIREMENT_PAGE_RECORD_READS.with(std::cell::Cell::get) } -/// One bounded page of identities to retire from a projection. +/// The identities of one record kind a deletion page retires, admitted +/// against the page's shared `budget`, and whether the scan admitted every +/// remaining record of that kind (`false` once the budget filled). /// /// Retirement deletes a generation page by page, so this scan must cost one /// page, not the projection: it reads owner-label candidates in index order @@ -936,26 +979,20 @@ pub(crate) fn retirement_page_record_reads() -> usize { fn projection_identity_deletion_page_checked( database: &GrafeoDB, owner_label: &str, - record_label: &str, - identity_property: &str, - maximum_records: usize, - description: &str, + kind: &DeletionRecordKind, + budget: &mut DeletionPageBudget, check: &dyn Fn() -> Result<(), GraphDbError>, -) -> Result, GraphDbError> { +) -> Result<(Vec, bool), GraphDbError> { check()?; let store = database.graph_store(); require_generation_capacity( - if record_label == ENTITY_LABEL { - "entities" - } else { - "relations" - }, + kind.capacity, nodes_with_label_count(store.as_ref(), owner_label), 0, - maximum_records, + kind.maximum_records, )?; let mut identities = BTreeSet::new(); - let mut live_bytes = 0usize; + let mut complete = true; for node in nodes_with_label(store.as_ref(), owner_label) { check()?; #[cfg(test)] @@ -964,34 +1001,42 @@ fn projection_identity_deletion_page_checked( // a record of this kind; the labeled scan skips those the same way. let Some(record) = store .get_node(node) - .filter(|record| has_native_label(record, record_label)) + .filter(|record| has_native_label(record, kind.record_label)) else { continue; }; let identity = identity_arc( - record.get_property(identity_property), - &format!("native graph {description} identity"), + record.get_property(kind.identity_property), + &format!("native graph {} identity", kind.description), )?; - let next_live_bytes = live_bytes.checked_add(identity.len()).ok_or_else(|| { - GraphDbError::budget_exhausted_count( - crate::GraphBudgetKind::Write, - MAX_VERIFIED_GENERATION_BATCH_LIVE_BYTES, - ) - })?; - if identities.len() == MAX_VERIFIED_GENERATION_BATCH_MUTATIONS + let next_live_bytes = budget + .live_bytes + .checked_add(identity.len()) + .ok_or_else(|| { + GraphDbError::budget_exhausted_count( + crate::GraphBudgetKind::Write, + MAX_VERIFIED_GENERATION_BATCH_LIVE_BYTES, + ) + })?; + if budget.mutations == MAX_VERIFIED_GENERATION_BATCH_MUTATIONS || next_live_bytes > MAX_VERIFIED_GENERATION_BATCH_LIVE_BYTES { + complete = false; break; } - live_bytes = next_live_bytes; if !identities.insert(identity.as_str().to_owned()) { return Err(GraphDbError::Corrupt { - message: format!("native graph generation repeats a {description} identity"), + message: format!( + "native graph generation repeats a {} identity", + kind.description + ), }); } + budget.live_bytes = next_live_bytes; + budget.mutations += 1; } check()?; - Ok(identities.into_iter().collect()) + Ok((identities.into_iter().collect(), complete)) } pub(crate) fn projection_relation_nodes_sorted_checked( diff --git a/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs b/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs index 5407b7d71a4..2787ba27dba 100644 --- a/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs +++ b/crates/tracedecay/src/daemon/tests/terminal_housekeeping.rs @@ -439,6 +439,12 @@ fn terminal_backlog_does_not_block_an_unrelated_store_writer() { held.rollback().await.expect("release A"); let outcome = native_runtime::timeout(Duration::from_secs(10), tick.wait()).await.expect("terminal tick settled"); assert!(matches!(outcome, native_runtime::NativeTaskOutcome::Completed(_))); + // A queued maintenance admission is supervised independently of the + // census tick. Join its fair store turn before reading the committed + // page, without admitting a second maintenance page. + native_runtime::timeout(Duration::from_secs(10), + daemon.engine.store_administration.with_writer_in(a_scope, || async {}) + ).await.expect("terminal store writer settled"); assert!(progressed.is_ok(), "B must write while A is suspended with a terminal backlog"); assert_eq!(retired_signals(&shard).await, 16, "one admission runs one bounded terminal page"); assert_eq!(retirement_frontiers(&shard).await, 1);