From e40de810faac8d7dac331e823e0731381e299cf9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 5 Oct 2026 00:12:50 +0900 Subject: [PATCH 01/16] fix(topic): reject non-finite expectation sufficient statistics --- .../topic-expectation-finite-statistics.md | 6 ++ crates/topic_measurement/src/reference.rs | 10 +++ .../tests/support/expectation_finite.rs | 90 +++++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 CHANGELOG.d/topic-expectation-finite-statistics.md create mode 100644 crates/topic_measurement/tests/support/expectation_finite.rs diff --git a/CHANGELOG.d/topic-expectation-finite-statistics.md b/CHANGELOG.d/topic-expectation-finite-statistics.md new file mode 100644 index 000000000..638075ab8 --- /dev/null +++ b/CHANGELOG.d/topic-expectation-finite-statistics.md @@ -0,0 +1,6 @@ +# Refuse non-finite topic expectation sufficient statistics + +- Validate each accumulated document-topic and topic-term count immediately after its write. A finite log-likelihood does not establish that the sufficient statistics are finite. +- Return the existing `NonFiniteEstimate` error on overflow without changing the responsibility formula, arithmetic order, model target, smoothing, or convergence tolerance. +- Add private crate regressions for document and topic accumulation overflow and an ordinary finite positive control. These exercise injected private states, not an observed public-fit success or a production incident. +- This correction reinforces the existing ADR0012 finite-intermediate contract. It does not implement or attest GPU execution, CPU/GPU parity, parameter recovery, release readiness, or protected-main integration. diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs index a1e4665d0..916f4f2c7 100644 --- a/crates/topic_measurement/src/reference.rs +++ b/crates/topic_measurement/src/reference.rs @@ -1,5 +1,9 @@ //! Bounded CPU `f64` reference estimator for TRSL-TM prevalence and lineage. +#[cfg(test)] +#[path = "../tests/support/expectation_finite.rs"] +mod expectation_finite_tests; + use std::collections::{BTreeMap, BTreeSet, HashMap}; use corpus_split::CorpusSnapshot; @@ -781,6 +785,10 @@ fn topic_proportions(eta: &[Vec]) -> Result>, TopicMeasurement type ExpectationOutput = (Vec>, Vec>, f64); +/// Compute responsibilities without returning non-finite sufficient statistics. +/// +/// Check each accumulated document/topic count after its write; a finite +/// likelihood alone does not establish that these `f64` sums are representable. fn expectation( input: &ReferenceTopicInput, theta: &[Vec], @@ -801,7 +809,9 @@ fn expectation( for topic in 0..topic_count { let expected = count * theta[document][topic] * beta[topic][term] / probability; document_topic_counts[document][topic] += expected; + require_finite(document_topic_counts[document][topic])?; beta_counts[topic][term] += expected; + require_finite(beta_counts[topic][term])?; } } } diff --git a/crates/topic_measurement/tests/support/expectation_finite.rs b/crates/topic_measurement/tests/support/expectation_finite.rs new file mode 100644 index 000000000..01ab1b899 --- /dev/null +++ b/crates/topic_measurement/tests/support/expectation_finite.rs @@ -0,0 +1,90 @@ +//! Private regression tests for finite expectation sufficient statistics. + +use super::{PrevalenceFeature, ReferenceTopicInput, expectation}; +use crate::TopicMeasurementError; +use temporal_core::EventTime; +use uuid::Uuid; + +/// Construct the same two-document private shape used by reference unit tests. +fn input(term_rows: Vec>) -> ReferenceTopicInput { + ReferenceTopicInput { + document_ids: vec![Uuid::from_u128(1), Uuid::from_u128(2)], + event_times: vec![ + EventTime::parse_rfc3339("2026-01-01T00:00:00Z").expect("first event time"), + EventTime::parse_rfc3339("2026-01-02T00:00:00Z").expect("second event time"), + ], + term_rows, + vocabulary_size: 2, + design: vec![vec![1.0], vec![1.0]], + features: vec![PrevalenceFeature::Intercept], + transition_pairs: vec![(0, 1)], + } +} + +/// Finite likelihood and per-document counts must not admit an infinite topic count. +#[test] +fn topic_count_overflow_is_rejected_with_finite_likelihood() { + let input = input(vec![vec![(0, f64::MAX)], vec![(0, f64::MAX)]]); + let theta = vec![vec![0.75, 0.25]; 2]; + let beta = vec![vec![1.0 - 1e-12, 1e-12]; 2]; + let probability = 0.75 * beta[0][0] + 0.25 * beta[1][0]; + let expected = f64::MAX * 0.75 * beta[0][0] / probability; + let likelihood = 2.0 * (f64::MAX * probability.ln()); + assert!(expected.is_finite()); + assert!(!(expected + expected).is_finite()); + assert!(likelihood.is_finite()); + assert_eq!( + expectation(&input, &theta, &beta, 2), + Err(TopicMeasurementError::NonFiniteEstimate), + "finite likelihood cannot certify finite sufficient statistics" + ); +} + +/// Finite term-level contributions must not overflow a document-level sum. +#[test] +fn document_count_overflow_is_rejected_with_finite_likelihood() { + let input = input(vec![ + vec![(0, f64::MAX * 0.7), (1, f64::MAX * 0.7)], + vec![(0, 1.0)], + ]); + let theta = vec![vec![0.9, 0.1]; 2]; + let beta = vec![vec![0.5, 0.5]; 2]; + let expected = f64::MAX * 0.7 * 0.9 * 0.5 / 0.5; + assert!(expected.is_finite()); + assert!(!(expected + expected).is_finite()); + assert!((2.0 * (f64::MAX * 0.7 * 0.5_f64.ln())).is_finite()); + assert_eq!( + expectation(&input, &theta, &beta, 2), + Err(TopicMeasurementError::NonFiniteEstimate), + "finite term-level contributions cannot certify a finite document sum" + ); +} + +/// Ordinary representable counts retain every original responsibility and shape. +#[test] +fn ordinary_counts_preserve_finite_sufficient_statistics() { + let input = input(vec![vec![(0, 8.0)], vec![(1, 4.0)]]); + let theta = vec![vec![0.75, 0.25]; 2]; + let beta = vec![vec![0.75, 0.25], vec![0.25, 0.75]]; + let (document_counts, topic_counts, likelihood) = + expectation(&input, &theta, &beta, 2).expect("finite expectation"); + assert_eq!(document_counts.len(), 2); + assert_eq!(topic_counts.len(), 2); + assert!(document_counts.iter().all(|row| row.len() == 2)); + assert!(topic_counts.iter().all(|row| row.len() == 2)); + assert!( + document_counts + .iter() + .flatten() + .all(|value| value.is_finite()) + ); + assert!(topic_counts.iter().flatten().all(|value| value.is_finite())); + assert!(likelihood.is_finite()); + assert_eq!(document_counts, vec![vec![7.2, 0.8], vec![2.0, 2.0]]); + assert_eq!(topic_counts, vec![vec![7.2, 2.0], vec![0.8, 2.0]]); + assert_eq!( + likelihood.to_bits(), + (8.0 * 0.625_f64.ln() + 4.0 * 0.375_f64.ln()).to_bits() + ); + println!("finite values: document=4/4 topic=4/4 likelihood=1/1"); +} From ded04071ef2da22e5141868805549203564e6d71 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 10 Nov 2026 16:04:43 +0900 Subject: [PATCH 02/16] fix(topic): reject normalized probability underflow --- .../topic-expectation-finite-statistics.md | 2 + crates/topic_measurement/src/reference.rs | 4 + .../tests/reference_estimator_contract.rs | 110 ++++++++++++++++++ .../tests/support/expectation_finite.rs | 25 +++- 4 files changed, 140 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.d/topic-expectation-finite-statistics.md b/CHANGELOG.d/topic-expectation-finite-statistics.md index 638075ab8..3397a121c 100644 --- a/CHANGELOG.d/topic-expectation-finite-statistics.md +++ b/CHANGELOG.d/topic-expectation-finite-statistics.md @@ -4,3 +4,5 @@ - Return the existing `NonFiniteEstimate` error on overflow without changing the responsibility formula, arithmetic order, model target, smoothing, or convergence tolerance. - Add private crate regressions for document and topic accumulation overflow and an ordinary finite positive control. These exercise injected private states, not an observed public-fit success or a production incident. - This correction reinforces the existing ADR0012 finite-intermediate contract. It does not implement or attest GPU execution, CPU/GPU parity, parameter recovery, release readiness, or protected-main integration. + +- Reject positive topic smoothing that underflows to a zero probability after beta normalization. Preserve the supplied floor and formulas; return `NonFiniteEstimate` rather than emitting a degenerate fit or treating an explicit zero sparse entry differently from its omission. Public CSR/CSC regressions include a representable tiny-floor control. diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs index 6b8689f6c..6033a94cc 100644 --- a/crates/topic_measurement/src/reference.rs +++ b/crates/topic_measurement/src/reference.rs @@ -1159,6 +1159,10 @@ fn normalize(values: &mut [f64]) -> Result<(), TopicMeasurementError> { require_finite(sum.ln())?; for value in values { *value /= sum; + // Positive smoothing must remain representable after normalization. + if !value.is_finite() || *value <= 0.0 { + return Err(TopicMeasurementError::NonFiniteEstimate); + } } Ok(()) } diff --git a/crates/topic_measurement/tests/reference_estimator_contract.rs b/crates/topic_measurement/tests/reference_estimator_contract.rs index f4bc3004a..0c33fa058 100644 --- a/crates/topic_measurement/tests/reference_estimator_contract.rs +++ b/crates/topic_measurement/tests/reference_estimator_contract.rs @@ -668,3 +668,113 @@ fn invalid_structural_inputs_and_nonconvergence_fail_closed() { .expect("finite hyperparameters"); assert!(fit_reference_topic_model(&input, &unstable).is_err()); } + +fn normalization_underflow_input(explicit_zero: bool, csc: bool) -> ReferenceTopicInput { + let (snapshot, ids, times, memberships, relations) = fixture(); + let matrix = if csc { + SparseMatrix::from_csc( + 6, + 5, + if explicit_zero { + vec![0, 3, 6, 9, 12, 13] + } else { + vec![0, 3, 6, 9, 12, 12] + }, + if explicit_zero { + vec![0, 1, 2, 0, 1, 2, 3, 4, 5, 3, 4, 5, 0] + } else { + vec![0, 1, 2, 0, 1, 2, 3, 4, 5, 3, 4, 5] + }, + if explicit_zero { + vec![ + 90.0, 85.0, 80.0, 10.0, 15.0, 20.0, 10.0, 15.0, 20.0, 90.0, 85.0, 80.0, 0.0, + ] + } else { + vec![ + 90.0, 85.0, 80.0, 10.0, 15.0, 20.0, 10.0, 15.0, 20.0, 90.0, 85.0, 80.0, + ] + }, + ) + } else { + SparseMatrix::from_csr( + 6, + 5, + if explicit_zero { + vec![0, 3, 5, 7, 9, 11, 13] + } else { + vec![0, 2, 4, 6, 8, 10, 12] + }, + if explicit_zero { + vec![0, 1, 4, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3] + } else { + vec![0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3] + }, + if explicit_zero { + vec![ + 90.0, 10.0, 0.0, 85.0, 15.0, 80.0, 20.0, 10.0, 90.0, 15.0, 85.0, 20.0, 80.0, + ] + } else { + vec![ + 90.0, 10.0, 85.0, 15.0, 80.0, 20.0, 10.0, 90.0, 15.0, 85.0, 20.0, 80.0, + ] + }, + ) + } + .expect("finite nonnegative sparse matrix"); + ReferenceTopicInput::new( + &snapshot, + ids, + &matrix, + ×, + None, + &memberships, + &relations, + ) + .expect("legal input with 600 positive total tokens") +} +fn smoothing_config(smoothing: f64) -> ReferenceTopicModelConfig { + assert!(smoothing.is_finite() && smoothing > 0.0); + ReferenceTopicModelConfig::new(2, vec![7], 2000, 1e-5) + .unwrap() + .with_hyperparameters(1.0, 0.25, 0.01, smoothing, 0.2) + .expect("documented finite positive smoothing admitted") +} +#[test] +fn normal_smoothing_preserves_zero_entry_equivalence() { + let c = smoothing_config(f64::MIN_POSITIVE); + let baseline = + fit_reference_topic_model(&normalization_underflow_input(false, false), &c).unwrap(); + for csc in [false, true] { + for explicit in [false, true] { + let fitted = + fit_reference_topic_model(&normalization_underflow_input(explicit, csc), &c) + .unwrap(); + assert_eq!(fitted, baseline); + assert!( + fitted + .topic_term_probabilities + .iter() + .flatten() + .all(|x| x.is_finite() && *x > 0.0) + ); + } + } +} +#[test] +fn normalization_underflow_is_rejected_before_returning_fit() { + let c = smoothing_config(f64::from_bits(1)); + let mut failures = Vec::new(); + for csc in [false, true] { + for explicit in [false, true] { + let result = + fit_reference_topic_model(&normalization_underflow_input(explicit, csc), &c); + if result != Err(TopicMeasurementError::NonFiniteEstimate) { + failures.push((csc, explicit)); + } + } + } + assert!( + failures.is_empty(), + "positive smoothing must not return an underflowed zero beta: {failures:?}" + ); +} diff --git a/crates/topic_measurement/tests/support/expectation_finite.rs b/crates/topic_measurement/tests/support/expectation_finite.rs index 01ab1b899..5c2344694 100644 --- a/crates/topic_measurement/tests/support/expectation_finite.rs +++ b/crates/topic_measurement/tests/support/expectation_finite.rs @@ -1,6 +1,6 @@ //! Private regression tests for finite expectation sufficient statistics. -use super::{PrevalenceFeature, ReferenceTopicInput, expectation}; +use super::{PrevalenceFeature, ReferenceTopicInput, expectation, normalize}; use crate::TopicMeasurementError; use temporal_core::EventTime; use uuid::Uuid; @@ -88,3 +88,26 @@ fn ordinary_counts_preserve_finite_sufficient_statistics() { ); println!("finite values: document=4/4 topic=4/4 likelihood=1/1"); } + +/// The shared beta normalizer never returns a zero, negative, or infinite part. +#[test] +fn normalized_positive_parts_remain_representable() { + let mut ordinary = [300.0, f64::MIN_POSITIVE]; + normalize(&mut ordinary).expect("representable positive parts"); + assert!(ordinary.iter().all(|x| x.is_finite() && *x > 0.0)); + let mut smallest = [1.0, f64::from_bits(1)]; + normalize(&mut smallest).expect("smallest part remains representable at denominator one"); + assert_eq!(smallest[1].to_bits(), 1); + // These are helper-only injected states; the public underflow is tested separately. + for mut values in [[300.0, f64::from_bits(1)], [1.0, 0.0], [2.0, -1.0]] { + assert_eq!( + normalize(&mut values), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + } + let mut division_overflow = [f64::MAX, -f64::MAX, f64::MIN_POSITIVE]; + assert_eq!( + normalize(&mut division_overflow), + Err(TopicMeasurementError::NonFiniteEstimate) + ); +} From fac3638dd3535961fe95fd3b1ce8edbdf8d70f54 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 10 Nov 2026 16:53:32 +0900 Subject: [PATCH 03/16] docs(topic): document normalization guards and regression fixtures --- crates/topic_measurement/src/reference.rs | 4 ++++ .../topic_measurement/tests/reference_estimator_contract.rs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs index 6033a94cc..39c8bd958 100644 --- a/crates/topic_measurement/src/reference.rs +++ b/crates/topic_measurement/src/reference.rs @@ -1154,6 +1154,10 @@ fn standardize_event_time(times: &[EventTime]) -> Result, TopicMeasurem .collect()) } +/// Scale beta weights by their sum and require representable positive results. +/// +/// Return `NonFiniteEstimate` when the sum has a non-finite logarithm or a +/// normalized part is non-finite or nonpositive; do not clamp supplied weights. fn normalize(values: &mut [f64]) -> Result<(), TopicMeasurementError> { let sum = values.iter().sum::(); require_finite(sum.ln())?; diff --git a/crates/topic_measurement/tests/reference_estimator_contract.rs b/crates/topic_measurement/tests/reference_estimator_contract.rs index 0c33fa058..c4e1dfeb9 100644 --- a/crates/topic_measurement/tests/reference_estimator_contract.rs +++ b/crates/topic_measurement/tests/reference_estimator_contract.rs @@ -669,6 +669,7 @@ fn invalid_structural_inputs_and_nonconvergence_fail_closed() { assert!(fit_reference_topic_model(&input, &unstable).is_err()); } +/// Build equivalent CSR/CSC fixtures with an unused fifth term and optional zero. fn normalization_underflow_input(explicit_zero: bool, csc: bool) -> ReferenceTopicInput { let (snapshot, ids, times, memberships, relations) = fixture(); let matrix = if csc { @@ -732,6 +733,7 @@ fn normalization_underflow_input(explicit_zero: bool, csc: bool) -> ReferenceTop ) .expect("legal input with 600 positive total tokens") } +/// Keep the fit configuration fixed while varying only positive topic smoothing. fn smoothing_config(smoothing: f64) -> ReferenceTopicModelConfig { assert!(smoothing.is_finite() && smoothing > 0.0); ReferenceTopicModelConfig::new(2, vec![7], 2000, 1e-5) @@ -739,6 +741,7 @@ fn smoothing_config(smoothing: f64) -> ReferenceTopicModelConfig { .with_hyperparameters(1.0, 0.25, 0.01, smoothing, 0.2) .expect("documented finite positive smoothing admitted") } +/// A representable tiny floor gives identical positive fits for all sparse forms. #[test] fn normal_smoothing_preserves_zero_entry_equivalence() { let c = smoothing_config(f64::MIN_POSITIVE); @@ -760,6 +763,7 @@ fn normal_smoothing_preserves_zero_entry_equivalence() { } } } +/// An unrepresentable beta floor fails for both orientations and zero spellings. #[test] fn normalization_underflow_is_rejected_before_returning_fit() { let c = smoothing_config(f64::from_bits(1)); From 2ad782d8f1f8f6d8272878d4c5b7a203c1a4637f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 10 Nov 2026 18:28:35 +0900 Subject: [PATCH 04/16] fix(topic): reject backwards mapped document event times --- .../topic-expectation-finite-statistics.md | 2 + crates/topic_measurement/src/reference.rs | 10 ++- .../tests/reference_estimator_contract.rs | 70 +++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.d/topic-expectation-finite-statistics.md b/CHANGELOG.d/topic-expectation-finite-statistics.md index 3397a121c..11caa5907 100644 --- a/CHANGELOG.d/topic-expectation-finite-statistics.md +++ b/CHANGELOG.d/topic-expectation-finite-statistics.md @@ -6,3 +6,5 @@ - This correction reinforces the existing ADR0012 finite-intermediate contract. It does not implement or attest GPU execution, CPU/GPU parity, parameter recovery, release readiness, or protected-main integration. - Reject positive topic smoothing that underflows to a zero probability after beta normalization. Preserve the supplied floor and formulas; return `NonFiniteEstimate` rather than emitting a degenerate fit or treating an explicit zero sparse entry differently from its omission. Public CSR/CSC regressions include a representable tiny-floor control. + +Reject observed topic transitions when the mapped document event time moves backward, even when the independently supplied relation intervals are forward-validated. Preserve equal-time pairs and document-row permutation. diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs index 39c8bd958..45b3549f8 100644 --- a/crates/topic_measurement/src/reference.rs +++ b/crates/topic_measurement/src/reference.rs @@ -139,7 +139,9 @@ impl ReferenceTopicInput { /// observed forward transition edges with both endpoints in the corpus /// affect the relational objective and emitted lineage. Inferred proposals /// and other relation kinds remain provenance only. At least one observed - /// transition pair is required. + /// transition pair is required. Mapped source document event times must not + /// be later than their target document event times, independently of the + /// intervals previously validated by the relation graph. /// /// # Errors /// @@ -189,6 +191,12 @@ impl ReferenceTopicInput { let (design, features) = build_design(&document_ids, event_times, covariates, memberships)?; let transition_pairs = collect_transition_pairs(&index_by_id, relations)?; + if transition_pairs + .iter() + .any(|&(source, target)| event_times[source] > event_times[target]) + { + return Err(TopicMeasurementError::InvalidModelInput); + } Ok(Self { document_ids, diff --git a/crates/topic_measurement/tests/reference_estimator_contract.rs b/crates/topic_measurement/tests/reference_estimator_contract.rs index c4e1dfeb9..2a3221103 100644 --- a/crates/topic_measurement/tests/reference_estimator_contract.rs +++ b/crates/topic_measurement/tests/reference_estimator_contract.rs @@ -782,3 +782,73 @@ fn normalization_underflow_is_rejected_before_returning_fit() { "positive smoothing must not return an underflowed zero beta: {failures:?}" ); } + +#[test] +fn reference_input_rejects_reverse_mapped_document_event_times() { + let (snapshot, ids, times, memberships, relations) = fixture(); + let mut reversed = times.clone(); + reversed.reverse(); + let mut one_reverse_pair = times; + one_reverse_pair.swap(0, 1); + for supplied_times in [reversed, one_reverse_pair] { + assert!(matches!( + ReferenceTopicInput::new( + &snapshot, + ids.clone(), + &separated_counts(), + &supplied_times, + None, + &memberships, + &relations, + ), + Err(TopicMeasurementError::InvalidModelInput) + )); + } +} + +#[test] +fn reference_input_admits_equal_mapped_pair_times_with_other_time_variation() { + let (snapshot, ids, mut times, memberships, relations) = fixture(); + times[1] = times[0]; + let input = ReferenceTopicInput::new( + &snapshot, + ids, + &separated_counts(), + ×, + None, + &memberships, + &relations, + ) + .expect("equal pair times do not move backward"); + let config = ReferenceTopicModelConfig::new(2, vec![7], 4, 1e30).expect("config"); + let model = fit_reference_topic_model(&input, &config).expect("finite bounded fit"); + assert!(model.objective.is_finite()); +} + +#[test] +fn reference_input_admits_permuted_document_rows_with_forward_identity_times() { + let (snapshot, mut ids, mut times, memberships, relations) = fixture(); + ids.reverse(); + times.reverse(); + let counts = SparseMatrix::from_csr( + 6, + 4, + vec![0, 2, 4, 6, 8, 10, 12], + vec![2, 3, 2, 3, 2, 3, 0, 1, 0, 1, 0, 1], + vec![ + 20.0, 80.0, 15.0, 85.0, 10.0, 90.0, 80.0, 20.0, 85.0, 15.0, 90.0, 10.0, + ], + ) + .expect("permuted matrix"); + let input = ReferenceTopicInput::new( + &snapshot, + ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("row order does not define transition order"); + assert_eq!(input.document_count(), 6); +} From 95286423ff8adba071a5b584c30ff0cf8870cf18 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 10 Nov 2026 19:56:26 +0900 Subject: [PATCH 05/16] fix(topic): reject overflowing sparse offset dimensions --- .../topic-expectation-finite-statistics.md | 2 + crates/topic_measurement/src/sparse.rs | 42 ++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.d/topic-expectation-finite-statistics.md b/CHANGELOG.d/topic-expectation-finite-statistics.md index 11caa5907..09a29d542 100644 --- a/CHANGELOG.d/topic-expectation-finite-statistics.md +++ b/CHANGELOG.d/topic-expectation-finite-statistics.md @@ -8,3 +8,5 @@ - Reject positive topic smoothing that underflows to a zero probability after beta normalization. Preserve the supplied floor and formulas; return `NonFiniteEstimate` rather than emitting a degenerate fit or treating an explicit zero sparse entry differently from its omission. Public CSR/CSC regressions include a representable tiny-floor control. Reject observed topic transitions when the mapped document event time moves backward, even when the independently supplied relation intervals are forward-validated. Preserve equal-time pairs and document-row permutation. + +- Reject CSR/CSC outer dimensions whose required offset count is not representable as `usize`, using `InvalidSparseMatrix` rather than overflowing during validation. Preserve tiny canonical storage with maximum inner dimensions. This closes an inherited checked-arithmetic constructor boundary; it does not establish a release-mode panic or change estimator arithmetic. diff --git a/crates/topic_measurement/src/sparse.rs b/crates/topic_measurement/src/sparse.rs index 9749bd0a0..45c6a5df8 100644 --- a/crates/topic_measurement/src/sparse.rs +++ b/crates/topic_measurement/src/sparse.rs @@ -88,7 +88,7 @@ impl SparseMatrix { }; if rows == 0 || columns == 0 - || offsets.len() != outer + 1 + || outer.checked_add(1) != Some(offsets.len()) || offsets.first() != Some(&0) || offsets.last().copied() != Some(indices.len()) || indices.len() != values.len() @@ -177,6 +177,46 @@ mod tests { assert_eq!(csr.row_entries(), csc.row_entries()); } + #[test] + fn maximum_outer_dimensions_return_typed_sparse_errors() { + assert_eq!( + SparseMatrix::from_csr(usize::MAX, 2, vec![0], vec![], vec![]), + Err(TopicMeasurementError::InvalidSparseMatrix) + ); + assert_eq!( + SparseMatrix::from_csc(2, usize::MAX, vec![0], vec![], vec![]), + Err(TopicMeasurementError::InvalidSparseMatrix) + ); + } + + #[test] + fn maximum_inner_dimensions_preserve_tiny_canonical_storage() { + let csr = + SparseMatrix::from_csr(1, usize::MAX, vec![0, 1], vec![usize::MAX - 1], vec![2.0]) + .expect("representable maximum inner column"); + let csc = + SparseMatrix::from_csc(usize::MAX, 1, vec![0, 1], vec![usize::MAX - 1], vec![2.0]) + .expect("representable maximum inner row"); + assert_eq!(csr.columns(), usize::MAX); + assert_eq!(csc.rows(), usize::MAX); + // Do not expand the maximum-row CSC into dense row storage. + assert_eq!(csr.row_entries(), vec![vec![(usize::MAX - 1, 2.0)]]); + } + + #[test] + fn adjacent_outer_dimensions_reject_short_offsets_without_overflow() { + for outer in [usize::MAX - 1, usize::MAX] { + assert_eq!( + SparseMatrix::from_csr(outer, 2, vec![0, 0], vec![], vec![]), + Err(TopicMeasurementError::InvalidSparseMatrix) + ); + assert_eq!( + SparseMatrix::from_csc(2, outer, vec![0, 0], vec![], vec![]), + Err(TopicMeasurementError::InvalidSparseMatrix) + ); + } + } + #[test] fn malformed_sparse_storage_fails_closed() { let error = Err(TopicMeasurementError::InvalidSparseMatrix); From 19c4e1a3dc1689a2882b7c88c70113f6fcc7dd3c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 10 Nov 2026 20:52:30 +0900 Subject: [PATCH 06/16] docs(topic): describe sparse validation and normalization contracts --- crates/topic_measurement/src/reference.rs | 2 ++ crates/topic_measurement/src/sparse.rs | 7 +++++++ .../tests/reference_estimator_contract.rs | 3 +++ 3 files changed, 12 insertions(+) diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs index 45b3549f8..9f89c24e1 100644 --- a/crates/topic_measurement/src/reference.rs +++ b/crates/topic_measurement/src/reference.rs @@ -1166,6 +1166,8 @@ fn standardize_event_time(times: &[EventTime]) -> Result, TopicMeasurem /// /// Return `NonFiniteEstimate` when the sum has a non-finite logarithm or a /// normalized part is non-finite or nonpositive; do not clamp supplied weights. +/// +/// On error the slice may be partially normalized; callers must discard it. fn normalize(values: &mut [f64]) -> Result<(), TopicMeasurementError> { let sum = values.iter().sum::(); require_finite(sum.ln())?; diff --git a/crates/topic_measurement/src/sparse.rs b/crates/topic_measurement/src/sparse.rs index 45c6a5df8..424910e60 100644 --- a/crates/topic_measurement/src/sparse.rs +++ b/crates/topic_measurement/src/sparse.rs @@ -70,6 +70,10 @@ impl SparseMatrix { ) } + /// Validate canonical compressed storage before slicing indices or values. + /// + /// Return `InvalidSparseMatrix` for unrepresentable outer offset counts or + /// malformed dimensions, offsets, indices, and non-finite stored values. fn new( rows: usize, columns: usize, @@ -177,6 +181,7 @@ mod tests { assert_eq!(csr.row_entries(), csc.row_entries()); } + /// Refuse unrepresentable CSR and CSC offset counts with the typed error. #[test] fn maximum_outer_dimensions_return_typed_sparse_errors() { assert_eq!( @@ -189,6 +194,7 @@ mod tests { ); } + /// Admit maximum inner dimensions without expanding maximum-row CSC storage. #[test] fn maximum_inner_dimensions_preserve_tiny_canonical_storage() { let csr = @@ -203,6 +209,7 @@ mod tests { assert_eq!(csr.row_entries(), vec![vec![(usize::MAX - 1, 2.0)]]); } + /// Reject short offsets at adjacent outer limits without arithmetic overflow. #[test] fn adjacent_outer_dimensions_reject_short_offsets_without_overflow() { for outer in [usize::MAX - 1, usize::MAX] { diff --git a/crates/topic_measurement/tests/reference_estimator_contract.rs b/crates/topic_measurement/tests/reference_estimator_contract.rs index 2a3221103..bbe0833b6 100644 --- a/crates/topic_measurement/tests/reference_estimator_contract.rs +++ b/crates/topic_measurement/tests/reference_estimator_contract.rs @@ -783,6 +783,7 @@ fn normalization_underflow_is_rejected_before_returning_fit() { ); } +/// Reject mapped backward document times despite forward graph intervals. #[test] fn reference_input_rejects_reverse_mapped_document_event_times() { let (snapshot, ids, times, memberships, relations) = fixture(); @@ -806,6 +807,7 @@ fn reference_input_rejects_reverse_mapped_document_event_times() { } } +/// Admit equal pair times while retaining variation elsewhere in the corpus. #[test] fn reference_input_admits_equal_mapped_pair_times_with_other_time_variation() { let (snapshot, ids, mut times, memberships, relations) = fixture(); @@ -825,6 +827,7 @@ fn reference_input_admits_equal_mapped_pair_times_with_other_time_variation() { assert!(model.objective.is_finite()); } +/// Preserve forward endpoint identity when documents and matrix rows are permuted. #[test] fn reference_input_admits_permuted_document_rows_with_forward_identity_times() { let (snapshot, mut ids, mut times, memberships, relations) = fixture(); From 1e6e88ec6fc5ca82a2fd4c25e039e72f15abd126 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 11 Nov 2026 01:19:18 +0900 Subject: [PATCH 07/16] fix(topic): preserve scaled expectation counts and likelihood parity --- .../topic-expectation-finite-statistics.md | 10 +- .../tests/fitted_candidate_k_contract.rs | 38 ++++ crates/topic_measurement/src/reference.rs | 176 +++++++++++++-- .../tests/support/expectation_finite.rs | 157 +++++++++++++ .../tests/support/responsibility_product.rs | 208 ++++++++++++++++++ 5 files changed, 575 insertions(+), 14 deletions(-) create mode 100644 crates/topic_measurement/tests/support/responsibility_product.rs diff --git a/CHANGELOG.d/topic-expectation-finite-statistics.md b/CHANGELOG.d/topic-expectation-finite-statistics.md index 09a29d542..03bee5cc7 100644 --- a/CHANGELOG.d/topic-expectation-finite-statistics.md +++ b/CHANGELOG.d/topic-expectation-finite-statistics.md @@ -1,7 +1,7 @@ # Refuse non-finite topic expectation sufficient statistics - Validate each accumulated document-topic and topic-term count immediately after its write. A finite log-likelihood does not establish that the sufficient statistics are finite. -- Return the existing `NonFiniteEstimate` error on overflow without changing the responsibility formula, arithmetic order, model target, smoothing, or convergence tolerance. +- Return the existing `NonFiniteEstimate` error on overflow without changing the responsibility formula, model target, smoothing, or convergence tolerance. - Add private crate regressions for document and topic accumulation overflow and an ordinary finite positive control. These exercise injected private states, not an observed public-fit success or a production incident. - This correction reinforces the existing ADR0012 finite-intermediate contract. It does not implement or attest GPU execution, CPU/GPU parity, parameter recovery, release readiness, or protected-main integration. @@ -10,3 +10,11 @@ Reject observed topic transitions when the mapped document event time moves backward, even when the independently supplied relation intervals are forward-validated. Preserve equal-time pairs and document-row permutation. - Reject CSR/CSC outer dimensions whose required offset count is not representable as `usize`, using `InvalidSparseMatrix` rather than overflowing during validation. Preserve tiny canonical storage with maximum inner dimensions. This closes an inherited checked-arithmetic constructor boundary; it does not establish a release-mode panic or change estimator arithmetic. + + +- Form each posterior responsibility before multiplying by its observed count. This arithmetic regrouping preserves representable tiny expected counts that a count-first intermediate product can round to zero. Public CSR/CSC fit-based regressions check aggregate rare-term count conservation against a normal-count control; this is CPU `f64` acceptance, not GPU parity or parameter-recovery evidence. + +- When the mixture is normal but an individual topic mass underflows, divide its smaller simplex factor by that normal mixture before the final product, ordering the observed-count scaling around one. When the mixture itself is subnormal or rounds to zero, decompose all positive factors into binary mantissas/exponents and use one common scaled sum for both responsibilities and log-likelihood. Scale the observed count before final binary rounding so representable expected counts survive. Regressions retain the large-count/tiny-mass and public CSR/CSC cases and add subnormal-mixture posterior/count conservation, consistent likelihood, zero/small/large-count controls, and individually rounded-zero products. The new subnormal cases are injected private normalized states, not an observed public-fit trajectory or GPU acceptance. + + +- Use the same mixture normalization and log-likelihood helper in expectation and the public in-sample likelihood consumed by statistical K selection. Actual public likelihood regressions compare the same injected subnormal/rounded-zero model states; they do not claim a public fit reached those states. diff --git a/crates/model_selection/tests/fitted_candidate_k_contract.rs b/crates/model_selection/tests/fitted_candidate_k_contract.rs index 10c9f9bc2..efeeeb35a 100644 --- a/crates/model_selection/tests/fitted_candidate_k_contract.rs +++ b/crates/model_selection/tests/fitted_candidate_k_contract.rs @@ -419,3 +419,41 @@ fn statistical_candidate_from_fit_refuses_unusable_diagnostics() { Err(ModelSelectionError::InvalidDiagnostic) ); } + +/// Statistical K scoring reuses the fitting likelihood on injected subnormal model states. +#[test] +fn statistical_candidate_preserves_subnormal_mixture_likelihood() { + let input = two_document_input([1.0, 0.0, 0.0, 1.0]); + let q = f64::from_bits(1); + for (theta, beta, expected_likelihood) in [ + ( + vec![vec![0.625, 0.375], vec![0.5, 0.5]], + vec![vec![2.0 * q, 1.0], vec![q, 1.0]], + 1.625_f64.ln() - 1074.0 * std::f64::consts::LN_2, + ), + (vec![vec![0.5, 0.5]; 2], vec![vec![q, 1.0]; 2], q.ln()), + ] { + // Only the result DTO is supplied: this is not a public fitting trajectory. + let model = ReferenceTopicModel { + seed: 7, + iterations: 4, + objective: 0.0, + topic_term_probabilities: beta, + document_topic_proportions: theta, + document_coordinate_variances: vec![vec![1.0]; 2], + prevalence_coefficients: vec![vec![0.0]], + prevalence_features: Vec::new(), + sequence_edges: Vec::new(), + connected_post_count: 0, + lineage_count: 0, + }; + let candidate = statistical_candidate_from_fit(&input, 2, &model) + .expect("positive mixture admits statistical candidate"); + let score = candidate + .held_out_log_likelihood() + .expect("statistical score"); + let expected_score = expected_likelihood - 0.5 * 5.0 * 2.0_f64.ln(); + assert!((score - expected_score).abs() < 1e-12); + assert!(candidate.is_statistically_supported()); + } +} diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs index 9f89c24e1..2001dae96 100644 --- a/crates/topic_measurement/src/reference.rs +++ b/crates/topic_measurement/src/reference.rs @@ -287,13 +287,13 @@ impl ReferenceTopicInput { if term >= self.vocabulary_size { return Err(TopicMeasurementError::InvalidModelInput); } - let probability = (0..topic_count) - .map(|topic| theta[topic] * model.topic_term_probabilities[topic][term]) - .sum::(); - let log_probability = probability.ln(); - if !log_probability.is_finite() { - return Err(TopicMeasurementError::NonFiniteEstimate); - } + let mixture = mixture_normalization( + theta, + &model.topic_term_probabilities, + term, + topic_count, + )?; + let log_probability = mixture.log_probability()?; log_likelihood += count * log_probability; } } @@ -902,6 +902,9 @@ type ExpectationOutput = (Vec>, Vec>, f64); /// /// Check each accumulated document/topic count after its write; a finite /// likelihood alone does not establish that these `f64` sums are representable. +/// Use a responsibility-first product for normal topic mass. For a subnormal +/// mixture, normalize all product mantissas with one common binary exponent so +/// numerator recovery and the likelihood use the same representable denominator. fn expectation( input: &ReferenceTopicInput, theta: &[Vec], @@ -913,14 +916,32 @@ fn expectation( let mut log_likelihood = 0.0; for (document, terms) in input.term_rows.iter().enumerate() { for &(term, count) in terms { - let probability = (0..topic_count) - .map(|topic| theta[document][topic] * beta[topic][term]) - .sum::(); - let log_probability = probability.ln(); - require_finite(log_probability)?; + let mixture = mixture_normalization(&theta[document], beta, term, topic_count)?; + let probability = mixture.probability; + let scaled = mixture.scaled; + let log_probability = mixture.log_probability()?; log_likelihood += count * log_probability; for topic in 0..topic_count { - let expected = count * theta[document][topic] * beta[topic][term] / probability; + let theta_part = theta[document][topic]; + let beta_part = beta[topic][term]; + let mass = theta_part * beta_part; + let expected = if let Some((exponent, sum)) = scaled { + scaled_expected_count(count, theta_part, beta_part, exponent, sum)? + } else if mass.is_normal() { + count * (mass / probability) + } else { + // Positive simplex factors and a positive mixture keep this + // smaller-factor ratio finite. Scale before the last product + // when count >= 1; for smaller counts form responsibility first. + // This avoids both count-first and mass-first underflow. + let larger = theta_part.max(beta_part); + let ratio = theta_part.min(beta_part) / probability; + if count >= 1.0 { + (count * larger) * ratio + } else { + count * (larger * ratio) + } + }; document_topic_counts[document][topic] += expected; require_finite(document_topic_counts[document][topic])?; beta_counts[topic][term] += expected; @@ -934,6 +955,131 @@ fn expectation( Ok((document_topic_counts, beta_counts, log_likelihood)) } +/// Shared normal or binary-scaled denominator for fitting and public likelihood consumers. +struct MixtureNormalization { + /// Ordinary binary64 product sum, used by the normal-mixture count path. + probability: f64, + /// Common exponent and mantissa sum when ordinary products lose mixture precision. + scaled: Option<(i32, f64)>, +} + +impl MixtureNormalization { + /// Return the same finite log mixture in fitting, reporting, and model selection. + fn log_probability(&self) -> Result { + let value = self.scaled.map_or_else( + || self.probability.ln(), + |(exponent, sum)| sum.ln() + f64::from(exponent) * std::f64::consts::LN_2, + ); + require_finite(value)?; + Ok(value) + } +} + +/// Select a common mixture representation without changing the posterior target. +fn mixture_normalization( + theta: &[f64], + beta: &[Vec], + term: usize, + topic_count: usize, +) -> Result { + let probability = (0..topic_count) + .map(|topic| theta[topic] * beta[topic][term]) + .sum::(); + let scaled = if probability.is_normal() { + None + } else { + Some(scaled_mixture(theta, beta, term, topic_count)?) + }; + Ok(MixtureNormalization { + probability, + scaled, + }) +} + +/// Decompose a positive finite binary64 value into a mantissa in [1, 2) and exponent. +fn positive_binary_parts(value: f64) -> Result<(f64, i32), TopicMeasurementError> { + if !value.is_finite() || value <= 0.0 { + return Err(TopicMeasurementError::NonFiniteEstimate); + } + let bits = value.to_bits(); + let field = ((bits >> 52) & 0x7ff) as i32; + let fraction = bits & ((1_u64 << 52) - 1); + if field == 0 { + let highest = fraction.ilog2(); + let mantissa_bits = (fraction << (52 - highest)) & ((1_u64 << 52) - 1); + let exponent = + i32::try_from(highest).map_err(|_| TopicMeasurementError::NonFiniteEstimate)? - 1074; + Ok((f64::from_bits((1023_u64 << 52) | mantissa_bits), exponent)) + } else { + Ok((f64::from_bits((1023_u64 << 52) | fraction), field - 1023)) + } +} + +/// Apply a binary exponent without first rounding a representable result to zero. +fn scale_binary(value: f64, exponent: i32) -> Result { + if value == 0.0 { + return Ok(0.0); + } + let (mantissa, own_exponent) = positive_binary_parts(value)?; + let combined = own_exponent + exponent; + let result = if combined > 1023 { + f64::INFINITY + } else if combined >= -1022 { + let field = + u64::try_from(combined + 1023).map_err(|_| TopicMeasurementError::NonFiniteEstimate)?; + mantissa * f64::from_bits(field << 52) + } else if combined >= -1075 { + (mantissa * 2.0_f64.powi(combined + 1074)) * f64::from_bits(1) + } else { + 0.0 + }; + require_finite(result)?; + Ok(result) +} + +/// Normalize tiny mixture products in a shared exponent domain, retaining positive mass. +fn scaled_mixture( + theta: &[f64], + beta: &[Vec], + term: usize, + topic_count: usize, +) -> Result<(i32, f64), TopicMeasurementError> { + let mut exponent = i32::MIN; + for topic in 0..topic_count { + let (_, left) = positive_binary_parts(theta[topic])?; + let (_, right) = positive_binary_parts(beta[topic][term])?; + exponent = exponent.max(left + right); + } + let mut sum = 0.0; + for topic in 0..topic_count { + let (left, left_exponent) = positive_binary_parts(theta[topic])?; + let (right, right_exponent) = positive_binary_parts(beta[topic][term])?; + sum += scale_binary(left * right, left_exponent + right_exponent - exponent)?; + } + require_finite(sum.ln())?; + Ok((exponent, sum)) +} + +/// Scale each count and unrounded product against the common tiny-mixture normalizer. +fn scaled_expected_count( + count: f64, + theta: f64, + beta: f64, + mixture_exponent: i32, + mixture_sum: f64, +) -> Result { + if count == 0.0 { + return Ok(0.0); + } + let (count_mantissa, count_exponent) = positive_binary_parts(count)?; + let (left, left_exponent) = positive_binary_parts(theta)?; + let (right, right_exponent) = positive_binary_parts(beta)?; + scale_binary( + count_mantissa * (left * right / mixture_sum), + count_exponent + left_exponent + right_exponent - mixture_exponent, + ) +} + fn update_beta( mut counts: Vec>, smoothing: f64, @@ -1611,3 +1757,7 @@ mod tests { validate_positive_definite(&[vec![2.0, 0.5], vec![0.5, 1.0]]).expect("positive definite"); } } + +#[cfg(test)] +#[path = "../tests/support/responsibility_product.rs"] +mod responsibility_product; diff --git a/crates/topic_measurement/tests/support/expectation_finite.rs b/crates/topic_measurement/tests/support/expectation_finite.rs index 5c2344694..d16ba9ac6 100644 --- a/crates/topic_measurement/tests/support/expectation_finite.rs +++ b/crates/topic_measurement/tests/support/expectation_finite.rs @@ -111,3 +111,160 @@ fn normalized_positive_parts_remain_representable() { Err(TopicMeasurementError::NonFiniteEstimate) ); } + +/// Preserve a tiny topic contribution even when its unscaled mixture mass underflows. +#[test] +fn tiny_topic_mass_preserves_representable_scaled_counts() { + let tiny = 2.0_f64.powi(-600); + for (count, other_mass, expected) in [ + (2.0_f64.powi(1000), 0.5, 2.0_f64.powi(-199)), + (1.0, tiny, tiny), + (0.5, tiny, tiny * 0.5), + (0.0, tiny, 0.0), + ] { + let input = input(vec![vec![(0, count)], vec![(1, 1.0)]]); + let theta = vec![vec![tiny, 1.0], vec![0.5, 0.5]]; + let beta = vec![vec![tiny, 1.0], vec![other_mass, 1.0 - other_mass]]; + assert_eq!(theta[0].iter().sum::().to_bits(), 1.0_f64.to_bits()); + assert_eq!(beta[0].iter().sum::().to_bits(), 1.0_f64.to_bits()); + assert_eq!(beta[1].iter().sum::().to_bits(), 1.0_f64.to_bits()); + let (document_counts, topic_counts, likelihood) = + expectation(&input, &theta, &beta, 2).expect("finite sufficient statistics"); + assert!(likelihood.is_finite()); + assert_eq!(document_counts[0][0].to_bits(), expected.to_bits()); + assert_eq!(topic_counts[0][0].to_bits(), expected.to_bits()); + } +} + +/// A subnormal mixture must still allocate exactly the observed count across topics. +#[test] +fn subnormal_mixture_preserves_posterior_count_conservation() { + let q = f64::from_bits(1); + let input = input(vec![vec![(0, 1.0)], vec![(1, 1.0)]]); + let theta = vec![vec![0.625, 0.375], vec![0.5, 0.5]]; + let beta = vec![vec![2.0 * q, 1.0], vec![q, 1.0]]; + for row in theta.iter().chain(&beta) { + assert!(row.iter().all(|x| x.is_finite() && *x > 0.0)); + assert_eq!(row.iter().sum::().to_bits(), 1.0_f64.to_bits()); + } + let (documents, topics, likelihood) = + expectation(&input, &theta, &beta, 2).expect("finite normalized state"); + let actual = documents[0].iter().sum::(); + println!( + "NUM-02 observed=1 actual={actual} document={:?} topic0={:?} likelihood={likelihood}", + documents[0], topics[0] + ); + assert!(likelihood.is_finite()); + assert!( + documents[0] + .iter() + .all(|x| x.is_finite() && *x >= 0.0 && *x <= 1.0) + ); + assert!((actual - 1.0).abs() <= 1e-14); + assert!((documents[0][0] - 10.0 / 13.0).abs() <= 1e-14); + assert!((documents[0][1] - 3.0 / 13.0).abs() <= 1e-14); +} + +/// Common scaling preserves posterior normalization, likelihood, and large/small count units. +#[test] +fn subnormal_mixture_uses_consistent_likelihood_and_count_scale() { + let q = f64::from_bits(1); + for count in [0.0, 0.5, 1.0, 2.0_f64.powi(900)] { + let input = input(vec![vec![(0, count)], vec![(1, 1.0)]]); + let theta = vec![vec![0.625, 0.375], vec![0.5, 0.5]]; + let beta = vec![vec![2.0 * q, 1.0], vec![q, 1.0]]; + let (documents, topics, likelihood) = + expectation(&input, &theta, &beta, 2).expect("finite subnormal mixture"); + let log_probability = 1.625_f64.ln() - 1074.0 * std::f64::consts::LN_2; + assert!(likelihood.is_finite()); + assert!( + (likelihood - count * log_probability).abs() + <= 1e-14 * (1.0 + (count * log_probability).abs()) + ); + if count == 0.0 { + assert_eq!(documents[0], vec![0.0, 0.0]); + } else { + assert!((documents[0].iter().sum::() / count - 1.0).abs() < 1e-14); + for (topic, fraction) in [10.0 / 13.0, 3.0 / 13.0].into_iter().enumerate() { + assert!((documents[0][topic] / count - fraction).abs() < 1e-14); + assert!((topics[topic][0] / count - fraction).abs() < 1e-14); + } + } + } +} + +/// Two individually rounded-zero products can still have positive representable mixture mass. +#[test] +fn all_rounded_zero_masses_keep_a_positive_mixture() { + let q = f64::from_bits(1); + let input = input(vec![vec![(0, 1.0)], vec![(1, 1.0)]]); + let theta = vec![vec![0.5, 0.5]; 2]; + let beta = vec![vec![q, 1.0]; 2]; + let (documents, topics, likelihood) = expectation(&input, &theta, &beta, 2) + .expect("positive exact mixture despite rounded products"); + assert_eq!(documents[0], vec![0.5, 0.5]); + assert_eq!(topics[0][0].to_bits(), 0.5_f64.to_bits()); + assert_eq!(topics[1][0].to_bits(), 0.5_f64.to_bits()); + assert!((likelihood - q.ln()).abs() < 1e-12); +} + +/// Construct a public result DTO at a private normalized state; this is not a fit trajectory. +fn model_at_state(theta: Vec>, beta: Vec>) -> super::ReferenceTopicModel { + super::ReferenceTopicModel { + seed: 7, + iterations: 4, + objective: 0.0, + topic_term_probabilities: beta, + document_topic_proportions: theta, + document_coordinate_variances: vec![vec![1.0]; 2], + prevalence_coefficients: vec![vec![0.0]], + prevalence_features: vec![PrevalenceFeature::Intercept], + sequence_edges: vec![], + connected_post_count: 0, + lineage_count: 0, + } +} + +/// Public likelihood must share the expectation mixture and log normalization. +#[test] +fn num03_public_subnormal_likelihood_matches_expectation() { + let q = f64::from_bits(1); + let input = input(vec![vec![(0, 1.0)], vec![(1, 1.0)]]); + let model = model_at_state( + vec![vec![0.625, 0.375], vec![0.5, 0.5]], + vec![vec![2.0 * q, 1.0], vec![q, 1.0]], + ); + let (_, _, expected) = expectation( + &input, + &model.document_topic_proportions, + &model.topic_term_probabilities, + 2, + ) + .expect("finite expectation"); + let public = input + .in_sample_log_likelihood(&model) + .expect("finite public likelihood"); + println!( + "NUM-03 expected={expected} public={public} difference={}", + expected - public + ); + assert!((expected - public).abs() < 1e-12); +} + +/// Rounded-zero topic products must not disagree about positive mixture admission. +#[test] +fn num03_public_rounded_zero_mixture_matches_expectation() { + let q = f64::from_bits(1); + let input = input(vec![vec![(0, 1.0)], vec![(1, 1.0)]]); + let model = model_at_state(vec![vec![0.5, 0.5]; 2], vec![vec![q, 1.0]; 2]); + let (_, _, expected) = expectation( + &input, + &model.document_topic_proportions, + &model.topic_term_probabilities, + 2, + ) + .expect("positive exact mixture"); + let public = input.in_sample_log_likelihood(&model); + println!("NUM-03 rounded-zero expectation={expected} public={public:?}"); + assert!(public.is_ok_and(|x| (x - expected).abs() < 1e-12)); +} diff --git a/crates/topic_measurement/tests/support/responsibility_product.rs b/crates/topic_measurement/tests/support/responsibility_product.rs new file mode 100644 index 000000000..1a7e53424 --- /dev/null +++ b/crates/topic_measurement/tests/support/responsibility_product.rs @@ -0,0 +1,208 @@ +//! Regressions for count conservation on public finite count inputs. + +use super::{ + ReferenceTopicInput, ReferenceTopicModelConfig, SparseMatrix, expectation, + fit_reference_topic_model, +}; +use corpus_split::{CorpusDocument, CorpusSnapshot}; +use membership_core::{ + GroupId, MemberId, MembershipAssignment, MembershipNetwork, MembershipRole, MembershipWeight, +}; +use relation_graph::{ + RelationEdge, RelationEndpointId, RelationEvidenceStatus, RelationGraph, RelationKind, +}; +use temporal_core::{ + AvailableTime, EventTime, KnowledgeCutoff, TemporalBoundary, TemporalInterval, + TemporalPrecision, +}; +use uuid::Uuid; + +/// Parse a deterministic UTC event time for the cutoff-safe corpus. +fn event_time(day: u8) -> EventTime { + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T00:00:00Z")).expect("event time") +} + +/// Build an observed forward transition between the supplied document identities. +fn relation(source: Uuid, target: Uuid, source_day: u8, target_day: u8) -> RelationEdge { + let interval = |day| { + TemporalInterval::bounded( + TemporalBoundary::Included(event_time(day)), + TemporalBoundary::Included( + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T12:00:00Z")) + .expect("interval end"), + ), + TemporalPrecision::Second, + ) + .expect("bounded interval") + }; + RelationEdge::new( + RelationKind::TransitionsTo, + RelationEndpointId::from_uuid(source), + RelationEndpointId::from_uuid(target), + RelationEvidenceStatus::Observed, + interval(source_day), + interval(target_day), + ) + .expect("forward relation") +} + +/// Build six eligible documents with crossed organization/project memberships. +fn fixture() -> ( + CorpusSnapshot, + Vec, + Vec, + MembershipNetwork, + RelationGraph, +) { + let document_ids: Vec<_> = (1_u128..=6).map(Uuid::from_u128).collect(); + let times: Vec<_> = (1_u8..=6).map(event_time).collect(); + let available = AvailableTime::parse_rfc3339("2026-01-10T00:00:00Z").expect("available"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + let mut snapshot = CorpusSnapshot::new(); + for id in &document_ids { + snapshot + .insert_if_eligible(CorpusDocument::new(*id, available), &cutoff) + .expect("eligible"); + } + + let organization = GroupId::from_uuid(Uuid::from_u128(100)); + let projects = [ + GroupId::from_uuid(Uuid::from_u128(101)), + GroupId::from_uuid(Uuid::from_u128(102)), + ]; + let validity_start = event_time(1); + let validity_end = event_time(9); + let mut memberships = MembershipNetwork::new(); + for (index, id) in document_ids.iter().enumerate() { + let member = MemberId::from_uuid(*id); + memberships + .insert( + MembershipAssignment::new( + member, + organization, + MembershipRole::Organization, + MembershipWeight::full().expect("full"), + validity_start, + validity_end, + ) + .expect("organization membership"), + ) + .expect("insert organization"); + memberships + .insert( + MembershipAssignment::new( + member, + projects[usize::from(index >= 3)], + MembershipRole::Project, + MembershipWeight::new(0.75).expect("partial"), + validity_start, + validity_end, + ) + .expect("project membership"), + ) + .expect("insert project"); + } + + let mut relations = RelationGraph::new(); + for (source, target, source_day, target_day) in [ + (0, 1, 1, 2), + (1, 2, 2, 3), + (2, 3, 3, 4), + (3, 4, 4, 5), + (4, 5, 5, 6), + ] { + relations + .insert(relation( + document_ids[source], + document_ids[target], + source_day, + target_day, + )) + .expect("insert relation"); + } + (snapshot, document_ids, times, memberships, relations) +} + +/// Fit the public CPU reference with identical finite CSR or CSC rare-term counts. +fn fit_tiny_term(count: f64, csc: bool) -> (ReferenceTopicInput, super::ReferenceTopicModel) { + let (snapshot, ids, times, memberships, relations) = fixture(); + let matrix = if csc { + SparseMatrix::from_csc( + 6, + 2, + vec![0, 6, 12], + vec![0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5], + [vec![100.0; 6], vec![count; 6]].concat(), + ) + } else { + SparseMatrix::from_csr( + 6, + 2, + vec![0, 2, 4, 6, 8, 10, 12], + vec![0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1], + (0..6).flat_map(|_| [100.0, count]).collect(), + ) + } + .expect("finite positive canonical matrix"); + let input = ReferenceTopicInput::new( + &snapshot, + ids, + &matrix, + ×, + None, + &memberships, + &relations, + ) + .expect("actual public constructor admits counts"); + let config = ReferenceTopicModelConfig::new(2, vec![7], 2000, 1e-5) + .unwrap() + .with_hyperparameters(1.0, 0.25, 0.01, 1e-300, 0.2) + .unwrap(); + let fit = + fit_reference_topic_model(&input, &config).expect("public fit must return finite model"); + assert!(fit.objective.is_finite()); + assert!(input.in_sample_log_likelihood(&fit).unwrap().is_finite()); + assert!( + fit.topic_term_probabilities + .iter() + .all(|row| row.len() == 2 && row.iter().all(|x| x.is_finite() && *x > 0.0)) + ); + (input, fit) +} +/// Each observed term distributes all its representable count across topics. +fn assert_rare_count_conservation(count: f64, csc: bool) { + let (input, fit) = fit_tiny_term(count, csc); + let (_, counts, ll) = expectation( + &input, + &fit.document_topic_proportions, + &fit.topic_term_probabilities, + 2, + ) + .expect("actual unchanged expectation on public fitted state"); + let actual = counts.iter().map(|row| row[1]).sum::(); + let expected = 6.0 * count; + println!( + "csc={csc} count={count:e} expected_rare_total={expected:e} actual_rare_total={actual:e} beta={:?} likelihood={ll:e}", + fit.topic_term_probabilities + ); + assert!(expected.is_finite() && expected > 0.0); + assert!(ll.is_finite()); + assert!( + (actual / expected - 1.0).abs() < 1e-12, + "real expectation must conserve representable rare counts: actual={actual:e} expected={expected:e}" + ); +} +/// Preserve aggregate rare-term counts when all intermediate products are normal. +#[test] +fn normal_product_control_preserves_rare_term_counts() { + for csc in [false, true] { + assert_rare_count_conservation(1e-100, csc); + } +} +/// Preserve representable rare counts even when a count-first product underflows. +#[test] +fn representable_rare_term_counts_survive_intermediate_products() { + for csc in [false, true] { + assert_rare_count_conservation(1e-200, csc); + } +} From 4dfc478b95e903f4b9811b99ff83c0349be0aa74 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 11 Nov 2026 13:22:50 +0900 Subject: [PATCH 08/16] fix(model-selection): bind candidate K to fitted topic dimension --- crates/model_selection/src/fitted.rs | 9 +- .../tests/fitted_k_identity_contract.rs | 136 ++++++++++++++++++ 2 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 crates/model_selection/tests/fitted_k_identity_contract.rs diff --git a/crates/model_selection/src/fitted.rs b/crates/model_selection/src/fitted.rs index a65ca8ed0..3db4528ec 100644 --- a/crates/model_selection/src/fitted.rs +++ b/crates/model_selection/src/fitted.rs @@ -158,12 +158,19 @@ impl FittedCandidateKConfig { /// /// Returns [`ModelSelectionError::NonPositiveCandidateK`] when `candidate_k` /// is less than two, or [`ModelSelectionError::InvalidDiagnostic`] when the -/// fitted dimensions, likelihood, or parameter count are unusable. +/// fitted dimensions, likelihood, or parameter count are unusable, including +/// a candidate topic count that disagrees with the supplied fitted model. pub fn statistical_candidate_from_fit( input: &ReferenceTopicInput, candidate_k: u32, model: &ReferenceTopicModel, ) -> Result { + if candidate_k < 2 { + return Err(ModelSelectionError::NonPositiveCandidateK); + } + if usize::try_from(candidate_k).ok() != Some(model.topic_term_probabilities.len()) { + return Err(ModelSelectionError::InvalidDiagnostic); + } let log_likelihood = input .in_sample_log_likelihood(model) .map_err(|_| ModelSelectionError::InvalidDiagnostic)?; diff --git a/crates/model_selection/tests/fitted_k_identity_contract.rs b/crates/model_selection/tests/fitted_k_identity_contract.rs new file mode 100644 index 000000000..f51f9e7aa --- /dev/null +++ b/crates/model_selection/tests/fitted_k_identity_contract.rs @@ -0,0 +1,136 @@ +//! Fitted candidate-`K` scoring uses real TRSL-TM reference fits. + +use corpus_split::{CorpusDocument, CorpusSnapshot}; +use membership_core::{ + GroupId, MemberId, MembershipAssignment, MembershipNetwork, MembershipRole, MembershipWeight, +}; +use model_selection::{ModelSelectionError, statistical_candidate_from_fit}; +use relation_graph::{ + RelationEdge, RelationEndpointId, RelationEvidenceStatus, RelationGraph, RelationKind, +}; +use temporal_core::{ + AvailableTime, EventTime, KnowledgeCutoff, TemporalBoundary, TemporalInterval, + TemporalPrecision, +}; +use topic_measurement::{ + ReferenceTopicInput, ReferenceTopicModelConfig, SparseMatrix, fit_reference_topic_model, +}; +use uuid::Uuid; + +fn event_time(day: u8) -> EventTime { + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T00:00:00Z")).expect("event time") +} + +fn relation(source: Uuid, target: Uuid, source_day: u8, target_day: u8) -> RelationEdge { + let interval = |day| { + TemporalInterval::bounded( + TemporalBoundary::Included(event_time(day)), + TemporalBoundary::Included( + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T12:00:00Z")) + .expect("interval end"), + ), + TemporalPrecision::Second, + ) + .expect("bounded interval") + }; + RelationEdge::new( + RelationKind::TransitionsTo, + RelationEndpointId::from_uuid(source), + RelationEndpointId::from_uuid(target), + RelationEvidenceStatus::Observed, + interval(source_day), + interval(target_day), + ) + .expect("forward relation") +} + +fn two_document_input(term_values: [f64; 4]) -> ReferenceTopicInput { + let document_ids = vec![Uuid::from_u128(1), Uuid::from_u128(2)]; + let times = vec![event_time(1), event_time(2)]; + let available = AvailableTime::parse_rfc3339("2026-01-10T00:00:00Z").expect("available"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + let mut snapshot = CorpusSnapshot::new(); + for id in &document_ids { + snapshot + .insert_if_eligible(CorpusDocument::new(*id, available), &cutoff) + .expect("eligible"); + } + let mut memberships = MembershipNetwork::new(); + memberships + .insert( + MembershipAssignment::new( + MemberId::from_uuid(document_ids[0]), + GroupId::from_uuid(Uuid::from_u128(100)), + MembershipRole::Organization, + MembershipWeight::full().expect("full"), + event_time(1), + event_time(9), + ) + .expect("membership"), + ) + .expect("insert"); + memberships + .insert( + MembershipAssignment::new( + MemberId::from_uuid(document_ids[1]), + GroupId::from_uuid(Uuid::from_u128(100)), + MembershipRole::Organization, + MembershipWeight::full().expect("full"), + event_time(1), + event_time(9), + ) + .expect("membership"), + ) + .expect("insert"); + let mut relations = RelationGraph::new(); + relations + .insert(relation(document_ids[0], document_ids[1], 1, 2)) + .expect("relation"); + let counts = + SparseMatrix::from_csr(2, 2, vec![0, 2, 4], vec![0, 1, 0, 1], term_values.to_vec()) + .expect("counts"); + ReferenceTopicInput::new( + &snapshot, + document_ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("two-document input") +} + +/// Matching candidate identity uses a real public fit, not an injected model DTO. +#[test] +fn candidate_k_matches_real_fit() { + let input = two_document_input([8.0, 0.0, 0.0, 4.0]); + let config = ReferenceTopicModelConfig::new(2, vec![7], 2000, 1e-5).expect("config"); + let model = fit_reference_topic_model(&input, &config).expect("actual CPU fit"); + assert_eq!(model.topic_term_probabilities.len(), 2); + let candidate = statistical_candidate_from_fit(&input, 2, &model).expect("matching K"); + assert_eq!(candidate.candidate_k(), 2); + assert!(candidate.is_statistically_supported()); +} +/// A real two-topic fit must not supply evidence for any other candidate K. +#[test] +fn wrong_candidate_k_is_rejected() { + let input = two_document_input([8.0, 0.0, 0.0, 4.0]); + let config = ReferenceTopicModelConfig::new(2, vec![7], 2000, 1e-5).expect("config"); + let model = fit_reference_topic_model(&input, &config).expect("actual CPU fit"); + assert_eq!(model.topic_term_probabilities.len(), 2); + for k in [0, 1, 3, 4, u32::MAX] { + let actual = statistical_candidate_from_fit(&input, k, &model); + println!("REAL_FIT_K actual_model_topics=2 supplied_k={k} result={actual:?}"); + let expected = if k < 2 { + ModelSelectionError::NonPositiveCandidateK + } else { + ModelSelectionError::InvalidDiagnostic + }; + assert_eq!( + actual, + Err(expected), + "candidate K must bind to the real fitted dimension" + ); + } +} From b56a54cbb9b4358c0721d8d62483c3a59dc2cf66 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 11 Nov 2026 14:24:54 +0900 Subject: [PATCH 09/16] test(topic-measurement): cover binary scale boundary outcomes --- .../tests/fitted_k_identity_contract.rs | 4 +- crates/topic_measurement/src/reference.rs | 4 ++ .../tests/support/binary_scale_contract.rs | 47 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 crates/topic_measurement/tests/support/binary_scale_contract.rs diff --git a/crates/model_selection/tests/fitted_k_identity_contract.rs b/crates/model_selection/tests/fitted_k_identity_contract.rs index f51f9e7aa..d35e48955 100644 --- a/crates/model_selection/tests/fitted_k_identity_contract.rs +++ b/crates/model_selection/tests/fitted_k_identity_contract.rs @@ -17,10 +17,12 @@ use topic_measurement::{ }; use uuid::Uuid; +/// Parse a deterministic event-time fixture for a January day. fn event_time(day: u8) -> EventTime { EventTime::parse_rfc3339(&format!("2026-01-{day:02}T00:00:00Z")).expect("event time") } +/// Construct an observed forward relation with bounded same-day intervals. fn relation(source: Uuid, target: Uuid, source_day: u8, target_day: u8) -> RelationEdge { let interval = |day| { TemporalInterval::bounded( @@ -44,6 +46,7 @@ fn relation(source: Uuid, target: Uuid, source_day: u8, target_day: u8) -> Relat .expect("forward relation") } +/// Build a cutoff-eligible two-document corpus with shared membership and a transition. fn two_document_input(term_values: [f64; 4]) -> ReferenceTopicInput { let document_ids = vec![Uuid::from_u128(1), Uuid::from_u128(2)]; let times = vec![event_time(1), event_time(2)]; @@ -121,7 +124,6 @@ fn wrong_candidate_k_is_rejected() { assert_eq!(model.topic_term_probabilities.len(), 2); for k in [0, 1, 3, 4, u32::MAX] { let actual = statistical_candidate_from_fit(&input, k, &model); - println!("REAL_FIT_K actual_model_topics=2 supplied_k={k} result={actual:?}"); let expected = if k < 2 { ModelSelectionError::NonPositiveCandidateK } else { diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs index 2001dae96..33df1e904 100644 --- a/crates/topic_measurement/src/reference.rs +++ b/crates/topic_measurement/src/reference.rs @@ -1761,3 +1761,7 @@ mod tests { #[cfg(test)] #[path = "../tests/support/responsibility_product.rs"] mod responsibility_product; + +#[cfg(test)] +#[path = "../tests/support/binary_scale_contract.rs"] +mod binary_scale_contract; diff --git a/crates/topic_measurement/tests/support/binary_scale_contract.rs b/crates/topic_measurement/tests/support/binary_scale_contract.rs new file mode 100644 index 000000000..3a7d77eeb --- /dev/null +++ b/crates/topic_measurement/tests/support/binary_scale_contract.rs @@ -0,0 +1,47 @@ +//! Direct numeric boundary controls for the shared binary64 scale helpers. + +use super::{positive_binary_parts, scale_binary, scaled_expected_count}; +use crate::TopicMeasurementError; + +/// Zero, overflow, normal, subnormal, and below-half-ulp scaling keep typed outcomes. +#[test] +fn binary_scale_covers_representable_and_refused_boundaries() { + assert_eq!(scale_binary(0.0, 0), Ok(0.0)); + assert_eq!(scale_binary(1.0, 0), Ok(1.0)); + assert_eq!(scale_binary(1.0, 1023), Ok(2.0_f64.powi(1023))); + assert_eq!( + scale_binary(1.0, 1024), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + assert_eq!(scale_binary(1.0, -1022), Ok(f64::MIN_POSITIVE)); + assert_eq!(scale_binary(1.0, -1074), Ok(f64::from_bits(1))); + assert_eq!(scale_binary(1.0, -1075), Ok(0.0)); + assert_eq!(scale_binary(1.5, -1075), Ok(f64::from_bits(1))); + assert_eq!(scale_binary(1.0, -1076), Ok(0.0)); + assert_eq!( + scale_binary(-1.0, 0), + Err(TopicMeasurementError::NonFiniteEstimate) + ); +} + +/// Invalid count or factor inputs must not bypass direct helper finite admission. +#[test] +fn binary_parts_and_scaled_counts_reject_invalid_inputs() { + for value in [0.0, -1.0, f64::INFINITY, f64::NAN] { + assert_eq!( + positive_binary_parts(value), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + } + assert_eq!(positive_binary_parts(f64::from_bits(1)), Ok((1.0, -1074))); + assert_eq!(positive_binary_parts(f64::MIN_POSITIVE), Ok((1.0, -1022))); + assert_eq!(positive_binary_parts(1.5), Ok((1.5, 0))); + assert_eq!(scaled_expected_count(0.0, 1.0, 1.0, 0, 1.0), Ok(0.0)); + assert_eq!(scaled_expected_count(2.0, 0.5, 0.5, -2, 1.0), Ok(2.0)); + for (count, theta, beta) in [(-1.0, 1.0, 1.0), (1.0, 0.0, 1.0), (1.0, 1.0, 0.0)] { + assert_eq!( + scaled_expected_count(count, theta, beta, 0, 1.0), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + } +} From d2541ccfcc77e741d5b76a76b01fefdd5038bd70 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 11 Nov 2026 16:07:50 +0900 Subject: [PATCH 10/16] test(topic-measurement): preserve provenance exclusion at fit boundary --- .../tests/reference_estimator_contract.rs | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/crates/topic_measurement/tests/reference_estimator_contract.rs b/crates/topic_measurement/tests/reference_estimator_contract.rs index bbe0833b6..2397630ec 100644 --- a/crates/topic_measurement/tests/reference_estimator_contract.rs +++ b/crates/topic_measurement/tests/reference_estimator_contract.rs @@ -855,3 +855,101 @@ fn reference_input_admits_permuted_document_rows_with_forward_identity_times() { .expect("row order does not define transition order"); assert_eq!(input.document_count(), 6); } + +/// Construct every non-transition relation pointing to an earlier corpus document. +fn provenance_only_graph(ids: &[Uuid]) -> RelationGraph { + let mut graph = RelationGraph::new(); + for kind in [ + RelationKind::References, + RelationKind::Summarizes, + RelationKind::Revises, + RelationKind::Translates, + RelationKind::RetrospectivelyReports, + RelationKind::Supports, + RelationKind::Contradicts, + RelationKind::OutcomeOf, + ] { + let temporal = relation(ids[0], ids[5], 1, 6); + graph + .insert( + RelationEdge::new( + kind, + temporal.target(), + temporal.source(), + RelationEvidenceStatus::Observed, + temporal.target_event_time(), + temporal.source_event_time(), + ) + .expect("backward provenance remains valid"), + ) + .expect("distinct provenance kind"); + } + assert_eq!(graph.edges().count(), 8); + assert!(graph.edges().all(|edge| !edge.is_transition_edge())); + graph +} + +/// Backward provenance alone cannot satisfy the observed-transition requirement. +#[test] +fn provenance_only_input_does_not_become_a_state_transition() { + let (snapshot, ids, times, memberships, _) = fixture(); + let graph = provenance_only_graph(&ids); + assert!(matches!( + ReferenceTopicInput::new( + &snapshot, + ids, + &separated_counts(), + ×, + None, + &memberships, + &graph, + ), + Err(TopicMeasurementError::InvalidModelInput) + )); + assert_eq!(graph.edges().count(), 8); +} + +/// Valid provenance is retained without changing fitted parameters, lineage, or precision. +#[test] +fn mixed_provenance_is_numerically_inert_at_public_fit_boundary() { + let (snapshot, ids, times, memberships, observed) = fixture(); + let provenance = provenance_only_graph(&ids); + let mut mixed = observed.clone(); + for edge in provenance.edges() { + mixed.insert(*edge).expect("retain provenance"); + } + assert_eq!(mixed.edges().count(), observed.edges().count() + 8); + let counts = separated_counts(); + let build = |graph: &RelationGraph| { + ReferenceTopicInput::new( + &snapshot, + ids.clone(), + &counts, + ×, + None, + &memberships, + graph, + ) + .expect("observed transitions remain available") + }; + let baseline_input = build(&observed); + let mixed_input = build(&mixed); + let config = ReferenceTopicModelConfig::new(2, vec![7], 2_000, 1e-5).expect("config"); + let baseline = fit_reference_topic_model(&baseline_input, &config).expect("baseline fit"); + let candidate = fit_reference_topic_model(&mixed_input, &config).expect("mixed fit"); + assert!(baseline.objective.is_finite()); + assert_eq!(candidate, baseline); + let topic_ids = vec![Uuid::from_u128(201), Uuid::from_u128(202)]; + let baseline_precision = baseline_input + .build_joint_coordinate_precision(&baseline, &config, topic_ids.clone()) + .expect("baseline precision"); + let mixed_precision = mixed_input + .build_joint_coordinate_precision(&candidate, &config, topic_ids) + .expect("mixed precision"); + assert_eq!(mixed_precision.values(), baseline_precision.values()); + assert_eq!( + mixed_precision.coordinate_means(), + baseline_precision.coordinate_means() + ); + assert_eq!(mixed.edges().count(), observed.edges().count() + 8); +} From 2bb2373867d1a21771d8dd3bdc47972c93f1d9dc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 11 Nov 2026 17:10:49 +0900 Subject: [PATCH 11/16] fix(topic-measurement): retain zero factors in tiny mixtures --- .../topic-expectation-finite-statistics.md | 2 + .../tests/fitted_k_identity_contract.rs | 55 +++++++++++++++++++ crates/topic_measurement/src/reference.rs | 20 ++++++- .../tests/support/expectation_finite.rs | 55 +++++++++++++++++++ 4 files changed, 131 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.d/topic-expectation-finite-statistics.md b/CHANGELOG.d/topic-expectation-finite-statistics.md index 03bee5cc7..7d42a7389 100644 --- a/CHANGELOG.d/topic-expectation-finite-statistics.md +++ b/CHANGELOG.d/topic-expectation-finite-statistics.md @@ -18,3 +18,5 @@ Reject observed topic transitions when the mapped document event time moves back - Use the same mixture normalization and log-likelihood helper in expectation and the public in-sample likelihood consumed by statistical K selection. Actual public likelihood regressions compare the same injected subnormal/rounded-zero model states; they do not claim a public fit reached those states. + +- Preserve exact-zero topic factors as zero mass when another topic supplies a positive subnormal mixture. Keep the positive-factor decomposition strict, reject nonfinite or negative partners before skipping a zero product, and retain all-zero-mixture refusal. Private expectation/count and public likelihood/statistical-candidate regressions distinguish injected model states from the unchanged positive fitted-model path. diff --git a/crates/model_selection/tests/fitted_k_identity_contract.rs b/crates/model_selection/tests/fitted_k_identity_contract.rs index d35e48955..85029788c 100644 --- a/crates/model_selection/tests/fitted_k_identity_contract.rs +++ b/crates/model_selection/tests/fitted_k_identity_contract.rs @@ -136,3 +136,58 @@ fn wrong_candidate_k_is_rejected() { ); } } + +/// Public likelihood and candidate scoring admit exact zeros beside a positive tiny mixture. +#[test] +fn zero_factor_subnormal_public_likelihood_and_candidate() { + let input = two_document_input([1.0, 0.0, 0.0, 1.0]); + let config = ReferenceTopicModelConfig::new(2, vec![7], 2000, 1e-5).expect("config"); + let fitted = fit_reference_topic_model(&input, &config).expect("actual CPU fit"); + for tiny in [0.25, f64::from_bits(1_u64 << 44)] { + for (theta, beta) in [ + ( + vec![1.0, 0.0], + vec![vec![tiny, 1.0 - tiny], vec![tiny, 1.0 - tiny]], + ), + (vec![0.5, 0.5], vec![vec![tiny, 1.0 - tiny], vec![0.0, 1.0]]), + ( + vec![0.0, 1.0], + vec![vec![tiny, 1.0 - tiny], vec![tiny, 1.0 - tiny]], + ), + ( + vec![0.5, 0.5], + vec![vec![-0.0, 1.0], vec![tiny, 1.0 - tiny]], + ), + ] { + let mut model = fitted.clone(); + model.document_topic_proportions = vec![theta, vec![0.5, 0.5]]; + model.topic_term_probabilities = beta; + let expected = model + .document_topic_proportions + .iter() + .enumerate() + .map(|(row, weights)| { + let probability = weights + .iter() + .enumerate() + .map(|(topic, weight)| weight * model.topic_term_probabilities[topic][row]) + .sum::(); + assert!(probability.is_finite() && probability > 0.0); + probability.ln() + }) + .sum::(); + let actual = input + .in_sample_log_likelihood(&model) + .expect("finite positive mixtures"); + assert!((actual - expected).abs() <= 1e-12); + let candidate = statistical_candidate_from_fit(&input, 2, &model) + .expect("same public likelihood consumer"); + assert!(candidate.is_statistically_supported()); + } + } + let mut invalid = fitted; + invalid.document_topic_proportions = vec![vec![1.0, 0.0]; 2]; + invalid.topic_term_probabilities = vec![vec![0.0, 1.0], vec![1.0, 0.0]]; + assert!(input.in_sample_log_likelihood(&invalid).is_err()); + assert!(statistical_candidate_from_fit(&input, 2, &invalid).is_err()); +} diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs index 33df1e904..7382871c1 100644 --- a/crates/topic_measurement/src/reference.rs +++ b/crates/topic_measurement/src/reference.rs @@ -926,7 +926,11 @@ fn expectation( let beta_part = beta[topic][term]; let mass = theta_part * beta_part; let expected = if let Some((exponent, sum)) = scaled { - scaled_expected_count(count, theta_part, beta_part, exponent, sum)? + if has_zero_mass(theta_part, beta_part)? { + 0.0 + } else { + scaled_expected_count(count, theta_part, beta_part, exponent, sum)? + } } else if mass.is_normal() { count * (mass / probability) } else { @@ -1037,6 +1041,14 @@ fn scale_binary(value: f64, exponent: i32) -> Result Ok(result) } +/// Admit a zero product only when both original factors are finite and nonnegative. +fn has_zero_mass(left: f64, right: f64) -> Result { + if !left.is_finite() || !right.is_finite() || left < 0.0 || right < 0.0 { + return Err(TopicMeasurementError::NonFiniteEstimate); + } + Ok(left == 0.0 || right == 0.0) +} + /// Normalize tiny mixture products in a shared exponent domain, retaining positive mass. fn scaled_mixture( theta: &[f64], @@ -1046,12 +1058,18 @@ fn scaled_mixture( ) -> Result<(i32, f64), TopicMeasurementError> { let mut exponent = i32::MIN; for topic in 0..topic_count { + if has_zero_mass(theta[topic], beta[topic][term])? { + continue; + } let (_, left) = positive_binary_parts(theta[topic])?; let (_, right) = positive_binary_parts(beta[topic][term])?; exponent = exponent.max(left + right); } let mut sum = 0.0; for topic in 0..topic_count { + if has_zero_mass(theta[topic], beta[topic][term])? { + continue; + } let (left, left_exponent) = positive_binary_parts(theta[topic])?; let (right, right_exponent) = positive_binary_parts(beta[topic][term])?; sum += scale_binary(left * right, left_exponent + right_exponent - exponent)?; diff --git a/crates/topic_measurement/tests/support/expectation_finite.rs b/crates/topic_measurement/tests/support/expectation_finite.rs index d16ba9ac6..8d5a9f463 100644 --- a/crates/topic_measurement/tests/support/expectation_finite.rs +++ b/crates/topic_measurement/tests/support/expectation_finite.rs @@ -268,3 +268,58 @@ fn num03_public_rounded_zero_mixture_matches_expectation() { println!("NUM-03 rounded-zero expectation={expected} public={public:?}"); assert!(public.is_ok_and(|x| (x - expected).abs() < 1e-12)); } + +/// Exact-zero topic factors preserve zero responsibility beside positive subnormal mass. +#[test] +fn zero_factor_subnormal_expectation_preserves_counts() { + let tiny = f64::from_bits(1_u64 << 44); + for count in [0.0, 0.5, 1.0] { + for (first_theta, first_beta, fraction) in [ + (vec![1.0, 0.0], vec![tiny, tiny], 1.0), + (vec![0.5, 0.5], vec![tiny, 0.0], 1.0), + (vec![0.0, 1.0], vec![tiny, tiny], 0.0), + (vec![0.5, 0.5], vec![-0.0, tiny], 0.0), + ] { + let input = input(vec![vec![(0, count)], vec![(1, 1.0)]]); + let theta = vec![first_theta.clone(), vec![0.5, 0.5]]; + let beta = vec![vec![first_beta[0], 1.0], vec![first_beta[1], 1.0]]; + let probability = first_theta[0] * first_beta[0] + first_theta[1] * first_beta[1]; + assert!(probability > 0.0 && probability.is_subnormal()); + let (documents, topics, likelihood) = + expectation(&input, &theta, &beta, 2).expect("zero factors are zero mass"); + assert_eq!( + documents[0], + vec![count * fraction, count * (1.0 - fraction)] + ); + assert_eq!(topics[0][0].to_bits(), (count * fraction).to_bits()); + assert_eq!(topics[1][0].to_bits(), (count * (1.0 - fraction)).to_bits()); + assert!((likelihood - count * probability.ln()).abs() <= 1e-12); + } + } +} + +/// A zero mixture and nonfinite or negative partners remain invalid scaled inputs. +#[test] +fn zero_factor_subnormal_expectation_preserves_refusals() { + let tiny = f64::from_bits(1_u64 << 44); + let input = input(vec![vec![(0, 1.0)], vec![(1, 1.0)]]); + for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY, -1.0] { + for (first_theta, first_beta) in [ + (vec![0.5, 0.0], vec![tiny, bad]), + (vec![0.5, bad], vec![tiny, 0.0]), + ] { + let theta = vec![first_theta, vec![0.5, 0.5]]; + let beta = vec![vec![first_beta[0], 1.0], vec![first_beta[1], 1.0]]; + assert_eq!( + expectation(&input, &theta, &beta, 2), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + } + } + let theta = vec![vec![1.0, 0.0], vec![0.5, 0.5]]; + let beta = vec![vec![0.0, 1.0], vec![tiny, 1.0]]; + assert_eq!( + expectation(&input, &theta, &beta, 2), + Err(TopicMeasurementError::NonFiniteEstimate) + ); +} From deb2afc89a28ac49d42d67af7eed95e125a3a77a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 11 Nov 2026 18:00:07 +0900 Subject: [PATCH 12/16] fix(topic-measurement): reject nonfinite raw mixtures --- .../topic-expectation-finite-statistics.md | 3 + .../tests/fitted_k_identity_contract.rs | 81 +++++++++++++++++++ crates/topic_measurement/src/reference.rs | 1 + .../tests/support/expectation_finite.rs | 19 +++++ 4 files changed, 104 insertions(+) diff --git a/CHANGELOG.d/topic-expectation-finite-statistics.md b/CHANGELOG.d/topic-expectation-finite-statistics.md index 7d42a7389..8b02298b4 100644 --- a/CHANGELOG.d/topic-expectation-finite-statistics.md +++ b/CHANGELOG.d/topic-expectation-finite-statistics.md @@ -20,3 +20,6 @@ Reject observed topic transitions when the mapped document event time moves back - Use the same mixture normalization and log-likelihood helper in expectation and the public in-sample likelihood consumed by statistical K selection. Actual public likelihood regressions compare the same injected subnormal/rounded-zero model states; they do not claim a public fit reached those states. - Preserve exact-zero topic factors as zero mass when another topic supplies a positive subnormal mixture. Keep the positive-factor decomposition strict, reject nonfinite or negative partners before skipping a zero product, and retain all-zero-mixture refusal. Private expectation/count and public likelihood/statistical-candidate regressions distinguish injected model states from the unchanged positive fitted-model path. + + +- Reject nonfinite ordinary mixture sums before selecting binary-scaled normalization. Product overflow and summation overflow must retain the existing `NonFiniteEstimate` refusal instead of becoming finite log scores; finite zero sums caused by positive-product underflow still use scaled recovery. Private expectation and public likelihood/statistical-candidate regressions cover overflowing injected model states alongside ordinary, subnormal and rounded-zero positive controls. This does not assert that a valid normalized fit produces overflowing probabilities. diff --git a/crates/model_selection/tests/fitted_k_identity_contract.rs b/crates/model_selection/tests/fitted_k_identity_contract.rs index 85029788c..468f7e5f1 100644 --- a/crates/model_selection/tests/fitted_k_identity_contract.rs +++ b/crates/model_selection/tests/fitted_k_identity_contract.rs @@ -191,3 +191,84 @@ fn zero_factor_subnormal_public_likelihood_and_candidate() { assert!(input.in_sample_log_likelihood(&invalid).is_err()); assert!(statistical_candidate_from_fit(&input, 2, &invalid).is_err()); } + +/// Public likelihood rejects an overflowing raw mixture instead of rescuing its logarithm. +#[test] +fn nonfinite_mixture_public_likelihood_and_candidate_are_rejected() { + let input = two_document_input([1.0, 0.0, 0.0, 1.0]); + let config = ReferenceTopicModelConfig::new(2, vec![7], 2000, 1e-5).expect("config"); + let fitted = fit_reference_topic_model(&input, &config).expect("actual CPU fit"); + for (theta, beta) in [ + ( + vec![f64::MAX, 0.0], + vec![vec![f64::MAX, 1.0], vec![0.0, 1.0]], + ), + ( + vec![f64::MAX, f64::MAX], + vec![vec![1.0, 1.0], vec![1.0, 1.0]], + ), + ] { + let raw = theta + .iter() + .enumerate() + .map(|(topic, weight)| weight * beta[topic][0]) + .sum::(); + assert!(raw.is_infinite() && raw.is_sign_positive()); + let mut model = fitted.clone(); + model.document_topic_proportions = vec![theta, vec![0.5, 0.5]]; + model.topic_term_probabilities = beta; + assert_eq!( + input.in_sample_log_likelihood(&model), + Err(topic_measurement::TopicMeasurementError::NonFiniteEstimate) + ); + assert!(statistical_candidate_from_fit(&input, 2, &model).is_err()); + } +} + +/// Finite ordinary, subnormal and rounded-zero positive mixtures retain shared scoring. +#[test] +fn nonfinite_mixture_guard_preserves_finite_and_underflow_controls() { + let input = two_document_input([1.0, 0.0, 0.0, 1.0]); + let config = ReferenceTopicModelConfig::new(2, vec![7], 2000, 1e-5).expect("config"); + let fitted = fit_reference_topic_model(&input, &config).expect("actual CPU fit"); + let tiny = 2.0_f64.powi(-600); + let subnormal = f64::from_bits(1_u64 << 44); + for (theta, beta, first_log, raw_is_zero) in [ + ( + vec![0.5, 0.5], + vec![vec![0.5, 0.5], vec![0.5, 0.5]], + 0.5_f64.ln(), + false, + ), + ( + vec![1.0, 0.0], + vec![vec![subnormal, 1.0], vec![0.0, 1.0]], + subnormal.ln(), + false, + ), + ( + vec![tiny, 1.0], + vec![vec![tiny, 1.0], vec![0.0, 1.0]], + -1200.0 * std::f64::consts::LN_2, + true, + ), + ] { + let raw = theta + .iter() + .enumerate() + .map(|(topic, weight)| weight * beta[topic][0]) + .sum::(); + assert!(raw.is_finite()); + assert_eq!(raw.to_bits() == 0.0_f64.to_bits(), raw_is_zero); + let mut model = fitted.clone(); + model.document_topic_proportions = vec![theta, vec![0.5, 0.5]]; + model.topic_term_probabilities = beta; + let second = + 0.5 * model.topic_term_probabilities[0][1] + 0.5 * model.topic_term_probabilities[1][1]; + let actual = input + .in_sample_log_likelihood(&model) + .expect("finite or recovered tiny mixture"); + assert!((actual - (first_log + second.ln())).abs() <= 1e-12); + assert!(statistical_candidate_from_fit(&input, 2, &model).is_ok()); + } +} diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs index 7382871c1..1246d8e41 100644 --- a/crates/topic_measurement/src/reference.rs +++ b/crates/topic_measurement/src/reference.rs @@ -989,6 +989,7 @@ fn mixture_normalization( let probability = (0..topic_count) .map(|topic| theta[topic] * beta[topic][term]) .sum::(); + require_finite(probability)?; let scaled = if probability.is_normal() { None } else { diff --git a/crates/topic_measurement/tests/support/expectation_finite.rs b/crates/topic_measurement/tests/support/expectation_finite.rs index 8d5a9f463..73bbfc62f 100644 --- a/crates/topic_measurement/tests/support/expectation_finite.rs +++ b/crates/topic_measurement/tests/support/expectation_finite.rs @@ -323,3 +323,22 @@ fn zero_factor_subnormal_expectation_preserves_refusals() { Err(TopicMeasurementError::NonFiniteEstimate) ); } + +/// Product and summation overflow are refused before scaled expectation normalization. +#[test] +fn nonfinite_mixture_expectation_rejects_raw_overflow() { + let input = input(vec![vec![(0, 1.0)], vec![(1, 1.0)]]); + for (first_theta, first_beta) in [ + (vec![f64::MAX, 0.0], vec![f64::MAX, 0.0]), + (vec![f64::MAX, f64::MAX], vec![1.0, 1.0]), + ] { + let raw = first_theta[0] * first_beta[0] + first_theta[1] * first_beta[1]; + assert!(raw.is_infinite() && raw.is_sign_positive()); + let theta = vec![first_theta, vec![0.5, 0.5]]; + let beta = vec![vec![first_beta[0], 1.0], vec![first_beta[1], 1.0]]; + assert_eq!( + expectation(&input, &theta, &beta, 2), + Err(TopicMeasurementError::NonFiniteEstimate) + ); + } +} From 59f92e5443d2af6530c4ad9ef50730f222679546 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 11 Nov 2026 21:07:36 +0900 Subject: [PATCH 13/16] feat(topic-measurement): add opt-in fit-origin-bound precision --- .../topic_measurement/src/fit_origin_bound.rs | 83 ++++ crates/topic_measurement/src/lib.rs | 5 + .../src/origin_guard_cases.rs | 353 ++++++++++++++++++ crates/topic_measurement/src/reference.rs | 5 + .../tests/bound_origin_proposal.rs | 307 +++++++++++++++ 5 files changed, 753 insertions(+) create mode 100644 crates/topic_measurement/src/fit_origin_bound.rs create mode 100644 crates/topic_measurement/src/origin_guard_cases.rs create mode 100644 crates/topic_measurement/tests/bound_origin_proposal.rs diff --git a/crates/topic_measurement/src/fit_origin_bound.rs b/crates/topic_measurement/src/fit_origin_bound.rs new file mode 100644 index 000000000..20fd86622 --- /dev/null +++ b/crates/topic_measurement/src/fit_origin_bound.rs @@ -0,0 +1,83 @@ +//! Additive private-proposal route; legacy DTO precision remains unbound. + +use super::{ + JointCoordinatePrecision, ReferenceTopicInput, ReferenceTopicModel, ReferenceTopicModelConfig, + fit_reference_topic_model, +}; +use crate::TopicMeasurementError; +use uuid::Uuid; + +/// An immutable actual CPU fit paired with the exact admitted numerical input. +/// +/// This additive proposal does not make the legacy public model DTO or its +/// precision entry safe. Callers must explicitly migrate to this route. +#[derive(Clone, Debug)] +pub struct FitOriginBoundModel { + input: ReferenceTopicInput, + config: ReferenceTopicModelConfig, + model: ReferenceTopicModel, +} + +/// Run the existing CPU f64 estimator and retain its actual admitted origin. +/// +/// The input is moved, not cloned; estimator equations and convergence remain +/// unchanged. The small configuration is retained with the resulting model. +/// +/// # Errors +/// Propagates the existing estimator's admission, finite and convergence errors. +pub fn fit_origin_bound_topic_model( + input: ReferenceTopicInput, + config: ReferenceTopicModelConfig, +) -> Result { + let model = fit_reference_topic_model(&input, &config)?; + Ok(FitOriginBoundModel { + input, + config, + model, + }) +} + +impl FitOriginBoundModel { + /// Inspect the fitted result without obtaining mutable access to its owner. + #[must_use] + pub const fn model(&self) -> &ReferenceTopicModel { + &self.model + } + + /// Build precision only for the actual fitted input and configuration. + /// + /// Equality covers ordered documents/times, sparse numerical rows, + /// vocabulary, expanded design/features and modeled transition pairs. + /// Configuration equality includes both numerical and execution controls. + /// This validates admitted numerical identity, not external run/snapshot/ + /// cutoff authority or completeness of a posterior artifact. Topic identity + /// still has the legacy explicit caller-assigned basis semantics. + /// + /// # Errors + /// Returns invalid input on any origin mismatch, otherwise propagates the + /// unchanged precision builder's errors. + pub fn build_joint_coordinate_precision( + &self, + input: &ReferenceTopicInput, + config: &ReferenceTopicModelConfig, + topic_ids: Vec, + ) -> Result { + let original = &self.input; + if config != &self.config + || input.document_ids != original.document_ids + || input.event_times != original.event_times + || input.term_rows != original.term_rows + || input.vocabulary_size != original.vocabulary_size + || input.design != original.design + || input.features != original.features + || input.transition_pairs != original.transition_pairs + { + return Err(TopicMeasurementError::InvalidModelInput); + } + original.build_joint_coordinate_precision(&self.model, &self.config, topic_ids) + } +} + +#[cfg(test)] +#[path = "origin_guard_cases.rs"] +mod origin_guard_cases; diff --git a/crates/topic_measurement/src/lib.rs b/crates/topic_measurement/src/lib.rs index 257b32563..2e747f40e 100644 --- a/crates/topic_measurement/src/lib.rs +++ b/crates/topic_measurement/src/lib.rs @@ -65,3 +65,8 @@ pub use reference::fit_reference_topic_model; pub use sparse::SparseMatrix; /// Whether compressed values are grouped by row or by column. pub use sparse::SparseOrientation; + +/// Opaque CPU fit retaining its admitted input and numerical configuration. +pub use reference::FitOriginBoundModel; +/// Fit through the opaque input/configuration-bound CPU route. +pub use reference::fit_origin_bound_topic_model; diff --git a/crates/topic_measurement/src/origin_guard_cases.rs b/crates/topic_measurement/src/origin_guard_cases.rs new file mode 100644 index 000000000..e7708a1da --- /dev/null +++ b/crates/topic_measurement/src/origin_guard_cases.rs @@ -0,0 +1,353 @@ +//! Isolated guard characterization; internal mutations are not public admission proofs. + +use crate::{ + ReferenceTopicInput, ReferenceTopicModelConfig, SparseMatrix, fit_origin_bound_topic_model, +}; +use corpus_split::{CorpusDocument, CorpusSnapshot}; +use membership_core::{ + GroupId, MemberId, MembershipAssignment, MembershipNetwork, MembershipRole, MembershipWeight, +}; +use relation_graph::{ + RelationEdge, RelationEndpointId, RelationEvidenceStatus, RelationGraph, RelationKind, +}; +use temporal_core::{ + AvailableTime, EventTime, KnowledgeCutoff, TemporalBoundary, TemporalInterval, + TemporalPrecision, +}; +use uuid::Uuid; + +fn event_time(day: u8) -> EventTime { + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T00:00:00Z")).expect("event time") +} + +fn relation(source: Uuid, target: Uuid, source_day: u8, target_day: u8) -> RelationEdge { + let interval = |day| { + TemporalInterval::bounded( + TemporalBoundary::Included(event_time(day)), + TemporalBoundary::Included( + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T12:00:00Z")) + .expect("interval end"), + ), + TemporalPrecision::Second, + ) + .expect("bounded interval") + }; + RelationEdge::new( + RelationKind::TransitionsTo, + RelationEndpointId::from_uuid(source), + RelationEndpointId::from_uuid(target), + RelationEvidenceStatus::Observed, + interval(source_day), + interval(target_day), + ) + .expect("forward relation") +} + +fn fixture() -> ( + CorpusSnapshot, + Vec, + Vec, + MembershipNetwork, + RelationGraph, +) { + let document_ids: Vec<_> = (1_u128..=6).map(Uuid::from_u128).collect(); + let times: Vec<_> = (1_u8..=6).map(event_time).collect(); + let available = AvailableTime::parse_rfc3339("2026-01-10T00:00:00Z").expect("available"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + let mut snapshot = CorpusSnapshot::new(); + for id in &document_ids { + snapshot + .insert_if_eligible(CorpusDocument::new(*id, available), &cutoff) + .expect("eligible"); + } + + let organization = GroupId::from_uuid(Uuid::from_u128(100)); + let projects = [ + GroupId::from_uuid(Uuid::from_u128(101)), + GroupId::from_uuid(Uuid::from_u128(102)), + ]; + let validity_start = event_time(1); + let validity_end = event_time(9); + let mut memberships = MembershipNetwork::new(); + for (index, id) in document_ids.iter().enumerate() { + let member = MemberId::from_uuid(*id); + memberships + .insert( + MembershipAssignment::new( + member, + organization, + MembershipRole::Organization, + MembershipWeight::full().expect("full"), + validity_start, + validity_end, + ) + .expect("organization membership"), + ) + .expect("insert organization"); + memberships + .insert( + MembershipAssignment::new( + member, + projects[usize::from(index >= 3)], + MembershipRole::Project, + MembershipWeight::new(0.75).expect("partial"), + validity_start, + validity_end, + ) + .expect("project membership"), + ) + .expect("insert project"); + } + + let mut relations = RelationGraph::new(); + for (source, target, source_day, target_day) in [ + (0, 1, 1, 2), + (1, 2, 2, 3), + (2, 3, 3, 4), + (3, 4, 4, 5), + (4, 5, 5, 6), + ] { + relations + .insert(relation( + document_ids[source], + document_ids[target], + source_day, + target_day, + )) + .expect("insert relation"); + } + (snapshot, document_ids, times, memberships, relations) +} + +fn separated_counts() -> SparseMatrix { + SparseMatrix::from_csr( + 6, + 4, + vec![0, 2, 4, 6, 8, 10, 12], + vec![0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3], + vec![ + 90.0, 10.0, 85.0, 15.0, 80.0, 20.0, 10.0, 90.0, 15.0, 85.0, 20.0, 80.0, + ], + ) + .expect("counts") +} + +/// Build the admitted synthetic corpus used by these origin-binding diagnostics. +fn precision_origin_input() -> ReferenceTopicInput { + let (snapshot, ids, times, memberships, relations) = fixture(); + ReferenceTopicInput::new( + &snapshot, + ids, + &separated_counts(), + ×, + None, + &memberships, + &relations, + ) + .expect("admitted input") +} + +/// Fit only the synthetic six-document corpus through the public CPU f64 entry. +fn precision_origin_fit( + input: &ReferenceTopicInput, +) -> (ReferenceTopicModelConfig, crate::FitOriginBoundModel) { + let config = ReferenceTopicModelConfig::new(2, vec![7], 2_000, 1e-5) + .expect("config") + .with_hyperparameters(1.0, 0.5, 0.01, 0.05, 0.2) + .expect("hyperparameters"); + let model = fit_origin_bound_topic_model(input.clone(), config.clone()) + .expect("real converged CPU fit"); + assert!(model.model().objective.is_finite()); + (config, model) +} + +/// Every input guard independently refuses one changed private numerical field. +/// These synthetic private mutations characterize the guard, not public reachability. +#[test] +fn origin_supplement_each_input_field_is_checked() { + let input = precision_origin_input(); + let (config, bound) = precision_origin_fit(&input); + let topics = vec![Uuid::from_u128(201), Uuid::from_u128(202)]; + let mut cases = Vec::new(); + let mut changed = input.clone(); + changed.document_ids[0] = Uuid::from_u128(900); + cases.push(("document_ids", changed)); + let mut changed = input.clone(); + changed.event_times[0] = event_time(2); + cases.push(("event_times", changed)); + let mut changed = input.clone(); + changed.term_rows[0][0].1 += 1.0; + cases.push(("term_rows", changed)); + let mut changed = input.clone(); + changed.vocabulary_size += 1; + cases.push(("vocabulary_size", changed)); + let mut changed = input.clone(); + changed.design[0][0] += 0.25; + cases.push(("design", changed)); + let mut changed = input.clone(); + changed.features.swap(0, 1); + cases.push(("features", changed)); + let mut changed = input.clone(); + changed.transition_pairs.pop().expect("observed pair"); + cases.push(("transition_pairs", changed)); + for (name, changed) in cases { + let equal_fields = [ + input.document_ids == changed.document_ids, + input.event_times == changed.event_times, + input.term_rows == changed.term_rows, + input.vocabulary_size == changed.vocabulary_size, + input.design == changed.design, + input.features == changed.features, + input.transition_pairs == changed.transition_pairs, + ]; + assert_eq!( + equal_fields.iter().filter(|same| !**same).count(), + 1, + "single input axis {name}" + ); + assert!( + matches!( + bound.build_joint_coordinate_precision(&changed, &config, topics.clone()), + Err(crate::TopicMeasurementError::InvalidModelInput) + ), + "unbound input axis {name}" + ); + println!("INPUT_AXIS {name} rejected_typed=true"); + } + assert!( + bound + .build_joint_coordinate_precision(&input, &config, topics) + .is_ok() + ); +} + +/// Every current config field is compared independently, including execution controls. +#[test] +fn origin_supplement_each_config_field_is_checked() { + let input = precision_origin_input(); + let (config, bound) = precision_origin_fit(&input); + let topics = vec![Uuid::from_u128(201), Uuid::from_u128(202)]; + let mut cases = Vec::new(); + let mut changed = config.clone(); + changed.topic_count = 3; + cases.push(("topic_count", changed)); + let mut changed = config.clone(); + changed.seeds = vec![8]; + cases.push(("seeds", changed)); + let mut changed = config.clone(); + changed.maximum_iterations += 1; + cases.push(("maximum_iterations", changed)); + let mut changed = config.clone(); + changed.tolerance *= 2.0; + cases.push(("tolerance", changed)); + let mut changed = config.clone(); + changed.prior_variance = 4.0; + cases.push(("prior_variance", changed)); + let mut changed = config.clone(); + changed.relation_strength = 0.0; + cases.push(("relation_strength", changed)); + let mut changed = config.clone(); + changed.ridge = 0.02; + cases.push(("ridge", changed)); + let mut changed = config.clone(); + changed.topic_smoothing = 0.1; + cases.push(("topic_smoothing", changed)); + let mut changed = config.clone(); + changed.step_size = 0.1; + cases.push(("step_size", changed)); + for (name, changed) in cases { + changed + .validate() + .expect("independently valid configuration"); + let equal_fields = [ + config.topic_count == changed.topic_count, + config.seeds == changed.seeds, + config.maximum_iterations == changed.maximum_iterations, + config.tolerance.to_bits() == changed.tolerance.to_bits(), + config.prior_variance.to_bits() == changed.prior_variance.to_bits(), + config.relation_strength.to_bits() == changed.relation_strength.to_bits(), + config.ridge.to_bits() == changed.ridge.to_bits(), + config.topic_smoothing.to_bits() == changed.topic_smoothing.to_bits(), + config.step_size.to_bits() == changed.step_size.to_bits(), + ]; + assert_eq!( + equal_fields.iter().filter(|same| !**same).count(), + 1, + "single config axis {name}" + ); + assert!( + matches!( + bound.build_joint_coordinate_precision(&input, &changed, topics.clone()), + Err(crate::TopicMeasurementError::InvalidModelInput) + ), + "unbound config axis {name}" + ); + println!("CONFIG_AXIS {name} rejected_typed=true"); + } + assert!( + bound + .build_joint_coordinate_precision(&input, &config, topics) + .is_ok() + ); +} + +/// Equivalent public reconstruction, clone immutability and successful labels survive. +#[test] +fn origin_supplement_successful_draw_and_plausible_labels() { + let input = precision_origin_input(); + let reconstructed = precision_origin_input(); + let (config, bound) = precision_origin_fit(&input); + let topics = vec![Uuid::from_u128(201), Uuid::from_u128(202)]; + let reference = bound + .build_joint_coordinate_precision(&input, &config, topics.clone()) + .expect("matching precision"); + let equivalent = bound + .build_joint_coordinate_precision(&reconstructed, &config.clone(), topics.clone()) + .expect("equal value public reconstruction"); + assert_eq!(reference, equivalent); + let cloned = bound.clone(); + let mut detached = cloned.model().clone(); + detached.document_topic_proportions[0][0] = 0.0; + assert_ne!( + detached.document_topic_proportions, + bound.model().document_topic_proportions + ); + assert_eq!(cloned.model(), bound.model()); + let draws = cloned + .build_joint_coordinate_precision(&reconstructed, &config, topics.clone()) + .expect("clone precision") + .draw_joint_gaussian(11, 2) + .expect("matching draws"); + let expected_ids: Vec<_> = (1_u128..=6).map(Uuid::from_u128).collect(); + let expected_times: Vec<_> = (1_u8..=6).map(event_time).collect(); + assert_eq!(draws.document_ids(), expected_ids); + assert_eq!(draws.event_times(), expected_times); + assert_eq!(draws.topic_ids(), topics); + assert_eq!(draws.seed(), 11); + assert_eq!(draws.draws().len(), 2); + assert!( + draws + .draws() + .iter() + .all(|row| row.len() == 6 && row.iter().all(|x| x.is_finite())) + ); + let records = draws.plausible_values(); + assert_eq!(records.len(), 12); + for (document, (id, time)) in expected_ids.iter().zip(&expected_times).enumerate() { + for draw in 0..2 { + let record = &records[document * 2 + draw]; + assert_eq!(record.document_id, *id); + assert_eq!(record.event_time, *time); + assert_eq!( + record.draw_index, + u64::try_from(draw).expect("bounded index") + ); + assert_eq!(record.logistic_normal_coordinates.len(), 1); + assert_eq!( + record.logistic_normal_coordinates[0].to_bits(), + draws.draws()[draw][document].to_bits() + ); + } + } + println!("SUCCESS_LABELS documents=6 draws=2 records=12 clone_immutable=true"); +} diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs index 1246d8e41..01bb99fbc 100644 --- a/crates/topic_measurement/src/reference.rs +++ b/crates/topic_measurement/src/reference.rs @@ -4,6 +4,11 @@ #[path = "../tests/support/expectation_finite.rs"] mod expectation_finite_tests; +#[path = "fit_origin_bound.rs"] +mod fit_origin_bound; + +pub use fit_origin_bound::{FitOriginBoundModel, fit_origin_bound_topic_model}; + use std::collections::{BTreeMap, BTreeSet, HashMap}; use corpus_split::CorpusSnapshot; diff --git a/crates/topic_measurement/tests/bound_origin_proposal.rs b/crates/topic_measurement/tests/bound_origin_proposal.rs new file mode 100644 index 000000000..4908f8668 --- /dev/null +++ b/crates/topic_measurement/tests/bound_origin_proposal.rs @@ -0,0 +1,307 @@ +//! Known-truth recovery contract for the CPU `f64` TRSL-TM reference. + +use corpus_split::{CorpusDocument, CorpusSnapshot}; +use membership_core::{ + GroupId, MemberId, MembershipAssignment, MembershipNetwork, MembershipRole, MembershipWeight, +}; +use relation_graph::{ + RelationEdge, RelationEndpointId, RelationEvidenceStatus, RelationGraph, RelationKind, +}; +use temporal_core::{ + AvailableTime, EventTime, KnowledgeCutoff, TemporalBoundary, TemporalInterval, + TemporalPrecision, +}; +use topic_measurement::{ + ReferenceTopicInput, ReferenceTopicModelConfig, SparseMatrix, fit_origin_bound_topic_model, +}; +use uuid::Uuid; + +fn event_time(day: u8) -> EventTime { + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T00:00:00Z")).expect("event time") +} + +fn relation(source: Uuid, target: Uuid, source_day: u8, target_day: u8) -> RelationEdge { + let interval = |day| { + TemporalInterval::bounded( + TemporalBoundary::Included(event_time(day)), + TemporalBoundary::Included( + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T12:00:00Z")) + .expect("interval end"), + ), + TemporalPrecision::Second, + ) + .expect("bounded interval") + }; + RelationEdge::new( + RelationKind::TransitionsTo, + RelationEndpointId::from_uuid(source), + RelationEndpointId::from_uuid(target), + RelationEvidenceStatus::Observed, + interval(source_day), + interval(target_day), + ) + .expect("forward relation") +} + +fn fixture() -> ( + CorpusSnapshot, + Vec, + Vec, + MembershipNetwork, + RelationGraph, +) { + let document_ids: Vec<_> = (1_u128..=6).map(Uuid::from_u128).collect(); + let times: Vec<_> = (1_u8..=6).map(event_time).collect(); + let available = AvailableTime::parse_rfc3339("2026-01-10T00:00:00Z").expect("available"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + let mut snapshot = CorpusSnapshot::new(); + for id in &document_ids { + snapshot + .insert_if_eligible(CorpusDocument::new(*id, available), &cutoff) + .expect("eligible"); + } + + let organization = GroupId::from_uuid(Uuid::from_u128(100)); + let projects = [ + GroupId::from_uuid(Uuid::from_u128(101)), + GroupId::from_uuid(Uuid::from_u128(102)), + ]; + let validity_start = event_time(1); + let validity_end = event_time(9); + let mut memberships = MembershipNetwork::new(); + for (index, id) in document_ids.iter().enumerate() { + let member = MemberId::from_uuid(*id); + memberships + .insert( + MembershipAssignment::new( + member, + organization, + MembershipRole::Organization, + MembershipWeight::full().expect("full"), + validity_start, + validity_end, + ) + .expect("organization membership"), + ) + .expect("insert organization"); + memberships + .insert( + MembershipAssignment::new( + member, + projects[usize::from(index >= 3)], + MembershipRole::Project, + MembershipWeight::new(0.75).expect("partial"), + validity_start, + validity_end, + ) + .expect("project membership"), + ) + .expect("insert project"); + } + + let mut relations = RelationGraph::new(); + for (source, target, source_day, target_day) in [ + (0, 1, 1, 2), + (1, 2, 2, 3), + (2, 3, 3, 4), + (3, 4, 4, 5), + (4, 5, 5, 6), + ] { + relations + .insert(relation( + document_ids[source], + document_ids[target], + source_day, + target_day, + )) + .expect("insert relation"); + } + (snapshot, document_ids, times, memberships, relations) +} + +fn separated_counts() -> SparseMatrix { + SparseMatrix::from_csr( + 6, + 4, + vec![0, 2, 4, 6, 8, 10, 12], + vec![0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3], + vec![ + 90.0, 10.0, 85.0, 15.0, 80.0, 20.0, 10.0, 90.0, 15.0, 85.0, 20.0, 80.0, + ], + ) + .expect("counts") +} + +/// Build the admitted synthetic corpus used by these origin-binding diagnostics. +fn precision_origin_input() -> ReferenceTopicInput { + let (snapshot, ids, times, memberships, relations) = fixture(); + ReferenceTopicInput::new( + &snapshot, + ids, + &separated_counts(), + ×, + None, + &memberships, + &relations, + ) + .expect("admitted input") +} + +/// Fit only the synthetic six-document corpus through the public CPU f64 entry. +fn precision_origin_fit( + input: &ReferenceTopicInput, +) -> ( + ReferenceTopicModelConfig, + topic_measurement::FitOriginBoundModel, +) { + let config = ReferenceTopicModelConfig::new(2, vec![7], 2_000, 1e-5) + .expect("config") + .with_hyperparameters(1.0, 0.5, 0.01, 0.05, 0.2) + .expect("hyperparameters"); + let model = fit_origin_bound_topic_model(input.clone(), config.clone()) + .expect("real converged CPU fit"); + assert!(model.model().objective.is_finite()); + (config, model) +} + +/// Matching input, configuration, and actual fit retain identified finite draws. +#[test] +fn precision_origin_matching_public_fit_positive() { + let input = precision_origin_input(); + let (config, model) = precision_origin_fit(&input); + let ids = vec![Uuid::from_u128(201), Uuid::from_u128(202)]; + let precision = model + .build_joint_coordinate_precision(&input, &config, ids.clone()) + .expect("matching precision"); + assert_eq!( + precision.document_ids(), + (1_u128..=6).map(Uuid::from_u128).collect::>() + ); + assert_eq!(precision.topic_ids(), ids); + assert_eq!(precision.coordinate_means().len(), 6); + assert_eq!(precision.values().len(), 6); + assert!( + precision + .values() + .iter() + .all(|r| r.len() == 6 && r.iter().all(|x| x.is_finite())) + ); + let draws = precision + .draw_joint_gaussian(11, 2) + .expect("finite public draws"); + assert_eq!(draws.draws().len(), 2); + assert!( + draws + .draws() + .iter() + .all(|r| r.len() == 6 && r.iter().all(|x| x.is_finite())) + ); + println!( + "MATCHING iterations={} objective={} precision00={} draw_set_id={}", + model.model().iterations, + model.model().objective, + precision.values()[0][0], + draws.draw_set_id() + ); +} + +/// A fit-bound claim must not silently replace the fitted prior and network penalty. +#[test] +fn precision_origin_rejects_different_fitted_hyperparameters() { + let input = precision_origin_input(); + let (config, model) = precision_origin_fit(&input); + let ids = vec![Uuid::from_u128(201), Uuid::from_u128(202)]; + let original = model + .build_joint_coordinate_precision(&input, &config, ids.clone()) + .expect("matching precision"); + let changed = config + .clone() + .with_hyperparameters(4.0, 0.0, 0.01, 0.05, 0.2) + .expect("independently valid but not fitted configuration"); + let result = model.build_joint_coordinate_precision(&input, &changed, ids); + if let Ok(other) = &result { + assert_eq!(other.coordinate_means(), original.coordinate_means()); + assert_ne!(other.values(), original.values()); + let a = original.draw_joint_gaussian(11, 2).expect("original draws"); + let b = other + .draw_joint_gaussian(11, 2) + .expect("mismatched draws still admitted"); + assert_ne!(a.draw_set_id(), b.draw_set_id()); + println!( + "MISMATCH_CONFIG admitted=true original00={} other00={} original_draw={} other_draw={}", + original.values()[0][0], + other.values()[0][0], + a.draw_set_id(), + b.draw_set_id() + ); + } + assert!( + result.is_err(), + "fit-origin configuration mismatch must not be admitted as fit-bound precision" + ); +} + +/// A fit-bound claim must not relabel unchanged fitted rows with another input order. +#[test] +fn precision_origin_rejects_different_fitted_document_order() { + let input = precision_origin_input(); + let (config, model) = precision_origin_fit(&input); + let (snapshot, mut ids, mut times, memberships, relations) = fixture(); + ids.reverse(); + times.reverse(); + let counts = SparseMatrix::from_csr( + 6, + 4, + vec![0, 2, 4, 6, 8, 10, 12], + vec![2, 3, 2, 3, 2, 3, 0, 1, 0, 1, 0, 1], + vec![ + 20.0, 80.0, 15.0, 85.0, 10.0, 90.0, 80.0, 20.0, 85.0, 15.0, 90.0, 10.0, + ], + ) + .expect("aligned reversed sparse rows"); + let reversed = ReferenceTopicInput::new( + &snapshot, + ids.clone(), + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("aligned row permutation is a valid independent input"); + let (_, reverse_fit) = precision_origin_fit(&reversed); + let topics = vec![Uuid::from_u128(201), Uuid::from_u128(202)]; + let positive = reverse_fit + .build_joint_coordinate_precision(&reversed, &config, topics.clone()) + .expect("matching reversed fit"); + assert_eq!(positive.document_ids(), ids); + let original = model + .build_joint_coordinate_precision(&input, &config, topics.clone()) + .expect("original matching precision"); + let result = model.build_joint_coordinate_precision(&reversed, &config, topics); + if let Ok(other) = &result { + assert_eq!(other.document_ids(), ids); + assert_eq!(other.coordinate_means(), original.coordinate_means()); + assert_ne!(other.coordinate_means(), positive.coordinate_means()); + let a = original.draw_joint_gaussian(11, 2).expect("original draws"); + let b = other + .draw_joint_gaussian(11, 2) + .expect("foreign-origin draws still admitted"); + assert_ne!(a.document_ids(), b.document_ids()); + assert_eq!(b.event_times(), times); + let records = b.plausible_values(); + assert_eq!(records[0].document_id, ids[0]); + assert_eq!(records[0].event_time, times[0]); + assert_eq!(records[0].logistic_normal_coordinates, b.draws()[0][..1]); + println!( + "MISMATCH_ORDER admitted=true original_first={} other_first={} same_coordinate_means=true same_numerical_draws={}", + original.document_ids()[0], + other.document_ids()[0], + a.draws() == b.draws() + ); + } + assert!( + result.is_err(), + "different originating document order must not relabel the original fit" + ); +} From 698e3d7873148f2933112f2bbbb5fc220905ecc0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 11 Nov 2026 21:58:23 +0900 Subject: [PATCH 14/16] docs(topic-measurement): state fit-origin test scope accurately --- crates/topic_measurement/tests/bound_origin_proposal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/topic_measurement/tests/bound_origin_proposal.rs b/crates/topic_measurement/tests/bound_origin_proposal.rs index 4908f8668..1cb9b4598 100644 --- a/crates/topic_measurement/tests/bound_origin_proposal.rs +++ b/crates/topic_measurement/tests/bound_origin_proposal.rs @@ -1,4 +1,4 @@ -//! Known-truth recovery contract for the CPU `f64` TRSL-TM reference. +//! Fit-origin binding, precision, and draw contracts for the CPU `f64` reference. use corpus_split::{CorpusDocument, CorpusSnapshot}; use membership_core::{ From 62dca9d552ed6b53473ce59d4aaf45ff7ed77652 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 12 Nov 2026 02:28:08 +0900 Subject: [PATCH 15/16] test(topic-measurement): characterize correlated joint draws --- .../src/correlated_draw_contract.rs | 74 +++++++++++++++++++ .../topic_measurement/src/posterior_draw.rs | 4 + 2 files changed, 78 insertions(+) create mode 100644 crates/topic_measurement/src/correlated_draw_contract.rs diff --git a/crates/topic_measurement/src/correlated_draw_contract.rs b/crates/topic_measurement/src/correlated_draw_contract.rs new file mode 100644 index 000000000..522cb336f --- /dev/null +++ b/crates/topic_measurement/src/correlated_draw_contract.rs @@ -0,0 +1,74 @@ +//! Numerical covariance characterization with synthetic identified precisions. + +use crate::JointCoordinatePrecision; +use temporal_core::EventTime; +use uuid::Uuid; + +/// Recover a non-diagonal analytic covariance, including both correlation signs. +/// +/// This is a synthetic precision-to-draw check, not fitted-parameter recovery. +/// Chebyshev plus a six-moment union bound sets family error at most 0.001 +/// under the ideal independent Gaussian reference; this deterministic seed is +/// one numerical realization, not a PRNG quality or nominal-coverage proof. +#[test] +fn correlated_precision_draws_recover_analytic_covariance() { + const COUNT: usize = 100_000; + const COUNT_F64: f64 = 100_000.0; + const FAMILY_ERROR: f64 = 0.001; + const MOMENTS: f64 = 6.0; + let document_ids = vec![Uuid::from_u128(1), Uuid::from_u128(2)]; + let topic_ids = vec![Uuid::from_u128(11), Uuid::from_u128(12)]; + let event_times = vec![ + EventTime::parse_rfc3339("2026-01-01T00:00:00Z").expect("time"), + EventTime::parse_rfc3339("2026-01-02T00:00:00Z").expect("time"), + ]; + for off_diagonal in [-3.2_f64, 3.2_f64] { + let precision = JointCoordinatePrecision { + document_ids: document_ids.clone(), + topic_ids: topic_ids.clone(), + event_times: event_times.clone(), + coordinate_means: vec![0.25, -0.5], + values: vec![vec![4.0, off_diagonal], vec![off_diagonal, 4.0]], + }; + let determinant = 16.0 - off_diagonal * off_diagonal; + assert!(determinant.is_finite() && determinant > 0.0); + let expected_variance = 4.0 / determinant; + let expected_covariance = -off_diagonal / determinant; + let draws = precision.draw_joint_gaussian(19, COUNT).expect("draws"); + assert_eq!(draws.document_ids(), document_ids); + assert_eq!(draws.topic_ids(), topic_ids); + assert_eq!(draws.event_times(), event_times); + assert_eq!(draws.draws().len(), COUNT); + assert!( + draws + .draws() + .iter() + .all(|row| { row.len() == 2 && row.iter().all(|value| value.is_finite()) }) + ); + let mut moments = [0.0_f64; 3]; + for row in draws.draws() { + let first = row[0] - 0.25; + let second = row[1] + 0.5; + moments[0] += first * first; + moments[1] += second * second; + moments[2] += first * second; + } + for value in &mut moments { + *value /= COUNT_F64; + } + assert!(moments.iter().all(|value| value.is_finite())); + let per_moment_error = FAMILY_ERROR / MOMENTS; + let variance_bound = expected_variance * (2.0 / (COUNT_F64 * per_moment_error)).sqrt(); + let covariance_bound = ((expected_variance * expected_variance + + expected_covariance * expected_covariance) + / (COUNT_F64 * per_moment_error)) + .sqrt(); + println!( + "CORRELATED off_diagonal={off_diagonal} determinant={determinant} expected_variance={expected_variance} expected_covariance={expected_covariance} observed={moments:?} variance_bound={variance_bound} covariance_bound={covariance_bound}" + ); + assert!((moments[0] - expected_variance).abs() <= variance_bound); + assert!((moments[1] - expected_variance).abs() <= variance_bound); + assert!((moments[2] - expected_covariance).abs() <= covariance_bound); + assert!(moments[2] * expected_covariance > 0.0); + } +} diff --git a/crates/topic_measurement/src/posterior_draw.rs b/crates/topic_measurement/src/posterior_draw.rs index 223a578c6..e20b946e0 100644 --- a/crates/topic_measurement/src/posterior_draw.rs +++ b/crates/topic_measurement/src/posterior_draw.rs @@ -359,3 +359,7 @@ mod tests { ); } } + +#[cfg(test)] +#[path = "correlated_draw_contract.rs"] +mod correlated_draw_supplement; From 5eaed1fa997ca2c108716ecc049cd7c222c1b38d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 12 Nov 2026 20:51:01 +0900 Subject: [PATCH 16/16] fix(topic-measurement): preserve precision formation symmetry --- crates/topic_measurement/src/reference.rs | 4 +- .../tests/precision_dynamic_range.rs | 171 ++++++++++++++++ .../tests/precision_formation_reachability.rs | 189 +++++++++++++++++ .../precision_high_count_reachability.rs | 190 ++++++++++++++++++ 4 files changed, 553 insertions(+), 1 deletion(-) create mode 100644 crates/topic_measurement/tests/precision_dynamic_range.rs create mode 100644 crates/topic_measurement/tests/precision_formation_reachability.rs create mode 100644 crates/topic_measurement/tests/precision_high_count_reachability.rs diff --git a/crates/topic_measurement/src/reference.rs b/crates/topic_measurement/src/reference.rs index 01bb99fbc..1288615bb 100644 --- a/crates/topic_measurement/src/reference.rs +++ b/crates/topic_measurement/src/reference.rs @@ -671,7 +671,7 @@ impl ReferenceTopicInput { .map(|(_, count)| count) .sum::(); for row in 0..coordinate_count { - for column in 0..coordinate_count { + for column in row..coordinate_count { let information = if row == column { tokens * theta[row] * (1.0 - theta[row]) + 1.0 / config.prior_variance } else { @@ -679,6 +679,8 @@ impl ReferenceTopicInput { }; precision[document * coordinate_count + row] [document * coordinate_count + column] = information; + precision[document * coordinate_count + column] + [document * coordinate_count + row] = information; } } } diff --git a/crates/topic_measurement/tests/precision_dynamic_range.rs b/crates/topic_measurement/tests/precision_dynamic_range.rs new file mode 100644 index 000000000..8ecfdaed6 --- /dev/null +++ b/crates/topic_measurement/tests/precision_dynamic_range.rs @@ -0,0 +1,171 @@ +//! Fit-origin binding, precision, and draw contracts for the CPU `f64` reference. + +use corpus_split::{CorpusDocument, CorpusSnapshot}; +use membership_core::{ + GroupId, MemberId, MembershipAssignment, MembershipNetwork, MembershipRole, MembershipWeight, +}; +use relation_graph::{ + RelationEdge, RelationEndpointId, RelationEvidenceStatus, RelationGraph, RelationKind, +}; +use temporal_core::{ + AvailableTime, EventTime, KnowledgeCutoff, TemporalBoundary, TemporalInterval, + TemporalPrecision, +}; +use topic_measurement::{ + ReferenceTopicInput, ReferenceTopicModelConfig, SparseMatrix, fit_reference_topic_model, +}; +use uuid::Uuid; + +fn event_time(day: u8) -> EventTime { + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T00:00:00Z")).expect("event time") +} + +fn relation(source: Uuid, target: Uuid, source_day: u8, target_day: u8) -> RelationEdge { + let interval = |day| { + TemporalInterval::bounded( + TemporalBoundary::Included(event_time(day)), + TemporalBoundary::Included( + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T12:00:00Z")) + .expect("interval end"), + ), + TemporalPrecision::Second, + ) + .expect("bounded interval") + }; + RelationEdge::new( + RelationKind::TransitionsTo, + RelationEndpointId::from_uuid(source), + RelationEndpointId::from_uuid(target), + RelationEvidenceStatus::Observed, + interval(source_day), + interval(target_day), + ) + .expect("forward relation") +} + +fn fixture() -> ( + CorpusSnapshot, + Vec, + Vec, + MembershipNetwork, + RelationGraph, +) { + let document_ids: Vec<_> = (1_u128..=6).map(Uuid::from_u128).collect(); + let times: Vec<_> = (1_u8..=6).map(event_time).collect(); + let available = AvailableTime::parse_rfc3339("2026-01-10T00:00:00Z").expect("available"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + let mut snapshot = CorpusSnapshot::new(); + for id in &document_ids { + snapshot + .insert_if_eligible(CorpusDocument::new(*id, available), &cutoff) + .expect("eligible"); + } + + let organization = GroupId::from_uuid(Uuid::from_u128(100)); + let projects = [ + GroupId::from_uuid(Uuid::from_u128(101)), + GroupId::from_uuid(Uuid::from_u128(102)), + ]; + let validity_start = event_time(1); + let validity_end = event_time(9); + let mut memberships = MembershipNetwork::new(); + for (index, id) in document_ids.iter().enumerate() { + let member = MemberId::from_uuid(*id); + memberships + .insert( + MembershipAssignment::new( + member, + organization, + MembershipRole::Organization, + MembershipWeight::full().expect("full"), + validity_start, + validity_end, + ) + .expect("organization membership"), + ) + .expect("insert organization"); + memberships + .insert( + MembershipAssignment::new( + member, + projects[usize::from(index >= 3)], + MembershipRole::Project, + MembershipWeight::new(0.75).expect("partial"), + validity_start, + validity_end, + ) + .expect("project membership"), + ) + .expect("insert project"); + } + + let mut relations = RelationGraph::new(); + for (source, target, source_day, target_day) in [ + (0, 1, 1, 2), + (1, 2, 2, 3), + (2, 3, 3, 4), + (3, 4, 4, 5), + (4, 5, 5, 6), + ] { + relations + .insert(relation( + document_ids[source], + document_ids[target], + source_day, + target_day, + )) + .expect("insert relation"); + } + (snapshot, document_ids, times, memberships, relations) +} + +/// An injected positive simplex characterizes builder range, not actual-fit reachability. +#[test] +fn precision_preserves_representable_tiny_off_diagonal_with_large_count() { + let (snapshot, ids, times, memberships, relations) = fixture(); + let ordinary = + SparseMatrix::from_csr(6, 4, vec![0, 1, 2, 3, 4, 5, 6], vec![0; 6], vec![100.; 6]) + .expect("small counts"); + let input = ReferenceTopicInput::new( + &snapshot, + ids.clone(), + &ordinary, + ×, + None, + &memberships, + &relations, + ) + .expect("admitted input"); + let config = ReferenceTopicModelConfig::new(3, vec![7], 5_000, 1e-5) + .expect("config") + .with_hyperparameters(1., 0., 0.01, 0.05, 0.2) + .expect("hyperparameters"); + let mut model = fit_reference_topic_model(&input, &config) + .expect("actual small CPU fit supplies DTO shape"); + model.document_topic_proportions = vec![vec![1e-170, 1e-170, 1.]; 6]; + let counts = + SparseMatrix::from_csr(6, 4, vec![0, 1, 2, 3, 4, 5, 6], vec![0; 6], vec![1e100; 6]) + .expect("large finite counts"); + let input = ReferenceTopicInput::new( + &snapshot, + ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("admitted larger counts"); + let expected = (-1e100_f64 * 1e-170_f64) * 1e-170_f64; + assert!(expected.is_normal() && expected < 0.); + let topics = (201_u128..=203).map(Uuid::from_u128).collect(); + let precision = input + .build_joint_coordinate_precision(&model, &config, topics) + .expect("injected positive simplex"); + println!( + "RANGE expected={expected} actual={}", + precision.values()[0][1] + ); + assert_eq!(precision.values()[0][1].to_bits(), expected.to_bits()); + assert_eq!(precision.values()[1][0].to_bits(), expected.to_bits()); +} diff --git a/crates/topic_measurement/tests/precision_formation_reachability.rs b/crates/topic_measurement/tests/precision_formation_reachability.rs new file mode 100644 index 000000000..94329f846 --- /dev/null +++ b/crates/topic_measurement/tests/precision_formation_reachability.rs @@ -0,0 +1,189 @@ +//! Fit-origin binding, precision, and draw contracts for the CPU `f64` reference. + +use corpus_split::{CorpusDocument, CorpusSnapshot}; +use membership_core::{ + GroupId, MemberId, MembershipAssignment, MembershipNetwork, MembershipRole, MembershipWeight, +}; +use relation_graph::{ + RelationEdge, RelationEndpointId, RelationEvidenceStatus, RelationGraph, RelationKind, +}; +use temporal_core::{ + AvailableTime, EventTime, KnowledgeCutoff, TemporalBoundary, TemporalInterval, + TemporalPrecision, +}; +use topic_measurement::{ + ReferenceTopicInput, ReferenceTopicModelConfig, SparseMatrix, fit_reference_topic_model, +}; +use uuid::Uuid; + +fn event_time(day: u8) -> EventTime { + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T00:00:00Z")).expect("event time") +} + +fn relation(source: Uuid, target: Uuid, source_day: u8, target_day: u8) -> RelationEdge { + let interval = |day| { + TemporalInterval::bounded( + TemporalBoundary::Included(event_time(day)), + TemporalBoundary::Included( + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T12:00:00Z")) + .expect("interval end"), + ), + TemporalPrecision::Second, + ) + .expect("bounded interval") + }; + RelationEdge::new( + RelationKind::TransitionsTo, + RelationEndpointId::from_uuid(source), + RelationEndpointId::from_uuid(target), + RelationEvidenceStatus::Observed, + interval(source_day), + interval(target_day), + ) + .expect("forward relation") +} + +fn fixture() -> ( + CorpusSnapshot, + Vec, + Vec, + MembershipNetwork, + RelationGraph, +) { + let document_ids: Vec<_> = (1_u128..=6).map(Uuid::from_u128).collect(); + let times: Vec<_> = (1_u8..=6).map(event_time).collect(); + let available = AvailableTime::parse_rfc3339("2026-01-10T00:00:00Z").expect("available"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + let mut snapshot = CorpusSnapshot::new(); + for id in &document_ids { + snapshot + .insert_if_eligible(CorpusDocument::new(*id, available), &cutoff) + .expect("eligible"); + } + + let organization = GroupId::from_uuid(Uuid::from_u128(100)); + let projects = [ + GroupId::from_uuid(Uuid::from_u128(101)), + GroupId::from_uuid(Uuid::from_u128(102)), + ]; + let validity_start = event_time(1); + let validity_end = event_time(9); + let mut memberships = MembershipNetwork::new(); + for (index, id) in document_ids.iter().enumerate() { + let member = MemberId::from_uuid(*id); + memberships + .insert( + MembershipAssignment::new( + member, + organization, + MembershipRole::Organization, + MembershipWeight::full().expect("full"), + validity_start, + validity_end, + ) + .expect("organization membership"), + ) + .expect("insert organization"); + memberships + .insert( + MembershipAssignment::new( + member, + projects[usize::from(index >= 3)], + MembershipRole::Project, + MembershipWeight::new(0.75).expect("partial"), + validity_start, + validity_end, + ) + .expect("project membership"), + ) + .expect("insert project"); + } + + let mut relations = RelationGraph::new(); + for (source, target, source_day, target_day) in [ + (0, 1, 1, 2), + (1, 2, 2, 3), + (2, 3, 3, 4), + (3, 4, 4, 5), + (4, 5, 5, 6), + ] { + relations + .insert(relation( + document_ids[source], + document_ids[target], + source_day, + target_day, + )) + .expect("insert relation"); + } + (snapshot, document_ids, times, memberships, relations) +} + +/// New characterization: public input and a real K=3 CPU fit feed the complete public builder. +#[test] +fn actual_three_topic_fit_retains_joint_precision_at_two_count_scales() { + for scale in [1.0_f64, 1_000.0_f64] { + let counts = SparseMatrix::from_csr( + 6, + 4, + vec![0, 2, 4, 6, 8, 10, 12], + vec![0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3], + vec![90., 10., 85., 15., 80., 20., 10., 90., 15., 85., 20., 80.] + .into_iter() + .map(|x| x * scale) + .collect(), + ) + .expect("finite positive public CSR counts"); + let (snapshot, ids, times, memberships, relations) = fixture(); + let input = ReferenceTopicInput::new( + &snapshot, + ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("fully admitted public input"); + let config = ReferenceTopicModelConfig::new(3, vec![7], 5_000, 1e-5) + .expect("K=3 configuration") + .with_hyperparameters(1.0, 0.5, 0.01, 0.05, 0.2) + .expect("hyperparameters"); + let fit = + fit_reference_topic_model(&input, &config).expect("actual converged public CPU fit"); + assert!(fit.objective.is_finite()); + assert_eq!(fit.document_topic_proportions.len(), 6); + assert_eq!(fit.topic_term_probabilities.len(), 3); + assert!(fit.document_topic_proportions.iter().all(|r| r.len() == 3 + && r.iter().all(|x| x.is_finite() && *x > 0.) + && (r.iter().sum::() - 1.).abs() < 1e-12)); + let tokens = 100. * scale; + let asymmetries: Vec = fit + .document_topic_proportions + .iter() + .map(|r| ((-tokens * r[0] * r[1]) - (-tokens * r[1] * r[0])).abs()) + .collect(); + println!( + "FIT scale={scale} tokens_per_document={tokens} iterations={} objective={} theta={:?} likelihood_asymmetries={asymmetries:?}", + fit.iterations, fit.objective, fit.document_topic_proportions + ); + let topics = (201_u128..=203).map(Uuid::from_u128).collect(); + let result = input.build_joint_coordinate_precision(&fit, &config, topics); + println!( + "BUILDER scale={scale} result={:?}", + result.as_ref().map(|x| ( + x.values().len(), + x.values().iter().map(Vec::len).collect::>() + )) + ); + let precision = + result.expect("actual admitted fitted model should retain finite joint precision"); + assert_eq!(precision.values().len(), 12); + assert!( + precision + .values() + .iter() + .all(|r| r.len() == 12 && r.iter().all(|x| x.is_finite())) + ); + } +} diff --git a/crates/topic_measurement/tests/precision_high_count_reachability.rs b/crates/topic_measurement/tests/precision_high_count_reachability.rs new file mode 100644 index 000000000..1b1db9a93 --- /dev/null +++ b/crates/topic_measurement/tests/precision_high_count_reachability.rs @@ -0,0 +1,190 @@ +//! Fit-origin binding, precision, and draw contracts for the CPU `f64` reference. + +use corpus_split::{CorpusDocument, CorpusSnapshot}; +use membership_core::{ + GroupId, MemberId, MembershipAssignment, MembershipNetwork, MembershipRole, MembershipWeight, +}; +use relation_graph::{ + RelationEdge, RelationEndpointId, RelationEvidenceStatus, RelationGraph, RelationKind, +}; +use temporal_core::{ + AvailableTime, EventTime, KnowledgeCutoff, TemporalBoundary, TemporalInterval, + TemporalPrecision, +}; +use topic_measurement::{ + ReferenceTopicInput, ReferenceTopicModelConfig, SparseMatrix, fit_reference_topic_model, +}; +use uuid::Uuid; + +fn event_time(day: u8) -> EventTime { + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T00:00:00Z")).expect("event time") +} + +fn relation(source: Uuid, target: Uuid, source_day: u8, target_day: u8) -> RelationEdge { + let interval = |day| { + TemporalInterval::bounded( + TemporalBoundary::Included(event_time(day)), + TemporalBoundary::Included( + EventTime::parse_rfc3339(&format!("2026-01-{day:02}T12:00:00Z")) + .expect("interval end"), + ), + TemporalPrecision::Second, + ) + .expect("bounded interval") + }; + RelationEdge::new( + RelationKind::TransitionsTo, + RelationEndpointId::from_uuid(source), + RelationEndpointId::from_uuid(target), + RelationEvidenceStatus::Observed, + interval(source_day), + interval(target_day), + ) + .expect("forward relation") +} + +fn fixture() -> ( + CorpusSnapshot, + Vec, + Vec, + MembershipNetwork, + RelationGraph, +) { + let document_ids: Vec<_> = (1_u128..=6).map(Uuid::from_u128).collect(); + let times: Vec<_> = (1_u8..=6).map(event_time).collect(); + let available = AvailableTime::parse_rfc3339("2026-01-10T00:00:00Z").expect("available"); + let cutoff = KnowledgeCutoff::parse_rfc3339("2026-02-01T00:00:00Z").expect("cutoff"); + let mut snapshot = CorpusSnapshot::new(); + for id in &document_ids { + snapshot + .insert_if_eligible(CorpusDocument::new(*id, available), &cutoff) + .expect("eligible"); + } + + let organization = GroupId::from_uuid(Uuid::from_u128(100)); + let projects = [ + GroupId::from_uuid(Uuid::from_u128(101)), + GroupId::from_uuid(Uuid::from_u128(102)), + ]; + let validity_start = event_time(1); + let validity_end = event_time(9); + let mut memberships = MembershipNetwork::new(); + for (index, id) in document_ids.iter().enumerate() { + let member = MemberId::from_uuid(*id); + memberships + .insert( + MembershipAssignment::new( + member, + organization, + MembershipRole::Organization, + MembershipWeight::full().expect("full"), + validity_start, + validity_end, + ) + .expect("organization membership"), + ) + .expect("insert organization"); + memberships + .insert( + MembershipAssignment::new( + member, + projects[usize::from(index >= 3)], + MembershipRole::Project, + MembershipWeight::new(0.75).expect("partial"), + validity_start, + validity_end, + ) + .expect("project membership"), + ) + .expect("insert project"); + } + + let mut relations = RelationGraph::new(); + for (source, target, source_day, target_day) in [ + (0, 1, 1, 2), + (1, 2, 2, 3), + (2, 3, 3, 4), + (3, 4, 4, 5), + (4, 5, 5, 6), + ] { + relations + .insert(relation( + document_ids[source], + document_ids[target], + source_day, + target_day, + )) + .expect("insert relation"); + } + (snapshot, document_ids, times, memberships, relations) +} + +/// New characterization: public input and a real K=3 CPU fit feed the complete public builder. +#[test] +fn actual_three_topic_high_count_fit_retains_joint_precision() { + { + let scale = 1_000_000.0_f64; + let counts = SparseMatrix::from_csr( + 6, + 4, + vec![0, 2, 4, 6, 8, 10, 12], + vec![0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3], + vec![90., 10., 85., 15., 80., 20., 10., 90., 15., 85., 20., 80.] + .into_iter() + .map(|x| x * scale) + .collect(), + ) + .expect("finite positive public CSR counts"); + let (snapshot, ids, times, memberships, relations) = fixture(); + let input = ReferenceTopicInput::new( + &snapshot, + ids, + &counts, + ×, + None, + &memberships, + &relations, + ) + .expect("fully admitted public input"); + let config = ReferenceTopicModelConfig::new(3, vec![7], 5_000, 1e-5) + .expect("K=3 configuration") + .with_hyperparameters(1.0, 0.5, 0.01, 0.05, 0.2) + .expect("hyperparameters"); + let fit = + fit_reference_topic_model(&input, &config).expect("actual converged public CPU fit"); + assert!(fit.objective.is_finite()); + assert_eq!(fit.document_topic_proportions.len(), 6); + assert_eq!(fit.topic_term_probabilities.len(), 3); + assert!(fit.document_topic_proportions.iter().all(|r| r.len() == 3 + && r.iter().all(|x| x.is_finite() && *x > 0.) + && (r.iter().sum::() - 1.).abs() < 1e-12)); + let tokens = 100. * scale; + let asymmetries: Vec = fit + .document_topic_proportions + .iter() + .map(|r| ((-tokens * r[0] * r[1]) - (-tokens * r[1] * r[0])).abs()) + .collect(); + println!( + "FIT scale={scale} tokens_per_document={tokens} iterations={} objective={} theta={:?} likelihood_asymmetries={asymmetries:?}", + fit.iterations, fit.objective, fit.document_topic_proportions + ); + let topics = (201_u128..=203).map(Uuid::from_u128).collect(); + let result = input.build_joint_coordinate_precision(&fit, &config, topics); + println!( + "BUILDER scale={scale} result={:?}", + result.as_ref().map(|x| ( + x.values().len(), + x.values().iter().map(Vec::len).collect::>() + )) + ); + let precision = + result.expect("actual admitted fitted model should retain finite joint precision"); + assert_eq!(precision.values().len(), 12); + assert!( + precision + .values() + .iter() + .all(|r| r.len() == 12 && r.iter().all(|x| x.is_finite())) + ); + } +}