From ecd2e248f51eb4d2a634051deabaab916fd2bc21 Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Mon, 9 Nov 2026 13:40:02 +0700 Subject: [PATCH 1/2] test(breg): wait for timer warnings before stopping replicas Readiness refuses as soon as the successor activation is applied, before the parked predecessor worker leaves its checkpoint. The removed-timer test then crashed the replica before the worker wrote its record_timer.worker.iteration_failed line, so the log check saw no fenced attempt. Wait for the line itself before the crash. The operator retry journey has the same ordering: each warning follows the commit that makes its attempt visible. Wait for the expected count before stopping those replicas too. Signed-off-by: Jeremi Joslin --- .../tests/postgres_record_timer_native.rs | 12 +++++++++++ .../tests/postgres_record_timer_operator.rs | 19 +++++++++++++++++ .../tests/support/record_timer_native.rs | 21 +++++++++++++++++-- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/crates/registry-breg/tests/postgres_record_timer_native.rs b/crates/registry-breg/tests/postgres_record_timer_native.rs index 51195389dc..102d2f5da0 100644 --- a/crates/registry-breg/tests/postgres_record_timer_native.rs +++ b/crates/registry-breg/tests/postgres_record_timer_native.rs @@ -887,6 +887,18 @@ async fn removed_timer_fences_predecessor_process( harness .wait_replica_unready(predecessor, Duration::from_secs(10)) .await?; + // Readiness already refuses once the successor is active, before the + // parked worker resumes. The worker writes its fenced-iteration warning + // only after it leaves the checkpoint, so wait for that line: it proves + // the predecessor resumed and was fenced before the unchanged-lease check + // below, and the crash cannot cut it off. + harness + .wait_for_log_code( + predecessor, + "record_timer.worker.iteration_failed", + Duration::from_secs(10), + ) + .await?; harness.expect_non_info_code_range( predecessor, "record_timer.worker.iteration_failed", diff --git a/crates/registry-breg/tests/postgres_record_timer_operator.rs b/crates/registry-breg/tests/postgres_record_timer_operator.rs index 75810234e9..fbadd37b1a 100644 --- a/crates/registry-breg/tests/postgres_record_timer_operator.rs +++ b/crates/registry-breg/tests/postgres_record_timer_operator.rs @@ -1191,6 +1191,16 @@ async fn operator_retry_after_actual_exhaustion( } } harness.expect_non_info_code(owner, "record_timer.worker.iteration_failed", 5)?; + // Each warning follows its attempt's commit, so the fifth attempt can be + // visible before its warning is written; the crash must not cut it off. + harness + .wait_for_log_code_count( + owner, + "record_timer.worker.iteration_failed", + 5, + Duration::from_secs(5), + ) + .await?; harness.crash_all_replicas()?; let operator_config = harness.write_operator_runtime_config("actual-retry")?; @@ -1243,6 +1253,15 @@ async fn operator_retry_after_actual_exhaustion( let blocked = wait_for_technical_attempt_count(harness, &record_id, 6, Duration::from_secs(8)).await?; harness.expect_non_info_code(retry_owner, "record_timer.worker.iteration_failed", 1)?; + // The warning follows the commit that made attempt six visible, and the + // replica stops at cleanup; wait so the stop cannot cut it off. + harness + .wait_for_log_code( + retry_owner, + "record_timer.worker.iteration_failed", + Duration::from_secs(5), + ) + .await?; if blocked.occurrence_id != occurrence_id || blocked.state != "blocked" || blocked.block_kind.as_deref() != Some("technical") diff --git a/crates/registry-breg/tests/support/record_timer_native.rs b/crates/registry-breg/tests/support/record_timer_native.rs index d2a0776228..d56cec4e51 100644 --- a/crates/registry-breg/tests/support/record_timer_native.rs +++ b/crates/registry-breg/tests/support/record_timer_native.rs @@ -2903,6 +2903,21 @@ impl NativeTimerHarness { replica: usize, code: &str, timeout: Duration, + ) -> Result<(), String> { + self.wait_for_log_code_count(replica, code, 1, timeout) + .await + } + + /// Wait until one replica's log holds at least `count` lines carrying + /// `code`. A worker writes its iteration warning only after the step it + /// reports, so neither database evidence nor readiness proves the line + /// exists yet; stopping the replica first can cut it off. + pub async fn wait_for_log_code_count( + &self, + replica: usize, + code: &str, + count: usize, + timeout: Duration, ) -> Result<(), String> { let replica = self .replicas @@ -2913,14 +2928,16 @@ impl NativeTimerHarness { loop { let rendered = fs::read_to_string(&replica.log_path) .map_err(|_| "native timer process log could not be read".to_owned())?; - if rendered.contains(&needle) { + if rendered.matches(&needle).count() >= count { return Ok(()); } tokio::time::sleep(Duration::from_millis(25)).await; } }) .await - .map_err(|_| "native timer process did not emit the expected log code".to_owned())? + .map_err(|_| { + format!("native timer process did not emit {count} {code} log line(s) before the bound") + })? } /// Terminate only the PostgreSQL sessions owned by one native process. From bf351a727e55db3a0c4e6b763ceb1f49a482950d Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Mon, 9 Nov 2026 13:42:16 +0700 Subject: [PATCH 2/2] fix(ci): give nightly coverage an absolute target dir Cargo runs tests from their crate directory, so the relative CARGO_TARGET_DIR sent the registry-rstack bootstrap tests to a scratch parent that does not exist and failed the developer-tools shard every night. Signed-off-by: Jeremi Joslin --- .github/scripts/test_nightly_rust_coverage.py | 9 +++++++++ .github/workflows/nightly-rust-coverage.yml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/scripts/test_nightly_rust_coverage.py b/.github/scripts/test_nightly_rust_coverage.py index fb2c3b11c0..81c477e0fa 100644 --- a/.github/scripts/test_nightly_rust_coverage.py +++ b/.github/scripts/test_nightly_rust_coverage.py @@ -58,6 +58,15 @@ def test_all_live_shards_have_stable_flags_and_owned_packages(self) -> None: str(entry["name"] == "platform").lower(), ) + def test_target_dir_is_absolute_for_tests_that_run_in_crate_dirs(self) -> None: + # Cargo runs each test with its crate directory as the working + # directory, so a relative CARGO_TARGET_DIR points tests that create + # scratch space under it at a directory that does not exist. + workflow = WORKFLOW.read_text(encoding="utf-8") + target = re.search(r"^ CARGO_TARGET_DIR: (.+)$", workflow, re.M) + self.assertIsNotNone(target) + self.assertTrue(target.group(1).startswith("${{ github.workspace }}/")) + def test_plan_validates_locked_metadata_and_appends_github_output(self) -> None: with tempfile.TemporaryDirectory() as directory: output = Path(directory) / "output" diff --git a/.github/workflows/nightly-rust-coverage.yml b/.github/workflows/nightly-rust-coverage.yml index 13b74d30ba..bf7e5eb6d0 100644 --- a/.github/workflows/nightly-rust-coverage.yml +++ b/.github/workflows/nightly-rust-coverage.yml @@ -49,7 +49,7 @@ jobs: max-parallel: 4 matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} env: - CARGO_TARGET_DIR: target/nightly-rust-coverage + CARGO_TARGET_DIR: ${{ github.workspace }}/target/nightly-rust-coverage COVERAGE_ALL_FEATURES: ${{ matrix.all_features }} COVERAGE_FEATURES: ${{ matrix.features }} COVERAGE_PACKAGES: ${{ matrix.packages }}