From e21e443d4654be21a5a5837b4a9f48103d9c36bc Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 10 Nov 2026 14:39:49 +0100 Subject: [PATCH] test(runner): pass an explicit per-test timeout and grace to program rows Program rows relied on the program's 900 s default, so one hung test could use much of the 45-minute run timeout. Derive the per-test timeout from the same constants as the nextest profile (120 s) and pass it with a 10 s grace to the program. Part of #5416. Signed-off-by: Evan Lezar --- tests/runner/src/bundle.rs | 33 ++++++++++++++++++++++++++------- tests/runner/src/execute.rs | 18 ++++++++++++++++-- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/tests/runner/src/bundle.rs b/tests/runner/src/bundle.rs index 95d0b914ea..7942412b7e 100644 --- a/tests/runner/src/bundle.rs +++ b/tests/runner/src/bundle.rs @@ -31,17 +31,29 @@ pub const PROFILE: &str = "openshell"; /// The readiness check every bundle carries, at the bundle root. pub const READY_BINARY: &str = "testkit-ready"; -/// Nextest flags a test as slow after the period and terminates it after -/// `terminate-after` periods, so a hung test fails after two minutes instead -/// of holding the lane until the job times out. -const NEXTEST_CONFIG: &str = r#"# Written by the OpenShell test runner. +/// Nextest flags a test as slow after this period and terminates it after +/// [`SLOW_TERMINATE_AFTER`] periods. +const SLOW_PERIOD_SECS: u64 = 30; +const SLOW_TERMINATE_AFTER: u64 = 4; + +/// How long one test may run before the runner stops it, on either kind of +/// bundle, so a hung test fails after two minutes instead of holding the lane +/// until the job times out. +pub const TEST_TIMEOUT_SECS: u64 = SLOW_PERIOD_SECS * SLOW_TERMINATE_AFTER; + +/// The nextest configuration written into a nextest bundle. +fn nextest_config() -> String { + format!( + r#"# Written by the OpenShell test runner. [profile.openshell] fail-fast = false -slow-timeout = { period = "30s", terminate-after = 4 } +slow-timeout = {{ period = "{SLOW_PERIOD_SECS}s", terminate-after = {SLOW_TERMINATE_AFTER} }} [profile.openshell.junit] path = "junit.xml" -"#; +"# + ) +} /// The prebuilt program's name in a program bundle. pub const PROGRAM_BINARY: &str = "test-program"; @@ -204,7 +216,7 @@ pub fn build(root: &Path, family: &str, packages: &[String], out: &Path) -> Resu run(&mut command).with_context(|| format!("build the {} test archive", workspace.display()))?; copy_manifests(&workspace, out)?; - std::fs::write(out.join(CONFIG), NEXTEST_CONFIG)?; + std::fs::write(out.join(CONFIG), nextest_config())?; std::fs::copy(&ready, out.join(READY_BINARY)) .with_context(|| format!("copy {}", ready.display()))?; let manifest = Manifest { @@ -343,6 +355,13 @@ fn run(command: &mut Command) -> Result<()> { mod tests { use super::*; + #[test] + fn nextest_profile_stops_a_test_at_the_shared_timeout() { + let config = nextest_config(); + assert!(config.contains(r#"slow-timeout = { period = "30s", terminate-after = 4 }"#)); + assert_eq!(TEST_TIMEOUT_SECS, 120); + } + #[test] fn program_bundle_holds_only_the_program() { let dir = tempfile::tempdir().unwrap(); diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index 2f477134f9..7a7aa62cc2 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -20,7 +20,7 @@ use anyhow::{Context, Result, ensure}; use tmachine::Session; use tmachine::config::Config; -use crate::bundle::{Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY}; +use crate::bundle::{Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY, TEST_TIMEOUT_SECS}; use crate::test_env; const GUEST_DIR: &str = "/var/tmp/openshell-tests"; @@ -42,6 +42,9 @@ const READY_SLACK: Duration = Duration::from_mins(1); const NEXTEST_TEST_FAILED: i32 = 100; /// nextest's exit code when `--no-tests fail` found no tests to run. const NEXTEST_NO_TESTS_RUN: i32 = 4; +/// How long a stopped test has between SIGTERM and SIGKILL. Matches nextest's +/// default, so both kinds of bundle stop a hung test the same way. +const TEST_GRACE_SECS: u64 = 10; /// A testkit program's exit code when a test failed. It exits 2 when tests /// could not run. const PROGRAM_TEST_FAILED: i32 = 1; @@ -308,6 +311,8 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { let archive = format!("{guest}/tests.tar.zst"); let extract = format!("{guest}/extract"); let config = format!("{guest}/nextest.toml"); + let timeout = TEST_TIMEOUT_SECS.to_string(); + let grace = TEST_GRACE_SECS.to_string(); let mut tests = vec!["env"]; tests.extend(env.iter().map(String::as_str)); match kind { @@ -342,7 +347,16 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { // The program runs its tests one at a time, and writes the report // where `collect_results` looks for it. Kind::Program => { - tests.extend([program.as_str(), "run", "--junit", &junit]); + tests.extend([ + program.as_str(), + "run", + "--timeout", + &timeout, + "--grace", + &grace, + "--junit", + &junit, + ]); // `run` takes name filters as arguments, after `--` so that one // starting with a dash isn't read as an option. if !run.filters.is_empty() {