From e2ba494186dd9cacf91d2f0937f95067d1072e43 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 15:16:57 -0800 Subject: [PATCH 1/9] test(conformance): port sandbox lifecycle and API e2e tests to testkit Move the driver-agnostic sandbox lifecycle, labels, templates, settings, image, process, Landlock and upload tests from e2e/rust to the conformance suite, the Podman OCI identity test to the podman driver suite, and the doctor, proxy-preflight and WebSocket tunnel checks to crate tests. Move the live-Internet benchmark to mise run perf:live-internet. Delete the legacy files and their references. Signed-off-by: Matthew Grossman --- TESTING.md | 2 +- .../openshell-cli/tests/doctor_integration.rs | 142 ++++++ .../tests/proxy_config_preflight.rs | 137 ++++++ crates/openshell-server/src/lib.rs | 74 ++- e2e/rust/Cargo.toml | 24 - e2e/rust/e2e-podman.sh | 7 - e2e/rust/e2e-vm.sh | 1 - e2e/rust/src/harness/sandbox.rs | 16 +- e2e/rust/tests/core_dump_hardening.rs | 26 -- e2e/rust/tests/default_image.rs | 40 -- e2e/rust/tests/docker_corporate_proxy.rs | 60 --- e2e/rust/tests/docker_preflight.rs | 171 ------- e2e/rust/tests/edge_tunnel_e2e.rs | 220 --------- e2e/rust/tests/ephemeral_cleanup.rs | 189 -------- e2e/rust/tests/internet_network_perf.rs | 170 ------- e2e/rust/tests/landlock.rs | 150 ------ e2e/rust/tests/live_internet_traffic_perf.rs | 337 -------------- e2e/rust/tests/podman_oci_identity.rs | 213 --------- e2e/rust/tests/podman_resource_limits.rs | 2 +- e2e/rust/tests/sandbox_labels.rs | 230 ---------- e2e/rust/tests/sandbox_lifecycle.rs | 429 +----------------- e2e/rust/tests/sandbox_templates.rs | 275 ----------- e2e/rust/tests/settings_management.rs | 329 -------------- e2e/rust/tests/upload_backpressure.rs | 121 ----- e2e/rust/tests/vm_corporate_proxy.rs | 38 -- e2e/with-docker-gateway.sh | 2 - e2e/with-podman-gateway.sh | 2 - tasks/rust.toml | 4 + tasks/scripts/live_internet_perf.py | 163 +++++++ tasks/scripts/live_internet_perf_round.py | 255 +++++++++++ tests/suites/conformance/Cargo.lock | 1 + tests/suites/conformance/Cargo.toml | 1 + tests/suites/conformance/README.md | 10 +- tests/suites/conformance/src/file_transfer.rs | 144 +++++- .../conformance/src/filesystem_policy.rs | 162 +++++++ tests/suites/conformance/src/image.rs | 53 +++ tests/suites/conformance/src/labels.rs | 93 ++++ tests/suites/conformance/src/lib.rs | 6 + tests/suites/conformance/src/lifecycle.rs | 399 +++++++++++++++- tests/suites/conformance/src/process.rs | 34 ++ tests/suites/conformance/src/settings.rs | 196 ++++++++ tests/suites/conformance/src/templates.rs | 190 ++++++++ tests/suites/drivers/podman/Cargo.lock | 1 + tests/suites/drivers/podman/Cargo.toml | 1 + .../drivers/podman/tests/oci_identity.rs | 115 +++++ tests/testkit/testkit/src/cli.rs | 11 + 46 files changed, 2178 insertions(+), 3068 deletions(-) create mode 100644 crates/openshell-cli/tests/doctor_integration.rs create mode 100644 crates/openshell-gateway/tests/proxy_config_preflight.rs delete mode 100644 e2e/rust/tests/core_dump_hardening.rs delete mode 100644 e2e/rust/tests/default_image.rs delete mode 100644 e2e/rust/tests/docker_preflight.rs delete mode 100644 e2e/rust/tests/edge_tunnel_e2e.rs delete mode 100644 e2e/rust/tests/ephemeral_cleanup.rs delete mode 100644 e2e/rust/tests/internet_network_perf.rs delete mode 100644 e2e/rust/tests/landlock.rs delete mode 100644 e2e/rust/tests/live_internet_traffic_perf.rs delete mode 100644 e2e/rust/tests/podman_oci_identity.rs delete mode 100644 e2e/rust/tests/sandbox_labels.rs delete mode 100644 e2e/rust/tests/sandbox_templates.rs delete mode 100644 e2e/rust/tests/settings_management.rs delete mode 100644 e2e/rust/tests/upload_backpressure.rs create mode 100644 tasks/scripts/live_internet_perf.py create mode 100644 tasks/scripts/live_internet_perf_round.py create mode 100644 tests/suites/conformance/src/filesystem_policy.rs create mode 100644 tests/suites/conformance/src/image.rs create mode 100644 tests/suites/conformance/src/labels.rs create mode 100644 tests/suites/conformance/src/process.rs create mode 100644 tests/suites/conformance/src/settings.rs create mode 100644 tests/suites/conformance/src/templates.rs create mode 100644 tests/suites/drivers/podman/tests/oci_identity.rs diff --git a/TESTING.md b/TESTING.md index f49be898a5..faf06a5e51 100644 --- a/TESTING.md +++ b/TESTING.md @@ -809,7 +809,7 @@ cargo test --manifest-path e2e/rust/Cargo.toml --features e2e --test sync Run a single Docker-only test directly with cargo: ```shell -cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test docker_preflight +cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test docker_corporate_proxy ``` The harness (`e2e/rust/src/harness/`) provides: diff --git a/crates/openshell-cli/tests/doctor_integration.rs b/crates/openshell-cli/tests/doctor_integration.rs new file mode 100644 index 0000000000..343dc7a9c2 --- /dev/null +++ b/crates/openshell-cli/tests/doctor_integration.rs @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `openshell doctor check` against a fake `docker` on `PATH`, so the result +//! doesn't depend on the host's Docker. + +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +struct Doctor { + combined: String, + code: i32, + elapsed: Duration, +} + +/// Run `openshell doctor check` with only `bin` on `PATH`. +fn doctor_check(bin: &Path) -> Doctor { + let home = tempfile::tempdir().expect("create isolated config dir"); + let start = Instant::now(); + let output = Command::new(env!("CARGO_BIN_EXE_openshell")) + .args(["doctor", "check"]) + .env("XDG_CONFIG_HOME", home.path()) + .env("HOME", home.path()) + .env("PATH", bin) + .env("NO_COLOR", "1") + .env_remove("DOCKER_HOST") + .env_remove("OPENSHELL_GATEWAY") + .env_remove("OPENSHELL_GATEWAY_ENDPOINT") + .stdin(Stdio::null()) + .output() + .expect("run openshell doctor check"); + Doctor { + combined: format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ), + code: output.status.code().unwrap_or(-1), + elapsed: start.elapsed(), + } +} + +/// Write an executable `docker` script to `bin` that records its arguments +/// in `bin/docker-args` and then runs `body`. +fn fake_docker(bin: &Path, body: &str) { + let docker = bin.join("docker"); + let args = bin.join("docker-args"); + fs::write( + &docker, + format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" > '{}'\n{body}\n", + args.display() + ), + ) + .expect("write fake docker"); + fs::set_permissions(&docker, fs::Permissions::from_mode(0o755)).expect("chmod fake docker"); +} + +fn docker_args(bin: &Path) -> String { + fs::read_to_string(bin.join("docker-args")).unwrap_or_default() +} + +#[test] +fn doctor_check_passes_when_docker_info_succeeds() { + let bin = tempfile::tempdir().expect("create fake bin dir"); + fake_docker(bin.path(), "echo 27.3.1"); + + let doctor = doctor_check(bin.path()); + assert_eq!(doctor.code, 0, "doctor check failed:\n{}", doctor.combined); + assert_eq!( + docker_args(bin.path()).trim(), + "info --format {{.ServerVersion}}", + "doctor check ran docker with unexpected arguments" + ); + for expected in ["Docker", "ok (version 27.3.1)", "All checks passed"] { + assert!( + doctor.combined.contains(expected), + "expected {expected:?} in doctor check output:\n{}", + doctor.combined + ); + } +} + +#[test] +fn doctor_check_fails_fast_when_docker_info_fails() { + let bin = tempfile::tempdir().expect("create fake bin dir"); + fake_docker( + bin.path(), + "echo 'Cannot connect to the Docker daemon' >&2\nexit 1", + ); + + let doctor = doctor_check(bin.path()); + assert_ne!( + doctor.code, 0, + "doctor check passed although docker info failed:\n{}", + doctor.combined + ); + assert!( + doctor.elapsed < Duration::from_secs(10), + "doctor check took {:?}", + doctor.elapsed + ); + for expected in [ + "Docker", + "FAILED", + "docker info failed", + "Cannot connect to the Docker daemon", + ] { + assert!( + doctor.combined.contains(expected), + "expected {expected:?} in doctor check output:\n{}", + doctor.combined + ); + } + assert!( + !doctor.combined.contains("All checks passed"), + "doctor check reported success:\n{}", + doctor.combined + ); +} + +#[test] +fn doctor_check_fails_without_docker_on_path() { + let bin = tempfile::tempdir().expect("create empty bin dir"); + + let doctor = doctor_check(bin.path()); + assert_ne!( + doctor.code, 0, + "doctor check passed without docker:\n{}", + doctor.combined + ); + assert!( + doctor.combined.contains("failed to execute docker info"), + "doctor check did not report that docker could not run:\n{}", + doctor.combined + ); +} diff --git a/crates/openshell-gateway/tests/proxy_config_preflight.rs b/crates/openshell-gateway/tests/proxy_config_preflight.rs new file mode 100644 index 0000000000..768d995f70 --- /dev/null +++ b/crates/openshell-gateway/tests/proxy_config_preflight.rs @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! An incoherent driver proxy table is fatal, rather than leaving every dial +//! direct while looking configured. A bypass list without a proxy URL means +//! the operator believed proxying was in effect. + +#![cfg(unix)] + +use std::fs; +use std::path::PathBuf; +use std::process::Output; +use std::time::Duration; + +/// Drivers with an upstream proxy table, compiled into this gateway. +const DRIVERS: &[&str] = &[ + #[cfg(feature = "compute-driver-docker")] + "docker", + #[cfg(feature = "compute-driver-podman")] + "podman", + #[cfg(feature = "compute-driver-vm")] + "vm", +]; + +struct Fixture { + root: tempfile::TempDir, + config: PathBuf, +} + +impl Fixture { + /// A gateway config that selects `driver` and sets only `no_proxy` in its + /// table. + fn incoherent(driver: &str) -> Self { + let root = tempfile::tempdir().expect("fixture root"); + let config = root.path().join("gateway.toml"); + fs::write( + &config, + format!( + "[openshell] +version = 2 +[openshell.gateway] +disable_tls = true +bind_address = \"127.0.0.1:0\" +compute_driver = {driver:?} +[openshell.gateway.auth] +allow_unauthenticated_users = true +[openshell.drivers.{driver}] +no_proxy = \"10.0.0.0/8\" +" + ), + ) + .expect("gateway configuration"); + Self { root, config } + } + + async fn run(&self, args: &[&str]) -> Output { + let mut command = tokio::process::Command::new(env!("CARGO_BIN_EXE_openshell-gateway")); + command + .env_clear() + .env("HOME", self.root.path()) + .env("PATH", "/usr/bin:/bin") + .env("NO_COLOR", "1") + .env("XDG_CONFIG_HOME", self.root.path().join("config-home")) + .env("XDG_STATE_HOME", self.root.path().join("state-home")) + .args(args) + .kill_on_drop(true); + tokio::time::timeout(Duration::from_secs(30), command.output()) + .await + .expect("the gateway must exit instead of starting") + .expect("run openshell-gateway") + } + + fn config_path(&self) -> &str { + self.config.to_str().expect("UTF-8 config path") + } +} + +fn combined(output: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +#[test] +fn proxy_drivers_are_compiled_in() { + assert!( + !DRIVERS.is_empty(), + "build with a compute driver that has a proxy table" + ); +} + +#[tokio::test] +async fn preflight_rejects_no_proxy_without_proxy() { + for driver in DRIVERS { + let fixture = Fixture::incoherent(driver); + let original = fs::read(&fixture.config).expect("original config"); + let output = fixture + .run(&["config", "preflight", "--path", fixture.config_path()]) + .await; + assert!( + !output.status.success(), + "preflight accepted an incoherent [openshell.drivers.{driver}] proxy table:\n{}", + combined(&output) + ); + assert!( + combined(&output).contains("category=malformed"), + "preflight did not classify the {driver} config as malformed:\n{}", + combined(&output) + ); + assert_eq!( + fs::read(&fixture.config).expect("config after preflight"), + original, + "preflight changed the {driver} config" + ); + } +} + +#[tokio::test] +async fn startup_rejects_no_proxy_without_proxy_and_names_the_keys() { + for driver in DRIVERS { + let fixture = Fixture::incoherent(driver); + let output = fixture.run(&["--config", fixture.config_path()]).await; + let report = combined(&output); + assert!( + !output.status.success(), + "the gateway started with an incoherent [openshell.drivers.{driver}] proxy table:\n{report}" + ); + // Matched on the key names, not the wording, so error wrapping can + // change. + assert!( + report.contains("no_proxy") && report.contains("https_proxy"), + "the {driver} startup error does not name the offending keys:\n{report}" + ); + } +} diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 0c9a352b26..082021033b 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -3103,11 +3103,21 @@ mod tests { async fn start_plaintext_gateway_listener() -> (SocketAddr, watch::Sender, tokio::task::JoinHandle<()>) { + start_plaintext_gateway_listener_with_tunnel(false).await + } + + async fn start_plaintext_gateway_listener_with_tunnel( + enable_websocket_tunnel: bool, + ) -> (SocketAddr, watch::Sender, tokio::task::JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0") .await .expect("failed to bind test listener"); let listen_addr = listener.local_addr().expect("failed to read local addr"); - let state = test_state(listen_addr, false).await; + let mut state = test_state(listen_addr, false).await; + Arc::get_mut(&mut state) + .expect("test server state should be uniquely owned") + .config + .enable_websocket_tunnel = enable_websocket_tunnel; let service = MultiplexService::new(state); let (shutdown_tx, shutdown_rx) = watch::channel(false); let handle = tokio::spawn(serve_gateway_listener( @@ -3163,6 +3173,68 @@ mod tests { stop_listener(shutdown, handle).await; } + /// The edge-tunnel client path: gRPC carried in binary frames over a + /// WebSocket to `/_ws_tunnel` on the production listener reaches the + /// gateway's own services. + #[tokio::test] + async fn production_gateway_listener_serves_grpc_through_websocket_tunnel() { + use futures::{SinkExt as _, StreamExt as _}; + use openshell_core::proto::{ + HealthRequest, ServiceStatus, open_shell_client::OpenShellClient, + }; + use tokio_tungstenite::tungstenite::Message; + + let (addr, shutdown, handle) = start_plaintext_gateway_listener_with_tunnel(true).await; + let ws_url = format!("ws://127.0.0.1:{}/_ws_tunnel", addr.port()); + let channel = tonic::transport::Endpoint::from_static("http://edge-tunnel.invalid") + .connect_with_connector(tower::service_fn(move |_| { + let ws_url = ws_url.clone(); + async move { + let (ws, _) = tokio_tungstenite::connect_async(ws_url) + .await + .map_err(Error::other)?; + let (mut ws_sink, mut ws_source) = ws.split(); + let (client, bridge) = tokio::io::duplex(64 * 1024); + let (mut bridge_read, mut bridge_write) = tokio::io::split(bridge); + tokio::spawn(async move { + let mut buf = vec![0u8; 32 * 1024]; + while let Ok(n @ 1..) = bridge_read.read(&mut buf).await { + if ws_sink + .send(Message::Binary(buf[..n].to_vec().into())) + .await + .is_err() + { + break; + } + } + let _ = ws_sink.close().await; + }); + tokio::spawn(async move { + while let Some(Ok(Message::Binary(data))) = ws_source.next().await { + if bridge_write.write_all(&data).await.is_err() { + break; + } + } + let _ = bridge_write.shutdown().await; + }); + Ok::<_, Error>(hyper_util::rt::TokioIo::new(client)) + } + })) + .await + .expect("gRPC channel through the WebSocket tunnel"); + + let response = tokio::time::timeout( + Duration::from_secs(10), + OpenShellClient::new(channel).health(HealthRequest {}), + ) + .await + .expect("health RPC through the tunnel timed out") + .expect("health RPC through the tunnel"); + assert_eq!(response.get_ref().status, ServiceStatus::Healthy as i32); + + stop_listener(shutdown, handle).await; + } + async fn send_plain_http(addr: SocketAddr, request: String) -> String { let connect_addr: SocketAddr = format!("127.0.0.1:{}", addr.port()) .parse() diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 2af824018c..4849bbda0b 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -89,11 +89,6 @@ name = "vm_bind_mount" path = "tests/vm_bind_mount.rs" required-features = ["e2e-vm"] -[[test]] -name = "docker_preflight" -path = "tests/docker_preflight.rs" -required-features = ["e2e-docker"] - [[test]] name = "docker_corporate_proxy" path = "tests/docker_corporate_proxy.rs" @@ -119,11 +114,6 @@ name = "local_driver_token_restart" path = "tests/local_driver_token_restart.rs" required-features = ["e2e"] -[[test]] -name = "ephemeral_cleanup" -path = "tests/ephemeral_cleanup.rs" -required-features = ["e2e"] - [[test]] name = "podman_gateway_start" path = "tests/podman_gateway_start.rs" @@ -144,11 +134,6 @@ name = "podman_corporate_proxy" path = "tests/podman_corporate_proxy.rs" required-features = ["e2e-podman"] -[[test]] -name = "podman_oci_identity" -path = "tests/podman_oci_identity.rs" -required-features = ["e2e-podman"] - [[test]] name = "podman_resource_limits" path = "tests/podman_resource_limits.rs" @@ -234,11 +219,6 @@ name = "workspace_lifecycle" path = "tests/workspace_lifecycle.rs" required-features = ["e2e"] -[[test]] -name = "sandbox_templates" -path = "tests/sandbox_templates.rs" -required-features = ["e2e"] - [[test]] name = "proxy_egress_pipeline" path = "tests/proxy_egress_pipeline.rs" @@ -310,7 +290,3 @@ module_name_repetitions = "allow" must_use_candidate = "allow" missing_errors_doc = "allow" missing_panics_doc = "allow" -[[test]] -name = "live_internet_traffic_perf" -path = "tests/live_internet_traffic_perf.rs" -required-features = ["e2e-host-gateway"] diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index e718f0afea..6891ecf221 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -23,16 +23,11 @@ source "${ROOT}/e2e/support/conformance.sh" # stabilized and can be added here. PODMAN_CI_TESTS=( bypass_detection - ephemeral_cleanup - core_dump_hardening - default_image driver_config_volume - edge_tunnel_e2e forward_proxy_graphql_l7 forward_proxy_jsonrpc_l7 forward_proxy_l7_bypass host_gateway_alias - landlock live_policy_update local_driver_token_restart no_proxy @@ -41,9 +36,7 @@ PODMAN_CI_TESTS=( podman_corporate_proxy podman_gateway_start podman_host_gateway - podman_oci_identity provider_token_exchange - upload_backpressure ) if [ -n "${E2E_TEST}" ] && [ -n "${E2E_TEST_SET}" ]; then diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index bafaadbc71..9ccb82f0af 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -493,7 +493,6 @@ run_e2e_test() { if [ -n "${E2E_TEST_OVERRIDE}" ]; then run_e2e_test "${E2E_TEST_OVERRIDE}" else - run_e2e_test ephemeral_cleanup run_e2e_test host_gateway_alias run_e2e_test vm_overlay run_e2e_test vm_bind_mount diff --git a/e2e/rust/src/harness/sandbox.rs b/e2e/rust/src/harness/sandbox.rs index c0f0c1aedc..9f57a29daa 100644 --- a/e2e/rust/src/harness/sandbox.rs +++ b/e2e/rust/src/harness/sandbox.rs @@ -129,18 +129,6 @@ impl SandboxGuard { /// Returns an error if the CLI exits with a non-zero status or the sandbox /// name cannot be parsed from the output. pub async fn create(args: &[&str]) -> Result { - Self::create_inner(args, true).await - } - - /// Create a sandbox using the gateway's configured default image. - /// - /// Most E2E tests use [`Self::create`], which supplies the tool-capable E2E - /// image. This variant is reserved for coverage of the product default. - pub async fn create_with_gateway_default(args: &[&str]) -> Result { - Self::create_inner(args, false).await - } - - async fn create_inner(args: &[&str], use_test_image: bool) -> Result { let separator = args.iter().position(|arg| *arg == "--"); let (create_args, command) = separator.map_or((args, &[][..]), |index| { (&args[..index], &args[index + 1..]) @@ -156,9 +144,7 @@ impl SandboxGuard { let mut cmd = openshell_cmd(); cmd.arg("sandbox").arg("create").arg("--detach"); add_unique_name_if_missing(&mut cmd, create_args); - if use_test_image { - add_test_image_if_missing(&mut cmd, create_args); - } + add_test_image_if_missing(&mut cmd, create_args); for arg in create_args { cmd.arg(arg); } diff --git a/e2e/rust/tests/core_dump_hardening.rs b/e2e/rust/tests/core_dump_hardening.rs deleted file mode 100644 index a400b6c218..0000000000 --- a/e2e/rust/tests/core_dump_hardening.rs +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -use openshell_e2e::harness::sandbox::SandboxGuard; - -#[tokio::test] -async fn sandbox_processes_disable_core_dumps() { - let mut sb = SandboxGuard::create(&[ - "--", - "sh", - "-lc", - "test \"$(ulimit -c)\" = 0 && echo core-limit-ok", - ]) - .await - .expect("sandbox create should succeed"); - - assert!( - sb.create_output.contains("core-limit-ok"), - "expected sandbox output to confirm core dumps are disabled:\n{}", - sb.create_output, - ); - - sb.cleanup().await; -} diff --git a/e2e/rust/tests/default_image.rs b/e2e/rust/tests/default_image.rs deleted file mode 100644 index 6a1dbb3cd8..0000000000 --- a/e2e/rust/tests/default_image.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for the default NVIDIA Ubuntu workload image. - -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; - -#[tokio::test] -async fn sandbox_from_default_image() { - let mut guard = SandboxGuard::create_with_gateway_default(&["--", "cat", "/etc/os-release"]) - .await - .expect("sandbox create from default image"); - - let clean_output = strip_ansi(&guard.create_output); - assert!( - clean_output.contains("ID=ubuntu"), - "expected Ubuntu OS release info in sandbox output:\n{clean_output}" - ); - - guard.cleanup().await; -} - -#[tokio::test] -async fn sandbox_from_explicit_nvidia_ubuntu_image() { - let image = "nvcr.io/nvidia/base/ubuntu:24.04"; - let mut guard = SandboxGuard::create(&["--from", image, "--", "cat", "/etc/os-release"]) - .await - .expect("sandbox create from explicit NVIDIA Ubuntu image"); - - let clean_output = strip_ansi(&guard.create_output); - assert!( - clean_output.contains("ID=ubuntu"), - "expected Ubuntu OS release info in sandbox output:\n{clean_output}" - ); - - guard.cleanup().await; -} diff --git a/e2e/rust/tests/docker_corporate_proxy.rs b/e2e/rust/tests/docker_corporate_proxy.rs index 85584b966f..659c6c5f8f 100644 --- a/e2e/rust/tests/docker_corporate_proxy.rs +++ b/e2e/rust/tests/docker_corporate_proxy.rs @@ -23,8 +23,6 @@ //! 5. An `https://` proxy works when its CA is supplied via `proxy_ca_bundle`. //! 6. A TLS-intercepting proxy CA is trusted by both inspected supervisor //! upstream TLS and a raw workload TLS connection. -//! 7. An incoherent setting is fatal at gateway startup rather than -//! degrading to a direct dial. //! //! Fixtures run as host processes and are reached by the host-networked //! supervisor. `host.openshell.internal` is normalized to the gateway host @@ -681,28 +679,6 @@ fn insert_docker_driver_settings(config: &str, extra: &str) -> Result Result { @@ -1142,39 +1118,3 @@ async fn docker_corporate_proxy_trusts_intercepted_destination_tls() { .await .expect("restore gateway config"); } - -#[tokio::test] -#[serial(docker_corporate_proxy)] -async fn docker_corporate_proxy_rejects_incoherent_configuration() { - if !should_run("corporate proxy fail-closed test") { - return; - } - - // A bypass list without a proxy URL means the operator believed proxying - // was in effect. Accepting it would leave every dial direct while looking - // configured, so the gateway must refuse to start instead. - let mut gateway_config = GatewayProxyConfig::apply_raw("no_proxy = \"10.0.0.0/8\"\n") - .expect("apply incoherent corporate proxy gateway config"); - - let health = wait_for_healthy(Duration::from_secs(20)).await; - assert!( - health.is_err(), - "gateway must not serve traffic with an incoherent [openshell.drivers.docker] proxy table" - ); - - let log_path = std::env::var("OPENSHELL_E2E_GATEWAY_LOG").expect("gateway log path"); - let log = std::fs::read_to_string(&log_path).expect("read gateway log"); - // The message must name the offending key rather than surfacing as an - // opaque driver-readiness timeout. Matched loosely on the key names so - // this does not break on error-wrapping changes. - assert!( - log.contains("no_proxy") && log.contains("https_proxy"), - "the startup error must name the offending key; gateway log tail:\n{}", - log.lines().rev().take(40).collect::>().join("\n") - ); - - gateway_config - .restore() - .await - .expect("restore gateway config"); -} diff --git a/e2e/rust/tests/docker_preflight.rs b/e2e/rust/tests/docker_preflight.rs deleted file mode 100644 index bf05a132d9..0000000000 --- a/e2e/rust/tests/docker_preflight.rs +++ /dev/null @@ -1,171 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Doctor Docker preflight e2e tests. -//! -//! These tests verify that `openshell doctor check` reports actionable guidance -//! when Docker is not available. -//! -//! The tests do NOT require a running gateway or Docker — they intentionally -//! point `DOCKER_HOST` at a non-existent socket to simulate Docker being -//! unavailable. - -use std::process::Stdio; -use std::time::Instant; -use std::{env, fs}; - -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::strip_ansi; - -/// Run `openshell ` in an isolated environment where Docker is -/// guaranteed to be unreachable. -/// -/// Sets `DOCKER_HOST` to a non-existent socket so the preflight check -/// fails immediately regardless of the host's Docker configuration. -async fn run_without_docker(args: &[&str]) -> (String, i32, std::time::Duration) { - let tmpdir = tempfile::tempdir().expect("create isolated config dir"); - let bin_dir = tmpdir.path().join("bin"); - fs::create_dir(&bin_dir).expect("create fake bin dir"); - let fake_docker = bin_dir.join("docker"); - fs::write( - &fake_docker, - "#!/bin/sh\n\ - echo 'Cannot connect to Docker daemon. Check DOCKER_HOST and run docker info.' >&2\n\ - exit 1\n", - ) - .expect("write fake docker"); - #[cfg(unix)] - fs::set_permissions(&fake_docker, fs::Permissions::from_mode(0o755)) - .expect("chmod fake docker"); - - let old_path = env::var("PATH").unwrap_or_default(); - let path = format!("{}:{old_path}", bin_dir.display()); - let start = Instant::now(); - - let mut cmd = openshell_cmd(); - cmd.args(args) - .env("XDG_CONFIG_HOME", tmpdir.path()) - .env("HOME", tmpdir.path()) - .env("PATH", path) - .env("DOCKER_HOST", "unix:///tmp/openshell-e2e-nonexistent.sock") - .env_remove("OPENSHELL_GATEWAY") - .env_remove("OPENSHELL_GATEWAY_ENDPOINT") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let elapsed = start.elapsed(); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - (combined, code, elapsed) -} - -// ------------------------------------------------------------------- -// doctor check: validates system prerequisites -// ------------------------------------------------------------------- - -/// `openshell doctor check` with Docker unavailable should fail fast -/// and report the Docker check as FAILED. -#[tokio::test] -async fn doctor_check_fails_without_docker() { - let (output, code, elapsed) = run_without_docker(&["doctor", "check"]).await; - - assert_ne!( - code, 0, - "doctor check should fail when Docker is unavailable, output:\n{output}" - ); - - assert!( - elapsed.as_secs() < 10, - "doctor check should complete quickly (took {}s)", - elapsed.as_secs() - ); - - let clean = strip_ansi(&output); - assert!( - clean.contains("FAILED"), - "doctor check should report Docker as FAILED:\n{clean}" - ); -} - -/// `openshell doctor check` output should include the check label -/// so the user knows what was tested. -#[tokio::test] -async fn doctor_check_output_shows_docker_label() { - let (output, _, _) = run_without_docker(&["doctor", "check"]).await; - let clean = strip_ansi(&output); - - assert!( - clean.contains("Docker"), - "doctor check output should include 'Docker' label:\n{clean}" - ); -} - -/// `openshell doctor check` with Docker unavailable should include -/// actionable guidance in the error output. -#[tokio::test] -async fn doctor_check_error_includes_guidance() { - let (output, code, _) = run_without_docker(&["doctor", "check"]).await; - - assert_ne!(code, 0); - let clean = strip_ansi(&output); - - assert!( - clean.contains("DOCKER_HOST"), - "doctor check error should mention DOCKER_HOST:\n{clean}" - ); - assert!( - clean.contains("docker info"), - "doctor check error should suggest 'docker info':\n{clean}" - ); -} - -/// When Docker IS available, `openshell doctor check` should pass and -/// report the version. -/// -/// This test only runs when Docker is actually reachable on the host -/// (i.e., it will pass in CI with Docker but be skipped locally if -/// Docker is not running). We detect this by checking if the default -/// socket exists. -#[tokio::test] -async fn doctor_check_passes_with_docker() { - if !std::path::Path::new("/var/run/docker.sock").exists() { - eprintln!("skipping: /var/run/docker.sock not found"); - return; - } - - let tmpdir = tempfile::tempdir().expect("create isolated config dir"); - let mut cmd = openshell_cmd(); - cmd.args(["doctor", "check"]) - .env("XDG_CONFIG_HOME", tmpdir.path()) - .env("HOME", tmpdir.path()) - .env_remove("OPENSHELL_GATEWAY") - .env_remove("OPENSHELL_GATEWAY_ENDPOINT") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - let clean = strip_ansi(&combined); - - assert_eq!( - code, 0, - "doctor check should pass when Docker is available, output:\n{clean}" - ); - assert!( - clean.contains("All checks passed"), - "doctor check should report success:\n{clean}" - ); - assert!( - clean.contains("ok"), - "doctor check should show 'ok' for Docker:\n{clean}" - ); -} diff --git a/e2e/rust/tests/edge_tunnel_e2e.rs b/e2e/rust/tests/edge_tunnel_e2e.rs deleted file mode 100644 index 80ee4a6ed8..0000000000 --- a/e2e/rust/tests/edge_tunnel_e2e.rs +++ /dev/null @@ -1,220 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E tests for edge tunnel auth flow against a running gateway. -//! -//! Prerequisites: -//! - A running openshell gateway -//! - For WS tunnel coverage, the gateway's HTTP endpoint is accessible (no TLS) -//! - The `openshell` binary (built automatically from the workspace) -//! -//! These tests exercise the full CLI → WS tunnel → gRPC flow. -//! -//! Environment variables: -//! - `OPENSHELL_GATEWAY`: Name of the active gateway (standard e2e var) -//! - `OPENSHELL_GATEWAY_ENDPOINT`: Optional direct plaintext endpoint. -//! -//! The edge-tunnel path requires a gateway endpoint that accepts plaintext HTTP. - -use std::process::Stdio; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::strip_ansi; - -/// Run `openshell ` using the system's configured gateway. -async fn run_cli(args: &[&str]) -> (String, i32) { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - (combined, code) -} - -/// Run `openshell ` with a custom config directory so the CLI reads -/// our seeded gateway metadata and edge token instead of the real config. -async fn run_cli_with_config(config_dir: &std::path::Path, args: &[&str]) -> (String, i32) { - let mut cmd = openshell_cmd(); - cmd.args(args) - .env("XDG_CONFIG_HOME", config_dir) - .env("HOME", config_dir) - .env_remove("OPENSHELL_GATEWAY") - .env_remove("OPENSHELL_GATEWAY_ENDPOINT") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - (combined, code) -} - -/// Seed a temporary config directory with gateway metadata that has -/// `auth_mode: "cloudflare_jwt"`, a stored edge token, and an active gateway -/// pointing at the given endpoint. -fn seed_edge_gateway_config( - config_dir: &std::path::Path, - gateway_name: &str, - gateway_endpoint: &str, - edge_token: &str, -) { - let openshell_dir = config_dir.join("openshell"); - let gateways_dir = openshell_dir.join("gateways"); - - std::fs::create_dir_all(&openshell_dir).expect("create openshell config dir"); - std::fs::write(openshell_dir.join("active_gateway"), gateway_name) - .expect("write active_gateway"); - - // Write gateway metadata JSON. - let gateway_dir = gateways_dir.join(gateway_name); - std::fs::create_dir_all(&gateway_dir).expect("create gateway dir"); - let metadata = serde_json::json!({ - "name": gateway_name, - "gateway_endpoint": gateway_endpoint, - "is_remote": false, - "gateway_port": 0, - "auth_mode": "cloudflare_jwt" - }); - std::fs::write( - gateway_dir.join("metadata.json"), - serde_json::to_string_pretty(&metadata).unwrap(), - ) - .expect("write gateway metadata"); - - // Write edge token file. - std::fs::write(gateway_dir.join("edge_token"), edge_token).expect("write edge_token"); -} - -// ------------------------------------------------------------------- -// Test 12: gRPC health check against a gateway -// ------------------------------------------------------------------- - -/// `openshell status` should report a healthy gateway when connected to a -/// configured gateway. -/// -/// This test verifies the normal gateway path: -/// - CLI resolves gateway metadata -/// - gRPC client connects -/// - Server responds to health check -#[tokio::test] -async fn gateway_status_reports_healthy() { - let (output, code) = run_cli(&["status"]).await; - let clean = strip_ansi(&output); - - assert_eq!( - code, 0, - "openshell status should exit 0 against gateway:\n{clean}" - ); - - // The status output should show the gateway as healthy/connected. - assert!( - clean.to_lowercase().contains("healthy") - || clean.to_lowercase().contains("running") - || clean.to_lowercase().contains("connected") - || clean.contains("✓"), - "status should report healthy gateway:\n{clean}" - ); -} - -// ------------------------------------------------------------------- -// Test 13: gRPC through the WS tunnel proxy (edge token path) -// ------------------------------------------------------------------- - -/// When a gateway's metadata has `auth_mode == "cloudflare_jwt"` and a -/// stored edge token, the CLI routes gRPC through the WebSocket tunnel proxy. -/// This test verifies the full tunnel path: -/// -/// CLI → local TCP proxy → WebSocket → /_ws_tunnel → loopback TCP → gRPC -/// -/// The test seeds a temporary config directory with edge auth metadata and a -/// dummy token, then runs `openshell status` against the live plaintext -/// gateway. -/// -/// Note: The dummy token won't be validated (no edge auth middleware on -/// the plaintext gateway), but it triggers the CLI's tunnel proxy codepath. -#[tokio::test] -async fn ws_tunnel_status_through_edge_proxy() { - let (original_status, _) = run_cli(&["status"]).await; - let clean_status = strip_ansi(&original_status); - - // Only run this test if we have a healthy gateway to test against. - if !clean_status.to_lowercase().contains("healthy") - && !clean_status.to_lowercase().contains("running") - && !clean_status.to_lowercase().contains("connected") - && !clean_status.contains("✓") - { - eprintln!("Skipping ws_tunnel test: no healthy gateway available"); - return; - } - - // Get the gateway endpoint from the gateway metadata. - let (info_output, info_code) = run_cli(&["gateway", "info"]).await; - assert_eq!(info_code, 0, "gateway info should succeed:\n{info_output}"); - - let info_clean = strip_ansi(&info_output); - - // Extract the gateway endpoint from the info output. - // The format varies, but it should contain a URL-like string. - let endpoint = info_clean.lines().find_map(|line| { - if line.to_lowercase().contains("endpoint") || line.to_lowercase().contains("gateway") { - // Try to extract a URL from the line - line.split_whitespace() - .find(|word| word.starts_with("http://") || word.starts_with("https://")) - .map(String::from) - } else { - None - } - }); - - let Some(endpoint) = endpoint else { - eprintln!( - "Skipping ws_tunnel test: could not extract gateway endpoint from:\n{info_clean}" - ); - return; - }; - - // For the WS tunnel test, we need the endpoint to be HTTP (plaintext). - // If it's HTTPS, the WS tunnel test requires TLS negotiation which - // complicates things. Skip if the gateway isn't plaintext. - if !endpoint.starts_with("http://") { - eprintln!( - "Skipping ws_tunnel test: gateway endpoint is not plaintext HTTP: {endpoint}\n\ - Use a plaintext local gateway such as `mise run gateway:docker` for this test." - ); - return; - } - - // Seed a temporary config directory with edge auth metadata pointing at - // the live gateway. The dummy token triggers the WS tunnel codepath - // without requiring real edge auth middleware. - let tmpdir = tempfile::tempdir().expect("create temp config dir"); - seed_edge_gateway_config( - tmpdir.path(), - "edge-tunnel-test", - &endpoint, - "dummy-test-jwt", - ); - - let (output, code) = - run_cli_with_config(tmpdir.path(), &["--gateway", "edge-tunnel-test", "status"]).await; - - let clean = strip_ansi(&output); - assert_eq!( - code, 0, - "openshell status through WS tunnel should exit 0:\n{clean}" - ); - assert!( - clean.to_lowercase().contains("healthy") - || clean.to_lowercase().contains("running") - || clean.to_lowercase().contains("connected") - || clean.contains("✓"), - "status through WS tunnel should report healthy:\n{clean}" - ); -} diff --git a/e2e/rust/tests/ephemeral_cleanup.rs b/e2e/rust/tests/ephemeral_cleanup.rs deleted file mode 100644 index eca4bb1d68..0000000000 --- a/e2e/rust/tests/ephemeral_cleanup.rs +++ /dev/null @@ -1,189 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Detached ephemeral lifecycle coverage shared by supervisor-based drivers. - -#![cfg(feature = "e2e")] - -use std::path::PathBuf; -use std::process::{Command, Stdio}; -use std::time::Duration; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::cli::run_cli; -use openshell_e2e::harness::container::{ContainerEngine, e2e_driver}; -use openshell_e2e::harness::sandbox::{E2E_WORKLOAD_IMAGE, unique_sandbox_name}; -use serial_test::serial; -use tokio::time::{Instant, sleep}; - -const CLEANUP_TIMEOUT: Duration = Duration::from_secs(90); - -struct DeleteOnFailure { - name: String, - armed: bool, -} - -impl Drop for DeleteOnFailure { - fn drop(&mut self) { - if self.armed { - let _ = Command::new(openshell_bin()) - .args(["sandbox", "delete", &self.name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } - } -} - -fn command_output(mut command: Command) -> Result { - let output = command - .output() - .map_err(|error| format!("run driver resource query: {error}"))?; - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - if !output.status.success() { - return Err(format!( - "driver resource query failed ({}): {stdout}{stderr}", - output.status - )); - } - Ok(stdout.trim().to_string()) -} - -fn driver_resources_present(sandbox_id: &str) -> Result { - match e2e_driver().as_deref() { - Some("docker" | "podman") => { - let engine = ContainerEngine::from_env()?; - let mut command = engine.command(); - command.args([ - "ps", - "-aq", - "--filter", - &format!("label=openshell.ai/sandbox-id={sandbox_id}"), - ]); - Ok(!command_output(command)?.is_empty()) - } - Some("kubernetes") => { - let mut command = Command::new("kubectl"); - command.args([ - "get", - "pods,sandboxes.agents.x-k8s.io", - "--all-namespaces", - "--selector", - &format!("openshell.ai/sandbox-id={sandbox_id}"), - "--output=name", - ]); - Ok(!command_output(command)?.is_empty()) - } - Some("vm") => { - let state_dir = std::env::var_os("OPENSHELL_E2E_VM_STATE_DIR") - .map(PathBuf::from) - .ok_or("OPENSHELL_E2E_VM_STATE_DIR must be set for VM resource checks")?; - Ok(state_dir.join("sandboxes").join(sandbox_id).exists()) - } - other => Err(format!( - "unsupported e2e driver for ephemeral cleanup: {other:?}" - )), - } -} - -async fn run_detached_ephemeral_cleanup(exit_code: i32) -> Result<(), String> { - let name = unique_sandbox_name(); - let mut cleanup = DeleteOnFailure { - name: name.clone(), - armed: true, - }; - let release_path = format!("/sandbox/.ephemeral-release-{name}"); - let script = - format!("while [ ! -e '{release_path}' ]; do sleep 0.1; done; sleep 1; exit {exit_code}"); - let mut create = openshell_cmd(); - create.args([ - "sandbox", - "create", - "--name", - &name, - "--from", - E2E_WORKLOAD_IMAGE, - "--no-keep", - "--detach", - "--", - "sh", - "-c", - &script, - ]); - let created = create - .output() - .await - .map_err(|error| format!("create detached ephemeral sandbox: {error}"))?; - if !created.status.success() { - return Err(format!( - "create detached ephemeral sandbox failed ({}): {}{}", - created.status, - String::from_utf8_lossy(&created.stdout), - String::from_utf8_lossy(&created.stderr) - )); - } - - let (details, get_code) = run_cli(&["sandbox", "get", &name, "--output", "json"]).await; - if get_code != 0 { - return Err(format!("get detached sandbox failed: {details}")); - } - let details: serde_json::Value = - serde_json::from_str(&details).map_err(|error| format!("parse sandbox JSON: {error}"))?; - let sandbox_id = details["id"] - .as_str() - .ok_or_else(|| format!("sandbox has no id: {details}"))?; - if !driver_resources_present(sandbox_id)? { - return Err(format!( - "driver resources were never observed for sandbox {name} ({sandbox_id})" - )); - } - - let (release, release_code) = run_cli(&[ - "sandbox", - "exec", - "--name", - &name, - "--no-tty", - "--no-login-shell", - "--", - "touch", - &release_path, - ]) - .await; - if release_code != 0 { - return Err(format!("release canonical process failed: {release}")); - } - - let deadline = Instant::now() + CLEANUP_TIMEOUT; - loop { - let (names, list_code) = run_cli(&["sandbox", "list", "--names"]).await; - if list_code != 0 { - return Err(format!("list sandboxes failed: {names}")); - } - let record_present = names.lines().any(|line| line.trim() == name); - let resources_present = driver_resources_present(sandbox_id)?; - if !record_present && !resources_present { - cleanup.armed = false; - return Ok(()); - } - if Instant::now() >= deadline { - return Err(format!( - "ephemeral sandbox {name} ({sandbox_id}) remained after {CLEANUP_TIMEOUT:?}: record_present={record_present}, resources_present={resources_present}" - )); - } - sleep(Duration::from_millis(500)).await; - } -} - -#[tokio::test] -#[serial(ephemeral_cleanup)] -async fn detached_ephemeral_success_removes_sandbox_and_driver_resources() { - run_detached_ephemeral_cleanup(0).await.unwrap(); -} - -#[tokio::test] -#[serial(ephemeral_cleanup)] -async fn detached_ephemeral_failure_removes_sandbox_and_driver_resources() { - run_detached_ephemeral_cleanup(17).await.unwrap(); -} diff --git a/e2e/rust/tests/internet_network_perf.rs b/e2e/rust/tests/internet_network_perf.rs deleted file mode 100644 index 70c0f1af25..0000000000 --- a/e2e/rust/tests/internet_network_perf.rs +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Opt-in Internet benchmark for the complete sandbox-to-supervisor data path. - -#![cfg(feature = "e2e-host-gateway")] - -use std::io::Write as _; -use std::process::Stdio; - -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const BENCHMARK: &str = r#" -import http.client -import json -import os -import socket -import ssl -import statistics -import time - -HOST = "example.com" -ITERATIONS = int(os.environ.get("OPENSHELL_INET_PERF_ITERATIONS", "20")) - -def percentile(values, fraction): - values = sorted(values) - return values[min(len(values) - 1, int((len(values) - 1) * fraction))] - -def measure(name, operation, iterations=ITERATIONS): - operation() - samples = [] - started = time.perf_counter_ns() - for _ in range(iterations): - before = time.perf_counter_ns() - operation() - samples.append((time.perf_counter_ns() - before) / 1_000_000) - elapsed = (time.perf_counter_ns() - started) / 1_000_000_000 - return { - "name": name, - "iterations": iterations, - "mean_ms": statistics.fmean(samples), - "p50_ms": percentile(samples, 0.50), - "p95_ms": percentile(samples, 0.95), - "ops_per_second": iterations / elapsed, - } - -def dns_lookup(): - result = socket.getaddrinfo(HOST, 443, socket.AF_INET, socket.SOCK_STREAM) - if not result: - raise RuntimeError("DNS returned no IPv4 addresses") - -def tcp_connect(): - with socket.create_connection((HOST, 443), timeout=10): - pass - -tls_context = ssl.create_default_context() - -def https_cold(): - connection = http.client.HTTPSConnection(HOST, 443, timeout=10, context=tls_context) - try: - connection.request("HEAD", "/", headers={"Connection": "close"}) - response = connection.getresponse() - response.read() - if response.status != 200: - raise RuntimeError(f"unexpected HTTP status {response.status}") - finally: - connection.close() - -warm_connection = http.client.HTTPSConnection(HOST, 443, timeout=10, context=tls_context) - -def https_reuse(): - warm_connection.request("HEAD", "/", headers={"Connection": "keep-alive"}) - response = warm_connection.getresponse() - response.read() - if response.status != 200: - raise RuntimeError(f"unexpected HTTP status {response.status}") - -try: - metrics = [ - measure("dns_lookup", dns_lookup), - measure("tcp_connect", tcp_connect), - measure("https_cold", https_cold, max(5, ITERATIONS // 2)), - measure("https_reuse", https_reuse), - ] -finally: - warm_connection.close() - -print(json.dumps({"metrics": metrics}, separators=(",", ":"))) -"#; - -fn write_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - write!( - file, - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -network_policies: - internet_performance: - name: internet_performance - endpoints: - - host: example.com - port: 80 - protocol: tcp - - host: example.com - port: 443 - protocol: tcp - binaries: - - path: "/**" -"# - ) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -async fn run_host_benchmark() -> Result { - let output = tokio::process::Command::new("python3") - .args(["-c", BENCHMARK]) - .env("OPENSHELL_INET_PERF_ITERATIONS", "20") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await - .map_err(|error| format!("run host benchmark: {error}"))?; - let combined = format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - if !output.status.success() { - return Err(format!("host benchmark failed: {combined}")); - } - Ok(combined) -} - -#[tokio::test] -#[ignore = "manual Internet performance benchmark"] -async fn benchmark_complete_internet_path() { - let policy = write_policy().expect("write Internet benchmark policy"); - let policy_path = policy.path().to_str().expect("UTF-8 policy path"); - let sandbox = SandboxGuard::create(&["--policy", policy_path]) - .await - .expect("create benchmark sandbox"); - - for round in 1..=3 { - let host = run_host_benchmark().await.expect("host benchmark"); - println!("INTERNET_PERF host round={round} {}", host.trim()); - let mediated = sandbox - .exec(&[ - "sh", - "-c", - "OPENSHELL_INET_PERF_ITERATIONS=20 python3 -c \"$1\"", - "openshell-internet-perf", - BENCHMARK, - ]) - .await - .expect("sandbox benchmark"); - println!("INTERNET_PERF sandbox round={round} {}", mediated.trim()); - } -} diff --git a/e2e/rust/tests/landlock.rs b/e2e/rust/tests/landlock.rs deleted file mode 100644 index cfbdcd5846..0000000000 --- a/e2e/rust/tests/landlock.rs +++ /dev/null @@ -1,150 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! End-to-end coverage for Landlock filesystem enforcement. - -use std::io::Write; -use std::time::Duration; - -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const SUCCESS_MARKER: &str = "landlock-hard-requirement-ok"; - -fn write_hard_requirement_policy() -> Result { - write_policy("hard_requirement", "/sandbox") -} - -fn write_policy(compatibility: &str, extra_path: &str) -> Result { - let mut file = - NamedTempFile::new().map_err(|error| format!("create temporary policy: {error}"))?; - let policy = r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /etc - - /proc - read_write: - - /sandbox - - /tmp - -landlock: - compatibility: hard_requirement - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - landlock_regression: - name: landlock_regression - endpoints: - - host: example.com - port: 443 - binaries: - - path: "/**" -"#; - - let policy = policy - .replace( - "compatibility: hard_requirement", - &format!("compatibility: {compatibility}"), - ) - .replace( - " - /sandbox\n", - &format!(" - /sandbox\n - {extra_path}\n"), - ); - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write temporary policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush temporary policy: {error}"))?; - Ok(file) -} - -#[tokio::test] -async fn hard_requirement_accepts_enriched_device_path() { - let policy = write_hard_requirement_policy().expect("write hard-requirement policy"); - let policy_path = policy.path().to_string_lossy().into_owned(); - let script = concat!( - "set -eu; ", - "bytes=$(head -c 16 /dev/urandom | wc -c); ", - "test \"$bytes\" -eq 16; ", - "printf landlock-ok > /tmp/landlock-check; ", - "test \"$(cat /tmp/landlock-check)\" = landlock-ok; ", - "echo landlock-hard-requirement-ok", - ); - - let mut sandbox = SandboxGuard::create(&["--policy", &policy_path, "--", "sh", "-lc", script]) - .await - .expect("hard_requirement should accept the enriched /dev/urandom path"); - - assert!(sandbox.create_output.contains(SUCCESS_MARKER)); - sandbox.cleanup().await; -} - -async fn check_policy_path_replacement(compatibility: &str, replace_ancestor: bool) { - let policy = write_policy(compatibility, "/sandbox/policy-parent/leaf") - .expect("write filesystem policy"); - let policy_path = policy.path().to_string_lossy().into_owned(); - let mut sandbox = SandboxGuard::create(&["--policy", &policy_path]) - .await - .expect("create sandbox with filesystem policy"); - - sandbox - .exec(&["sh", "-c", "mkdir -p /sandbox/policy-parent/leaf"]) - .await - .expect("create declared directory under writable workspace"); - sandbox - .exec(&[ - "sh", - "-c", - "echo allowed > /sandbox/policy-parent/leaf/check", - ]) - .await - .expect("normal directory should remain usable on the next exec"); - - let script = if replace_ancestor { - "mv /sandbox/policy-parent /sandbox/original-parent; ln -s /var/tmp /sandbox/policy-parent" - } else { - "mv /sandbox/policy-parent/leaf /sandbox/original-leaf; ln -s /var/tmp /sandbox/policy-parent/leaf" - }; - sandbox - .exec(&["sh", "-c", script]) - .await - .expect("replace workspace directory"); - let result = tokio::time::timeout(Duration::from_secs(20), sandbox.exec(&["true"])).await; - // Delete through the control plane even when further workload launches fail. - sandbox.cleanup().await; - let error = result - .expect("rejected launch must close its SSH channel promptly") - .expect_err("mutable policy symlink must prevent the next launch"); - assert!( - error.to_lowercase().contains("symlink"), - "unexpected launch error: {error}" - ); -} - -#[tokio::test] -async fn best_effort_rejects_replaced_policy_path() { - check_policy_path_replacement("best_effort", false).await; -} - -#[tokio::test] -async fn hard_requirement_rejects_replaced_policy_path() { - check_policy_path_replacement("hard_requirement", false).await; -} - -#[tokio::test] -async fn best_effort_rejects_replaced_policy_ancestor() { - check_policy_path_replacement("best_effort", true).await; -} - -#[tokio::test] -async fn hard_requirement_rejects_replaced_policy_ancestor() { - check_policy_path_replacement("hard_requirement", true).await; -} diff --git a/e2e/rust/tests/live_internet_traffic_perf.rs b/e2e/rust/tests/live_internet_traffic_perf.rs deleted file mode 100644 index e8727bbce3..0000000000 --- a/e2e/rust/tests/live_internet_traffic_perf.rs +++ /dev/null @@ -1,337 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Opt-in live-Internet benchmark shaped like a short coding-agent session. - -#![cfg(feature = "e2e-host-gateway")] - -use std::io::Write as _; -use std::process::Stdio; -use std::time::Instant; - -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const DEFAULT_ROUNDS: usize = 10; - -const BENCHMARK: &str = r#" -import concurrent.futures -import http.client -import json -import os -import shutil -import socket -import ssl -import statistics -import subprocess -import tempfile -import time -import urllib.parse - -MODE = os.environ.get("OPENSHELL_LIVE_PERF_MODE", "direct") -TLS = ssl.create_default_context() -USER_AGENT = "OpenShell-live-Internet-benchmark/1" - -METADATA_URLS = [ - "https://raw.githubusercontent.com/NVIDIA/OpenShell/main/README.md", - "https://pypi.org/pypi/requests/json", - "https://registry.npmjs.org/typescript/latest", - "https://docs.python.org/3/", -] -DNS_HOSTS = [ - "github.com", - "raw.githubusercontent.com", - "pypi.org", - "registry.npmjs.org", - "crates.io", - "docs.python.org", - "speed.cloudflare.com", -] - -def percentile(values, fraction): - values = sorted(values) - return values[min(len(values) - 1, int((len(values) - 1) * fraction))] - -def request(url, method="GET", max_bytes=None): - parsed = urllib.parse.urlsplit(url) - connection = http.client.HTTPSConnection(parsed.hostname, parsed.port or 443, timeout=20, context=TLS) - path = parsed.path or "/" - if parsed.query: - path += "?" + parsed.query - started = time.perf_counter_ns() - connection.request(method, path, headers={"User-Agent": USER_AGENT, "Connection": "close"}) - response = connection.getresponse() - first = response.read(1) - first_byte_ms = (time.perf_counter_ns() - started) / 1_000_000 - body_bytes = len(first) - while max_bytes is None or body_bytes < max_bytes: - remaining = None if max_bytes is None else max_bytes - body_bytes - chunk = response.read(65536 if remaining is None else min(65536, remaining)) - if not chunk: - break - body_bytes += len(chunk) - total_ms = (time.perf_counter_ns() - started) / 1_000_000 - status = response.status - connection.close() - if status < 200 or status >= 400: - raise RuntimeError(f"{url} returned HTTP {status}") - return {"status": status, "bytes": body_bytes, "first_byte_ms": first_byte_ms, "total_ms": total_ms} - -def metric(name, operation): - started = time.perf_counter_ns() - try: - detail = operation() - return { - "name": name, - "ok": True, - "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, - "detail": detail, - } - except Exception as error: - return { - "name": name, - "ok": False, - "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, - "error": f"{type(error).__name__}: {error}", - } - -def dns_set(): - samples = [] - for host in DNS_HOSTS: - started = time.perf_counter_ns() - addresses = socket.getaddrinfo(host, 443, socket.AF_UNSPEC, socket.SOCK_STREAM) - samples.append({ - "host": host, - "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, - "addresses": len(addresses), - }) - timings = [sample["elapsed_ms"] for sample in samples] - return { - "lookups": samples, - "p50_ms": percentile(timings, 0.50), - "p95_ms": percentile(timings, 0.95), - } - -def metadata_serial(): - return {"requests": [request(url, max_bytes=2_000_000) for url in METADATA_URLS]} - -def metadata_concurrent(): - urls = METADATA_URLS + METADATA_URLS - with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: - responses = list(pool.map(lambda url: request(url, max_bytes=2_000_000), urls)) - return {"requests": responses} - -def https_reuse(): - connection = http.client.HTTPSConnection("example.com", 443, timeout=20, context=TLS) - samples = [] - try: - for _ in range(20): - started = time.perf_counter_ns() - connection.request("HEAD", "/", headers={"User-Agent": USER_AGENT, "Connection": "keep-alive"}) - response = connection.getresponse() - response.read() - if response.status != 200: - raise RuntimeError(f"example.com returned HTTP {response.status}") - samples.append((time.perf_counter_ns() - started) / 1_000_000) - finally: - connection.close() - return { - "requests": len(samples), - "p50_ms": percentile(samples, 0.50), - "p95_ms": percentile(samples, 0.95), - "mean_ms": statistics.fmean(samples), - } - -def git_clone(): - if shutil.which("git") is None: - return {"skipped": "git is not installed in the workload image"} - with tempfile.TemporaryDirectory(prefix="openshell-live-git-") as directory: - checkout = os.path.join(directory, "sampleproject") - command = [ - "git", "-c", "advice.detachedHead=false", "clone", "--quiet", - "--depth", "1", "--filter=blob:none", - "https://github.com/pypa/sampleproject.git", checkout, - ] - output = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, check=False) - if output.returncode != 0: - raise RuntimeError(output.stderr.decode(errors="replace")[-500:]) - files = sum(len(names) for _, _, names in os.walk(checkout)) - size = sum(os.path.getsize(os.path.join(root, name)) for root, _, names in os.walk(checkout) for name in names) - return {"files": files, "checkout_bytes": size} - -def package_download(): - metadata = request("https://pypi.org/pypi/idna/json", max_bytes=2_000_000) - connection = http.client.HTTPSConnection("pypi.org", 443, timeout=20, context=TLS) - connection.request("GET", "/pypi/idna/json", headers={"User-Agent": USER_AGENT}) - response = connection.getresponse() - document = json.loads(response.read()) - connection.close() - wheels = [entry for entry in document["urls"] if entry["packagetype"] == "bdist_wheel"] - if not wheels: - raise RuntimeError("PyPI returned no idna wheel") - artifact = request(wheels[0]["url"], max_bytes=5_000_000) - return {"metadata": metadata, "artifact": artifact, "filename": wheels[0]["filename"]} - -def bulk_download(): - return request("https://speed.cloudflare.com/__down?bytes=5242880", max_bytes=5242880) - -def denied_destination(): - if MODE != "sandbox": - return {"skipped": "policy denial applies only to sandbox mode"} - started = time.perf_counter_ns() - try: - socket.create_connection(("example.net", 443), timeout=5).close() - except OSError as error: - return { - "denied": True, - "latency_ms": (time.perf_counter_ns() - started) / 1_000_000, - "error": str(error), - } - raise RuntimeError("destination omitted from policy was reachable") - -started = time.perf_counter_ns() -metrics = [ - metric("dns_service_set", dns_set), - metric("https_metadata_serial", metadata_serial), - metric("https_metadata_concurrent", metadata_concurrent), - metric("https_reuse", https_reuse), - metric("git_clone", git_clone), - metric("package_download", package_download), - metric("bulk_download_5mib", bulk_download), - metric("policy_denial", denied_destination), -] -document = { - "schema": "openshell.live-internet-perf.v1", - "mode": MODE, - "total_ms": (time.perf_counter_ns() - started) / 1_000_000, - "metrics": metrics, -} -print(json.dumps(document, separators=(",", ":"))) -if any(item["name"] == "policy_denial" and not item["ok"] for item in metrics): - raise SystemExit(1) -"#; - -fn rounds() -> usize { - std::env::var("OPENSHELL_LIVE_PERF_ROUNDS") - .ok() - .and_then(|value| value.parse().ok()) - .filter(|value| *value > 0) - .unwrap_or(DEFAULT_ROUNDS) -} - -fn write_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - write!( - file, - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -network_policies: - live_internet_performance: - name: live_internet_performance - endpoints: - - {{ host: example.com, port: 443, protocol: tcp }} - - {{ host: github.com, port: 443, protocol: tcp }} - - {{ host: raw.githubusercontent.com, port: 443, protocol: tcp }} - - {{ host: pypi.org, port: 443, protocol: tcp }} - - {{ host: files.pythonhosted.org, port: 443, protocol: tcp }} - - {{ host: registry.npmjs.org, port: 443, protocol: tcp }} - - {{ host: crates.io, port: 443, protocol: tcp }} - - {{ host: docs.python.org, port: 443, protocol: tcp }} - - {{ host: speed.cloudflare.com, port: 443, protocol: tcp }} - binaries: - - path: "/**" -"# - ) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -async fn run_direct() -> Result { - run_python("direct").await -} - -async fn run_python(mode: &str) -> Result { - let output = tokio::process::Command::new("python3") - .args(["-c", BENCHMARK]) - .env("OPENSHELL_LIVE_PERF_MODE", mode) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await - .map_err(|error| format!("run {mode} live Internet benchmark: {error}"))?; - let combined = format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - if !output.status.success() { - return Err(format!( - "{mode} live Internet benchmark failed (exit {:?}): {combined}", - output.status.code() - )); - } - Ok(combined) -} - -async fn run_sandbox(sandbox: &SandboxGuard) -> Result { - sandbox - .exec(&[ - "sh", - "-c", - "OPENSHELL_LIVE_PERF_MODE=sandbox python3 -c \"$1\"", - "openshell-live-internet-perf", - BENCHMARK, - ]) - .await -} - -#[tokio::test] -#[ignore = "manual live-Internet performance benchmark"] -async fn benchmark_live_internet_agent_traffic() { - let policy = write_policy().expect("write live Internet benchmark policy"); - let policy_path = policy.path().to_str().expect("UTF-8 policy path"); - let create_started = Instant::now(); - let mut sandbox = SandboxGuard::create(&["--policy", policy_path]) - .await - .expect("create live Internet benchmark sandbox"); - println!( - "LIVE_INTERNET_PERF create {{\"elapsed_ms\":{}}}", - create_started.elapsed().as_secs_f64() * 1000.0 - ); - - for round in 1..=rounds() { - if round % 2 == 1 { - let direct = run_direct().await.expect("direct live Internet benchmark"); - println!("LIVE_INTERNET_PERF direct round={round} {}", direct.trim()); - let mediated = run_sandbox(&sandbox) - .await - .expect("sandbox live Internet benchmark"); - println!( - "LIVE_INTERNET_PERF sandbox round={round} {}", - mediated.trim() - ); - } else { - let mediated = run_sandbox(&sandbox) - .await - .expect("sandbox live Internet benchmark"); - println!( - "LIVE_INTERNET_PERF sandbox round={round} {}", - mediated.trim() - ); - let direct = run_direct().await.expect("direct live Internet benchmark"); - println!("LIVE_INTERNET_PERF direct round={round} {}", direct.trim()); - } - } - - sandbox.cleanup().await; -} diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs deleted file mode 100644 index 6b137d1a2b..0000000000 --- a/e2e/rust/tests/podman_oci_identity.rs +++ /dev/null @@ -1,213 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-podman")] - -//! Podman-specific E2E coverage for the supervisor companion container. -//! -//! Podman runs the supervisor as the resolved OCI workload identity with no -//! effective capabilities; Docker runs it as a fixed unprivileged user. The -//! test builds an image through the selected Podman engine, creates a sandbox -//! from it, and inspects the real supervisor container. The workload's image -//! ID, identity, network, and mounts, and what the main process and -//! `sandbox exec` see, are shared with Docker and covered by the `oci-image` -//! feature suite in `tests/suites/features`. - -use std::process::Stdio; - -use openshell_e2e::harness::container::{ContainerEngine, is_e2e_driver}; -use openshell_e2e::harness::sandbox::SandboxGuard; - -const BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; -const READY_MARKER: &str = "podman-oci-identity-ready"; -const OCI_UID: &str = "2345"; -const OCI_GID: &str = "2346"; -const OCI_FALLBACK_POLICY: &str = r"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /etc] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort - -network_policies: {} -"; - -struct ImageGuard { - engine: ContainerEngine, - tag: String, -} - -impl ImageGuard { - fn build() -> Result { - let engine = ContainerEngine::from_env()?; - if engine.name() != "podman" { - return Err(format!( - "Podman OCI identity E2E requires podman, got {}", - engine.name() - )); - } - - let context = tempfile::tempdir().map_err(|err| format!("create build context: {err}"))?; - let containerfile = context.path().join("Containerfile"); - std::fs::write( - &containerfile, - format!( - r"FROM {BASE_IMAGE} -USER 0:0 -RUN mkdir -p /home/app/project && \ - chown {OCI_UID}:{OCI_GID} /home/app /home/app/project && \ - chmod 0700 /home/app /home/app/project -WORKDIR /home/app/project -USER {OCI_UID}:{OCI_GID} -" - ), - ) - .map_err(|err| format!("write Containerfile: {err}"))?; - - let tag = format!( - "localhost/openshell-e2e-podman-oci-identity:{}", - std::process::id() - ); - run_engine( - &engine, - &[ - "build", - "--pull=never", - "--file", - containerfile - .to_str() - .ok_or_else(|| "Containerfile path is not UTF-8".to_string())?, - "--tag", - &tag, - context - .path() - .to_str() - .ok_or_else(|| "build context path is not UTF-8".to_string())?, - ], - )?; - let user = run_engine( - &engine, - &["image", "inspect", "--format", "{{.Config.User}}", &tag], - )?; - if user != format!("{OCI_UID}:{OCI_GID}") { - return Err(format!( - "Podman-built image has OCI user '{user}', expected {OCI_UID}:{OCI_GID}" - )); - } - - Ok(Self { engine, tag }) - } -} - -impl Drop for ImageGuard { - fn drop(&mut self) { - let _ = self - .engine - .command() - .args(["image", "rm", "--force", &self.tag]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -fn run_engine(engine: &ContainerEngine, args: &[&str]) -> Result { - let output = engine - .command() - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .map_err(|err| format!("failed to run {} {}: {err}", engine.name(), args.join(" ")))?; - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - if !output.status.success() { - return Err(format!( - "{} {} failed (exit {:?}):\n{stdout}{stderr}", - engine.name(), - args.join(" "), - output.status.code() - )); - } - Ok(stdout.trim().to_string()) -} - -fn container_id_for_role( - engine: &ContainerEngine, - sandbox_name: &str, - role: &str, -) -> Result { - let name_filter = format!("label=openshell.ai/sandbox-name={sandbox_name}"); - let role_filter = format!("label=openshell.ai/isolation-role={role}"); - let stdout = run_engine( - engine, - &[ - "ps", - "-aq", - "--filter", - "label=openshell.managed=true", - "--filter", - &name_filter, - "--filter", - &role_filter, - ], - )?; - let ids = stdout - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .collect::>(); - match ids.as_slice() { - [id] => Ok((*id).to_string()), - [] => Err(format!( - "no Podman container found for sandbox '{sandbox_name}'" - )), - _ => Err(format!( - "multiple Podman containers found for sandbox '{sandbox_name}': {ids:?}" - )), - } -} - -#[tokio::test] -async fn podman_supervisor_runs_as_oci_identity_without_capabilities() { - if !is_e2e_driver("podman") { - eprintln!("Skipping Podman OCI identity test: e2e driver is not podman"); - return; - } - - let image = ImageGuard::build().expect("build Podman OCI identity image"); - // The fixture image contains a policy with an - // explicit `sandbox` process identity. Supply a complete policy that - // intentionally omits `process` so this test exercises OCI fallback. - let policy = tempfile::NamedTempFile::new().expect("create OCI fallback policy"); - std::fs::write(policy.path(), OCI_FALLBACK_POLICY).expect("write OCI fallback policy"); - let policy_path = policy.path().to_str().expect("policy path is UTF-8"); - let mut sandbox = SandboxGuard::create_keep_with_args( - &["--from", &image.tag, "--policy", policy_path, "--no-tty"], - &["sh", "-c", "echo podman-oci-identity-ready; sleep infinity"], - READY_MARKER, - ) - .await - .expect("create sandbox from Podman-built OCI identity image"); - - let supervisor_id = container_id_for_role(&image.engine, &sandbox.name, "supervisor") - .expect("find separate supervisor companion"); - let supervisor_user = run_engine( - &image.engine, - &["inspect", "--format", "{{.Config.User}}", &supervisor_id], - ) - .unwrap(); - assert_eq!(supervisor_user, format!("{OCI_UID}:{OCI_GID}")); - let supervisor_caps = run_engine( - &image.engine, - &["inspect", "--format", "{{.EffectiveCaps}}", &supervisor_id], - ) - .unwrap(); - assert_eq!( - supervisor_caps, "[]", - "the supervisor companion may not have effective capabilities" - ); - sandbox.cleanup().await; -} diff --git a/e2e/rust/tests/podman_resource_limits.rs b/e2e/rust/tests/podman_resource_limits.rs index a0282f0e09..225f77ff8f 100644 --- a/e2e/rust/tests/podman_resource_limits.rs +++ b/e2e/rust/tests/podman_resource_limits.rs @@ -7,7 +7,7 @@ //! resource limits are actually enforced by cgroups inside the workload, not //! just echoed back by the API/template layer. //! -//! `e2e/rust/tests/sandbox_templates.rs` already verifies that +//! The conformance `templates` suite already verifies that //! `sandbox template create --cpu ... --memory ...` is stored and returned //! correctly, but it never inspects the resulting container's real resource //! state. This test creates a sandbox with those flags directly and reads the diff --git a/e2e/rust/tests/sandbox_labels.rs b/e2e/rust/tests/sandbox_labels.rs deleted file mode 100644 index 937171ead3..0000000000 --- a/e2e/rust/tests/sandbox_labels.rs +++ /dev/null @@ -1,230 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use std::process::Stdio; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::{extract_field, strip_ansi}; -use openshell_e2e::harness::sandbox::SandboxGuard; - -fn normalize_output(output: &str) -> String { - let stripped = strip_ansi(output).replace('\r', ""); - let mut cleaned = String::with_capacity(stripped.len()); - - for ch in stripped.chars() { - match ch { - '\u{8}' => { - cleaned.pop(); - } - '\u{4}' => {} - _ => cleaned.push(ch), - } - } - - cleaned -} - -fn extract_sandbox_name(output: &str) -> Option { - if let Some((_, rest)) = output.split_once("Created sandbox:") { - return rest.split_whitespace().next().map(ToOwned::to_owned); - } - - extract_field(output, "Created sandbox").or_else(|| extract_field(output, "Name")) -} - -async fn create_sandbox_with_labels(name: &str, labels: &[(&str, &str)]) -> String { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "create", "--detach", "--name", name]); - - for (key, value) in labels { - cmd.arg("--label").arg(format!("{key}={value}")); - } - - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox create"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - assert!( - output.status.success(), - "sandbox create should succeed (exit {:?}):\n{combined}", - output.status.code() - ); - - extract_sandbox_name(&combined).expect("sandbox name should be present in output") -} - -async fn list_sandboxes_with_selector(selector: &str) -> Vec { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "list", "--names"]); - - if !selector.is_empty() { - cmd.arg("--selector").arg(selector); - } - - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox list"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - assert!( - output.status.success(), - "sandbox list should succeed (exit {:?}):\n{combined}", - output.status.code() - ); - - combined - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .map(ToOwned::to_owned) - .collect() -} - -async fn get_sandbox_details(name: &str) -> String { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "get", name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox get"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - assert!( - output.status.success(), - "sandbox get should succeed (exit {:?}):\n{combined}", - output.status.code() - ); - - combined -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] // end-to-end test exercises full label lifecycle -async fn sandbox_labels_are_stored_and_filterable() { - // Keep the unique names below the public 19-character routable-name - // limit. Hex keeps the per-process suffix compact on busy CI hosts. - let suffix = format!("{:x}", std::process::id()); - let dev_backend = format!("lbl-db-{suffix}"); - let staging_backend = format!("lbl-sb-{suffix}"); - let prod_frontend = format!("lbl-pf-{suffix}"); - let dev_data = format!("lbl-dd-{suffix}"); - - // Arm guards before create so a partial create still cleans up. - let _cleanup: Vec<_> = [&dev_backend, &staging_backend, &prod_frontend, &dev_data] - .into_iter() - .map(|name| SandboxGuard::manage_existing(name.clone())) - .collect(); - - // Create sandboxes with different labels - let name1 = - create_sandbox_with_labels(&dev_backend, &[("env", "dev"), ("team", "backend")]).await; - - let name2 = - create_sandbox_with_labels(&staging_backend, &[("env", "staging"), ("team", "backend")]) - .await; - - let name3 = - create_sandbox_with_labels(&prod_frontend, &[("env", "prod"), ("team", "frontend")]).await; - - let name4 = create_sandbox_with_labels(&dev_data, &[("env", "dev"), ("team", "data")]).await; - - // Test 1: Verify labels are stored in sandbox metadata - let details = get_sandbox_details(&name1).await; - assert!( - details.contains("env: dev"), - "sandbox should have env=dev label in details:\n{details}" - ); - assert!( - details.contains("team: backend"), - "sandbox should have team=backend label in details:\n{details}" - ); - - // Test 2: Filter by single label (env=dev) - let dev_sandboxes = list_sandboxes_with_selector("env=dev").await; - assert!( - dev_sandboxes.contains(&name1), - "env=dev filter should include {name1}, got: {dev_sandboxes:?}" - ); - assert!( - dev_sandboxes.contains(&name4), - "env=dev filter should include {name4}, got: {dev_sandboxes:?}" - ); - assert!( - !dev_sandboxes.contains(&name2), - "env=dev filter should not include staging sandbox {name2}, got: {dev_sandboxes:?}" - ); - assert!( - !dev_sandboxes.contains(&name3), - "env=dev filter should not include prod sandbox {name3}, got: {dev_sandboxes:?}" - ); - - // Test 3: Filter by single label (team=backend) - let backend_sandboxes = list_sandboxes_with_selector("team=backend").await; - assert!( - backend_sandboxes.contains(&name1), - "team=backend filter should include {name1}, got: {backend_sandboxes:?}" - ); - assert!( - backend_sandboxes.contains(&name2), - "team=backend filter should include {name2}, got: {backend_sandboxes:?}" - ); - assert!( - !backend_sandboxes.contains(&name3), - "team=backend filter should not include frontend sandbox {name3}, got: {backend_sandboxes:?}" - ); - assert!( - !backend_sandboxes.contains(&name4), - "team=backend filter should not include data sandbox {name4}, got: {backend_sandboxes:?}" - ); - - // Test 4: Filter by multiple labels (AND logic: env=dev,team=backend) - let dev_backend_sandboxes = list_sandboxes_with_selector("env=dev,team=backend").await; - assert_eq!( - dev_backend_sandboxes - .iter() - .filter(|name| [&name1, &name2, &name3, &name4].contains(name)) - .count(), - 1, - "env=dev,team=backend filter should return exactly 1 sandbox, got: {dev_backend_sandboxes:?}" - ); - assert!( - dev_backend_sandboxes.contains(&name1), - "env=dev,team=backend filter should include {name1}, got: {dev_backend_sandboxes:?}" - ); - - // Test 5: Filter by non-existent label value - let qa_sandboxes = list_sandboxes_with_selector("team=qa").await; - assert!( - !qa_sandboxes.contains(&name1) - && !qa_sandboxes.contains(&name2) - && !qa_sandboxes.contains(&name3) - && !qa_sandboxes.contains(&name4), - "team=qa filter should not return any of our test sandboxes, got: {qa_sandboxes:?}" - ); - - // Test 6: List all sandboxes (no filter) - let all_sandboxes = list_sandboxes_with_selector("").await; - assert!( - all_sandboxes.contains(&name1), - "list without filter should include all test sandboxes" - ); - assert!( - all_sandboxes.contains(&name2), - "list without filter should include all test sandboxes" - ); - assert!( - all_sandboxes.contains(&name3), - "list without filter should include all test sandboxes" - ); - assert!( - all_sandboxes.contains(&name4), - "list without filter should include all test sandboxes" - ); -} diff --git a/e2e/rust/tests/sandbox_lifecycle.rs b/e2e/rust/tests/sandbox_lifecycle.rs index b299a5d1d7..2497288042 100644 --- a/e2e/rust/tests/sandbox_lifecycle.rs +++ b/e2e/rust/tests/sandbox_lifecycle.rs @@ -9,18 +9,14 @@ use std::process::Stdio; use std::time::Duration; use openshell_e2e::harness::binary::{openshell_cmd, openshell_tty_cmd}; -use openshell_e2e::harness::cli::{ - run_cli, wait_for_sandbox_exec_contains, wait_for_sandbox_phase, -}; use openshell_e2e::harness::output::{extract_field, strip_ansi}; -use openshell_e2e::harness::sandbox::{E2E_WORKLOAD_IMAGE, SandboxGuard, unique_sandbox_name}; +use openshell_e2e::harness::sandbox::{SandboxGuard, unique_sandbox_name}; use serial_test::serial; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::time::{Instant, sleep}; const SANDBOX_PRESENCE_TIMEOUT: Duration = Duration::from_secs(30); const SANDBOX_LIST_POLL_INTERVAL: Duration = Duration::from_millis(500); -const SANDBOX_RESTART_TIMEOUT: Duration = Duration::from_secs(60); fn normalize_output(output: &str) -> String { let stripped = strip_ansi(output).replace('\r', ""); @@ -114,28 +110,6 @@ async fn delete_sandbox(name: &str) { let _ = cmd.status().await; } -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_exec_outlives_startup_recovery_deadline() { - let mut sandbox = SandboxGuard::create(&[]) - .await - .expect("create sandbox for exec recovery deadline"); - let output = tokio::time::timeout( - Duration::from_secs(60), - sandbox.exec(&["sh", "-c", "sleep 31; printf past-recovery-deadline"]), - ) - .await; - let next = sandbox.exec(&["printf", "fresh-exec"]).await; - sandbox.cleanup().await; - - assert_eq!( - output.expect("exec timed out").expect("exec failed"), - "past-recovery-deadline", - "expiration must not terminate an admitted command" - ); - assert_eq!(next.expect("fresh exec after expiration"), "fresh-exec"); -} - #[tokio::test] #[serial(sandbox_lifecycle)] async fn sandbox_exec_large_output_is_complete() { @@ -387,39 +361,6 @@ async fn piped_exec_stdin_crosses_grpc_message_limit() { sandbox.cleanup().await; } -async fn run_sandbox_lifecycle_command(operation: &str, name: &str) -> String { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", operation, name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .unwrap_or_else(|error| panic!("spawn openshell sandbox {operation}: {error}")); - let combined = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - assert!( - output.status.success(), - "sandbox {operation} should succeed (exit {:?}):\n{combined}", - output.status.code(), - ); - combined -} - -async fn sandbox_details(name: &str) -> String { - let (details, exit_code) = run_cli(&["sandbox", "get", name]).await; - let details = normalize_output(&details); - assert_eq!( - exit_code, 0, - "sandbox get should succeed for {name}:\n{details}" - ); - details -} - async fn reconnect_with_input_ownership( sandbox_name: &str, ) -> (tokio::process::Child, Vec) { @@ -547,125 +488,6 @@ async fn wait_for_process_with_args(expected_args: &[&str]) -> u32 { .unwrap_or_else(|_| panic!("process with arguments {expected_args:?} did not start")) } -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_stop_start_preserves_workspace() { - const SENTINEL: &str = "openshell-stop-start-sentinel"; - const SENTINEL_PATH: &str = "/sandbox/.openshell-stop-start-e2e"; - const RUN_COUNT_PATH: &str = "/sandbox/.openshell-main-run-count"; - let write_sentinel = format!("printf '%s\\n' '{SENTINEL}' > '{SENTINEL_PATH}'"); - let main = format!( - "count=0; test ! -f '{RUN_COUNT_PATH}' || count=$(cat '{RUN_COUNT_PATH}'); \ - count=$((count + 1)); printf '%s\\n' \"$count\" > '{RUN_COUNT_PATH}'; \ - echo lifecycle-ready-$count; exec sleep infinity" - ); - - let mut sandbox = SandboxGuard::create_keep(&["sh", "-c", &main], "lifecycle-ready") - .await - .expect("sandbox create should start a durable main process"); - sandbox - .exec(&["sh", "-lc", &write_sentinel]) - .await - .expect("sandbox exec should write the workspace sentinel"); - - let stop_output = run_sandbox_lifecycle_command("stop", &sandbox.name).await; - assert!( - stop_output.contains("Stopped sandbox"), - "expected stop confirmation in:\n{stop_output}", - ); - - let mut exec_cmd = openshell_cmd(); - exec_cmd - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--", - "cat", - SENTINEL_PATH, - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let stopped_exec = exec_cmd - .output() - .await - .expect("spawn openshell sandbox exec while stopped"); - assert!( - !stopped_exec.status.success(), - "sandbox exec should fail while stopped" - ); - - let start_output = run_sandbox_lifecycle_command("start", &sandbox.name).await; - assert!( - start_output.contains("Started sandbox"), - "expected start confirmation in:\n{start_output}", - ); - - let sentinel = sandbox - .exec(&["cat", SENTINEL_PATH]) - .await - .expect("sandbox exec should succeed after start"); - assert!( - sentinel.lines().any(|line| line.trim() == SENTINEL), - "workspace sentinel should survive stop and start:\n{sentinel}", - ); - let run_count = sandbox - .exec(&["cat", RUN_COUNT_PATH]) - .await - .expect("sandbox exec should read the canonical main run count"); - assert_eq!( - run_count.trim(), - "2", - "start should launch a fresh canonical main instance:\n{run_count}", - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_can_be_deleted_while_stopped() { - let mut sandbox = SandboxGuard::create_keep( - &["sh", "-c", "echo stop-ready; exec sleep infinity"], - "stop-ready", - ) - .await - .expect("sandbox create should start a durable main process"); - - let stop_output = run_sandbox_lifecycle_command("stop", &sandbox.name).await; - assert!( - stop_output.contains("Stopped sandbox"), - "expected stop confirmation in:\n{stop_output}", - ); - - let delete_output = run_sandbox_lifecycle_command("delete", &sandbox.name).await; - // Deletion may return before the owned cleanup worker finishes. Both - // outcomes must still reach absence, which is checked below. - assert!( - delete_output.contains(&format!("Deleted sandbox {}", sandbox.name)) - || delete_output.contains(&format!( - "Sandbox {} deletion accepted; cleanup is pending", - sandbox.name - )), - "expected completed or accepted deletion in:\n{delete_output}", - ); - - if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox.name, false).await { - sandbox.cleanup().await; - panic!( - "stopped sandbox {} should be deleted without starting after \ - {SANDBOX_PRESENCE_TIMEOUT:?}; last observed sandbox list: {last_sandbox_list:?}", - sandbox.name, - ); - } - - // Mark the guard cleaned up. Its idempotent delete is harmless now that - // the lifecycle operation above has removed the sandbox. - sandbox.cleanup().await; -} - #[tokio::test] #[serial(sandbox_lifecycle)] async fn canonical_main_exit_zero_completes_persistent_sandbox() { @@ -779,166 +601,6 @@ async fn canonical_main_nonzero_exit_preserves_status() { ); } -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_canonical_main_exit_zero_reaches_completed() { - const RELEASE_PATH: &str = "/sandbox/.openshell-detached-success-release"; - let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 0"); - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script]) - .await - .expect("create detached successful canonical main"); - sandbox - .exec(&["touch", RELEASE_PATH]) - .await - .expect("release detached successful canonical main"); - - wait_for_sandbox_phase(&sandbox.name, "Completed", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("detached successful main did not complete:\n{err}")); - let details = sandbox_details(&sandbox.name).await; - assert!( - details.contains("Phase: Completed"), - "detached successful main should retain Completed status:\n{details}" - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_canonical_main_nonzero_exit_reaches_error() { - const RELEASE_PATH: &str = "/sandbox/.openshell-detached-failure-release"; - let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 11"); - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script]) - .await - .expect("create detached failing canonical main"); - sandbox - .exec(&["touch", RELEASE_PATH]) - .await - .expect("release detached failing canonical main"); - - wait_for_sandbox_phase(&sandbox.name, "Error", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("detached failing main did not reach Error:\n{err}")); - let details = sandbox_details(&sandbox.name).await; - assert!( - details.contains("Phase: Error") && details.contains("Exit Code: 11"), - "detached failing main should retain its terminal result:\n{details}" - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_and_exec_receive_declared_environment() { - for mode in ["--tty", "--no-tty"] { - let script = r#"printf 'declared_env=%s\n' "${REPRO_SENTINEL:-missing}"; while true; do sleep 1; done"#; - let mut sandbox = SandboxGuard::create_keep_with_args( - &[ - mode, - "--no-auto-providers", - "--env", - "REPRO_SENTINEL=present", - ], - &["sh", "-c", script], - "declared_env=", - ) - .await - .expect("create canonical process with declared environment"); - let initial = normalize_output(&sandbox.create_output); - let later = sandbox - .exec(&[ - "sh", - "-c", - r#"printf 'declared_env=%s\n' "${REPRO_SENTINEL:-missing}""#, - ]) - .await; - sandbox.cleanup().await; - - assert!( - initial.lines().any(|line| line == "declared_env=present"), - "initial process must receive declared environment ({mode}): {initial}" - ); - let later = normalize_output(&later.expect("exec environment probe")); - assert!( - later.lines().any(|line| line == "declared_env=present"), - "exec must receive the same declared environment ({mode}): {later}" - ); - } -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_main_exit_during_provisioning_is_classified_as_workload_result() { - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", "exit 11"]) - .await - .expect("fast detached main exit should not be reported as a provisioning failure"); - - wait_for_sandbox_phase(&sandbox.name, "Error", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("fast detached main did not reach Error:\n{err}")); - - let mut get_cmd = openshell_cmd(); - get_cmd - .args(["sandbox", "get", &sandbox.name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let get_output = get_cmd.output().await.expect("spawn openshell sandbox get"); - let details = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&get_output.stdout), - String::from_utf8_lossy(&get_output.stderr), - )); - assert!( - get_output.status.success(), - "sandbox get failed:\n{details}" - ); - assert!( - details.contains("Phase: Error") && details.contains("Exit Code: 11"), - "fast detached main should retain its workload result:\n{details}" - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_tty_main_uses_sandbox_environment() { - let script = r#"printf 'canonical_env home=%s user=%s term=%s\n' "$HOME" "$USER" "$TERM"; while true; do sleep 1; done"#; - let mut sandbox = - SandboxGuard::create_keep_with_args(&["--tty"], &["sh", "-lc", script], "canonical_env") - .await - .expect("create canonical TTY process"); - - let output = normalize_output(&sandbox.create_output); - let environment = output - .lines() - .find(|line| line.contains("canonical_env")) - .expect("canonical environment output"); - let field = |name: &str| { - environment - .split_whitespace() - .find_map(|value| value.strip_prefix(&format!("{name}="))) - .unwrap_or_default() - }; - - assert!( - !field("home").is_empty() && field("home") != "/root", - "canonical process must not inherit the supervisor HOME: {environment}" - ); - assert!( - !field("user").is_empty(), - "canonical process USER must identify the sandbox user: {environment}" - ); - assert!( - !field("term").is_empty() && field("term") != "dumb", - "canonical TTY process must receive a usable TERM: {environment}" - ); - - sandbox.cleanup().await; -} - #[tokio::test] #[serial(sandbox_lifecycle)] async fn canonical_main_disconnect_reconnect_replays_history_for_same_process() { @@ -1299,95 +961,6 @@ async fn canonical_main_exit_255_is_not_retried_as_transport_failure() { assert_eq!(status.code(), Some(255)); } -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn on_failure_policy_replaces_runtime_and_preserves_workspace() { - const SCRIPT: &str = r#" -marker=/sandbox/.openshell-restart-e2e -if [ -e "$marker" ]; then - printf 'replacement-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/replacement-run - sleep 300 -else - touch "$marker" - printf 'initial-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/initial-run - while [ ! -e /sandbox/.openshell-restart-release ]; do sleep 0.05; done - exit 17 -fi -"#; - - let mut sandbox = SandboxGuard::manage_existing(unique_sandbox_name()); - let (output, exit_code) = run_cli(&[ - "sandbox", - "create", - "--name", - &sandbox.name, - "--from", - E2E_WORKLOAD_IMAGE, - "--detach", - "--restart-policy", - "on-failure", - "--", - "sh", - "-lc", - SCRIPT, - ]) - .await; - assert_eq!( - exit_code, 0, - "create sandbox with OnFailure policy: {output}" - ); - wait_for_sandbox_exec_contains( - &sandbox.name, - &["cat", "/sandbox/initial-run"], - "initial-", - SANDBOX_RESTART_TIMEOUT, - ) - .await - .expect("initial main process should write its workspace marker"); - - let initial_details = sandbox_details(&sandbox.name).await; - let initial_instance = extract_field(&initial_details, "Main process instance") - .expect("initial main process instance"); - sandbox - .exec(&["touch", "/sandbox/.openshell-restart-release"]) - .await - .expect("release initial main process"); - - let deadline = Instant::now() + SANDBOX_RESTART_TIMEOUT; - let mut last_runs = Err("replacement is not ready".to_string()); - let final_details = loop { - let details = sandbox_details(&sandbox.name).await; - let replacement_instance = extract_field(&details, "Main process instance") - .is_some_and(|instance| instance != "-" && instance != initial_instance); - if details.contains("Phase: Ready") - && extract_field(&details, "Restart count").as_deref() == Some("1") - && replacement_instance - { - last_runs = sandbox - .exec(&["cat", "/sandbox/initial-run", "/sandbox/replacement-run"]) - .await; - if let Ok(runs) = &last_runs - && runs.contains("initial-") - && runs.contains("replacement-") - { - break details; - } - } - assert!( - Instant::now() < deadline, - "sandbox did not complete its policy-driven restart within \ - {SANDBOX_RESTART_TIMEOUT:?}; last details:\n{details}\nworkspace: {last_runs:?}" - ); - sleep(Duration::from_millis(250)).await; - }; - - assert!( - final_details.contains("Restart policy: on-failure"), - "restart policy should remain visible after replacement:\n{final_details}" - ); - sandbox.cleanup().await; -} - #[tokio::test] async fn sandbox_create_with_no_keep_cleans_up_after_tty_command() { let name = format!("tty-{:015x}", rand::random::() & 0x0fff_ffff_ffff_ffff); diff --git a/e2e/rust/tests/sandbox_templates.rs b/e2e/rust/tests/sandbox_templates.rs deleted file mode 100644 index 21c9233daf..0000000000 --- a/e2e/rust/tests/sandbox_templates.rs +++ /dev/null @@ -1,275 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for reusable sandbox workload templates. - -use std::process::Stdio; -use std::time::{SystemTime, UNIX_EPOCH}; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use serde_json::Value; - -struct CliResult { - output: String, - success: bool, -} - -struct TemplateGuard { - name: String, -} - -impl TemplateGuard { - fn new(name: String) -> Self { - Self { name } - } - - async fn cleanup(mut self) { - delete_template(&self.name).await; - self.name.clear(); - } -} - -impl Drop for TemplateGuard { - fn drop(&mut self) { - if self.name.is_empty() { - return; - } - let bin = openshell_bin(); - let _ = std::process::Command::new(&bin) - .args(["sandbox", "template", "delete", &self.name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - CliResult { - output: strip_ansi(&combined), - success: output.status.success(), - } -} - -async fn delete_template(name: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "template", "delete", name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -fn unique_name(prefix: &str) -> String { - let millis = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock before epoch") - .as_millis(); - let suffix = millis % 1_000_000; - format!("{prefix}-{suffix:06}") -} - -fn output_contains(output: &str, needle: &str) -> bool { - output.to_lowercase().contains(&needle.to_lowercase()) -} - -fn output_mentions_template_not_found(output: &str) -> bool { - output_contains(output, "sandbox template") && output_contains(output, "not found") -} - -#[tokio::test] -async fn sandbox_create_from_template_uses_reusable_workload() { - let template_name = unique_name("tmpl"); - let sandbox_name = unique_name("sb-tmpl"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&[ - "sandbox", - "template", - "create", - &template_name, - "--cpu", - "500m", - "--memory", - "512Mi", - "--label", - "e2e=sandbox-template", - "--env", - "FEATURE_FLAG=on", - ]) - .await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let get_template = run_cli(&[ - "sandbox", - "template", - "get", - &template_name, - "--output", - "json", - ]) - .await; - assert!( - get_template.success, - "sandbox template get failed:\n{}", - get_template.output - ); - let template_json: Value = - serde_json::from_str(&get_template.output).expect("template JSON output"); - assert_eq!(template_json["name"].as_str(), Some(template_name.as_str())); - assert_eq!( - template_json["labels"]["e2e"].as_str(), - Some("sandbox-template") - ); - assert_eq!( - template_json["environment"]["FEATURE_FLAG"].as_str(), - Some("on") - ); - assert_eq!(template_json["resources"]["cpu"].as_str(), Some("500m")); - assert_eq!(template_json["resources"]["memory"].as_str(), Some("512Mi")); - - let list_templates = run_cli(&["sandbox", "template", "list", "--names"]).await; - assert!( - list_templates.success, - "sandbox template list failed:\n{}", - list_templates.output - ); - assert!( - list_templates - .output - .lines() - .any(|line| line.trim() == template_name), - "template list should include {template_name}:\n{}", - list_templates.output - ); - - let mut sandbox = SandboxGuard::create(&[ - "--name", - &sandbox_name, - "--template", - &template_name, - "--", - "sh", - "-lc", - "test \"$FEATURE_FLAG\" = on && echo template-env-ok", - ]) - .await - .expect("sandbox create from template should succeed"); - - assert!( - sandbox.create_output.contains("template-env-ok"), - "sandbox should inherit template environment:\n{}", - sandbox.create_output - ); - - sandbox.cleanup().await; - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_template_get_after_delete_returns_not_found() { - let template_name = unique_name("tmpl-del"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let delete_template = run_cli(&["sandbox", "template", "delete", &template_name]).await; - assert!( - delete_template.success, - "sandbox template delete failed:\n{}", - delete_template.output - ); - - let get_deleted = run_cli(&["sandbox", "template", "get", &template_name]).await; - assert!( - !get_deleted.success, - "deleted template should not be fetchable:\n{}", - get_deleted.output - ); - assert!( - output_mentions_template_not_found(&get_deleted.output), - "deleted template should return not-found:\n{}", - get_deleted.output - ); - - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_template_create_rejects_duplicate_name() { - let template_name = unique_name("tmpl-dupe"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let duplicate = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - !duplicate.success, - "duplicate sandbox template create should fail:\n{}", - duplicate.output - ); - assert!( - output_contains(&duplicate.output, "already exists"), - "duplicate sandbox template create should report already-exists:\n{}", - duplicate.output - ); - - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_create_from_missing_template_returns_not_found() { - let template_name = unique_name("tmpl-missing"); - let sandbox_name = unique_name("sb-missing"); - - let create_sandbox = run_cli(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--template", - &template_name, - "--", - "true", - ]) - .await; - - if create_sandbox.success { - let _ = run_cli(&["sandbox", "delete", &sandbox_name]).await; - } - - assert!( - !create_sandbox.success, - "sandbox create from missing template should fail:\n{}", - create_sandbox.output - ); - assert!( - output_mentions_template_not_found(&create_sandbox.output), - "sandbox create from missing template should report not-found:\n{}", - create_sandbox.output - ); -} diff --git a/e2e/rust/tests/settings_management.rs b/e2e/rust/tests/settings_management.rs deleted file mode 100644 index a3089590f7..0000000000 --- a/e2e/rust/tests/settings_management.rs +++ /dev/null @@ -1,329 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E tests for sandbox/global settings CLI workflows. -//! -//! Covers: -//! - Sandbox settings start as `` -//! - Sandbox setting set + read -//! - Global override blocks sandbox writes for that key -//! - Global get + global delete -//! - Sandbox-level control resumes after global delete - -use std::process::Stdio; -use std::sync::Mutex; -use std::time::Duration; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tokio::time::{Instant, sleep}; - -const TEST_KEY: &str = "ocsf_json_enabled"; -static SETTINGS_E2E_LOCK: Mutex<()> = Mutex::new(()); - -struct CliResult { - clean_output: String, - success: bool, - exit_code: Option, -} - -/// Best-effort global setting cleanup that runs even on test panic. -struct GlobalSettingCleanup { - key: &'static str, -} - -impl GlobalSettingCleanup { - fn new(key: &'static str) -> Self { - Self { key } - } -} - -impl Drop for GlobalSettingCleanup { - fn drop(&mut self) { - let _ = std::process::Command::new(openshell_bin()) - .args(["settings", "delete", "--global", "--key", self.key, "--yes"]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - CliResult { - clean_output: strip_ansi(&combined), - success: output.status.success(), - exit_code: output.status.code(), - } -} - -fn assert_setting_line(output: &str, key: &str, expected: &str) { - let needle = format!("{key} = {expected}"); - assert!( - output.contains(&needle), - "expected setting line '{needle}' in output:\n{output}" - ); -} - -fn assert_setting_line_with_scope(output: &str, key: &str, expected: &str, scope: &str) { - let needle = format!("{key} = {expected} ({scope})"); - assert!( - output.contains(&needle), - "expected setting line '{needle}' in output:\n{output}" - ); -} - -/// Poll `settings get` until the expected value and scope appear for a key. -async fn wait_for_setting_value( - sandbox_name: &str, - key: &str, - expected_value: &str, - expected_scope: &str, - timeout_duration: Duration, -) { - let needle = format!("{key} = {expected_value} ({expected_scope})"); - let start = Instant::now(); - loop { - let result = run_cli(&["settings", "get", sandbox_name]).await; - if result.success && result.clean_output.contains(&needle) { - return; - } - if start.elapsed() > timeout_duration { - panic!( - "timed out after {:?} waiting for setting line '{needle}' in output:\n{}", - timeout_duration, result.clean_output - ); - } - sleep(Duration::from_secs(1)).await; - } -} - -#[tokio::test] -async fn settings_global_override_round_trip() { - let _settings_lock = SETTINGS_E2E_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let _global_cleanup = GlobalSettingCleanup::new(TEST_KEY); - - let cleanup_before = - run_cli(&["settings", "delete", "--global", "--key", TEST_KEY, "--yes"]).await; - assert!( - cleanup_before.success, - "initial global setting cleanup should succeed (exit {:?}):\n{}", - cleanup_before.exit_code, cleanup_before.clean_output - ); - - let mut guard = - SandboxGuard::create_keep(&["sh", "-c", "echo Ready && sleep infinity"], "Ready") - .await - .expect("create keep sandbox"); - - let initial = run_cli(&["settings", "get", &guard.name]).await; - assert!( - initial.success, - "settings get should succeed (exit {:?}):\n{}", - initial.exit_code, initial.clean_output - ); - assert_setting_line_with_scope(&initial.clean_output, TEST_KEY, "", "unset"); - - let set_sandbox = run_cli(&[ - "settings", - "set", - &guard.name, - "--key", - TEST_KEY, - "--value", - "true", - ]) - .await; - assert!( - set_sandbox.success, - "sandbox setting set should succeed (exit {:?}):\n{}", - set_sandbox.exit_code, set_sandbox.clean_output - ); - - // Wait for the gateway to reflect the setting change. The setting is stored - // server-side, so `settings get` reads it immediately. Poll to ensure the - // config_revision has been updated (visible in the output). - wait_for_setting_value( - &guard.name, - TEST_KEY, - "true", - "sandbox", - Duration::from_secs(10), - ) - .await; - - let after_sandbox_set = run_cli(&["settings", "get", &guard.name]).await; - assert!( - after_sandbox_set.success, - "settings get after sandbox set should succeed (exit {:?}):\n{}", - after_sandbox_set.exit_code, after_sandbox_set.clean_output - ); - assert_setting_line_with_scope(&after_sandbox_set.clean_output, TEST_KEY, "true", "sandbox"); - - // Sandbox-scoped delete should succeed when not globally managed. - let sandbox_delete = run_cli(&["settings", "delete", &guard.name, "--key", TEST_KEY]).await; - assert!( - sandbox_delete.success, - "sandbox setting delete should succeed (exit {:?}):\n{}", - sandbox_delete.exit_code, sandbox_delete.clean_output - ); - assert!( - sandbox_delete - .clean_output - .contains("Deleted sandbox setting"), - "expected sandbox delete confirmation:\n{}", - sandbox_delete.clean_output - ); - - // After delete, the key should be unset again. - let after_sandbox_delete = run_cli(&["settings", "get", &guard.name]).await; - assert!( - after_sandbox_delete.success, - "settings get after sandbox delete should succeed:\n{}", - after_sandbox_delete.clean_output - ); - assert_setting_line_with_scope( - &after_sandbox_delete.clean_output, - TEST_KEY, - "", - "unset", - ); - - // Re-set at sandbox scope so we can test global override next. - let re_set = run_cli(&[ - "settings", - "set", - &guard.name, - "--key", - TEST_KEY, - "--value", - "true", - ]) - .await; - assert!( - re_set.success, - "re-set should succeed:\n{}", - re_set.clean_output - ); - - let set_global = run_cli(&[ - "settings", "set", "--global", "--key", TEST_KEY, "--value", "false", "--yes", - ]) - .await; - assert!( - set_global.success, - "global setting set should succeed (exit {:?}):\n{}", - set_global.exit_code, set_global.clean_output - ); - assert!( - set_global - .clean_output - .contains(&format!("Set global setting {TEST_KEY}=false")), - "expected global set output:\n{}", - set_global.clean_output - ); - - let blocked_sandbox_set = run_cli(&[ - "settings", - "set", - &guard.name, - "--key", - TEST_KEY, - "--value", - "true", - ]) - .await; - assert!( - !blocked_sandbox_set.success, - "sandbox setting should fail while key is global-managed:\n{}", - blocked_sandbox_set.clean_output - ); - assert!( - blocked_sandbox_set.clean_output.contains("is managed"), - "expected 'managed globally' error:\n{}", - blocked_sandbox_set.clean_output - ); - - // Sandbox-scoped delete should also be blocked while globally managed. - let blocked_sandbox_delete = - run_cli(&["settings", "delete", &guard.name, "--key", TEST_KEY]).await; - assert!( - !blocked_sandbox_delete.success, - "sandbox delete should fail while key is global-managed:\n{}", - blocked_sandbox_delete.clean_output - ); - - let global_get = run_cli(&["settings", "get", "--global"]).await; - assert!( - global_get.success, - "global settings get should succeed (exit {:?}):\n{}", - global_get.exit_code, global_get.clean_output - ); - assert_setting_line(&global_get.clean_output, TEST_KEY, "false"); - - let delete_global = - run_cli(&["settings", "delete", "--global", "--key", TEST_KEY, "--yes"]).await; - assert!( - delete_global.success, - "global settings delete should succeed (exit {:?}):\n{}", - delete_global.exit_code, delete_global.clean_output - ); - assert!( - delete_global - .clean_output - .contains(&format!("Deleted global setting {TEST_KEY}")), - "expected global delete confirmation in output:\n{}", - delete_global.clean_output - ); - - let global_after_delete = run_cli(&["settings", "get", "--global"]).await; - assert!( - global_after_delete.success, - "global settings get after delete should succeed (exit {:?}):\n{}", - global_after_delete.exit_code, global_after_delete.clean_output - ); - assert_setting_line(&global_after_delete.clean_output, TEST_KEY, ""); - - let sandbox_set_after_delete = run_cli(&[ - "settings", - "set", - &guard.name, - "--key", - TEST_KEY, - "--value", - "false", - ]) - .await; - assert!( - sandbox_set_after_delete.success, - "sandbox setting set after global delete should succeed (exit {:?}):\n{}", - sandbox_set_after_delete.exit_code, sandbox_set_after_delete.clean_output - ); - - let sandbox_after_delete = run_cli(&["settings", "get", &guard.name]).await; - assert!( - sandbox_after_delete.success, - "settings get after global delete should succeed (exit {:?}):\n{}", - sandbox_after_delete.exit_code, sandbox_after_delete.clean_output - ); - assert_setting_line_with_scope( - &sandbox_after_delete.clean_output, - TEST_KEY, - "false", - "sandbox", - ); - - guard.cleanup().await; -} diff --git a/e2e/rust/tests/upload_backpressure.rs b/e2e/rust/tests/upload_backpressure.rs deleted file mode 100644 index e766303c37..0000000000 --- a/e2e/rust/tests/upload_backpressure.rs +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(all(feature = "e2e", unix))] - -//! Regression for large built-in uploads through the CLI, gateway, supervisor, -//! and sandbox. A fast sender previously overflowed the supervisor's 4 MiB -//! pending-stdin budget while tar was creating many files (issue #4430). - -use std::fmt::Write as _; -use std::fs; -use std::os::unix::fs::{PermissionsExt, symlink}; -use std::time::Duration; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use rand::{RngCore, SeedableRng, rngs::StdRng}; -use sha2::{Digest, Sha256}; -use tokio::time::timeout; - -const FILE_COUNT: usize = 12_000; -const FILE_BYTES: usize = 8_192; -const TRANSFER_TIMEOUT: Duration = Duration::from_secs(300); - -/// An archive substantially larger than the stdin budget must finish without -/// pacing at the caller, preserving every file's content and tar semantics. -#[tokio::test] -async fn large_upload_preserves_content_and_metadata_outside_workdir() { - let tmpdir = tempfile::tempdir().expect("create upload fixture tempdir"); - let source = tmpdir.path().join("large-project"); - fs::create_dir_all(source.join("payload")).expect("create payload directory"); - - // Distinct deterministic pseudorandom bytes keep the fixture reproducible - // without letting compression or identical-file checks hide dropped data. - let mut rng = StdRng::seed_from_u64(4430); - let mut payload = [0_u8; FILE_BYTES]; - let mut checksums = String::new(); - for index in 0..FILE_COUNT { - rng.fill_bytes(&mut payload); - let relative = format!("payload/{index:05}.bin"); - fs::write(source.join(&relative), payload).expect("write payload file"); - writeln!(checksums, "{:x} {relative}", Sha256::digest(payload)) - .expect("append file checksum"); - } - let manifest_digest = format!("{:x}", Sha256::digest(checksums.as_bytes())); - fs::write(source.join("SHA256SUMS"), checksums).expect("write checksum manifest"); - fs::write( - source.join("run.sh"), - "#!/bin/sh\nprintf '%s\\n' upload-executable-ok\n", - ) - .expect("write executable fixture"); - fs::set_permissions(source.join("run.sh"), fs::Permissions::from_mode(0o751)) - .expect("set executable mode"); - symlink("payload/00000.bin", source.join("payload-link")).expect("create relative symlink"); - symlink("missing-target", source.join("dangling-link")).expect("create dangling symlink"); - fs::write(source.join("conflict.txt"), "local-conflict").expect("write merge fixture"); - - let mut sandbox = SandboxGuard::create(&[]) - .await - .expect("create persistent upload sandbox"); - sandbox - .exec(&[ - "sh", - "-c", - "set -eu; test \"$(pwd -P)\" != /tmp/openshell-upload-backpressure; \ - mkdir -p /tmp/openshell-upload-backpressure/large-project; \ - printf remote-conflict > /tmp/openshell-upload-backpressure/large-project/conflict.txt; \ - printf remote-preserved > /tmp/openshell-upload-backpressure/large-project/unrelated.txt", - ]) - .await - .expect("prepare destination outside the canonical workdir"); - - // No sleep, bandwidth limit, or retry: the protocol must backpressure the - // normal CLI tar stream, even when the local sender outruns extraction. - let mut upload = openshell_cmd(); - upload - .env("RUST_LOG", "warn") - .args(["sandbox", "upload", &sandbox.name]) - .arg(&source) - .args(["/tmp/openshell-upload-backpressure", "--no-git-ignore"]); - let result = timeout(TRANSFER_TIMEOUT, upload.output()) - .await - .expect("large upload must not hang waiting for credit") - .expect("start large upload"); - let output = format!( - "{}{}", - String::from_utf8_lossy(&result.stdout), - String::from_utf8_lossy(&result.stderr) - ); - assert!(result.status.success(), "large upload failed: {output}"); - assert!( - !output.contains("sandbox sync operation failed; retrying"), - "large upload required a retry: {output}" - ); - - let verify = format!( - "set -eu; cd /tmp/openshell-upload-backpressure/large-project; \ - test \"$(find payload -type f | wc -l)\" -eq {FILE_COUNT}; \ - test \"$(sha256sum SHA256SUMS | cut -d ' ' -f 1)\" = {manifest_digest}; \ - sha256sum --check --quiet SHA256SUMS; \ - test \"$(stat -c %a run.sh)\" = 751; \ - test \"$(./run.sh)\" = upload-executable-ok; \ - test -L payload-link; test \"$(readlink payload-link)\" = payload/00000.bin; \ - test -L dangling-link; test ! -e dangling-link; \ - test \"$(readlink dangling-link)\" = missing-target; \ - test \"$(cat conflict.txt)\" = local-conflict; \ - test \"$(cat unrelated.txt)\" = remote-preserved; \ - printf large-upload-content-and-metadata-ok" - ); - let output = timeout(TRANSFER_TIMEOUT, sandbox.exec(&["sh", "-c", &verify])) - .await - .expect("large upload verification must not hang") - .expect("every uploaded file and metadata item must match"); - assert!( - strip_ansi(&output).contains("large-upload-content-and-metadata-ok"), - "expected complete upload verification marker:\n{output}" - ); - - sandbox.cleanup().await; -} diff --git a/e2e/rust/tests/vm_corporate_proxy.rs b/e2e/rust/tests/vm_corporate_proxy.rs index d914cb1c03..32d9934125 100644 --- a/e2e/rust/tests/vm_corporate_proxy.rs +++ b/e2e/rust/tests/vm_corporate_proxy.rs @@ -21,8 +21,6 @@ //! 407 to an unauthenticated CONNECT, so a 200 proves delivery. //! 4. A port-qualified `no_proxy` entry bypasses the proxy for that port only. //! 5. An `https://` proxy works when its CA is supplied via `proxy_ca_bundle`. -//! 6. An incoherent setting is fatal at gateway startup rather than -//! degrading to a direct dial. //! //! Fixtures run as host processes and are reached by the host supervisor. //! `host.openshell.internal` is normalized to host loopback for both libkrun @@ -850,39 +848,3 @@ async fn vm_corporate_proxy_trusts_ca_bundle_for_https_proxy() { .await .expect("restore gateway config"); } - -#[tokio::test] -#[serial(vm_corporate_proxy)] -async fn vm_corporate_proxy_rejects_incoherent_configuration() { - if !should_run("corporate proxy fail-closed test") { - return; - } - - // A bypass list without a proxy URL means the operator believed proxying - // was in effect. Accepting it would leave every dial direct while looking - // configured, so the gateway must refuse to start instead. - let mut gateway_config = GatewayProxyConfig::apply_raw("no_proxy = \"10.0.0.0/8\"\n") - .expect("apply incoherent corporate proxy gateway config"); - - let health = wait_for_healthy(Duration::from_secs(20)).await; - assert!( - health.is_err(), - "gateway must not serve traffic with an incoherent [openshell.drivers.vm] proxy table" - ); - - let log_path = std::env::var("OPENSHELL_E2E_GATEWAY_LOG").expect("gateway log path"); - let log = std::fs::read_to_string(&log_path).expect("read gateway log"); - // The message must name the offending key rather than surfacing as an - // opaque driver-readiness timeout. Matched loosely on the key names so - // this does not break on error-wrapping changes. - assert!( - log.contains("no_proxy") && log.contains("https_proxy"), - "the startup error must name the offending key; gateway log tail:\n{}", - log.lines().rev().take(40).collect::>().join("\n") - ); - - gateway_config - .restore() - .await - .expect("restore gateway config"); -} diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index e97d7f961e..5b6c4c97c5 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -713,8 +713,6 @@ GATEWAY_ARGS=( --compute-driver docker --tls-cert "${PKI_DIR}/server/tls.crt" --tls-key "${PKI_DIR}/server/tls.key" - # edge_tunnel_e2e exercises the opt-in WebSocket tunnel. - --enable-websocket-tunnel true --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" ) diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 224eecaaf3..0310db6bcd 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -636,8 +636,6 @@ GATEWAY_ARGS=( --health-port "${HEALTH_PORT}" --tls-cert "${PKI_DIR}/server/tls.crt" --tls-key "${PKI_DIR}/server/tls.key" - # edge_tunnel_e2e exercises the opt-in WebSocket tunnel. - --enable-websocket-tunnel true --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" --log-level info ) diff --git a/tasks/rust.toml b/tasks/rust.toml index c995e74f8e..58d29303df 100644 --- a/tasks/rust.toml +++ b/tasks/rust.toml @@ -7,6 +7,10 @@ description = "Compare native and sandbox-filtered TCP socket performance" run = "cargo run --release -p openshell-sandbox --features perf-harness --bin openshell-seccomp-perf --" +["perf:live-internet"] +description = "Compare live-Internet traffic on the host and in a sandbox on the active gateway" +run = "uv run --no-project python tasks/scripts/live_internet_perf.py" + ["rust:check"] description = "Check all Rust crates for errors" depends = ["rust:lockfiles:check"] diff --git a/tasks/scripts/live_internet_perf.py b/tasks/scripts/live_internet_perf.py new file mode 100644 index 0000000000..9ccad8e2fd --- /dev/null +++ b/tasks/scripts/live_internet_perf.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compare live-Internet traffic on the host with the same traffic in a sandbox. + +Creates one sandbox on the active gateway with a policy that allows the +benchmark's destinations, then alternates direct and sandbox rounds of +`live_internet_perf_round.py` so drift in the network affects both sides +alike. Each round prints one `LIVE_INTERNET_PERF` line with the round's JSON. +The sandbox is deleted on exit. + +This is a manual benchmark: it needs the live Internet, and CI doesn't run it. + + mise run perf:live-internet -- --rounds 4 --from +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import time +import uuid +from pathlib import Path + +ROUND = Path(__file__).with_name("live_internet_perf_round.py") + +POLICY = """version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] + +landlock: + compatibility: best_effort + +network_policies: + live_internet_performance: + name: live_internet_performance + endpoints: + - { host: example.com, port: 443, protocol: tcp } + - { host: github.com, port: 443, protocol: tcp } + - { host: raw.githubusercontent.com, port: 443, protocol: tcp } + - { host: pypi.org, port: 443, protocol: tcp } + - { host: files.pythonhosted.org, port: 443, protocol: tcp } + - { host: registry.npmjs.org, port: 443, protocol: tcp } + - { host: crates.io, port: 443, protocol: tcp } + - { host: docs.python.org, port: 443, protocol: tcp } + - { host: speed.cloudflare.com, port: 443, protocol: tcp } + binaries: + - path: "/**" +""" + + +def run(command: list[str], **kwargs) -> str: + result = subprocess.run( + command, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + **kwargs, + ) + if result.returncode != 0: + raise SystemExit( + f"{' '.join(command[:4])} ... failed (exit {result.returncode}):\n{result.stdout}" + ) + return result.stdout + + +def direct_round() -> str: + env = dict(os.environ, OPENSHELL_LIVE_PERF_MODE="direct") + return run([sys.executable, str(ROUND)], env=env) + + +def sandbox_round(openshell: str, name: str, source: str) -> str: + return run( + [ + openshell, + "sandbox", + "exec", + "--name", + name, + "--no-tty", + "--", + "sh", + "-c", + 'OPENSHELL_LIVE_PERF_MODE=sandbox python3 -c "$1"', + "openshell-live-internet-perf", + source, + ] + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--rounds", type=int, default=10, help="rounds per side (default 10)" + ) + parser.add_argument( + "--from", dest="image", help="sandbox image (default: the gateway's)" + ) + parser.add_argument("--openshell", default="openshell", help="openshell CLI to use") + args = parser.parse_args() + if args.rounds < 1: + parser.error("--rounds must be at least 1") + + source = ROUND.read_text() + name = f"live-perf-{uuid.uuid4().hex[:8]}" + with tempfile.NamedTemporaryFile("w", suffix=".yaml") as policy: + policy.write(POLICY) + policy.flush() + create = [ + args.openshell, + "sandbox", + "create", + "--name", + name, + "--policy", + policy.name, + ] + if args.image: + create += ["--from", args.image] + started = time.perf_counter() + try: + run([*create, "--detach"]) + elapsed_ms = (time.perf_counter() - started) * 1000 + print( + f"LIVE_INTERNET_PERF create {json.dumps({'elapsed_ms': elapsed_ms})}", + flush=True, + ) + for number in range(1, args.rounds + 1): + # Alternate which side goes first so neither always runs on a + # warmer network. + sides = [ + ("direct", direct_round), + ("sandbox", lambda: sandbox_round(args.openshell, name, source)), + ] + if number % 2 == 0: + sides.reverse() + for mode, round_fn in sides: + print( + f"LIVE_INTERNET_PERF {mode} round={number} {round_fn().strip()}", + flush=True, + ) + finally: + subprocess.run( + [args.openshell, "sandbox", "delete", name], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + +if __name__ == "__main__": + main() diff --git a/tasks/scripts/live_internet_perf_round.py b/tasks/scripts/live_internet_perf_round.py new file mode 100644 index 0000000000..0684269486 --- /dev/null +++ b/tasks/scripts/live_internet_perf_round.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""One round of the live-Internet benchmark, shaped like a short coding-agent +session. Prints one JSON document. `live_internet_perf.py` runs it on the host +and inside a sandbox; it needs only the Python standard library and, for the +clone step, git. +""" + +import concurrent.futures +import http.client +import json +import os +import shutil +import socket +import ssl +import statistics +import subprocess +import tempfile +import time +import urllib.parse +from pathlib import Path + +MODE = os.environ.get("OPENSHELL_LIVE_PERF_MODE", "direct") +TLS = ssl.create_default_context() +USER_AGENT = "OpenShell-live-Internet-benchmark/1" + +METADATA_URLS = [ + "https://raw.githubusercontent.com/NVIDIA/OpenShell/main/README.md", + "https://pypi.org/pypi/requests/json", + "https://registry.npmjs.org/typescript/latest", + "https://docs.python.org/3/", +] +DNS_HOSTS = [ + "github.com", + "raw.githubusercontent.com", + "pypi.org", + "registry.npmjs.org", + "crates.io", + "docs.python.org", + "speed.cloudflare.com", +] + + +def percentile(values, fraction): + values = sorted(values) + return values[min(len(values) - 1, int((len(values) - 1) * fraction))] + + +def request(url, method="GET", max_bytes=None): + parsed = urllib.parse.urlsplit(url) + connection = http.client.HTTPSConnection( + parsed.hostname, parsed.port or 443, timeout=20, context=TLS + ) + path = parsed.path or "/" + if parsed.query: + path += "?" + parsed.query + started = time.perf_counter_ns() + connection.request( + method, path, headers={"User-Agent": USER_AGENT, "Connection": "close"} + ) + response = connection.getresponse() + first = response.read(1) + first_byte_ms = (time.perf_counter_ns() - started) / 1_000_000 + body_bytes = len(first) + while max_bytes is None or body_bytes < max_bytes: + remaining = None if max_bytes is None else max_bytes - body_bytes + chunk = response.read(65536 if remaining is None else min(65536, remaining)) + if not chunk: + break + body_bytes += len(chunk) + total_ms = (time.perf_counter_ns() - started) / 1_000_000 + status = response.status + connection.close() + if status < 200 or status >= 400: + raise RuntimeError(f"{url} returned HTTP {status}") + return { + "status": status, + "bytes": body_bytes, + "first_byte_ms": first_byte_ms, + "total_ms": total_ms, + } + + +def metric(name, operation): + started = time.perf_counter_ns() + try: + detail = operation() + return { + "name": name, + "ok": True, + "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, + "detail": detail, + } + except Exception as error: + return { + "name": name, + "ok": False, + "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, + "error": f"{type(error).__name__}: {error}", + } + + +def dns_set(): + samples = [] + for host in DNS_HOSTS: + started = time.perf_counter_ns() + addresses = socket.getaddrinfo(host, 443, socket.AF_UNSPEC, socket.SOCK_STREAM) + samples.append( + { + "host": host, + "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, + "addresses": len(addresses), + } + ) + timings = [sample["elapsed_ms"] for sample in samples] + return { + "lookups": samples, + "p50_ms": percentile(timings, 0.50), + "p95_ms": percentile(timings, 0.95), + } + + +def metadata_serial(): + return {"requests": [request(url, max_bytes=2_000_000) for url in METADATA_URLS]} + + +def metadata_concurrent(): + urls = METADATA_URLS + METADATA_URLS + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + responses = list(pool.map(lambda url: request(url, max_bytes=2_000_000), urls)) + return {"requests": responses} + + +def https_reuse(): + connection = http.client.HTTPSConnection( + "example.com", 443, timeout=20, context=TLS + ) + samples = [] + try: + for _ in range(20): + started = time.perf_counter_ns() + connection.request( + "HEAD", + "/", + headers={"User-Agent": USER_AGENT, "Connection": "keep-alive"}, + ) + response = connection.getresponse() + response.read() + if response.status != 200: + raise RuntimeError(f"example.com returned HTTP {response.status}") + samples.append((time.perf_counter_ns() - started) / 1_000_000) + finally: + connection.close() + return { + "requests": len(samples), + "p50_ms": percentile(samples, 0.50), + "p95_ms": percentile(samples, 0.95), + "mean_ms": statistics.fmean(samples), + } + + +def git_clone(): + if shutil.which("git") is None: + return {"skipped": "git is not installed in the workload image"} + with tempfile.TemporaryDirectory(prefix="openshell-live-git-") as directory: + checkout = Path(directory) / "sampleproject" + command = [ + "git", + "-c", + "advice.detachedHead=false", + "clone", + "--quiet", + "--depth", + "1", + "--filter=blob:none", + "https://github.com/pypa/sampleproject.git", + str(checkout), + ] + output = subprocess.run( + command, + capture_output=True, + timeout=60, + check=False, + ) + if output.returncode != 0: + raise RuntimeError(output.stderr.decode(errors="replace")[-500:]) + paths = [path for path in checkout.rglob("*") if path.is_file()] + files = len(paths) + size = sum(path.stat().st_size for path in paths) + return {"files": files, "checkout_bytes": size} + + +def package_download(): + metadata = request("https://pypi.org/pypi/idna/json", max_bytes=2_000_000) + connection = http.client.HTTPSConnection("pypi.org", 443, timeout=20, context=TLS) + connection.request("GET", "/pypi/idna/json", headers={"User-Agent": USER_AGENT}) + response = connection.getresponse() + document = json.loads(response.read()) + connection.close() + wheels = [ + entry for entry in document["urls"] if entry["packagetype"] == "bdist_wheel" + ] + if not wheels: + raise RuntimeError("PyPI returned no idna wheel") + artifact = request(wheels[0]["url"], max_bytes=5_000_000) + return { + "metadata": metadata, + "artifact": artifact, + "filename": wheels[0]["filename"], + } + + +def bulk_download(): + return request( + "https://speed.cloudflare.com/__down?bytes=5242880", max_bytes=5242880 + ) + + +def denied_destination(): + if MODE != "sandbox": + return {"skipped": "policy denial applies only to sandbox mode"} + started = time.perf_counter_ns() + try: + socket.create_connection(("example.net", 443), timeout=5).close() + except OSError as error: + return { + "denied": True, + "latency_ms": (time.perf_counter_ns() - started) / 1_000_000, + "error": str(error), + } + raise RuntimeError("destination omitted from policy was reachable") + + +started = time.perf_counter_ns() +metrics = [ + metric("dns_service_set", dns_set), + metric("https_metadata_serial", metadata_serial), + metric("https_metadata_concurrent", metadata_concurrent), + metric("https_reuse", https_reuse), + metric("git_clone", git_clone), + metric("package_download", package_download), + metric("bulk_download_5mib", bulk_download), + metric("policy_denial", denied_destination), +] +document = { + "schema": "openshell.live-internet-perf.v1", + "mode": MODE, + "total_ms": (time.perf_counter_ns() - started) / 1_000_000, + "metrics": metrics, +} +print(json.dumps(document, separators=(",", ":"))) +if any(item["name"] == "policy_denial" and not item["ok"] for item in metrics): + raise SystemExit(1) diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 0d4ab62225..0bc14ac2a8 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -517,6 +517,7 @@ name = "conformance" version = "0.0.0" dependencies = [ "host-server-support", + "openshell-crypto", "openshell-testkit", "serde_json", "tempfile", diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index 7201953c9d..9715526af2 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -22,6 +22,7 @@ test = false [dependencies] host-server-support = { path = "../../support/host-server" } +openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-testkit = { path = "../../testkit/testkit" } serde_json = "1" tempfile = "3" diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index c51df37c16..fab8a9b33b 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -9,12 +9,18 @@ nextest run the tests as the library's unit tests. The crate also builds the | Suite | Covers | |---|---| | `smoke` | Create a sandbox, run `echo` in it, delete it | -| `lifecycle` | SDK: long-running and run-to-completion sandboxes, prompt delete when the workload is slow to handle TERM, the default command, restart of a stopped sandbox, deleting a stopped sandbox | +| `lifecycle` | SDK: long-running and run-to-completion sandboxes, prompt delete when the workload is slow to handle TERM, the default command, restart of a stopped sandbox, deleting a stopped sandbox, exec past the startup recovery deadline, the final phase when the main command exits, the main command's environment, `on_failure` restarts, `--no-keep` cleanup | | `cli` | The same lifecycle through the `openshell` CLI: status, create, get, list, exec, stop, start, delete | | `persistent-workspace` | Files in `/sandbox` survive stop and start | | `policy-advisor` | `policy.local` accepts a proposed rule; a denied address or hostname yields one draft rule | -| `file-transfer` | `sandbox upload` and `download`: round trips, `.gitignore` filtering, workspace-escape and option-looking names | +| `file-transfer` | `sandbox upload` and `download`: round trips, file modes and symlinks outside the workspace, `.gitignore` filtering, workspace-escape and option-looking names, a large upload | | `credential-gating` | Provider credentials: admission of credentialed endpoints, REST body placeholders and rewriting, WebSocket binary frames, live policy binding | +| `image` | A sandbox runs the default image or the one it is created from | +| `labels` | Labels are stored and select sandboxes in `sandbox list` | +| `templates` | A sandbox created from a template uses its workload; duplicate, deleted and missing templates return the right errors | +| `settings` | Sandbox-scoped settings, and gateway-global settings that override them | +| `process` | Core dumps are disabled for sandbox processes | +| `filesystem-policy` | Landlock: a hard requirement accepts added device paths; a replaced policy path rejects the next launch | `smoke` is the smallest end-to-end check that the runner can run a real test in a tmachine environment on CI. The other suites are ports of the old diff --git a/tests/suites/conformance/src/file_transfer.rs b/tests/suites/conformance/src/file_transfer.rs index 5bd24563c0..6525595ffe 100644 --- a/tests/suites/conformance/src/file_transfer.rs +++ b/tests/suites/conformance/src/file_transfer.rs @@ -11,6 +11,7 @@ //! why they run in every environment. use std::fs; +use std::os::unix::fs::{PermissionsExt, symlink}; use std::path::{Path, PathBuf}; use tempfile::TempDir; @@ -25,6 +26,7 @@ const LARGE_FILE_SIZE: usize = 512 * 1024; async fn upload_download_round_trip(cx: &mut Context) -> testkit::Result { let ft = Transfer::new(cx, "fr").await?; ft.upload_into_existing_directories().await?; + ft.upload_metadata_outside_workdir().await?; ft.round_trip().await?; ft.download_file().await?; ft.download_directory().await?; @@ -53,6 +55,85 @@ async fn download_stays_inside_workspace(cx: &mut Context) -> testkit::Result { ft.sandbox.delete().await } +const BACKPRESSURE_FILE_COUNT: usize = 12_000; +const BACKPRESSURE_FILE_BYTES: usize = 8_192; + +/// An upload far larger than the supervisor's 4 MiB pending-stdin budget, +/// made of many small files that keep tar busy creating them, completes on +/// the first attempt with every file intact. The sender is not paced, so the +/// transfer must apply backpressure (#4430). +#[testkit::test] +async fn large_upload_backpressure(cx: &mut Context) -> testkit::Result { + let ft = Transfer::new(cx, "fb").await?; + let source = ft.local("large-project"); + create_dir_all(&source.join("payload"))?; + + // Distinct pseudorandom bytes, so compression or identical-file checks + // can't hide dropped data. + let mut state: u64 = 4430; + let mut payload = vec![0_u8; BACKPRESSURE_FILE_BYTES]; + let mut checksums = String::new(); + for index in 0..BACKPRESSURE_FILE_COUNT { + for chunk in payload.chunks_mut(8) { + chunk.copy_from_slice(&splitmix64(&mut state).to_le_bytes()[..chunk.len()]); + } + let relative = format!("payload/{index:05}.bin"); + fs::write(source.join(&relative), &payload) + .map_err(|error| Error::msg(format!("write {relative}: {error}")))?; + checksums.push_str(&format!("{} {relative}\n", sha256_hex(&payload)?)); + } + write(&source.join("SHA256SUMS"), &checksums)?; + + let remote = format!("{}/backpressure", ft.remote_root); + let uploaded = ft + .cx + .cli_with_env( + &[("RUST_LOG", "warn")], + [ + "sandbox", + "upload", + ft.sandbox.name(), + &source.display().to_string(), + &remote, + "--no-git-ignore", + ], + ) + .await? + .success()?; + testkit::ensure!( + !uploaded + .message() + .contains("sandbox sync operation failed; retrying"), + "the large upload needed a retry: {}", + uploaded.message() + ); + + ft.sh(&format!( + "set -eu; cd '{remote}/large-project'; \ + test \"$(find payload -type f | wc -l)\" -eq {BACKPRESSURE_FILE_COUNT}; \ + test \"$(sha256sum SHA256SUMS | cut -d ' ' -f 1)\" = {}; \ + sha256sum --check --quiet SHA256SUMS", + sha256_hex(checksums.as_bytes())? + )) + .await?; + ft.sandbox.delete().await +} + +/// One step of SplitMix64, a small deterministic generator for fixture bytes. +fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +fn sha256_hex(bytes: &[u8]) -> testkit::Result { + let digest = + openshell_crypto::sha256(bytes).map_err(|error| Error::msg(format!("SHA-256: {error}")))?; + Ok(digest.iter().map(|byte| format!("{byte:02x}")).collect()) +} + /// A sandbox, a directory in its workspace, and a local scratch directory. struct Transfer<'a> { cx: &'a Context, @@ -169,6 +250,53 @@ impl<'a> Transfer<'a> { Ok(()) } + /// An upload to a directory outside the workspace keeps file modes and + /// symlinks, dangling ones included, overwrites files it carries, and + /// leaves other files in the destination alone. + async fn upload_metadata_outside_workdir(&self) -> testkit::Result { + let source = self.local("metadata-project"); + create_dir_all(&source.join("payload"))?; + write(&source.join("payload/00000.bin"), "payload")?; + let script = source.join("run.sh"); + write(&script, "#!/bin/sh\nprintf '%s\\n' upload-executable-ok\n")?; + fs::set_permissions(&script, fs::Permissions::from_mode(0o751)) + .map_err(|error| Error::msg(format!("chmod run.sh: {error}")))?; + for (target, link) in [ + ("payload/00000.bin", "payload-link"), + ("missing-target", "dangling-link"), + ] { + symlink(target, source.join(link)) + .map_err(|error| Error::msg(format!("symlink {link}: {error}")))?; + } + write(&source.join("conflict.txt"), "local-conflict")?; + + let remote = format!( + "/tmp/file-transfer-{}-{}", + self.cx.run_id(), + self.sandbox.name() + ); + let project = format!("{remote}/metadata-project"); + self.sh(&format!( + "set -eu; test \"$(pwd -P)\" != '{remote}'; mkdir -p '{project}'; \ + printf remote-conflict > '{project}/conflict.txt'; \ + printf remote-preserved > '{project}/unrelated.txt'" + )) + .await?; + + self.upload(&source, &remote, true).await?.success()?; + self.sh(&format!( + "set -eu; cd '{project}'; \ + test \"$(stat -c %a run.sh)\" = 751; \ + test \"$(./run.sh)\" = upload-executable-ok; \ + test -L payload-link; test \"$(readlink payload-link)\" = payload/00000.bin; \ + test -L dangling-link; test ! -e dangling-link; \ + test \"$(readlink dangling-link)\" = missing-target; \ + test \"$(cat conflict.txt)\" = local-conflict; \ + test \"$(cat unrelated.txt)\" = remote-preserved" + )) + .await + } + /// A directory with a nested file and a large binary file, and a single /// file, survive upload then download. async fn round_trip(&self) -> testkit::Result { @@ -182,7 +310,7 @@ impl<'a> Transfer<'a> { let remote = format!("{}/roundtrip", self.remote_root); let uploaded = self.upload(&source, &remote, false).await?.success()?; - let warning = message(&uploaded); + let warning = uploaded.message(); testkit::ensure!( warning.contains("outside a Git work tree") && warning.contains(".gitignore rules are not applied"), @@ -310,7 +438,7 @@ impl<'a> Transfer<'a> { let remote = format!("{}/reuploaded", self.remote_root); let rejected = self.upload(&runs, &remote, false).await?.failure()?; - let output = message(&rejected); + let output = rejected.message(); testkit::ensure!( output.contains("filtering selected no files") && output.contains("--no-git-ignore"), "the rejected upload did not explain the --no-git-ignore override: {output}" @@ -336,7 +464,7 @@ impl<'a> Transfer<'a> { for source in [etc_link.clone(), passwd_link, format!("{etc_link}/passwd")] { let rejected = self.download(&source, &destination).await?.failure()?; - let output = message(&rejected); + let output = rejected.message(); testkit::ensure!( output.contains("resolves to") && output.contains("outside the") @@ -362,16 +490,6 @@ impl<'a> Transfer<'a> { } } -/// The CLI's stdout and stderr as one line of words. Error reports wrap to -/// the terminal width and draw a gutter, so match against this instead. -fn message(output: &CliOutput) -> String { - format!("{} {}", output.stdout, output.stderr) - .replace(['│', '×', '╰', '─', '╭', '├'], " ") - .split_whitespace() - .collect::>() - .join(" ") -} - fn git(repository: &Path, args: &[&str]) -> testkit::Result { let output = std::process::Command::new("git") .args(args) diff --git a/tests/suites/conformance/src/filesystem_policy.rs b/tests/suites/conformance/src/filesystem_policy.rs new file mode 100644 index 0000000000..09f6d63559 --- /dev/null +++ b/tests/suites/conformance/src/filesystem_policy.rs @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Filesystem policy suite: Landlock enforcement of a sandbox's filesystem +//! policy. + +use std::collections::HashMap; +use std::time::Duration; + +use testkit::Context; +use testkit::sdk::raw::proto; + +/// A rejected launch must close promptly instead of hanging the exec. +const REJECTED_LAUNCH_TIMEOUT: Duration = Duration::from_secs(20); + +/// The filesystem policy the tests start from, with `extra_read_write` added +/// to the writable paths. It names no device paths; the sandbox adds the ones +/// every process needs, such as `/dev/urandom`. +/// +/// It sets no process identity, so the workload runs as the image's `USER`: +/// Landlock applies whatever the identity, and the test image need not have a +/// particular user. +fn policy(compatibility: &str, extra_read_write: &[&str]) -> proto::SandboxPolicy { + let paths = |paths: &[&str]| paths.iter().map(ToString::to_string).collect::>(); + let mut read_write = paths(&["/sandbox"]); + read_write.extend(paths(extra_read_write)); + read_write.push("/tmp".to_string()); + proto::SandboxPolicy { + version: 1, + filesystem: Some(proto::FilesystemPolicy { + include_workdir: true, + read_only: paths(&["/usr", "/lib", "/etc", "/proc"]), + read_write, + }), + landlock: Some(proto::LandlockPolicy { + compatibility: compatibility.to_string(), + }), + network_policies: HashMap::from([( + "landlock_regression".to_string(), + proto::NetworkPolicyRule { + name: "landlock_regression".to_string(), + endpoints: vec![proto::NetworkEndpoint { + host: "example.com".to_string(), + port: 443, + ..proto::NetworkEndpoint::default() + }], + binaries: vec![proto::NetworkBinary { + path: "/**".to_string(), + }], + }, + )]), + ..proto::SandboxPolicy::default() + } +} + +/// Under `hard_requirement`, the device paths the sandbox adds to the policy +/// don't make Landlock setup fail: processes start, read `/dev/urandom`, and +/// write to a writable path. +#[testkit::test] +async fn hard_requirement_accepts_added_device_paths(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("ld") + .command(["sleep", "infinity"]) + .policy(policy("hard_requirement", &[])) + .create() + .await?; + sandbox.wait_ready().await?; + + let output = sandbox + .exec([ + "sh", + "-c", + "set -eu; \ + bytes=$(head -c 16 /dev/urandom | wc -c); \ + test \"$bytes\" -eq 16; \ + printf landlock-ok > /tmp/landlock-check; \ + test \"$(cat /tmp/landlock-check)\" = landlock-ok", + ]) + .await?; + testkit::ensure!( + output.exit_code == 0, + "the hard_requirement sandbox could not read /dev/urandom or write /tmp: exit {}; stderr: {}", + output.exit_code, + output.stderr + ); + Ok(()) +} + +/// When a workload replaces a policy path, or one of its ancestors, with a +/// symlink, the next launch is rejected rather than following the symlink, +/// in both Landlock compatibility modes. +#[testkit::test] +async fn replaced_policy_path_rejects_next_launch(cx: &mut Context) -> testkit::Result { + const LEAF: &str = "/sandbox/policy-parent/leaf"; + let cases = [ + ("lbp", "best_effort", false), + ("lhp", "hard_requirement", false), + ("lba", "best_effort", true), + ("lha", "hard_requirement", true), + ]; + + let mut sandboxes = Vec::new(); + for (suffix, compatibility, ancestor) in cases { + let sandbox = cx + .sandbox(suffix) + .command(["sleep", "infinity"]) + .policy(policy(compatibility, &[LEAF])) + .create() + .await?; + sandboxes.push((sandbox, compatibility, ancestor)); + } + + for (sandbox, compatibility, ancestor) in &sandboxes { + let case = format!( + "{compatibility}, replacing the {}", + if *ancestor { "ancestor" } else { "path" } + ); + sandbox.wait_ready().await?; + for script in [ + format!("mkdir -p {LEAF}"), + // A normal directory stays usable on the next launch. + format!("echo allowed > {LEAF}/check"), + if *ancestor { + "mv /sandbox/policy-parent /sandbox/original-parent; \ + ln -s /var/tmp /sandbox/policy-parent" + .to_string() + } else { + format!("mv {LEAF} /sandbox/original-leaf; ln -s /var/tmp {LEAF}") + }, + ] { + let output = sandbox.exec(["sh", "-c", script.as_str()]).await?; + testkit::ensure!( + output.exit_code == 0, + "({case}) `{script}` exited {}; stderr: {}", + output.exit_code, + output.stderr + ); + } + + let next = tokio::time::timeout(REJECTED_LAUNCH_TIMEOUT, sandbox.exec(["true"])) + .await + .map_err(|_| { + testkit::Error::msg(format!( + "({case}) the rejected launch did not close within {REJECTED_LAUNCH_TIMEOUT:?}" + )) + })?; + let report = match next { + Ok(output) if output.exit_code == 0 => { + return Err(testkit::Error::msg(format!( + "({case}) a launch succeeded after a policy path was replaced with a symlink" + ))); + } + Ok(output) => output.stderr, + Err(error) => error.to_string(), + }; + testkit::ensure!( + report.to_lowercase().contains("symlink"), + "({case}) the rejected launch does not say a policy path is a symlink: {report}" + ); + } + Ok(()) +} diff --git a/tests/suites/conformance/src/image.rs b/tests/suites/conformance/src/image.rs new file mode 100644 index 0000000000..84d8ea7ed8 --- /dev/null +++ b/tests/suites/conformance/src/image.rs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox image suite: a sandbox runs the image it is created from. + +use testkit::{Context, Sandbox}; + +/// A sandbox created with an explicit image runs that image. +#[testkit::test] +async fn explicit_image_is_used(cx: &mut Context) -> testkit::Result { + let image = testkit::base_image(); + let sandbox = cx + .sandbox("ei") + .command(["sleep", "infinity"]) + .spec(|spec| spec.image = Some(image.clone())) + .create() + .await?; + sandbox.wait_ready().await?; + expect_ubuntu(&sandbox) + .await + .map_err(|error| error.context(format!("sandbox from image {image}"))) +} + +/// Fail unless the sandbox's `/etc/os-release` names Ubuntu and a version, +/// as OpenShell's base image does. +pub(crate) async fn expect_ubuntu(sandbox: &Sandbox) -> testkit::Result { + let output = sandbox.exec(["cat", "/etc/os-release"]).await?; + testkit::ensure!( + output.exit_code == 0, + "reading /etc/os-release exited {}; stderr: {}", + output.exit_code, + output.stderr + ); + let field = |key: &str| { + output.stdout.lines().find_map(|line| { + line.strip_prefix(key) + .and_then(|rest| rest.strip_prefix('=')) + .map(|value| value.trim_matches('"')) + }) + }; + testkit::ensure!( + field("ID") == Some("ubuntu"), + "/etc/os-release has ID {:?}; expected \"ubuntu\":\n{}", + field("ID"), + output.stdout + ); + testkit::ensure!( + field("VERSION_ID").is_some_and(|version| !version.is_empty()), + "/etc/os-release has no VERSION_ID:\n{}", + output.stdout + ); + Ok(()) +} diff --git a/tests/suites/conformance/src/labels.rs b/tests/suites/conformance/src/labels.rs new file mode 100644 index 0000000000..97ffe8f7be --- /dev/null +++ b/tests/suites/conformance/src/labels.rs @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox labels suite: labels are stored and select sandboxes in list. + +use std::collections::{BTreeSet, HashMap}; + +use testkit::Context; +use testkit::sdk::ListOptions; + +/// Labels given at create are returned by get, and list filters by one label +/// or by several (all must match). Label values are unique to the run, so +/// other tests' sandboxes can't match. +#[testkit::test] +async fn labels_are_stored_and_filter_list(cx: &mut Context) -> testkit::Result { + let run = cx.run_id().to_string(); + let dev = format!("dev-{run}"); + let staging = format!("staging-{run}"); + let backend = format!("backend-{run}"); + let data = format!("data-{run}"); + let cases = [ + ("ldb", dev.as_str(), backend.as_str()), + ("lsb", staging.as_str(), backend.as_str()), + ("ldd", dev.as_str(), data.as_str()), + ]; + + let mut names = Vec::new(); + for (suffix, env, team) in cases { + let labels = HashMap::from([ + ("env".to_string(), env.to_string()), + ("team".to_string(), team.to_string()), + ]); + let sandbox = cx + .sandbox(suffix) + .spec(|spec| spec.labels = labels.clone()) + .create() + .await?; + let got = sandbox.get().await?; + testkit::ensure!( + labels + .iter() + .all(|(key, value)| got.labels.get(key) == Some(value)), + "sandbox {} has labels {:?}; expected {labels:?}", + sandbox.name(), + got.labels + ); + names.push(sandbox.name().to_string()); + } + let [dev_backend, staging_backend, dev_data] = names.as_slice() else { + unreachable!("one name per case"); + }; + + let checks = [ + (format!("env={dev}"), vec![dev_backend, dev_data]), + ( + format!("team={backend}"), + vec![dev_backend, staging_backend], + ), + (format!("env={dev},team={backend}"), vec![dev_backend]), + (format!("team=qa-{run}"), vec![]), + ]; + for (selector, expected) in checks { + let listed: BTreeSet = cx + .client() + .list_all_sandboxes(ListOptions { + label_selector: Some(selector.clone()), + ..ListOptions::default() + }) + .await? + .into_iter() + .map(|sandbox| sandbox.name) + .collect(); + let expected: BTreeSet = expected.into_iter().cloned().collect(); + testkit::ensure!( + listed == expected, + "list with selector {selector:?} returned {listed:?}; expected {expected:?}" + ); + } + + let all: BTreeSet = cx + .client() + .list_all_sandboxes(ListOptions::default()) + .await? + .into_iter() + .map(|sandbox| sandbox.name) + .collect(); + let missing: Vec<&String> = names.iter().filter(|name| !all.contains(*name)).collect(); + testkit::ensure!( + missing.is_empty(), + "list without a selector is missing {missing:?}" + ); + Ok(()) +} diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 98b916092f..cabd7f3a3b 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -11,7 +11,13 @@ mod cli; mod credential_gating; mod file_transfer; +mod filesystem_policy; +mod image; +mod labels; mod lifecycle; mod persistent_workspace; mod policy_advisor; +mod process; +mod settings; mod smoke; +mod templates; diff --git a/tests/suites/conformance/src/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs index dd47c0b4d6..ed5cf60ea2 100644 --- a/tests/suites/conformance/src/lifecycle.rs +++ b/tests/suites/conformance/src/lifecycle.rs @@ -9,8 +9,9 @@ use std::time::Duration; -use testkit::Context; -use testkit::sdk::{SandboxPhase, SandboxRestartPolicy}; +use serde_json::Value; +use testkit::sdk::{SandboxPhase, SandboxRestartPolicy, SdkError}; +use testkit::{Context, Poll, Sandbox, poll_until}; /// A long-running sandbox becomes ready, runs a command with exec, and is gone /// from get and list after delete. @@ -42,6 +43,49 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result { cx.expect_absent(&sandbox).await } +/// Longer than the supervisor's 30s startup recovery deadline. +const PAST_RECOVERY_DEADLINE: Duration = Duration::from_secs(31); + +/// An exec admitted before the startup recovery deadline expires keeps +/// running past it, and a new exec still works afterwards. +#[testkit::test] +async fn exec_outlives_startup_recovery_deadline(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("rd") + .command(["sleep", "infinity"]) + .create() + .await?; + sandbox.wait_ready().await?; + + let script = format!( + "sleep {}; printf past-recovery-deadline", + PAST_RECOVERY_DEADLINE.as_secs() + ); + let output = tokio::time::timeout( + Duration::from_mins(1), + sandbox.exec(["sh", "-c", script.as_str()]), + ) + .await + .map_err(|_| testkit::Error::msg("exec past the recovery deadline did not finish in 60s"))??; + testkit::ensure!( + output.exit_code == 0 && output.stdout == "past-recovery-deadline", + "the recovery deadline must not end an admitted exec: exited {} with stdout {:?}; stderr: {}", + output.exit_code, + output.stdout, + output.stderr + ); + + let next = sandbox.exec(["printf", "fresh-exec"]).await?; + testkit::ensure!( + next.exit_code == 0 && next.stdout == "fresh-exec", + "exec after the recovery deadline exited {} with stdout {:?}; stderr: {}", + next.exit_code, + next.stdout, + next.stderr + ); + Ok(()) +} + /// Long enough that the workload is still handling TERM when sandbox shutdown /// begins, and below the supervisor's 5s wait before it kills the workload. const TERM_HANDLING_DELAY: Duration = Duration::from_secs(3); @@ -155,6 +199,74 @@ async fn run_to_completion_exit_delete(cx: &mut Context) -> testkit::Result { cx.expect_absent(&sandbox).await } +/// A main command that exits during provisioning is reported as the +/// workload's result, not as a provisioning failure: the create succeeds and +/// the sandbox reaches Error with the command's exit code. +#[testkit::test] +async fn main_exit_during_provisioning_is_workload_result(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("pe") + .command(["sh", "-c", "exit 11"]) + .restart_policy(SandboxRestartPolicy::Never) + .create() + .await?; + + let finished = sandbox.wait_finished().await?; + testkit::ensure!( + finished.phase == SandboxPhase::Error && finished.exit_code == Some(11), + "sandbox finished in phase {:?} with exit code {:?}; expected Error with Some(11)", + finished.phase, + finished.exit_code + ); + Ok(()) +} + +/// A main command that exits after the sandbox is Ready leaves its result: +/// exit 0 reaches Completed, and a non-zero exit reaches Error with that code. +#[testkit::test] +async fn main_exit_after_ready_sets_final_phase(cx: &mut Context) -> testkit::Result { + const RELEASE_PATH: &str = "/sandbox/.openshell-main-release"; + let cases = [ + ("mz", 0, SandboxPhase::Completed), + ("mn", 11, SandboxPhase::Error), + ]; + + // Create every sandbox before waiting, so they provision concurrently. + let mut sandboxes = Vec::new(); + for (suffix, code, phase) in cases { + let script = + format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; sleep 1; exit {code}"); + let sandbox = cx + .sandbox(suffix) + .command(["sh", "-c", script.as_str()]) + .restart_policy(SandboxRestartPolicy::Never) + .create() + .await?; + sandboxes.push((sandbox, code, phase)); + } + + for (sandbox, code, phase) in &sandboxes { + sandbox.wait_ready().await?; + let release = sandbox.exec(["touch", RELEASE_PATH]).await?; + testkit::ensure!( + release.exit_code == 0, + "releasing the main command of {} exited {}; stderr: {}", + sandbox.name(), + release.exit_code, + release.stderr + ); + let finished = sandbox.wait_finished().await?; + testkit::ensure!( + finished.phase == *phase && finished.exit_code == Some(*code), + "main exiting {code} left sandbox {} in phase {:?} with exit code {:?}; expected {phase:?} with Some({code})", + sandbox.name(), + finished.phase, + finished.exit_code + ); + } + Ok(()) +} + /// A sandbox created with the gateway's default command becomes Ready, is /// listed as Ready, runs a command with exec, and is gone after delete. #[testkit::test] @@ -178,6 +290,8 @@ async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::R output.stdout, output.stderr ); + // The gateway's default image is OpenShell's Ubuntu base image. + crate::image::expect_ubuntu(&sandbox).await?; sandbox.delete().await?; cx.expect_absent(&sandbox).await @@ -235,3 +349,284 @@ async fn stopped_sandbox_can_be_deleted(cx: &mut Context) -> testkit::Result { sandbox.delete().await?; cx.expect_absent(&sandbox).await } + +/// Where the main command of [`canonical_main_environment`] records what it +/// sees. +const MAIN_ENV_PATH: &str = "/sandbox/.openshell-main-env"; + +/// Declared environment reaches both the main command and exec, with and +/// without a TTY. A TTY main command also gets the sandbox user's `HOME` and +/// `USER`, not the supervisor's, and a usable `TERM`. +#[testkit::test] +async fn canonical_main_environment(cx: &mut Context) -> testkit::Result { + let declared = format!("present-{}", cx.run_id()); + let script = format!( + "printf 'declared=%s\\nhome=%s\\nuser=%s\\nterm=%s\\n' \ + \"${{REPRO_SENTINEL:-missing}}\" \"$HOME\" \"$USER\" \"$TERM\" > '{MAIN_ENV_PATH}'; \ + exec sleep infinity" + ); + + let mut sandboxes = Vec::new(); + for (suffix, tty) in [("et", true), ("en", false)] { + let sandbox = cx + .sandbox(suffix) + .command(["sh", "-c", script.as_str()]) + .spec(|spec| { + spec.tty = tty; + spec.environment + .insert("REPRO_SENTINEL".to_string(), declared.clone()); + }) + .create() + .await?; + sandboxes.push((sandbox, tty)); + } + + for (sandbox, tty) in &sandboxes { + sandbox.wait_ready().await?; + let main = main_environment(sandbox).await?; + let field = |name: &str| main.get(name).map_or("", String::as_str); + testkit::ensure!( + field("declared") == declared, + "the main command (tty {tty}) saw REPRO_SENTINEL={:?}; expected {declared:?}", + field("declared") + ); + + let exec = sandbox + .exec(["sh", "-c", "printf '%s' \"${REPRO_SENTINEL:-missing}\""]) + .await?; + testkit::ensure!( + exec.exit_code == 0 && exec.stdout == declared, + "exec (tty {tty}) saw REPRO_SENTINEL={:?}, exit {}; expected {declared:?}; stderr: {}", + exec.stdout, + exec.exit_code, + exec.stderr + ); + + if *tty { + testkit::ensure!( + !field("home").is_empty() && field("home") != "/root", + "the TTY main command inherited the supervisor's HOME: {main:?}" + ); + testkit::ensure!( + !field("user").is_empty(), + "the TTY main command has no USER: {main:?}" + ); + testkit::ensure!( + !field("term").is_empty() && field("term") != "dumb", + "the TTY main command has no usable TERM: {main:?}" + ); + } + } + Ok(()) +} + +/// The `key=value` lines the main command wrote to [`MAIN_ENV_PATH`]. The +/// sandbox can be Ready before the main command writes them. +async fn main_environment( + sandbox: &Sandbox, +) -> testkit::Result> { + let contents = poll_until( + &format!( + "the main command of {} to record its environment", + sandbox.name() + ), + Duration::from_secs(30), + Duration::from_millis(250), + async || { + let output = sandbox.exec(["cat", MAIN_ENV_PATH]).await?; + Ok( + if output.exit_code == 0 && output.stdout.contains("term=") { + Poll::Ready(output.stdout) + } else { + Poll::Pending(format!( + "cat exited {}; stderr: {}", + output.exit_code, output.stderr + )) + }, + ) + }, + ) + .await?; + Ok(contents + .lines() + .filter_map(|line| line.split_once('=')) + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect()) +} + +/// How long a restart under the on-failure policy may take. The runner kills +/// a test after two minutes, so the test's deadlines must add up to less. +const RESTART_TIMEOUT: Duration = Duration::from_secs(60); +/// How long a Ready sandbox's main command may take to write its marker. +const MARKER_TIMEOUT: Duration = Duration::from_secs(20); + +/// Under the on-failure policy, a main command that fails is replaced by a new +/// instance of it. The workspace survives the restart, the restart is counted, +/// and the policy is still reported afterwards. +#[testkit::test] +async fn on_failure_policy_restarts_main_and_keeps_workspace(cx: &mut Context) -> testkit::Result { + const SCRIPT: &str = r#" +marker=/sandbox/.openshell-restart-e2e +if [ -e "$marker" ]; then + printf 'replacement-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/replacement-run + sleep 300 +else + touch "$marker" + printf 'initial-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/initial-run + while [ ! -e /sandbox/.openshell-restart-release ]; do sleep 0.05; done + exit 17 +fi +"#; + let sandbox = cx + .sandbox("of") + .command(["sh", "-c", SCRIPT]) + .restart_policy(SandboxRestartPolicy::OnFailure) + .create() + .await?; + sandbox.wait_ready().await?; + poll_until( + "the initial main command to write its workspace marker", + MARKER_TIMEOUT, + Duration::from_millis(250), + async || { + let output = sandbox.exec(["cat", "/sandbox/initial-run"]).await?; + Ok(if output.stdout.contains("initial-") { + Poll::Ready(()) + } else { + Poll::Pending(format!("cat exited {}", output.exit_code)) + }) + }, + ) + .await?; + + let initial = sandbox_details(cx, &sandbox).await?; + let initial_instance = initial["main_process_instance_id"] + .as_str() + .unwrap_or_default() + .to_string(); + testkit::ensure!( + !initial_instance.is_empty(), + "sandbox get reports no main process instance: {initial}" + ); + let release = sandbox + .exec(["touch", "/sandbox/.openshell-restart-release"]) + .await?; + testkit::ensure!( + release.exit_code == 0, + "releasing the initial main command exited {}; stderr: {}", + release.exit_code, + release.stderr + ); + + let restarted = poll_until( + "the on-failure restart", + RESTART_TIMEOUT, + Duration::from_millis(250), + async || { + let details = sandbox_details(cx, &sandbox).await?; + let instance = details["main_process_instance_id"] + .as_str() + .unwrap_or_default(); + let replaced = !instance.is_empty() && instance != initial_instance; + if details["phase"] != "Ready" || details["restart_count"] != 1 || !replaced { + return Ok(Poll::Pending(format!( + "phase {}, restart count {}, main process instance {instance:?}", + details["phase"], details["restart_count"] + ))); + } + let runs = sandbox + .exec(["cat", "/sandbox/initial-run", "/sandbox/replacement-run"]) + .await?; + Ok( + if runs.stdout.contains("initial-") && runs.stdout.contains("replacement-") { + Poll::Ready(details) + } else { + Poll::Pending(format!( + "restarted, but the workspace has {:?}; stderr: {}", + runs.stdout, runs.stderr + )) + }, + ) + }, + ) + .await?; + testkit::ensure!( + restarted["restart_policy"] == "on-failure", + "the restart policy after the restart is {}; expected \"on-failure\"", + restarted["restart_policy"] + ); + Ok(()) +} + +/// `sandbox get --output json`, which reports the main process instance and +/// restart policy that the SDK does not expose. +async fn sandbox_details(cx: &Context, sandbox: &Sandbox) -> testkit::Result { + cx.cli(["sandbox", "get", sandbox.name(), "--output", "json"]) + .await? + .success()? + .json() +} + +/// How long the gateway may take to remove an ephemeral sandbox after its +/// main command exits. Both sandboxes are released before the test waits, so +/// their removals overlap. +const EPHEMERAL_CLEANUP_TIMEOUT: Duration = Duration::from_secs(60); + +/// A detached sandbox created with `--no-keep` is removed once its main +/// command exits, whether the command succeeds or fails. +#[testkit::test] +async fn detached_no_keep_sandbox_is_removed_after_main_exits(cx: &mut Context) -> testkit::Result { + let mut sandboxes = Vec::new(); + for (suffix, code) in [("ez", 0), ("ef", 17)] { + let name = cx.name(suffix); + let release_path = format!("/sandbox/.ephemeral-release-{name}"); + let script = + format!("while [ ! -e '{release_path}' ]; do sleep 0.1; done; sleep 1; exit {code}"); + cx.track(&name); + cx.cli([ + "sandbox", + "create", + "--name", + &name, + "--no-keep", + "--detach", + "--", + "sh", + "-c", + &script, + ]) + .await? + .success()?; + sandboxes.push((cx.adopt(&name).await?, release_path, code)); + } + + for (sandbox, release_path, _) in &sandboxes { + sandbox.wait_ready().await?; + let release = sandbox.exec(["touch", release_path.as_str()]).await?; + testkit::ensure!( + release.exit_code == 0, + "releasing the main command of {} exited {}; stderr: {}", + sandbox.name(), + release.exit_code, + release.stderr + ); + } + for (sandbox, _, code) in &sandboxes { + poll_until( + &format!( + "ephemeral sandbox {} to be removed after exiting {code}", + sandbox.name() + ), + EPHEMERAL_CLEANUP_TIMEOUT, + Duration::from_millis(500), + async || match cx.client().get_sandbox(sandbox.name()).await { + Err(SdkError::NotFound { .. }) => Ok(Poll::Ready(())), + Err(error) => Err(error.into()), + Ok(found) => Ok(Poll::Pending(format!("phase {:?}", found.phase))), + }, + ) + .await?; + cx.expect_absent(sandbox).await?; + } + Ok(()) +} diff --git a/tests/suites/conformance/src/process.rs b/tests/suites/conformance/src/process.rs new file mode 100644 index 0000000000..109376bc0e --- /dev/null +++ b/tests/suites/conformance/src/process.rs @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox process suite: limits the sandbox applies to its processes. + +use testkit::Context; + +/// Sandbox processes can't write core dumps: both the soft and the hard core +/// file size limits are 0, so a process can't raise its own limit either. +#[testkit::test] +async fn core_dumps_are_disabled(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("cd") + .command(["sleep", "infinity"]) + .create() + .await?; + sandbox.wait_ready().await?; + + let output = sandbox + .exec([ + "sh", + "-c", + "printf '%s %s' \"$(ulimit -c)\" \"$(ulimit -Hc)\"", + ]) + .await?; + testkit::ensure!( + output.exit_code == 0 && output.stdout == "0 0", + "core file size limits (soft hard) are {:?}, exit {}; expected \"0 0\"; stderr: {}", + output.stdout, + output.exit_code, + output.stderr + ); + Ok(()) +} diff --git a/tests/suites/conformance/src/settings.rs b/tests/suites/conformance/src/settings.rs new file mode 100644 index 0000000000..35490f2b38 --- /dev/null +++ b/tests/suites/conformance/src/settings.rs @@ -0,0 +1,196 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Settings suite: sandbox-scoped settings and the gateway-global settings +//! that override them. +//! +//! The global test changes a setting for the whole gateway. That is safe +//! because the runner runs one test at a time; a family that runs tests +//! concurrently needs to give it a test group of its own. + +use std::time::Duration; + +use serde_json::Value; +use testkit::{Context, Poll, Sandbox, poll_until}; + +/// A registered boolean setting that does not change what these tests do. +const KEY: &str = "ocsf_json_enabled"; +/// How the CLI shows a setting with no value. +const UNSET: &str = ""; + +/// With no global value, a sandbox setting starts unset, reads back after it is set, and is unset +/// again after it is deleted. +#[testkit::test] +async fn sandbox_setting_round_trip(cx: &mut Context) -> testkit::Result { + // A global value would hide the sandbox's own, so clear it for the test. + restore_global_setting_on_cleanup(cx, KEY).await?; + cx.cli(["settings", "delete", "--global", "--key", KEY, "--yes"]) + .await? + .success()?; + let sandbox = settings_sandbox(cx, "sr").await?; + expect_sandbox_setting(cx, &sandbox, UNSET, "unset").await?; + + set_sandbox_setting(cx, &sandbox, "true").await?.success()?; + expect_sandbox_setting(cx, &sandbox, "true", "sandbox").await?; + + cx.cli(["settings", "delete", sandbox.name(), "--key", KEY]) + .await? + .success()?; + expect_sandbox_setting(cx, &sandbox, UNSET, "unset").await +} + +/// A global setting overrides the sandbox's value and blocks sandbox writes +/// and deletes for that key. Once the global setting is deleted, the sandbox +/// controls the key again. +#[testkit::test] +async fn global_setting_overrides_sandbox_scope(cx: &mut Context) -> testkit::Result { + restore_global_setting_on_cleanup(cx, KEY).await?; + cx.cli(["settings", "delete", "--global", "--key", KEY, "--yes"]) + .await? + .success()?; + + let sandbox = settings_sandbox(cx, "sg").await?; + set_sandbox_setting(cx, &sandbox, "true").await?.success()?; + expect_sandbox_setting(cx, &sandbox, "true", "sandbox").await?; + + cx.cli([ + "settings", "set", "--global", "--key", KEY, "--value", "false", "--yes", + ]) + .await? + .success()?; + testkit::ensure!( + global_setting(cx, KEY).await? == "false", + "the global {KEY} does not read back as false" + ); + expect_sandbox_setting(cx, &sandbox, "false", "global").await?; + + let blocked = set_sandbox_setting(cx, &sandbox, "true").await?.failure()?; + testkit::ensure!( + blocked.message().contains("managed globally"), + "setting a globally managed key on the sandbox failed without saying the key is managed globally: {}", + blocked.message() + ); + cx.cli(["settings", "delete", sandbox.name(), "--key", KEY]) + .await? + .failure()?; + + cx.cli(["settings", "delete", "--global", "--key", KEY, "--yes"]) + .await? + .success()?; + let global = global_setting(cx, KEY).await?; + testkit::ensure!( + global == UNSET, + "the global {KEY} is {global:?} after delete; expected {UNSET:?}" + ); + + set_sandbox_setting(cx, &sandbox, "false") + .await? + .success()?; + expect_sandbox_setting(cx, &sandbox, "false", "sandbox").await +} + +async fn settings_sandbox(cx: &Context, suffix: &str) -> testkit::Result { + let sandbox = cx + .sandbox(suffix) + .command(["sleep", "infinity"]) + .create() + .await?; + sandbox.wait_ready().await?; + Ok(sandbox) +} + +async fn set_sandbox_setting( + cx: &Context, + sandbox: &Sandbox, + value: &str, +) -> testkit::Result { + cx.cli([ + "settings", + "set", + sandbox.name(), + "--key", + KEY, + "--value", + value, + ]) + .await +} + +/// Wait until `settings get` for the sandbox shows [`KEY`] with `value` from +/// `scope`. +async fn expect_sandbox_setting( + cx: &Context, + sandbox: &Sandbox, + value: &str, + scope: &str, +) -> testkit::Result { + poll_until( + &format!( + "{KEY} on sandbox {} to be {value} ({scope})", + sandbox.name() + ), + Duration::from_secs(10), + Duration::from_secs(1), + async || { + let settings: Value = cx + .cli(["settings", "get", sandbox.name(), "--json"]) + .await? + .success()? + .json()?; + let setting = &settings["settings"][KEY]; + Ok(if setting["value"] == value && setting["scope"] == scope { + Poll::Ready(()) + } else { + Poll::Pending(format!("{KEY} is {setting}")) + }) + }, + ) + .await +} + +/// The gateway-global value of `key` as the CLI shows it, [`UNSET`] if none. +async fn global_setting(cx: &Context, key: &str) -> testkit::Result { + read_global_setting(&cx.cli_handle(), key).await +} + +async fn read_global_setting(cli: &testkit::Cli, key: &str) -> testkit::Result { + let settings: Value = cli + .run(["settings", "get", "--global", "--json"]) + .await? + .success()? + .json()?; + Ok(settings["settings"][key] + .as_str() + .unwrap_or(UNSET) + .to_string()) +} + +/// Record the gateway-global value of `key` and put it back after the test: +/// set it again if it had one, delete it otherwise. +async fn restore_global_setting_on_cleanup(cx: &Context, key: &'static str) -> testkit::Result { + let cli = cx.cli_handle(); + let prior = read_global_setting(&cli, key).await?; + testkit::ensure!( + !prior.starts_with(", + workload: SandboxWorkloadConfig, +) -> SandboxWorkloadTemplate { + SandboxWorkloadTemplate { + metadata: Some(ObjectMeta { + name: name.to_string(), + labels, + ..ObjectMeta::default() + }), + spec: Some(SandboxWorkloadTemplateSpec { + workload: Some(workload), + ..SandboxWorkloadTemplateSpec::default() + }), + } +} + +/// A template stores its labels, environment and resources, is listed, and a +/// sandbox created from it runs with its environment and records which +/// template it came from. +#[testkit::test] +async fn sandbox_from_template_uses_its_workload(cx: &mut Context) -> testkit::Result { + let template_name = cx.name("tp"); + delete_template_on_cleanup(cx, &template_name); + let labels = HashMap::from([("e2e".to_string(), "sandbox-template".to_string())]); + let workload = SandboxWorkloadConfig { + environment: HashMap::from([("FEATURE_FLAG".to_string(), "on".to_string())]), + resources: Some(SandboxResources { + cpu: "500m".to_string(), + memory: "512Mi".to_string(), + ..SandboxResources::default() + }), + ..SandboxWorkloadConfig::default() + }; + cx.client() + .create_sandbox_template(template(&template_name, labels.clone(), workload)) + .await?; + + let stored = cx.client().get_sandbox_template(&template_name).await?; + let meta = stored.metadata.clone().unwrap_or_default(); + testkit::ensure!( + meta.name == template_name && meta.labels.get("e2e") == labels.get("e2e"), + "template get returned name {:?} with labels {:?}; expected {template_name:?} with {labels:?}", + meta.name, + meta.labels + ); + let stored_workload = stored + .spec + .as_ref() + .and_then(|spec| spec.workload.clone()) + .unwrap_or_default(); + let resources = stored_workload.resources.clone().unwrap_or_default(); + testkit::ensure!( + stored_workload + .environment + .get("FEATURE_FLAG") + .map(String::as_str) + == Some("on") + && resources.cpu == "500m" + && resources.memory == "512Mi", + "template get returned workload {stored_workload:?}; expected FEATURE_FLAG=on, cpu 500m, memory 512Mi" + ); + + let listed = cx + .client() + .list_all_sandbox_templates(SandboxTemplateListOptions::default()) + .await?; + testkit::ensure!( + listed.iter().any(|entry| entry + .metadata + .as_ref() + .is_some_and(|m| m.name == template_name)), + "template list does not include {template_name}" + ); + + let sandbox_name = cx.name("ts"); + cx.track(&sandbox_name); + cx.client() + .create_sandbox_from_template(SandboxTemplateCreateSpec { + name: Some(sandbox_name.clone()), + template_name: template_name.clone(), + command: vec!["sleep".to_string(), "infinity".to_string()], + ..SandboxTemplateCreateSpec::default() + }) + .await?; + let sandbox = cx.adopt(&sandbox_name).await?; + let ready = sandbox.wait_ready().await?; + let provenance = ready.created_from_workload_template.as_ref(); + testkit::ensure!( + provenance.is_some_and(|p| p.name == template_name), + "sandbox {sandbox_name} records template {provenance:?}; expected {template_name:?}" + ); + + let output = sandbox + .exec(["sh", "-c", "printf '%s' \"${FEATURE_FLAG:-missing}\""]) + .await?; + testkit::ensure!( + output.exit_code == 0 && output.stdout == "on", + "FEATURE_FLAG in a sandbox from the template is {:?}, exit {}; expected \"on\"; stderr: {}", + output.stdout, + output.exit_code, + output.stderr + ); + Ok(()) +} + +/// Template operations fail with the right status: a duplicate name is +/// `AlreadyExists`, a deleted template is `NotFound`, and creating a sandbox +/// from a missing template is `NotFound`. +#[testkit::test] +async fn template_crud_errors(cx: &mut Context) -> testkit::Result { + let name = cx.name("te"); + delete_template_on_cleanup(cx, &name); + let empty = || template(&name, HashMap::new(), SandboxWorkloadConfig::default()); + + cx.client().create_sandbox_template(empty()).await?; + match cx.client().create_sandbox_template(empty()).await { + Err(SdkError::AlreadyExists { .. }) => {} + other => { + return Err(testkit::Error::msg(format!( + "creating template {name} again returned {other:?}; expected AlreadyExists" + ))); + } + } + + cx.client() + .delete_sandbox_template(&name, DeleteOptions::default()) + .await?; + match cx.client().get_sandbox_template(&name).await { + Err(SdkError::NotFound { .. }) => {} + other => { + return Err(testkit::Error::msg(format!( + "getting deleted template {name} returned {other:?}; expected NotFound" + ))); + } + } + + let sandbox_name = cx.name("tm"); + cx.track(&sandbox_name); + let missing = cx.name("tx"); + match cx + .client() + .create_sandbox_from_template(SandboxTemplateCreateSpec { + name: Some(sandbox_name), + template_name: missing.clone(), + command: vec!["true".to_string()], + ..SandboxTemplateCreateSpec::default() + }) + .await + { + Err(SdkError::NotFound { .. }) => Ok(()), + other => Err(testkit::Error::msg(format!( + "creating a sandbox from missing template {missing} returned {other:?}; expected NotFound" + ))), + } +} diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index 6657ca1da6..57953fc563 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2448,6 +2448,7 @@ dependencies = [ "container-engine-support", "openshell-testkit", "podman-support", + "tempfile", ] [[package]] diff --git a/tests/suites/drivers/podman/Cargo.toml b/tests/suites/drivers/podman/Cargo.toml index 02c0304a1c..6ea6bb4f99 100644 --- a/tests/suites/drivers/podman/Cargo.toml +++ b/tests/suites/drivers/podman/Cargo.toml @@ -16,5 +16,6 @@ publish = false container-engine-support = { path = "../../../support/container-engine" } openshell-testkit = { path = "../../../testkit/testkit" } podman-support = { path = "../../../support/podman" } +tempfile = "3" [workspace] diff --git a/tests/suites/drivers/podman/tests/oci_identity.rs b/tests/suites/drivers/podman/tests/oci_identity.rs new file mode 100644 index 0000000000..412215e888 --- /dev/null +++ b/tests/suites/drivers/podman/tests/oci_identity.rs @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Podman driver supervisor identity. +//! +//! Podman runs a sandbox's supervisor container as the workload's resolved +//! OCI identity with no effective capabilities; Docker runs it as a fixed +//! unprivileged user. The workload side of the same image is covered by the +//! `oci-image` feature suite in `tests/suites/features`. + +use container_engine_support::Engine; +use testkit::{Context, Error, ResultExt as _}; + +const OCI_IDENTITY: &str = "2345:2346"; +/// A complete policy without a `process` section, so the sandbox identity +/// falls back to the image `USER`. +const IMAGE_IDENTITY_POLICY: &str = "version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /etc] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort + +network_policies: {} +"; + +/// The supervisor container runs as the image's numeric `USER`, not root or a +/// fixed user, and has no effective capabilities. +#[testkit::test] +async fn supervisor_runs_as_oci_identity_without_capabilities(cx: &mut Context) -> testkit::Result { + cx.expect_compute_driver("podman").await?; + let engine = Engine::from_env()?; + + let tag = format!("localhost/openshell-test-oci-identity:{}", cx.run_id()); + engine + .build_image( + &tag, + &format!( + "FROM {} +USER 0:0 +RUN mkdir -p /home/app/project && \\ + chown {OCI_IDENTITY} /home/app /home/app/project && \\ + chmod 0700 /home/app /home/app/project +WORKDIR /home/app/project +USER {OCI_IDENTITY} +", + testkit::base_image() + ), + ) + .await?; + // Cleanup runs last registered first, so the sandbox goes before the + // image it uses. + let (cleanup_engine, built) = (engine.clone(), tag.clone()); + cx.on_cleanup(format!("remove image {tag}"), async move || { + cleanup_engine.remove_image(&built).await + }); + let image_user = engine + .output(["image", "inspect", "--format", "{{.Config.User}}", &tag]) + .await?; + testkit::ensure!( + image_user == OCI_IDENTITY, + "the test image has USER {image_user:?}; expected {OCI_IDENTITY}" + ); + + let policy = tempfile::NamedTempFile::new() + .map_err(|error| Error::from_error(&error)) + .context("create policy file")?; + std::fs::write(policy.path(), IMAGE_IDENTITY_POLICY) + .map_err(|error| Error::from_error(&error)) + .context("write policy file")?; + let policy_path = policy + .path() + .to_str() + .ok_or_else(|| Error::msg("policy path is not UTF-8"))?; + + let name = cx.name("oi"); + cx.track(&name); + cx.cli([ + "sandbox", + "create", + "--name", + &name, + "--from", + &tag, + "--policy", + policy_path, + "--detach", + "--", + "sleep", + "infinity", + ]) + .await? + .success() + .context("sandbox starts from the OCI identity image")?; + cx.adopt(&name).await?.wait_ready().await?; + + let supervisor = engine.container_for_role(&name, "supervisor").await?; + let user = engine + .inspect_container(&supervisor, "{{.Config.User}}") + .await?; + testkit::ensure!( + user == OCI_IDENTITY, + "the supervisor container runs as {user:?}; expected the image identity {OCI_IDENTITY}" + ); + let caps = engine + .inspect_container(&supervisor, "{{.EffectiveCaps}}") + .await?; + testkit::ensure!( + caps == "[]", + "the supervisor container has effective capabilities {caps}; expected none" + ); + Ok(()) +} diff --git a/tests/testkit/testkit/src/cli.rs b/tests/testkit/testkit/src/cli.rs index 5ac54e4108..18b598260f 100644 --- a/tests/testkit/testkit/src/cli.rs +++ b/tests/testkit/testkit/src/cli.rs @@ -173,6 +173,17 @@ impl CliOutput { Ok(self) } + /// Stdout and stderr as one line of words, for matching an error report. + /// The CLI wraps reports to the terminal width and draws a gutter, so + /// match against this rather than the raw output. + pub fn message(&self) -> String { + format!("{} {}", self.stdout, self.stderr) + .replace(['│', '×', '╰', '─', '╭', '├'], " ") + .split_whitespace() + .collect::>() + .join(" ") + } + /// Parse stdout as JSON, from a command run with `--output json`. pub fn json(&self) -> Result { serde_json::from_str(&self.stdout) From 61a290103d729f631a368c0ab2c380691d554520 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 15:38:30 -0800 Subject: [PATCH 2/9] test(testkit): add CLI sandbox, exec success, details, wait_absent and test image helpers Add cx.cli_sandbox (with policy files), ExecOutput::success, cx.sandbox_details, cx.wait_absent and Engine::build_test_image, use them in the ported lifecycle tests, and replace the duplicate policy file, sandbox get and test image code in the host-gateway, oci-image and podman suites. Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/cli.rs | 38 ++--- .../conformance/src/filesystem_policy.rs | 26 ++-- tests/suites/conformance/src/image.rs | 8 +- tests/suites/conformance/src/lifecycle.rs | 131 ++++++------------ tests/suites/conformance/src/process.rs | 11 +- tests/suites/conformance/src/templates.rs | 11 +- tests/suites/drivers/podman/Cargo.lock | 1 - tests/suites/drivers/podman/Cargo.toml | 1 - .../drivers/podman/tests/oci_identity.rs | 68 ++++----- tests/suites/drivers/podman/tests/userns.rs | 19 +-- tests/suites/features/Cargo.lock | 1 - tests/suites/features/oci-image/Cargo.toml | 1 - .../features/oci-image/tests/oci_image.rs | 102 ++++---------- tests/support/container-engine/src/lib.rs | 26 +++- tests/testkit/testkit/src/cli_sandbox.rs | 128 +++++++++++++++++ tests/testkit/testkit/src/context.rs | 36 ++++- tests/testkit/testkit/src/lib.rs | 2 + 17 files changed, 313 insertions(+), 297 deletions(-) create mode 100644 tests/testkit/testkit/src/cli_sandbox.rs diff --git a/tests/suites/conformance/src/cli.rs b/tests/suites/conformance/src/cli.rs index d26eed9bb5..5dbef95a23 100644 --- a/tests/suites/conformance/src/cli.rs +++ b/tests/suites/conformance/src/cli.rs @@ -23,18 +23,9 @@ const MAX_PAGES: usize = 1000; async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { cx.cli(["status"]).await?.success()?; - let name = cx.name("cs"); - cx.track(&name); - cx.cli(["sandbox", "create", "--name", &name, "--detach"]) - .await? - .success()?; - - let sandbox = cx.adopt(&name).await?; - let got: Value = cx - .cli(["sandbox", "get", &name, "--output", "json"]) - .await? - .success()? - .json()?; + let sandbox = cx.cli_sandbox("cs").create().await?; + let name = sandbox.name().to_string(); + let got = cx.sandbox_details(&name).await?; testkit::ensure!( got["name"] == name.as_str(), "sandbox get returned {:?}; expected {name:?}", @@ -72,23 +63,12 @@ async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { /// stopped, starts again, and can be stopped and deleted. #[testkit::test] async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { - let name = cx.name("cl"); - cx.track(&name); - cx.cli([ - "sandbox", - "create", - "--name", - &name, - "--detach", - "--no-tty", - "--", - "sh", - "-c", - "exec sleep infinity", - ]) - .await? - .success()?; - let sandbox = cx.adopt(&name).await?; + let sandbox = cx + .cli_sandbox("cl") + .command(["sh", "-c", "exec sleep infinity"]) + .create() + .await?; + let name = sandbox.name().to_string(); sandbox.wait_ready().await?; cx.cli(["sandbox", "stop", &name]).await?.success()?; diff --git a/tests/suites/conformance/src/filesystem_policy.rs b/tests/suites/conformance/src/filesystem_policy.rs index 09f6d63559..ddb95977f8 100644 --- a/tests/suites/conformance/src/filesystem_policy.rs +++ b/tests/suites/conformance/src/filesystem_policy.rs @@ -7,8 +7,8 @@ use std::collections::HashMap; use std::time::Duration; -use testkit::Context; use testkit::sdk::raw::proto; +use testkit::{Context, ResultExt as _}; /// A rejected launch must close promptly instead of hanging the exec. const REJECTED_LAUNCH_TIMEOUT: Duration = Duration::from_secs(20); @@ -66,7 +66,7 @@ async fn hard_requirement_accepts_added_device_paths(cx: &mut Context) -> testki .await?; sandbox.wait_ready().await?; - let output = sandbox + sandbox .exec([ "sh", "-c", @@ -76,13 +76,9 @@ async fn hard_requirement_accepts_added_device_paths(cx: &mut Context) -> testki printf landlock-ok > /tmp/landlock-check; \ test \"$(cat /tmp/landlock-check)\" = landlock-ok", ]) - .await?; - testkit::ensure!( - output.exit_code == 0, - "the hard_requirement sandbox could not read /dev/urandom or write /tmp: exit {}; stderr: {}", - output.exit_code, - output.stderr - ); + .await? + .success() + .context("the hard_requirement sandbox must read /dev/urandom and write /tmp")?; Ok(()) } @@ -128,13 +124,11 @@ async fn replaced_policy_path_rejects_next_launch(cx: &mut Context) -> testkit:: format!("mv {LEAF} /sandbox/original-leaf; ln -s /var/tmp {LEAF}") }, ] { - let output = sandbox.exec(["sh", "-c", script.as_str()]).await?; - testkit::ensure!( - output.exit_code == 0, - "({case}) `{script}` exited {}; stderr: {}", - output.exit_code, - output.stderr - ); + sandbox + .exec(["sh", "-c", script.as_str()]) + .await? + .success() + .context(format!("({case}) prepare the policy path"))?; } let next = tokio::time::timeout(REJECTED_LAUNCH_TIMEOUT, sandbox.exec(["true"])) diff --git a/tests/suites/conformance/src/image.rs b/tests/suites/conformance/src/image.rs index 84d8ea7ed8..36282f8fe4 100644 --- a/tests/suites/conformance/src/image.rs +++ b/tests/suites/conformance/src/image.rs @@ -24,13 +24,7 @@ async fn explicit_image_is_used(cx: &mut Context) -> testkit::Result { /// Fail unless the sandbox's `/etc/os-release` names Ubuntu and a version, /// as OpenShell's base image does. pub(crate) async fn expect_ubuntu(sandbox: &Sandbox) -> testkit::Result { - let output = sandbox.exec(["cat", "/etc/os-release"]).await?; - testkit::ensure!( - output.exit_code == 0, - "reading /etc/os-release exited {}; stderr: {}", - output.exit_code, - output.stderr - ); + let output = sandbox.exec(["cat", "/etc/os-release"]).await?.success()?; let field = |key: &str| { output.stdout.lines().find_map(|line| { line.strip_prefix(key) diff --git a/tests/suites/conformance/src/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs index ed5cf60ea2..b14a424f4a 100644 --- a/tests/suites/conformance/src/lifecycle.rs +++ b/tests/suites/conformance/src/lifecycle.rs @@ -9,9 +9,8 @@ use std::time::Duration; -use serde_json::Value; -use testkit::sdk::{SandboxPhase, SandboxRestartPolicy, SdkError}; -use testkit::{Context, Poll, Sandbox, poll_until}; +use testkit::sdk::{SandboxPhase, SandboxRestartPolicy}; +use testkit::{Context, Poll, ResultExt as _, Sandbox, poll_until}; /// A long-running sandbox becomes ready, runs a command with exec, and is gone /// from get and list after delete. @@ -66,22 +65,24 @@ async fn exec_outlives_startup_recovery_deadline(cx: &mut Context) -> testkit::R sandbox.exec(["sh", "-c", script.as_str()]), ) .await - .map_err(|_| testkit::Error::msg("exec past the recovery deadline did not finish in 60s"))??; + .map_err(|_| testkit::Error::msg("exec past the recovery deadline did not finish in 60s"))?? + .success() + .context("the recovery deadline must not end an admitted exec")?; testkit::ensure!( - output.exit_code == 0 && output.stdout == "past-recovery-deadline", - "the recovery deadline must not end an admitted exec: exited {} with stdout {:?}; stderr: {}", - output.exit_code, - output.stdout, - output.stderr + output.stdout == "past-recovery-deadline", + "the exec past the recovery deadline printed {:?}", + output.stdout ); - let next = sandbox.exec(["printf", "fresh-exec"]).await?; + let next = sandbox + .exec(["printf", "fresh-exec"]) + .await? + .success() + .context("exec after the recovery deadline")?; testkit::ensure!( - next.exit_code == 0 && next.stdout == "fresh-exec", - "exec after the recovery deadline exited {} with stdout {:?}; stderr: {}", - next.exit_code, - next.stdout, - next.stderr + next.stdout == "fresh-exec", + "exec after the recovery deadline printed {:?}", + next.stdout ); Ok(()) } @@ -247,14 +248,7 @@ async fn main_exit_after_ready_sets_final_phase(cx: &mut Context) -> testkit::Re for (sandbox, code, phase) in &sandboxes { sandbox.wait_ready().await?; - let release = sandbox.exec(["touch", RELEASE_PATH]).await?; - testkit::ensure!( - release.exit_code == 0, - "releasing the main command of {} exited {}; stderr: {}", - sandbox.name(), - release.exit_code, - release.stderr - ); + sandbox.exec(["touch", RELEASE_PATH]).await?.success()?; let finished = sandbox.wait_finished().await?; testkit::ensure!( finished.phase == *phase && finished.exit_code == Some(*code), @@ -393,13 +387,12 @@ async fn canonical_main_environment(cx: &mut Context) -> testkit::Result { let exec = sandbox .exec(["sh", "-c", "printf '%s' \"${REPRO_SENTINEL:-missing}\""]) - .await?; + .await? + .success()?; testkit::ensure!( - exec.exit_code == 0 && exec.stdout == declared, - "exec (tty {tty}) saw REPRO_SENTINEL={:?}, exit {}; expected {declared:?}; stderr: {}", - exec.stdout, - exec.exit_code, - exec.stderr + exec.stdout == declared, + "exec (tty {tty}) saw REPRO_SENTINEL={:?}; expected {declared:?}", + exec.stdout ); if *tty { @@ -499,7 +492,7 @@ fi ) .await?; - let initial = sandbox_details(cx, &sandbox).await?; + let initial = cx.sandbox_details(sandbox.name()).await?; let initial_instance = initial["main_process_instance_id"] .as_str() .unwrap_or_default() @@ -508,22 +501,17 @@ fi !initial_instance.is_empty(), "sandbox get reports no main process instance: {initial}" ); - let release = sandbox + sandbox .exec(["touch", "/sandbox/.openshell-restart-release"]) - .await?; - testkit::ensure!( - release.exit_code == 0, - "releasing the initial main command exited {}; stderr: {}", - release.exit_code, - release.stderr - ); + .await? + .success()?; let restarted = poll_until( "the on-failure restart", RESTART_TIMEOUT, Duration::from_millis(250), async || { - let details = sandbox_details(cx, &sandbox).await?; + let details = cx.sandbox_details(sandbox.name()).await?; let instance = details["main_process_instance_id"] .as_str() .unwrap_or_default(); @@ -558,15 +546,6 @@ fi Ok(()) } -/// `sandbox get --output json`, which reports the main process instance and -/// restart policy that the SDK does not expose. -async fn sandbox_details(cx: &Context, sandbox: &Sandbox) -> testkit::Result { - cx.cli(["sandbox", "get", sandbox.name(), "--output", "json"]) - .await? - .success()? - .json() -} - /// How long the gateway may take to remove an ephemeral sandbox after its /// main command exits. Both sandboxes are released before the test waits, so /// their removals overlap. @@ -578,55 +557,29 @@ const EPHEMERAL_CLEANUP_TIMEOUT: Duration = Duration::from_secs(60); async fn detached_no_keep_sandbox_is_removed_after_main_exits(cx: &mut Context) -> testkit::Result { let mut sandboxes = Vec::new(); for (suffix, code) in [("ez", 0), ("ef", 17)] { - let name = cx.name(suffix); - let release_path = format!("/sandbox/.ephemeral-release-{name}"); + let builder = cx.cli_sandbox(suffix); + let release_path = format!("/sandbox/.ephemeral-release-{}", builder.name()); let script = format!("while [ ! -e '{release_path}' ]; do sleep 0.1; done; sleep 1; exit {code}"); - cx.track(&name); - cx.cli([ - "sandbox", - "create", - "--name", - &name, - "--no-keep", - "--detach", - "--", - "sh", - "-c", - &script, - ]) - .await? - .success()?; - sandboxes.push((cx.adopt(&name).await?, release_path, code)); + let sandbox = builder + .args(["--no-keep"]) + .command(["sh", "-c", &script]) + .create() + .await?; + sandboxes.push((sandbox, release_path, code)); } for (sandbox, release_path, _) in &sandboxes { sandbox.wait_ready().await?; - let release = sandbox.exec(["touch", release_path.as_str()]).await?; - testkit::ensure!( - release.exit_code == 0, - "releasing the main command of {} exited {}; stderr: {}", - sandbox.name(), - release.exit_code, - release.stderr - ); + sandbox + .exec(["touch", release_path.as_str()]) + .await? + .success()?; } for (sandbox, _, code) in &sandboxes { - poll_until( - &format!( - "ephemeral sandbox {} to be removed after exiting {code}", - sandbox.name() - ), - EPHEMERAL_CLEANUP_TIMEOUT, - Duration::from_millis(500), - async || match cx.client().get_sandbox(sandbox.name()).await { - Err(SdkError::NotFound { .. }) => Ok(Poll::Ready(())), - Err(error) => Err(error.into()), - Ok(found) => Ok(Poll::Pending(format!("phase {:?}", found.phase))), - }, - ) - .await?; - cx.expect_absent(sandbox).await?; + cx.wait_absent(sandbox, EPHEMERAL_CLEANUP_TIMEOUT) + .await + .context(format!("ephemeral sandbox exited {code}"))?; } Ok(()) } diff --git a/tests/suites/conformance/src/process.rs b/tests/suites/conformance/src/process.rs index 109376bc0e..68cc2dc94b 100644 --- a/tests/suites/conformance/src/process.rs +++ b/tests/suites/conformance/src/process.rs @@ -22,13 +22,12 @@ async fn core_dumps_are_disabled(cx: &mut Context) -> testkit::Result { "-c", "printf '%s %s' \"$(ulimit -c)\" \"$(ulimit -Hc)\"", ]) - .await?; + .await? + .success()?; testkit::ensure!( - output.exit_code == 0 && output.stdout == "0 0", - "core file size limits (soft hard) are {:?}, exit {}; expected \"0 0\"; stderr: {}", - output.stdout, - output.exit_code, - output.stderr + output.stdout == "0 0", + "core file size limits (soft hard) are {:?}; expected \"0 0\"", + output.stdout ); Ok(()) } diff --git a/tests/suites/conformance/src/templates.rs b/tests/suites/conformance/src/templates.rs index b4ba4f98a5..13c2efb674 100644 --- a/tests/suites/conformance/src/templates.rs +++ b/tests/suites/conformance/src/templates.rs @@ -127,13 +127,12 @@ async fn sandbox_from_template_uses_its_workload(cx: &mut Context) -> testkit::R let output = sandbox .exec(["sh", "-c", "printf '%s' \"${FEATURE_FLAG:-missing}\""]) - .await?; + .await? + .success()?; testkit::ensure!( - output.exit_code == 0 && output.stdout == "on", - "FEATURE_FLAG in a sandbox from the template is {:?}, exit {}; expected \"on\"; stderr: {}", - output.stdout, - output.exit_code, - output.stderr + output.stdout == "on", + "FEATURE_FLAG in a sandbox from the template is {:?}; expected \"on\"", + output.stdout ); Ok(()) } diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index 57953fc563..6657ca1da6 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2448,7 +2448,6 @@ dependencies = [ "container-engine-support", "openshell-testkit", "podman-support", - "tempfile", ] [[package]] diff --git a/tests/suites/drivers/podman/Cargo.toml b/tests/suites/drivers/podman/Cargo.toml index 6ea6bb4f99..02c0304a1c 100644 --- a/tests/suites/drivers/podman/Cargo.toml +++ b/tests/suites/drivers/podman/Cargo.toml @@ -16,6 +16,5 @@ publish = false container-engine-support = { path = "../../../support/container-engine" } openshell-testkit = { path = "../../../testkit/testkit" } podman-support = { path = "../../../support/podman" } -tempfile = "3" [workspace] diff --git a/tests/suites/drivers/podman/tests/oci_identity.rs b/tests/suites/drivers/podman/tests/oci_identity.rs index 412215e888..d9d6841ed1 100644 --- a/tests/suites/drivers/podman/tests/oci_identity.rs +++ b/tests/suites/drivers/podman/tests/oci_identity.rs @@ -9,7 +9,7 @@ //! `oci-image` feature suite in `tests/suites/features`. use container_engine_support::Engine; -use testkit::{Context, Error, ResultExt as _}; +use testkit::{Context, ResultExt as _}; const OCI_IDENTITY: &str = "2345:2346"; /// A complete policy without a `process` section, so the sandbox identity @@ -33,10 +33,10 @@ async fn supervisor_runs_as_oci_identity_without_capabilities(cx: &mut Context) cx.expect_compute_driver("podman").await?; let engine = Engine::from_env()?; - let tag = format!("localhost/openshell-test-oci-identity:{}", cx.run_id()); - engine - .build_image( - &tag, + let image = engine + .build_test_image( + cx, + "oci-identity", &format!( "FROM {} USER 0:0 @@ -50,53 +50,33 @@ USER {OCI_IDENTITY} ), ) .await?; - // Cleanup runs last registered first, so the sandbox goes before the - // image it uses. - let (cleanup_engine, built) = (engine.clone(), tag.clone()); - cx.on_cleanup(format!("remove image {tag}"), async move || { - cleanup_engine.remove_image(&built).await - }); let image_user = engine - .output(["image", "inspect", "--format", "{{.Config.User}}", &tag]) + .output([ + "image", + "inspect", + "--format", + "{{.Config.User}}", + &image.tag, + ]) .await?; testkit::ensure!( image_user == OCI_IDENTITY, "the test image has USER {image_user:?}; expected {OCI_IDENTITY}" ); - let policy = tempfile::NamedTempFile::new() - .map_err(|error| Error::from_error(&error)) - .context("create policy file")?; - std::fs::write(policy.path(), IMAGE_IDENTITY_POLICY) - .map_err(|error| Error::from_error(&error)) - .context("write policy file")?; - let policy_path = policy - .path() - .to_str() - .ok_or_else(|| Error::msg("policy path is not UTF-8"))?; + let sandbox = cx + .cli_sandbox("oi") + .image(&image.tag) + .policy(IMAGE_IDENTITY_POLICY) + .command(["sleep", "infinity"]) + .create() + .await + .context("sandbox starts from the OCI identity image")?; + sandbox.wait_ready().await?; - let name = cx.name("oi"); - cx.track(&name); - cx.cli([ - "sandbox", - "create", - "--name", - &name, - "--from", - &tag, - "--policy", - policy_path, - "--detach", - "--", - "sleep", - "infinity", - ]) - .await? - .success() - .context("sandbox starts from the OCI identity image")?; - cx.adopt(&name).await?.wait_ready().await?; - - let supervisor = engine.container_for_role(&name, "supervisor").await?; + let supervisor = engine + .container_for_role(sandbox.name(), "supervisor") + .await?; let user = engine .inspect_container(&supervisor, "{{.Config.User}}") .await?; diff --git a/tests/suites/drivers/podman/tests/userns.rs b/tests/suites/drivers/podman/tests/userns.rs index 5dff4a8a44..545554bfff 100644 --- a/tests/suites/drivers/podman/tests/userns.rs +++ b/tests/suites/drivers/podman/tests/userns.rs @@ -37,22 +37,11 @@ async fn configured_userns_matches_podman_reference(cx: &mut Context) -> testkit let image = testkit::base_image(); let reference = reference_uid_map(&engine, userns, &image).await?; - let name = cx.name("pu"); - cx.track(&name); let run = cx - .cli([ - "sandbox", - "create", - "--name", - &name, - "--from", - &image, - "--no-tty", - "--", - "sh", - "-c", - WORKSPACE_AND_UID_MAP_PROBE, - ]) + .cli_sandbox("pu") + .image(&image) + .command(["sh", "-c", WORKSPACE_AND_UID_MAP_PROBE]) + .run() .await? .success() .context("sandbox can write to its owned workspace and prints its UID map")?; diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index bcc428d57d..0059ebdf31 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2131,7 +2131,6 @@ version = "0.0.0" dependencies = [ "container-engine-support", "openshell-testkit", - "serde_json", "tempfile", ] diff --git a/tests/suites/features/oci-image/Cargo.toml b/tests/suites/features/oci-image/Cargo.toml index b966d2c38d..4298cd0686 100644 --- a/tests/suites/features/oci-image/Cargo.toml +++ b/tests/suites/features/oci-image/Cargo.toml @@ -14,5 +14,4 @@ publish = false [dependencies] container-engine-support = { path = "../../../support/container-engine" } openshell-testkit = { path = "../../../testkit/testkit" } -serde_json = "1" tempfile = "3" diff --git a/tests/suites/features/oci-image/tests/oci_image.rs b/tests/suites/features/oci-image/tests/oci_image.rs index 4206c1ccb5..287e44c96b 100644 --- a/tests/suites/features/oci-image/tests/oci_image.rs +++ b/tests/suites/features/oci-image/tests/oci_image.rs @@ -11,7 +11,6 @@ //! driver created, through the same engine. use container_engine_support::{Engine, Image}; -use serde_json::Value; use testkit::{Context, Error, Result, ResultExt as _}; /// Sandbox condition reason for an image `WORKDIR` the sandbox identity @@ -87,10 +86,9 @@ USER 2345:2346 ), ) .await?; - let policy = PolicyFile::new(IMAGE_IDENTITY_POLICY)?; let checks = workspace_checks("2345:2346", "/home/app/project", true); let sandbox = oci - .create_sandbox("pu", &image, Some(policy.path()?), &checks) + .create_sandbox("pu", &image, Some(IMAGE_IDENTITY_POLICY), &checks) .await?; oci.container_checks(&sandbox, &image, "2345:2346").await }) @@ -106,13 +104,12 @@ async fn userless_image_uses_default_identity(cx: &mut Context) -> testkit::Resu let image = oci .build_image("userless", &format!("FROM {}\n", oci.base_image)) .await?; - let policy = PolicyFile::new(IMAGE_IDENTITY_POLICY)?; let checks = format!( "{} test \"$(id -G)\" = 1000;", workspace_checks("1000:1000", "/sandbox", false) ); let sandbox = oci - .create_sandbox("ud", &image, Some(policy.path()?), &checks) + .create_sandbox("ud", &image, Some(IMAGE_IDENTITY_POLICY), &checks) .await?; oci.container_checks(&sandbox, &image, "1000:1000").await }) @@ -137,19 +134,19 @@ USER root ), ) .await?; - let policy = PolicyFile::new(&format!( + let policy = format!( "{IMAGE_IDENTITY_POLICY} process: run_as_user: \"4234\" run_as_group: \"4235\" " - ))?; + ); let checks = format!( "{} test \"$(id -G)\" = 4235;", workspace_checks("4234:4235", "/sandbox", false) ); let sandbox = oci - .create_sandbox("pr", &image, Some(policy.path()?), &checks) + .create_sandbox("pr", &image, Some(&policy), &checks) .await?; oci.container_checks(&sandbox, &image, "4234:4235").await }) @@ -198,22 +195,12 @@ USER app ), ) .await?; - let name = oci.track(suffix); - let create = oci - .cx - .cli([ - "sandbox", - "create", - "--name", - &name, - "--from", - &image.tag, - "--no-tty", - "--", - "sh", - "-c", - "echo workspace-access-marker", - ]) + let builder = oci.cx.cli_sandbox(suffix); + let name = builder.name().to_string(); + let create = builder + .image(&image.tag) + .command(["sh", "-c", "echo workspace-access-marker"]) + .run() .await?; if writable { let create = create @@ -238,12 +225,7 @@ USER app ); // Require the condition reason, so another provisioning failure // cannot pass as a workspace rejection. - let sandbox: Value = oci - .cx - .cli(["sandbox", "get", &name, "--output", "json"]) - .await? - .success()? - .json()?; + let sandbox = oci.cx.sandbox_details(&name).await?; let reasons: Vec<&str> = sandbox["conditions"] .as_array() .into_iter() @@ -278,27 +260,12 @@ async fn run(cx: &Context, test: impl AsyncFnOnce(&Oci<'_>) -> Result) -> Result } impl Oci<'_> { - /// Build `containerfile` with a tag unique to this run, and remove the - /// image when the test ends. Cleanup runs last registered first, so the - /// sandboxes created from the image are deleted before it is removed. + /// Build `containerfile` as a test image named `oci-`, removed when + /// the test ends. async fn build_image(&self, name: &str, containerfile: &str) -> Result { - let tag = format!("localhost/openshell-test-oci-{name}:{}", self.cx.run_id()); - self.engine.build_image(&tag, containerfile).await?; - let (engine, built) = (self.engine.clone(), tag.clone()); - self.cx - .on_cleanup(format!("remove image {tag}"), async move || { - engine.remove_image(&built).await - }); - self.engine.image(&tag).await - } - - /// Name a sandbox for this test and register it for cleanup. Sandbox - /// names are limited to 19 characters on some drivers, so keep `short` - /// to a few characters. - fn track(&self, short: &str) -> String { - let name = self.cx.name(short); - self.cx.track(&name); - name + self.engine + .build_test_image(self.cx, &format!("oci-{name}"), containerfile) + .await } /// Create a detached sandbox whose main process runs `checks` and writes @@ -311,21 +278,21 @@ impl Oci<'_> { policy: Option<&str>, checks: &str, ) -> Result { - let name = self.track(short); let main = format!( "(set -eu; {checks} touch main-write) >/tmp/oci-main.log 2>&1; \ echo $? >/tmp/oci-main.status; exec sleep infinity" ); - let mut args = vec!["sandbox", "create", "--name", &name, "--from", &image.tag]; + let mut builder = self.cx.cli_sandbox(short).image(&image.tag); if let Some(policy) = policy { - args.extend(["--policy", policy]); + builder = builder.policy(policy); } - args.extend(["--detach", "--", "sh", "-c", &main]); - self.cx - .cli(args) - .await? - .success() - .context("sandbox starts from the test image")?; + let name = builder + .command(["sh", "-c", &main]) + .create() + .await + .context("sandbox starts from the test image")? + .name() + .to_string(); let script = format!( "set -eu; i=0; \ @@ -527,23 +494,6 @@ fn workspace_checks(identity: &str, workspace: &str, image_file: bool) -> String checks } -/// A sandbox policy written to a temporary file for `sandbox create --policy`. -struct PolicyFile(tempfile::NamedTempFile); - -impl PolicyFile { - fn new(contents: &str) -> Result { - let file = tempfile::NamedTempFile::new() - .map_err(|error| Error::from_error(&error)) - .context("create policy file")?; - write(file.path(), contents)?; - Ok(Self(file)) - } - - fn path(&self) -> Result<&str> { - utf8(self.0.path()) - } -} - fn write(path: &std::path::Path, contents: &str) -> Result { std::fs::write(path, contents) .map_err(|error| Error::from_error(&error)) diff --git a/tests/support/container-engine/src/lib.rs b/tests/support/container-engine/src/lib.rs index f380a9815f..861e0848ff 100644 --- a/tests/support/container-engine/src/lib.rs +++ b/tests/support/container-engine/src/lib.rs @@ -20,7 +20,7 @@ use std::path::Path; -use testkit::{Error, Result, ResultExt as _}; +use testkit::{Context, Error, Result, ResultExt as _}; use tokio::process::Command; pub const ENGINE_ENV: &str = "OPENSHELL_TEST_CONTAINER_ENGINE"; @@ -64,9 +64,8 @@ impl Engine { Ok(Self { command }) } - /// Build `containerfile` as `tag`. The build context is empty. Remove - /// the image with [`Engine::remove_image`], for example in a cleanup - /// hook registered right after the build. + /// Build `containerfile` as `tag`. The build context is empty. Prefer + /// [`Engine::build_test_image`], which also removes the image. pub async fn build_image(&self, tag: &str, containerfile: &str) -> Result { let context = tempfile::tempdir() .map_err(|error| Error::from_error(&error)) @@ -87,6 +86,25 @@ impl Engine { .map(drop) } + /// Build `containerfile` as `localhost/openshell-test-:` + /// and remove the image when the test ends. Cleanup runs last registered + /// first, so sandboxes created from the image afterwards are deleted + /// before it is removed. + pub async fn build_test_image( + &self, + cx: &Context, + name: &str, + containerfile: &str, + ) -> Result { + let tag = format!("localhost/openshell-test-{name}:{}", cx.run_id()); + self.build_image(&tag, containerfile).await?; + let (engine, built) = (self.clone(), tag.clone()); + cx.on_cleanup(format!("remove image {tag}"), async move || { + engine.remove_image(&built).await + }); + self.image(&tag).await + } + /// Image `tag` and the ID it has now. pub async fn image(&self, tag: &str) -> Result { let id = self diff --git a/tests/testkit/testkit/src/cli_sandbox.rs b/tests/testkit/testkit/src/cli_sandbox.rs new file mode 100644 index 0000000000..858e27cec8 --- /dev/null +++ b/tests/testkit/testkit/src/cli_sandbox.rs @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandboxes created through `openshell sandbox create`, for tests of what +//! only the CLI offers, such as `--policy` files, `--no-keep`, or an attached +//! main command. + +use crate::cli::CliOutput; +use crate::context::{Context, Sandbox}; +use crate::error::{Error, Result, ResultExt}; + +/// A sandbox to create with `openshell sandbox create`. Start one with +/// [`Context::cli_sandbox`]. +/// +/// The sandbox is tracked for cleanup before the CLI runs, so a create the +/// gateway accepted but the CLI reported as failed is still cleaned up. The +/// CLI runs with `--no-tty`: tests never have a terminal. +pub struct CliSandboxBuilder<'a> { + cx: &'a Context, + name: String, + args: Vec, + policy: Option, + command: Vec, +} + +impl<'a> CliSandboxBuilder<'a> { + pub(crate) fn new(cx: &'a Context, name: String) -> Self { + Self { + cx, + name, + args: Vec::new(), + policy: None, + command: Vec::new(), + } + } + + /// The sandbox's name, for building the command before the create. + pub fn name(&self) -> &str { + &self.name + } + + /// Create the sandbox from `image` (`--from`). + #[must_use] + pub fn image(self, image: &str) -> Self { + self.args(["--from", image]) + } + + /// Create the sandbox with this policy YAML (`--policy`). It is written + /// to a temporary file that lives until the CLI exits. + #[must_use] + pub fn policy(mut self, yaml: &str) -> Self { + self.policy = Some(yaml.to_string()); + self + } + + /// Other `sandbox create` arguments, such as `--no-keep` or `--label`. + #[must_use] + pub fn args(mut self, args: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.args.extend(args.into_iter().map(Into::into)); + self + } + + /// The sandbox's main command. Empty selects the gateway's default. + #[must_use] + pub fn command(mut self, command: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.command = command.into_iter().map(Into::into).collect(); + self + } + + /// Create the sandbox with `--detach`, require the CLI to succeed, and + /// return the sandbox. It need not be Ready yet. + pub async fn create(mut self) -> Result { + self.args.push("--detach".to_string()); + let (cx, name) = (self.cx, self.name.clone()); + self.run() + .await? + .success() + .context(format!("create sandbox {name}"))?; + cx.adopt(&name).await + } + + /// Create the sandbox attached to its main command and return the CLI's + /// output, whatever its exit code. For a test of the main command's + /// result, or of a create that must fail. + pub async fn run(self) -> Result { + self.cx.track(&self.name); + let policy = self.policy.as_deref().map(PolicyFile::new).transpose()?; + let mut args = vec!["sandbox", "create", "--name", &self.name, "--no-tty"]; + if let Some(policy) = &policy { + args.extend(["--policy", policy.path()?]); + } + args.extend(self.args.iter().map(String::as_str)); + if !self.command.is_empty() { + args.push("--"); + args.extend(self.command.iter().map(String::as_str)); + } + self.cx.cli(args).await + } +} + +/// A policy written to a temporary file for `sandbox create --policy`. +struct PolicyFile(tempfile::NamedTempFile); + +impl PolicyFile { + fn new(contents: &str) -> Result { + let file = tempfile::NamedTempFile::new() + .map_err(|error| Error::from_error(&error)) + .context("create policy file")?; + std::fs::write(file.path(), contents) + .map_err(|error| Error::from_error(&error)) + .context(format!("write {}", file.path().display()))?; + Ok(Self(file)) + } + + fn path(&self) -> Result<&str> { + let path = self.0.path(); + path.to_str() + .ok_or_else(|| Error::msg(format!("{} is not UTF-8", path.display()))) + } +} diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 7f09ad7397..f176c1c936 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -14,6 +14,7 @@ use rand::Rng; use crate::cleanup::{CLEANUP_TIMEOUT, Cleanup, Key}; use crate::cli::{Cli, CliOutput}; +use crate::cli_sandbox::CliSandboxBuilder; use crate::error::{Error, Result, ResultExt}; use crate::poll::{Poll, poll_until}; @@ -113,6 +114,24 @@ impl Context { } } + /// Start describing a sandbox named `cx.name(suffix)` that the + /// `openshell` CLI creates. Use it for what only `sandbox create` + /// offers, such as a `--policy` file or `--no-keep`; prefer + /// [`Context::sandbox`] otherwise. + pub fn cli_sandbox(&self, suffix: &str) -> CliSandboxBuilder<'_> { + CliSandboxBuilder::new(self, self.name(suffix)) + } + + /// `sandbox get --output json` for sandbox `name`. It reports fields the + /// SDK does not expose, such as `restart_count`, `restart_policy`, + /// `main_process_instance_id`, and `conditions`. + pub async fn sandbox_details(&self, name: &str) -> Result { + self.cli(["sandbox", "get", name, "--output", "json"]) + .await? + .success()? + .json() + } + /// Track a sandbox the test is about to create through the CLI, so cleanup /// deletes it even if the test fails after the gateway accepted the create. /// Call this before the CLI command; use [`Context::adopt`] afterwards. @@ -179,6 +198,17 @@ impl Context { }) } + /// Wait up to `timeout` for the gateway to remove the sandbox by itself, + /// for example after a `--no-keep` main command exits, then check it is + /// gone from both get and list. + pub async fn wait_absent(&self, sandbox: &Sandbox, timeout: Duration) -> Result { + self.client + .wait_deleted(sandbox.name(), timeout, Some(sandbox.id())) + .await + .context(format!("wait for sandbox {} to be removed", sandbox.name()))?; + self.expect_absent(sandbox).await + } + /// Fail unless the sandbox is gone from both get and list. pub async fn expect_absent(&self, sandbox: &Sandbox) -> Result { let name = sandbox.name(); @@ -446,6 +476,8 @@ pub struct Sandbox { /// The buffered output of an exec. #[derive(Debug)] pub struct ExecOutput { + /// The command and the sandbox it ran in, for reports. + pub command: String, pub exit_code: i32, pub stdout: String, pub stderr: String, @@ -456,7 +488,8 @@ impl ExecOutput { pub fn success(self) -> Result { crate::ensure!( self.exit_code == 0, - "exec exited with {}; stdout: {}; stderr: {}", + "{} exited with {}; stdout: {}; stderr: {}", + self.command, self.exit_code, self.stdout.trim(), self.stderr.trim() @@ -625,6 +658,7 @@ impl Sandbox { .await .context(format!("exec {command:?} in sandbox {}", self.name))?; Ok(ExecOutput { + command: format!("exec {command:?} in sandbox {}", self.name), exit_code: result.exit_code, stdout: String::from_utf8_lossy(&result.stdout).into_owned(), stderr: String::from_utf8_lossy(&result.stderr).into_owned(), diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 8520fb80b3..83a4e426b8 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -28,6 +28,7 @@ mod cleanup; mod cli; +mod cli_sandbox; mod context; mod error; pub mod gateway; @@ -43,6 +44,7 @@ mod yaml_file; pub use cleanup::CLEANUP_TIMEOUT; pub use cli::{CLI_TIMEOUT, Cli, CliOutput}; +pub use cli_sandbox::CliSandboxBuilder; pub use context::{ Context, DELETE_TIMEOUT, EXEC_TIMEOUT, ExecOutput, READY_TIMEOUT, STOP_TIMEOUT, Sandbox, SandboxBuilder, From f32859f52b1034021a86ce04993fa1787a78edea Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 15:46:53 -0800 Subject: [PATCH 3/9] test(conformance): use ExecOutput::success for exec checks Replace hand-written exit-code checks in the smoke, lifecycle, persistent_workspace, file_transfer and policy_advisor suites. Polls and checks that expect a failure keep reading the exit code. Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/file_transfer.rs | 13 +++--- tests/suites/conformance/src/lifecycle.rs | 31 ++++++------- .../conformance/src/persistent_workspace.rs | 14 +++--- .../suites/conformance/src/policy_advisor.rs | 44 +++++++++---------- tests/suites/conformance/src/smoke.rs | 8 +--- 5 files changed, 45 insertions(+), 65 deletions(-) diff --git a/tests/suites/conformance/src/file_transfer.rs b/tests/suites/conformance/src/file_transfer.rs index 6525595ffe..3bcf30fe36 100644 --- a/tests/suites/conformance/src/file_transfer.rs +++ b/tests/suites/conformance/src/file_transfer.rs @@ -169,14 +169,11 @@ impl<'a> Transfer<'a> { /// Run a shell script in the sandbox and require it to succeed. async fn sh(&self, script: &str) -> testkit::Result { - let output = self.sandbox.exec(["sh", "-c", script]).await?; - testkit::ensure!( - output.exit_code == 0, - "`{script}` exited with {}; stderr: {}", - output.exit_code, - output.stderr - ); - Ok(()) + self.sandbox + .exec(["sh", "-c", script]) + .await? + .success() + .map(drop) } async fn upload( diff --git a/tests/suites/conformance/src/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs index b14a424f4a..e07732ecf7 100644 --- a/tests/suites/conformance/src/lifecycle.rs +++ b/tests/suites/conformance/src/lifecycle.rs @@ -24,13 +24,7 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result { sandbox.wait_ready().await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?; - testkit::ensure!( - output.exit_code == 0, - "exec exited with {}; stderr: {}", - output.exit_code, - output.stderr - ); + let output = sandbox.exec(["echo", marker.as_str()]).await?.success()?; testkit::ensure!( output.stdout == format!("{marker}\n"), "exec stdout was {:?}; expected {:?}", @@ -276,13 +270,12 @@ async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::R cx.expect_listed(&sandbox, SandboxPhase::Ready).await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?; + let output = sandbox.exec(["echo", marker.as_str()]).await?.success()?; testkit::ensure!( - output.exit_code == 0 && output.stdout == format!("{marker}\n"), - "exec exited {} with stdout {:?}; stderr: {}", - output.exit_code, + output.stdout == format!("{marker}\n"), + "exec stdout was {:?}; expected {:?}", output.stdout, - output.stderr + format!("{marker}\n") ); // The gateway's default image is OpenShell's Ubuntu base image. crate::image::expect_ubuntu(&sandbox).await?; @@ -319,13 +312,15 @@ async fn restart_stopped_sandbox(cx: &mut Context) -> testkit::Result { ); sandbox.start().await?; - let output = sandbox.exec(["echo", "started"]).await?; + let output = sandbox + .exec(["echo", "started"]) + .await? + .success() + .context("exec after start")?; testkit::ensure!( - output.exit_code == 0 && output.stdout == "started\n", - "exec after start exited {} with stdout {:?}; stderr: {}", - output.exit_code, - output.stdout, - output.stderr + output.stdout == "started\n", + "exec after start printed {:?}; expected \"started\\n\"", + output.stdout ); Ok(()) } diff --git a/tests/suites/conformance/src/persistent_workspace.rs b/tests/suites/conformance/src/persistent_workspace.rs index de2940e571..a8b2eadfb3 100644 --- a/tests/suites/conformance/src/persistent_workspace.rs +++ b/tests/suites/conformance/src/persistent_workspace.rs @@ -10,7 +10,7 @@ use std::time::Duration; -use testkit::{Context, Poll}; +use testkit::{Context, Poll, ResultExt as _}; /// Files in the workspace survive stop and start, and start runs the main /// command again. @@ -31,19 +31,15 @@ async fn persistent_workspace_survives_stop_start(cx: &mut Context) -> testkit:: sandbox.wait_ready().await?; let sentinel = format!("openshell-stop-start-{}", cx.run_id()); - let write = sandbox + sandbox .exec([ "sh", "-c", &format!("printf '%s\\n' '{sentinel}' > '{SENTINEL}' && sync"), ]) - .await?; - testkit::ensure!( - write.exit_code == 0, - "writing the sentinel exited {}; stderr: {}", - write.exit_code, - write.stderr - ); + .await? + .success() + .context("write the sentinel")?; sandbox.stop().await?; sandbox.start().await?; diff --git a/tests/suites/conformance/src/policy_advisor.rs b/tests/suites/conformance/src/policy_advisor.rs index 003917163c..49ee54ce9d 100644 --- a/tests/suites/conformance/src/policy_advisor.rs +++ b/tests/suites/conformance/src/policy_advisor.rs @@ -13,7 +13,7 @@ use std::time::{Duration, Instant}; use serde_json::Value; use testkit::sdk::raw::proto; -use testkit::{Context, Error, Sandbox}; +use testkit::{Context, Error, ResultExt as _, Sandbox}; const READY_TIMEOUT: Duration = Duration::from_mins(4); const PROPOSAL_TIMEOUT: Duration = Duration::from_secs(90); @@ -161,12 +161,13 @@ async fn denied_address_produces_draft_rule(cx: &mut Context) -> testkit::Result "printf 'BINARY=%s\\n' \"$(readlink -f /proc/$$/exe)\"; \ if exec 3<>/dev/tcp/1.1.1.1/443; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi", ]) - .await?; + .await? + .success() + .context("the connection to 1.1.1.1:443 must be denied")?; testkit::ensure!( - probe.exit_code == 0 && probe.stdout.lines().any(|line| line == "DENIED"), - "the connection to 1.1.1.1:443 was not denied: stdout {:?}, stderr {:?}", - probe.stdout, - probe.stderr + probe.stdout.lines().any(|line| line == "DENIED"), + "the connection to 1.1.1.1:443 was not denied: stdout {:?}", + probe.stdout ); let binary = probe .stdout @@ -199,12 +200,13 @@ async fn denied_hostname_produces_draft_rule(cx: &mut Context) -> testkit::Resul "-c", "if exec 3<>/dev/tcp/pypi.org/80; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi", ]) - .await?; + .await? + .success() + .context("the connection to pypi.org:80 must be denied")?; testkit::ensure!( - probe.exit_code == 0 && probe.stdout.lines().any(|line| line == "DENIED"), - "the connection to pypi.org:80 was not denied: stdout {:?}, stderr {:?}", - probe.stdout, - probe.stderr + probe.stdout.lines().any(|line| line == "DENIED"), + "the connection to pypi.org:80 was not denied: stdout {:?}", + probe.stdout ); expect_draft( @@ -289,13 +291,13 @@ async fn bash_path(sandbox: &Sandbox) -> testkit::Result { "-c", "printf '%s\\n' \"$(readlink -f /proc/$$/exe)\"", ]) - .await?; + .await? + .success()?; let path = output.stdout.trim(); testkit::ensure!( - output.exit_code == 0 && is_bash_path(path), - "expected one absolute Bash path; got {:?}, stderr {:?}", - output.stdout, - output.stderr + is_bash_path(path), + "expected one absolute Bash path; got {:?}", + output.stdout ); Ok(path.to_string()) } @@ -326,13 +328,9 @@ async fn policy_local( path, body, ]) - .await?; - testkit::ensure!( - output.exit_code == 0, - "{method} http://policy.local{path} exited {}; stderr: {}", - output.exit_code, - output.stderr - ); + .await? + .success() + .context(format!("{method} http://policy.local{path}"))?; let (headers, json) = output.stdout.split_once("\r\n\r\n").ok_or_else(|| { Error::msg(format!( "{method} http://policy.local{path} returned no complete HTTP response: {:?}", diff --git a/tests/suites/conformance/src/smoke.rs b/tests/suites/conformance/src/smoke.rs index d7255d44ea..0f39bff143 100644 --- a/tests/suites/conformance/src/smoke.rs +++ b/tests/suites/conformance/src/smoke.rs @@ -17,13 +17,7 @@ async fn create_exec_delete(cx: &mut Context) -> testkit::Result { sandbox.wait_ready().await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?; - testkit::ensure!( - output.exit_code == 0, - "exec exited with {}; stderr: {}", - output.exit_code, - output.stderr - ); + let output = sandbox.exec(["echo", marker.as_str()]).await?.success()?; testkit::ensure!( output.stdout == format!("{marker}\n"), "exec stdout was {:?}; expected {:?}", From 37ad549d019cbdcb824faa19c30bf99732ce239e Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 20:33:42 -0800 Subject: [PATCH 4/9] test(testkit): add create_ready, expect_stdout, poll_exec and deadline helpers Add SandboxBuilder::create_ready, ExecOutput::expect_stdout, Sandbox::poll_exec, Context::delete_and_expect_absent and testkit::with_deadline, and convert std::io::Error into testkit::Error so I/O results take .context directly. Use them across the conformance suite, which also gets shared template, global-setting and label-list helpers and a split on-failure restart poll. Make Engine::build_image private so callers go through build_test_image, which removes the image. Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/file_transfer.rs | 34 +-- .../conformance/src/filesystem_policy.rs | 20 +- tests/suites/conformance/src/image.rs | 3 +- tests/suites/conformance/src/labels.rs | 57 ++-- tests/suites/conformance/src/lifecycle.rs | 263 +++++++----------- .../conformance/src/persistent_workspace.rs | 39 +-- .../suites/conformance/src/policy_advisor.rs | 3 +- tests/suites/conformance/src/process.rs | 15 +- tests/suites/conformance/src/settings.rs | 80 +++--- tests/suites/conformance/src/smoke.rs | 17 +- tests/suites/conformance/src/templates.rs | 124 ++++----- .../tests/provider_refresh.rs | 4 +- .../features/oci-image/tests/oci_image.rs | 16 +- tests/support/container-engine/src/lib.rs | 14 +- tests/support/keycloak/src/lib.rs | 9 +- tests/testkit/testkit/src/cli.rs | 7 +- tests/testkit/testkit/src/cli_sandbox.rs | 12 +- tests/testkit/testkit/src/context.rs | 61 +++- tests/testkit/testkit/src/error.rs | 6 + tests/testkit/testkit/src/lib.rs | 2 +- tests/testkit/testkit/src/poll.rs | 12 + 21 files changed, 368 insertions(+), 430 deletions(-) diff --git a/tests/suites/conformance/src/file_transfer.rs b/tests/suites/conformance/src/file_transfer.rs index 3bcf30fe36..7e04dbe27d 100644 --- a/tests/suites/conformance/src/file_transfer.rs +++ b/tests/suites/conformance/src/file_transfer.rs @@ -15,7 +15,7 @@ use std::os::unix::fs::{PermissionsExt, symlink}; use std::path::{Path, PathBuf}; use tempfile::TempDir; -use testkit::{CliOutput, Context, Error, Sandbox}; +use testkit::{CliOutput, Context, Error, ResultExt as _, Sandbox}; const LARGE_FILE_SIZE: usize = 512 * 1024; @@ -78,8 +78,7 @@ async fn large_upload_backpressure(cx: &mut Context) -> testkit::Result { chunk.copy_from_slice(&splitmix64(&mut state).to_le_bytes()[..chunk.len()]); } let relative = format!("payload/{index:05}.bin"); - fs::write(source.join(&relative), &payload) - .map_err(|error| Error::msg(format!("write {relative}: {error}")))?; + fs::write(source.join(&relative), &payload).context(format!("write {relative}"))?; checksums.push_str(&format!("{} {relative}\n", sha256_hex(&payload)?)); } write(&source.join("SHA256SUMS"), &checksums)?; @@ -147,12 +146,10 @@ impl<'a> Transfer<'a> { let sandbox = cx .sandbox(suffix) .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let remote_root = format!("/sandbox/file-transfer-{}-{suffix}", cx.run_id()); - let local = tempfile::tempdir() - .map_err(|error| Error::msg(format!("create a local scratch directory: {error}")))?; + let local = tempfile::tempdir().context("create a local scratch directory")?; let ft = Self { cx, sandbox, @@ -256,14 +253,12 @@ impl<'a> Transfer<'a> { write(&source.join("payload/00000.bin"), "payload")?; let script = source.join("run.sh"); write(&script, "#!/bin/sh\nprintf '%s\\n' upload-executable-ok\n")?; - fs::set_permissions(&script, fs::Permissions::from_mode(0o751)) - .map_err(|error| Error::msg(format!("chmod run.sh: {error}")))?; + fs::set_permissions(&script, fs::Permissions::from_mode(0o751)).context("chmod run.sh")?; for (target, link) in [ ("payload/00000.bin", "payload-link"), ("missing-target", "dangling-link"), ] { - symlink(target, source.join(link)) - .map_err(|error| Error::msg(format!("symlink {link}: {error}")))?; + symlink(target, source.join(link)).context(format!("symlink {link}"))?; } write(&source.join("conflict.txt"), "local-conflict")?; @@ -302,8 +297,7 @@ impl<'a> Transfer<'a> { write(&source.join("greeting.txt"), "hello-from-local")?; write(&source.join("subdir/nested.txt"), "nested-content")?; let large: Vec = (0u8..=250).cycle().take(LARGE_FILE_SIZE).collect(); - fs::write(source.join("large.bin"), &large) - .map_err(|error| Error::msg(format!("write large.bin: {error}")))?; + fs::write(source.join("large.bin"), &large).context("write large.bin")?; let remote = format!("{}/roundtrip", self.remote_root); let uploaded = self.upload(&source, &remote, false).await?.success()?; @@ -321,8 +315,7 @@ impl<'a> Transfer<'a> { .success()?; require_text(&destination.join("greeting.txt"), "hello-from-local")?; require_text(&destination.join("subdir/nested.txt"), "nested-content")?; - let actual = fs::read(destination.join("large.bin")) - .map_err(|error| Error::msg(format!("read large.bin: {error}")))?; + let actual = fs::read(destination.join("large.bin")).context("read large.bin")?; testkit::ensure!( actual == large, "large file changed in the round trip: sent {} bytes, received {}", @@ -492,7 +485,7 @@ fn git(repository: &Path, args: &[&str]) -> testkit::Result { .args(args) .current_dir(repository) .output() - .map_err(|error| Error::msg(format!("run git {}: {error}", args.join(" "))))?; + .context(format!("run git {}", args.join(" ")))?; testkit::ensure!( output.status.success(), "git {} failed in {}: {}", @@ -504,18 +497,15 @@ fn git(repository: &Path, args: &[&str]) -> testkit::Result { } fn write(path: &Path, contents: &str) -> testkit::Result { - fs::write(path, contents) - .map_err(|error| Error::msg(format!("write {}: {error}", path.display()))) + fs::write(path, contents).context(format!("write {}", path.display())) } fn create_dir_all(path: &Path) -> testkit::Result { - fs::create_dir_all(path) - .map_err(|error| Error::msg(format!("create {}: {error}", path.display()))) + fs::create_dir_all(path).context(format!("create {}", path.display())) } fn require_text(path: &Path, expected: &str) -> testkit::Result { - let actual = fs::read_to_string(path) - .map_err(|error| Error::msg(format!("read {}: {error}", path.display())))?; + let actual = fs::read_to_string(path).context(format!("read {}", path.display()))?; testkit::ensure!( actual == expected, "{} contains {actual:?}; expected {expected:?}", diff --git a/tests/suites/conformance/src/filesystem_policy.rs b/tests/suites/conformance/src/filesystem_policy.rs index ddb95977f8..4f67ed3f07 100644 --- a/tests/suites/conformance/src/filesystem_policy.rs +++ b/tests/suites/conformance/src/filesystem_policy.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use std::time::Duration; use testkit::sdk::raw::proto; -use testkit::{Context, ResultExt as _}; +use testkit::{Context, ResultExt as _, with_deadline}; /// A rejected launch must close promptly instead of hanging the exec. const REJECTED_LAUNCH_TIMEOUT: Duration = Duration::from_secs(20); @@ -62,9 +62,8 @@ async fn hard_requirement_accepts_added_device_paths(cx: &mut Context) -> testki .sandbox("ld") .command(["sleep", "infinity"]) .policy(policy("hard_requirement", &[])) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; sandbox .exec([ @@ -131,13 +130,14 @@ async fn replaced_policy_path_rejects_next_launch(cx: &mut Context) -> testkit:: .context(format!("({case}) prepare the policy path"))?; } - let next = tokio::time::timeout(REJECTED_LAUNCH_TIMEOUT, sandbox.exec(["true"])) - .await - .map_err(|_| { - testkit::Error::msg(format!( - "({case}) the rejected launch did not close within {REJECTED_LAUNCH_TIMEOUT:?}" - )) - })?; + // The exec's own error is the rejection report, so keep it apart from + // the deadline's. + let next = with_deadline( + REJECTED_LAUNCH_TIMEOUT, + &format!("({case}) the rejected launch"), + async { Ok(sandbox.exec(["true"]).await) }, + ) + .await?; let report = match next { Ok(output) if output.exit_code == 0 => { return Err(testkit::Error::msg(format!( diff --git a/tests/suites/conformance/src/image.rs b/tests/suites/conformance/src/image.rs index 36282f8fe4..f8faf80d71 100644 --- a/tests/suites/conformance/src/image.rs +++ b/tests/suites/conformance/src/image.rs @@ -13,9 +13,8 @@ async fn explicit_image_is_used(cx: &mut Context) -> testkit::Result { .sandbox("ei") .command(["sleep", "infinity"]) .spec(|spec| spec.image = Some(image.clone())) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; expect_ubuntu(&sandbox) .await .map_err(|error| error.context(format!("sandbox from image {image}"))) diff --git a/tests/suites/conformance/src/labels.rs b/tests/suites/conformance/src/labels.rs index 97ffe8f7be..fb9d9082f0 100644 --- a/tests/suites/conformance/src/labels.rs +++ b/tests/suites/conformance/src/labels.rs @@ -18,14 +18,7 @@ async fn labels_are_stored_and_filter_list(cx: &mut Context) -> testkit::Result let staging = format!("staging-{run}"); let backend = format!("backend-{run}"); let data = format!("data-{run}"); - let cases = [ - ("ldb", dev.as_str(), backend.as_str()), - ("lsb", staging.as_str(), backend.as_str()), - ("ldd", dev.as_str(), data.as_str()), - ]; - - let mut names = Vec::new(); - for (suffix, env, team) in cases { + let labeled = async |suffix: &str, env: &str, team: &str| -> testkit::Result { let labels = HashMap::from([ ("env".to_string(), env.to_string()), ("team".to_string(), team.to_string()), @@ -44,32 +37,30 @@ async fn labels_are_stored_and_filter_list(cx: &mut Context) -> testkit::Result sandbox.name(), got.labels ); - names.push(sandbox.name().to_string()); - } - let [dev_backend, staging_backend, dev_data] = names.as_slice() else { - unreachable!("one name per case"); + Ok(sandbox.name().to_string()) }; + let dev_backend = labeled("ldb", &dev, &backend).await?; + let staging_backend = labeled("lsb", &staging, &backend).await?; + let dev_data = labeled("ldd", &dev, &data).await?; let checks = [ - (format!("env={dev}"), vec![dev_backend, dev_data]), + (format!("env={dev}"), vec![&dev_backend, &dev_data]), ( format!("team={backend}"), - vec![dev_backend, staging_backend], + vec![&dev_backend, &staging_backend], ), - (format!("env={dev},team={backend}"), vec![dev_backend]), + (format!("env={dev},team={backend}"), vec![&dev_backend]), (format!("team=qa-{run}"), vec![]), ]; for (selector, expected) in checks { - let listed: BTreeSet = cx - .client() - .list_all_sandboxes(ListOptions { + let listed = listed_names( + cx, + ListOptions { label_selector: Some(selector.clone()), ..ListOptions::default() - }) - .await? - .into_iter() - .map(|sandbox| sandbox.name) - .collect(); + }, + ) + .await?; let expected: BTreeSet = expected.into_iter().cloned().collect(); testkit::ensure!( listed == expected, @@ -77,17 +68,25 @@ async fn labels_are_stored_and_filter_list(cx: &mut Context) -> testkit::Result ); } - let all: BTreeSet = cx - .client() - .list_all_sandboxes(ListOptions::default()) - .await? + let all = listed_names(cx, ListOptions::default()).await?; + let missing: Vec<&String> = [&dev_backend, &staging_backend, &dev_data] .into_iter() - .map(|sandbox| sandbox.name) + .filter(|name| !all.contains(*name)) .collect(); - let missing: Vec<&String> = names.iter().filter(|name| !all.contains(*name)).collect(); testkit::ensure!( missing.is_empty(), "list without a selector is missing {missing:?}" ); Ok(()) } + +/// The names of the sandboxes `list` returns, across all pages. +async fn listed_names(cx: &Context, options: ListOptions) -> testkit::Result> { + Ok(cx + .client() + .list_all_sandboxes(options) + .await? + .into_iter() + .map(|sandbox| sandbox.name) + .collect()) +} diff --git a/tests/suites/conformance/src/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs index e07732ecf7..c120477a0e 100644 --- a/tests/suites/conformance/src/lifecycle.rs +++ b/tests/suites/conformance/src/lifecycle.rs @@ -10,7 +10,7 @@ use std::time::Duration; use testkit::sdk::{SandboxPhase, SandboxRestartPolicy}; -use testkit::{Context, Poll, ResultExt as _, Sandbox, poll_until}; +use testkit::{Context, Poll, ResultExt as _, Sandbox, poll_until, with_deadline}; /// A long-running sandbox becomes ready, runs a command with exec, and is gone /// from get and list after delete. @@ -19,25 +19,23 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("lr") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?.success()?; - testkit::ensure!( - output.stdout == format!("{marker}\n"), - "exec stdout was {:?}; expected {:?}", - output.stdout, - format!("{marker}\n") - ); + sandbox + .exec(["echo", marker.as_str()]) + .await? + .expect_stdout(&format!("{marker}\n"))?; - sandbox.delete().await?; - cx.expect_absent(&sandbox).await + cx.delete_and_expect_absent(&sandbox).await } /// Longer than the supervisor's 30s startup recovery deadline. const PAST_RECOVERY_DEADLINE: Duration = Duration::from_secs(31); +/// The exec's own timeout is the runner's two-minute kill, so fail sooner, +/// with a message that says which step hung. +const PAST_RECOVERY_EXEC_DEADLINE: Duration = Duration::from_mins(1); /// An exec admitted before the startup recovery deadline expires keeps /// running past it, and a new exec still works afterwards. @@ -46,38 +44,27 @@ async fn exec_outlives_startup_recovery_deadline(cx: &mut Context) -> testkit::R let sandbox = cx .sandbox("rd") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let script = format!( "sleep {}; printf past-recovery-deadline", PAST_RECOVERY_DEADLINE.as_secs() ); - let output = tokio::time::timeout( - Duration::from_mins(1), + with_deadline( + PAST_RECOVERY_EXEC_DEADLINE, + "the exec past the recovery deadline", sandbox.exec(["sh", "-c", script.as_str()]), ) - .await - .map_err(|_| testkit::Error::msg("exec past the recovery deadline did not finish in 60s"))?? - .success() + .await? + .expect_stdout("past-recovery-deadline") .context("the recovery deadline must not end an admitted exec")?; - testkit::ensure!( - output.stdout == "past-recovery-deadline", - "the exec past the recovery deadline printed {:?}", - output.stdout - ); - let next = sandbox + sandbox .exec(["printf", "fresh-exec"]) .await? - .success() + .expect_stdout("fresh-exec") .context("exec after the recovery deadline")?; - testkit::ensure!( - next.stdout == "fresh-exec", - "exec after the recovery deadline printed {:?}", - next.stdout - ); Ok(()) } @@ -110,45 +97,28 @@ async fn delete_is_prompt_when_workload_is_slow_to_handle_term( let sandbox = cx .sandbox("dt") .command(["sh", "-c", script.as_str()]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; // Ready alone does not prove that the workload has installed its trap. - testkit::poll_until( - "workload TERM handler readiness", - Duration::from_secs(10), - Duration::from_millis(100), - async || { - let output = sandbox - .exec(["sh", "-c", "test -f /tmp/delayed-term-ready"]) - .await?; - Ok(if output.exit_code == 0 { - testkit::Poll::Ready(()) - } else { - testkit::Poll::Pending(format!( - "readiness check exited {}; stderr: {}", - output.exit_code, output.stderr - )) - }) - }, - ) - .await?; + sandbox + .poll_exec( + "workload TERM handler readiness", + Duration::from_secs(10), + ["test", "-f", "/tmp/delayed-term-ready"], + |output| output.exit_code == 0, + ) + .await?; // Use one deadline starting before delete, not a fresh timeout per step. // Boundary containment may kill the workload before its handler finishes; // this asserts bounded deletion, not a graceful workload exit. - tokio::time::timeout(DELETE_DEADLINE, async { - sandbox.delete().await?; - cx.expect_absent(&sandbox).await - }) + with_deadline( + DELETE_DEADLINE, + &format!("deleting sandbox {}", sandbox.name()), + cx.delete_and_expect_absent(&sandbox), + ) .await - .map_err(|_| { - testkit::Error::msg(format!( - "sandbox {} was not fully deleted within {DELETE_DEADLINE:?}", - sandbox.name() - )) - })? } /// A sandbox whose main command exits 0, with restart policy Never, reaches @@ -190,8 +160,7 @@ async fn run_to_completion_exit_delete(cx: &mut Context) -> testkit::Result { again.restart_count ); - sandbox.delete().await?; - cx.expect_absent(&sandbox).await + cx.delete_and_expect_absent(&sandbox).await } /// A main command that exits during provisioning is reported as the @@ -270,18 +239,14 @@ async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::R cx.expect_listed(&sandbox, SandboxPhase::Ready).await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?.success()?; - testkit::ensure!( - output.stdout == format!("{marker}\n"), - "exec stdout was {:?}; expected {:?}", - output.stdout, - format!("{marker}\n") - ); + sandbox + .exec(["echo", marker.as_str()]) + .await? + .expect_stdout(&format!("{marker}\n"))?; // The gateway's default image is OpenShell's Ubuntu base image. crate::image::expect_ubuntu(&sandbox).await?; - sandbox.delete().await?; - cx.expect_absent(&sandbox).await + cx.delete_and_expect_absent(&sandbox).await } /// A stopped sandbox rejects exec, accepts a second stop, and returns to @@ -291,18 +256,15 @@ async fn restart_stopped_sandbox(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("rs") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; sandbox.stop().await?; - match sandbox.exec(["true"]).await { - Err(_) => {} - Ok(output) => testkit::ensure!( - output.exit_code != 0, - "exec succeeded while the sandbox was stopped" - ), - } + let rejected = sandbox.exec(["true"]).await; + testkit::ensure!( + !matches!(&rejected, Ok(output) if output.exit_code == 0), + "exec succeeded while the sandbox was stopped" + ); // Stop is idempotent. let again = sandbox.stop().await?; testkit::ensure!( @@ -312,16 +274,11 @@ async fn restart_stopped_sandbox(cx: &mut Context) -> testkit::Result { ); sandbox.start().await?; - let output = sandbox + sandbox .exec(["echo", "started"]) .await? - .success() + .expect_stdout("started\n") .context("exec after start")?; - testkit::ensure!( - output.stdout == "started\n", - "exec after start printed {:?}; expected \"started\\n\"", - output.stdout - ); Ok(()) } @@ -331,12 +288,10 @@ async fn stopped_sandbox_can_be_deleted(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("sd") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; sandbox.stop().await?; - sandbox.delete().await?; - cx.expect_absent(&sandbox).await + cx.delete_and_expect_absent(&sandbox).await } /// Where the main command of [`canonical_main_environment`] records what it @@ -380,15 +335,11 @@ async fn canonical_main_environment(cx: &mut Context) -> testkit::Result { field("declared") ); - let exec = sandbox + sandbox .exec(["sh", "-c", "printf '%s' \"${REPRO_SENTINEL:-missing}\""]) .await? - .success()?; - testkit::ensure!( - exec.stdout == declared, - "exec (tty {tty}) saw REPRO_SENTINEL={:?}; expected {declared:?}", - exec.stdout - ); + .expect_stdout(&declared) + .context(format!("REPRO_SENTINEL in exec (tty {tty})"))?; if *tty { testkit::ensure!( @@ -413,40 +364,31 @@ async fn canonical_main_environment(cx: &mut Context) -> testkit::Result { async fn main_environment( sandbox: &Sandbox, ) -> testkit::Result> { - let contents = poll_until( - &format!( - "the main command of {} to record its environment", - sandbox.name() - ), - Duration::from_secs(30), - Duration::from_millis(250), - async || { - let output = sandbox.exec(["cat", MAIN_ENV_PATH]).await?; - Ok( - if output.exit_code == 0 && output.stdout.contains("term=") { - Poll::Ready(output.stdout) - } else { - Poll::Pending(format!( - "cat exited {}; stderr: {}", - output.exit_code, output.stderr - )) - }, - ) - }, - ) - .await?; - Ok(contents + let output = sandbox + .poll_exec( + &format!( + "the main command of {} to record its environment", + sandbox.name() + ), + Duration::from_secs(30), + ["cat", MAIN_ENV_PATH], + |output| output.exit_code == 0 && output.stdout.contains("term="), + ) + .await?; + Ok(output + .stdout .lines() .filter_map(|line| line.split_once('=')) .map(|(key, value)| (key.to_string(), value.to_string())) .collect()) } -/// How long a restart under the on-failure policy may take. The runner kills -/// a test after two minutes, so the test's deadlines must add up to less. -const RESTART_TIMEOUT: Duration = Duration::from_secs(60); -/// How long a Ready sandbox's main command may take to write its marker. -const MARKER_TIMEOUT: Duration = Duration::from_secs(20); +/// How long a restart under the on-failure policy may take to be reported. +/// The runner kills a test after two minutes, so the test's deadlines must +/// add up to less. +const RESTART_TIMEOUT: Duration = Duration::from_secs(45); +/// How long a running main command may take to write its workspace marker. +const MARKER_TIMEOUT: Duration = Duration::from_secs(15); /// Under the on-failure policy, a main command that fails is replaced by a new /// instance of it. The workspace survives the restart, the restart is counted, @@ -469,29 +411,19 @@ fi .sandbox("of") .command(["sh", "-c", SCRIPT]) .restart_policy(SandboxRestartPolicy::OnFailure) - .create() + .create_ready() + .await?; + sandbox + .poll_exec( + "the initial main command to write its workspace marker", + MARKER_TIMEOUT, + ["cat", "/sandbox/initial-run"], + |output| output.stdout.contains("initial-"), + ) .await?; - sandbox.wait_ready().await?; - poll_until( - "the initial main command to write its workspace marker", - MARKER_TIMEOUT, - Duration::from_millis(250), - async || { - let output = sandbox.exec(["cat", "/sandbox/initial-run"]).await?; - Ok(if output.stdout.contains("initial-") { - Poll::Ready(()) - } else { - Poll::Pending(format!("cat exited {}", output.exit_code)) - }) - }, - ) - .await?; let initial = cx.sandbox_details(sandbox.name()).await?; - let initial_instance = initial["main_process_instance_id"] - .as_str() - .unwrap_or_default() - .to_string(); + let initial_instance = main_instance(&initial).to_string(); testkit::ensure!( !initial_instance.is_empty(), "sandbox get reports no main process instance: {initial}" @@ -507,32 +439,29 @@ fi Duration::from_millis(250), async || { let details = cx.sandbox_details(sandbox.name()).await?; - let instance = details["main_process_instance_id"] - .as_str() - .unwrap_or_default(); + let instance = main_instance(&details); let replaced = !instance.is_empty() && instance != initial_instance; - if details["phase"] != "Ready" || details["restart_count"] != 1 || !replaced { - return Ok(Poll::Pending(format!( - "phase {}, restart count {}, main process instance {instance:?}", - details["phase"], details["restart_count"] - ))); - } - let runs = sandbox - .exec(["cat", "/sandbox/initial-run", "/sandbox/replacement-run"]) - .await?; Ok( - if runs.stdout.contains("initial-") && runs.stdout.contains("replacement-") { + if details["phase"] == "Ready" && details["restart_count"] == 1 && replaced { Poll::Ready(details) } else { Poll::Pending(format!( - "restarted, but the workspace has {:?}; stderr: {}", - runs.stdout, runs.stderr + "phase {}, restart count {}, main process instance {instance:?}", + details["phase"], details["restart_count"] )) }, ) }, ) .await?; + sandbox + .poll_exec( + "the replacement main command to find the workspace and write its marker", + MARKER_TIMEOUT, + ["cat", "/sandbox/initial-run", "/sandbox/replacement-run"], + |output| output.stdout.contains("initial-") && output.stdout.contains("replacement-"), + ) + .await?; testkit::ensure!( restarted["restart_policy"] == "on-failure", "the restart policy after the restart is {}; expected \"on-failure\"", @@ -541,6 +470,14 @@ fi Ok(()) } +/// The ID of the sandbox's current main process instance, from +/// [`Context::sandbox_details`]. Empty if none is reported. +fn main_instance(details: &serde_json::Value) -> &str { + details["main_process_instance_id"] + .as_str() + .unwrap_or_default() +} + /// How long the gateway may take to remove an ephemeral sandbox after its /// main command exits. Both sandboxes are released before the test waits, so /// their removals overlap. diff --git a/tests/suites/conformance/src/persistent_workspace.rs b/tests/suites/conformance/src/persistent_workspace.rs index a8b2eadfb3..1b123fe618 100644 --- a/tests/suites/conformance/src/persistent_workspace.rs +++ b/tests/suites/conformance/src/persistent_workspace.rs @@ -10,7 +10,7 @@ use std::time::Duration; -use testkit::{Context, Poll, ResultExt as _}; +use testkit::{Context, ResultExt as _}; /// Files in the workspace survive stop and start, and start runs the main /// command again. @@ -26,9 +26,8 @@ async fn persistent_workspace_survives_stop_start(cx: &mut Context) -> testkit:: let sandbox = cx .sandbox("kw") .command(["sh", "-c", main.as_str()]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let sentinel = format!("openshell-stop-start-{}", cx.run_id()); sandbox @@ -44,28 +43,20 @@ async fn persistent_workspace_survives_stop_start(cx: &mut Context) -> testkit:: sandbox.stop().await?; sandbox.start().await?; - let read = sandbox.exec(["cat", SENTINEL]).await?; - testkit::ensure!( - read.stdout == format!("{sentinel}\n"), - "sentinel after restart is {:?}; stderr: {}", - read.stdout, - read.stderr - ); + sandbox + .exec(["cat", SENTINEL]) + .await? + .expect_stdout(&format!("{sentinel}\n")) + .context("read the sentinel after restart")?; // The sandbox is Ready before the restarted main command has necessarily // rewritten the count, so wait for it to say it ran again. - testkit::poll_until( - "the main command to run again", - Duration::from_secs(30), - Duration::from_secs(1), - async || { - let runs = sandbox.exec(["cat", RUN_COUNT]).await?; - Ok(if runs.stdout == "2\n" { - Poll::Ready(()) - } else { - Poll::Pending(format!("run count {:?}", runs.stdout.trim())) - }) - }, - ) - .await?; + sandbox + .poll_exec( + "the main command to run again", + Duration::from_secs(30), + ["cat", RUN_COUNT], + |runs| runs.stdout == "2\n", + ) + .await?; Ok(()) } diff --git a/tests/suites/conformance/src/policy_advisor.rs b/tests/suites/conformance/src/policy_advisor.rs index 49ee54ce9d..82ff54cc09 100644 --- a/tests/suites/conformance/src/policy_advisor.rs +++ b/tests/suites/conformance/src/policy_advisor.rs @@ -230,8 +230,7 @@ async fn create( if let Some(policy) = policy { builder = builder.policy(policy); } - let sandbox = builder.create().await?; - sandbox.wait_ready().await?; + let sandbox = builder.create_ready().await?; Ok(sandbox) } diff --git a/tests/suites/conformance/src/process.rs b/tests/suites/conformance/src/process.rs index 68cc2dc94b..1f7e9d1eae 100644 --- a/tests/suites/conformance/src/process.rs +++ b/tests/suites/conformance/src/process.rs @@ -3,7 +3,7 @@ //! Sandbox process suite: limits the sandbox applies to its processes. -use testkit::Context; +use testkit::{Context, ResultExt as _}; /// Sandbox processes can't write core dumps: both the soft and the hard core /// file size limits are 0, so a process can't raise its own limit either. @@ -12,22 +12,17 @@ async fn core_dumps_are_disabled(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("cd") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; - let output = sandbox + sandbox .exec([ "sh", "-c", "printf '%s %s' \"$(ulimit -c)\" \"$(ulimit -Hc)\"", ]) .await? - .success()?; - testkit::ensure!( - output.stdout == "0 0", - "core file size limits (soft hard) are {:?}; expected \"0 0\"", - output.stdout - ); + .expect_stdout("0 0") + .context("core file size limits (soft hard)")?; Ok(()) } diff --git a/tests/suites/conformance/src/settings.rs b/tests/suites/conformance/src/settings.rs index 35490f2b38..c667663d16 100644 --- a/tests/suites/conformance/src/settings.rs +++ b/tests/suites/conformance/src/settings.rs @@ -18,16 +18,18 @@ const KEY: &str = "ocsf_json_enabled"; /// How the CLI shows a setting with no value. const UNSET: &str = ""; -/// With no global value, a sandbox setting starts unset, reads back after it is set, and is unset -/// again after it is deleted. +/// With no global value, a sandbox setting starts unset, reads back after it +/// is set, and is unset again after it is deleted. #[testkit::test] async fn sandbox_setting_round_trip(cx: &mut Context) -> testkit::Result { // A global value would hide the sandbox's own, so clear it for the test. restore_global_setting_on_cleanup(cx, KEY).await?; - cx.cli(["settings", "delete", "--global", "--key", KEY, "--yes"]) - .await? - .success()?; - let sandbox = settings_sandbox(cx, "sr").await?; + delete_global_setting(&cx.cli_handle(), KEY).await?; + let sandbox = cx + .sandbox("sr") + .command(["sleep", "infinity"]) + .create_ready() + .await?; expect_sandbox_setting(cx, &sandbox, UNSET, "unset").await?; set_sandbox_setting(cx, &sandbox, "true").await?.success()?; @@ -44,22 +46,21 @@ async fn sandbox_setting_round_trip(cx: &mut Context) -> testkit::Result { /// controls the key again. #[testkit::test] async fn global_setting_overrides_sandbox_scope(cx: &mut Context) -> testkit::Result { + let cli = cx.cli_handle(); restore_global_setting_on_cleanup(cx, KEY).await?; - cx.cli(["settings", "delete", "--global", "--key", KEY, "--yes"]) - .await? - .success()?; + delete_global_setting(&cli, KEY).await?; - let sandbox = settings_sandbox(cx, "sg").await?; + let sandbox = cx + .sandbox("sg") + .command(["sleep", "infinity"]) + .create_ready() + .await?; set_sandbox_setting(cx, &sandbox, "true").await?.success()?; expect_sandbox_setting(cx, &sandbox, "true", "sandbox").await?; - cx.cli([ - "settings", "set", "--global", "--key", KEY, "--value", "false", "--yes", - ]) - .await? - .success()?; + set_global_setting(&cli, KEY, "false").await?; testkit::ensure!( - global_setting(cx, KEY).await? == "false", + read_global_setting(&cli, KEY).await? == "false", "the global {KEY} does not read back as false" ); expect_sandbox_setting(cx, &sandbox, "false", "global").await?; @@ -74,10 +75,8 @@ async fn global_setting_overrides_sandbox_scope(cx: &mut Context) -> testkit::Re .await? .failure()?; - cx.cli(["settings", "delete", "--global", "--key", KEY, "--yes"]) - .await? - .success()?; - let global = global_setting(cx, KEY).await?; + delete_global_setting(&cli, KEY).await?; + let global = read_global_setting(&cli, KEY).await?; testkit::ensure!( global == UNSET, "the global {KEY} is {global:?} after delete; expected {UNSET:?}" @@ -89,16 +88,6 @@ async fn global_setting_overrides_sandbox_scope(cx: &mut Context) -> testkit::Re expect_sandbox_setting(cx, &sandbox, "false", "sandbox").await } -async fn settings_sandbox(cx: &Context, suffix: &str) -> testkit::Result { - let sandbox = cx - .sandbox(suffix) - .command(["sleep", "infinity"]) - .create() - .await?; - sandbox.wait_ready().await?; - Ok(sandbox) -} - async fn set_sandbox_setting( cx: &Context, sandbox: &Sandbox, @@ -149,10 +138,6 @@ async fn expect_sandbox_setting( } /// The gateway-global value of `key` as the CLI shows it, [`UNSET`] if none. -async fn global_setting(cx: &Context, key: &str) -> testkit::Result { - read_global_setting(&cx.cli_handle(), key).await -} - async fn read_global_setting(cli: &testkit::Cli, key: &str) -> testkit::Result { let settings: Value = cli .run(["settings", "get", "--global", "--json"]) @@ -165,6 +150,22 @@ async fn read_global_setting(cli: &testkit::Cli, key: &str) -> testkit::Result testkit::Result { + cli.run([ + "settings", "set", "--global", "--key", key, "--value", value, "--yes", + ]) + .await? + .success()?; + Ok(()) +} + +async fn delete_global_setting(cli: &testkit::Cli, key: &str) -> testkit::Result { + cli.run(["settings", "delete", "--global", "--key", key, "--yes"]) + .await? + .success()?; + Ok(()) +} + /// Record the gateway-global value of `key` and put it back after the test: /// set it again if it had one, delete it otherwise. async fn restore_global_setting_on_cleanup(cx: &Context, key: &'static str) -> testkit::Result { @@ -180,17 +181,10 @@ async fn restore_global_setting_on_cleanup(cx: &Context, key: &'static str) -> t return Ok(()); } if prior == UNSET { - cli.run(["settings", "delete", "--global", "--key", key, "--yes"]) - .await? - .success()?; + delete_global_setting(&cli, key).await } else { - cli.run([ - "settings", "set", "--global", "--key", key, "--value", &prior, "--yes", - ]) - .await? - .success()?; + set_global_setting(&cli, key, &prior).await } - Ok(()) }); Ok(()) } diff --git a/tests/suites/conformance/src/smoke.rs b/tests/suites/conformance/src/smoke.rs index 0f39bff143..8047b6a5f4 100644 --- a/tests/suites/conformance/src/smoke.rs +++ b/tests/suites/conformance/src/smoke.rs @@ -12,19 +12,14 @@ async fn create_exec_delete(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("smoke") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?.success()?; - testkit::ensure!( - output.stdout == format!("{marker}\n"), - "exec stdout was {:?}; expected {:?}", - output.stdout, - format!("{marker}\n") - ); + sandbox + .exec(["echo", marker.as_str()]) + .await? + .expect_stdout(&format!("{marker}\n"))?; - sandbox.delete().await?; - cx.expect_absent(&sandbox).await + cx.delete_and_expect_absent(&sandbox).await } diff --git a/tests/suites/conformance/src/templates.rs b/tests/suites/conformance/src/templates.rs index 13c2efb674..9e6e2196d0 100644 --- a/tests/suites/conformance/src/templates.rs +++ b/tests/suites/conformance/src/templates.rs @@ -6,16 +6,22 @@ use std::collections::HashMap; -use testkit::Context; use testkit::sdk::raw::proto::datamodel::v1::ObjectMeta; use testkit::sdk::{ DeleteOptions, SandboxResources, SandboxTemplateCreateSpec, SandboxTemplateListOptions, SandboxWorkloadConfig, SandboxWorkloadTemplate, SandboxWorkloadTemplateSpec, SdkError, }; +use testkit::{Context, ResultExt as _}; -/// Delete template `name` after the test. Registered before the create, so it -/// also covers a create the gateway applied but reported as failed. -fn delete_template_on_cleanup(cx: &Context, name: &str) { +/// Create template `name`, and delete it after the test. The cleanup is +/// registered before the create, so it also covers a create the gateway +/// applied but reported as failed. +async fn create_template( + cx: &Context, + name: &str, + labels: HashMap, + workload: SandboxWorkloadConfig, +) -> Result { let client = cx.client().clone(); let owned = name.to_string(); cx.on_cleanup(format!("delete sandbox template {name}"), async move || { @@ -29,24 +35,19 @@ fn delete_template_on_cleanup(cx: &Context, name: &str) { .await?; Ok(()) }); -} - -fn template( - name: &str, - labels: HashMap, - workload: SandboxWorkloadConfig, -) -> SandboxWorkloadTemplate { - SandboxWorkloadTemplate { - metadata: Some(ObjectMeta { - name: name.to_string(), - labels, - ..ObjectMeta::default() - }), - spec: Some(SandboxWorkloadTemplateSpec { - workload: Some(workload), - ..SandboxWorkloadTemplateSpec::default() - }), - } + cx.client() + .create_sandbox_template(SandboxWorkloadTemplate { + metadata: Some(ObjectMeta { + name: name.to_string(), + labels, + ..ObjectMeta::default() + }), + spec: Some(SandboxWorkloadTemplateSpec { + workload: Some(workload), + ..SandboxWorkloadTemplateSpec::default() + }), + }) + .await } /// A template stores its labels, environment and resources, is listed, and a @@ -55,7 +56,6 @@ fn template( #[testkit::test] async fn sandbox_from_template_uses_its_workload(cx: &mut Context) -> testkit::Result { let template_name = cx.name("tp"); - delete_template_on_cleanup(cx, &template_name); let labels = HashMap::from([("e2e".to_string(), "sandbox-template".to_string())]); let workload = SandboxWorkloadConfig { environment: HashMap::from([("FEATURE_FLAG".to_string(), "on".to_string())]), @@ -66,9 +66,7 @@ async fn sandbox_from_template_uses_its_workload(cx: &mut Context) -> testkit::R }), ..SandboxWorkloadConfig::default() }; - cx.client() - .create_sandbox_template(template(&template_name, labels.clone(), workload)) - .await?; + create_template(cx, &template_name, labels.clone(), workload.clone()).await?; let stored = cx.client().get_sandbox_template(&template_name).await?; let meta = stored.metadata.clone().unwrap_or_default(); @@ -83,16 +81,17 @@ async fn sandbox_from_template_uses_its_workload(cx: &mut Context) -> testkit::R .as_ref() .and_then(|spec| spec.workload.clone()) .unwrap_or_default(); + // Field by field: the gateway may fill in fields the template left unset. + let sent = workload.resources.clone().unwrap_or_default(); let resources = stored_workload.resources.clone().unwrap_or_default(); testkit::ensure!( - stored_workload - .environment - .get("FEATURE_FLAG") - .map(String::as_str) - == Some("on") - && resources.cpu == "500m" - && resources.memory == "512Mi", - "template get returned workload {stored_workload:?}; expected FEATURE_FLAG=on, cpu 500m, memory 512Mi" + stored_workload.environment == workload.environment + && resources.cpu == sent.cpu + && resources.memory == sent.memory, + "template get returned workload {stored_workload:?}; expected environment {:?}, cpu {:?}, memory {:?}", + workload.environment, + sent.cpu, + sent.memory ); let listed = cx @@ -125,15 +124,11 @@ async fn sandbox_from_template_uses_its_workload(cx: &mut Context) -> testkit::R "sandbox {sandbox_name} records template {provenance:?}; expected {template_name:?}" ); - let output = sandbox + sandbox .exec(["sh", "-c", "printf '%s' \"${FEATURE_FLAG:-missing}\""]) .await? - .success()?; - testkit::ensure!( - output.stdout == "on", - "FEATURE_FLAG in a sandbox from the template is {:?}; expected \"on\"", - output.stdout - ); + .expect_stdout(&workload.environment["FEATURE_FLAG"]) + .context("FEATURE_FLAG in a sandbox from the template")?; Ok(()) } @@ -143,35 +138,29 @@ async fn sandbox_from_template_uses_its_workload(cx: &mut Context) -> testkit::R #[testkit::test] async fn template_crud_errors(cx: &mut Context) -> testkit::Result { let name = cx.name("te"); - delete_template_on_cleanup(cx, &name); - let empty = || template(&name, HashMap::new(), SandboxWorkloadConfig::default()); + let empty = + async || create_template(cx, &name, HashMap::new(), SandboxWorkloadConfig::default()).await; - cx.client().create_sandbox_template(empty()).await?; - match cx.client().create_sandbox_template(empty()).await { - Err(SdkError::AlreadyExists { .. }) => {} - other => { - return Err(testkit::Error::msg(format!( - "creating template {name} again returned {other:?}; expected AlreadyExists" - ))); - } - } + empty().await?; + let again = empty().await; + testkit::ensure!( + matches!(again, Err(SdkError::AlreadyExists { .. })), + "creating template {name} again returned {again:?}; expected AlreadyExists" + ); cx.client() .delete_sandbox_template(&name, DeleteOptions::default()) .await?; - match cx.client().get_sandbox_template(&name).await { - Err(SdkError::NotFound { .. }) => {} - other => { - return Err(testkit::Error::msg(format!( - "getting deleted template {name} returned {other:?}; expected NotFound" - ))); - } - } + let deleted = cx.client().get_sandbox_template(&name).await; + testkit::ensure!( + matches!(deleted, Err(SdkError::NotFound { .. })), + "getting deleted template {name} returned {deleted:?}; expected NotFound" + ); let sandbox_name = cx.name("tm"); cx.track(&sandbox_name); let missing = cx.name("tx"); - match cx + let from_missing = cx .client() .create_sandbox_from_template(SandboxTemplateCreateSpec { name: Some(sandbox_name), @@ -179,11 +168,10 @@ async fn template_crud_errors(cx: &mut Context) -> testkit::Result { command: vec!["true".to_string()], ..SandboxTemplateCreateSpec::default() }) - .await - { - Err(SdkError::NotFound { .. }) => Ok(()), - other => Err(testkit::Error::msg(format!( - "creating a sandbox from missing template {missing} returned {other:?}; expected NotFound" - ))), - } + .await; + testkit::ensure!( + matches!(from_missing, Err(SdkError::NotFound { .. })), + "creating a sandbox from missing template {missing} returned {from_missing:?}; expected NotFound" + ); + Ok(()) } diff --git a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs index 8035d0694e..a617c24843 100644 --- a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs +++ b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs @@ -9,7 +9,7 @@ use std::io::Write as _; use keycloak_support::KeycloakContext; -use testkit::{CliOutput, Context, Error, ResultExt as _}; +use testkit::{CliOutput, Context, ResultExt as _}; const CREDENTIAL_KEY: &str = "KEYCLOAK_ACCESS_TOKEN"; @@ -172,11 +172,9 @@ binaries: let mut file = tempfile::Builder::new() .suffix(".yaml") .tempfile() - .map_err(|error| Error::from_error(&error)) .context("create provider profile")?; file.write_all(profile.as_bytes()) .and_then(|()| file.flush()) - .map_err(|error| Error::from_error(&error)) .context("write provider profile")?; Ok(file) } diff --git a/tests/suites/features/oci-image/tests/oci_image.rs b/tests/suites/features/oci-image/tests/oci_image.rs index 287e44c96b..ccb44b48e4 100644 --- a/tests/suites/features/oci-image/tests/oci_image.rs +++ b/tests/suites/features/oci-image/tests/oci_image.rs @@ -313,9 +313,7 @@ impl Oci<'_> { /// Upload and download default to paths relative to the image workspace. async fn file_transfer_uses_workspace(&self, sandbox: &str) -> Result { - let local = tempfile::tempdir() - .map_err(|error| Error::from_error(&error)) - .context("create temp dir")?; + let local = tempfile::tempdir().context("create temp dir")?; let upload = local.path().join("oci-transfer.txt"); write(&upload, "oci-transfer-ok")?; self.cx @@ -348,9 +346,7 @@ impl Oci<'_> { .await? .success() .context("download resolves relative paths in the workspace")?; - let downloaded = std::fs::read_to_string(&download) - .map_err(|error| Error::from_error(&error)) - .context("read download")?; + let downloaded = std::fs::read_to_string(&download).context("read download")?; testkit::ensure!( downloaded == "oci-transfer-ok", "downloaded file has unexpected content: {downloaded:?}" @@ -358,9 +354,7 @@ impl Oci<'_> { // A directory upload merges into an existing workspace directory. let merge = local.path().join("merge-upload"); - std::fs::create_dir(&merge) - .map_err(|error| Error::from_error(&error)) - .context("create merge dir")?; + std::fs::create_dir(&merge).context("create merge dir")?; write(&merge.join("conflict.txt"), "local-conflict")?; write(&merge.join("added.txt"), "local-added")?; self.exec( @@ -495,9 +489,7 @@ fn workspace_checks(identity: &str, workspace: &str, image_file: bool) -> String } fn write(path: &std::path::Path, contents: &str) -> Result { - std::fs::write(path, contents) - .map_err(|error| Error::from_error(&error)) - .context(format!("write {}", path.display())) + std::fs::write(path, contents).context(format!("write {}", path.display())) } fn utf8(path: &std::path::Path) -> Result<&str> { diff --git a/tests/support/container-engine/src/lib.rs b/tests/support/container-engine/src/lib.rs index 861e0848ff..7d47f2d33f 100644 --- a/tests/support/container-engine/src/lib.rs +++ b/tests/support/container-engine/src/lib.rs @@ -64,16 +64,11 @@ impl Engine { Ok(Self { command }) } - /// Build `containerfile` as `tag`. The build context is empty. Prefer - /// [`Engine::build_test_image`], which also removes the image. - pub async fn build_image(&self, tag: &str, containerfile: &str) -> Result { - let context = tempfile::tempdir() - .map_err(|error| Error::from_error(&error)) - .context("create build context")?; + /// Build `containerfile` as `tag`. The build context is empty. + async fn build_image(&self, tag: &str, containerfile: &str) -> Result { + let context = tempfile::tempdir().context("create build context")?; let file = context.path().join("Containerfile"); - std::fs::write(&file, containerfile) - .map_err(|error| Error::from_error(&error)) - .context("write Containerfile")?; + std::fs::write(&file, containerfile).context("write Containerfile")?; self.output([ "build", "--file", @@ -175,7 +170,6 @@ impl Engine { .kill_on_drop(true) .output() .await - .map_err(|error| Error::from_error(&error)) .context(format!("run {command}"))?; testkit::ensure!( output.status.success(), diff --git a/tests/support/keycloak/src/lib.rs b/tests/support/keycloak/src/lib.rs index d5b0b58f31..7e96c5b41c 100644 --- a/tests/support/keycloak/src/lib.rs +++ b/tests/support/keycloak/src/lib.rs @@ -158,7 +158,6 @@ impl Keycloak { .kill_on_drop(true) .output() .await - .map_err(|error| Error::from_error(&error)) .context("run curl")?; Ok(if output.status.success() { Poll::Ready(()) @@ -253,20 +252,14 @@ async fn post_form(url: &str, fields: &[&str], secret: (&str, &str)) -> Result Result { @@ -178,7 +181,7 @@ impl CliOutput { /// match against this rather than the raw output. pub fn message(&self) -> String { format!("{} {}", self.stdout, self.stderr) - .replace(['│', '×', '╰', '─', '╭', '├'], " ") + .replace(REPORT_GUTTER, " ") .split_whitespace() .collect::>() .join(" ") diff --git a/tests/testkit/testkit/src/cli_sandbox.rs b/tests/testkit/testkit/src/cli_sandbox.rs index 858e27cec8..52c545ce83 100644 --- a/tests/testkit/testkit/src/cli_sandbox.rs +++ b/tests/testkit/testkit/src/cli_sandbox.rs @@ -79,18 +79,17 @@ impl<'a> CliSandboxBuilder<'a> { /// return the sandbox. It need not be Ready yet. pub async fn create(mut self) -> Result { self.args.push("--detach".to_string()); - let (cx, name) = (self.cx, self.name.clone()); self.run() .await? .success() - .context(format!("create sandbox {name}"))?; - cx.adopt(&name).await + .context(format!("create sandbox {}", self.name))?; + self.cx.adopt(&self.name).await } /// Create the sandbox attached to its main command and return the CLI's /// output, whatever its exit code. For a test of the main command's /// result, or of a create that must fail. - pub async fn run(self) -> Result { + pub async fn run(&self) -> Result { self.cx.track(&self.name); let policy = self.policy.as_deref().map(PolicyFile::new).transpose()?; let mut args = vec!["sandbox", "create", "--name", &self.name, "--no-tty"]; @@ -111,11 +110,8 @@ struct PolicyFile(tempfile::NamedTempFile); impl PolicyFile { fn new(contents: &str) -> Result { - let file = tempfile::NamedTempFile::new() - .map_err(|error| Error::from_error(&error)) - .context("create policy file")?; + let file = tempfile::NamedTempFile::new().context("create policy file")?; std::fs::write(file.path(), contents) - .map_err(|error| Error::from_error(&error)) .context(format!("write {}", file.path().display()))?; Ok(Self(file)) } diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index f176c1c936..e8029ec511 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -209,6 +209,12 @@ impl Context { self.expect_absent(sandbox).await } + /// Delete the sandbox, then check it is gone from both get and list. + pub async fn delete_and_expect_absent(&self, sandbox: &Sandbox) -> Result { + sandbox.delete().await?; + self.expect_absent(sandbox).await + } + /// Fail unless the sandbox is gone from both get and list. pub async fn expect_absent(&self, sandbox: &Sandbox) -> Result { let name = sandbox.name(); @@ -375,6 +381,13 @@ impl SandboxBuilder { self } + /// Create the sandbox and wait until it is Ready. + pub async fn create_ready(self) -> Result { + let sandbox = self.create().await?; + sandbox.wait_ready().await?; + Ok(sandbox) + } + /// Create the sandbox. It is tracked for cleanup before the request is /// sent, so a create that fails after the gateway accepted it is still /// cleaned up. @@ -496,6 +509,18 @@ impl ExecOutput { ); Ok(self) } + + /// Fail unless the command exited 0 and printed exactly `expected`. + pub fn expect_stdout(self, expected: &str) -> Result { + let output = self.success()?; + crate::ensure!( + output.stdout == expected, + "{} printed {:?}; expected {expected:?}", + output.command, + output.stdout + ); + Ok(output) + } } impl Sandbox { @@ -648,6 +673,7 @@ impl Sandbox { S: Into, { let command: Vec = command.into_iter().map(Into::into).collect(); + let description = format!("exec {command:?} in sandbox {}", self.name); let options = ExecOptions { timeout: Some(EXEC_TIMEOUT), ..ExecOptions::default() @@ -656,15 +682,46 @@ impl Sandbox { .client .exec(&self.name, &command, options) .await - .context(format!("exec {command:?} in sandbox {}", self.name))?; + .context(&description)?; Ok(ExecOutput { - command: format!("exec {command:?} in sandbox {}", self.name), + command: description, exit_code: result.exit_code, stdout: String::from_utf8_lossy(&result.stdout).into_owned(), stderr: String::from_utf8_lossy(&result.stderr).into_owned(), }) } + /// Run `command` every 250 ms until `ready` accepts its output, and return + /// that output. An exec error fails at once; a timeout reports the last + /// output. + pub async fn poll_exec( + &self, + what: &str, + timeout: Duration, + command: I, + ready: impl Fn(&ExecOutput) -> bool, + ) -> Result + where + I: IntoIterator, + S: Into, + { + let command: Vec = command.into_iter().map(Into::into).collect(); + poll_until(what, timeout, Duration::from_millis(250), async || { + let output = self.exec(command.iter().cloned()).await?; + Ok(if ready(&output) { + Poll::Ready(output) + } else { + Poll::Pending(format!( + "exited {} with stdout {:?}; stderr: {}", + output.exit_code, + output.stdout, + output.stderr.trim() + )) + }) + }) + .await + } + /// Delete the sandbox and wait until it is gone. pub async fn delete(&self) -> Result { self.client diff --git a/tests/testkit/testkit/src/error.rs b/tests/testkit/testkit/src/error.rs index 51d2fc3904..11ad20c853 100644 --- a/tests/testkit/testkit/src/error.rs +++ b/tests/testkit/testkit/src/error.rs @@ -85,6 +85,12 @@ impl From for Error { } } +impl From for Error { + fn from(error: std::io::Error) -> Self { + Self::from_error(&error) + } +} + /// Add context to a failing step. pub trait ResultExt { fn context(self, context: impl fmt::Display) -> Result; diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 83a4e426b8..0137ee1164 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -53,7 +53,7 @@ pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; pub use image::{PYTHON_BINARIES, base_image, python_image}; pub use openshell_sdk as sdk; -pub use poll::{Poll, poll_until}; +pub use poll::{Poll, poll_until, with_deadline}; pub use program::main; pub use ready::check as check_ready; pub use registry::Test; diff --git a/tests/testkit/testkit/src/poll.rs b/tests/testkit/testkit/src/poll.rs index 838d89526d..a9bd0041f5 100644 --- a/tests/testkit/testkit/src/poll.rs +++ b/tests/testkit/testkit/src/poll.rs @@ -15,6 +15,18 @@ pub enum Poll { Pending(String), } +/// Fail unless `future` finishes within `timeout`. For a deadline shorter +/// than the step's own, such as one that must fit inside the runner's kill. +pub async fn with_deadline( + timeout: Duration, + what: &str, + future: impl Future>, +) -> Result { + tokio::time::timeout(timeout, future) + .await + .map_err(|_| Error::msg(format!("{what} did not finish within {timeout:?}")))? +} + /// Observe until the condition holds or `timeout` elapses. /// /// An error from `observe` fails immediately. On timeout, the error reports From 9a461dcff3c5591351627d8995e06aa7476b3541 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 20:33:42 -0800 Subject: [PATCH 5/9] test(server): drive the WebSocket tunnel test through the SDK tunnel proxy Replace the hand-written WebSocket bridge with openshell_sdk::edge_tunnel::start_tunnel_proxy, so the test covers the client half of the tunnel and its edge-token headers as well as the router. Signed-off-by: Matthew Grossman --- Cargo.lock | 1 + crates/openshell-server/Cargo.toml | 1 + crates/openshell-server/src/lib.rs | 57 ++++++++---------------------- 3 files changed, 17 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d8b2f36221..7e6e0f5ff6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5152,6 +5152,7 @@ dependencies = [ "openshell-policy", "openshell-prover", "openshell-providers", + "openshell-sdk", "openshell-supervisor-control", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index a7c13be2cb..869dce9291 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -142,6 +142,7 @@ hyper-rustls = { version = "0.27", default-features = false, features = ["native rcgen = { workspace = true } rsa = { version = "0.9", features = ["pem"] } tokio-tungstenite = { workspace = true } +openshell-sdk = { path = "../openshell-sdk" } futures-util = "0.3" wiremock = "0.6" openshell-otel-test-support = { path = "../openshell-otel-test-support" } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 082021033b..600444ab49 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -3173,55 +3173,28 @@ mod tests { stop_listener(shutdown, handle).await; } - /// The edge-tunnel client path: gRPC carried in binary frames over a - /// WebSocket to `/_ws_tunnel` on the production listener reaches the - /// gateway's own services. + /// The edge-tunnel client path: the SDK's tunnel proxy carries gRPC in + /// WebSocket binary frames to `/_ws_tunnel` on the production listener, + /// and the request reaches the gateway's own services. #[tokio::test] async fn production_gateway_listener_serves_grpc_through_websocket_tunnel() { - use futures::{SinkExt as _, StreamExt as _}; use openshell_core::proto::{ HealthRequest, ServiceStatus, open_shell_client::OpenShellClient, }; - use tokio_tungstenite::tungstenite::Message; let (addr, shutdown, handle) = start_plaintext_gateway_listener_with_tunnel(true).await; - let ws_url = format!("ws://127.0.0.1:{}/_ws_tunnel", addr.port()); - let channel = tonic::transport::Endpoint::from_static("http://edge-tunnel.invalid") - .connect_with_connector(tower::service_fn(move |_| { - let ws_url = ws_url.clone(); - async move { - let (ws, _) = tokio_tungstenite::connect_async(ws_url) - .await - .map_err(Error::other)?; - let (mut ws_sink, mut ws_source) = ws.split(); - let (client, bridge) = tokio::io::duplex(64 * 1024); - let (mut bridge_read, mut bridge_write) = tokio::io::split(bridge); - tokio::spawn(async move { - let mut buf = vec![0u8; 32 * 1024]; - while let Ok(n @ 1..) = bridge_read.read(&mut buf).await { - if ws_sink - .send(Message::Binary(buf[..n].to_vec().into())) - .await - .is_err() - { - break; - } - } - let _ = ws_sink.close().await; - }); - tokio::spawn(async move { - while let Some(Ok(Message::Binary(data))) = ws_source.next().await { - if bridge_write.write_all(&data).await.is_err() { - break; - } - } - let _ = bridge_write.shutdown().await; - }); - Ok::<_, Error>(hyper_util::rt::TokioIo::new(client)) - } - })) - .await - .expect("gRPC channel through the WebSocket tunnel"); + let proxy = openshell_sdk::edge_tunnel::start_tunnel_proxy( + &format!("http://127.0.0.1:{}", addr.port()), + "edge-token", + ) + .await + .expect("start the SDK tunnel proxy"); + let channel = + tonic::transport::Endpoint::from_shared(format!("http://{}", proxy.local_addr)) + .expect("tunnel proxy endpoint") + .connect() + .await + .expect("gRPC channel through the WebSocket tunnel"); let response = tokio::time::timeout( Duration::from_secs(10), From ffc9ce3c96e062131052800f91051c239612c920 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 20:33:42 -0800 Subject: [PATCH 6/9] test(gateway): build the preflight report once Signed-off-by: Matthew Grossman --- crates/openshell-gateway/tests/proxy_config_preflight.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/openshell-gateway/tests/proxy_config_preflight.rs b/crates/openshell-gateway/tests/proxy_config_preflight.rs index 768d995f70..dccaf7fba6 100644 --- a/crates/openshell-gateway/tests/proxy_config_preflight.rs +++ b/crates/openshell-gateway/tests/proxy_config_preflight.rs @@ -99,15 +99,14 @@ async fn preflight_rejects_no_proxy_without_proxy() { let output = fixture .run(&["config", "preflight", "--path", fixture.config_path()]) .await; + let report = combined(&output); assert!( !output.status.success(), - "preflight accepted an incoherent [openshell.drivers.{driver}] proxy table:\n{}", - combined(&output) + "preflight accepted an incoherent [openshell.drivers.{driver}] proxy table:\n{report}" ); assert!( - combined(&output).contains("category=malformed"), - "preflight did not classify the {driver} config as malformed:\n{}", - combined(&output) + report.contains("category=malformed"), + "preflight did not classify the {driver} config as malformed:\n{report}" ); assert_eq!( fs::read(&fixture.config).expect("config after preflight"), From 27138af91129d19aa153debda1f66d3b50dc984e Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 20:53:11 -0800 Subject: [PATCH 7/9] refactor(testkit): make the sandbox and output helpers uniform Move expect_listed, expect_absent, wait_absent and the CLI's sandbox get JSON (now get_json) from Context onto Sandbox, and drop delete_and_expect_absent. Give ExecOutput failure() and CliOutput expect_stdout(), so both outputs offer the same checks. Put with_deadline's description first, like poll_until. Make CliSandboxBuilder::run consume the builder like create, and rename its policy setter to policy_yaml. Update the testkit README's list of what a test can do. Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/cli.rs | 23 +-- .../conformance/src/filesystem_policy.rs | 2 +- tests/suites/conformance/src/lifecycle.rs | 32 ++-- tests/suites/conformance/src/smoke.rs | 3 +- .../drivers/podman/tests/oci_identity.rs | 2 +- .../features/oci-image/tests/oci_image.rs | 4 +- tests/testkit/README.md | 21 ++- tests/testkit/testkit/src/cli.rs | 12 ++ tests/testkit/testkit/src/cli_sandbox.rs | 10 +- tests/testkit/testkit/src/context.rs | 154 ++++++++++-------- tests/testkit/testkit/src/poll.rs | 2 +- 11 files changed, 151 insertions(+), 114 deletions(-) diff --git a/tests/suites/conformance/src/cli.rs b/tests/suites/conformance/src/cli.rs index 5dbef95a23..25286c2dc7 100644 --- a/tests/suites/conformance/src/cli.rs +++ b/tests/suites/conformance/src/cli.rs @@ -25,7 +25,7 @@ async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { let sandbox = cx.cli_sandbox("cs").create().await?; let name = sandbox.name().to_string(); - let got = cx.sandbox_details(&name).await?; + let got = sandbox.get_json().await?; testkit::ensure!( got["name"] == name.as_str(), "sandbox get returned {:?}; expected {name:?}", @@ -41,22 +41,15 @@ async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { testkit::ensure!(listed.is_some(), "sandbox {name} is not in sandbox list"); let marker = format!("openshell-e2e-{}", cx.run_id()); - let exec = cx - .cli([ - "sandbox", "exec", "--name", &name, "--no-tty", "--", "echo", &marker, - ]) - .await? - .success()?; - testkit::ensure!( - exec.stdout == format!("{marker}\n"), - "sandbox exec stdout was {:?}; expected {:?}", - exec.stdout, - format!("{marker}\n") - ); + cx.cli([ + "sandbox", "exec", "--name", &name, "--no-tty", "--", "echo", &marker, + ]) + .await? + .expect_stdout(&format!("{marker}\n"))?; cx.cli(["sandbox", "delete", &name]).await?.success()?; wait_unlisted(cx, &name).await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await } /// A sandbox created with the CLI can be stopped, rejects `sandbox exec` while @@ -88,7 +81,7 @@ async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { cx.cli(["sandbox", "delete", &name]).await?.success()?; wait_unlisted(cx, &name).await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await } /// Wait until `sandbox list` no longer shows the sandbox. diff --git a/tests/suites/conformance/src/filesystem_policy.rs b/tests/suites/conformance/src/filesystem_policy.rs index 4f67ed3f07..79e6937ad8 100644 --- a/tests/suites/conformance/src/filesystem_policy.rs +++ b/tests/suites/conformance/src/filesystem_policy.rs @@ -133,8 +133,8 @@ async fn replaced_policy_path_rejects_next_launch(cx: &mut Context) -> testkit:: // The exec's own error is the rejection report, so keep it apart from // the deadline's. let next = with_deadline( - REJECTED_LAUNCH_TIMEOUT, &format!("({case}) the rejected launch"), + REJECTED_LAUNCH_TIMEOUT, async { Ok(sandbox.exec(["true"]).await) }, ) .await?; diff --git a/tests/suites/conformance/src/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs index c120477a0e..41ec709f06 100644 --- a/tests/suites/conformance/src/lifecycle.rs +++ b/tests/suites/conformance/src/lifecycle.rs @@ -28,7 +28,8 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result { .await? .expect_stdout(&format!("{marker}\n"))?; - cx.delete_and_expect_absent(&sandbox).await + sandbox.delete().await?; + sandbox.expect_absent().await } /// Longer than the supervisor's 30s startup recovery deadline. @@ -52,8 +53,8 @@ async fn exec_outlives_startup_recovery_deadline(cx: &mut Context) -> testkit::R PAST_RECOVERY_DEADLINE.as_secs() ); with_deadline( - PAST_RECOVERY_EXEC_DEADLINE, "the exec past the recovery deadline", + PAST_RECOVERY_EXEC_DEADLINE, sandbox.exec(["sh", "-c", script.as_str()]), ) .await? @@ -114,9 +115,12 @@ async fn delete_is_prompt_when_workload_is_slow_to_handle_term( // Boundary containment may kill the workload before its handler finishes; // this asserts bounded deletion, not a graceful workload exit. with_deadline( - DELETE_DEADLINE, &format!("deleting sandbox {}", sandbox.name()), - cx.delete_and_expect_absent(&sandbox), + DELETE_DEADLINE, + async { + sandbox.delete().await?; + sandbox.expect_absent().await + }, ) .await } @@ -160,7 +164,8 @@ async fn run_to_completion_exit_delete(cx: &mut Context) -> testkit::Result { again.restart_count ); - cx.delete_and_expect_absent(&sandbox).await + sandbox.delete().await?; + sandbox.expect_absent().await } /// A main command that exits during provisioning is reported as the @@ -236,7 +241,7 @@ async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::R ready.name, sandbox.name() ); - cx.expect_listed(&sandbox, SandboxPhase::Ready).await?; + sandbox.expect_listed(SandboxPhase::Ready).await?; let marker = format!("openshell-e2e-{}", cx.run_id()); sandbox @@ -246,7 +251,8 @@ async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::R // The gateway's default image is OpenShell's Ubuntu base image. crate::image::expect_ubuntu(&sandbox).await?; - cx.delete_and_expect_absent(&sandbox).await + sandbox.delete().await?; + sandbox.expect_absent().await } /// A stopped sandbox rejects exec, accepts a second stop, and returns to @@ -291,7 +297,8 @@ async fn stopped_sandbox_can_be_deleted(cx: &mut Context) -> testkit::Result { .create_ready() .await?; sandbox.stop().await?; - cx.delete_and_expect_absent(&sandbox).await + sandbox.delete().await?; + sandbox.expect_absent().await } /// Where the main command of [`canonical_main_environment`] records what it @@ -422,7 +429,7 @@ fi ) .await?; - let initial = cx.sandbox_details(sandbox.name()).await?; + let initial = sandbox.get_json().await?; let initial_instance = main_instance(&initial).to_string(); testkit::ensure!( !initial_instance.is_empty(), @@ -438,7 +445,7 @@ fi RESTART_TIMEOUT, Duration::from_millis(250), async || { - let details = cx.sandbox_details(sandbox.name()).await?; + let details = sandbox.get_json().await?; let instance = main_instance(&details); let replaced = !instance.is_empty() && instance != initial_instance; Ok( @@ -471,7 +478,7 @@ fi } /// The ID of the sandbox's current main process instance, from -/// [`Context::sandbox_details`]. Empty if none is reported. +/// [`Sandbox::get_json`]. Empty if none is reported. fn main_instance(details: &serde_json::Value) -> &str { details["main_process_instance_id"] .as_str() @@ -509,7 +516,8 @@ async fn detached_no_keep_sandbox_is_removed_after_main_exits(cx: &mut Context) .success()?; } for (sandbox, _, code) in &sandboxes { - cx.wait_absent(sandbox, EPHEMERAL_CLEANUP_TIMEOUT) + sandbox + .wait_absent(EPHEMERAL_CLEANUP_TIMEOUT) .await .context(format!("ephemeral sandbox exited {code}"))?; } diff --git a/tests/suites/conformance/src/smoke.rs b/tests/suites/conformance/src/smoke.rs index 8047b6a5f4..eba5ed4995 100644 --- a/tests/suites/conformance/src/smoke.rs +++ b/tests/suites/conformance/src/smoke.rs @@ -21,5 +21,6 @@ async fn create_exec_delete(cx: &mut Context) -> testkit::Result { .await? .expect_stdout(&format!("{marker}\n"))?; - cx.delete_and_expect_absent(&sandbox).await + sandbox.delete().await?; + sandbox.expect_absent().await } diff --git a/tests/suites/drivers/podman/tests/oci_identity.rs b/tests/suites/drivers/podman/tests/oci_identity.rs index d9d6841ed1..baad4235a1 100644 --- a/tests/suites/drivers/podman/tests/oci_identity.rs +++ b/tests/suites/drivers/podman/tests/oci_identity.rs @@ -67,7 +67,7 @@ USER {OCI_IDENTITY} let sandbox = cx .cli_sandbox("oi") .image(&image.tag) - .policy(IMAGE_IDENTITY_POLICY) + .policy_yaml(IMAGE_IDENTITY_POLICY) .command(["sleep", "infinity"]) .create() .await diff --git a/tests/suites/features/oci-image/tests/oci_image.rs b/tests/suites/features/oci-image/tests/oci_image.rs index ccb44b48e4..8e156eb8eb 100644 --- a/tests/suites/features/oci-image/tests/oci_image.rs +++ b/tests/suites/features/oci-image/tests/oci_image.rs @@ -225,7 +225,7 @@ USER app ); // Require the condition reason, so another provisioning failure // cannot pass as a workspace rejection. - let sandbox = oci.cx.sandbox_details(&name).await?; + let sandbox = oci.cx.adopt(&name).await?.get_json().await?; let reasons: Vec<&str> = sandbox["conditions"] .as_array() .into_iter() @@ -284,7 +284,7 @@ impl Oci<'_> { ); let mut builder = self.cx.cli_sandbox(short).image(&image.tag); if let Some(policy) = policy { - builder = builder.policy(policy); + builder = builder.policy_yaml(policy); } let name = builder .command(["sh", "-c", &main]) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 09fcd05052..f3db403b75 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -63,15 +63,21 @@ fails. A killed test process skips cleanup, so names are unique to the run. `Context` currently provides: -- `cx.sandbox(suffix)` to build and create a sandbox with a run-unique name. - The returned `Sandbox` can wait for ready or finished, stop, start, exec, - read its config and draft policy, set a setting, and delete. `exec` output - has `success()`. +- `cx.sandbox(suffix)` to build a sandbox with a run-unique name, then + `create()` it or `create_ready()` to also wait for Ready. The returned + `Sandbox` can wait for ready or finished, stop, start, exec or + `poll_exec` until the output matches, read its config and draft policy, set + a setting, read `sandbox get` JSON with `get_json()`, and delete. Its + `expect_listed`, `expect_absent` and `wait_absent` check list and get. + Exec output has `success()`, `failure()` and `expect_stdout()`. +- `cx.cli_sandbox(suffix)` for a sandbox only `openshell sandbox create` + can make, such as one with `policy_yaml(...)` or `--no-keep`. `create()` + detaches and returns the `Sandbox`; `run()` returns the attached CLI + output. - `cx.track(name)` to register a sandbox created some other way, such as through the CLI, for cleanup. Call it before the create, so a create that fails after the gateway accepted it is still cleaned up. `cx.adopt(name)` returns a `Sandbox` handle for one the CLI created. -- `cx.expect_listed` and `cx.expect_absent` to check a sandbox in list and get. - `cx.expect_compute_driver(name)` to fail unless the gateway runs only that compute driver. Driver suites call it first. - `testkit::base_image()` for the image to create sandboxes from or build on: @@ -82,7 +88,7 @@ fails. A killed test process skips cleanup, so names are unique to the run. every environment; `mise run e2e:workload:build` builds it locally. An override image must run `python3` from one of `testkit::PYTHON_BINARIES`. - `cx.cli([...])` to run the installed CLI against the same gateway, with - `success()`, `failure()`, and `json()` on the output. `cx.cli_handle()` + `success()`, `failure()`, `expect_stdout()`, and `json()` on the output. `cx.cli_handle()` returns that CLI for code that can't borrow `cx`, such as a cleanup hook. - `cx.on_cleanup(name, hook)` to remove something when the test ends. - `testkit::policy` for sandbox policies: `policy::base()` reads @@ -101,7 +107,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `cx.client()` for SDK calls testkit does not wrap. - `error.grpc_code()` for the gRPC status code (`testkit::GrpcCode`) of a request the gateway refused, such as a sandbox create. -- `poll_until` for waiting on other state. +- `poll_until` for waiting on other state, and `with_deadline(what, timeout, + future)` for a step that must finish sooner than its own timeout. ## Running against a gateway diff --git a/tests/testkit/testkit/src/cli.rs b/tests/testkit/testkit/src/cli.rs index 34e02a3758..153bc3c214 100644 --- a/tests/testkit/testkit/src/cli.rs +++ b/tests/testkit/testkit/src/cli.rs @@ -176,6 +176,18 @@ impl CliOutput { Ok(self) } + /// Fail unless the command exited 0 and printed exactly `expected`. + pub fn expect_stdout(self, expected: &str) -> Result { + let output = self.success()?; + crate::ensure!( + output.stdout == expected, + "`{}` printed {:?}; expected {expected:?}", + output.command, + output.stdout + ); + Ok(output) + } + /// Stdout and stderr as one line of words, for matching an error report. /// The CLI wraps reports to the terminal width and draws a gutter, so /// match against this rather than the raw output. diff --git a/tests/testkit/testkit/src/cli_sandbox.rs b/tests/testkit/testkit/src/cli_sandbox.rs index 52c545ce83..1e762792a0 100644 --- a/tests/testkit/testkit/src/cli_sandbox.rs +++ b/tests/testkit/testkit/src/cli_sandbox.rs @@ -48,7 +48,7 @@ impl<'a> CliSandboxBuilder<'a> { /// Create the sandbox with this policy YAML (`--policy`). It is written /// to a temporary file that lives until the CLI exits. #[must_use] - pub fn policy(mut self, yaml: &str) -> Self { + pub fn policy_yaml(mut self, yaml: &str) -> Self { self.policy = Some(yaml.to_string()); self } @@ -79,7 +79,7 @@ impl<'a> CliSandboxBuilder<'a> { /// return the sandbox. It need not be Ready yet. pub async fn create(mut self) -> Result { self.args.push("--detach".to_string()); - self.run() + self.invoke() .await? .success() .context(format!("create sandbox {}", self.name))?; @@ -89,7 +89,11 @@ impl<'a> CliSandboxBuilder<'a> { /// Create the sandbox attached to its main command and return the CLI's /// output, whatever its exit code. For a test of the main command's /// result, or of a create that must fail. - pub async fn run(&self) -> Result { + pub async fn run(self) -> Result { + self.invoke().await + } + + async fn invoke(&self) -> Result { self.cx.track(&self.name); let policy = self.policy.as_deref().map(PolicyFile::new).transpose()?; let mut args = vec!["sandbox", "create", "--name", &self.name, "--no-tty"]; diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index e8029ec511..3b3ee0b8fc 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -105,6 +105,7 @@ impl Context { SandboxBuilder { client: self.client.clone(), grpc: self.grpc.clone(), + cli: self.cli.clone(), policy: None, cleanup: self.cleanup.clone(), spec: SandboxSpec { @@ -122,16 +123,6 @@ impl Context { CliSandboxBuilder::new(self, self.name(suffix)) } - /// `sandbox get --output json` for sandbox `name`. It reports fields the - /// SDK does not expose, such as `restart_count`, `restart_policy`, - /// `main_process_instance_id`, and `conditions`. - pub async fn sandbox_details(&self, name: &str) -> Result { - self.cli(["sandbox", "get", name, "--output", "json"]) - .await? - .success()? - .json() - } - /// Track a sandbox the test is about to create through the CLI, so cleanup /// deletes it even if the test fails after the gateway accepted the create. /// Call this before the CLI command; use [`Context::adopt`] afterwards. @@ -192,54 +183,13 @@ impl Context { Ok(Sandbox { client: self.client.clone(), grpc: self.grpc.clone(), + cli: self.cli.clone(), cleanup: self.cleanup.clone(), name: name.to_string(), id: found.id, }) } - /// Wait up to `timeout` for the gateway to remove the sandbox by itself, - /// for example after a `--no-keep` main command exits, then check it is - /// gone from both get and list. - pub async fn wait_absent(&self, sandbox: &Sandbox, timeout: Duration) -> Result { - self.client - .wait_deleted(sandbox.name(), timeout, Some(sandbox.id())) - .await - .context(format!("wait for sandbox {} to be removed", sandbox.name()))?; - self.expect_absent(sandbox).await - } - - /// Delete the sandbox, then check it is gone from both get and list. - pub async fn delete_and_expect_absent(&self, sandbox: &Sandbox) -> Result { - sandbox.delete().await?; - self.expect_absent(sandbox).await - } - - /// Fail unless the sandbox is gone from both get and list. - pub async fn expect_absent(&self, sandbox: &Sandbox) -> Result { - let name = sandbox.name(); - match self.client.get_sandbox(name).await { - Err(SdkError::NotFound { .. }) => {} - Err(error) => return Err(Error::from(error).context(format!("get sandbox {name}"))), - Ok(found) => { - return Err(Error::msg(format!( - "sandbox {name} still exists (id {}, phase {:?})", - found.id, found.phase - ))); - } - } - let listed = self - .client - .list_all_sandboxes(ListOptions::default()) - .await - .context("list sandboxes")?; - crate::ensure!( - !listed.iter().any(|entry| entry.name == name), - "sandbox {name} is absent from get but still listed" - ); - Ok(()) - } - /// Fail unless the gateway runs exactly one compute driver, `name`, as /// gateway info reports it. Driver suites check this first, so a suite /// pointed at another driver's gateway fails instead of testing it. @@ -271,25 +221,6 @@ impl Context { Ok(()) } - /// Fail unless `list` shows the sandbox in `phase`. - pub async fn expect_listed(&self, sandbox: &Sandbox, phase: SandboxPhase) -> Result { - let name = sandbox.name(); - let listed = self - .client - .list_all_sandboxes(ListOptions::default()) - .await - .context("list sandboxes")?; - let entry = listed.iter().find(|entry| entry.name == name); - crate::ensure!(entry.is_some(), "sandbox {name} is not listed"); - let entry = entry.expect("checked above"); - crate::ensure!( - entry.phase == phase, - "sandbox {name} is listed in phase {:?}; expected {phase:?}", - entry.phase - ); - Ok(()) - } - /// Run the cleanup hooks, including deleting the sandboxes the test /// created and has not deleted. Returns the cleanup failures; they are /// reported separately from the test result. @@ -344,6 +275,7 @@ impl Grpc { pub struct SandboxBuilder { client: WorkspaceScopedClient, grpc: Grpc, + cli: Cli, policy: Option, cleanup: Cleanup, spec: SandboxSpec, @@ -426,6 +358,7 @@ impl SandboxBuilder { Ok(Sandbox { client: self.client, grpc: self.grpc, + cli: self.cli, cleanup: self.cleanup, name, id, @@ -481,6 +414,7 @@ fn policy_create_request( pub struct Sandbox { client: WorkspaceScopedClient, grpc: Grpc, + cli: Cli, cleanup: Cleanup, name: String, id: String, @@ -510,6 +444,18 @@ impl ExecOutput { Ok(self) } + /// Fail unless the command exited non-zero, e.g. when a request must be + /// rejected. + pub fn failure(self) -> Result { + crate::ensure!( + self.exit_code != 0, + "{} succeeded but was expected to fail; stdout: {}", + self.command, + self.stdout.trim() + ); + Ok(self) + } + /// Fail unless the command exited 0 and printed exactly `expected`. pub fn expect_stdout(self, expected: &str) -> Result { let output = self.success()?; @@ -722,6 +668,72 @@ impl Sandbox { .await } + /// `sandbox get --output json`. It reports fields the SDK does not + /// expose, such as `restart_count`, `restart_policy`, + /// `main_process_instance_id`, and `conditions`. + pub async fn get_json(&self) -> Result { + self.cli + .run(["sandbox", "get", &self.name, "--output", "json"]) + .await? + .success()? + .json() + } + + /// Fail unless `list` shows the sandbox in `phase`. + pub async fn expect_listed(&self, phase: SandboxPhase) -> Result { + let name = &self.name; + let listed = self + .client + .list_all_sandboxes(ListOptions::default()) + .await + .context("list sandboxes")?; + let entry = listed.iter().find(|entry| entry.name == *name); + crate::ensure!(entry.is_some(), "sandbox {name} is not listed"); + let entry = entry.expect("checked above"); + crate::ensure!( + entry.phase == phase, + "sandbox {name} is listed in phase {:?}; expected {phase:?}", + entry.phase + ); + Ok(()) + } + + /// Fail unless the sandbox is gone from both get and list. + pub async fn expect_absent(&self) -> Result { + let name = &self.name; + match self.client.get_sandbox(name).await { + Err(SdkError::NotFound { .. }) => {} + Err(error) => return Err(Error::from(error).context(format!("get sandbox {name}"))), + Ok(found) => { + return Err(Error::msg(format!( + "sandbox {name} still exists (id {}, phase {:?})", + found.id, found.phase + ))); + } + } + let listed = self + .client + .list_all_sandboxes(ListOptions::default()) + .await + .context("list sandboxes")?; + crate::ensure!( + !listed.iter().any(|entry| entry.name == *name), + "sandbox {name} is absent from get but still listed" + ); + Ok(()) + } + + /// Wait up to `timeout` for the gateway to remove the sandbox by itself, + /// for example after a `--no-keep` main command exits, then check it is + /// gone from both get and list. + pub async fn wait_absent(&self, timeout: Duration) -> Result { + self.client + .wait_deleted(&self.name, timeout, Some(&self.id)) + .await + .context(format!("wait for sandbox {} to be removed", self.name))?; + self.expect_absent().await + } + /// Delete the sandbox and wait until it is gone. pub async fn delete(&self) -> Result { self.client diff --git a/tests/testkit/testkit/src/poll.rs b/tests/testkit/testkit/src/poll.rs index a9bd0041f5..f65592be88 100644 --- a/tests/testkit/testkit/src/poll.rs +++ b/tests/testkit/testkit/src/poll.rs @@ -18,8 +18,8 @@ pub enum Poll { /// Fail unless `future` finishes within `timeout`. For a deadline shorter /// than the step's own, such as one that must fit inside the runner's kill. pub async fn with_deadline( - timeout: Duration, what: &str, + timeout: Duration, future: impl Future>, ) -> Result { tokio::time::timeout(timeout, future) From 559f3756f9e95511a7f0d2b092ba63cf5b99b057 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 21:22:19 -0800 Subject: [PATCH 8/9] refactor(testkit): reuse YamlFile and report stdout when CLI checks fail Replace CliSandboxBuilder's private PolicyFile with testkit::YamlFile, and drop YamlFile's map_err calls now that I/O errors convert into testkit::Error. CliOutput::success now includes stdout in its failure message, like ExecOutput::success, since many commands report errors there. Signed-off-by: Matthew Grossman --- tests/testkit/testkit/src/cli.rs | 3 ++- tests/testkit/testkit/src/cli_sandbox.rs | 23 +++-------------------- tests/testkit/testkit/src/yaml_file.rs | 2 -- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/tests/testkit/testkit/src/cli.rs b/tests/testkit/testkit/src/cli.rs index 153bc3c214..6f4e38cd75 100644 --- a/tests/testkit/testkit/src/cli.rs +++ b/tests/testkit/testkit/src/cli.rs @@ -156,9 +156,10 @@ impl CliOutput { pub fn success(self) -> Result { crate::ensure!( self.exit_code == 0, - "`{}` exited with {}; stderr: {}", + "`{}` exited with {}; stdout: {}; stderr: {}", self.command, self.exit_code, + self.stdout.trim(), self.stderr.trim() ); Ok(self) diff --git a/tests/testkit/testkit/src/cli_sandbox.rs b/tests/testkit/testkit/src/cli_sandbox.rs index 1e762792a0..33e91af3e4 100644 --- a/tests/testkit/testkit/src/cli_sandbox.rs +++ b/tests/testkit/testkit/src/cli_sandbox.rs @@ -7,7 +7,8 @@ use crate::cli::CliOutput; use crate::context::{Context, Sandbox}; -use crate::error::{Error, Result, ResultExt}; +use crate::error::{Result, ResultExt}; +use crate::yaml_file::YamlFile; /// A sandbox to create with `openshell sandbox create`. Start one with /// [`Context::cli_sandbox`]. @@ -95,7 +96,7 @@ impl<'a> CliSandboxBuilder<'a> { async fn invoke(&self) -> Result { self.cx.track(&self.name); - let policy = self.policy.as_deref().map(PolicyFile::new).transpose()?; + let policy = self.policy.as_deref().map(YamlFile::new).transpose()?; let mut args = vec!["sandbox", "create", "--name", &self.name, "--no-tty"]; if let Some(policy) = &policy { args.extend(["--policy", policy.path()?]); @@ -108,21 +109,3 @@ impl<'a> CliSandboxBuilder<'a> { self.cx.cli(args).await } } - -/// A policy written to a temporary file for `sandbox create --policy`. -struct PolicyFile(tempfile::NamedTempFile); - -impl PolicyFile { - fn new(contents: &str) -> Result { - let file = tempfile::NamedTempFile::new().context("create policy file")?; - std::fs::write(file.path(), contents) - .context(format!("write {}", file.path().display()))?; - Ok(Self(file)) - } - - fn path(&self) -> Result<&str> { - let path = self.0.path(); - path.to_str() - .ok_or_else(|| Error::msg(format!("{} is not UTF-8", path.display()))) - } -} diff --git a/tests/testkit/testkit/src/yaml_file.rs b/tests/testkit/testkit/src/yaml_file.rs index 04ca3d815d..4549c7e95b 100644 --- a/tests/testkit/testkit/src/yaml_file.rs +++ b/tests/testkit/testkit/src/yaml_file.rs @@ -13,10 +13,8 @@ impl YamlFile { let file = tempfile::Builder::new() .suffix(".yaml") .tempfile() - .map_err(|error| Error::from_error(&error)) .context("create a temporary YAML file")?; std::fs::write(file.path(), contents) - .map_err(|error| Error::from_error(&error)) .context(format!("write {}", file.path().display()))?; Ok(Self(file)) } From f8b1ceddc17d1d56181c979fed3f44bbe808f004 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 21:36:12 -0800 Subject: [PATCH 9/9] refactor(testkit): use the new sandbox helpers with the credential gating changes Add SandboxBuilder::image and CliSandboxBuilder::create_ready, so both builders set an image and wait for Ready the same way. Convert tonic::Status into testkit::Error, keeping its code, so gRPC results take .context() directly, and drop Error::from_status. Create the host-gateway sandbox with cx.sandbox(..).image(..).policy(..) instead of a hand-written CLI create. In credential gating, use create_ready and poll_exec. Drop the remaining map_err calls on I/O results in host-server-support. Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/cli.rs | 3 +- .../conformance/src/credential_gating.rs | 42 +++++++---------- tests/suites/conformance/src/image.rs | 2 +- .../drivers/podman/tests/oci_identity.rs | 3 +- .../host-gateway/tests/host_gateway.rs | 45 ++++++------------- .../kubernetes-updates/tests/update.rs | 4 +- tests/support/host-server/src/lib.rs | 4 +- tests/testkit/README.md | 7 +-- tests/testkit/testkit/src/cli_sandbox.rs | 7 +++ tests/testkit/testkit/src/context.rs | 12 ++--- tests/testkit/testkit/src/error.rs | 18 ++++---- 11 files changed, 63 insertions(+), 84 deletions(-) diff --git a/tests/suites/conformance/src/cli.rs b/tests/suites/conformance/src/cli.rs index 25286c2dc7..f06f16d2d0 100644 --- a/tests/suites/conformance/src/cli.rs +++ b/tests/suites/conformance/src/cli.rs @@ -59,10 +59,9 @@ async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { let sandbox = cx .cli_sandbox("cl") .command(["sh", "-c", "exec sleep infinity"]) - .create() + .create_ready() .await?; let name = sandbox.name().to_string(); - sandbox.wait_ready().await?; cx.cli(["sandbox", "stop", &name]).await?.success()?; sandbox diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index ea67464e35..2d67926ef0 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -14,7 +14,7 @@ use host_server_support::{HttpRecorder, Request, WebSocketRecorder}; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; use testkit::{ - Context, Error, GrpcCode, PYTHON_BINARIES, Poll, Result, ResultExt as _, Sandbox, YamlFile, + Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox, YamlFile, }; const TEST_HOST: &str = "host.openshell.internal"; @@ -298,10 +298,8 @@ fn sandbox_with( ) -> testkit::SandboxBuilder { cx.sandbox(suffix) .policy(policy) - .spec(|spec| { - spec.image = Some(testkit::python_image()); - spec.providers = providers.iter().map(ToString::to_string).collect(); - }) + .image(testkit::python_image()) + .spec(|spec| spec.providers = providers.iter().map(ToString::to_string).collect()) .command(["sleep", "infinity"]) } @@ -313,9 +311,9 @@ async fn create_sandbox( policy: proto::SandboxPolicy, providers: &[&str], ) -> Result { - let sandbox = sandbox_with(cx, suffix, policy, providers).create().await?; - sandbox.wait_ready().await?; - Ok(sandbox) + sandbox_with(cx, suffix, policy, providers) + .create_ready() + .await } /// Fail unless the gateway rejects the sandbox as a failed precondition, and @@ -384,25 +382,15 @@ fn redact(request: &Request) -> String { /// Wait until new exec sessions report `expected` from [`TOKEN_PRESENCE`]. async fn wait_for_token(sandbox: &Sandbox, expected: &str) -> Result { - testkit::poll_until( - &format!("the provider environment to report {expected}"), - Duration::from_secs(30), - Duration::from_millis(500), - async || { - let output = sandbox.exec(["sh", "-lc", TOKEN_PRESENCE]).await?; - Ok(if output.stdout.contains(expected) { - Poll::Ready(()) - } else { - Poll::Pending(format!( - "exit {}; stdout: {}; stderr: {}", - output.exit_code, - output.stdout.trim(), - output.stderr.trim() - )) - }) - }, - ) - .await + sandbox + .poll_exec( + &format!("the provider environment to report {expected}"), + Duration::from_secs(30), + ["sh", "-lc", TOKEN_PRESENCE], + |output| output.stdout.contains(expected), + ) + .await?; + Ok(()) } /// Replace `sandbox`'s policy with `policy` and wait until it is applied. diff --git a/tests/suites/conformance/src/image.rs b/tests/suites/conformance/src/image.rs index f8faf80d71..58fb294253 100644 --- a/tests/suites/conformance/src/image.rs +++ b/tests/suites/conformance/src/image.rs @@ -12,7 +12,7 @@ async fn explicit_image_is_used(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("ei") .command(["sleep", "infinity"]) - .spec(|spec| spec.image = Some(image.clone())) + .image(image.clone()) .create_ready() .await?; expect_ubuntu(&sandbox) diff --git a/tests/suites/drivers/podman/tests/oci_identity.rs b/tests/suites/drivers/podman/tests/oci_identity.rs index baad4235a1..ae214ed283 100644 --- a/tests/suites/drivers/podman/tests/oci_identity.rs +++ b/tests/suites/drivers/podman/tests/oci_identity.rs @@ -69,10 +69,9 @@ USER {OCI_IDENTITY} .image(&image.tag) .policy_yaml(IMAGE_IDENTITY_POLICY) .command(["sleep", "infinity"]) - .create() + .create_ready() .await .context("sandbox starts from the OCI identity image")?; - sandbox.wait_ready().await?; let supervisor = engine .container_for_role(sandbox.name(), "supervisor") diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index 472197dfd1..7d285116e4 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -11,7 +11,8 @@ use host_server_support::HttpRecorder; use testkit::policy; -use testkit::{Context, PYTHON_BINARIES, Result, ResultExt as _, YamlFile}; +use testkit::sdk::raw::proto; +use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; /// The image's Python, which needs nothing beyond its standard library. The /// policy allows [`PYTHON_BINARIES`], the real paths behind `python3` in @@ -42,8 +43,6 @@ print(response.decode(errors="replace")) async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result { let marker = format!("host-gateway-{}", cx.run_id()); let server = HttpRecorder::start(marker.clone()).await?; - let policy = YamlFile::new(&policy(server.port)?)?; - let port = server.port.to_string(); let image = testkit::python_image(); // The client runs through `sandbox exec` in a Ready sandbox, not as the @@ -51,33 +50,17 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result // attach to a main process that already finished with "sandbox is not // ready" when the request reaches a replica that doesn't hold the // sandbox's session. - let name = cx.name("hg"); - cx.track(&name); - cx.cli([ - "sandbox", - "create", - "--name", - &name, - "--from", - &image, - "--policy", - policy.path()?, - "--detach", - "--no-tty", - "--", - "sh", - "-c", - "exec sleep infinity", - ]) - .await? - .success() - .context(format!("create a sandbox from {image}"))?; - cx.adopt(&name).await?.wait_ready().await?; + let sandbox = cx + .sandbox("hg") + .image(image.clone()) + .policy(policy(server.port)) + .command(["sh", "-c", "exec sleep infinity"]) + .create_ready() + .await + .context(format!("create a sandbox from {image}"))?; - let client = cx - .cli([ - "sandbox", "exec", "--name", &name, "--no-tty", "--", PYTHON, "-c", CLIENT, &port, - ]) + let client = sandbox + .exec([PYTHON, "-c", CLIENT, &server.port.to_string()]) .await? .success() .context("run the client in the sandbox")?; @@ -94,7 +77,7 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result /// connect to `host.openshell.internal` on `port`. The endpoint names only the /// host and port, as the policy docs describe, so the driver's address for the /// name is what makes it reachable, not an `allowed_ips` range. -fn policy(port: u16) -> Result { +fn policy(port: u16) -> proto::SandboxPolicy { let mut policy = policy::base(); policy::allow( &mut policy, @@ -103,5 +86,5 @@ fn policy(port: u16) -> Result { port, PYTHON_BINARIES, ); - policy::to_yaml(&policy) + policy } diff --git a/tests/suites/features/kubernetes-updates/tests/update.rs b/tests/suites/features/kubernetes-updates/tests/update.rs index 61ac57e9fa..ab71a028a1 100644 --- a/tests/suites/features/kubernetes-updates/tests/update.rs +++ b/tests/suites/features/kubernetes-updates/tests/update.rs @@ -458,7 +458,7 @@ async fn image_update_retains_workspace_and_identity(cx: &mut Context) -> testki let sandbox = cx // The testkit's unique prefix leaves five characters of the name limit. .sandbox("image") - .spec(|spec| spec.image = Some(ORIGINAL_IMAGE.to_string())) + .image(ORIGINAL_IMAGE.as_str()) .create() .await?; sandbox.wait_ready().await?; @@ -597,7 +597,7 @@ async fn terminal_workload_can_stop_update_and_restart(cx: &mut Context) -> test let sandbox = cx .sandbox("term") .restart_policy(testkit::sdk::SandboxRestartPolicy::Never) - .spec(|spec| spec.image = Some(ORIGINAL_IMAGE.to_string())) + .image(ORIGINAL_IMAGE.as_str()) .create() .await?; sandbox.wait_ready().await?; diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 0c11e4ceca..8ac8528570 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -15,7 +15,7 @@ mod websocket; use std::time::Duration; -use testkit::{Error, Result, ResultExt as _}; +use testkit::{Result, ResultExt as _}; use tokio::net::TcpListener; pub use http::{HttpRecorder, Request}; @@ -29,11 +29,9 @@ const MAX_REQUEST: usize = 1024 * 1024; async fn bind() -> Result<(TcpListener, u16)> { let listener = TcpListener::bind(("0.0.0.0", 0)) .await - .map_err(|error| Error::from_error(&error)) .context("bind a host server")?; let port = listener .local_addr() - .map_err(|error| Error::from_error(&error)) .context("read the host server's address")? .port(); Ok((listener, port)) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index f3db403b75..061c9edee8 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -64,7 +64,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. `Context` currently provides: - `cx.sandbox(suffix)` to build a sandbox with a run-unique name, then - `create()` it or `create_ready()` to also wait for Ready. The returned + `create()` it or `create_ready()` to also wait for Ready. Set its image + with `image(..)` and its policy with `policy(..)`. The returned `Sandbox` can wait for ready or finished, stop, start, exec or `poll_exec` until the output matches, read its config and draft policy, set a setting, read `sandbox get` JSON with `get_json()`, and delete. Its @@ -72,8 +73,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. Exec output has `success()`, `failure()` and `expect_stdout()`. - `cx.cli_sandbox(suffix)` for a sandbox only `openshell sandbox create` can make, such as one with `policy_yaml(...)` or `--no-keep`. `create()` - detaches and returns the `Sandbox`; `run()` returns the attached CLI - output. + detaches and returns the `Sandbox`, `create_ready()` also waits for Ready, + and `run()` returns the attached CLI output. - `cx.track(name)` to register a sandbox created some other way, such as through the CLI, for cleanup. Call it before the create, so a create that fails after the gateway accepted it is still cleaned diff --git a/tests/testkit/testkit/src/cli_sandbox.rs b/tests/testkit/testkit/src/cli_sandbox.rs index 33e91af3e4..8d58968cd2 100644 --- a/tests/testkit/testkit/src/cli_sandbox.rs +++ b/tests/testkit/testkit/src/cli_sandbox.rs @@ -87,6 +87,13 @@ impl<'a> CliSandboxBuilder<'a> { self.cx.adopt(&self.name).await } + /// Create the sandbox and wait until it is Ready. + pub async fn create_ready(self) -> Result { + let sandbox = self.create().await?; + sandbox.wait_ready().await?; + Ok(sandbox) + } + /// Create the sandbox attached to its main command and return the CLI's /// output, whatever its exit code. For a test of the main command's /// result, or of a create that must fail. diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 3b3ee0b8fc..5172213c93 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -200,7 +200,6 @@ impl Context { .await? .get_gateway_info(proto::GetGatewayInfoRequest {}) .await - .map_err(|status| Error::from_error(&status)) .context("get gateway info")? .into_inner(); let drivers: Vec<(&str, &str)> = info @@ -293,6 +292,13 @@ impl SandboxBuilder { self } + /// Create the sandbox from `image` instead of the gateway's default. + #[must_use] + pub fn image(mut self, image: impl Into) -> Self { + self.spec.image = Some(image.into()); + self + } + #[must_use] pub fn restart_policy(mut self, policy: SandboxRestartPolicy) -> Self { self.spec.restart_policy = policy; @@ -346,7 +352,6 @@ impl SandboxBuilder { .await? .create_sandbox(request) .await - .map_err(|status| Error::from_status(&status)) .context(format!("create sandbox {name}"))?; self.client .get_sandbox(&name) @@ -564,7 +569,6 @@ impl Sandbox { .await? .update_config(request) .await - .map_err(|status| Error::from_error(&status)) .context(format!("set {key} on sandbox {}", self.name))?; Ok(()) } @@ -581,7 +585,6 @@ impl Sandbox { .await? .get_sandbox_config(request) .await - .map_err(|status| Error::from_error(&status)) .context(format!("get config of sandbox {}", self.name))? .into_inner()) } @@ -599,7 +602,6 @@ impl Sandbox { .await? .get_draft_policy(request) .await - .map_err(|status| Error::from_error(&status)) .context(format!("get draft policy of sandbox {}", self.name))? .into_inner()) } diff --git a/tests/testkit/testkit/src/error.rs b/tests/testkit/testkit/src/error.rs index 11ad20c853..3434e3fb1b 100644 --- a/tests/testkit/testkit/src/error.rs +++ b/tests/testkit/testkit/src/error.rs @@ -26,14 +26,6 @@ impl Error { } } - /// A gateway's refusal, keeping its status code for [`Error::grpc_code`]. - pub fn from_status(status: &tonic::Status) -> Self { - Self { - code: Some(status.code()), - ..Self::from_error(status) - } - } - /// The gRPC status code, when the gateway refused a request. pub fn grpc_code(&self) -> Option { self.code @@ -85,6 +77,16 @@ impl From for Error { } } +/// A gateway's refusal, keeping its status code for [`Error::grpc_code`]. +impl From for Error { + fn from(status: tonic::Status) -> Self { + Self { + code: Some(status.code()), + ..Self::from_error(&status) + } + } +} + impl From for Error { fn from(error: std::io::Error) -> Self { Self::from_error(&error)