diff --git a/.config/nextest.toml b/.config/nextest.toml index fef1880a35..636c10a725 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -10,10 +10,6 @@ failure-output = "immediate-final" slow-timeout = { period = "30s", terminate-after = 4 } [profile.e2e-kubernetes] -# Temporary waiver: port-forward reconnect can attach read-only while the old -# SSH session retains stdin ownership. Keep recovery coverage on other drivers. -# Remove this exclusion after resolving https://github.com/NVIDIA/OpenShell/issues/4445. -default-filter = "not test(/^canonical_main_connect_recovers_its_ssh_transport$/)" fail-fast = false status-level = "slow" final-status-level = "fail" diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 791761befb..b5bdd44c4e 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -82,6 +82,7 @@ jobs: {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"cli-attach","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-userns-default","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"}, diff --git a/Cargo.lock b/Cargo.lock index d178930a4a..8fcc24130f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5151,6 +5151,7 @@ dependencies = [ "openshell-policy", "openshell-prover", "openshell-providers", + "openshell-sdk", "openshell-supervisor-control", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", diff --git a/TESTING.md b/TESTING.md index a3936ad649..cd983f2139 100644 --- a/TESTING.md +++ b/TESTING.md @@ -424,11 +424,14 @@ Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. - Conformance (`tests/suites/conformance`) - the testkit `conformance` crate: - lifecycle, the CLI, persistent workspaces, file transfer, and the policy - advisor. Driver E2E runs it with `openshell-conformance` against the lane's - gateway and writes a JUnit report under `results/`; set - `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program, as Branch E2E does - with the static build it makes once per architecture. The + lifecycle, the CLI, persistent workspaces, file transfer, the policy + advisor, and credential gating. Driver E2E runs it with + `openshell-conformance` against the lane's gateway and writes a JUnit report + under `results/`; set `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program, + as Branch E2E does with the static build it makes once per architecture. + The Docker and Podman runners build `openshell/e2e-python:dev` for it; the + Kubernetes and VM runners set `OPENSHELL_TEST_PYTHON_IMAGE` to a pullable + Python image unless it is already set. The installed-artifact conformance lanes run the same crate through `openshell-test-runner` (see above). - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or @@ -593,6 +596,11 @@ rootful and rootless, and the `k3s` and `k3s-ha` installers. The tests run on the K3s node, so the K3s installers set the chart's `server.hostGatewayIP` to the node's address. +The `cli-attach` feature suite (`tests/suites/features/cli-attach`) checks how +`sandbox connect` supervises its `ssh` process: it reconnects after losing the +SSH transport and forwards a SIGTERM to `ssh`. The tests find those processes +in `/proc`, so they need a Linux test host. CI runs them on Docker rootful. + Run the VM-backed Rust CLI e2e suite: ```shell @@ -656,6 +664,10 @@ explains the isolation backend separation behind the capability-free runtime. On a **remote** cluster, drop the `e2e-host-gateway` feature. Those tests rely on the sandbox-side `host.openshell.internal` alias reaching the machine running the tests, which is unreachable from pods on a remote cluster, so they fail. +The conformance suite's `credential_gating` tests need the alias to reach that +machine too, and the feature selection doesn't skip them, so they fail on a +remote cluster. [#5409](https://github.com/NVIDIA/OpenShell/issues/5409) +tracks handling that. Left enabled, the `host_gateway_alias` suite fails because `host.openshell.internal` does not resolve inside the pod, so the gateway SSRF-denies the request (`DNS resolution failed` / `ssrf_denied`) — a networking @@ -788,7 +800,7 @@ cargo test --manifest-path e2e/rust/Cargo.toml --features e2e --test sync Run a single Docker-only test directly with cargo: ```shell -cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test docker_preflight +cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test docker_corporate_proxy ``` The harness (`e2e/rust/src/harness/`) provides: diff --git a/crates/openshell-cli/tests/doctor_integration.rs b/crates/openshell-cli/tests/doctor_integration.rs new file mode 100644 index 0000000000..343dc7a9c2 --- /dev/null +++ b/crates/openshell-cli/tests/doctor_integration.rs @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `openshell doctor check` against a fake `docker` on `PATH`, so the result +//! doesn't depend on the host's Docker. + +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +struct Doctor { + combined: String, + code: i32, + elapsed: Duration, +} + +/// Run `openshell doctor check` with only `bin` on `PATH`. +fn doctor_check(bin: &Path) -> Doctor { + let home = tempfile::tempdir().expect("create isolated config dir"); + let start = Instant::now(); + let output = Command::new(env!("CARGO_BIN_EXE_openshell")) + .args(["doctor", "check"]) + .env("XDG_CONFIG_HOME", home.path()) + .env("HOME", home.path()) + .env("PATH", bin) + .env("NO_COLOR", "1") + .env_remove("DOCKER_HOST") + .env_remove("OPENSHELL_GATEWAY") + .env_remove("OPENSHELL_GATEWAY_ENDPOINT") + .stdin(Stdio::null()) + .output() + .expect("run openshell doctor check"); + Doctor { + combined: format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ), + code: output.status.code().unwrap_or(-1), + elapsed: start.elapsed(), + } +} + +/// Write an executable `docker` script to `bin` that records its arguments +/// in `bin/docker-args` and then runs `body`. +fn fake_docker(bin: &Path, body: &str) { + let docker = bin.join("docker"); + let args = bin.join("docker-args"); + fs::write( + &docker, + format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" > '{}'\n{body}\n", + args.display() + ), + ) + .expect("write fake docker"); + fs::set_permissions(&docker, fs::Permissions::from_mode(0o755)).expect("chmod fake docker"); +} + +fn docker_args(bin: &Path) -> String { + fs::read_to_string(bin.join("docker-args")).unwrap_or_default() +} + +#[test] +fn doctor_check_passes_when_docker_info_succeeds() { + let bin = tempfile::tempdir().expect("create fake bin dir"); + fake_docker(bin.path(), "echo 27.3.1"); + + let doctor = doctor_check(bin.path()); + assert_eq!(doctor.code, 0, "doctor check failed:\n{}", doctor.combined); + assert_eq!( + docker_args(bin.path()).trim(), + "info --format {{.ServerVersion}}", + "doctor check ran docker with unexpected arguments" + ); + for expected in ["Docker", "ok (version 27.3.1)", "All checks passed"] { + assert!( + doctor.combined.contains(expected), + "expected {expected:?} in doctor check output:\n{}", + doctor.combined + ); + } +} + +#[test] +fn doctor_check_fails_fast_when_docker_info_fails() { + let bin = tempfile::tempdir().expect("create fake bin dir"); + fake_docker( + bin.path(), + "echo 'Cannot connect to the Docker daemon' >&2\nexit 1", + ); + + let doctor = doctor_check(bin.path()); + assert_ne!( + doctor.code, 0, + "doctor check passed although docker info failed:\n{}", + doctor.combined + ); + assert!( + doctor.elapsed < Duration::from_secs(10), + "doctor check took {:?}", + doctor.elapsed + ); + for expected in [ + "Docker", + "FAILED", + "docker info failed", + "Cannot connect to the Docker daemon", + ] { + assert!( + doctor.combined.contains(expected), + "expected {expected:?} in doctor check output:\n{}", + doctor.combined + ); + } + assert!( + !doctor.combined.contains("All checks passed"), + "doctor check reported success:\n{}", + doctor.combined + ); +} + +#[test] +fn doctor_check_fails_without_docker_on_path() { + let bin = tempfile::tempdir().expect("create empty bin dir"); + + let doctor = doctor_check(bin.path()); + assert_ne!( + doctor.code, 0, + "doctor check passed without docker:\n{}", + doctor.combined + ); + assert!( + doctor.combined.contains("failed to execute docker info"), + "doctor check did not report that docker could not run:\n{}", + doctor.combined + ); +} diff --git a/crates/openshell-gateway/tests/proxy_config_preflight.rs b/crates/openshell-gateway/tests/proxy_config_preflight.rs new file mode 100644 index 0000000000..dccaf7fba6 --- /dev/null +++ b/crates/openshell-gateway/tests/proxy_config_preflight.rs @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! An incoherent driver proxy table is fatal, rather than leaving every dial +//! direct while looking configured. A bypass list without a proxy URL means +//! the operator believed proxying was in effect. + +#![cfg(unix)] + +use std::fs; +use std::path::PathBuf; +use std::process::Output; +use std::time::Duration; + +/// Drivers with an upstream proxy table, compiled into this gateway. +const DRIVERS: &[&str] = &[ + #[cfg(feature = "compute-driver-docker")] + "docker", + #[cfg(feature = "compute-driver-podman")] + "podman", + #[cfg(feature = "compute-driver-vm")] + "vm", +]; + +struct Fixture { + root: tempfile::TempDir, + config: PathBuf, +} + +impl Fixture { + /// A gateway config that selects `driver` and sets only `no_proxy` in its + /// table. + fn incoherent(driver: &str) -> Self { + let root = tempfile::tempdir().expect("fixture root"); + let config = root.path().join("gateway.toml"); + fs::write( + &config, + format!( + "[openshell] +version = 2 +[openshell.gateway] +disable_tls = true +bind_address = \"127.0.0.1:0\" +compute_driver = {driver:?} +[openshell.gateway.auth] +allow_unauthenticated_users = true +[openshell.drivers.{driver}] +no_proxy = \"10.0.0.0/8\" +" + ), + ) + .expect("gateway configuration"); + Self { root, config } + } + + async fn run(&self, args: &[&str]) -> Output { + let mut command = tokio::process::Command::new(env!("CARGO_BIN_EXE_openshell-gateway")); + command + .env_clear() + .env("HOME", self.root.path()) + .env("PATH", "/usr/bin:/bin") + .env("NO_COLOR", "1") + .env("XDG_CONFIG_HOME", self.root.path().join("config-home")) + .env("XDG_STATE_HOME", self.root.path().join("state-home")) + .args(args) + .kill_on_drop(true); + tokio::time::timeout(Duration::from_secs(30), command.output()) + .await + .expect("the gateway must exit instead of starting") + .expect("run openshell-gateway") + } + + fn config_path(&self) -> &str { + self.config.to_str().expect("UTF-8 config path") + } +} + +fn combined(output: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +#[test] +fn proxy_drivers_are_compiled_in() { + assert!( + !DRIVERS.is_empty(), + "build with a compute driver that has a proxy table" + ); +} + +#[tokio::test] +async fn preflight_rejects_no_proxy_without_proxy() { + for driver in DRIVERS { + let fixture = Fixture::incoherent(driver); + let original = fs::read(&fixture.config).expect("original config"); + let output = fixture + .run(&["config", "preflight", "--path", fixture.config_path()]) + .await; + let report = combined(&output); + assert!( + !output.status.success(), + "preflight accepted an incoherent [openshell.drivers.{driver}] proxy table:\n{report}" + ); + assert!( + report.contains("category=malformed"), + "preflight did not classify the {driver} config as malformed:\n{report}" + ); + assert_eq!( + fs::read(&fixture.config).expect("config after preflight"), + original, + "preflight changed the {driver} config" + ); + } +} + +#[tokio::test] +async fn startup_rejects_no_proxy_without_proxy_and_names_the_keys() { + for driver in DRIVERS { + let fixture = Fixture::incoherent(driver); + let output = fixture.run(&["--config", fixture.config_path()]).await; + let report = combined(&output); + assert!( + !output.status.success(), + "the gateway started with an incoherent [openshell.drivers.{driver}] proxy table:\n{report}" + ); + // Matched on the key names, not the wording, so error wrapping can + // change. + assert!( + report.contains("no_proxy") && report.contains("https_proxy"), + "the {driver} startup error does not name the offending keys:\n{report}" + ); + } +} diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index a7c13be2cb..869dce9291 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -142,6 +142,7 @@ hyper-rustls = { version = "0.27", default-features = false, features = ["native rcgen = { workspace = true } rsa = { version = "0.9", features = ["pem"] } tokio-tungstenite = { workspace = true } +openshell-sdk = { path = "../openshell-sdk" } futures-util = "0.3" wiremock = "0.6" openshell-otel-test-support = { path = "../openshell-otel-test-support" } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index d4911d50fd..714559024b 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -3099,11 +3099,21 @@ mod tests { async fn start_plaintext_gateway_listener() -> (SocketAddr, watch::Sender, tokio::task::JoinHandle<()>) { + start_plaintext_gateway_listener_with_tunnel(false).await + } + + async fn start_plaintext_gateway_listener_with_tunnel( + enable_websocket_tunnel: bool, + ) -> (SocketAddr, watch::Sender, tokio::task::JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0") .await .expect("failed to bind test listener"); let listen_addr = listener.local_addr().expect("failed to read local addr"); - let state = test_state(listen_addr, false).await; + let mut state = test_state(listen_addr, false).await; + Arc::get_mut(&mut state) + .expect("test server state should be uniquely owned") + .config + .enable_websocket_tunnel = enable_websocket_tunnel; let service = MultiplexService::new(state); let (shutdown_tx, shutdown_rx) = watch::channel(false); let handle = tokio::spawn(serve_gateway_listener( @@ -3159,6 +3169,41 @@ mod tests { stop_listener(shutdown, handle).await; } + /// The edge-tunnel client path: the SDK's tunnel proxy carries gRPC in + /// WebSocket binary frames to `/_ws_tunnel` on the production listener, + /// and the request reaches the gateway's own services. + #[tokio::test] + async fn production_gateway_listener_serves_grpc_through_websocket_tunnel() { + use openshell_core::proto::{ + HealthRequest, ServiceStatus, open_shell_client::OpenShellClient, + }; + + let (addr, shutdown, handle) = start_plaintext_gateway_listener_with_tunnel(true).await; + let proxy = openshell_sdk::edge_tunnel::start_tunnel_proxy( + &format!("http://127.0.0.1:{}", addr.port()), + "edge-token", + ) + .await + .expect("start the SDK tunnel proxy"); + let channel = + tonic::transport::Endpoint::from_shared(format!("http://{}", proxy.local_addr)) + .expect("tunnel proxy endpoint") + .connect() + .await + .expect("gRPC channel through the WebSocket tunnel"); + + let response = tokio::time::timeout( + Duration::from_secs(10), + OpenShellClient::new(channel).health(HealthRequest {}), + ) + .await + .expect("health RPC through the tunnel timed out") + .expect("health RPC through the tunnel"); + assert_eq!(response.get_ref().status, ServiceStatus::Healthy as i32); + + stop_listener(shutdown, handle).await; + } + async fn send_plain_http(addr: SocketAddr, request: String) -> String { let connect_addr: SocketAddr = format!("127.0.0.1:{}", addr.port()) .parse() diff --git a/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/driver-env-patch.yaml b/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/driver-env-patch.yaml new file mode 100644 index 0000000000..3c829a6cb8 --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/driver-env-patch.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openshell +spec: + template: + spec: + containers: + - name: kubernetes-compute-driver + env: + - name: OPENSHELL_HOST_GATEWAY_IP + value: replaced-by-kustomize diff --git a/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/kustomization.yaml b/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/kustomization.yaml new file mode 100644 index 0000000000..2afb0e5262 --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/kustomization.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The in-process driver reads server.hostGatewayIP from the rendered gateway +# TOML. The external driver reads it from its own environment, so copy the +# chart's rendered value into that container. +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +patches: + - path: driver-env-patch.yaml + target: + group: apps + version: v1 + kind: StatefulSet + name: openshell +replacements: + - source: + group: apps + version: v1 + kind: StatefulSet + name: openshell + fieldPath: spec.template.spec.hostAliases.0.ip + targets: + - select: + group: apps + version: v1 + kind: StatefulSet + name: openshell + fieldPaths: + - spec.template.spec.containers.[name=kubernetes-compute-driver].env.[name=OPENSHELL_HOST_GATEWAY_IP].value diff --git a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh index 43ee52140a..24faf26b62 100755 --- a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh +++ b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh @@ -16,4 +16,18 @@ cp "${plugin_dir}/kustomization.yaml" "${work_dir}/kustomization.yaml" cp "${plugin_dir}/workload-patch.yaml" "${work_dir}/workload-patch.yaml" tee "${work_dir}/rendered.yaml" >/dev/null +# server.hostGatewayIP renders the gateway pod's only hostAliases entry +# (deploy/helm/openshell/templates/_gateway-workload.tpl), which the +# host-gateway component copies from hostAliases.0.ip. The lane exports +# OPENSHELL_E2E_EXPECTED_HOST_GATEWAY_IP when it sets server.hostGatewayIP, so +# a chart change that moves the alias fails here instead of leaving the +# external driver without the IP. +if grep -q '^ *- host\.openshell\.internal$' "${work_dir}/rendered.yaml"; then + cp -R "${plugin_dir}/host-gateway" "${work_dir}/host-gateway" + printf 'components:\n - host-gateway\n' >>"${work_dir}/kustomization.yaml" +elif [ -n "${OPENSHELL_E2E_EXPECTED_HOST_GATEWAY_IP:-}" ]; then + echo "ERROR: server.hostGatewayIP is set, but the rendered chart has no host.openshell.internal hostAliases entry." >&2 + exit 1 +fi + kubectl kustomize "${work_dir}" diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index e664206f5c..376991e991 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -389,12 +389,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - [[package]] name = "chacha20" version = "0.10.2" @@ -1855,18 +1849,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "nix" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", -] - [[package]] name = "nom" version = "7.1.3" @@ -2030,7 +2012,6 @@ dependencies = [ "hyper", "hyper-util", "jsonwebtoken", - "nix", "noyalib", "openshell-crypto", "openshell-ocsf-validate", diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index b0938d9625..8123ae14aa 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -89,11 +89,6 @@ name = "vm_bind_mount" path = "tests/vm_bind_mount.rs" required-features = ["e2e-vm"] -[[test]] -name = "docker_preflight" -path = "tests/docker_preflight.rs" -required-features = ["e2e-docker"] - [[test]] name = "docker_corporate_proxy" path = "tests/docker_corporate_proxy.rs" @@ -119,11 +114,6 @@ name = "local_driver_token_restart" path = "tests/local_driver_token_restart.rs" required-features = ["e2e"] -[[test]] -name = "ephemeral_cleanup" -path = "tests/ephemeral_cleanup.rs" -required-features = ["e2e"] - [[test]] name = "podman_gateway_start" path = "tests/podman_gateway_start.rs" @@ -144,11 +134,6 @@ name = "podman_corporate_proxy" path = "tests/podman_corporate_proxy.rs" required-features = ["e2e-podman"] -[[test]] -name = "podman_oci_identity" -path = "tests/podman_oci_identity.rs" -required-features = ["e2e-podman"] - [[test]] name = "podman_resource_limits" path = "tests/podman_resource_limits.rs" @@ -204,11 +189,6 @@ name = "websocket_conformance" path = "tests/websocket_conformance.rs" required-features = ["e2e-host-gateway"] -[[test]] -name = "credential_gating" -path = "tests/credential_gating.rs" -required-features = ["e2e-host-gateway"] - [[test]] name = "user_namespaces" path = "tests/user_namespaces.rs" @@ -239,11 +219,6 @@ name = "workspace_lifecycle" path = "tests/workspace_lifecycle.rs" required-features = ["e2e"] -[[test]] -name = "sandbox_templates" -path = "tests/sandbox_templates.rs" -required-features = ["e2e"] - [[test]] name = "proxy_egress_pipeline" path = "tests/proxy_egress_pipeline.rs" @@ -298,7 +273,6 @@ tonic-prost = "0.14" tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] } tower = "0.5" url = "2" -nix = { version = "0.29", features = ["process", "signal", "term", "user"] } [dev-dependencies] openshell-ocsf-validate = { path = "../../crates/openshell-ocsf-validate" } @@ -315,7 +289,3 @@ module_name_repetitions = "allow" must_use_candidate = "allow" missing_errors_doc = "allow" missing_panics_doc = "allow" -[[test]] -name = "live_internet_traffic_perf" -path = "tests/live_internet_traffic_perf.rs" -required-features = ["e2e-host-gateway"] diff --git a/e2e/rust/e2e-docker.sh b/e2e/rust/e2e-docker.sh index 8db220ca1b..68d48e7370 100755 --- a/e2e/rust/e2e-docker.sh +++ b/e2e/rust/e2e-docker.sh @@ -54,9 +54,9 @@ if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then exit 0 fi -if [ "${E2E_FEATURES}" != "e2e-conformance-only" ]; then - CONTAINER_ENGINE=docker bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" -fi +# The conformance suite runs on every lane, including e2e-conformance-only, +# and creates sandboxes from this image (testkit::python_image()). +CONTAINER_ENGINE=docker bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" exec "${ROOT}/e2e/with-docker-gateway.sh" \ bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index fc90a827ee..7ef5576eab 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -80,6 +80,7 @@ run_conformance() { fi OPENSHELL_CONFORMANCE_JUNIT="${OPENSHELL_CONFORMANCE_JUNIT:-${ROOT}/results/${OPENSHELL_E2E_REPORT_NAME:-e2e-kubernetes}-conformance.xml}" \ + OPENSHELL_TEST_PYTHON_IMAGE="${OPENSHELL_TEST_PYTHON_IMAGE:-${E2E_PULLABLE_PYTHON_IMAGE}}" \ e2e_run_openshell_conformance "Kubernetes" } diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index 0d6cd5008c..6891ecf221 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -23,17 +23,11 @@ source "${ROOT}/e2e/support/conformance.sh" # stabilized and can be added here. PODMAN_CI_TESTS=( bypass_detection - ephemeral_cleanup - core_dump_hardening - credential_gating - default_image driver_config_volume - edge_tunnel_e2e forward_proxy_graphql_l7 forward_proxy_jsonrpc_l7 forward_proxy_l7_bypass host_gateway_alias - landlock live_policy_update local_driver_token_restart no_proxy @@ -42,9 +36,7 @@ PODMAN_CI_TESTS=( podman_corporate_proxy podman_gateway_start podman_host_gateway - podman_oci_identity provider_token_exchange - upload_backpressure ) if [ -n "${E2E_TEST}" ] && [ -n "${E2E_TEST_SET}" ]; then @@ -117,9 +109,9 @@ if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ] && [ -z "${OPENSHELL_BIN:-}" ]; then export OPENSHELL_BIN="${ROOT}/target/debug/openshell" fi -if [ "${E2E_FEATURES}" != "e2e-conformance-only" ]; then - CONTAINER_ENGINE=podman bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" -fi +# The conformance suite runs on every lane, including e2e-conformance-only, +# and creates sandboxes from this image (testkit::python_image()). +CONTAINER_ENGINE=podman bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" exec "${ROOT}/e2e/with-podman-gateway.sh" \ bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 0cb2c46858..9ccb82f0af 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -463,7 +463,8 @@ e2e_export_gateway_restart_metadata \ # preparation; allow 180s for slower CI runners. export OPENSHELL_PROVISION_TIMEOUT="${SANDBOX_PROVISION_TIMEOUT}" -e2e_run_openshell_conformance "VM" +OPENSHELL_TEST_PYTHON_IMAGE="${OPENSHELL_TEST_PYTHON_IMAGE:-${E2E_PULLABLE_PYTHON_IMAGE}}" \ + e2e_run_openshell_conformance "VM" if [ "${E2E_FEATURES}" = "e2e-conformance-only" ]; then echo "::notice::Skipping Rust e2e tests (e2e-conformance-only)" exit 0 @@ -492,7 +493,6 @@ run_e2e_test() { if [ -n "${E2E_TEST_OVERRIDE}" ]; then run_e2e_test "${E2E_TEST_OVERRIDE}" else - run_e2e_test ephemeral_cleanup run_e2e_test host_gateway_alias run_e2e_test vm_overlay run_e2e_test vm_bind_mount diff --git a/e2e/rust/src/harness/binary.rs b/e2e/rust/src/harness/binary.rs index 4cf87c4c87..cd853906b9 100644 --- a/e2e/rust/src/harness/binary.rs +++ b/e2e/rust/src/harness/binary.rs @@ -52,41 +52,6 @@ pub fn openshell_cmd() -> tokio::process::Command { cmd } -fn shell_escape(arg: &str) -> String { - if arg - .chars() - .all(|c| c.is_ascii_alphanumeric() || "-_./:@".contains(c)) - { - return arg.to_string(); - } - - format!("'{}'", arg.replace('\'', "'\\''")) -} - -/// Create a [`tokio::process::Command`] that runs `openshell` under a PTY. -pub fn openshell_tty_cmd(args: &[&str]) -> tokio::process::Command { - let bin = openshell_bin(); - let mut cmd = tokio::process::Command::new("script"); - - if cfg!(target_os = "macos") { - cmd.arg("-q").arg("/dev/null").arg(bin).args(args); - } else { - let mut shell_command = shell_escape(bin.to_str().expect("openshell path is utf-8")); - for arg in args { - shell_command.push(' '); - shell_command.push_str(&shell_escape(arg)); - } - cmd.arg("-q") - .arg("-e") - .arg("-c") - .arg(shell_command) - .arg("/dev/null"); - } - - cmd.kill_on_drop(true); - cmd -} - #[cfg(test)] mod tests { use super::*; diff --git a/e2e/rust/src/harness/sandbox.rs b/e2e/rust/src/harness/sandbox.rs index c0f0c1aedc..9f57a29daa 100644 --- a/e2e/rust/src/harness/sandbox.rs +++ b/e2e/rust/src/harness/sandbox.rs @@ -129,18 +129,6 @@ impl SandboxGuard { /// Returns an error if the CLI exits with a non-zero status or the sandbox /// name cannot be parsed from the output. pub async fn create(args: &[&str]) -> Result { - Self::create_inner(args, true).await - } - - /// Create a sandbox using the gateway's configured default image. - /// - /// Most E2E tests use [`Self::create`], which supplies the tool-capable E2E - /// image. This variant is reserved for coverage of the product default. - pub async fn create_with_gateway_default(args: &[&str]) -> Result { - Self::create_inner(args, false).await - } - - async fn create_inner(args: &[&str], use_test_image: bool) -> Result { let separator = args.iter().position(|arg| *arg == "--"); let (create_args, command) = separator.map_or((args, &[][..]), |index| { (&args[..index], &args[index + 1..]) @@ -156,9 +144,7 @@ impl SandboxGuard { let mut cmd = openshell_cmd(); cmd.arg("sandbox").arg("create").arg("--detach"); add_unique_name_if_missing(&mut cmd, create_args); - if use_test_image { - add_test_image_if_missing(&mut cmd, create_args); - } + add_test_image_if_missing(&mut cmd, create_args); for arg in create_args { cmd.arg(arg); } diff --git a/e2e/rust/tests/core_dump_hardening.rs b/e2e/rust/tests/core_dump_hardening.rs deleted file mode 100644 index a400b6c218..0000000000 --- a/e2e/rust/tests/core_dump_hardening.rs +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -use openshell_e2e::harness::sandbox::SandboxGuard; - -#[tokio::test] -async fn sandbox_processes_disable_core_dumps() { - let mut sb = SandboxGuard::create(&[ - "--", - "sh", - "-lc", - "test \"$(ulimit -c)\" = 0 && echo core-limit-ok", - ]) - .await - .expect("sandbox create should succeed"); - - assert!( - sb.create_output.contains("core-limit-ok"), - "expected sandbox output to confirm core dumps are disabled:\n{}", - sb.create_output, - ); - - sb.cleanup().await; -} diff --git a/e2e/rust/tests/credential_gating.rs b/e2e/rust/tests/credential_gating.rs deleted file mode 100644 index 110e02cb1e..0000000000 --- a/e2e/rust/tests/credential_gating.rs +++ /dev/null @@ -1,1057 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for credentialed endpoint admission and REST body backstops. -//! -//! Each test installs its own provider and profile under a unique name, so the -//! tests can run in parallel. Each test prints its steps to stderr with the -//! elapsed time, so a timeout shows which step was slow. - -use std::fmt::Display; -use std::io::Write; -use std::process::Stdio; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; - -use base64::Engine as _; -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::sandbox::SandboxGuard; -use sha1::{Digest, Sha1}; -use tempfile::NamedTempFile; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::task::JoinHandle; - -const TEST_HOST: &str = "host.openshell.internal"; -const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; -const TEST_SECRET: &str = "e2e-gating-secret-value"; -const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; -const KEY_PRESENCE_SCRIPT: &str = r#"if [ "${E2E_GATING_TOKEN+x}" != x ]; then - echo TOKEN_ABSENT -else - case "$E2E_GATING_TOKEN" in - openshell:resolve:env:*) echo TOKEN_PLACEHOLDER ;; - *) echo TOKEN_UNSAFE ;; - esac -fi"#; -const WEBSOCKET_GUID: &str = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; - -async fn run_cli(args: &[&str]) -> (bool, String) { - let mut command = openshell_cmd(); - command - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let output = command.output().await.expect("spawn openshell CLI"); - ( - output.status.success(), - format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ), - ) -} - -/// Prints a test's steps to stderr with the time since the test started. -#[derive(Clone, Copy)] -struct Progress { - scenario: &'static str, - started: Instant, -} - -impl Progress { - fn start(scenario: &'static str) -> Self { - let progress = Self { - scenario, - started: Instant::now(), - }; - progress.step("start"); - progress - } - - fn step(self, message: impl Display) { - eprintln!( - "credential_gating[{}] +{:.1}s {message}", - self.scenario, - self.started.elapsed().as_secs_f64() - ); - } -} - -fn utf8_path(file: &NamedTempFile) -> Result<&str, String> { - file.path() - .to_str() - .ok_or_else(|| format!("{} is not UTF-8", file.path().display())) -} - -/// Retry a delete until it takes effect. -/// -/// The gateway refuses to delete a provider or a profile that a sandbox still -/// references, and sandbox teardown drains asynchronously, so a single attempt -/// can be rejected right after a sandbox was removed. This runs from `Drop`, -/// so it blocks instead of awaiting. -fn delete_until_gone(args: &[&str]) -> Result<(), String> { - const ATTEMPTS: u32 = 40; - let binary = openshell_bin(); - let mut last_output = String::new(); - for _ in 0..ATTEMPTS { - match std::process::Command::new(&binary) - .args(args) - .stdin(Stdio::null()) - .output() - { - Ok(output) => { - let combined = format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - if output.status.success() || combined.to_lowercase().contains("not found") { - return Ok(()); - } - last_output = combined; - } - Err(error) => last_output = format!("spawn openshell CLI: {error}"), - } - std::thread::sleep(Duration::from_millis(500)); - } - Err(format!( - "'{}' still failing after {ATTEMPTS} attempts:\n{last_output}", - args.join(" ") - )) -} - -/// A provider and the profile it uses, both under one name unique to the -/// test. Dropping it deletes both, also when the test fails. -struct GatingProvider { - name: String, - progress: Progress, -} - -impl GatingProvider { - async fn install_with_endpoints( - progress: Progress, - rest_port: u16, - websocket_port: u16, - ) -> Result { - let name = unique_name(progress.scenario); - let profile = write_provider_profile(&name, rest_port, websocket_port)?; - Self::install(progress, name, &profile).await - } - - async fn install_endpointless(progress: Progress) -> Result { - let name = unique_name(progress.scenario); - let profile = write_endpointless_provider_profile(&name)?; - Self::install(progress, name, &profile).await - } - - async fn install( - progress: Progress, - name: String, - profile: &NamedTempFile, - ) -> Result { - progress.step(format_args!("importing profile {name}")); - let (imported, output) = - run_cli(&["profile", "import", "--file", utf8_path(profile)?]).await; - if !imported { - return Err(format!("profile import failed:\n{output}")); - } - let provider = Self { name, progress }; - progress.step(format_args!("creating provider {}", provider.name)); - let credential = format!("{TOKEN_ENV}={TEST_SECRET}"); - let (created, output) = run_cli(&[ - "provider", - "create", - "--name", - &provider.name, - "--type", - &provider.name, - "--credential", - &credential, - ]) - .await; - if !created { - return Err(format!("provider create failed:\n{output}")); - } - Ok(provider) - } -} - -impl Drop for GatingProvider { - fn drop(&mut self) { - self.progress - .step(format_args!("deleting provider and profile {}", self.name)); - for kind in ["provider", "profile"] { - if let Err(error) = delete_until_gone(&[kind, "delete", &self.name]) { - self.progress - .step(format_args!("{kind} cleanup did not settle: {error}")); - } - } - self.progress.step("cleanup finished"); - } -} - -/// A provider and profile name that does not collide with parallel tests or -/// earlier runs against the same gateway. -fn unique_name(scenario: &str) -> String { - format!( - "e2e-cg-{scenario}-{}-{:08x}", - std::process::id(), - rand::random::() - ) -} - -fn write_provider_profile( - profile_id: &str, - rest_port: u16, - websocket_port: u16, -) -> Result { - let mut file = tempfile::Builder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| format!("create profile: {error}"))?; - let profile = format!( - r"id: {profile_id} -display_name: E2E Credential Gating -category: other -credentials: - - name: token - env_vars: [{TOKEN_ENV}] - required: true - auth_style: bearer - header_name: authorization -endpoints: - - host: {TEST_HOST} - port: {rest_port} - protocol: rest - access: full - - host: {TEST_HOST} - port: {websocket_port} - protocol: websocket - access: read-write -binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -", - ); - file.write_all(profile.as_bytes()) - .map_err(|error| format!("write profile: {error}"))?; - file.flush() - .map_err(|error| format!("flush profile: {error}"))?; - Ok(file) -} - -fn write_endpointless_provider_profile(profile_id: &str) -> Result { - let mut file = tempfile::Builder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| format!("create endpointless profile: {error}"))?; - let profile = format!( - r"id: {profile_id} -display_name: E2E Endpointless Credential Gating -category: other -credentials: - - name: token - env_vars: [{TOKEN_ENV}] - required: true - auth_style: bearer - header_name: authorization -binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -", - ); - file.write_all(profile.as_bytes()) - .map_err(|error| format!("write endpointless profile: {error}"))?; - file.flush() - .map_err(|error| format!("flush endpointless profile: {error}"))?; - Ok(file) -} - -#[derive(Clone, Copy)] -enum EndpointMode { - L4, - TlsSkip, - L4OptIn, - RestBody { rewrite: bool }, - WebSocket, -} - -#[derive(Clone, Copy)] -enum CredentialSource { - ProviderProfile, - PolicyBinding, -} - -fn write_policy( - port: u16, - mode: EndpointMode, - credential_source: CredentialSource, - provider: &str, -) -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let endpoint_options = match mode { - EndpointMode::L4 => String::new(), - EndpointMode::TlsSkip => { - " protocol: rest\n access: full\n tls: skip\n".to_string() - } - EndpointMode::L4OptIn => " allow_uninspected_credentials: true\n".to_string(), - EndpointMode::RestBody { rewrite } => format!( - " protocol: rest\n access: full\n request_body_credential_rewrite: {rewrite}\n" - ), - EndpointMode::WebSocket => { - " protocol: websocket\n access: read-write\n".to_string() - } - }; - let credential_binding = match credential_source { - CredentialSource::ProviderProfile => String::new(), - CredentialSource::PolicyBinding => { - format!(" credential_binding:\n provider: {provider}\n") - } - }; - let policy = format!( - r"version: 1 -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox -network_policies: - credential_gating: - name: credential_gating - endpoints: - - host: {TEST_HOST} - port: {port} -{endpoint_options}{credential_binding} binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -", - ); - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -#[derive(Debug, Default, Clone, Copy)] -struct BodyObservation { - saw_placeholder: bool, - saw_secret: bool, -} - -struct HttpProbeServer { - port: u16, - observations: Arc>>, - task: JoinHandle<()>, -} - -impl HttpProbeServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind HTTP probe: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read HTTP probe address: {error}"))? - .port(); - let observations = Arc::new(Mutex::new(Vec::new())); - let task_observations = Arc::clone(&observations); - let task = tokio::spawn(async move { - loop { - let Ok((stream, _)) = listener.accept().await else { - break; - }; - let observations = Arc::clone(&task_observations); - tokio::spawn(async move { - let _ = handle_http_probe(stream, observations).await; - }); - } - }); - Ok(Self { - port, - observations, - task, - }) - } - - async fn wait_for_observations(&self, count: usize) -> Vec { - for _ in 0..100 { - let observations = self.observations.lock().unwrap().clone(); - if observations.len() >= count { - return observations; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - self.observations.lock().unwrap().clone() - } -} - -impl Drop for HttpProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -struct BinaryWebSocketProbeServer { - port: u16, - handshake_seen: Arc, - binary_seen: Arc, - task: JoinHandle<()>, -} - -impl BinaryWebSocketProbeServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind WebSocket probe: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read WebSocket probe address: {error}"))? - .port(); - let handshake_seen = Arc::new(AtomicBool::new(false)); - let binary_seen = Arc::new(AtomicBool::new(false)); - let task_handshake_seen = Arc::clone(&handshake_seen); - let task_binary_seen = Arc::clone(&binary_seen); - let task = tokio::spawn(async move { - loop { - let Ok((stream, _)) = listener.accept().await else { - break; - }; - let handshake_seen = Arc::clone(&task_handshake_seen); - let binary_seen = Arc::clone(&task_binary_seen); - tokio::spawn(async move { - let _ = - handle_binary_websocket_probe(stream, handshake_seen, binary_seen).await; - }); - } - }); - Ok(Self { - port, - handshake_seen, - binary_seen, - task, - }) - } - - async fn wait_for_handshake(&self) -> bool { - for _ in 0..100 { - if self.handshake_seen.load(Ordering::Acquire) { - return true; - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - false - } -} - -impl Drop for BinaryWebSocketProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn recv_until(stream: &mut TcpStream, marker: &[u8]) -> std::io::Result> { - let mut received = Vec::new(); - let mut buffer = [0_u8; 1024]; - loop { - let read = stream.read(&mut buffer).await?; - if read == 0 { - return Ok(received); - } - received.extend_from_slice(&buffer[..read]); - if received - .windows(marker.len()) - .any(|window| window == marker) - { - return Ok(received); - } - } -} - -fn websocket_header_value(request: &str, name: &str) -> Option { - request.lines().find_map(|line| { - let (header, value) = line.split_once(':')?; - header - .trim() - .eq_ignore_ascii_case(name) - .then(|| value.trim().to_string()) - }) -} - -fn websocket_accept(key: &str) -> String { - let mut hasher = Sha1::new(); - hasher.update(key.as_bytes()); - hasher.update(WEBSOCKET_GUID.as_bytes()); - base64::engine::general_purpose::STANDARD.encode(hasher.finalize()) -} - -async fn handle_binary_websocket_probe( - mut stream: TcpStream, - handshake_seen: Arc, - binary_seen: Arc, -) -> std::io::Result<()> { - let request_bytes = recv_until(&mut stream, b"\r\n\r\n").await?; - let request = String::from_utf8_lossy(&request_bytes); - let key = websocket_header_value(&request, "Sec-WebSocket-Key").ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::InvalidData, "missing WebSocket key") - })?; - let accept = websocket_accept(&key); - let response = format!( - "HTTP/1.1 101 Switching Protocols\r\n\ - Upgrade: websocket\r\n\ - Connection: Upgrade\r\n\ - Sec-WebSocket-Accept: {accept}\r\n\ - \r\n" - ); - stream.write_all(response.as_bytes()).await?; - handshake_seen.store(true, Ordering::Release); - - let mut header = [0_u8; 2]; - if tokio::time::timeout(Duration::from_secs(5), stream.read_exact(&mut header)) - .await - .is_ok_and(|result| result.is_ok()) - && header[0] & 0x0f == 0x02 - { - binary_seen.store(true, Ordering::Release); - } - Ok(()) -} - -async fn handle_http_probe( - mut stream: TcpStream, - observations: Arc>>, -) -> std::io::Result<()> { - let mut received = Vec::new(); - let mut buffer = [0_u8; 4096]; - let mut expected_total = None; - loop { - let read = tokio::time::timeout(Duration::from_secs(10), stream.read(&mut buffer)).await; - let Ok(Ok(read)) = read else { - break; - }; - if read == 0 { - break; - } - received.extend_from_slice(&buffer[..read]); - if expected_total.is_none() - && let Some(header_end) = received.windows(4).position(|window| window == b"\r\n\r\n") - { - let header_end = header_end + 4; - let headers = String::from_utf8_lossy(&received[..header_end]); - let content_length = headers - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - expected_total = Some(header_end + content_length); - } - if expected_total.is_some_and(|expected| received.len() >= expected) { - break; - } - } - - let observation = BodyObservation { - saw_placeholder: received - .windows(PLACEHOLDER_PREFIX.len()) - .any(|window| window == PLACEHOLDER_PREFIX.as_bytes()), - saw_secret: received - .windows(TEST_SECRET.len()) - .any(|window| window == TEST_SECRET.as_bytes()), - }; - observations.lock().unwrap().push(observation); - if expected_total.is_some_and(|expected| received.len() >= expected) { - let result = if observation.saw_secret && !observation.saw_placeholder { - "BODY_REWRITTEN" - } else if observation.saw_placeholder && !observation.saw_secret { - "BODY_TEXT" - } else { - "BODY_BAD" - }; - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{result}", - result.len() - ); - stream.write_all(response.as_bytes()).await?; - } - Ok(()) -} - -fn body_client_script(port: u16) -> String { - format!( - r#" -import os -import socket - -host = {TEST_HOST:?} -port = {port} -token = os.environ[{TOKEN_ENV:?}] - -with socket.create_connection((host, port), timeout=10) as sock: - target = f"{{host}}:{{port}}" - body = ("prefix-" + token + "-suffix").encode("utf-8") - request = ( - f"POST /token HTTP/1.1\r\nHost: {{target}}\r\n" - f"Content-Type: text/plain\r\nContent-Length: {{len(body)}}\r\nConnection: close\r\n\r\n" - ).encode("ascii") + body - sock.sendall(request) - sock.settimeout(3) - response = b"" - while True: - try: - chunk = sock.recv(4096) - except socket.timeout: - break - if not chunk: - break - response += chunk - print("BODY_REWRITTEN" if b"BODY_REWRITTEN" in response else "BODY_TEXT" if b"BODY_TEXT" in response else "BODY_DENIED") -"# - ) -} - -fn binary_websocket_client_script(port: u16) -> String { - format!( - r#" -import base64 -import os -import socket -import struct - -host = {TEST_HOST:?} -port = {port} - -def recv_until(sock, marker): - data = b"" - while marker not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def recv_exact(sock, size): - data = b"" - while len(data) < size: - chunk = sock.recv(size - len(data)) - if not chunk: - break - data += chunk - return data - -with socket.create_connection((host, port), timeout=10) as sock: - target = f"{{host}}:{{port}}" - key = base64.b64encode(os.urandom(16)).decode("ascii") - request = ( - f"GET /ws HTTP/1.1\r\nHost: {{target}}\r\n" - "Upgrade: websocket\r\nConnection: Upgrade\r\n" - f"Sec-WebSocket-Key: {{key}}\r\nSec-WebSocket-Version: 13\r\n\r\n" - ) - sock.sendall(request.encode("ascii")) - if not recv_until(sock, b"\r\n\r\n").startswith(b"HTTP/1.1 101"): - raise RuntimeError("upgrade failed") - payload = b"binary-credential-channel" - mask = os.urandom(4) - masked = bytes(byte ^ mask[index % 4] for index, byte in enumerate(payload)) - frame = bytes([0x82, 0x80 | len(payload)]) + mask + masked - sock.sendall(frame) - sock.settimeout(3) - denied = False - try: - header = recv_exact(sock, 2) - if not header: - denied = True - elif len(header) == 2: - opcode = header[0] & 0x0f - masked = bool(header[1] & 0x80) - payload_length = header[1] & 0x7f - if opcode == 0x08 and not masked and payload_length < 126: - close_payload = recv_exact(sock, payload_length) - denied = ( - len(close_payload) >= 2 - and struct.unpack("!H", close_payload[:2])[0] == 1008 - ) - except socket.timeout: - denied = True - print("BINARY_DENIED" if denied else "BINARY_FORWARDED") -"# - ) -} - -async fn sandbox_create_failure( - progress: Progress, - label: &str, - provider: &str, - policy: &NamedTempFile, -) -> String { - progress.step(format_args!( - "{label}: creating sandbox, expecting rejection" - )); - let policy_path = policy.path().to_str().expect("policy path is UTF-8"); - let (success, output) = run_cli(&[ - "sandbox", - "create", - "--policy", - policy_path, - "--provider", - provider, - "--", - "echo", - "must-not-run", - ]) - .await; - assert!(!success, "sandbox create unexpectedly succeeded:\n{output}"); - progress.step(format_args!("{label}: rejected")); - output -} - -/// Create a sandbox, run one command in it, and delete it. -/// -/// Returns the create output followed by the command output. -async fn run_in_sandbox( - progress: Progress, - label: &str, - create_args: &[&str], - command: &[&str], -) -> Result { - progress.step(format_args!("{label}: creating sandbox")); - let mut sandbox = SandboxGuard::create(create_args).await?; - progress.step(format_args!( - "{label}: sandbox {} ready, running command", - sandbox.name - )); - let output = sandbox.exec(command).await; - progress.step(format_args!("{label}: deleting sandbox {}", sandbox.name)); - sandbox.cleanup().await; - progress.step(format_args!("{label}: sandbox deleted")); - Ok(format!("{}{}", sandbox.create_output, output?)) -} - -async fn assert_gateway_admission( - progress: Progress, - provider: &str, - port: u16, - credential_source: CredentialSource, -) -> Result<(), String> { - let l4 = write_policy(port, EndpointMode::L4, credential_source, provider)?; - let l4_error = sandbox_create_failure(progress, "L4-only policy", provider, &l4).await; - assert!(l4_error.contains("credentialed endpoint"), "{l4_error}"); - assert!(l4_error.contains("L4-only"), "{l4_error}"); - - let tls_skip = write_policy(port, EndpointMode::TlsSkip, credential_source, provider)?; - let tls_error = sandbox_create_failure(progress, "tls: skip policy", provider, &tls_skip).await; - assert!(tls_error.contains("credentialed endpoint"), "{tls_error}"); - assert!( - tls_error.contains("tls:") && tls_error.contains("skip"), - "{tls_error}" - ); - - let opt_in = write_policy(port, EndpointMode::L4OptIn, credential_source, provider)?; - let accepted_marker = match credential_source { - CredentialSource::ProviderProfile => "OPT_IN_ACCEPTED", - CredentialSource::PolicyBinding => "ENDPOINTLESS_OPT_IN_ACCEPTED", - }; - let output = run_in_sandbox( - progress, - "opt-in policy", - &["--policy", utf8_path(&opt_in)?, "--provider", provider], - &["echo", accepted_marker], - ) - .await?; - assert!(output.contains(accepted_marker), "{output}"); - Ok(()) -} - -async fn wait_for_provider_key_presence( - sandbox: &SandboxGuard, - expected_present: bool, -) -> Result<(), String> { - let expected = if expected_present { - "TOKEN_PLACEHOLDER" - } else { - "TOKEN_ABSENT" - }; - let mut last_output = String::new(); - for _ in 0..60 { - last_output = sandbox.exec(&["sh", "-lc", KEY_PRESENCE_SCRIPT]).await?; - if last_output.contains(expected) { - return Ok(()); - } - tokio::time::sleep(Duration::from_millis(500)).await; - } - Err(format!( - "provider environment did not converge to {expected}; last key-presence output:\n{last_output}" - )) -} - -async fn set_policy(progress: Progress, sandbox: &str, policy_path: &str) -> Result<(), String> { - let (success, output) = run_cli(&[ - "policy", - "set", - sandbox, - "--policy", - policy_path, - "--wait", - "--timeout", - "120", - ]) - .await; - if !success { - return Err(format!("policy update failed:\n{output}")); - } - progress.step("policy applied"); - Ok(()) -} - -async fn assert_endpointless_provider_env_live_update( - progress: Progress, - provider: &str, - port: u16, -) -> Result<(), String> { - let unbound = write_policy( - port, - EndpointMode::RestBody { rewrite: false }, - CredentialSource::ProviderProfile, - provider, - )?; - let bound = write_policy( - port, - EndpointMode::RestBody { rewrite: false }, - CredentialSource::PolicyBinding, - provider, - )?; - let unbound_path = utf8_path(&unbound)?; - let bound_path = utf8_path(&bound)?; - - progress.step("creating long-running sandbox with the unbound policy"); - let mut sandbox = SandboxGuard::create_keep_with_args( - &["--policy", unbound_path, "--provider", provider, "--no-tty"], - &["sh", "-c", "echo PROVIDER_UPDATE_READY && sleep infinity"], - "PROVIDER_UPDATE_READY", - ) - .await?; - - let result = async { - progress.step(format_args!( - "sandbox {} ready, waiting for the token to be absent", - sandbox.name - )); - wait_for_provider_key_presence(&sandbox, false).await?; - - progress.step("binding the provider with policy set --wait"); - set_policy(progress, &sandbox.name, bound_path) - .await - .map_err(|error| format!("binding {error}"))?; - progress.step("waiting for the token placeholder"); - wait_for_provider_key_presence(&sandbox, true).await?; - - progress.step("unbinding the provider with policy set --wait"); - set_policy(progress, &sandbox.name, unbound_path) - .await - .map_err(|error| format!("unbinding {error}"))?; - progress.step("waiting for the token to be absent"); - wait_for_provider_key_presence(&sandbox, false).await - } - .await; - - progress.step(format_args!("deleting sandbox {}", sandbox.name)); - sandbox.cleanup().await; - progress.step("sandbox deleted"); - result -} - -async fn run_body_sandbox( - progress: Progress, - provider: &str, - port: u16, - mode: EndpointMode, - credential_source: CredentialSource, -) -> Result { - let policy = write_policy(port, mode, credential_source, provider)?; - let script = body_client_script(port); - run_in_sandbox( - progress, - "REST body client", - &["--policy", utf8_path(&policy)?, "--provider", provider], - &["python3", "-c", &script], - ) - .await -} - -async fn assert_websocket_binary_denied( - progress: Progress, - provider: &str, - server: &BinaryWebSocketProbeServer, -) -> Result<(), String> { - let policy = write_policy( - server.port, - EndpointMode::WebSocket, - CredentialSource::ProviderProfile, - provider, - )?; - let script = binary_websocket_client_script(server.port); - let output = run_in_sandbox( - progress, - "WebSocket client", - &["--policy", utf8_path(&policy)?, "--provider", provider], - &["python3", "-c", &script], - ) - .await?; - assert!(output.contains("BINARY_DENIED"), "{output}"); - assert!( - server.wait_for_handshake().await, - "upstream should receive the WebSocket handshake" - ); - assert!( - !server.binary_seen.load(Ordering::Acquire), - "credentialed WebSocket binary frame reached upstream" - ); - Ok(()) -} - -/// Start both probe servers and install a provider whose profile declares -/// their endpoints. -async fn profile_endpoint_fixture( - progress: Progress, -) -> (HttpProbeServer, BinaryWebSocketProbeServer, GatingProvider) { - let server = HttpProbeServer::start().await.expect("start HTTP probe"); - let websocket_server = BinaryWebSocketProbeServer::start() - .await - .expect("start WebSocket probe"); - let provider = - GatingProvider::install_with_endpoints(progress, server.port, websocket_server.port) - .await - .expect("install credentialed provider"); - (server, websocket_server, provider) -} - -/// Start the HTTP probe server and install a provider whose profile declares -/// no endpoints, so policies must bind it explicitly. -async fn endpointless_fixture(progress: Progress) -> (HttpProbeServer, GatingProvider) { - let server = HttpProbeServer::start().await.expect("start HTTP probe"); - let provider = GatingProvider::install_endpointless(progress) - .await - .expect("install endpointless provider"); - (server, provider) -} - -#[tokio::test] -async fn profile_rest_endpoint_keeps_body_placeholder() { - let progress = Progress::start("profile-rest"); - let (server, _websocket_server, provider) = profile_endpoint_fixture(progress).await; - - let output = run_body_sandbox( - progress, - &provider.name, - server.port, - EndpointMode::RestBody { rewrite: false }, - CredentialSource::ProviderProfile, - ) - .await - .expect("run REST body client"); - assert!(output.contains("BODY_TEXT"), "{output}"); - let observations = server.wait_for_observations(1).await; - assert_eq!(observations.len(), 1, "observations: {observations:?}"); - assert!(observations[0].saw_placeholder); - assert!(!observations[0].saw_secret); - progress.step("checks passed"); -} - -#[tokio::test] -async fn profile_websocket_endpoint_denies_binary_frames() { - let progress = Progress::start("profile-websocket"); - let (_server, websocket_server, provider) = profile_endpoint_fixture(progress).await; - - assert_websocket_binary_denied(progress, &provider.name, &websocket_server) - .await - .expect("run WebSocket client"); - progress.step("checks passed"); -} - -#[tokio::test] -async fn policy_binding_requires_inspection_or_opt_in() { - let progress = Progress::start("binding-admission"); - let (server, provider) = endpointless_fixture(progress).await; - - assert_gateway_admission( - progress, - &provider.name, - server.port, - CredentialSource::PolicyBinding, - ) - .await - .expect("gateway admission"); - progress.step("checks passed"); -} - -#[tokio::test] -async fn policy_binding_rest_body_keeps_placeholder_without_rewrite() { - let progress = Progress::start("binding-body"); - let (server, provider) = endpointless_fixture(progress).await; - - let literal = run_body_sandbox( - progress, - &provider.name, - server.port, - EndpointMode::RestBody { rewrite: false }, - CredentialSource::PolicyBinding, - ) - .await - .expect("run REST body client"); - assert!(literal.contains("BODY_TEXT"), "{literal}"); - let observations = server.wait_for_observations(1).await; - assert_eq!(observations.len(), 1, "observations: {observations:?}"); - assert!(observations[0].saw_placeholder); - assert!(!observations[0].saw_secret); - progress.step("checks passed"); -} - -#[tokio::test] -async fn policy_binding_rest_body_rewrite_sends_secret_upstream() { - let progress = Progress::start("binding-rewrite"); - let (server, provider) = endpointless_fixture(progress).await; - - let rewritten = run_body_sandbox( - progress, - &provider.name, - server.port, - EndpointMode::RestBody { rewrite: true }, - CredentialSource::PolicyBinding, - ) - .await - .expect("run REST body client"); - assert!(rewritten.contains("BODY_REWRITTEN")); - assert!(!rewritten.contains(TEST_SECRET)); - assert!(!rewritten.contains(PLACEHOLDER_PREFIX)); - let observations = server.wait_for_observations(1).await; - assert_eq!(observations.len(), 1, "observations: {observations:?}"); - assert!(!observations[0].saw_placeholder); - assert!(observations[0].saw_secret); - progress.step("checks passed"); -} - -#[tokio::test] -async fn policy_binding_updates_provider_environment_live() { - let progress = Progress::start("binding-live-update"); - let (server, provider) = endpointless_fixture(progress).await; - - assert_endpointless_provider_env_live_update(progress, &provider.name, server.port) - .await - .expect("provider environment live update"); - progress.step("checks passed"); -} diff --git a/e2e/rust/tests/default_image.rs b/e2e/rust/tests/default_image.rs deleted file mode 100644 index 6a1dbb3cd8..0000000000 --- a/e2e/rust/tests/default_image.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for the default NVIDIA Ubuntu workload image. - -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; - -#[tokio::test] -async fn sandbox_from_default_image() { - let mut guard = SandboxGuard::create_with_gateway_default(&["--", "cat", "/etc/os-release"]) - .await - .expect("sandbox create from default image"); - - let clean_output = strip_ansi(&guard.create_output); - assert!( - clean_output.contains("ID=ubuntu"), - "expected Ubuntu OS release info in sandbox output:\n{clean_output}" - ); - - guard.cleanup().await; -} - -#[tokio::test] -async fn sandbox_from_explicit_nvidia_ubuntu_image() { - let image = "nvcr.io/nvidia/base/ubuntu:24.04"; - let mut guard = SandboxGuard::create(&["--from", image, "--", "cat", "/etc/os-release"]) - .await - .expect("sandbox create from explicit NVIDIA Ubuntu image"); - - let clean_output = strip_ansi(&guard.create_output); - assert!( - clean_output.contains("ID=ubuntu"), - "expected Ubuntu OS release info in sandbox output:\n{clean_output}" - ); - - guard.cleanup().await; -} diff --git a/e2e/rust/tests/docker_corporate_proxy.rs b/e2e/rust/tests/docker_corporate_proxy.rs index 85584b966f..659c6c5f8f 100644 --- a/e2e/rust/tests/docker_corporate_proxy.rs +++ b/e2e/rust/tests/docker_corporate_proxy.rs @@ -23,8 +23,6 @@ //! 5. An `https://` proxy works when its CA is supplied via `proxy_ca_bundle`. //! 6. A TLS-intercepting proxy CA is trusted by both inspected supervisor //! upstream TLS and a raw workload TLS connection. -//! 7. An incoherent setting is fatal at gateway startup rather than -//! degrading to a direct dial. //! //! Fixtures run as host processes and are reached by the host-networked //! supervisor. `host.openshell.internal` is normalized to the gateway host @@ -681,28 +679,6 @@ fn insert_docker_driver_settings(config: &str, extra: &str) -> Result Result { @@ -1142,39 +1118,3 @@ async fn docker_corporate_proxy_trusts_intercepted_destination_tls() { .await .expect("restore gateway config"); } - -#[tokio::test] -#[serial(docker_corporate_proxy)] -async fn docker_corporate_proxy_rejects_incoherent_configuration() { - if !should_run("corporate proxy fail-closed test") { - return; - } - - // A bypass list without a proxy URL means the operator believed proxying - // was in effect. Accepting it would leave every dial direct while looking - // configured, so the gateway must refuse to start instead. - let mut gateway_config = GatewayProxyConfig::apply_raw("no_proxy = \"10.0.0.0/8\"\n") - .expect("apply incoherent corporate proxy gateway config"); - - let health = wait_for_healthy(Duration::from_secs(20)).await; - assert!( - health.is_err(), - "gateway must not serve traffic with an incoherent [openshell.drivers.docker] proxy table" - ); - - let log_path = std::env::var("OPENSHELL_E2E_GATEWAY_LOG").expect("gateway log path"); - let log = std::fs::read_to_string(&log_path).expect("read gateway log"); - // The message must name the offending key rather than surfacing as an - // opaque driver-readiness timeout. Matched loosely on the key names so - // this does not break on error-wrapping changes. - assert!( - log.contains("no_proxy") && log.contains("https_proxy"), - "the startup error must name the offending key; gateway log tail:\n{}", - log.lines().rev().take(40).collect::>().join("\n") - ); - - gateway_config - .restore() - .await - .expect("restore gateway config"); -} diff --git a/e2e/rust/tests/docker_preflight.rs b/e2e/rust/tests/docker_preflight.rs deleted file mode 100644 index bf05a132d9..0000000000 --- a/e2e/rust/tests/docker_preflight.rs +++ /dev/null @@ -1,171 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Doctor Docker preflight e2e tests. -//! -//! These tests verify that `openshell doctor check` reports actionable guidance -//! when Docker is not available. -//! -//! The tests do NOT require a running gateway or Docker — they intentionally -//! point `DOCKER_HOST` at a non-existent socket to simulate Docker being -//! unavailable. - -use std::process::Stdio; -use std::time::Instant; -use std::{env, fs}; - -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::strip_ansi; - -/// Run `openshell ` in an isolated environment where Docker is -/// guaranteed to be unreachable. -/// -/// Sets `DOCKER_HOST` to a non-existent socket so the preflight check -/// fails immediately regardless of the host's Docker configuration. -async fn run_without_docker(args: &[&str]) -> (String, i32, std::time::Duration) { - let tmpdir = tempfile::tempdir().expect("create isolated config dir"); - let bin_dir = tmpdir.path().join("bin"); - fs::create_dir(&bin_dir).expect("create fake bin dir"); - let fake_docker = bin_dir.join("docker"); - fs::write( - &fake_docker, - "#!/bin/sh\n\ - echo 'Cannot connect to Docker daemon. Check DOCKER_HOST and run docker info.' >&2\n\ - exit 1\n", - ) - .expect("write fake docker"); - #[cfg(unix)] - fs::set_permissions(&fake_docker, fs::Permissions::from_mode(0o755)) - .expect("chmod fake docker"); - - let old_path = env::var("PATH").unwrap_or_default(); - let path = format!("{}:{old_path}", bin_dir.display()); - let start = Instant::now(); - - let mut cmd = openshell_cmd(); - cmd.args(args) - .env("XDG_CONFIG_HOME", tmpdir.path()) - .env("HOME", tmpdir.path()) - .env("PATH", path) - .env("DOCKER_HOST", "unix:///tmp/openshell-e2e-nonexistent.sock") - .env_remove("OPENSHELL_GATEWAY") - .env_remove("OPENSHELL_GATEWAY_ENDPOINT") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let elapsed = start.elapsed(); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - (combined, code, elapsed) -} - -// ------------------------------------------------------------------- -// doctor check: validates system prerequisites -// ------------------------------------------------------------------- - -/// `openshell doctor check` with Docker unavailable should fail fast -/// and report the Docker check as FAILED. -#[tokio::test] -async fn doctor_check_fails_without_docker() { - let (output, code, elapsed) = run_without_docker(&["doctor", "check"]).await; - - assert_ne!( - code, 0, - "doctor check should fail when Docker is unavailable, output:\n{output}" - ); - - assert!( - elapsed.as_secs() < 10, - "doctor check should complete quickly (took {}s)", - elapsed.as_secs() - ); - - let clean = strip_ansi(&output); - assert!( - clean.contains("FAILED"), - "doctor check should report Docker as FAILED:\n{clean}" - ); -} - -/// `openshell doctor check` output should include the check label -/// so the user knows what was tested. -#[tokio::test] -async fn doctor_check_output_shows_docker_label() { - let (output, _, _) = run_without_docker(&["doctor", "check"]).await; - let clean = strip_ansi(&output); - - assert!( - clean.contains("Docker"), - "doctor check output should include 'Docker' label:\n{clean}" - ); -} - -/// `openshell doctor check` with Docker unavailable should include -/// actionable guidance in the error output. -#[tokio::test] -async fn doctor_check_error_includes_guidance() { - let (output, code, _) = run_without_docker(&["doctor", "check"]).await; - - assert_ne!(code, 0); - let clean = strip_ansi(&output); - - assert!( - clean.contains("DOCKER_HOST"), - "doctor check error should mention DOCKER_HOST:\n{clean}" - ); - assert!( - clean.contains("docker info"), - "doctor check error should suggest 'docker info':\n{clean}" - ); -} - -/// When Docker IS available, `openshell doctor check` should pass and -/// report the version. -/// -/// This test only runs when Docker is actually reachable on the host -/// (i.e., it will pass in CI with Docker but be skipped locally if -/// Docker is not running). We detect this by checking if the default -/// socket exists. -#[tokio::test] -async fn doctor_check_passes_with_docker() { - if !std::path::Path::new("/var/run/docker.sock").exists() { - eprintln!("skipping: /var/run/docker.sock not found"); - return; - } - - let tmpdir = tempfile::tempdir().expect("create isolated config dir"); - let mut cmd = openshell_cmd(); - cmd.args(["doctor", "check"]) - .env("XDG_CONFIG_HOME", tmpdir.path()) - .env("HOME", tmpdir.path()) - .env_remove("OPENSHELL_GATEWAY") - .env_remove("OPENSHELL_GATEWAY_ENDPOINT") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - let clean = strip_ansi(&combined); - - assert_eq!( - code, 0, - "doctor check should pass when Docker is available, output:\n{clean}" - ); - assert!( - clean.contains("All checks passed"), - "doctor check should report success:\n{clean}" - ); - assert!( - clean.contains("ok"), - "doctor check should show 'ok' for Docker:\n{clean}" - ); -} diff --git a/e2e/rust/tests/edge_tunnel_e2e.rs b/e2e/rust/tests/edge_tunnel_e2e.rs deleted file mode 100644 index 80ee4a6ed8..0000000000 --- a/e2e/rust/tests/edge_tunnel_e2e.rs +++ /dev/null @@ -1,220 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E tests for edge tunnel auth flow against a running gateway. -//! -//! Prerequisites: -//! - A running openshell gateway -//! - For WS tunnel coverage, the gateway's HTTP endpoint is accessible (no TLS) -//! - The `openshell` binary (built automatically from the workspace) -//! -//! These tests exercise the full CLI → WS tunnel → gRPC flow. -//! -//! Environment variables: -//! - `OPENSHELL_GATEWAY`: Name of the active gateway (standard e2e var) -//! - `OPENSHELL_GATEWAY_ENDPOINT`: Optional direct plaintext endpoint. -//! -//! The edge-tunnel path requires a gateway endpoint that accepts plaintext HTTP. - -use std::process::Stdio; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::strip_ansi; - -/// Run `openshell ` using the system's configured gateway. -async fn run_cli(args: &[&str]) -> (String, i32) { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - (combined, code) -} - -/// Run `openshell ` with a custom config directory so the CLI reads -/// our seeded gateway metadata and edge token instead of the real config. -async fn run_cli_with_config(config_dir: &std::path::Path, args: &[&str]) -> (String, i32) { - let mut cmd = openshell_cmd(); - cmd.args(args) - .env("XDG_CONFIG_HOME", config_dir) - .env("HOME", config_dir) - .env_remove("OPENSHELL_GATEWAY") - .env_remove("OPENSHELL_GATEWAY_ENDPOINT") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let code = output.status.code().unwrap_or(-1); - (combined, code) -} - -/// Seed a temporary config directory with gateway metadata that has -/// `auth_mode: "cloudflare_jwt"`, a stored edge token, and an active gateway -/// pointing at the given endpoint. -fn seed_edge_gateway_config( - config_dir: &std::path::Path, - gateway_name: &str, - gateway_endpoint: &str, - edge_token: &str, -) { - let openshell_dir = config_dir.join("openshell"); - let gateways_dir = openshell_dir.join("gateways"); - - std::fs::create_dir_all(&openshell_dir).expect("create openshell config dir"); - std::fs::write(openshell_dir.join("active_gateway"), gateway_name) - .expect("write active_gateway"); - - // Write gateway metadata JSON. - let gateway_dir = gateways_dir.join(gateway_name); - std::fs::create_dir_all(&gateway_dir).expect("create gateway dir"); - let metadata = serde_json::json!({ - "name": gateway_name, - "gateway_endpoint": gateway_endpoint, - "is_remote": false, - "gateway_port": 0, - "auth_mode": "cloudflare_jwt" - }); - std::fs::write( - gateway_dir.join("metadata.json"), - serde_json::to_string_pretty(&metadata).unwrap(), - ) - .expect("write gateway metadata"); - - // Write edge token file. - std::fs::write(gateway_dir.join("edge_token"), edge_token).expect("write edge_token"); -} - -// ------------------------------------------------------------------- -// Test 12: gRPC health check against a gateway -// ------------------------------------------------------------------- - -/// `openshell status` should report a healthy gateway when connected to a -/// configured gateway. -/// -/// This test verifies the normal gateway path: -/// - CLI resolves gateway metadata -/// - gRPC client connects -/// - Server responds to health check -#[tokio::test] -async fn gateway_status_reports_healthy() { - let (output, code) = run_cli(&["status"]).await; - let clean = strip_ansi(&output); - - assert_eq!( - code, 0, - "openshell status should exit 0 against gateway:\n{clean}" - ); - - // The status output should show the gateway as healthy/connected. - assert!( - clean.to_lowercase().contains("healthy") - || clean.to_lowercase().contains("running") - || clean.to_lowercase().contains("connected") - || clean.contains("✓"), - "status should report healthy gateway:\n{clean}" - ); -} - -// ------------------------------------------------------------------- -// Test 13: gRPC through the WS tunnel proxy (edge token path) -// ------------------------------------------------------------------- - -/// When a gateway's metadata has `auth_mode == "cloudflare_jwt"` and a -/// stored edge token, the CLI routes gRPC through the WebSocket tunnel proxy. -/// This test verifies the full tunnel path: -/// -/// CLI → local TCP proxy → WebSocket → /_ws_tunnel → loopback TCP → gRPC -/// -/// The test seeds a temporary config directory with edge auth metadata and a -/// dummy token, then runs `openshell status` against the live plaintext -/// gateway. -/// -/// Note: The dummy token won't be validated (no edge auth middleware on -/// the plaintext gateway), but it triggers the CLI's tunnel proxy codepath. -#[tokio::test] -async fn ws_tunnel_status_through_edge_proxy() { - let (original_status, _) = run_cli(&["status"]).await; - let clean_status = strip_ansi(&original_status); - - // Only run this test if we have a healthy gateway to test against. - if !clean_status.to_lowercase().contains("healthy") - && !clean_status.to_lowercase().contains("running") - && !clean_status.to_lowercase().contains("connected") - && !clean_status.contains("✓") - { - eprintln!("Skipping ws_tunnel test: no healthy gateway available"); - return; - } - - // Get the gateway endpoint from the gateway metadata. - let (info_output, info_code) = run_cli(&["gateway", "info"]).await; - assert_eq!(info_code, 0, "gateway info should succeed:\n{info_output}"); - - let info_clean = strip_ansi(&info_output); - - // Extract the gateway endpoint from the info output. - // The format varies, but it should contain a URL-like string. - let endpoint = info_clean.lines().find_map(|line| { - if line.to_lowercase().contains("endpoint") || line.to_lowercase().contains("gateway") { - // Try to extract a URL from the line - line.split_whitespace() - .find(|word| word.starts_with("http://") || word.starts_with("https://")) - .map(String::from) - } else { - None - } - }); - - let Some(endpoint) = endpoint else { - eprintln!( - "Skipping ws_tunnel test: could not extract gateway endpoint from:\n{info_clean}" - ); - return; - }; - - // For the WS tunnel test, we need the endpoint to be HTTP (plaintext). - // If it's HTTPS, the WS tunnel test requires TLS negotiation which - // complicates things. Skip if the gateway isn't plaintext. - if !endpoint.starts_with("http://") { - eprintln!( - "Skipping ws_tunnel test: gateway endpoint is not plaintext HTTP: {endpoint}\n\ - Use a plaintext local gateway such as `mise run gateway:docker` for this test." - ); - return; - } - - // Seed a temporary config directory with edge auth metadata pointing at - // the live gateway. The dummy token triggers the WS tunnel codepath - // without requiring real edge auth middleware. - let tmpdir = tempfile::tempdir().expect("create temp config dir"); - seed_edge_gateway_config( - tmpdir.path(), - "edge-tunnel-test", - &endpoint, - "dummy-test-jwt", - ); - - let (output, code) = - run_cli_with_config(tmpdir.path(), &["--gateway", "edge-tunnel-test", "status"]).await; - - let clean = strip_ansi(&output); - assert_eq!( - code, 0, - "openshell status through WS tunnel should exit 0:\n{clean}" - ); - assert!( - clean.to_lowercase().contains("healthy") - || clean.to_lowercase().contains("running") - || clean.to_lowercase().contains("connected") - || clean.contains("✓"), - "status through WS tunnel should report healthy:\n{clean}" - ); -} diff --git a/e2e/rust/tests/ephemeral_cleanup.rs b/e2e/rust/tests/ephemeral_cleanup.rs deleted file mode 100644 index eca4bb1d68..0000000000 --- a/e2e/rust/tests/ephemeral_cleanup.rs +++ /dev/null @@ -1,189 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Detached ephemeral lifecycle coverage shared by supervisor-based drivers. - -#![cfg(feature = "e2e")] - -use std::path::PathBuf; -use std::process::{Command, Stdio}; -use std::time::Duration; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::cli::run_cli; -use openshell_e2e::harness::container::{ContainerEngine, e2e_driver}; -use openshell_e2e::harness::sandbox::{E2E_WORKLOAD_IMAGE, unique_sandbox_name}; -use serial_test::serial; -use tokio::time::{Instant, sleep}; - -const CLEANUP_TIMEOUT: Duration = Duration::from_secs(90); - -struct DeleteOnFailure { - name: String, - armed: bool, -} - -impl Drop for DeleteOnFailure { - fn drop(&mut self) { - if self.armed { - let _ = Command::new(openshell_bin()) - .args(["sandbox", "delete", &self.name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } - } -} - -fn command_output(mut command: Command) -> Result { - let output = command - .output() - .map_err(|error| format!("run driver resource query: {error}"))?; - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - if !output.status.success() { - return Err(format!( - "driver resource query failed ({}): {stdout}{stderr}", - output.status - )); - } - Ok(stdout.trim().to_string()) -} - -fn driver_resources_present(sandbox_id: &str) -> Result { - match e2e_driver().as_deref() { - Some("docker" | "podman") => { - let engine = ContainerEngine::from_env()?; - let mut command = engine.command(); - command.args([ - "ps", - "-aq", - "--filter", - &format!("label=openshell.ai/sandbox-id={sandbox_id}"), - ]); - Ok(!command_output(command)?.is_empty()) - } - Some("kubernetes") => { - let mut command = Command::new("kubectl"); - command.args([ - "get", - "pods,sandboxes.agents.x-k8s.io", - "--all-namespaces", - "--selector", - &format!("openshell.ai/sandbox-id={sandbox_id}"), - "--output=name", - ]); - Ok(!command_output(command)?.is_empty()) - } - Some("vm") => { - let state_dir = std::env::var_os("OPENSHELL_E2E_VM_STATE_DIR") - .map(PathBuf::from) - .ok_or("OPENSHELL_E2E_VM_STATE_DIR must be set for VM resource checks")?; - Ok(state_dir.join("sandboxes").join(sandbox_id).exists()) - } - other => Err(format!( - "unsupported e2e driver for ephemeral cleanup: {other:?}" - )), - } -} - -async fn run_detached_ephemeral_cleanup(exit_code: i32) -> Result<(), String> { - let name = unique_sandbox_name(); - let mut cleanup = DeleteOnFailure { - name: name.clone(), - armed: true, - }; - let release_path = format!("/sandbox/.ephemeral-release-{name}"); - let script = - format!("while [ ! -e '{release_path}' ]; do sleep 0.1; done; sleep 1; exit {exit_code}"); - let mut create = openshell_cmd(); - create.args([ - "sandbox", - "create", - "--name", - &name, - "--from", - E2E_WORKLOAD_IMAGE, - "--no-keep", - "--detach", - "--", - "sh", - "-c", - &script, - ]); - let created = create - .output() - .await - .map_err(|error| format!("create detached ephemeral sandbox: {error}"))?; - if !created.status.success() { - return Err(format!( - "create detached ephemeral sandbox failed ({}): {}{}", - created.status, - String::from_utf8_lossy(&created.stdout), - String::from_utf8_lossy(&created.stderr) - )); - } - - let (details, get_code) = run_cli(&["sandbox", "get", &name, "--output", "json"]).await; - if get_code != 0 { - return Err(format!("get detached sandbox failed: {details}")); - } - let details: serde_json::Value = - serde_json::from_str(&details).map_err(|error| format!("parse sandbox JSON: {error}"))?; - let sandbox_id = details["id"] - .as_str() - .ok_or_else(|| format!("sandbox has no id: {details}"))?; - if !driver_resources_present(sandbox_id)? { - return Err(format!( - "driver resources were never observed for sandbox {name} ({sandbox_id})" - )); - } - - let (release, release_code) = run_cli(&[ - "sandbox", - "exec", - "--name", - &name, - "--no-tty", - "--no-login-shell", - "--", - "touch", - &release_path, - ]) - .await; - if release_code != 0 { - return Err(format!("release canonical process failed: {release}")); - } - - let deadline = Instant::now() + CLEANUP_TIMEOUT; - loop { - let (names, list_code) = run_cli(&["sandbox", "list", "--names"]).await; - if list_code != 0 { - return Err(format!("list sandboxes failed: {names}")); - } - let record_present = names.lines().any(|line| line.trim() == name); - let resources_present = driver_resources_present(sandbox_id)?; - if !record_present && !resources_present { - cleanup.armed = false; - return Ok(()); - } - if Instant::now() >= deadline { - return Err(format!( - "ephemeral sandbox {name} ({sandbox_id}) remained after {CLEANUP_TIMEOUT:?}: record_present={record_present}, resources_present={resources_present}" - )); - } - sleep(Duration::from_millis(500)).await; - } -} - -#[tokio::test] -#[serial(ephemeral_cleanup)] -async fn detached_ephemeral_success_removes_sandbox_and_driver_resources() { - run_detached_ephemeral_cleanup(0).await.unwrap(); -} - -#[tokio::test] -#[serial(ephemeral_cleanup)] -async fn detached_ephemeral_failure_removes_sandbox_and_driver_resources() { - run_detached_ephemeral_cleanup(17).await.unwrap(); -} diff --git a/e2e/rust/tests/internet_network_perf.rs b/e2e/rust/tests/internet_network_perf.rs deleted file mode 100644 index 70c0f1af25..0000000000 --- a/e2e/rust/tests/internet_network_perf.rs +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Opt-in Internet benchmark for the complete sandbox-to-supervisor data path. - -#![cfg(feature = "e2e-host-gateway")] - -use std::io::Write as _; -use std::process::Stdio; - -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const BENCHMARK: &str = r#" -import http.client -import json -import os -import socket -import ssl -import statistics -import time - -HOST = "example.com" -ITERATIONS = int(os.environ.get("OPENSHELL_INET_PERF_ITERATIONS", "20")) - -def percentile(values, fraction): - values = sorted(values) - return values[min(len(values) - 1, int((len(values) - 1) * fraction))] - -def measure(name, operation, iterations=ITERATIONS): - operation() - samples = [] - started = time.perf_counter_ns() - for _ in range(iterations): - before = time.perf_counter_ns() - operation() - samples.append((time.perf_counter_ns() - before) / 1_000_000) - elapsed = (time.perf_counter_ns() - started) / 1_000_000_000 - return { - "name": name, - "iterations": iterations, - "mean_ms": statistics.fmean(samples), - "p50_ms": percentile(samples, 0.50), - "p95_ms": percentile(samples, 0.95), - "ops_per_second": iterations / elapsed, - } - -def dns_lookup(): - result = socket.getaddrinfo(HOST, 443, socket.AF_INET, socket.SOCK_STREAM) - if not result: - raise RuntimeError("DNS returned no IPv4 addresses") - -def tcp_connect(): - with socket.create_connection((HOST, 443), timeout=10): - pass - -tls_context = ssl.create_default_context() - -def https_cold(): - connection = http.client.HTTPSConnection(HOST, 443, timeout=10, context=tls_context) - try: - connection.request("HEAD", "/", headers={"Connection": "close"}) - response = connection.getresponse() - response.read() - if response.status != 200: - raise RuntimeError(f"unexpected HTTP status {response.status}") - finally: - connection.close() - -warm_connection = http.client.HTTPSConnection(HOST, 443, timeout=10, context=tls_context) - -def https_reuse(): - warm_connection.request("HEAD", "/", headers={"Connection": "keep-alive"}) - response = warm_connection.getresponse() - response.read() - if response.status != 200: - raise RuntimeError(f"unexpected HTTP status {response.status}") - -try: - metrics = [ - measure("dns_lookup", dns_lookup), - measure("tcp_connect", tcp_connect), - measure("https_cold", https_cold, max(5, ITERATIONS // 2)), - measure("https_reuse", https_reuse), - ] -finally: - warm_connection.close() - -print(json.dumps({"metrics": metrics}, separators=(",", ":"))) -"#; - -fn write_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - write!( - file, - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -network_policies: - internet_performance: - name: internet_performance - endpoints: - - host: example.com - port: 80 - protocol: tcp - - host: example.com - port: 443 - protocol: tcp - binaries: - - path: "/**" -"# - ) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -async fn run_host_benchmark() -> Result { - let output = tokio::process::Command::new("python3") - .args(["-c", BENCHMARK]) - .env("OPENSHELL_INET_PERF_ITERATIONS", "20") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await - .map_err(|error| format!("run host benchmark: {error}"))?; - let combined = format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - if !output.status.success() { - return Err(format!("host benchmark failed: {combined}")); - } - Ok(combined) -} - -#[tokio::test] -#[ignore = "manual Internet performance benchmark"] -async fn benchmark_complete_internet_path() { - let policy = write_policy().expect("write Internet benchmark policy"); - let policy_path = policy.path().to_str().expect("UTF-8 policy path"); - let sandbox = SandboxGuard::create(&["--policy", policy_path]) - .await - .expect("create benchmark sandbox"); - - for round in 1..=3 { - let host = run_host_benchmark().await.expect("host benchmark"); - println!("INTERNET_PERF host round={round} {}", host.trim()); - let mediated = sandbox - .exec(&[ - "sh", - "-c", - "OPENSHELL_INET_PERF_ITERATIONS=20 python3 -c \"$1\"", - "openshell-internet-perf", - BENCHMARK, - ]) - .await - .expect("sandbox benchmark"); - println!("INTERNET_PERF sandbox round={round} {}", mediated.trim()); - } -} diff --git a/e2e/rust/tests/landlock.rs b/e2e/rust/tests/landlock.rs deleted file mode 100644 index cfbdcd5846..0000000000 --- a/e2e/rust/tests/landlock.rs +++ /dev/null @@ -1,150 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! End-to-end coverage for Landlock filesystem enforcement. - -use std::io::Write; -use std::time::Duration; - -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const SUCCESS_MARKER: &str = "landlock-hard-requirement-ok"; - -fn write_hard_requirement_policy() -> Result { - write_policy("hard_requirement", "/sandbox") -} - -fn write_policy(compatibility: &str, extra_path: &str) -> Result { - let mut file = - NamedTempFile::new().map_err(|error| format!("create temporary policy: {error}"))?; - let policy = r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /etc - - /proc - read_write: - - /sandbox - - /tmp - -landlock: - compatibility: hard_requirement - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - landlock_regression: - name: landlock_regression - endpoints: - - host: example.com - port: 443 - binaries: - - path: "/**" -"#; - - let policy = policy - .replace( - "compatibility: hard_requirement", - &format!("compatibility: {compatibility}"), - ) - .replace( - " - /sandbox\n", - &format!(" - /sandbox\n - {extra_path}\n"), - ); - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write temporary policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush temporary policy: {error}"))?; - Ok(file) -} - -#[tokio::test] -async fn hard_requirement_accepts_enriched_device_path() { - let policy = write_hard_requirement_policy().expect("write hard-requirement policy"); - let policy_path = policy.path().to_string_lossy().into_owned(); - let script = concat!( - "set -eu; ", - "bytes=$(head -c 16 /dev/urandom | wc -c); ", - "test \"$bytes\" -eq 16; ", - "printf landlock-ok > /tmp/landlock-check; ", - "test \"$(cat /tmp/landlock-check)\" = landlock-ok; ", - "echo landlock-hard-requirement-ok", - ); - - let mut sandbox = SandboxGuard::create(&["--policy", &policy_path, "--", "sh", "-lc", script]) - .await - .expect("hard_requirement should accept the enriched /dev/urandom path"); - - assert!(sandbox.create_output.contains(SUCCESS_MARKER)); - sandbox.cleanup().await; -} - -async fn check_policy_path_replacement(compatibility: &str, replace_ancestor: bool) { - let policy = write_policy(compatibility, "/sandbox/policy-parent/leaf") - .expect("write filesystem policy"); - let policy_path = policy.path().to_string_lossy().into_owned(); - let mut sandbox = SandboxGuard::create(&["--policy", &policy_path]) - .await - .expect("create sandbox with filesystem policy"); - - sandbox - .exec(&["sh", "-c", "mkdir -p /sandbox/policy-parent/leaf"]) - .await - .expect("create declared directory under writable workspace"); - sandbox - .exec(&[ - "sh", - "-c", - "echo allowed > /sandbox/policy-parent/leaf/check", - ]) - .await - .expect("normal directory should remain usable on the next exec"); - - let script = if replace_ancestor { - "mv /sandbox/policy-parent /sandbox/original-parent; ln -s /var/tmp /sandbox/policy-parent" - } else { - "mv /sandbox/policy-parent/leaf /sandbox/original-leaf; ln -s /var/tmp /sandbox/policy-parent/leaf" - }; - sandbox - .exec(&["sh", "-c", script]) - .await - .expect("replace workspace directory"); - let result = tokio::time::timeout(Duration::from_secs(20), sandbox.exec(&["true"])).await; - // Delete through the control plane even when further workload launches fail. - sandbox.cleanup().await; - let error = result - .expect("rejected launch must close its SSH channel promptly") - .expect_err("mutable policy symlink must prevent the next launch"); - assert!( - error.to_lowercase().contains("symlink"), - "unexpected launch error: {error}" - ); -} - -#[tokio::test] -async fn best_effort_rejects_replaced_policy_path() { - check_policy_path_replacement("best_effort", false).await; -} - -#[tokio::test] -async fn hard_requirement_rejects_replaced_policy_path() { - check_policy_path_replacement("hard_requirement", false).await; -} - -#[tokio::test] -async fn best_effort_rejects_replaced_policy_ancestor() { - check_policy_path_replacement("best_effort", true).await; -} - -#[tokio::test] -async fn hard_requirement_rejects_replaced_policy_ancestor() { - check_policy_path_replacement("hard_requirement", true).await; -} diff --git a/e2e/rust/tests/live_internet_traffic_perf.rs b/e2e/rust/tests/live_internet_traffic_perf.rs deleted file mode 100644 index e8727bbce3..0000000000 --- a/e2e/rust/tests/live_internet_traffic_perf.rs +++ /dev/null @@ -1,337 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Opt-in live-Internet benchmark shaped like a short coding-agent session. - -#![cfg(feature = "e2e-host-gateway")] - -use std::io::Write as _; -use std::process::Stdio; -use std::time::Instant; - -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const DEFAULT_ROUNDS: usize = 10; - -const BENCHMARK: &str = r#" -import concurrent.futures -import http.client -import json -import os -import shutil -import socket -import ssl -import statistics -import subprocess -import tempfile -import time -import urllib.parse - -MODE = os.environ.get("OPENSHELL_LIVE_PERF_MODE", "direct") -TLS = ssl.create_default_context() -USER_AGENT = "OpenShell-live-Internet-benchmark/1" - -METADATA_URLS = [ - "https://raw.githubusercontent.com/NVIDIA/OpenShell/main/README.md", - "https://pypi.org/pypi/requests/json", - "https://registry.npmjs.org/typescript/latest", - "https://docs.python.org/3/", -] -DNS_HOSTS = [ - "github.com", - "raw.githubusercontent.com", - "pypi.org", - "registry.npmjs.org", - "crates.io", - "docs.python.org", - "speed.cloudflare.com", -] - -def percentile(values, fraction): - values = sorted(values) - return values[min(len(values) - 1, int((len(values) - 1) * fraction))] - -def request(url, method="GET", max_bytes=None): - parsed = urllib.parse.urlsplit(url) - connection = http.client.HTTPSConnection(parsed.hostname, parsed.port or 443, timeout=20, context=TLS) - path = parsed.path or "/" - if parsed.query: - path += "?" + parsed.query - started = time.perf_counter_ns() - connection.request(method, path, headers={"User-Agent": USER_AGENT, "Connection": "close"}) - response = connection.getresponse() - first = response.read(1) - first_byte_ms = (time.perf_counter_ns() - started) / 1_000_000 - body_bytes = len(first) - while max_bytes is None or body_bytes < max_bytes: - remaining = None if max_bytes is None else max_bytes - body_bytes - chunk = response.read(65536 if remaining is None else min(65536, remaining)) - if not chunk: - break - body_bytes += len(chunk) - total_ms = (time.perf_counter_ns() - started) / 1_000_000 - status = response.status - connection.close() - if status < 200 or status >= 400: - raise RuntimeError(f"{url} returned HTTP {status}") - return {"status": status, "bytes": body_bytes, "first_byte_ms": first_byte_ms, "total_ms": total_ms} - -def metric(name, operation): - started = time.perf_counter_ns() - try: - detail = operation() - return { - "name": name, - "ok": True, - "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, - "detail": detail, - } - except Exception as error: - return { - "name": name, - "ok": False, - "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, - "error": f"{type(error).__name__}: {error}", - } - -def dns_set(): - samples = [] - for host in DNS_HOSTS: - started = time.perf_counter_ns() - addresses = socket.getaddrinfo(host, 443, socket.AF_UNSPEC, socket.SOCK_STREAM) - samples.append({ - "host": host, - "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, - "addresses": len(addresses), - }) - timings = [sample["elapsed_ms"] for sample in samples] - return { - "lookups": samples, - "p50_ms": percentile(timings, 0.50), - "p95_ms": percentile(timings, 0.95), - } - -def metadata_serial(): - return {"requests": [request(url, max_bytes=2_000_000) for url in METADATA_URLS]} - -def metadata_concurrent(): - urls = METADATA_URLS + METADATA_URLS - with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: - responses = list(pool.map(lambda url: request(url, max_bytes=2_000_000), urls)) - return {"requests": responses} - -def https_reuse(): - connection = http.client.HTTPSConnection("example.com", 443, timeout=20, context=TLS) - samples = [] - try: - for _ in range(20): - started = time.perf_counter_ns() - connection.request("HEAD", "/", headers={"User-Agent": USER_AGENT, "Connection": "keep-alive"}) - response = connection.getresponse() - response.read() - if response.status != 200: - raise RuntimeError(f"example.com returned HTTP {response.status}") - samples.append((time.perf_counter_ns() - started) / 1_000_000) - finally: - connection.close() - return { - "requests": len(samples), - "p50_ms": percentile(samples, 0.50), - "p95_ms": percentile(samples, 0.95), - "mean_ms": statistics.fmean(samples), - } - -def git_clone(): - if shutil.which("git") is None: - return {"skipped": "git is not installed in the workload image"} - with tempfile.TemporaryDirectory(prefix="openshell-live-git-") as directory: - checkout = os.path.join(directory, "sampleproject") - command = [ - "git", "-c", "advice.detachedHead=false", "clone", "--quiet", - "--depth", "1", "--filter=blob:none", - "https://github.com/pypa/sampleproject.git", checkout, - ] - output = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, check=False) - if output.returncode != 0: - raise RuntimeError(output.stderr.decode(errors="replace")[-500:]) - files = sum(len(names) for _, _, names in os.walk(checkout)) - size = sum(os.path.getsize(os.path.join(root, name)) for root, _, names in os.walk(checkout) for name in names) - return {"files": files, "checkout_bytes": size} - -def package_download(): - metadata = request("https://pypi.org/pypi/idna/json", max_bytes=2_000_000) - connection = http.client.HTTPSConnection("pypi.org", 443, timeout=20, context=TLS) - connection.request("GET", "/pypi/idna/json", headers={"User-Agent": USER_AGENT}) - response = connection.getresponse() - document = json.loads(response.read()) - connection.close() - wheels = [entry for entry in document["urls"] if entry["packagetype"] == "bdist_wheel"] - if not wheels: - raise RuntimeError("PyPI returned no idna wheel") - artifact = request(wheels[0]["url"], max_bytes=5_000_000) - return {"metadata": metadata, "artifact": artifact, "filename": wheels[0]["filename"]} - -def bulk_download(): - return request("https://speed.cloudflare.com/__down?bytes=5242880", max_bytes=5242880) - -def denied_destination(): - if MODE != "sandbox": - return {"skipped": "policy denial applies only to sandbox mode"} - started = time.perf_counter_ns() - try: - socket.create_connection(("example.net", 443), timeout=5).close() - except OSError as error: - return { - "denied": True, - "latency_ms": (time.perf_counter_ns() - started) / 1_000_000, - "error": str(error), - } - raise RuntimeError("destination omitted from policy was reachable") - -started = time.perf_counter_ns() -metrics = [ - metric("dns_service_set", dns_set), - metric("https_metadata_serial", metadata_serial), - metric("https_metadata_concurrent", metadata_concurrent), - metric("https_reuse", https_reuse), - metric("git_clone", git_clone), - metric("package_download", package_download), - metric("bulk_download_5mib", bulk_download), - metric("policy_denial", denied_destination), -] -document = { - "schema": "openshell.live-internet-perf.v1", - "mode": MODE, - "total_ms": (time.perf_counter_ns() - started) / 1_000_000, - "metrics": metrics, -} -print(json.dumps(document, separators=(",", ":"))) -if any(item["name"] == "policy_denial" and not item["ok"] for item in metrics): - raise SystemExit(1) -"#; - -fn rounds() -> usize { - std::env::var("OPENSHELL_LIVE_PERF_ROUNDS") - .ok() - .and_then(|value| value.parse().ok()) - .filter(|value| *value > 0) - .unwrap_or(DEFAULT_ROUNDS) -} - -fn write_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - write!( - file, - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -network_policies: - live_internet_performance: - name: live_internet_performance - endpoints: - - {{ host: example.com, port: 443, protocol: tcp }} - - {{ host: github.com, port: 443, protocol: tcp }} - - {{ host: raw.githubusercontent.com, port: 443, protocol: tcp }} - - {{ host: pypi.org, port: 443, protocol: tcp }} - - {{ host: files.pythonhosted.org, port: 443, protocol: tcp }} - - {{ host: registry.npmjs.org, port: 443, protocol: tcp }} - - {{ host: crates.io, port: 443, protocol: tcp }} - - {{ host: docs.python.org, port: 443, protocol: tcp }} - - {{ host: speed.cloudflare.com, port: 443, protocol: tcp }} - binaries: - - path: "/**" -"# - ) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -async fn run_direct() -> Result { - run_python("direct").await -} - -async fn run_python(mode: &str) -> Result { - let output = tokio::process::Command::new("python3") - .args(["-c", BENCHMARK]) - .env("OPENSHELL_LIVE_PERF_MODE", mode) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await - .map_err(|error| format!("run {mode} live Internet benchmark: {error}"))?; - let combined = format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - if !output.status.success() { - return Err(format!( - "{mode} live Internet benchmark failed (exit {:?}): {combined}", - output.status.code() - )); - } - Ok(combined) -} - -async fn run_sandbox(sandbox: &SandboxGuard) -> Result { - sandbox - .exec(&[ - "sh", - "-c", - "OPENSHELL_LIVE_PERF_MODE=sandbox python3 -c \"$1\"", - "openshell-live-internet-perf", - BENCHMARK, - ]) - .await -} - -#[tokio::test] -#[ignore = "manual live-Internet performance benchmark"] -async fn benchmark_live_internet_agent_traffic() { - let policy = write_policy().expect("write live Internet benchmark policy"); - let policy_path = policy.path().to_str().expect("UTF-8 policy path"); - let create_started = Instant::now(); - let mut sandbox = SandboxGuard::create(&["--policy", policy_path]) - .await - .expect("create live Internet benchmark sandbox"); - println!( - "LIVE_INTERNET_PERF create {{\"elapsed_ms\":{}}}", - create_started.elapsed().as_secs_f64() * 1000.0 - ); - - for round in 1..=rounds() { - if round % 2 == 1 { - let direct = run_direct().await.expect("direct live Internet benchmark"); - println!("LIVE_INTERNET_PERF direct round={round} {}", direct.trim()); - let mediated = run_sandbox(&sandbox) - .await - .expect("sandbox live Internet benchmark"); - println!( - "LIVE_INTERNET_PERF sandbox round={round} {}", - mediated.trim() - ); - } else { - let mediated = run_sandbox(&sandbox) - .await - .expect("sandbox live Internet benchmark"); - println!( - "LIVE_INTERNET_PERF sandbox round={round} {}", - mediated.trim() - ); - let direct = run_direct().await.expect("direct live Internet benchmark"); - println!("LIVE_INTERNET_PERF direct round={round} {}", direct.trim()); - } - } - - sandbox.cleanup().await; -} diff --git a/e2e/rust/tests/oidc_pkce.rs b/e2e/rust/tests/oidc_pkce.rs index 929f21b676..409d99bf30 100644 --- a/e2e/rust/tests/oidc_pkce.rs +++ b/e2e/rust/tests/oidc_pkce.rs @@ -1457,8 +1457,8 @@ async fn delete_workspace(admin: &LoginSession, workspace: &str) { } // Keep RBAC fixtures running until cleanup so authorization checks do not also -// exercise fast canonical-process completion. That lifecycle belongs to the -// dedicated sandbox_lifecycle suite. +// exercise fast canonical-process completion. The conformance `lifecycle` +// suite covers that. async fn assert_can_create_sandbox(session: &LoginSession, workspace: &str, sandbox_name: &str) { let create = run_workspace_cli( session, diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs deleted file mode 100644 index 6b137d1a2b..0000000000 --- a/e2e/rust/tests/podman_oci_identity.rs +++ /dev/null @@ -1,213 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-podman")] - -//! Podman-specific E2E coverage for the supervisor companion container. -//! -//! Podman runs the supervisor as the resolved OCI workload identity with no -//! effective capabilities; Docker runs it as a fixed unprivileged user. The -//! test builds an image through the selected Podman engine, creates a sandbox -//! from it, and inspects the real supervisor container. The workload's image -//! ID, identity, network, and mounts, and what the main process and -//! `sandbox exec` see, are shared with Docker and covered by the `oci-image` -//! feature suite in `tests/suites/features`. - -use std::process::Stdio; - -use openshell_e2e::harness::container::{ContainerEngine, is_e2e_driver}; -use openshell_e2e::harness::sandbox::SandboxGuard; - -const BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; -const READY_MARKER: &str = "podman-oci-identity-ready"; -const OCI_UID: &str = "2345"; -const OCI_GID: &str = "2346"; -const OCI_FALLBACK_POLICY: &str = r"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /etc] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort - -network_policies: {} -"; - -struct ImageGuard { - engine: ContainerEngine, - tag: String, -} - -impl ImageGuard { - fn build() -> Result { - let engine = ContainerEngine::from_env()?; - if engine.name() != "podman" { - return Err(format!( - "Podman OCI identity E2E requires podman, got {}", - engine.name() - )); - } - - let context = tempfile::tempdir().map_err(|err| format!("create build context: {err}"))?; - let containerfile = context.path().join("Containerfile"); - std::fs::write( - &containerfile, - format!( - r"FROM {BASE_IMAGE} -USER 0:0 -RUN mkdir -p /home/app/project && \ - chown {OCI_UID}:{OCI_GID} /home/app /home/app/project && \ - chmod 0700 /home/app /home/app/project -WORKDIR /home/app/project -USER {OCI_UID}:{OCI_GID} -" - ), - ) - .map_err(|err| format!("write Containerfile: {err}"))?; - - let tag = format!( - "localhost/openshell-e2e-podman-oci-identity:{}", - std::process::id() - ); - run_engine( - &engine, - &[ - "build", - "--pull=never", - "--file", - containerfile - .to_str() - .ok_or_else(|| "Containerfile path is not UTF-8".to_string())?, - "--tag", - &tag, - context - .path() - .to_str() - .ok_or_else(|| "build context path is not UTF-8".to_string())?, - ], - )?; - let user = run_engine( - &engine, - &["image", "inspect", "--format", "{{.Config.User}}", &tag], - )?; - if user != format!("{OCI_UID}:{OCI_GID}") { - return Err(format!( - "Podman-built image has OCI user '{user}', expected {OCI_UID}:{OCI_GID}" - )); - } - - Ok(Self { engine, tag }) - } -} - -impl Drop for ImageGuard { - fn drop(&mut self) { - let _ = self - .engine - .command() - .args(["image", "rm", "--force", &self.tag]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -fn run_engine(engine: &ContainerEngine, args: &[&str]) -> Result { - let output = engine - .command() - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .map_err(|err| format!("failed to run {} {}: {err}", engine.name(), args.join(" ")))?; - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - if !output.status.success() { - return Err(format!( - "{} {} failed (exit {:?}):\n{stdout}{stderr}", - engine.name(), - args.join(" "), - output.status.code() - )); - } - Ok(stdout.trim().to_string()) -} - -fn container_id_for_role( - engine: &ContainerEngine, - sandbox_name: &str, - role: &str, -) -> Result { - let name_filter = format!("label=openshell.ai/sandbox-name={sandbox_name}"); - let role_filter = format!("label=openshell.ai/isolation-role={role}"); - let stdout = run_engine( - engine, - &[ - "ps", - "-aq", - "--filter", - "label=openshell.managed=true", - "--filter", - &name_filter, - "--filter", - &role_filter, - ], - )?; - let ids = stdout - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .collect::>(); - match ids.as_slice() { - [id] => Ok((*id).to_string()), - [] => Err(format!( - "no Podman container found for sandbox '{sandbox_name}'" - )), - _ => Err(format!( - "multiple Podman containers found for sandbox '{sandbox_name}': {ids:?}" - )), - } -} - -#[tokio::test] -async fn podman_supervisor_runs_as_oci_identity_without_capabilities() { - if !is_e2e_driver("podman") { - eprintln!("Skipping Podman OCI identity test: e2e driver is not podman"); - return; - } - - let image = ImageGuard::build().expect("build Podman OCI identity image"); - // The fixture image contains a policy with an - // explicit `sandbox` process identity. Supply a complete policy that - // intentionally omits `process` so this test exercises OCI fallback. - let policy = tempfile::NamedTempFile::new().expect("create OCI fallback policy"); - std::fs::write(policy.path(), OCI_FALLBACK_POLICY).expect("write OCI fallback policy"); - let policy_path = policy.path().to_str().expect("policy path is UTF-8"); - let mut sandbox = SandboxGuard::create_keep_with_args( - &["--from", &image.tag, "--policy", policy_path, "--no-tty"], - &["sh", "-c", "echo podman-oci-identity-ready; sleep infinity"], - READY_MARKER, - ) - .await - .expect("create sandbox from Podman-built OCI identity image"); - - let supervisor_id = container_id_for_role(&image.engine, &sandbox.name, "supervisor") - .expect("find separate supervisor companion"); - let supervisor_user = run_engine( - &image.engine, - &["inspect", "--format", "{{.Config.User}}", &supervisor_id], - ) - .unwrap(); - assert_eq!(supervisor_user, format!("{OCI_UID}:{OCI_GID}")); - let supervisor_caps = run_engine( - &image.engine, - &["inspect", "--format", "{{.EffectiveCaps}}", &supervisor_id], - ) - .unwrap(); - assert_eq!( - supervisor_caps, "[]", - "the supervisor companion may not have effective capabilities" - ); - sandbox.cleanup().await; -} diff --git a/e2e/rust/tests/podman_resource_limits.rs b/e2e/rust/tests/podman_resource_limits.rs index a0282f0e09..225f77ff8f 100644 --- a/e2e/rust/tests/podman_resource_limits.rs +++ b/e2e/rust/tests/podman_resource_limits.rs @@ -7,7 +7,7 @@ //! resource limits are actually enforced by cgroups inside the workload, not //! just echoed back by the API/template layer. //! -//! `e2e/rust/tests/sandbox_templates.rs` already verifies that +//! The conformance `templates` suite already verifies that //! `sandbox template create --cpu ... --memory ...` is stored and returned //! correctly, but it never inspects the resulting container's real resource //! state. This test creates a sandbox with those flags directly and reads the diff --git a/e2e/rust/tests/sandbox_labels.rs b/e2e/rust/tests/sandbox_labels.rs deleted file mode 100644 index 937171ead3..0000000000 --- a/e2e/rust/tests/sandbox_labels.rs +++ /dev/null @@ -1,230 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use std::process::Stdio; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::{extract_field, strip_ansi}; -use openshell_e2e::harness::sandbox::SandboxGuard; - -fn normalize_output(output: &str) -> String { - let stripped = strip_ansi(output).replace('\r', ""); - let mut cleaned = String::with_capacity(stripped.len()); - - for ch in stripped.chars() { - match ch { - '\u{8}' => { - cleaned.pop(); - } - '\u{4}' => {} - _ => cleaned.push(ch), - } - } - - cleaned -} - -fn extract_sandbox_name(output: &str) -> Option { - if let Some((_, rest)) = output.split_once("Created sandbox:") { - return rest.split_whitespace().next().map(ToOwned::to_owned); - } - - extract_field(output, "Created sandbox").or_else(|| extract_field(output, "Name")) -} - -async fn create_sandbox_with_labels(name: &str, labels: &[(&str, &str)]) -> String { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "create", "--detach", "--name", name]); - - for (key, value) in labels { - cmd.arg("--label").arg(format!("{key}={value}")); - } - - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox create"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - assert!( - output.status.success(), - "sandbox create should succeed (exit {:?}):\n{combined}", - output.status.code() - ); - - extract_sandbox_name(&combined).expect("sandbox name should be present in output") -} - -async fn list_sandboxes_with_selector(selector: &str) -> Vec { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "list", "--names"]); - - if !selector.is_empty() { - cmd.arg("--selector").arg(selector); - } - - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox list"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - assert!( - output.status.success(), - "sandbox list should succeed (exit {:?}):\n{combined}", - output.status.code() - ); - - combined - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .map(ToOwned::to_owned) - .collect() -} - -async fn get_sandbox_details(name: &str) -> String { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "get", name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox get"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - assert!( - output.status.success(), - "sandbox get should succeed (exit {:?}):\n{combined}", - output.status.code() - ); - - combined -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] // end-to-end test exercises full label lifecycle -async fn sandbox_labels_are_stored_and_filterable() { - // Keep the unique names below the public 19-character routable-name - // limit. Hex keeps the per-process suffix compact on busy CI hosts. - let suffix = format!("{:x}", std::process::id()); - let dev_backend = format!("lbl-db-{suffix}"); - let staging_backend = format!("lbl-sb-{suffix}"); - let prod_frontend = format!("lbl-pf-{suffix}"); - let dev_data = format!("lbl-dd-{suffix}"); - - // Arm guards before create so a partial create still cleans up. - let _cleanup: Vec<_> = [&dev_backend, &staging_backend, &prod_frontend, &dev_data] - .into_iter() - .map(|name| SandboxGuard::manage_existing(name.clone())) - .collect(); - - // Create sandboxes with different labels - let name1 = - create_sandbox_with_labels(&dev_backend, &[("env", "dev"), ("team", "backend")]).await; - - let name2 = - create_sandbox_with_labels(&staging_backend, &[("env", "staging"), ("team", "backend")]) - .await; - - let name3 = - create_sandbox_with_labels(&prod_frontend, &[("env", "prod"), ("team", "frontend")]).await; - - let name4 = create_sandbox_with_labels(&dev_data, &[("env", "dev"), ("team", "data")]).await; - - // Test 1: Verify labels are stored in sandbox metadata - let details = get_sandbox_details(&name1).await; - assert!( - details.contains("env: dev"), - "sandbox should have env=dev label in details:\n{details}" - ); - assert!( - details.contains("team: backend"), - "sandbox should have team=backend label in details:\n{details}" - ); - - // Test 2: Filter by single label (env=dev) - let dev_sandboxes = list_sandboxes_with_selector("env=dev").await; - assert!( - dev_sandboxes.contains(&name1), - "env=dev filter should include {name1}, got: {dev_sandboxes:?}" - ); - assert!( - dev_sandboxes.contains(&name4), - "env=dev filter should include {name4}, got: {dev_sandboxes:?}" - ); - assert!( - !dev_sandboxes.contains(&name2), - "env=dev filter should not include staging sandbox {name2}, got: {dev_sandboxes:?}" - ); - assert!( - !dev_sandboxes.contains(&name3), - "env=dev filter should not include prod sandbox {name3}, got: {dev_sandboxes:?}" - ); - - // Test 3: Filter by single label (team=backend) - let backend_sandboxes = list_sandboxes_with_selector("team=backend").await; - assert!( - backend_sandboxes.contains(&name1), - "team=backend filter should include {name1}, got: {backend_sandboxes:?}" - ); - assert!( - backend_sandboxes.contains(&name2), - "team=backend filter should include {name2}, got: {backend_sandboxes:?}" - ); - assert!( - !backend_sandboxes.contains(&name3), - "team=backend filter should not include frontend sandbox {name3}, got: {backend_sandboxes:?}" - ); - assert!( - !backend_sandboxes.contains(&name4), - "team=backend filter should not include data sandbox {name4}, got: {backend_sandboxes:?}" - ); - - // Test 4: Filter by multiple labels (AND logic: env=dev,team=backend) - let dev_backend_sandboxes = list_sandboxes_with_selector("env=dev,team=backend").await; - assert_eq!( - dev_backend_sandboxes - .iter() - .filter(|name| [&name1, &name2, &name3, &name4].contains(name)) - .count(), - 1, - "env=dev,team=backend filter should return exactly 1 sandbox, got: {dev_backend_sandboxes:?}" - ); - assert!( - dev_backend_sandboxes.contains(&name1), - "env=dev,team=backend filter should include {name1}, got: {dev_backend_sandboxes:?}" - ); - - // Test 5: Filter by non-existent label value - let qa_sandboxes = list_sandboxes_with_selector("team=qa").await; - assert!( - !qa_sandboxes.contains(&name1) - && !qa_sandboxes.contains(&name2) - && !qa_sandboxes.contains(&name3) - && !qa_sandboxes.contains(&name4), - "team=qa filter should not return any of our test sandboxes, got: {qa_sandboxes:?}" - ); - - // Test 6: List all sandboxes (no filter) - let all_sandboxes = list_sandboxes_with_selector("").await; - assert!( - all_sandboxes.contains(&name1), - "list without filter should include all test sandboxes" - ); - assert!( - all_sandboxes.contains(&name2), - "list without filter should include all test sandboxes" - ); - assert!( - all_sandboxes.contains(&name3), - "list without filter should include all test sandboxes" - ); - assert!( - all_sandboxes.contains(&name4), - "list without filter should include all test sandboxes" - ); -} diff --git a/e2e/rust/tests/sandbox_lifecycle.rs b/e2e/rust/tests/sandbox_lifecycle.rs deleted file mode 100644 index b299a5d1d7..0000000000 --- a/e2e/rust/tests/sandbox_lifecycle.rs +++ /dev/null @@ -1,1515 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -#[cfg(target_os = "linux")] -use std::fs; -use std::process::Stdio; -use std::time::Duration; - -use openshell_e2e::harness::binary::{openshell_cmd, openshell_tty_cmd}; -use openshell_e2e::harness::cli::{ - run_cli, wait_for_sandbox_exec_contains, wait_for_sandbox_phase, -}; -use openshell_e2e::harness::output::{extract_field, strip_ansi}; -use openshell_e2e::harness::sandbox::{E2E_WORKLOAD_IMAGE, SandboxGuard, unique_sandbox_name}; -use serial_test::serial; -use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; -use tokio::time::{Instant, sleep}; - -const SANDBOX_PRESENCE_TIMEOUT: Duration = Duration::from_secs(30); -const SANDBOX_LIST_POLL_INTERVAL: Duration = Duration::from_millis(500); -const SANDBOX_RESTART_TIMEOUT: Duration = Duration::from_secs(60); - -fn normalize_output(output: &str) -> String { - let stripped = strip_ansi(output).replace('\r', ""); - let mut cleaned = String::with_capacity(stripped.len()); - - for ch in stripped.chars() { - match ch { - '\u{8}' => { - cleaned.pop(); - } - '\u{4}' => {} - _ => cleaned.push(ch), - } - } - - cleaned -} - -fn extract_sandbox_name(output: &str) -> Option { - if let Some((_, rest)) = output.split_once("Created sandbox:") { - return rest.split_whitespace().next().map(ToOwned::to_owned); - } - - extract_field(output, "Created sandbox").or_else(|| extract_field(output, "Name")) -} - -async fn sandbox_list_names(deadline: Instant) -> Option> { - if Instant::now() >= deadline { - return None; - } - - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "list", "--names"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = match tokio::time::timeout_at(deadline, cmd.output()).await { - Ok(output) => output.expect("spawn openshell sandbox list"), - Err(_) => return None, - }; - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - assert!( - output.status.success(), - "sandbox list should succeed (exit {:?}):\n{combined}", - output.status.code() - ); - - Some( - combined - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .map(ToOwned::to_owned) - .collect(), - ) -} - -async fn assert_sandbox_presence_eventually( - sandbox_name: &str, - should_exist: bool, -) -> Result<(), Vec> { - let deadline = Instant::now() + SANDBOX_PRESENCE_TIMEOUT; - let mut last_sandbox_names = Vec::new(); - - loop { - let Some(sandbox_names) = sandbox_list_names(deadline).await else { - return Err(last_sandbox_names); - }; - let exists = sandbox_names.iter().any(|name| name == sandbox_name); - if exists == should_exist { - return Ok(()); - } - - let now = Instant::now(); - if now >= deadline { - return Err(sandbox_names); - } - - last_sandbox_names = sandbox_names; - sleep(SANDBOX_LIST_POLL_INTERVAL.min(deadline - now)).await; - } -} - -async fn delete_sandbox(name: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "delete", name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_exec_outlives_startup_recovery_deadline() { - let mut sandbox = SandboxGuard::create(&[]) - .await - .expect("create sandbox for exec recovery deadline"); - let output = tokio::time::timeout( - Duration::from_secs(60), - sandbox.exec(&["sh", "-c", "sleep 31; printf past-recovery-deadline"]), - ) - .await; - let next = sandbox.exec(&["printf", "fresh-exec"]).await; - sandbox.cleanup().await; - - assert_eq!( - output.expect("exec timed out").expect("exec failed"), - "past-recovery-deadline", - "expiration must not terminate an admitted command" - ); - assert_eq!(next.expect("fresh exec after expiration"), "fresh-exec"); -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_exec_large_output_is_complete() { - const BYTES: usize = 8 * 1024 * 1024; - let mut sandbox = SandboxGuard::create(&[]) - .await - .expect("create sandbox for large output"); - - for stderr in [false, true] { - let script = if stderr { - format!("yes A | head -c {BYTES} >&2") - } else { - format!("yes A | head -c {BYTES}") - }; - let mut command = openshell_cmd(); - command - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "sh", - "-c", - &script, - ]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let output = tokio::time::timeout(Duration::from_secs(30), command.output()) - .await - .expect("large exec output timed out") - .expect("run large exec output"); - assert!(output.status.success(), "large exec exited unsuccessfully"); - let bytes = if stderr { output.stderr } else { output.stdout }; - assert_eq!(bytes.len(), BYTES, "exec output was truncated"); - assert!(bytes.chunks_exact(2).all(|pair| pair == b"A\n")); - } - - let mut early_exit = openshell_cmd(); - let mut early_child = early_exit - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "head", - "-n1", - ]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .expect("start early-exit exec"); - let mut input = b"first\n".to_vec(); - input.extend(vec![b'x'; 128 * 1024]); - let write_result = early_child - .stdin - .take() - .expect("early-exit stdin") - .write_all(&input) - .await; - if let Err(error) = write_result { - assert_eq!(error.kind(), std::io::ErrorKind::BrokenPipe); - } - let early_output = - tokio::time::timeout(Duration::from_secs(30), early_child.wait_with_output()) - .await - .expect("early-exit exec timed out") - .expect("wait for early-exit exec"); - assert!(early_output.status.success(), "early-exit exec failed"); - assert_eq!(early_output.stdout, b"first\n"); - - let mut command = openshell_cmd(); - let output = tokio::time::timeout( - Duration::from_secs(10), - command - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "sh", - "-c", - "sleep 5 & echo ok", - ]) - .stdin(Stdio::null()) - .output(), - ) - .await - .expect("exec with background pipe holder timed out") - .expect("run exec with background pipe holder"); - assert!(output.status.success(), "unexpected output failure"); - assert_eq!(output.stdout, b"ok\n"); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn piped_exec_stdin_crosses_grpc_message_limit() { - let mut sandbox = SandboxGuard::create(&[]) - .await - .expect("create sandbox for streamed stdin"); - - for size in [5, 1_048_576, 4_194_304] { - let mut command = openshell_cmd(); - command - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "wc", - "-c", - ]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut child = command.spawn().expect("spawn sandbox exec"); - let mut input = child.stdin.take().expect("piped stdin"); - input - .write_all(&vec![b'x'; size]) - .await - .expect("write piped stdin"); - drop(input); - let output = tokio::time::timeout(Duration::from_secs(30), child.wait_with_output()) - .await - .expect("streamed exec timed out") - .expect("wait for streamed exec"); - assert!( - output.status.success(), - "streamed exec failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - assert_eq!( - String::from_utf8_lossy(&output.stdout).trim(), - size.to_string() - ); - } - - let mut oversized = openshell_cmd(); - let mut oversized_child = oversized - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "wc", - "-c", - ]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .expect("spawn oversized stdin exec"); - oversized_child - .stdin - .take() - .expect("oversized stdin pipe") - .write_all(&vec![b'x'; 4_194_305]) - .await - .expect("write oversized stdin"); - let oversized_output = oversized_child - .wait_with_output() - .await - .expect("wait for oversized stdin error"); - assert!(!oversized_output.status.success()); - assert!(oversized_output.stdout.is_empty()); - assert!(String::from_utf8_lossy(&oversized_output.stderr).contains("4 MiB limit")); - - // A forced PTY with a small pipe must keep using the unary RPC for - // compatibility with older gateways. - let mut tty_command = openshell_cmd(); - let mut tty_child = tty_command - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--tty", - "--no-login-shell", - "--", - "sh", - "-c", - "printf tty-ok", - ]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .expect("spawn forced TTY exec"); - tty_child - .stdin - .take() - .expect("piped TTY stdin") - .write_all(b"x") - .await - .expect("write TTY stdin"); - let tty_output = tokio::time::timeout(Duration::from_secs(30), tty_child.wait_with_output()) - .await - .expect("forced TTY exec timed out") - .expect("wait for forced TTY exec"); - assert!( - tty_output.status.success(), - "forced TTY exec failed: {}", - String::from_utf8_lossy(&tty_output.stderr) - ); - assert!(String::from_utf8_lossy(&tty_output.stdout).contains("tty-ok")); - - #[cfg(unix)] - { - let directory = std::fs::File::open("/").expect("open directory as stdin"); - let mut command = openshell_cmd(); - let output = command - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--no-login-shell", - "--", - "wc", - "-c", - ]) - .stdin(Stdio::from(directory)) - .output() - .await - .expect("run exec with unreadable stdin"); - assert!(!output.status.success(), "stdin read error was ignored"); - assert!(output.stdout.is_empty()); - } - - sandbox.cleanup().await; -} - -async fn run_sandbox_lifecycle_command(operation: &str, name: &str) -> String { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", operation, name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .unwrap_or_else(|error| panic!("spawn openshell sandbox {operation}: {error}")); - let combined = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - assert!( - output.status.success(), - "sandbox {operation} should succeed (exit {:?}):\n{combined}", - output.status.code(), - ); - combined -} - -async fn sandbox_details(name: &str) -> String { - let (details, exit_code) = run_cli(&["sandbox", "get", name]).await; - let details = normalize_output(&details); - assert_eq!( - exit_code, 0, - "sandbox get should succeed for {name}:\n{details}" - ); - details -} - -async fn reconnect_with_input_ownership( - sandbox_name: &str, -) -> (tokio::process::Child, Vec) { - let reconnect_deadline = Instant::now() + Duration::from_secs(30); - let mut attempt = 0; - - loop { - attempt += 1; - let token = format!("reconnect-{attempt}"); - let mut reconnect_cmd = openshell_cmd(); - reconnect_cmd - .args(["sandbox", "connect", sandbox_name]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut candidate = reconnect_cmd.spawn().expect("spawn reconnect attachment"); - let mut candidate_stdin = candidate.stdin.take().expect("reconnect stdin"); - candidate_stdin - .write_all(format!("{token}\n").as_bytes()) - .await - .expect("write reconnect ownership probe"); - candidate_stdin - .flush() - .await - .expect("flush reconnect ownership probe"); - - let reconnect_stdout = candidate.stdout.take().expect("reconnect stdout"); - let mut reconnect_lines = BufReader::new(reconnect_stdout).lines(); - let mut candidate_replay = Vec::new(); - let remaining = reconnect_deadline.saturating_duration_since(Instant::now()); - assert!( - !remaining.is_zero(), - "reconnect did not acquire input ownership" - ); - let attempt_timeout = Duration::from_secs(2).min(remaining); - let acquired = tokio::time::timeout(attempt_timeout, async { - loop { - let Some(line) = reconnect_lines - .next_line() - .await - .expect("read reconnect output") - else { - return false; - }; - if line.contains("main_pid=") { - candidate_replay.push(line.clone()); - } - if line.contains(&format!("input={token}")) { - return true; - } - } - }) - .await - .unwrap_or(false); - - if acquired { - return (candidate, candidate_replay); - } - - let _ = candidate.kill().await; - let _ = candidate.wait().await; - assert!( - Instant::now() < reconnect_deadline, - "reconnect did not acquire input ownership" - ); - sleep(Duration::from_millis(100)).await; - } -} - -#[cfg(target_os = "linux")] -fn find_process_with_args(expected_args: &[&str]) -> Option { - for entry in fs::read_dir("/proc").ok()?.filter_map(Result::ok) { - let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { - continue; - }; - let cmdline = fs::read(entry.path().join("cmdline")).unwrap_or_default(); - let args = cmdline.split(|byte| *byte == 0).collect::>(); - if expected_args - .iter() - .all(|expected| args.contains(&expected.as_bytes())) - { - return Some(pid); - } - } - None -} - -#[cfg(target_os = "linux")] -fn find_child_process_with_args(parent_pid: u32, expected_args: &[&str]) -> Option { - for entry in fs::read_dir("/proc").ok()?.filter_map(Result::ok) { - let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { - continue; - }; - let status = fs::read_to_string(entry.path().join("status")).unwrap_or_default(); - let process_parent = status.lines().find_map(|line| { - line.strip_prefix("PPid:") - .and_then(|value| value.trim().parse::().ok()) - }); - if process_parent != Some(parent_pid) { - continue; - } - let cmdline = fs::read(entry.path().join("cmdline")).unwrap_or_default(); - let args = cmdline.split(|byte| *byte == 0).collect::>(); - if expected_args - .iter() - .all(|expected| args.contains(&expected.as_bytes())) - { - return Some(pid); - } - } - None -} - -#[cfg(target_os = "linux")] -async fn wait_for_process_with_args(expected_args: &[&str]) -> u32 { - tokio::time::timeout(Duration::from_secs(10), async { - loop { - if let Some(pid) = find_process_with_args(expected_args) { - return pid; - } - sleep(Duration::from_millis(50)).await; - } - }) - .await - .unwrap_or_else(|_| panic!("process with arguments {expected_args:?} did not start")) -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_stop_start_preserves_workspace() { - const SENTINEL: &str = "openshell-stop-start-sentinel"; - const SENTINEL_PATH: &str = "/sandbox/.openshell-stop-start-e2e"; - const RUN_COUNT_PATH: &str = "/sandbox/.openshell-main-run-count"; - let write_sentinel = format!("printf '%s\\n' '{SENTINEL}' > '{SENTINEL_PATH}'"); - let main = format!( - "count=0; test ! -f '{RUN_COUNT_PATH}' || count=$(cat '{RUN_COUNT_PATH}'); \ - count=$((count + 1)); printf '%s\\n' \"$count\" > '{RUN_COUNT_PATH}'; \ - echo lifecycle-ready-$count; exec sleep infinity" - ); - - let mut sandbox = SandboxGuard::create_keep(&["sh", "-c", &main], "lifecycle-ready") - .await - .expect("sandbox create should start a durable main process"); - sandbox - .exec(&["sh", "-lc", &write_sentinel]) - .await - .expect("sandbox exec should write the workspace sentinel"); - - let stop_output = run_sandbox_lifecycle_command("stop", &sandbox.name).await; - assert!( - stop_output.contains("Stopped sandbox"), - "expected stop confirmation in:\n{stop_output}", - ); - - let mut exec_cmd = openshell_cmd(); - exec_cmd - .args([ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--", - "cat", - SENTINEL_PATH, - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let stopped_exec = exec_cmd - .output() - .await - .expect("spawn openshell sandbox exec while stopped"); - assert!( - !stopped_exec.status.success(), - "sandbox exec should fail while stopped" - ); - - let start_output = run_sandbox_lifecycle_command("start", &sandbox.name).await; - assert!( - start_output.contains("Started sandbox"), - "expected start confirmation in:\n{start_output}", - ); - - let sentinel = sandbox - .exec(&["cat", SENTINEL_PATH]) - .await - .expect("sandbox exec should succeed after start"); - assert!( - sentinel.lines().any(|line| line.trim() == SENTINEL), - "workspace sentinel should survive stop and start:\n{sentinel}", - ); - let run_count = sandbox - .exec(&["cat", RUN_COUNT_PATH]) - .await - .expect("sandbox exec should read the canonical main run count"); - assert_eq!( - run_count.trim(), - "2", - "start should launch a fresh canonical main instance:\n{run_count}", - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_can_be_deleted_while_stopped() { - let mut sandbox = SandboxGuard::create_keep( - &["sh", "-c", "echo stop-ready; exec sleep infinity"], - "stop-ready", - ) - .await - .expect("sandbox create should start a durable main process"); - - let stop_output = run_sandbox_lifecycle_command("stop", &sandbox.name).await; - assert!( - stop_output.contains("Stopped sandbox"), - "expected stop confirmation in:\n{stop_output}", - ); - - let delete_output = run_sandbox_lifecycle_command("delete", &sandbox.name).await; - // Deletion may return before the owned cleanup worker finishes. Both - // outcomes must still reach absence, which is checked below. - assert!( - delete_output.contains(&format!("Deleted sandbox {}", sandbox.name)) - || delete_output.contains(&format!( - "Sandbox {} deletion accepted; cleanup is pending", - sandbox.name - )), - "expected completed or accepted deletion in:\n{delete_output}", - ); - - if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox.name, false).await { - sandbox.cleanup().await; - panic!( - "stopped sandbox {} should be deleted without starting after \ - {SANDBOX_PRESENCE_TIMEOUT:?}; last observed sandbox list: {last_sandbox_list:?}", - sandbox.name, - ); - } - - // Mark the guard cleaned up. Its idempotent delete is harmless now that - // the lifecycle operation above has removed the sandbox. - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_exit_zero_completes_persistent_sandbox() { - // Armed before create so a failed create or parse still cleans up. - let sandbox_name = unique_sandbox_name(); - let _cleanup = SandboxGuard::manage_existing(sandbox_name.clone()); - - let mut cmd = openshell_tty_cmd(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--", - "echo", - "OK", - ]); - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox create"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - assert!(output.status.success(), "create failed:\n{combined}"); - assert!( - combined.contains("OK"), - "main output was not streamed:\n{combined}" - ); - - if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox_name, true).await { - panic!( - "sandbox {sandbox_name} should still exist by default after {SANDBOX_PRESENCE_TIMEOUT:?}; \ - last observed sandbox list: {last_sandbox_list:?}" - ); - } - - let mut get_cmd = openshell_cmd(); - get_cmd - .args(["sandbox", "get", &sandbox_name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let get_output = get_cmd.output().await.expect("spawn openshell sandbox get"); - let details = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&get_output.stdout), - String::from_utf8_lossy(&get_output.stderr), - )); - assert!( - get_output.status.success(), - "sandbox get failed:\n{details}" - ); - assert!( - details.contains("Phase: Completed"), - "expected terminal sandbox phase:\n{details}" - ); -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_nonzero_exit_preserves_status() { - // Armed before create so a failed create or parse still cleans up. - let sandbox_name = unique_sandbox_name(); - let _cleanup = SandboxGuard::manage_existing(sandbox_name.clone()); - - let mut cmd = openshell_tty_cmd(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--", - "sh", - "-c", - "echo failed-main; exit 7", - ]); - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox create"); - let combined = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - assert_eq!( - output.status.code(), - Some(7), - "unexpected result:\n{combined}" - ); - assert!( - combined.contains("failed-main"), - "main output was not streamed:\n{combined}" - ); - - let mut get_cmd = openshell_cmd(); - get_cmd - .args(["sandbox", "get", &sandbox_name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let get_output = get_cmd.output().await.expect("spawn openshell sandbox get"); - let details = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&get_output.stdout), - String::from_utf8_lossy(&get_output.stderr), - )); - assert!( - details.contains("Phase: Error"), - "unexpected phase:\n{details}" - ); - assert!( - details.contains("Exit Code: 7"), - "missing exit code:\n{details}" - ); -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_canonical_main_exit_zero_reaches_completed() { - const RELEASE_PATH: &str = "/sandbox/.openshell-detached-success-release"; - let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 0"); - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script]) - .await - .expect("create detached successful canonical main"); - sandbox - .exec(&["touch", RELEASE_PATH]) - .await - .expect("release detached successful canonical main"); - - wait_for_sandbox_phase(&sandbox.name, "Completed", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("detached successful main did not complete:\n{err}")); - let details = sandbox_details(&sandbox.name).await; - assert!( - details.contains("Phase: Completed"), - "detached successful main should retain Completed status:\n{details}" - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_canonical_main_nonzero_exit_reaches_error() { - const RELEASE_PATH: &str = "/sandbox/.openshell-detached-failure-release"; - let script = format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 11"); - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script]) - .await - .expect("create detached failing canonical main"); - sandbox - .exec(&["touch", RELEASE_PATH]) - .await - .expect("release detached failing canonical main"); - - wait_for_sandbox_phase(&sandbox.name, "Error", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("detached failing main did not reach Error:\n{err}")); - let details = sandbox_details(&sandbox.name).await; - assert!( - details.contains("Phase: Error") && details.contains("Exit Code: 11"), - "detached failing main should retain its terminal result:\n{details}" - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_and_exec_receive_declared_environment() { - for mode in ["--tty", "--no-tty"] { - let script = r#"printf 'declared_env=%s\n' "${REPRO_SENTINEL:-missing}"; while true; do sleep 1; done"#; - let mut sandbox = SandboxGuard::create_keep_with_args( - &[ - mode, - "--no-auto-providers", - "--env", - "REPRO_SENTINEL=present", - ], - &["sh", "-c", script], - "declared_env=", - ) - .await - .expect("create canonical process with declared environment"); - let initial = normalize_output(&sandbox.create_output); - let later = sandbox - .exec(&[ - "sh", - "-c", - r#"printf 'declared_env=%s\n' "${REPRO_SENTINEL:-missing}""#, - ]) - .await; - sandbox.cleanup().await; - - assert!( - initial.lines().any(|line| line == "declared_env=present"), - "initial process must receive declared environment ({mode}): {initial}" - ); - let later = normalize_output(&later.expect("exec environment probe")); - assert!( - later.lines().any(|line| line == "declared_env=present"), - "exec must receive the same declared environment ({mode}): {later}" - ); - } -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn detached_main_exit_during_provisioning_is_classified_as_workload_result() { - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", "exit 11"]) - .await - .expect("fast detached main exit should not be reported as a provisioning failure"); - - wait_for_sandbox_phase(&sandbox.name, "Error", SANDBOX_PRESENCE_TIMEOUT) - .await - .unwrap_or_else(|err| panic!("fast detached main did not reach Error:\n{err}")); - - let mut get_cmd = openshell_cmd(); - get_cmd - .args(["sandbox", "get", &sandbox.name]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let get_output = get_cmd.output().await.expect("spawn openshell sandbox get"); - let details = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&get_output.stdout), - String::from_utf8_lossy(&get_output.stderr), - )); - assert!( - get_output.status.success(), - "sandbox get failed:\n{details}" - ); - assert!( - details.contains("Phase: Error") && details.contains("Exit Code: 11"), - "fast detached main should retain its workload result:\n{details}" - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_tty_main_uses_sandbox_environment() { - let script = r#"printf 'canonical_env home=%s user=%s term=%s\n' "$HOME" "$USER" "$TERM"; while true; do sleep 1; done"#; - let mut sandbox = - SandboxGuard::create_keep_with_args(&["--tty"], &["sh", "-lc", script], "canonical_env") - .await - .expect("create canonical TTY process"); - - let output = normalize_output(&sandbox.create_output); - let environment = output - .lines() - .find(|line| line.contains("canonical_env")) - .expect("canonical environment output"); - let field = |name: &str| { - environment - .split_whitespace() - .find_map(|value| value.strip_prefix(&format!("{name}="))) - .unwrap_or_default() - }; - - assert!( - !field("home").is_empty() && field("home") != "/root", - "canonical process must not inherit the supervisor HOME: {environment}" - ); - assert!( - !field("user").is_empty(), - "canonical process USER must identify the sandbox user: {environment}" - ); - assert!( - !field("term").is_empty() && field("term") != "dumb", - "canonical TTY process must receive a usable TERM: {environment}" - ); - - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_disconnect_reconnect_replays_history_for_same_process() { - const FIRST_MARKER: &str = "sequence=0001"; - let script = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'main_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'main_pid=%s input=%s\n' "$$" "$line"; done"#; - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-lc", script]) - .await - .expect("create retained canonical main process"); - - let mut owner_cmd = openshell_cmd(); - owner_cmd - .args(["sandbox", "connect", &sandbox.name]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut owner = owner_cmd.spawn().expect("spawn input-owning attachment"); - let owner_stdout = owner.stdout.take().expect("owner stdout"); - let mut owner_lines = BufReader::new(owner_stdout).lines(); - let owner_line = tokio::time::timeout(Duration::from_secs(30), owner_lines.next_line()) - .await - .expect("owner output timeout") - .expect("read owner output") - .expect("owner output should remain open"); - assert!( - owner_line.contains("main_pid="), - "unexpected owner output: {owner_line}" - ); - - let mut observer_cmd = openshell_cmd(); - observer_cmd - .args(["sandbox", "connect", &sandbox.name]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut observer = observer_cmd.spawn().expect("spawn competing attachment"); - let observer_stderr = observer.stderr.take().expect("observer stderr"); - let mut observer_errors = BufReader::new(observer_stderr).lines(); - let warning = tokio::time::timeout(Duration::from_secs(30), observer_errors.next_line()) - .await - .expect("observer warning timeout") - .expect("read observer warning") - .expect("observer warning should remain open"); - assert!( - warning.contains("already has an input owner") && warning.contains("read-only"), - "competing attachment should become read-only: {warning}" - ); - let observer_stdout = observer.stdout.take().expect("observer stdout"); - let mut observer_lines = BufReader::new(observer_stdout).lines(); - let observer_output_line = - tokio::time::timeout(Duration::from_secs(30), observer_lines.next_line()) - .await - .expect("observer output timeout") - .expect("read observer output") - .expect("observer output should remain open"); - assert!( - observer_output_line.contains("main_pid="), - "read-only attachment should observe output: {observer_output_line}" - ); - - observer - .stdin - .as_mut() - .expect("observer stdin") - .write_all(b"\x03") - .await - .expect("send Ctrl-C to viewer"); - let observer_status = tokio::time::timeout(Duration::from_secs(30), observer.wait()) - .await - .expect("Ctrl-C should exit viewer") - .expect("wait for viewer"); - assert!(observer_status.success(), "viewer should exit successfully"); - - owner - .stdin - .as_mut() - .expect("owner stdin") - .write_all(b"owner-after-viewer-exit\n") - .await - .expect("send input after viewer exits"); - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let line = owner_lines - .next_line() - .await - .expect("read owner output after viewer exits") - .expect("main process should remain running"); - if line.contains("input=owner-after-viewer-exit") { - break; - } - } - }) - .await - .expect("owner should retain working stdin after viewer exits"); - - owner.kill().await.expect("disconnect input owner"); - owner.wait().await.expect("wait for input owner disconnect"); - - let (mut reconnect, replay) = reconnect_with_input_ownership(&sandbox.name).await; - - let first_pid = owner_line - .split_whitespace() - .find_map(|field| field.strip_prefix("main_pid=")) - .expect("owner output pid"); - assert!( - replay.iter().any(|line| line.contains(FIRST_MARKER)), - "reconnect should replay the beginning of retained history: {replay:?}" - ); - assert!( - replay - .iter() - .all(|line| line.contains(&format!("main_pid={first_pid}"))), - "reconnect should target the same canonical process: {replay:?}" - ); - assert!( - replay.iter().any(|line| !line.contains(FIRST_MARKER)), - "reconnect should observe output beyond the first record: {replay:?}" - ); - - reconnect - .kill() - .await - .expect("disconnect reconnect attachment"); - reconnect - .wait() - .await - .expect("wait for reconnect disconnect"); - sandbox.cleanup().await; -} - -#[cfg(target_os = "linux")] -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_connect_recovers_its_ssh_transport() { - let script = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'transport_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'transport_pid=%s input=%s\n' "$$" "$line"; done"#; - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-lc", script]) - .await - .expect("create retained canonical main process"); - - let mut connect_cmd = openshell_cmd(); - connect_cmd - .args(["sandbox", "connect", &sandbox.name]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut connect = connect_cmd.spawn().expect("spawn supervised attachment"); - let mut connect_stdin = connect.stdin.take().expect("connect stdin"); - let connect_stdout = connect.stdout.take().expect("connect stdout"); - let mut connect_lines = BufReader::new(connect_stdout).lines(); - let connect_stderr = connect.stderr.take().expect("connect stderr"); - let mut connect_errors = BufReader::new(connect_stderr).lines(); - - let initial_line = tokio::time::timeout(Duration::from_secs(30), connect_lines.next_line()) - .await - .expect("initial attachment output timeout") - .expect("read initial attachment output") - .expect("initial attachment output should remain open"); - assert!( - initial_line.contains("transport_pid="), - "unexpected initial attachment output: {initial_line}" - ); - - // Recovery intentionally starts only for an established attachment, so - // let the initial SSH process live beyond that setup guard before killing - // only its ProxyCommand transport. - sleep(Duration::from_secs(3)).await; - let proxy_pid = wait_for_process_with_args(&["ssh-proxy", "--sandbox", &sandbox.name]).await; - let kill_status = tokio::process::Command::new("kill") - .args(["-TERM", &proxy_pid.to_string()]) - .status() - .await - .expect("terminate SSH proxy transport"); - assert!(kill_status.success(), "terminate SSH proxy transport"); - - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let line = connect_errors - .next_line() - .await - .expect("read supervised attachment diagnostics") - .expect("supervised attachment diagnostics should remain open"); - if normalize_output(&line).contains("Connection to sandbox lost; reconnecting") { - break; - } - } - }) - .await - .expect("supervised attachment did not enter recovery"); - - let input_token = format!("after-transport-recovery-{:x}", rand::random::()); - connect_stdin - .write_all(format!("{input_token}\n").as_bytes()) - .await - .expect("write input after transport recovery"); - connect_stdin - .flush() - .await - .expect("flush input after transport recovery"); - - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let line = connect_lines - .next_line() - .await - .expect("read recovered attachment output") - .expect("recovered attachment output should remain open"); - if line.contains(&format!("input={input_token}")) { - break; - } - } - }) - .await - .expect("replacement SSH session did not reattach to canonical main"); - assert!( - connect - .try_wait() - .expect("inspect supervised attachment") - .is_none(), - "sandbox connect parent should remain alive after recovery" - ); - - connect - .kill() - .await - .expect("disconnect recovered attachment"); - connect - .wait() - .await - .expect("wait for recovered attachment disconnect"); - sandbox.cleanup().await; -} - -#[cfg(target_os = "linux")] -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_connect_forwards_pid_targeted_termination_and_reaps_ssh() { - use std::os::fd::OwnedFd; - - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-lc", "exec sleep infinity"]) - .await - .expect("create retained canonical main process"); - let pty = nix::pty::openpty(None, None).expect("open pseudo-terminal"); - let controller: OwnedFd = pty.master; - let follower: OwnedFd = pty.slave; - - let mut connect_cmd = openshell_cmd(); - connect_cmd - .args(["sandbox", "connect", &sandbox.name]) - .stdin( - follower - .try_clone() - .expect("duplicate PTY follower for stdin"), - ) - .stdout( - follower - .try_clone() - .expect("duplicate PTY follower for stdout"), - ) - .stderr( - follower - .try_clone() - .expect("duplicate PTY follower for stderr"), - ); - let mut connect = connect_cmd - .spawn() - .expect("spawn supervised PTY attachment"); - let connect_pid = connect.id().expect("connect process ID"); - drop(follower); - - let ssh_pid = tokio::time::timeout(Duration::from_secs(30), async { - loop { - if let Some(pid) = - find_child_process_with_args(connect_pid, &["-s", "sandbox", "openshell-main"]) - { - return pid; - } - sleep(Duration::from_millis(50)).await; - } - }) - .await - .expect("supervised SSH child did not start"); - - nix::sys::signal::kill( - nix::unistd::Pid::from_raw(i32::try_from(connect_pid).expect("connect PID fits i32")), - nix::sys::signal::Signal::SIGTERM, - ) - .expect("send SIGTERM to only the OpenShell parent"); - - let status = tokio::time::timeout(Duration::from_secs(10), connect.wait()) - .await - .expect("OpenShell parent did not terminate") - .expect("wait for OpenShell parent"); - assert_eq!(status.code(), Some(143)); - tokio::time::timeout(Duration::from_secs(5), async { - while fs::metadata(format!("/proc/{ssh_pid}")).is_ok() { - sleep(Duration::from_millis(25)).await; - } - }) - .await - .expect("SSH child was not reaped after parent termination"); - - drop(controller); - sandbox.cleanup().await; -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn canonical_main_exit_255_is_not_retried_as_transport_failure() { - const READY_MARKER: &str = "exit-255-ready"; - const RELEASE_PATH: &str = "/sandbox/.openshell-exit-255-release"; - let script = format!( - "echo {READY_MARKER}; while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; exit 255" - ); - let mut sandbox = SandboxGuard::create_detached_main(&["sh", "-c", &script]) - .await - .expect("create retained canonical main process"); - - let mut connect_cmd = openshell_cmd(); - connect_cmd - .args(["sandbox", "connect", &sandbox.name]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut connect = connect_cmd.spawn().expect("spawn supervised attachment"); - let connect_stdout = connect.stdout.take().expect("connect stdout"); - let mut connect_lines = BufReader::new(connect_stdout).lines(); - - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let line = connect_lines - .next_line() - .await - .expect("read attachment output") - .expect("attachment output should remain open"); - if line.contains(READY_MARKER) { - break; - } - } - }) - .await - .expect("attachment did not observe the canonical main process"); - - // Keep the attachment alive beyond the setup guard used to distinguish - // initial SSH failures from established transport failures. - sleep(Duration::from_secs(3)).await; - sandbox - .exec(&["touch", RELEASE_PATH]) - .await - .expect("release canonical main process"); - - let status = tokio::time::timeout(Duration::from_secs(10), connect.wait()).await; - if status.is_err() { - connect.kill().await.expect("stop stuck attachment"); - } - sandbox.cleanup().await; - let status = status - .expect("exit status 255 must not enter the transport recovery loop") - .expect("wait for canonical main attachment"); - assert_eq!(status.code(), Some(255)); -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn on_failure_policy_replaces_runtime_and_preserves_workspace() { - const SCRIPT: &str = r#" -marker=/sandbox/.openshell-restart-e2e -if [ -e "$marker" ]; then - printf 'replacement-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/replacement-run - sleep 300 -else - touch "$marker" - printf 'initial-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/initial-run - while [ ! -e /sandbox/.openshell-restart-release ]; do sleep 0.05; done - exit 17 -fi -"#; - - let mut sandbox = SandboxGuard::manage_existing(unique_sandbox_name()); - let (output, exit_code) = run_cli(&[ - "sandbox", - "create", - "--name", - &sandbox.name, - "--from", - E2E_WORKLOAD_IMAGE, - "--detach", - "--restart-policy", - "on-failure", - "--", - "sh", - "-lc", - SCRIPT, - ]) - .await; - assert_eq!( - exit_code, 0, - "create sandbox with OnFailure policy: {output}" - ); - wait_for_sandbox_exec_contains( - &sandbox.name, - &["cat", "/sandbox/initial-run"], - "initial-", - SANDBOX_RESTART_TIMEOUT, - ) - .await - .expect("initial main process should write its workspace marker"); - - let initial_details = sandbox_details(&sandbox.name).await; - let initial_instance = extract_field(&initial_details, "Main process instance") - .expect("initial main process instance"); - sandbox - .exec(&["touch", "/sandbox/.openshell-restart-release"]) - .await - .expect("release initial main process"); - - let deadline = Instant::now() + SANDBOX_RESTART_TIMEOUT; - let mut last_runs = Err("replacement is not ready".to_string()); - let final_details = loop { - let details = sandbox_details(&sandbox.name).await; - let replacement_instance = extract_field(&details, "Main process instance") - .is_some_and(|instance| instance != "-" && instance != initial_instance); - if details.contains("Phase: Ready") - && extract_field(&details, "Restart count").as_deref() == Some("1") - && replacement_instance - { - last_runs = sandbox - .exec(&["cat", "/sandbox/initial-run", "/sandbox/replacement-run"]) - .await; - if let Ok(runs) = &last_runs - && runs.contains("initial-") - && runs.contains("replacement-") - { - break details; - } - } - assert!( - Instant::now() < deadline, - "sandbox did not complete its policy-driven restart within \ - {SANDBOX_RESTART_TIMEOUT:?}; last details:\n{details}\nworkspace: {last_runs:?}" - ); - sleep(Duration::from_millis(250)).await; - }; - - assert!( - final_details.contains("Restart policy: on-failure"), - "restart policy should remain visible after replacement:\n{final_details}" - ); - sandbox.cleanup().await; -} - -#[tokio::test] -async fn sandbox_create_with_no_keep_cleans_up_after_tty_command() { - let name = format!("tty-{:015x}", rand::random::() & 0x0fff_ffff_ffff_ffff); - // Capture startup diagnostics before --no-keep removes a failed container. - // This is best-effort: the lifecycle assertions also run on other drivers. - let log_name = name.clone(); - let diagnostics = tokio::spawn(async move { - tokio::time::timeout(Duration::from_secs(30), async { - loop { - let containers = tokio::process::Command::new("docker") - .args(["ps", "--all", "--quiet", "--filter"]) - .arg(format!("label=openshell.ai/sandbox-name={log_name}")) - .kill_on_drop(true) - .output() - .await - .ok()?; - if !containers.status.success() { - return None; - } - let ids = String::from_utf8_lossy(&containers.stdout); - if let Some(id) = ids.split_whitespace().next() { - let logs = tokio::process::Command::new("docker") - .args(["logs", "--follow", id]) - .kill_on_drop(true) - .output() - .await - .ok()?; - return Some(normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&logs.stdout), - String::from_utf8_lossy(&logs.stderr) - ))); - } - sleep(Duration::from_millis(100)).await; - } - }) - .await - .ok() - .flatten() - }); - let mut cmd = openshell_tty_cmd(&[ - "sandbox", - "create", - "--name", - &name, - "--no-keep", - "--", - "echo", - "OK", - ]); - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox create"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = normalize_output(&format!("{stdout}{stderr}")); - - let startup_logs = if output.status.success() { - diagnostics.abort(); - None - } else { - diagnostics.await.ok().flatten() - }; - assert!( - output.status.success(), - "create failed:\n{combined}\nsupervisor logs:\n{}", - startup_logs.as_deref().unwrap_or("unavailable") - ); - assert!( - combined.contains("OK"), - "main output was not streamed:\n{combined}" - ); - let sandbox_name = - extract_sandbox_name(&combined).expect("sandbox name should be present in output"); - - if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox_name, false).await { - delete_sandbox(&sandbox_name).await; - panic!( - "sandbox {sandbox_name} should have been deleted automatically after \ - {SANDBOX_PRESENCE_TIMEOUT:?}; last observed sandbox list: {last_sandbox_list:?}" - ); - } -} - -#[tokio::test] -#[serial(sandbox_lifecycle)] -async fn sandbox_create_with_no_keep_preserves_failure_then_cleans_up() { - let mut cmd = openshell_tty_cmd(&[ - "sandbox", - "create", - "--no-keep", - "--", - "sh", - "-c", - "echo ephemeral-failure; exit 17", - ]); - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell sandbox create"); - let combined = normalize_output(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - - assert_eq!( - output.status.code(), - Some(17), - "ephemeral main should preserve its failure code:\n{combined}" - ); - assert!( - combined.contains("ephemeral-failure"), - "ephemeral main output was not streamed:\n{combined}" - ); - let sandbox_name = - extract_sandbox_name(&combined).expect("sandbox name should be present in output"); - - if let Err(last_sandbox_list) = assert_sandbox_presence_eventually(&sandbox_name, false).await { - delete_sandbox(&sandbox_name).await; - panic!( - "failed ephemeral sandbox {sandbox_name} should have been deleted automatically after \ - {SANDBOX_PRESENCE_TIMEOUT:?}; last observed sandbox list: {last_sandbox_list:?}" - ); - } -} diff --git a/e2e/rust/tests/sandbox_templates.rs b/e2e/rust/tests/sandbox_templates.rs deleted file mode 100644 index 21c9233daf..0000000000 --- a/e2e/rust/tests/sandbox_templates.rs +++ /dev/null @@ -1,275 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for reusable sandbox workload templates. - -use std::process::Stdio; -use std::time::{SystemTime, UNIX_EPOCH}; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use serde_json::Value; - -struct CliResult { - output: String, - success: bool, -} - -struct TemplateGuard { - name: String, -} - -impl TemplateGuard { - fn new(name: String) -> Self { - Self { name } - } - - async fn cleanup(mut self) { - delete_template(&self.name).await; - self.name.clear(); - } -} - -impl Drop for TemplateGuard { - fn drop(&mut self) { - if self.name.is_empty() { - return; - } - let bin = openshell_bin(); - let _ = std::process::Command::new(&bin) - .args(["sandbox", "template", "delete", &self.name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - CliResult { - output: strip_ansi(&combined), - success: output.status.success(), - } -} - -async fn delete_template(name: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["sandbox", "template", "delete", name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -fn unique_name(prefix: &str) -> String { - let millis = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock before epoch") - .as_millis(); - let suffix = millis % 1_000_000; - format!("{prefix}-{suffix:06}") -} - -fn output_contains(output: &str, needle: &str) -> bool { - output.to_lowercase().contains(&needle.to_lowercase()) -} - -fn output_mentions_template_not_found(output: &str) -> bool { - output_contains(output, "sandbox template") && output_contains(output, "not found") -} - -#[tokio::test] -async fn sandbox_create_from_template_uses_reusable_workload() { - let template_name = unique_name("tmpl"); - let sandbox_name = unique_name("sb-tmpl"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&[ - "sandbox", - "template", - "create", - &template_name, - "--cpu", - "500m", - "--memory", - "512Mi", - "--label", - "e2e=sandbox-template", - "--env", - "FEATURE_FLAG=on", - ]) - .await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let get_template = run_cli(&[ - "sandbox", - "template", - "get", - &template_name, - "--output", - "json", - ]) - .await; - assert!( - get_template.success, - "sandbox template get failed:\n{}", - get_template.output - ); - let template_json: Value = - serde_json::from_str(&get_template.output).expect("template JSON output"); - assert_eq!(template_json["name"].as_str(), Some(template_name.as_str())); - assert_eq!( - template_json["labels"]["e2e"].as_str(), - Some("sandbox-template") - ); - assert_eq!( - template_json["environment"]["FEATURE_FLAG"].as_str(), - Some("on") - ); - assert_eq!(template_json["resources"]["cpu"].as_str(), Some("500m")); - assert_eq!(template_json["resources"]["memory"].as_str(), Some("512Mi")); - - let list_templates = run_cli(&["sandbox", "template", "list", "--names"]).await; - assert!( - list_templates.success, - "sandbox template list failed:\n{}", - list_templates.output - ); - assert!( - list_templates - .output - .lines() - .any(|line| line.trim() == template_name), - "template list should include {template_name}:\n{}", - list_templates.output - ); - - let mut sandbox = SandboxGuard::create(&[ - "--name", - &sandbox_name, - "--template", - &template_name, - "--", - "sh", - "-lc", - "test \"$FEATURE_FLAG\" = on && echo template-env-ok", - ]) - .await - .expect("sandbox create from template should succeed"); - - assert!( - sandbox.create_output.contains("template-env-ok"), - "sandbox should inherit template environment:\n{}", - sandbox.create_output - ); - - sandbox.cleanup().await; - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_template_get_after_delete_returns_not_found() { - let template_name = unique_name("tmpl-del"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let delete_template = run_cli(&["sandbox", "template", "delete", &template_name]).await; - assert!( - delete_template.success, - "sandbox template delete failed:\n{}", - delete_template.output - ); - - let get_deleted = run_cli(&["sandbox", "template", "get", &template_name]).await; - assert!( - !get_deleted.success, - "deleted template should not be fetchable:\n{}", - get_deleted.output - ); - assert!( - output_mentions_template_not_found(&get_deleted.output), - "deleted template should return not-found:\n{}", - get_deleted.output - ); - - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_template_create_rejects_duplicate_name() { - let template_name = unique_name("tmpl-dupe"); - let template = TemplateGuard::new(template_name.clone()); - - let create_template = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - create_template.success, - "sandbox template create failed:\n{}", - create_template.output - ); - - let duplicate = run_cli(&["sandbox", "template", "create", &template_name]).await; - assert!( - !duplicate.success, - "duplicate sandbox template create should fail:\n{}", - duplicate.output - ); - assert!( - output_contains(&duplicate.output, "already exists"), - "duplicate sandbox template create should report already-exists:\n{}", - duplicate.output - ); - - template.cleanup().await; -} - -#[tokio::test] -async fn sandbox_create_from_missing_template_returns_not_found() { - let template_name = unique_name("tmpl-missing"); - let sandbox_name = unique_name("sb-missing"); - - let create_sandbox = run_cli(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--template", - &template_name, - "--", - "true", - ]) - .await; - - if create_sandbox.success { - let _ = run_cli(&["sandbox", "delete", &sandbox_name]).await; - } - - assert!( - !create_sandbox.success, - "sandbox create from missing template should fail:\n{}", - create_sandbox.output - ); - assert!( - output_mentions_template_not_found(&create_sandbox.output), - "sandbox create from missing template should report not-found:\n{}", - create_sandbox.output - ); -} diff --git a/e2e/rust/tests/settings_management.rs b/e2e/rust/tests/settings_management.rs deleted file mode 100644 index a3089590f7..0000000000 --- a/e2e/rust/tests/settings_management.rs +++ /dev/null @@ -1,329 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E tests for sandbox/global settings CLI workflows. -//! -//! Covers: -//! - Sandbox settings start as `` -//! - Sandbox setting set + read -//! - Global override blocks sandbox writes for that key -//! - Global get + global delete -//! - Sandbox-level control resumes after global delete - -use std::process::Stdio; -use std::sync::Mutex; -use std::time::Duration; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tokio::time::{Instant, sleep}; - -const TEST_KEY: &str = "ocsf_json_enabled"; -static SETTINGS_E2E_LOCK: Mutex<()> = Mutex::new(()); - -struct CliResult { - clean_output: String, - success: bool, - exit_code: Option, -} - -/// Best-effort global setting cleanup that runs even on test panic. -struct GlobalSettingCleanup { - key: &'static str, -} - -impl GlobalSettingCleanup { - fn new(key: &'static str) -> Self { - Self { key } - } -} - -impl Drop for GlobalSettingCleanup { - fn drop(&mut self) { - let _ = std::process::Command::new(openshell_bin()) - .args(["settings", "delete", "--global", "--key", self.key, "--yes"]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - CliResult { - clean_output: strip_ansi(&combined), - success: output.status.success(), - exit_code: output.status.code(), - } -} - -fn assert_setting_line(output: &str, key: &str, expected: &str) { - let needle = format!("{key} = {expected}"); - assert!( - output.contains(&needle), - "expected setting line '{needle}' in output:\n{output}" - ); -} - -fn assert_setting_line_with_scope(output: &str, key: &str, expected: &str, scope: &str) { - let needle = format!("{key} = {expected} ({scope})"); - assert!( - output.contains(&needle), - "expected setting line '{needle}' in output:\n{output}" - ); -} - -/// Poll `settings get` until the expected value and scope appear for a key. -async fn wait_for_setting_value( - sandbox_name: &str, - key: &str, - expected_value: &str, - expected_scope: &str, - timeout_duration: Duration, -) { - let needle = format!("{key} = {expected_value} ({expected_scope})"); - let start = Instant::now(); - loop { - let result = run_cli(&["settings", "get", sandbox_name]).await; - if result.success && result.clean_output.contains(&needle) { - return; - } - if start.elapsed() > timeout_duration { - panic!( - "timed out after {:?} waiting for setting line '{needle}' in output:\n{}", - timeout_duration, result.clean_output - ); - } - sleep(Duration::from_secs(1)).await; - } -} - -#[tokio::test] -async fn settings_global_override_round_trip() { - let _settings_lock = SETTINGS_E2E_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let _global_cleanup = GlobalSettingCleanup::new(TEST_KEY); - - let cleanup_before = - run_cli(&["settings", "delete", "--global", "--key", TEST_KEY, "--yes"]).await; - assert!( - cleanup_before.success, - "initial global setting cleanup should succeed (exit {:?}):\n{}", - cleanup_before.exit_code, cleanup_before.clean_output - ); - - let mut guard = - SandboxGuard::create_keep(&["sh", "-c", "echo Ready && sleep infinity"], "Ready") - .await - .expect("create keep sandbox"); - - let initial = run_cli(&["settings", "get", &guard.name]).await; - assert!( - initial.success, - "settings get should succeed (exit {:?}):\n{}", - initial.exit_code, initial.clean_output - ); - assert_setting_line_with_scope(&initial.clean_output, TEST_KEY, "", "unset"); - - let set_sandbox = run_cli(&[ - "settings", - "set", - &guard.name, - "--key", - TEST_KEY, - "--value", - "true", - ]) - .await; - assert!( - set_sandbox.success, - "sandbox setting set should succeed (exit {:?}):\n{}", - set_sandbox.exit_code, set_sandbox.clean_output - ); - - // Wait for the gateway to reflect the setting change. The setting is stored - // server-side, so `settings get` reads it immediately. Poll to ensure the - // config_revision has been updated (visible in the output). - wait_for_setting_value( - &guard.name, - TEST_KEY, - "true", - "sandbox", - Duration::from_secs(10), - ) - .await; - - let after_sandbox_set = run_cli(&["settings", "get", &guard.name]).await; - assert!( - after_sandbox_set.success, - "settings get after sandbox set should succeed (exit {:?}):\n{}", - after_sandbox_set.exit_code, after_sandbox_set.clean_output - ); - assert_setting_line_with_scope(&after_sandbox_set.clean_output, TEST_KEY, "true", "sandbox"); - - // Sandbox-scoped delete should succeed when not globally managed. - let sandbox_delete = run_cli(&["settings", "delete", &guard.name, "--key", TEST_KEY]).await; - assert!( - sandbox_delete.success, - "sandbox setting delete should succeed (exit {:?}):\n{}", - sandbox_delete.exit_code, sandbox_delete.clean_output - ); - assert!( - sandbox_delete - .clean_output - .contains("Deleted sandbox setting"), - "expected sandbox delete confirmation:\n{}", - sandbox_delete.clean_output - ); - - // After delete, the key should be unset again. - let after_sandbox_delete = run_cli(&["settings", "get", &guard.name]).await; - assert!( - after_sandbox_delete.success, - "settings get after sandbox delete should succeed:\n{}", - after_sandbox_delete.clean_output - ); - assert_setting_line_with_scope( - &after_sandbox_delete.clean_output, - TEST_KEY, - "", - "unset", - ); - - // Re-set at sandbox scope so we can test global override next. - let re_set = run_cli(&[ - "settings", - "set", - &guard.name, - "--key", - TEST_KEY, - "--value", - "true", - ]) - .await; - assert!( - re_set.success, - "re-set should succeed:\n{}", - re_set.clean_output - ); - - let set_global = run_cli(&[ - "settings", "set", "--global", "--key", TEST_KEY, "--value", "false", "--yes", - ]) - .await; - assert!( - set_global.success, - "global setting set should succeed (exit {:?}):\n{}", - set_global.exit_code, set_global.clean_output - ); - assert!( - set_global - .clean_output - .contains(&format!("Set global setting {TEST_KEY}=false")), - "expected global set output:\n{}", - set_global.clean_output - ); - - let blocked_sandbox_set = run_cli(&[ - "settings", - "set", - &guard.name, - "--key", - TEST_KEY, - "--value", - "true", - ]) - .await; - assert!( - !blocked_sandbox_set.success, - "sandbox setting should fail while key is global-managed:\n{}", - blocked_sandbox_set.clean_output - ); - assert!( - blocked_sandbox_set.clean_output.contains("is managed"), - "expected 'managed globally' error:\n{}", - blocked_sandbox_set.clean_output - ); - - // Sandbox-scoped delete should also be blocked while globally managed. - let blocked_sandbox_delete = - run_cli(&["settings", "delete", &guard.name, "--key", TEST_KEY]).await; - assert!( - !blocked_sandbox_delete.success, - "sandbox delete should fail while key is global-managed:\n{}", - blocked_sandbox_delete.clean_output - ); - - let global_get = run_cli(&["settings", "get", "--global"]).await; - assert!( - global_get.success, - "global settings get should succeed (exit {:?}):\n{}", - global_get.exit_code, global_get.clean_output - ); - assert_setting_line(&global_get.clean_output, TEST_KEY, "false"); - - let delete_global = - run_cli(&["settings", "delete", "--global", "--key", TEST_KEY, "--yes"]).await; - assert!( - delete_global.success, - "global settings delete should succeed (exit {:?}):\n{}", - delete_global.exit_code, delete_global.clean_output - ); - assert!( - delete_global - .clean_output - .contains(&format!("Deleted global setting {TEST_KEY}")), - "expected global delete confirmation in output:\n{}", - delete_global.clean_output - ); - - let global_after_delete = run_cli(&["settings", "get", "--global"]).await; - assert!( - global_after_delete.success, - "global settings get after delete should succeed (exit {:?}):\n{}", - global_after_delete.exit_code, global_after_delete.clean_output - ); - assert_setting_line(&global_after_delete.clean_output, TEST_KEY, ""); - - let sandbox_set_after_delete = run_cli(&[ - "settings", - "set", - &guard.name, - "--key", - TEST_KEY, - "--value", - "false", - ]) - .await; - assert!( - sandbox_set_after_delete.success, - "sandbox setting set after global delete should succeed (exit {:?}):\n{}", - sandbox_set_after_delete.exit_code, sandbox_set_after_delete.clean_output - ); - - let sandbox_after_delete = run_cli(&["settings", "get", &guard.name]).await; - assert!( - sandbox_after_delete.success, - "settings get after global delete should succeed (exit {:?}):\n{}", - sandbox_after_delete.exit_code, sandbox_after_delete.clean_output - ); - assert_setting_line_with_scope( - &sandbox_after_delete.clean_output, - TEST_KEY, - "false", - "sandbox", - ); - - guard.cleanup().await; -} diff --git a/e2e/rust/tests/ssh_host_identity.rs b/e2e/rust/tests/ssh_host_identity.rs deleted file mode 100644 index c348c2b3f4..0000000000 --- a/e2e/rust/tests/ssh_host_identity.rs +++ /dev/null @@ -1,156 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -use std::path::Path; -use std::time::Duration; - -use openshell_e2e::harness::cli::{ - run_cli, sandbox_names, wait_for_healthy, wait_for_sandbox_exec_contains, - wait_for_sandbox_phase, -}; -use openshell_e2e::harness::gateway::ManagedGateway; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; - -async fn fingerprint(name: &str) -> String { - let (output, code) = run_cli(&["sandbox", "get", name, "--output", "json"]).await; - assert_eq!(code, 0, "sandbox get failed: {output}"); - let sandbox: serde_json::Value = serde_json::from_str(&strip_ansi(&output)).unwrap(); - let fingerprint = sandbox["host_key_fingerprint"] - .as_str() - .unwrap() - .to_string(); - assert!(fingerprint.starts_with("SHA256:")); - let (output, code) = run_cli(&["sandbox", "list", "--output", "json"]).await; - assert_eq!(code, 0, "sandbox list failed: {output}"); - let list: serde_json::Value = serde_json::from_str(&strip_ansi(&output)).unwrap(); - let listed = list["sandboxes"] - .as_array() - .unwrap() - .iter() - .find(|sandbox| sandbox["name"] == name) - .unwrap(); - assert_eq!(listed["host_key_fingerprint"], fingerprint); - fingerprint -} - -async fn pinned_ssh(config: &Path, known_hosts: &Path, name: &str, first: bool) { - let output = tokio::time::timeout( - Duration::from_secs(30), - tokio::process::Command::new("ssh") - .kill_on_drop(true) - .arg("-F") - .arg(config) - .arg("-o") - .arg(if first { - "StrictHostKeyChecking=accept-new" - } else { - "StrictHostKeyChecking=yes" - }) - .arg("-o") - .arg(format!("UserKnownHostsFile={}", known_hosts.display())) - .arg("-o") - .arg("BatchMode=yes") - .arg(format!("openshell-{name}.default")) - .args(["printf", "ssh-identity-ok"]) - .output(), - ) - .await - .unwrap() - .unwrap(); - assert!( - output.status.success(), - "pinned SSH failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - assert_eq!(String::from_utf8_lossy(&output.stdout), "ssh-identity-ok"); -} - -#[tokio::test] -async fn ssh_host_identity_survives_restarts_and_changes_after_recreation() { - let mut sandbox = SandboxGuard::create(&[]).await.unwrap(); - let name = sandbox.name.clone(); - let original = fingerprint(&name).await; - let directory = tempfile::tempdir().unwrap(); - let config = directory.path().join("ssh_config"); - let known_hosts = directory.path().join("known_hosts"); - let (output, code) = run_cli(&["sandbox", "ssh-config", &name]).await; - assert_eq!(code, 0, "SSH config failed: {output}"); - std::fs::write(&config, output).unwrap(); - pinned_ssh(&config, &known_hosts, &name, true).await; - let key = tokio::process::Command::new("ssh-keygen") - .args(["-l", "-E", "sha256", "-f"]) - .arg(&known_hosts) - .output() - .await - .unwrap(); - assert!(key.status.success()); - assert!( - String::from_utf8_lossy(&key.stdout) - .split_whitespace() - .any(|word| word == original) - ); - sandbox - .exec(&["sh", "-c", "test ! -r /.openshell/supervisor/auth.json"]) - .await - .unwrap(); - - let (output, code) = run_cli(&["sandbox", "stop", &name]).await; - assert_eq!(code, 0, "stop failed: {output}"); - wait_for_sandbox_phase(&name, "Stopped", Duration::from_secs(60)) - .await - .unwrap(); - assert_eq!(fingerprint(&name).await, original); - let (output, code) = run_cli(&["sandbox", "start", &name]).await; - assert_eq!(code, 0, "start failed: {output}"); - wait_for_sandbox_phase(&name, "Ready", Duration::from_secs(120)) - .await - .unwrap(); - assert_eq!(fingerprint(&name).await, original); - pinned_ssh(&config, &known_hosts, &name, false).await; - - if let Some(gateway) = ManagedGateway::from_env().unwrap() { - gateway.stop().unwrap(); - gateway.start().unwrap(); - wait_for_healthy(Duration::from_secs(120)).await.unwrap(); - wait_for_sandbox_phase(&name, "Ready", Duration::from_secs(120)) - .await - .unwrap(); - wait_for_sandbox_exec_contains( - &name, - &["printf", "ssh-identity-ready"], - "ssh-identity-ready", - Duration::from_secs(240), - ) - .await - .unwrap(); - assert_eq!(fingerprint(&name).await, original); - pinned_ssh(&config, &known_hosts, &name, false).await; - } else { - eprintln!("Skipping gateway restart check: this run uses an existing gateway"); - } - - let (output, code) = run_cli(&["sandbox", "delete", &name]).await; - assert_eq!(code, 0, "delete failed: {output}"); - // Delete may return before the driver finishes removing the sandbox. - // Its name remains reserved until the gateway confirms removal. - tokio::time::timeout(Duration::from_secs(120), async { - loop { - let names = sandbox_names() - .await - .expect("list sandboxes after deletion"); - if !names.iter().any(|listed| listed == &name) { - break; - } - tokio::time::sleep(Duration::from_millis(500)).await; - } - }) - .await - .expect("original sandbox was not fully deleted before name reuse"); - sandbox.cleanup().await; - let mut replacement = SandboxGuard::create(&["--name", &name]).await.unwrap(); - assert_ne!(fingerprint(&name).await, original); - replacement.cleanup().await; -} diff --git a/e2e/rust/tests/upload_backpressure.rs b/e2e/rust/tests/upload_backpressure.rs deleted file mode 100644 index e766303c37..0000000000 --- a/e2e/rust/tests/upload_backpressure.rs +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(all(feature = "e2e", unix))] - -//! Regression for large built-in uploads through the CLI, gateway, supervisor, -//! and sandbox. A fast sender previously overflowed the supervisor's 4 MiB -//! pending-stdin budget while tar was creating many files (issue #4430). - -use std::fmt::Write as _; -use std::fs; -use std::os::unix::fs::{PermissionsExt, symlink}; -use std::time::Duration; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use rand::{RngCore, SeedableRng, rngs::StdRng}; -use sha2::{Digest, Sha256}; -use tokio::time::timeout; - -const FILE_COUNT: usize = 12_000; -const FILE_BYTES: usize = 8_192; -const TRANSFER_TIMEOUT: Duration = Duration::from_secs(300); - -/// An archive substantially larger than the stdin budget must finish without -/// pacing at the caller, preserving every file's content and tar semantics. -#[tokio::test] -async fn large_upload_preserves_content_and_metadata_outside_workdir() { - let tmpdir = tempfile::tempdir().expect("create upload fixture tempdir"); - let source = tmpdir.path().join("large-project"); - fs::create_dir_all(source.join("payload")).expect("create payload directory"); - - // Distinct deterministic pseudorandom bytes keep the fixture reproducible - // without letting compression or identical-file checks hide dropped data. - let mut rng = StdRng::seed_from_u64(4430); - let mut payload = [0_u8; FILE_BYTES]; - let mut checksums = String::new(); - for index in 0..FILE_COUNT { - rng.fill_bytes(&mut payload); - let relative = format!("payload/{index:05}.bin"); - fs::write(source.join(&relative), payload).expect("write payload file"); - writeln!(checksums, "{:x} {relative}", Sha256::digest(payload)) - .expect("append file checksum"); - } - let manifest_digest = format!("{:x}", Sha256::digest(checksums.as_bytes())); - fs::write(source.join("SHA256SUMS"), checksums).expect("write checksum manifest"); - fs::write( - source.join("run.sh"), - "#!/bin/sh\nprintf '%s\\n' upload-executable-ok\n", - ) - .expect("write executable fixture"); - fs::set_permissions(source.join("run.sh"), fs::Permissions::from_mode(0o751)) - .expect("set executable mode"); - symlink("payload/00000.bin", source.join("payload-link")).expect("create relative symlink"); - symlink("missing-target", source.join("dangling-link")).expect("create dangling symlink"); - fs::write(source.join("conflict.txt"), "local-conflict").expect("write merge fixture"); - - let mut sandbox = SandboxGuard::create(&[]) - .await - .expect("create persistent upload sandbox"); - sandbox - .exec(&[ - "sh", - "-c", - "set -eu; test \"$(pwd -P)\" != /tmp/openshell-upload-backpressure; \ - mkdir -p /tmp/openshell-upload-backpressure/large-project; \ - printf remote-conflict > /tmp/openshell-upload-backpressure/large-project/conflict.txt; \ - printf remote-preserved > /tmp/openshell-upload-backpressure/large-project/unrelated.txt", - ]) - .await - .expect("prepare destination outside the canonical workdir"); - - // No sleep, bandwidth limit, or retry: the protocol must backpressure the - // normal CLI tar stream, even when the local sender outruns extraction. - let mut upload = openshell_cmd(); - upload - .env("RUST_LOG", "warn") - .args(["sandbox", "upload", &sandbox.name]) - .arg(&source) - .args(["/tmp/openshell-upload-backpressure", "--no-git-ignore"]); - let result = timeout(TRANSFER_TIMEOUT, upload.output()) - .await - .expect("large upload must not hang waiting for credit") - .expect("start large upload"); - let output = format!( - "{}{}", - String::from_utf8_lossy(&result.stdout), - String::from_utf8_lossy(&result.stderr) - ); - assert!(result.status.success(), "large upload failed: {output}"); - assert!( - !output.contains("sandbox sync operation failed; retrying"), - "large upload required a retry: {output}" - ); - - let verify = format!( - "set -eu; cd /tmp/openshell-upload-backpressure/large-project; \ - test \"$(find payload -type f | wc -l)\" -eq {FILE_COUNT}; \ - test \"$(sha256sum SHA256SUMS | cut -d ' ' -f 1)\" = {manifest_digest}; \ - sha256sum --check --quiet SHA256SUMS; \ - test \"$(stat -c %a run.sh)\" = 751; \ - test \"$(./run.sh)\" = upload-executable-ok; \ - test -L payload-link; test \"$(readlink payload-link)\" = payload/00000.bin; \ - test -L dangling-link; test ! -e dangling-link; \ - test \"$(readlink dangling-link)\" = missing-target; \ - test \"$(cat conflict.txt)\" = local-conflict; \ - test \"$(cat unrelated.txt)\" = remote-preserved; \ - printf large-upload-content-and-metadata-ok" - ); - let output = timeout(TRANSFER_TIMEOUT, sandbox.exec(&["sh", "-c", &verify])) - .await - .expect("large upload verification must not hang") - .expect("every uploaded file and metadata item must match"); - assert!( - strip_ansi(&output).contains("large-upload-content-and-metadata-ok"), - "expected complete upload verification marker:\n{output}" - ); - - sandbox.cleanup().await; -} diff --git a/e2e/rust/tests/vm_corporate_proxy.rs b/e2e/rust/tests/vm_corporate_proxy.rs index d914cb1c03..32d9934125 100644 --- a/e2e/rust/tests/vm_corporate_proxy.rs +++ b/e2e/rust/tests/vm_corporate_proxy.rs @@ -21,8 +21,6 @@ //! 407 to an unauthenticated CONNECT, so a 200 proves delivery. //! 4. A port-qualified `no_proxy` entry bypasses the proxy for that port only. //! 5. An `https://` proxy works when its CA is supplied via `proxy_ca_bundle`. -//! 6. An incoherent setting is fatal at gateway startup rather than -//! degrading to a direct dial. //! //! Fixtures run as host processes and are reached by the host supervisor. //! `host.openshell.internal` is normalized to host loopback for both libkrun @@ -850,39 +848,3 @@ async fn vm_corporate_proxy_trusts_ca_bundle_for_https_proxy() { .await .expect("restore gateway config"); } - -#[tokio::test] -#[serial(vm_corporate_proxy)] -async fn vm_corporate_proxy_rejects_incoherent_configuration() { - if !should_run("corporate proxy fail-closed test") { - return; - } - - // A bypass list without a proxy URL means the operator believed proxying - // was in effect. Accepting it would leave every dial direct while looking - // configured, so the gateway must refuse to start instead. - let mut gateway_config = GatewayProxyConfig::apply_raw("no_proxy = \"10.0.0.0/8\"\n") - .expect("apply incoherent corporate proxy gateway config"); - - let health = wait_for_healthy(Duration::from_secs(20)).await; - assert!( - health.is_err(), - "gateway must not serve traffic with an incoherent [openshell.drivers.vm] proxy table" - ); - - let log_path = std::env::var("OPENSHELL_E2E_GATEWAY_LOG").expect("gateway log path"); - let log = std::fs::read_to_string(&log_path).expect("read gateway log"); - // The message must name the offending key rather than surfacing as an - // opaque driver-readiness timeout. Matched loosely on the key names so - // this does not break on error-wrapping changes. - assert!( - log.contains("no_proxy") && log.contains("https_proxy"), - "the startup error must name the offending key; gateway log tail:\n{}", - log.lines().rev().take(40).collect::>().join("\n") - ); - - gateway_config - .restore() - .await - .expect("restore gateway config"); -} diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index 56ce5a50f1..eef41ecb29 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -23,6 +23,12 @@ e2e_require_openshell_bin() { export OPENSHELL_BIN="${binary_dir}/$(basename "${OPENSHELL_BIN}")" } +# A pullable image for testkit::python_image() on lanes that can't load the +# locally built openshell/e2e-python:dev into their compute platform. It +# matches the Rust harness's E2E_WORKLOAD_IMAGE for those lanes. +# shellcheck disable=SC2034 # Used by the scripts that source this file. +E2E_PULLABLE_PYTHON_IMAGE="ghcr.io/astral-sh/uv:0.12.17-python3.12-trixie-slim@sha256:9a59bb7206905ccaae4f7dab222fbac47c125a21e5fc16f43f427cd6c940ade3" + # Print the path of the openshell-conformance program: OPENSHELL_CONFORMANCE_BIN # if set, else a debug build from this checkout. e2e_openshell_conformance_bin() { diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index e97d7f961e..5b6c4c97c5 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -713,8 +713,6 @@ GATEWAY_ARGS=( --compute-driver docker --tls-cert "${PKI_DIR}/server/tls.crt" --tls-key "${PKI_DIR}/server/tls.key" - # edge_tunnel_e2e exercises the opt-in WebSocket tunnel. - --enable-websocket-tunnel true --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" ) diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 31c0771dc8..b85e72b579 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -1331,6 +1331,7 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then exit 2 fi export HELM_PLUGINS="${ROOT}/e2e/helm-plugins" + export OPENSHELL_E2E_EXPECTED_HOST_GATEWAY_IP="${HOST_GATEWAY_IP}" helm_post_renderer_args+=( --post-renderer openshell-external-compute-driver ) diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 224eecaaf3..0310db6bcd 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -636,8 +636,6 @@ GATEWAY_ARGS=( --health-port "${HEALTH_PORT}" --tls-cert "${PKI_DIR}/server/tls.crt" --tls-key "${PKI_DIR}/server/tls.key" - # edge_tunnel_e2e exercises the opt-in WebSocket tunnel. - --enable-websocket-tunnel true --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" --log-level info ) diff --git a/tasks/rust.toml b/tasks/rust.toml index c995e74f8e..58d29303df 100644 --- a/tasks/rust.toml +++ b/tasks/rust.toml @@ -7,6 +7,10 @@ description = "Compare native and sandbox-filtered TCP socket performance" run = "cargo run --release -p openshell-sandbox --features perf-harness --bin openshell-seccomp-perf --" +["perf:live-internet"] +description = "Compare live-Internet traffic on the host and in a sandbox on the active gateway" +run = "uv run --no-project python tasks/scripts/live_internet_perf.py" + ["rust:check"] description = "Check all Rust crates for errors" depends = ["rust:lockfiles:check"] diff --git a/tasks/scripts/live_internet_perf.py b/tasks/scripts/live_internet_perf.py new file mode 100644 index 0000000000..9ccad8e2fd --- /dev/null +++ b/tasks/scripts/live_internet_perf.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compare live-Internet traffic on the host with the same traffic in a sandbox. + +Creates one sandbox on the active gateway with a policy that allows the +benchmark's destinations, then alternates direct and sandbox rounds of +`live_internet_perf_round.py` so drift in the network affects both sides +alike. Each round prints one `LIVE_INTERNET_PERF` line with the round's JSON. +The sandbox is deleted on exit. + +This is a manual benchmark: it needs the live Internet, and CI doesn't run it. + + mise run perf:live-internet -- --rounds 4 --from +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import time +import uuid +from pathlib import Path + +ROUND = Path(__file__).with_name("live_internet_perf_round.py") + +POLICY = """version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] + +landlock: + compatibility: best_effort + +network_policies: + live_internet_performance: + name: live_internet_performance + endpoints: + - { host: example.com, port: 443, protocol: tcp } + - { host: github.com, port: 443, protocol: tcp } + - { host: raw.githubusercontent.com, port: 443, protocol: tcp } + - { host: pypi.org, port: 443, protocol: tcp } + - { host: files.pythonhosted.org, port: 443, protocol: tcp } + - { host: registry.npmjs.org, port: 443, protocol: tcp } + - { host: crates.io, port: 443, protocol: tcp } + - { host: docs.python.org, port: 443, protocol: tcp } + - { host: speed.cloudflare.com, port: 443, protocol: tcp } + binaries: + - path: "/**" +""" + + +def run(command: list[str], **kwargs) -> str: + result = subprocess.run( + command, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + **kwargs, + ) + if result.returncode != 0: + raise SystemExit( + f"{' '.join(command[:4])} ... failed (exit {result.returncode}):\n{result.stdout}" + ) + return result.stdout + + +def direct_round() -> str: + env = dict(os.environ, OPENSHELL_LIVE_PERF_MODE="direct") + return run([sys.executable, str(ROUND)], env=env) + + +def sandbox_round(openshell: str, name: str, source: str) -> str: + return run( + [ + openshell, + "sandbox", + "exec", + "--name", + name, + "--no-tty", + "--", + "sh", + "-c", + 'OPENSHELL_LIVE_PERF_MODE=sandbox python3 -c "$1"', + "openshell-live-internet-perf", + source, + ] + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--rounds", type=int, default=10, help="rounds per side (default 10)" + ) + parser.add_argument( + "--from", dest="image", help="sandbox image (default: the gateway's)" + ) + parser.add_argument("--openshell", default="openshell", help="openshell CLI to use") + args = parser.parse_args() + if args.rounds < 1: + parser.error("--rounds must be at least 1") + + source = ROUND.read_text() + name = f"live-perf-{uuid.uuid4().hex[:8]}" + with tempfile.NamedTemporaryFile("w", suffix=".yaml") as policy: + policy.write(POLICY) + policy.flush() + create = [ + args.openshell, + "sandbox", + "create", + "--name", + name, + "--policy", + policy.name, + ] + if args.image: + create += ["--from", args.image] + started = time.perf_counter() + try: + run([*create, "--detach"]) + elapsed_ms = (time.perf_counter() - started) * 1000 + print( + f"LIVE_INTERNET_PERF create {json.dumps({'elapsed_ms': elapsed_ms})}", + flush=True, + ) + for number in range(1, args.rounds + 1): + # Alternate which side goes first so neither always runs on a + # warmer network. + sides = [ + ("direct", direct_round), + ("sandbox", lambda: sandbox_round(args.openshell, name, source)), + ] + if number % 2 == 0: + sides.reverse() + for mode, round_fn in sides: + print( + f"LIVE_INTERNET_PERF {mode} round={number} {round_fn().strip()}", + flush=True, + ) + finally: + subprocess.run( + [args.openshell, "sandbox", "delete", name], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + +if __name__ == "__main__": + main() diff --git a/tasks/scripts/live_internet_perf_round.py b/tasks/scripts/live_internet_perf_round.py new file mode 100644 index 0000000000..0684269486 --- /dev/null +++ b/tasks/scripts/live_internet_perf_round.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""One round of the live-Internet benchmark, shaped like a short coding-agent +session. Prints one JSON document. `live_internet_perf.py` runs it on the host +and inside a sandbox; it needs only the Python standard library and, for the +clone step, git. +""" + +import concurrent.futures +import http.client +import json +import os +import shutil +import socket +import ssl +import statistics +import subprocess +import tempfile +import time +import urllib.parse +from pathlib import Path + +MODE = os.environ.get("OPENSHELL_LIVE_PERF_MODE", "direct") +TLS = ssl.create_default_context() +USER_AGENT = "OpenShell-live-Internet-benchmark/1" + +METADATA_URLS = [ + "https://raw.githubusercontent.com/NVIDIA/OpenShell/main/README.md", + "https://pypi.org/pypi/requests/json", + "https://registry.npmjs.org/typescript/latest", + "https://docs.python.org/3/", +] +DNS_HOSTS = [ + "github.com", + "raw.githubusercontent.com", + "pypi.org", + "registry.npmjs.org", + "crates.io", + "docs.python.org", + "speed.cloudflare.com", +] + + +def percentile(values, fraction): + values = sorted(values) + return values[min(len(values) - 1, int((len(values) - 1) * fraction))] + + +def request(url, method="GET", max_bytes=None): + parsed = urllib.parse.urlsplit(url) + connection = http.client.HTTPSConnection( + parsed.hostname, parsed.port or 443, timeout=20, context=TLS + ) + path = parsed.path or "/" + if parsed.query: + path += "?" + parsed.query + started = time.perf_counter_ns() + connection.request( + method, path, headers={"User-Agent": USER_AGENT, "Connection": "close"} + ) + response = connection.getresponse() + first = response.read(1) + first_byte_ms = (time.perf_counter_ns() - started) / 1_000_000 + body_bytes = len(first) + while max_bytes is None or body_bytes < max_bytes: + remaining = None if max_bytes is None else max_bytes - body_bytes + chunk = response.read(65536 if remaining is None else min(65536, remaining)) + if not chunk: + break + body_bytes += len(chunk) + total_ms = (time.perf_counter_ns() - started) / 1_000_000 + status = response.status + connection.close() + if status < 200 or status >= 400: + raise RuntimeError(f"{url} returned HTTP {status}") + return { + "status": status, + "bytes": body_bytes, + "first_byte_ms": first_byte_ms, + "total_ms": total_ms, + } + + +def metric(name, operation): + started = time.perf_counter_ns() + try: + detail = operation() + return { + "name": name, + "ok": True, + "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, + "detail": detail, + } + except Exception as error: + return { + "name": name, + "ok": False, + "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, + "error": f"{type(error).__name__}: {error}", + } + + +def dns_set(): + samples = [] + for host in DNS_HOSTS: + started = time.perf_counter_ns() + addresses = socket.getaddrinfo(host, 443, socket.AF_UNSPEC, socket.SOCK_STREAM) + samples.append( + { + "host": host, + "elapsed_ms": (time.perf_counter_ns() - started) / 1_000_000, + "addresses": len(addresses), + } + ) + timings = [sample["elapsed_ms"] for sample in samples] + return { + "lookups": samples, + "p50_ms": percentile(timings, 0.50), + "p95_ms": percentile(timings, 0.95), + } + + +def metadata_serial(): + return {"requests": [request(url, max_bytes=2_000_000) for url in METADATA_URLS]} + + +def metadata_concurrent(): + urls = METADATA_URLS + METADATA_URLS + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + responses = list(pool.map(lambda url: request(url, max_bytes=2_000_000), urls)) + return {"requests": responses} + + +def https_reuse(): + connection = http.client.HTTPSConnection( + "example.com", 443, timeout=20, context=TLS + ) + samples = [] + try: + for _ in range(20): + started = time.perf_counter_ns() + connection.request( + "HEAD", + "/", + headers={"User-Agent": USER_AGENT, "Connection": "keep-alive"}, + ) + response = connection.getresponse() + response.read() + if response.status != 200: + raise RuntimeError(f"example.com returned HTTP {response.status}") + samples.append((time.perf_counter_ns() - started) / 1_000_000) + finally: + connection.close() + return { + "requests": len(samples), + "p50_ms": percentile(samples, 0.50), + "p95_ms": percentile(samples, 0.95), + "mean_ms": statistics.fmean(samples), + } + + +def git_clone(): + if shutil.which("git") is None: + return {"skipped": "git is not installed in the workload image"} + with tempfile.TemporaryDirectory(prefix="openshell-live-git-") as directory: + checkout = Path(directory) / "sampleproject" + command = [ + "git", + "-c", + "advice.detachedHead=false", + "clone", + "--quiet", + "--depth", + "1", + "--filter=blob:none", + "https://github.com/pypa/sampleproject.git", + str(checkout), + ] + output = subprocess.run( + command, + capture_output=True, + timeout=60, + check=False, + ) + if output.returncode != 0: + raise RuntimeError(output.stderr.decode(errors="replace")[-500:]) + paths = [path for path in checkout.rglob("*") if path.is_file()] + files = len(paths) + size = sum(path.stat().st_size for path in paths) + return {"files": files, "checkout_bytes": size} + + +def package_download(): + metadata = request("https://pypi.org/pypi/idna/json", max_bytes=2_000_000) + connection = http.client.HTTPSConnection("pypi.org", 443, timeout=20, context=TLS) + connection.request("GET", "/pypi/idna/json", headers={"User-Agent": USER_AGENT}) + response = connection.getresponse() + document = json.loads(response.read()) + connection.close() + wheels = [ + entry for entry in document["urls"] if entry["packagetype"] == "bdist_wheel" + ] + if not wheels: + raise RuntimeError("PyPI returned no idna wheel") + artifact = request(wheels[0]["url"], max_bytes=5_000_000) + return { + "metadata": metadata, + "artifact": artifact, + "filename": wheels[0]["filename"], + } + + +def bulk_download(): + return request( + "https://speed.cloudflare.com/__down?bytes=5242880", max_bytes=5242880 + ) + + +def denied_destination(): + if MODE != "sandbox": + return {"skipped": "policy denial applies only to sandbox mode"} + started = time.perf_counter_ns() + try: + socket.create_connection(("example.net", 443), timeout=5).close() + except OSError as error: + return { + "denied": True, + "latency_ms": (time.perf_counter_ns() - started) / 1_000_000, + "error": str(error), + } + raise RuntimeError("destination omitted from policy was reachable") + + +started = time.perf_counter_ns() +metrics = [ + metric("dns_service_set", dns_set), + metric("https_metadata_serial", metadata_serial), + metric("https_metadata_concurrent", metadata_concurrent), + metric("https_reuse", https_reuse), + metric("git_clone", git_clone), + metric("package_download", package_download), + metric("bulk_download_5mib", bulk_download), + metric("policy_denial", denied_destination), +] +document = { + "schema": "openshell.live-internet-perf.v1", + "mode": MODE, + "total_ms": (time.perf_counter_ns() - started) / 1_000_000, + "metrics": metrics, +} +print(json.dumps(document, separators=(",", ":"))) +if any(item["name"] == "policy_denial" and not item["ok"] for item in metrics): + raise SystemExit(1) diff --git a/tasks/test.toml b/tasks/test.toml index 2839c1cf0c..30c31ec753 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -177,6 +177,7 @@ run = "OPENSHELL_TEST_CONTAINER_ENGINE=podman e2e/with-podman-gateway.sh cargo t ["e2e:testkit"] description = "Run the testkit suites in tests/suites/conformance against a Docker-backed gateway" +depends = ["e2e:workload:build"] run = "e2e/with-docker-gateway.sh cargo nextest run --locked --manifest-path tests/suites/conformance/Cargo.toml --workspace --test-threads 1 --no-fail-fast" ["e2e:rust:push"] diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index ea04cd150d..0bc14ac2a8 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -493,6 +493,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -506,6 +516,8 @@ dependencies = [ name = "conformance" version = "0.0.0" dependencies = [ + "host-server-support", + "openshell-crypto", "openshell-testkit", "serde_json", "tempfile", @@ -561,6 +573,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1138,6 +1156,25 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "rand 0.10.2", + "ring", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "url", +] + [[package]] name = "hkdf" version = "0.13.0" @@ -1165,6 +1202,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "host-server-support" +version = "0.0.0" +dependencies = [ + "openshell-testkit", + "tokio", + "tungstenite", +] + [[package]] name = "hostname" version = "0.4.2" @@ -1518,6 +1564,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2021,6 +2116,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2109,6 +2208,19 @@ dependencies = [ "tower", ] +[[package]] +name = "openshell-policy" +version = "0.0.0" +dependencies = [ + "hickory-proto", + "miette", + "openshell-core", + "openshell-policy-schema", + "prost-types", + "serde", + "serde_json", +] + [[package]] name = "openshell-policy-schema" version = "0.0.0" @@ -2155,15 +2267,19 @@ dependencies = [ "libc", "libtest-mimic", "linkme", + "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", "serde", "serde_json", + "tempfile", "tokio", + "tonic", ] [[package]] @@ -2337,6 +2453,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.5" @@ -2755,6 +2877,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2837,6 +2968,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -3083,6 +3223,22 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -3951,6 +4107,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4098,6 +4264,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index 99274d8dfa..f75fa3aeed 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -21,9 +21,11 @@ path = "src/main.rs" test = false [dependencies] +host-server-support = { path = "../../support/host-server" } +openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-testkit = { path = "../../testkit/testkit" } serde_json = "1" tempfile = "3" -tokio = { version = "1", features = ["time"] } +tokio = { version = "1", features = ["process", "time"] } [workspace] diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index c756107cc1..52a4d639a0 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -9,11 +9,21 @@ nextest run the tests as the library's unit tests. The crate also builds the | Suite | Covers | |---|---| | `smoke` | Create a sandbox, run `echo` in it, delete it | -| `lifecycle` | SDK: long-running and run-to-completion sandboxes, prompt delete when the workload is slow to handle TERM, the default command, restart of a stopped sandbox, deleting a stopped sandbox | -| `cli` | The same lifecycle through the `openshell` CLI: status, create, get, list, exec, stop, start, delete | +| `lifecycle` | SDK: long-running and run-to-completion sandboxes, prompt delete when the workload is slow to handle TERM, the default command, restart of a stopped sandbox, deleting a stopped sandbox, exec past the startup recovery deadline, the final phase when the main command exits, the main command's environment, `on_failure` restarts, `--no-keep` cleanup | +| `cli` | The same lifecycle through the `openshell` CLI: status, create, get, list, exec, stop, start, delete; attached `sandbox create -- ` streams the main process and reports its exit code; `--no-keep` deletes the sandbox afterwards | +| `exec-cli` | `sandbox exec` through the CLI: complete large output, piped stdin up to the 4 MiB limit, early exit, a forced TTY, unreadable stdin | +| `attach` | `sandbox connect` to the main process: one input owner, read-only viewers, history replay on reconnect, stop and start while attached, a main exit of 255 | +| `ssh` | The SSH host key: the same in get, list, and `ssh`; kept across stop and start; new for a recreated sandbox | | `persistent-workspace` | Files in `/sandbox` survive stop and start | | `policy-advisor` | `policy.local` accepts a proposed rule; a denied address or hostname yields one draft rule | -| `file-transfer` | `sandbox upload` and `download`: round trips, `.gitignore` filtering, workspace-escape and option-looking names | +| `file-transfer` | `sandbox upload` and `download`: round trips, file modes and symlinks outside the workspace, `.gitignore` filtering, workspace-escape and option-looking names, a large upload | +| `credential-gating` | Provider credentials: admission of credentialed endpoints, REST body placeholders and rewriting, WebSocket binary frames, live policy binding | +| `image` | A sandbox runs the default image or the one it is created from | +| `labels` | Labels are stored and select sandboxes in `sandbox list` | +| `templates` | A sandbox created from a template uses its workload; duplicate, deleted and missing templates return the right errors | +| `settings` | Sandbox-scoped settings, and gateway-global settings that override them | +| `process` | Core dumps are disabled for sandbox processes | +| `filesystem-policy` | Landlock: a hard requirement accepts added device paths; a replaced policy path rejects the next launch | `smoke` is the smallest end-to-end check that the runner can run a real test in a tmachine environment on CI. The other suites are ports of the old @@ -21,7 +31,11 @@ tmachine environment on CI. The other suites are ports of the old (credited in each file). `file-transfer` runs `git` on the machine that runs the tests, so a test environment needs it installed (`tests/tmachine/ansible/playbooks/test-tools.yaml` does that for the tmachine -environments). +environments). `credential-gating`, ported from NVIDIA/OpenShell#5264, starts +servers (`tests/support/host-server`) on the machine that runs the tests and +reaches them from sandboxes created from `testkit::python_image()` through +`host.openshell.internal`. `ssh` runs `ssh` and `ssh-keygen` there and needs a named +gateway, which `sandbox ssh-config` writes into its `ProxyCommand`. ## Running it @@ -42,7 +56,10 @@ tests/suites/conformance/target/debug/openshell-conformance run --junit conforma ``` The program runs only this suite: passing it means a gateway meets the -conformance contract below. Feature and driver suites are not part of it. The +conformance contract below. The machine that runs it must be reachable from +sandboxes as `host.openshell.internal`, which rules out a gateway on a remote +Kubernetes cluster ([#5409](https://github.com/NVIDIA/OpenShell/issues/5409)). +Feature and driver suites are not part of it. The [testkit README](../../testkit/README.md#test-programs) describes the commands and exit codes. `tasks/scripts/check-conformance-program.sh` checks that it lists the same tests as nextest. diff --git a/tests/suites/conformance/src/attach.rs b/tests/suites/conformance/src/attach.rs new file mode 100644 index 0000000000..7562e31972 --- /dev/null +++ b/tests/suites/conformance/src/attach.rs @@ -0,0 +1,195 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Attaching to a sandbox's main process with `sandbox connect`: one input +//! owner, read-only viewers, retained history on reconnect, and the main +//! process's exit code. +//! +//! Ported from `e2e/rust/tests/sandbox_lifecycle.rs`. The CLI-side transport +//! recovery and signal tests are in `tests/suites/features/cli-attach`. + +use std::process::Stdio; +use std::time::Duration; + +use testkit::sdk::SandboxPhase; +use testkit::{CliChild, Context, ResultExt as _}; + +/// How long an attachment may take to show output. +const ATTACH_TIMEOUT: Duration = Duration::from_secs(30); +/// How long the CLI treats a new attachment as still connecting. A failure +/// after that is a transport failure it may recover from. +const ESTABLISHED: Duration = Duration::from_secs(3); + +/// A main process that prints a numbered line every 0.2 seconds and echoes +/// each line of its input, each with its PID. +const ECHO_MAIN: &str = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'main_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'main_pid=%s input=%s\n' "$$" "$line"; done"#; +const FIRST_RECORD: &str = "sequence=0001"; + +/// The first `sandbox connect` owns the main process's input. A second one +/// is told it is read-only, still sees output, and exits 0 on Ctrl-C without +/// taking input from the owner. After the owner disconnects, a new +/// attachment takes over input and replays the retained output from the +/// first record, all from the same process. Stopping and starting the +/// sandbox works while it is attached. +#[testkit::test] +async fn attach_reconnect_replays_history_for_the_same_main_process( + cx: &mut Context, +) -> testkit::Result { + let sandbox = cx + .sandbox("rc") + .command(["sh", "-lc", ECHO_MAIN]) + .create_ready() + .await?; + let connect = ["sandbox", "connect", sandbox.name()]; + + let mut owner = cx.cli_spawn(connect)?; + let owner_line = owner.stdout_line(ATTACH_TIMEOUT).await?; + let main_pid = owner_line + .split_whitespace() + .find_map(|field| field.strip_prefix("main_pid=")) + .ok_or_else(|| testkit::Error::msg(format!("unexpected owner output {owner_line:?}")))? + .to_string(); + + let mut viewer = cx.cli_spawn(connect)?; + let warning = viewer + .stderr_until(ATTACH_TIMEOUT, |line| { + line.contains("already has an input owner") + }) + .await? + .pop() + .unwrap_or_default(); + testkit::ensure!( + warning.contains("read-only"), + "a second attachment was not told it is read-only: {warning:?}" + ); + let viewer_line = viewer.stdout_line(ATTACH_TIMEOUT).await?; + testkit::ensure!( + viewer_line.contains("main_pid="), + "a read-only attachment did not see output: {viewer_line:?}" + ); + viewer + .write_stdin(b"\x03") + .await + .context("send Ctrl-C to the viewer")?; + let code = viewer.wait(ATTACH_TIMEOUT).await?; + testkit::ensure!(code == 0, "the viewer exited with {code} on Ctrl-C"); + + owner + .write_stdin(b"owner-after-viewer-exit\n") + .await + .context("send input after the viewer exits")?; + owner + .stdout_until(ATTACH_TIMEOUT, |line| { + line.contains("input=owner-after-viewer-exit") + }) + .await + .context("the owner keeps its input after the viewer exits")?; + owner.kill().await?; + + let (mut reconnect, replay) = reconnect_with_input(cx, sandbox.name()).await?; + testkit::ensure!( + replay.iter().any(|line| line.contains(FIRST_RECORD)), + "reconnect did not replay the retained history from {FIRST_RECORD}: {replay:?}" + ); + testkit::ensure!( + replay + .iter() + .all(|line| line.contains(&format!("main_pid={main_pid}"))), + "reconnect shows a process other than {main_pid}: {replay:?}" + ); + testkit::ensure!( + replay.iter().any(|line| !line.contains(FIRST_RECORD)), + "reconnect shows nothing beyond the first record: {replay:?}" + ); + + sandbox.stop().await?; + testkit::ensure!( + sandbox.get().await?.phase == SandboxPhase::Stopped, + "sandbox {} is not Stopped", + sandbox.name() + ); + sandbox.start().await?; + reconnect.kill().await +} + +/// A main process that exits 255 after the attachment is established ends +/// `sandbox connect` with 255. The CLI doesn't mistake it for a lost SSH +/// transport, which also exits 255, and reconnect. +#[testkit::test] +async fn attach_main_exit_255_is_not_retried_as_transport_failure( + cx: &mut Context, +) -> testkit::Result { + const READY: &str = "exit-255-ready"; + const RELEASE: &str = "/sandbox/.openshell-exit-255-release"; + let script = format!("echo {READY}; while [ ! -e '{RELEASE}' ]; do sleep 0.05; done; exit 255"); + let sandbox = cx + .sandbox("x255") + .command(["sh", "-c", &script]) + .create_ready() + .await?; + + let mut connect = + cx.cli_spawn_with_stdin(["sandbox", "connect", sandbox.name()], Stdio::null())?; + connect + .stdout_until(ATTACH_TIMEOUT, |line| line.contains(READY)) + .await?; + // Stay attached past the CLI's setup window, so a 255 would count as an + // established transport failure. + tokio::time::sleep(ESTABLISHED).await; + let release = sandbox.exec(["touch", RELEASE]).await?; + testkit::ensure!( + release.exit_code == 0, + "release the main process: {}", + release.stderr.trim() + ); + + let code = connect + .wait(Duration::from_secs(10)) + .await + .context("a main exit of 255 must not start transport recovery")?; + testkit::ensure!( + code == 255, + "`{}` exited with {code}; expected the main process's 255", + connect.command() + ); + Ok(()) +} + +/// Attach until one attachment takes over input, which it can only do once +/// the gateway has released the previous owner. Returns the attachment and +/// the output lines it showed up to its own input. +async fn reconnect_with_input( + cx: &Context, + name: &str, +) -> testkit::Result<(CliChild, Vec)> { + const DEADLINE: Duration = Duration::from_secs(30); + let deadline = tokio::time::Instant::now() + DEADLINE; + for attempt in 1.. { + let token = format!("reconnect-{attempt}"); + let mut candidate = cx.cli_spawn(["sandbox", "connect", name])?; + candidate + .write_stdin(format!("{token}\n").as_bytes()) + .await + .context("send reconnect input")?; + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + let read = candidate + .stdout_until(Duration::from_secs(2).min(remaining), |line| { + line.contains(&format!("input={token}")) + }) + .await; + if let Ok(lines) = read { + let replay = lines + .into_iter() + .filter(|line| line.contains("main_pid=")) + .collect(); + return Ok((candidate, replay)); + } + candidate.kill().await?; + testkit::ensure!( + tokio::time::Instant::now() < deadline, + "no reconnect to sandbox {name} took over input within {DEADLINE:?}" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + unreachable!("the attempts are unbounded") +} diff --git a/tests/suites/conformance/src/cli.rs b/tests/suites/conformance/src/cli.rs index d26eed9bb5..9dacadd9fd 100644 --- a/tests/suites/conformance/src/cli.rs +++ b/tests/suites/conformance/src/cli.rs @@ -23,18 +23,9 @@ const MAX_PAGES: usize = 1000; async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { cx.cli(["status"]).await?.success()?; - let name = cx.name("cs"); - cx.track(&name); - cx.cli(["sandbox", "create", "--name", &name, "--detach"]) - .await? - .success()?; - - let sandbox = cx.adopt(&name).await?; - let got: Value = cx - .cli(["sandbox", "get", &name, "--output", "json"]) - .await? - .success()? - .json()?; + let sandbox = cx.cli_sandbox("cs").create().await?; + let name = sandbox.name().to_string(); + let got = sandbox.get_json().await?; testkit::ensure!( got["name"] == name.as_str(), "sandbox get returned {:?}; expected {name:?}", @@ -50,46 +41,27 @@ async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result { testkit::ensure!(listed.is_some(), "sandbox {name} is not in sandbox list"); let marker = format!("openshell-e2e-{}", cx.run_id()); - let exec = cx - .cli([ - "sandbox", "exec", "--name", &name, "--no-tty", "--", "echo", &marker, - ]) - .await? - .success()?; - testkit::ensure!( - exec.stdout == format!("{marker}\n"), - "sandbox exec stdout was {:?}; expected {:?}", - exec.stdout, - format!("{marker}\n") - ); + cx.cli([ + "sandbox", "exec", "--name", &name, "--no-tty", "--", "echo", &marker, + ]) + .await? + .expect_stdout(&format!("{marker}\n"))?; cx.cli(["sandbox", "delete", &name]).await?.success()?; wait_unlisted(cx, &name).await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await } /// A sandbox created with the CLI can be stopped, rejects `sandbox exec` while /// stopped, starts again, and can be stopped and deleted. #[testkit::test] async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { - let name = cx.name("cl"); - cx.track(&name); - cx.cli([ - "sandbox", - "create", - "--name", - &name, - "--detach", - "--no-tty", - "--", - "sh", - "-c", - "exec sleep infinity", - ]) - .await? - .success()?; - let sandbox = cx.adopt(&name).await?; - sandbox.wait_ready().await?; + let sandbox = cx + .cli_sandbox("cl") + .command(["sh", "-c", "exec sleep infinity"]) + .create_ready() + .await?; + let name = sandbox.name().to_string(); cx.cli(["sandbox", "stop", &name]).await?.success()?; sandbox @@ -108,7 +80,122 @@ async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result { cx.cli(["sandbox", "delete", &name]).await?.success()?; wait_unlisted(cx, &name).await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await +} + +/// `sandbox create -- ` without `--detach` streams the main +/// process's output and exits with its exit code. The sandbox is kept by +/// default: one whose main exited 0 is listed and Completed, and one whose +/// main exited 7 is in Error with exit code 7. +#[testkit::test] +async fn cli_attached_create_streams_main_and_reports_exit(cx: &mut Context) -> testkit::Result { + let completed = cx.name("ac0"); + cx.track(&completed); + let output = cx + .cli([ + "sandbox", "create", "--name", &completed, "--", "echo", "OK", + ]) + .await? + .success()?; + expect_streamed(&output, "OK")?; + testkit::ensure!( + find_listed(cx, &completed).await?.is_some(), + "sandbox {completed} is not listed after its main exited; sandboxes are kept by default" + ); + // The phase can trail the CLI's exit by a status update. + cx.adopt(&completed).await?.wait_finished().await?; + let got = get_json(cx, &completed).await?; + testkit::ensure!( + got["phase"] == "Completed", + "sandbox {completed} is in phase {}; expected Completed", + got["phase"] + ); + + let failed = cx.name("ac7"); + cx.track(&failed); + let output = cx + .cli([ + "sandbox", + "create", + "--name", + &failed, + "--", + "sh", + "-c", + "echo failed-main; exit 7", + ]) + .await?; + testkit::ensure!( + output.exit_code == 7, + "`{}` exited with {}; expected the main process's 7; stderr: {}", + output.command, + output.exit_code, + output.stderr.trim() + ); + expect_streamed(&output, "failed-main")?; + cx.adopt(&failed).await?.wait_finished().await?; + let got = get_json(cx, &failed).await?; + testkit::ensure!( + got["phase"] == "Error" && got["exit_code"] == 7, + "sandbox {failed} is in phase {} with exit code {}; expected Error with 7", + got["phase"], + got["exit_code"] + ); + Ok(()) +} + +/// `sandbox create --no-keep -- ` streams the main process's output, +/// exits with its exit code, and then deletes the sandbox, whether the main +/// exited 0 or 17. +#[testkit::test] +async fn cli_no_keep_removes_sandbox_after_main_exit(cx: &mut Context) -> testkit::Result { + for (suffix, code, marker) in [("nk0", 0, "OK"), ("nk17", 17, "ephemeral-failure")] { + let name = cx.name(suffix); + cx.track(&name); + let script = format!("echo {marker}; exit {code}"); + let output = cx + .cli([ + "sandbox", + "create", + "--name", + &name, + "--no-keep", + "--", + "sh", + "-c", + &script, + ]) + .await?; + testkit::ensure!( + output.exit_code == code, + "`{}` exited with {}; expected the main process's {code}; stderr: {}", + output.command, + output.exit_code, + output.stderr.trim() + ); + expect_streamed(&output, marker)?; + wait_unlisted(cx, &name).await?; + } + Ok(()) +} + +/// Fail unless the CLI printed `marker` from the main process. +fn expect_streamed(output: &testkit::CliOutput, marker: &str) -> testkit::Result { + testkit::ensure!( + output.stdout.contains(marker) || output.stderr.contains(marker), + "`{}` did not stream {marker:?} from the main process; stdout: {}; stderr: {}", + output.command, + output.stdout.trim(), + output.stderr.trim() + ); + Ok(()) +} + +async fn get_json(cx: &Context, name: &str) -> testkit::Result { + cx.cli(["sandbox", "get", name, "--output", "json"]) + .await? + .success()? + .json() } /// Wait until `sandbox list` no longer shows the sandbox. diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs new file mode 100644 index 0000000000..2d67926ef0 --- /dev/null +++ b/tests/suites/conformance/src/credential_gating.rs @@ -0,0 +1,413 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Credential gating: which endpoints a provider credential may reach, and +//! what the supervisor forwards upstream in REST bodies and WebSocket frames. +//! +//! Ported from the e2e tests added in NVIDIA/OpenShell#5264. The upstream +//! servers run on the test host ([`host_server_support`]), so the suite +//! requires sandboxes to reach the test host as `host.openshell.internal`. + +use std::time::Duration; + +use host_server_support::{HttpRecorder, Request, WebSocketRecorder}; +use testkit::policy::{self, Access, Endpoint as _}; +use testkit::sdk::raw::proto; +use testkit::{ + Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox, YamlFile, +}; + +const TEST_HOST: &str = "host.openshell.internal"; +const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; +const TEST_SECRET: &str = "e2e-gating-secret-value"; +const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; + +/// POSTs a body containing the provider's token to `TEST_HOST` on the port +/// in `argv[1]`, directly rather than through a proxy. +const BODY_CLIENT: &str = r#" +import os, sys, urllib.request +body = ("prefix-" + os.environ["E2E_GATING_TOKEN"] + "-suffix").encode() +request = urllib.request.Request(f"http://host.openshell.internal:{sys.argv[1]}/token", data=body, headers={"Content-Type": "text/plain"}) +urllib.request.build_opener(urllib.request.ProxyHandler({})).open(request, timeout=10).read() +"#; + +/// Upgrades to WebSocket at `TEST_HOST` on the port in `argv[1]`, sends one +/// binary frame, and waits briefly for the connection to be closed. +const WEBSOCKET_CLIENT: &str = r#" +import base64, os, socket, sys +host, port = "host.openshell.internal", int(sys.argv[1]) +with socket.create_connection((host, port), timeout=10) as sock: + key = base64.b64encode(os.urandom(16)).decode() + sock.sendall(f"GET /ws HTTP/1.1\r\nHost: {host}:{port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode()) + response = b"" + while b"\r\n\r\n" not in response: + chunk = sock.recv(4096) + if not chunk: + sys.exit(f"closed during the upgrade: {response!r}") + response += chunk + if not response.startswith(b"HTTP/1.1 101"): + sys.exit(f"upgrade failed: {response!r}") + payload, mask = b"binary-credential-channel", os.urandom(4) + sock.sendall(bytes([0x82, 0x80 | len(payload)]) + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))) + sock.settimeout(3) + try: + sock.recv(4096) + except (TimeoutError, ConnectionError): + pass +"#; + +/// Prints `TOKEN_ABSENT`, `TOKEN_PLACEHOLDER`, or `TOKEN_UNSAFE` for the +/// provider's environment variable, without printing its value. +const TOKEN_PRESENCE: &str = r#"if [ "${E2E_GATING_TOKEN+x}" != x ]; then + echo TOKEN_ABSENT +else + case "$E2E_GATING_TOKEN" in + openshell:resolve:env:*) echo TOKEN_PLACEHOLDER ;; + *) echo TOKEN_UNSAFE ;; + esac +fi"#; + +/// A provider whose profile declares the REST and WebSocket servers as its +/// endpoints keeps the credential placeholder in a REST body. +#[testkit::test] +async fn profile_rest_endpoint_keeps_body_placeholder(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::start("ok").await?; + let websocket = WebSocketRecorder::start().await?; + let provider = install_gating_provider(cx, Some((server.port, websocket.port))).await?; + + let mut policy = policy::base(); + allow(&mut policy, server.port) + .rest(Access::Full) + .request_body_credential_rewrite(false); + let sandbox = create_sandbox(cx, "rest", policy, &[&provider]).await?; + run_python(&sandbox, BODY_CLIENT, server.port).await?; + expect_placeholder_body(&server).await +} + +/// A provider's WebSocket endpoint completes the handshake but doesn't +/// forward a binary frame, which the supervisor can't inspect for the +/// credential. The same client in a sandbox without the provider gets its +/// binary frame through, which shows the denial comes from the credential. +#[testkit::test] +async fn profile_websocket_endpoint_denies_binary_frames(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::start("ok").await?; + let websocket = WebSocketRecorder::start().await?; + let provider = install_gating_provider(cx, Some((server.port, websocket.port))).await?; + + let control = WebSocketRecorder::start().await?; + let binary_seen = send_binary_frame(cx, "wsctl", &control, &[]).await?; + testkit::ensure!( + binary_seen, + "the binary frame didn't reach upstream from a sandbox without the provider" + ); + + let binary_seen = send_binary_frame(cx, "ws", &websocket, &[&provider]).await?; + testkit::ensure!( + !binary_seen, + "credentialed WebSocket binary frame reached upstream" + ); + Ok(()) +} + +/// The gateway rejects a policy that binds a provider to an L4-only +/// endpoint or to an endpoint with `tls: skip`, and accepts an L4 endpoint +/// that opts in to uninspected credentials. +#[testkit::test] +async fn policy_binding_requires_inspection_or_opt_in(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::start("ok").await?; + let provider = install_gating_provider(cx, None).await?; + + let mut l4 = policy::base(); + allow(&mut l4, server.port).credential_binding(&provider); + // The wording checks follow the #5264 tests. + let error = expect_rejected(cx, "l4", l4, &provider).await?; + testkit::ensure!(error.contains("credentialed endpoint"), "{error}"); + testkit::ensure!(error.contains("L4-only"), "{error}"); + + let mut tls_skip = policy::base(); + allow(&mut tls_skip, server.port) + .rest(Access::Full) + .tls_skip() + .credential_binding(&provider); + let error = expect_rejected(cx, "tls", tls_skip, &provider).await?; + testkit::ensure!(error.contains("credentialed endpoint"), "{error}"); + testkit::ensure!(error.contains("tls:") && error.contains("skip"), "{error}"); + + let mut opt_in = policy::base(); + allow(&mut opt_in, server.port) + .allow_uninspected_credentials() + .credential_binding(&provider); + let sandbox = create_sandbox(cx, "optin", opt_in, &[&provider]).await?; + sandbox.exec(["true"]).await?.success()?; + Ok(()) +} + +/// A policy binding to a REST endpoint without body rewriting sends the +/// credential placeholder upstream, not the secret. +#[testkit::test] +async fn policy_binding_rest_body_keeps_placeholder_without_rewrite( + cx: &mut Context, +) -> testkit::Result { + let server = HttpRecorder::start("ok").await?; + let provider = install_gating_provider(cx, None).await?; + + let mut policy = policy::base(); + allow(&mut policy, server.port) + .rest(Access::Full) + .request_body_credential_rewrite(false) + .credential_binding(&provider); + let sandbox = create_sandbox(cx, "body", policy, &[&provider]).await?; + run_python(&sandbox, BODY_CLIENT, server.port).await?; + expect_placeholder_body(&server).await +} + +/// A policy binding with `request_body_credential_rewrite: true` replaces +/// the placeholder in the REST body with the secret upstream. +#[testkit::test] +async fn policy_binding_rest_body_rewrite_sends_secret_upstream( + cx: &mut Context, +) -> testkit::Result { + let server = HttpRecorder::start("ok").await?; + let provider = install_gating_provider(cx, None).await?; + + let mut policy = policy::base(); + allow(&mut policy, server.port) + .rest(Access::Full) + .request_body_credential_rewrite(true) + .credential_binding(&provider); + let sandbox = create_sandbox(cx, "rw", policy, &[&provider]).await?; + run_python(&sandbox, BODY_CLIENT, server.port).await?; + let request = one_request(&server).await?; + testkit::ensure!( + request.body == format!("prefix-{TEST_SECRET}-suffix"), + "expected the secret in the body upstream; got {}", + redact(&request) + ); + Ok(()) +} + +/// Binding and unbinding a provider with `policy set --wait` adds and +/// removes its environment variable in a running sandbox. +#[testkit::test] +async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::start("ok").await?; + let provider = install_gating_provider(cx, None).await?; + + let policy_binding = |bind: bool| { + let mut policy = policy::base(); + let endpoint = allow(&mut policy, server.port) + .rest(Access::Full) + .request_body_credential_rewrite(false); + if bind { + endpoint.credential_binding(&provider); + } + policy + }; + let (unbound, bound) = (policy_binding(false), policy_binding(true)); + + let sandbox = create_sandbox(cx, "live", unbound.clone(), &[&provider]).await?; + wait_for_token(&sandbox, "TOKEN_ABSENT").await?; + + set_policy(cx, &sandbox, &bound) + .await + .context("bind the provider")?; + wait_for_token(&sandbox, "TOKEN_PLACEHOLDER").await?; + + set_policy(cx, &sandbox, &unbound) + .await + .context("unbind the provider")?; + wait_for_token(&sandbox, "TOKEN_ABSENT").await +} + +/// Run [`WEBSOCKET_CLIENT`] against `server` in a sandbox with `providers`, +/// and return whether its binary frame reached `server`. +async fn send_binary_frame( + cx: &Context, + suffix: &str, + server: &WebSocketRecorder, + providers: &[&str], +) -> Result { + let mut policy = policy::base(); + allow(&mut policy, server.port).websocket(Access::ReadWrite); + let sandbox = create_sandbox(cx, suffix, policy, providers).await?; + run_python(&sandbox, WEBSOCKET_CLIENT, server.port).await?; + server + .wait_for_handshake() + .await + .context("upstream should receive the WebSocket handshake")?; + server.wait_until_finished().await +} + +/// Let the image's Python reach `TEST_HOST` on `port`. +fn allow(policy: &mut proto::SandboxPolicy, port: u16) -> &mut proto::NetworkEndpoint { + policy::allow( + policy, + "credential_gating", + TEST_HOST, + port, + PYTHON_BINARIES, + ) +} + +/// Install a run-unique provider holding `TEST_SECRET` in `TOKEN_ENV`. With +/// `endpoints`, its profile declares the REST and WebSocket ports as its +/// endpoints; without, a policy must bind the provider explicitly. +async fn install_gating_provider(cx: &Context, endpoints: Option<(u16, u16)>) -> Result { + let name = cx.name("cg"); + let endpoints = endpoints.map_or_else(String::new, |(rest, websocket)| { + format!( + "endpoints: + - host: {TEST_HOST} + port: {rest} + protocol: rest + access: full + - host: {TEST_HOST} + port: {websocket} + protocol: websocket + access: read-write +" + ) + }); + let binaries: String = PYTHON_BINARIES + .iter() + .map(|path| format!(" - path: {path}\n")) + .collect(); + let profile = format!( + "id: {name} +display_name: E2E Credential Gating +category: other +credentials: + - name: token + env_vars: [{TOKEN_ENV}] + required: true + auth_style: bearer + header_name: authorization +{endpoints}binaries: +{binaries}" + ); + cx.install_provider(&name, &profile, &[(TOKEN_ENV, TEST_SECRET)]) + .await?; + Ok(name) +} + +fn sandbox_with( + cx: &Context, + suffix: &str, + policy: proto::SandboxPolicy, + providers: &[&str], +) -> testkit::SandboxBuilder { + cx.sandbox(suffix) + .policy(policy) + .image(testkit::python_image()) + .spec(|spec| spec.providers = providers.iter().map(ToString::to_string).collect()) + .command(["sleep", "infinity"]) +} + +/// Create a long-running sandbox from the Python image with `policy` and +/// `providers`, and wait until it is Ready. +async fn create_sandbox( + cx: &Context, + suffix: &str, + policy: proto::SandboxPolicy, + providers: &[&str], +) -> Result { + sandbox_with(cx, suffix, policy, providers) + .create_ready() + .await +} + +/// Fail unless the gateway rejects the sandbox as a failed precondition, and +/// return the rejection. +async fn expect_rejected( + cx: &Context, + suffix: &str, + policy: proto::SandboxPolicy, + provider: &str, +) -> Result { + match sandbox_with(cx, suffix, policy, &[provider]).create().await { + Ok(sandbox) => Err(Error::msg(format!( + "the gateway accepted sandbox {}", + sandbox.name() + ))), + Err(error) if error.grpc_code() == Some(GrpcCode::FailedPrecondition) => { + Ok(error.to_string()) + } + Err(error) => Err(error.context("expected a failed precondition")), + } +} + +/// Run `script` with the image's Python and `port` as its argument. +async fn run_python(sandbox: &Sandbox, script: &str, port: u16) -> Result { + let port = port.to_string(); + sandbox + .exec(["python3", "-c", script, &port]) + .await? + .success() + .context("run the client in the sandbox")?; + Ok(()) +} + +/// The one request `server` saw. +async fn one_request(server: &HttpRecorder) -> Result { + let mut requests = server.wait_for_requests(1).await?; + testkit::ensure!( + requests.len() == 1, + "expected one request upstream; got {}", + requests.len() + ); + Ok(requests.remove(0)) +} + +/// Fail unless `server` saw one request whose body has the credential +/// placeholder in place of the token, and the secret appears nowhere in it. +async fn expect_placeholder_body(server: &HttpRecorder) -> Result { + let request = one_request(server).await?; + testkit::ensure!( + request + .body + .starts_with(&format!("prefix-{PLACEHOLDER_PREFIX}")) + && request.body.ends_with("-suffix") + && !request.head.contains(TEST_SECRET) + && !request.body.contains(TEST_SECRET), + "expected the placeholder in the body upstream and no secret; got {}", + redact(&request) + ); + Ok(()) +} + +/// `request` for a failure message, with the secret replaced. +fn redact(request: &Request) -> String { + format!("{}\n\n{}", request.head, request.body).replace(TEST_SECRET, "") +} + +/// Wait until new exec sessions report `expected` from [`TOKEN_PRESENCE`]. +async fn wait_for_token(sandbox: &Sandbox, expected: &str) -> Result { + sandbox + .poll_exec( + &format!("the provider environment to report {expected}"), + Duration::from_secs(30), + ["sh", "-lc", TOKEN_PRESENCE], + |output| output.stdout.contains(expected), + ) + .await?; + Ok(()) +} + +/// Replace `sandbox`'s policy with `policy` and wait until it is applied. +async fn set_policy(cx: &Context, sandbox: &Sandbox, policy: &proto::SandboxPolicy) -> Result { + let file = YamlFile::new(&policy::to_yaml(policy)?)?; + cx.cli([ + "policy", + "set", + sandbox.name(), + "--policy", + file.path()?, + "--wait", + "--timeout", + "120", + ]) + .await? + .success() + .context("policy set")?; + Ok(()) +} diff --git a/tests/suites/conformance/src/exec_cli.rs b/tests/suites/conformance/src/exec_cli.rs new file mode 100644 index 0000000000..35924cb351 --- /dev/null +++ b/tests/suites/conformance/src/exec_cli.rs @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `sandbox exec` through the `openshell` CLI: large output, piped stdin, and +//! early exit. +//! +//! Ported from `e2e/rust/tests/sandbox_lifecycle.rs`. + +use std::time::Duration; + +use testkit::{Context, ResultExt as _, Sandbox}; + +/// How long one exec of a few MiB may take. +const EXEC_TIMEOUT: Duration = Duration::from_secs(30); +/// The largest stdin a non-interactive exec accepts. +const STDIN_LIMIT: usize = 4 * 1024 * 1024; + +/// `sandbox exec --no-tty` returns all of a large stdout and stderr, exits 0 +/// when the command exits before reading all its stdin, and doesn't wait for +/// a background process that still holds the output pipe. +#[testkit::test] +async fn cli_exec_large_output_is_complete(cx: &mut Context) -> testkit::Result { + const BYTES: usize = 8 * 1024 * 1024; + let sandbox = ready_sandbox(cx, "lo").await?; + + for stream in ["stdout", "stderr"] { + let script = match stream { + "stdout" => format!("yes A | head -c {BYTES}"), + _ => format!("yes A | head -c {BYTES} >&2"), + }; + let output = tokio::time::timeout( + EXEC_TIMEOUT, + cx.cli(exec_args(&sandbox, &["sh", "-c", &script])), + ) + .await + .map_err(|_| testkit::Error::msg(format!("exec of 8 MiB on {stream} timed out")))?? + .success()?; + let bytes = match stream { + "stdout" => output.stdout.as_bytes(), + _ => output.stderr.as_bytes(), + }; + testkit::ensure!( + bytes.len() == BYTES, + "exec {stream} has {} bytes; expected {BYTES}", + bytes.len() + ); + testkit::ensure!( + bytes.chunks_exact(2).all(|pair| pair == b"A\n"), + "exec {stream} differs from the `yes A` output" + ); + } + + let mut early_exit = cx.cli_spawn(exec_args(&sandbox, &["head", "-n1"]))?; + let mut input = b"first\n".to_vec(); + input.extend(vec![b'x'; 128 * 1024]); + if let Err(error) = early_exit.write_stdin(&input).await { + testkit::ensure!( + error.kind() == std::io::ErrorKind::BrokenPipe, + "write stdin of `{}`: {error}", + early_exit.command() + ); + } + let output = early_exit.output(EXEC_TIMEOUT).await?.success()?; + testkit::ensure!( + output.stdout == "first\n", + "`head -n1` printed {:?}; expected \"first\\n\"", + output.stdout + ); + + let output = tokio::time::timeout( + Duration::from_secs(10), + cx.cli(exec_args(&sandbox, &["sh", "-c", "sleep 5 & echo ok"])), + ) + .await + .map_err(|_| testkit::Error::msg("exec waited for a background process holding its output"))?? + .success()?; + testkit::ensure!( + output.stdout == "ok\n", + "exec printed {:?}; expected \"ok\\n\"", + output.stdout + ); + Ok(()) +} + +/// Piped stdin larger than one gRPC message reaches the command intact, up to +/// the 4 MiB limit. A larger stdin is rejected with a message naming the +/// limit, a forced TTY with piped stdin still works, and a stdin that can't be +/// read fails the exec. +#[testkit::test] +async fn cli_exec_piped_stdin_crosses_grpc_message_limit(cx: &mut Context) -> testkit::Result { + let sandbox = ready_sandbox(cx, "si").await?; + let wc = exec_args(&sandbox, &["wc", "-c"]); + + for size in [5, 1024 * 1024, STDIN_LIMIT] { + let mut exec = cx.cli_spawn(&wc)?; + exec.write_stdin(&vec![b'x'; size]) + .await + .context(format!("write {size} bytes to `{}`", exec.command()))?; + let output = exec.output(EXEC_TIMEOUT).await?.success()?; + testkit::ensure!( + output.stdout.trim() == size.to_string(), + "`wc -c` counted {:?} of {size} piped bytes", + output.stdout.trim() + ); + } + + let mut oversized = cx.cli_spawn(&wc)?; + oversized + .write_stdin(&vec![b'x'; STDIN_LIMIT + 1]) + .await + .context(format!( + "write oversized stdin to `{}`", + oversized.command() + ))?; + let output = oversized.output(EXEC_TIMEOUT).await?.failure()?; + testkit::ensure!( + output.stdout.is_empty(), + "an exec with oversized stdin printed {:?}", + output.stdout + ); + testkit::ensure!( + output.stderr.contains("4 MiB limit"), + "an exec with oversized stdin did not name the limit; stderr: {}", + output.stderr.trim() + ); + + // A forced PTY with a small pipe must keep using the unary RPC for + // compatibility with older gateways. + let mut tty = cx.cli_spawn([ + "sandbox", + "exec", + "--name", + sandbox.name(), + "--tty", + "--no-login-shell", + "--", + "sh", + "-c", + "printf tty-ok", + ])?; + tty.write_stdin(b"x") + .await + .context(format!("write stdin to `{}`", tty.command()))?; + let output = tty.output(EXEC_TIMEOUT).await?.success()?; + testkit::ensure!( + output.stdout.contains("tty-ok"), + "a forced TTY exec printed {:?}", + output.stdout + ); + + #[cfg(unix)] + { + let directory = std::fs::File::open("/").context("open / as stdin")?; + let output = cx + .cli_spawn_with_stdin(&wc, directory)? + .output(EXEC_TIMEOUT) + .await? + .failure()?; + testkit::ensure!( + output.stdout.is_empty(), + "an exec whose stdin can't be read printed {:?}", + output.stdout + ); + } + Ok(()) +} + +async fn ready_sandbox(cx: &Context, suffix: &str) -> testkit::Result { + cx.sandbox(suffix) + .command(["sleep", "infinity"]) + .create_ready() + .await +} + +/// `sandbox exec` of `command` without a TTY or login shell. +fn exec_args(sandbox: &Sandbox, command: &[&str]) -> Vec { + [ + "sandbox", + "exec", + "--name", + sandbox.name(), + "--no-tty", + "--no-login-shell", + "--", + ] + .iter() + .chain(command) + .map(ToString::to_string) + .collect() +} diff --git a/tests/suites/conformance/src/file_transfer.rs b/tests/suites/conformance/src/file_transfer.rs index 5bd24563c0..7e04dbe27d 100644 --- a/tests/suites/conformance/src/file_transfer.rs +++ b/tests/suites/conformance/src/file_transfer.rs @@ -11,10 +11,11 @@ //! why they run in every environment. use std::fs; +use std::os::unix::fs::{PermissionsExt, symlink}; use std::path::{Path, PathBuf}; use tempfile::TempDir; -use testkit::{CliOutput, Context, Error, Sandbox}; +use testkit::{CliOutput, Context, Error, ResultExt as _, Sandbox}; const LARGE_FILE_SIZE: usize = 512 * 1024; @@ -25,6 +26,7 @@ const LARGE_FILE_SIZE: usize = 512 * 1024; async fn upload_download_round_trip(cx: &mut Context) -> testkit::Result { let ft = Transfer::new(cx, "fr").await?; ft.upload_into_existing_directories().await?; + ft.upload_metadata_outside_workdir().await?; ft.round_trip().await?; ft.download_file().await?; ft.download_directory().await?; @@ -53,6 +55,84 @@ async fn download_stays_inside_workspace(cx: &mut Context) -> testkit::Result { ft.sandbox.delete().await } +const BACKPRESSURE_FILE_COUNT: usize = 12_000; +const BACKPRESSURE_FILE_BYTES: usize = 8_192; + +/// An upload far larger than the supervisor's 4 MiB pending-stdin budget, +/// made of many small files that keep tar busy creating them, completes on +/// the first attempt with every file intact. The sender is not paced, so the +/// transfer must apply backpressure (#4430). +#[testkit::test] +async fn large_upload_backpressure(cx: &mut Context) -> testkit::Result { + let ft = Transfer::new(cx, "fb").await?; + let source = ft.local("large-project"); + create_dir_all(&source.join("payload"))?; + + // Distinct pseudorandom bytes, so compression or identical-file checks + // can't hide dropped data. + let mut state: u64 = 4430; + let mut payload = vec![0_u8; BACKPRESSURE_FILE_BYTES]; + let mut checksums = String::new(); + for index in 0..BACKPRESSURE_FILE_COUNT { + for chunk in payload.chunks_mut(8) { + chunk.copy_from_slice(&splitmix64(&mut state).to_le_bytes()[..chunk.len()]); + } + let relative = format!("payload/{index:05}.bin"); + fs::write(source.join(&relative), &payload).context(format!("write {relative}"))?; + checksums.push_str(&format!("{} {relative}\n", sha256_hex(&payload)?)); + } + write(&source.join("SHA256SUMS"), &checksums)?; + + let remote = format!("{}/backpressure", ft.remote_root); + let uploaded = ft + .cx + .cli_with_env( + &[("RUST_LOG", "warn")], + [ + "sandbox", + "upload", + ft.sandbox.name(), + &source.display().to_string(), + &remote, + "--no-git-ignore", + ], + ) + .await? + .success()?; + testkit::ensure!( + !uploaded + .message() + .contains("sandbox sync operation failed; retrying"), + "the large upload needed a retry: {}", + uploaded.message() + ); + + ft.sh(&format!( + "set -eu; cd '{remote}/large-project'; \ + test \"$(find payload -type f | wc -l)\" -eq {BACKPRESSURE_FILE_COUNT}; \ + test \"$(sha256sum SHA256SUMS | cut -d ' ' -f 1)\" = {}; \ + sha256sum --check --quiet SHA256SUMS", + sha256_hex(checksums.as_bytes())? + )) + .await?; + ft.sandbox.delete().await +} + +/// One step of SplitMix64, a small deterministic generator for fixture bytes. +fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +fn sha256_hex(bytes: &[u8]) -> testkit::Result { + let digest = + openshell_crypto::sha256(bytes).map_err(|error| Error::msg(format!("SHA-256: {error}")))?; + Ok(digest.iter().map(|byte| format!("{byte:02x}")).collect()) +} + /// A sandbox, a directory in its workspace, and a local scratch directory. struct Transfer<'a> { cx: &'a Context, @@ -66,12 +146,10 @@ impl<'a> Transfer<'a> { let sandbox = cx .sandbox(suffix) .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let remote_root = format!("/sandbox/file-transfer-{}-{suffix}", cx.run_id()); - let local = tempfile::tempdir() - .map_err(|error| Error::msg(format!("create a local scratch directory: {error}")))?; + let local = tempfile::tempdir().context("create a local scratch directory")?; let ft = Self { cx, sandbox, @@ -88,14 +166,11 @@ impl<'a> Transfer<'a> { /// Run a shell script in the sandbox and require it to succeed. async fn sh(&self, script: &str) -> testkit::Result { - let output = self.sandbox.exec(["sh", "-c", script]).await?; - testkit::ensure!( - output.exit_code == 0, - "`{script}` exited with {}; stderr: {}", - output.exit_code, - output.stderr - ); - Ok(()) + self.sandbox + .exec(["sh", "-c", script]) + .await? + .success() + .map(drop) } async fn upload( @@ -169,6 +244,51 @@ impl<'a> Transfer<'a> { Ok(()) } + /// An upload to a directory outside the workspace keeps file modes and + /// symlinks, dangling ones included, overwrites files it carries, and + /// leaves other files in the destination alone. + async fn upload_metadata_outside_workdir(&self) -> testkit::Result { + let source = self.local("metadata-project"); + create_dir_all(&source.join("payload"))?; + write(&source.join("payload/00000.bin"), "payload")?; + let script = source.join("run.sh"); + write(&script, "#!/bin/sh\nprintf '%s\\n' upload-executable-ok\n")?; + fs::set_permissions(&script, fs::Permissions::from_mode(0o751)).context("chmod run.sh")?; + for (target, link) in [ + ("payload/00000.bin", "payload-link"), + ("missing-target", "dangling-link"), + ] { + symlink(target, source.join(link)).context(format!("symlink {link}"))?; + } + write(&source.join("conflict.txt"), "local-conflict")?; + + let remote = format!( + "/tmp/file-transfer-{}-{}", + self.cx.run_id(), + self.sandbox.name() + ); + let project = format!("{remote}/metadata-project"); + self.sh(&format!( + "set -eu; test \"$(pwd -P)\" != '{remote}'; mkdir -p '{project}'; \ + printf remote-conflict > '{project}/conflict.txt'; \ + printf remote-preserved > '{project}/unrelated.txt'" + )) + .await?; + + self.upload(&source, &remote, true).await?.success()?; + self.sh(&format!( + "set -eu; cd '{project}'; \ + test \"$(stat -c %a run.sh)\" = 751; \ + test \"$(./run.sh)\" = upload-executable-ok; \ + test -L payload-link; test \"$(readlink payload-link)\" = payload/00000.bin; \ + test -L dangling-link; test ! -e dangling-link; \ + test \"$(readlink dangling-link)\" = missing-target; \ + test \"$(cat conflict.txt)\" = local-conflict; \ + test \"$(cat unrelated.txt)\" = remote-preserved" + )) + .await + } + /// A directory with a nested file and a large binary file, and a single /// file, survive upload then download. async fn round_trip(&self) -> testkit::Result { @@ -177,12 +297,11 @@ impl<'a> Transfer<'a> { write(&source.join("greeting.txt"), "hello-from-local")?; write(&source.join("subdir/nested.txt"), "nested-content")?; let large: Vec = (0u8..=250).cycle().take(LARGE_FILE_SIZE).collect(); - fs::write(source.join("large.bin"), &large) - .map_err(|error| Error::msg(format!("write large.bin: {error}")))?; + fs::write(source.join("large.bin"), &large).context("write large.bin")?; let remote = format!("{}/roundtrip", self.remote_root); let uploaded = self.upload(&source, &remote, false).await?.success()?; - let warning = message(&uploaded); + let warning = uploaded.message(); testkit::ensure!( warning.contains("outside a Git work tree") && warning.contains(".gitignore rules are not applied"), @@ -196,8 +315,7 @@ impl<'a> Transfer<'a> { .success()?; require_text(&destination.join("greeting.txt"), "hello-from-local")?; require_text(&destination.join("subdir/nested.txt"), "nested-content")?; - let actual = fs::read(destination.join("large.bin")) - .map_err(|error| Error::msg(format!("read large.bin: {error}")))?; + let actual = fs::read(destination.join("large.bin")).context("read large.bin")?; testkit::ensure!( actual == large, "large file changed in the round trip: sent {} bytes, received {}", @@ -310,7 +428,7 @@ impl<'a> Transfer<'a> { let remote = format!("{}/reuploaded", self.remote_root); let rejected = self.upload(&runs, &remote, false).await?.failure()?; - let output = message(&rejected); + let output = rejected.message(); testkit::ensure!( output.contains("filtering selected no files") && output.contains("--no-git-ignore"), "the rejected upload did not explain the --no-git-ignore override: {output}" @@ -336,7 +454,7 @@ impl<'a> Transfer<'a> { for source in [etc_link.clone(), passwd_link, format!("{etc_link}/passwd")] { let rejected = self.download(&source, &destination).await?.failure()?; - let output = message(&rejected); + let output = rejected.message(); testkit::ensure!( output.contains("resolves to") && output.contains("outside the") @@ -362,22 +480,12 @@ impl<'a> Transfer<'a> { } } -/// The CLI's stdout and stderr as one line of words. Error reports wrap to -/// the terminal width and draw a gutter, so match against this instead. -fn message(output: &CliOutput) -> String { - format!("{} {}", output.stdout, output.stderr) - .replace(['│', '×', '╰', '─', '╭', '├'], " ") - .split_whitespace() - .collect::>() - .join(" ") -} - fn git(repository: &Path, args: &[&str]) -> testkit::Result { let output = std::process::Command::new("git") .args(args) .current_dir(repository) .output() - .map_err(|error| Error::msg(format!("run git {}: {error}", args.join(" "))))?; + .context(format!("run git {}", args.join(" ")))?; testkit::ensure!( output.status.success(), "git {} failed in {}: {}", @@ -389,18 +497,15 @@ fn git(repository: &Path, args: &[&str]) -> testkit::Result { } fn write(path: &Path, contents: &str) -> testkit::Result { - fs::write(path, contents) - .map_err(|error| Error::msg(format!("write {}: {error}", path.display()))) + fs::write(path, contents).context(format!("write {}", path.display())) } fn create_dir_all(path: &Path) -> testkit::Result { - fs::create_dir_all(path) - .map_err(|error| Error::msg(format!("create {}: {error}", path.display()))) + fs::create_dir_all(path).context(format!("create {}", path.display())) } fn require_text(path: &Path, expected: &str) -> testkit::Result { - let actual = fs::read_to_string(path) - .map_err(|error| Error::msg(format!("read {}: {error}", path.display())))?; + let actual = fs::read_to_string(path).context(format!("read {}", path.display()))?; testkit::ensure!( actual == expected, "{} contains {actual:?}; expected {expected:?}", diff --git a/tests/suites/conformance/src/filesystem_policy.rs b/tests/suites/conformance/src/filesystem_policy.rs new file mode 100644 index 0000000000..79e6937ad8 --- /dev/null +++ b/tests/suites/conformance/src/filesystem_policy.rs @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Filesystem policy suite: Landlock enforcement of a sandbox's filesystem +//! policy. + +use std::collections::HashMap; +use std::time::Duration; + +use testkit::sdk::raw::proto; +use testkit::{Context, ResultExt as _, with_deadline}; + +/// A rejected launch must close promptly instead of hanging the exec. +const REJECTED_LAUNCH_TIMEOUT: Duration = Duration::from_secs(20); + +/// The filesystem policy the tests start from, with `extra_read_write` added +/// to the writable paths. It names no device paths; the sandbox adds the ones +/// every process needs, such as `/dev/urandom`. +/// +/// It sets no process identity, so the workload runs as the image's `USER`: +/// Landlock applies whatever the identity, and the test image need not have a +/// particular user. +fn policy(compatibility: &str, extra_read_write: &[&str]) -> proto::SandboxPolicy { + let paths = |paths: &[&str]| paths.iter().map(ToString::to_string).collect::>(); + let mut read_write = paths(&["/sandbox"]); + read_write.extend(paths(extra_read_write)); + read_write.push("/tmp".to_string()); + proto::SandboxPolicy { + version: 1, + filesystem: Some(proto::FilesystemPolicy { + include_workdir: true, + read_only: paths(&["/usr", "/lib", "/etc", "/proc"]), + read_write, + }), + landlock: Some(proto::LandlockPolicy { + compatibility: compatibility.to_string(), + }), + network_policies: HashMap::from([( + "landlock_regression".to_string(), + proto::NetworkPolicyRule { + name: "landlock_regression".to_string(), + endpoints: vec![proto::NetworkEndpoint { + host: "example.com".to_string(), + port: 443, + ..proto::NetworkEndpoint::default() + }], + binaries: vec![proto::NetworkBinary { + path: "/**".to_string(), + }], + }, + )]), + ..proto::SandboxPolicy::default() + } +} + +/// Under `hard_requirement`, the device paths the sandbox adds to the policy +/// don't make Landlock setup fail: processes start, read `/dev/urandom`, and +/// write to a writable path. +#[testkit::test] +async fn hard_requirement_accepts_added_device_paths(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("ld") + .command(["sleep", "infinity"]) + .policy(policy("hard_requirement", &[])) + .create_ready() + .await?; + + sandbox + .exec([ + "sh", + "-c", + "set -eu; \ + bytes=$(head -c 16 /dev/urandom | wc -c); \ + test \"$bytes\" -eq 16; \ + printf landlock-ok > /tmp/landlock-check; \ + test \"$(cat /tmp/landlock-check)\" = landlock-ok", + ]) + .await? + .success() + .context("the hard_requirement sandbox must read /dev/urandom and write /tmp")?; + Ok(()) +} + +/// When a workload replaces a policy path, or one of its ancestors, with a +/// symlink, the next launch is rejected rather than following the symlink, +/// in both Landlock compatibility modes. +#[testkit::test] +async fn replaced_policy_path_rejects_next_launch(cx: &mut Context) -> testkit::Result { + const LEAF: &str = "/sandbox/policy-parent/leaf"; + let cases = [ + ("lbp", "best_effort", false), + ("lhp", "hard_requirement", false), + ("lba", "best_effort", true), + ("lha", "hard_requirement", true), + ]; + + let mut sandboxes = Vec::new(); + for (suffix, compatibility, ancestor) in cases { + let sandbox = cx + .sandbox(suffix) + .command(["sleep", "infinity"]) + .policy(policy(compatibility, &[LEAF])) + .create() + .await?; + sandboxes.push((sandbox, compatibility, ancestor)); + } + + for (sandbox, compatibility, ancestor) in &sandboxes { + let case = format!( + "{compatibility}, replacing the {}", + if *ancestor { "ancestor" } else { "path" } + ); + sandbox.wait_ready().await?; + for script in [ + format!("mkdir -p {LEAF}"), + // A normal directory stays usable on the next launch. + format!("echo allowed > {LEAF}/check"), + if *ancestor { + "mv /sandbox/policy-parent /sandbox/original-parent; \ + ln -s /var/tmp /sandbox/policy-parent" + .to_string() + } else { + format!("mv {LEAF} /sandbox/original-leaf; ln -s /var/tmp {LEAF}") + }, + ] { + sandbox + .exec(["sh", "-c", script.as_str()]) + .await? + .success() + .context(format!("({case}) prepare the policy path"))?; + } + + // The exec's own error is the rejection report, so keep it apart from + // the deadline's. + let next = with_deadline( + &format!("({case}) the rejected launch"), + REJECTED_LAUNCH_TIMEOUT, + async { Ok(sandbox.exec(["true"]).await) }, + ) + .await?; + let report = match next { + Ok(output) if output.exit_code == 0 => { + return Err(testkit::Error::msg(format!( + "({case}) a launch succeeded after a policy path was replaced with a symlink" + ))); + } + Ok(output) => output.stderr, + Err(error) => error.to_string(), + }; + testkit::ensure!( + report.to_lowercase().contains("symlink"), + "({case}) the rejected launch does not say a policy path is a symlink: {report}" + ); + } + Ok(()) +} diff --git a/tests/suites/conformance/src/image.rs b/tests/suites/conformance/src/image.rs new file mode 100644 index 0000000000..58fb294253 --- /dev/null +++ b/tests/suites/conformance/src/image.rs @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox image suite: a sandbox runs the image it is created from. + +use testkit::{Context, Sandbox}; + +/// A sandbox created with an explicit image runs that image. +#[testkit::test] +async fn explicit_image_is_used(cx: &mut Context) -> testkit::Result { + let image = testkit::base_image(); + let sandbox = cx + .sandbox("ei") + .command(["sleep", "infinity"]) + .image(image.clone()) + .create_ready() + .await?; + expect_ubuntu(&sandbox) + .await + .map_err(|error| error.context(format!("sandbox from image {image}"))) +} + +/// Fail unless the sandbox's `/etc/os-release` names Ubuntu and a version, +/// as OpenShell's base image does. +pub(crate) async fn expect_ubuntu(sandbox: &Sandbox) -> testkit::Result { + let output = sandbox.exec(["cat", "/etc/os-release"]).await?.success()?; + let field = |key: &str| { + output.stdout.lines().find_map(|line| { + line.strip_prefix(key) + .and_then(|rest| rest.strip_prefix('=')) + .map(|value| value.trim_matches('"')) + }) + }; + testkit::ensure!( + field("ID") == Some("ubuntu"), + "/etc/os-release has ID {:?}; expected \"ubuntu\":\n{}", + field("ID"), + output.stdout + ); + testkit::ensure!( + field("VERSION_ID").is_some_and(|version| !version.is_empty()), + "/etc/os-release has no VERSION_ID:\n{}", + output.stdout + ); + Ok(()) +} diff --git a/tests/suites/conformance/src/labels.rs b/tests/suites/conformance/src/labels.rs new file mode 100644 index 0000000000..fb9d9082f0 --- /dev/null +++ b/tests/suites/conformance/src/labels.rs @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox labels suite: labels are stored and select sandboxes in list. + +use std::collections::{BTreeSet, HashMap}; + +use testkit::Context; +use testkit::sdk::ListOptions; + +/// Labels given at create are returned by get, and list filters by one label +/// or by several (all must match). Label values are unique to the run, so +/// other tests' sandboxes can't match. +#[testkit::test] +async fn labels_are_stored_and_filter_list(cx: &mut Context) -> testkit::Result { + let run = cx.run_id().to_string(); + let dev = format!("dev-{run}"); + let staging = format!("staging-{run}"); + let backend = format!("backend-{run}"); + let data = format!("data-{run}"); + let labeled = async |suffix: &str, env: &str, team: &str| -> testkit::Result { + let labels = HashMap::from([ + ("env".to_string(), env.to_string()), + ("team".to_string(), team.to_string()), + ]); + let sandbox = cx + .sandbox(suffix) + .spec(|spec| spec.labels = labels.clone()) + .create() + .await?; + let got = sandbox.get().await?; + testkit::ensure!( + labels + .iter() + .all(|(key, value)| got.labels.get(key) == Some(value)), + "sandbox {} has labels {:?}; expected {labels:?}", + sandbox.name(), + got.labels + ); + Ok(sandbox.name().to_string()) + }; + let dev_backend = labeled("ldb", &dev, &backend).await?; + let staging_backend = labeled("lsb", &staging, &backend).await?; + let dev_data = labeled("ldd", &dev, &data).await?; + + let checks = [ + (format!("env={dev}"), vec![&dev_backend, &dev_data]), + ( + format!("team={backend}"), + vec![&dev_backend, &staging_backend], + ), + (format!("env={dev},team={backend}"), vec![&dev_backend]), + (format!("team=qa-{run}"), vec![]), + ]; + for (selector, expected) in checks { + let listed = listed_names( + cx, + ListOptions { + label_selector: Some(selector.clone()), + ..ListOptions::default() + }, + ) + .await?; + let expected: BTreeSet = expected.into_iter().cloned().collect(); + testkit::ensure!( + listed == expected, + "list with selector {selector:?} returned {listed:?}; expected {expected:?}" + ); + } + + let all = listed_names(cx, ListOptions::default()).await?; + let missing: Vec<&String> = [&dev_backend, &staging_backend, &dev_data] + .into_iter() + .filter(|name| !all.contains(*name)) + .collect(); + testkit::ensure!( + missing.is_empty(), + "list without a selector is missing {missing:?}" + ); + Ok(()) +} + +/// The names of the sandboxes `list` returns, across all pages. +async fn listed_names(cx: &Context, options: ListOptions) -> testkit::Result> { + Ok(cx + .client() + .list_all_sandboxes(options) + .await? + .into_iter() + .map(|sandbox| sandbox.name) + .collect()) +} diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 00a4d45027..f0a568679c 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -8,9 +8,19 @@ //! `#[testkit::test]` also registers them, so the `openshell-conformance` //! program in `src/main.rs` runs them by linking this library. +mod attach; mod cli; +mod credential_gating; +mod exec_cli; mod file_transfer; +mod filesystem_policy; +mod image; +mod labels; mod lifecycle; mod persistent_workspace; mod policy_advisor; +mod process; +mod settings; mod smoke; +mod ssh; +mod templates; diff --git a/tests/suites/conformance/src/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs index dd47c0b4d6..41ec709f06 100644 --- a/tests/suites/conformance/src/lifecycle.rs +++ b/tests/suites/conformance/src/lifecycle.rs @@ -9,8 +9,8 @@ use std::time::Duration; -use testkit::Context; use testkit::sdk::{SandboxPhase, SandboxRestartPolicy}; +use testkit::{Context, Poll, ResultExt as _, Sandbox, poll_until, with_deadline}; /// A long-running sandbox becomes ready, runs a command with exec, and is gone /// from get and list after delete. @@ -19,27 +19,54 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("lr") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?; - testkit::ensure!( - output.exit_code == 0, - "exec exited with {}; stderr: {}", - output.exit_code, - output.stderr - ); - testkit::ensure!( - output.stdout == format!("{marker}\n"), - "exec stdout was {:?}; expected {:?}", - output.stdout, - format!("{marker}\n") - ); + sandbox + .exec(["echo", marker.as_str()]) + .await? + .expect_stdout(&format!("{marker}\n"))?; sandbox.delete().await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await +} + +/// Longer than the supervisor's 30s startup recovery deadline. +const PAST_RECOVERY_DEADLINE: Duration = Duration::from_secs(31); +/// The exec's own timeout is the runner's two-minute kill, so fail sooner, +/// with a message that says which step hung. +const PAST_RECOVERY_EXEC_DEADLINE: Duration = Duration::from_mins(1); + +/// An exec admitted before the startup recovery deadline expires keeps +/// running past it, and a new exec still works afterwards. +#[testkit::test] +async fn exec_outlives_startup_recovery_deadline(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("rd") + .command(["sleep", "infinity"]) + .create_ready() + .await?; + + let script = format!( + "sleep {}; printf past-recovery-deadline", + PAST_RECOVERY_DEADLINE.as_secs() + ); + with_deadline( + "the exec past the recovery deadline", + PAST_RECOVERY_EXEC_DEADLINE, + sandbox.exec(["sh", "-c", script.as_str()]), + ) + .await? + .expect_stdout("past-recovery-deadline") + .context("the recovery deadline must not end an admitted exec")?; + + sandbox + .exec(["printf", "fresh-exec"]) + .await? + .expect_stdout("fresh-exec") + .context("exec after the recovery deadline")?; + Ok(()) } /// Long enough that the workload is still handling TERM when sandbox shutdown @@ -71,45 +98,31 @@ async fn delete_is_prompt_when_workload_is_slow_to_handle_term( let sandbox = cx .sandbox("dt") .command(["sh", "-c", script.as_str()]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; // Ready alone does not prove that the workload has installed its trap. - testkit::poll_until( - "workload TERM handler readiness", - Duration::from_secs(10), - Duration::from_millis(100), - async || { - let output = sandbox - .exec(["sh", "-c", "test -f /tmp/delayed-term-ready"]) - .await?; - Ok(if output.exit_code == 0 { - testkit::Poll::Ready(()) - } else { - testkit::Poll::Pending(format!( - "readiness check exited {}; stderr: {}", - output.exit_code, output.stderr - )) - }) - }, - ) - .await?; + sandbox + .poll_exec( + "workload TERM handler readiness", + Duration::from_secs(10), + ["test", "-f", "/tmp/delayed-term-ready"], + |output| output.exit_code == 0, + ) + .await?; // Use one deadline starting before delete, not a fresh timeout per step. // Boundary containment may kill the workload before its handler finishes; // this asserts bounded deletion, not a graceful workload exit. - tokio::time::timeout(DELETE_DEADLINE, async { - sandbox.delete().await?; - cx.expect_absent(&sandbox).await - }) + with_deadline( + &format!("deleting sandbox {}", sandbox.name()), + DELETE_DEADLINE, + async { + sandbox.delete().await?; + sandbox.expect_absent().await + }, + ) .await - .map_err(|_| { - testkit::Error::msg(format!( - "sandbox {} was not fully deleted within {DELETE_DEADLINE:?}", - sandbox.name() - )) - })? } /// A sandbox whose main command exits 0, with restart policy Never, reaches @@ -152,7 +165,68 @@ async fn run_to_completion_exit_delete(cx: &mut Context) -> testkit::Result { ); sandbox.delete().await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await +} + +/// A main command that exits during provisioning is reported as the +/// workload's result, not as a provisioning failure: the create succeeds and +/// the sandbox reaches Error with the command's exit code. +#[testkit::test] +async fn main_exit_during_provisioning_is_workload_result(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("pe") + .command(["sh", "-c", "exit 11"]) + .restart_policy(SandboxRestartPolicy::Never) + .create() + .await?; + + let finished = sandbox.wait_finished().await?; + testkit::ensure!( + finished.phase == SandboxPhase::Error && finished.exit_code == Some(11), + "sandbox finished in phase {:?} with exit code {:?}; expected Error with Some(11)", + finished.phase, + finished.exit_code + ); + Ok(()) +} + +/// A main command that exits after the sandbox is Ready leaves its result: +/// exit 0 reaches Completed, and a non-zero exit reaches Error with that code. +#[testkit::test] +async fn main_exit_after_ready_sets_final_phase(cx: &mut Context) -> testkit::Result { + const RELEASE_PATH: &str = "/sandbox/.openshell-main-release"; + let cases = [ + ("mz", 0, SandboxPhase::Completed), + ("mn", 11, SandboxPhase::Error), + ]; + + // Create every sandbox before waiting, so they provision concurrently. + let mut sandboxes = Vec::new(); + for (suffix, code, phase) in cases { + let script = + format!("while [ ! -e '{RELEASE_PATH}' ]; do sleep 0.05; done; sleep 1; exit {code}"); + let sandbox = cx + .sandbox(suffix) + .command(["sh", "-c", script.as_str()]) + .restart_policy(SandboxRestartPolicy::Never) + .create() + .await?; + sandboxes.push((sandbox, code, phase)); + } + + for (sandbox, code, phase) in &sandboxes { + sandbox.wait_ready().await?; + sandbox.exec(["touch", RELEASE_PATH]).await?.success()?; + let finished = sandbox.wait_finished().await?; + testkit::ensure!( + finished.phase == *phase && finished.exit_code == Some(*code), + "main exiting {code} left sandbox {} in phase {:?} with exit code {:?}; expected {phase:?} with Some({code})", + sandbox.name(), + finished.phase, + finished.exit_code + ); + } + Ok(()) } /// A sandbox created with the gateway's default command becomes Ready, is @@ -167,20 +241,18 @@ async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::R ready.name, sandbox.name() ); - cx.expect_listed(&sandbox, SandboxPhase::Ready).await?; + sandbox.expect_listed(SandboxPhase::Ready).await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?; - testkit::ensure!( - output.exit_code == 0 && output.stdout == format!("{marker}\n"), - "exec exited {} with stdout {:?}; stderr: {}", - output.exit_code, - output.stdout, - output.stderr - ); + sandbox + .exec(["echo", marker.as_str()]) + .await? + .expect_stdout(&format!("{marker}\n"))?; + // The gateway's default image is OpenShell's Ubuntu base image. + crate::image::expect_ubuntu(&sandbox).await?; sandbox.delete().await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await } /// A stopped sandbox rejects exec, accepts a second stop, and returns to @@ -190,18 +262,15 @@ async fn restart_stopped_sandbox(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("rs") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; sandbox.stop().await?; - match sandbox.exec(["true"]).await { - Err(_) => {} - Ok(output) => testkit::ensure!( - output.exit_code != 0, - "exec succeeded while the sandbox was stopped" - ), - } + let rejected = sandbox.exec(["true"]).await; + testkit::ensure!( + !matches!(&rejected, Ok(output) if output.exit_code == 0), + "exec succeeded while the sandbox was stopped" + ); // Stop is idempotent. let again = sandbox.stop().await?; testkit::ensure!( @@ -211,14 +280,11 @@ async fn restart_stopped_sandbox(cx: &mut Context) -> testkit::Result { ); sandbox.start().await?; - let output = sandbox.exec(["echo", "started"]).await?; - testkit::ensure!( - output.exit_code == 0 && output.stdout == "started\n", - "exec after start exited {} with stdout {:?}; stderr: {}", - output.exit_code, - output.stdout, - output.stderr - ); + sandbox + .exec(["echo", "started"]) + .await? + .expect_stdout("started\n") + .context("exec after start")?; Ok(()) } @@ -228,10 +294,232 @@ async fn stopped_sandbox_can_be_deleted(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("sd") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; sandbox.stop().await?; sandbox.delete().await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await +} + +/// Where the main command of [`canonical_main_environment`] records what it +/// sees. +const MAIN_ENV_PATH: &str = "/sandbox/.openshell-main-env"; + +/// Declared environment reaches both the main command and exec, with and +/// without a TTY. A TTY main command also gets the sandbox user's `HOME` and +/// `USER`, not the supervisor's, and a usable `TERM`. +#[testkit::test] +async fn canonical_main_environment(cx: &mut Context) -> testkit::Result { + let declared = format!("present-{}", cx.run_id()); + let script = format!( + "printf 'declared=%s\\nhome=%s\\nuser=%s\\nterm=%s\\n' \ + \"${{REPRO_SENTINEL:-missing}}\" \"$HOME\" \"$USER\" \"$TERM\" > '{MAIN_ENV_PATH}'; \ + exec sleep infinity" + ); + + let mut sandboxes = Vec::new(); + for (suffix, tty) in [("et", true), ("en", false)] { + let sandbox = cx + .sandbox(suffix) + .command(["sh", "-c", script.as_str()]) + .spec(|spec| { + spec.tty = tty; + spec.environment + .insert("REPRO_SENTINEL".to_string(), declared.clone()); + }) + .create() + .await?; + sandboxes.push((sandbox, tty)); + } + + for (sandbox, tty) in &sandboxes { + sandbox.wait_ready().await?; + let main = main_environment(sandbox).await?; + let field = |name: &str| main.get(name).map_or("", String::as_str); + testkit::ensure!( + field("declared") == declared, + "the main command (tty {tty}) saw REPRO_SENTINEL={:?}; expected {declared:?}", + field("declared") + ); + + sandbox + .exec(["sh", "-c", "printf '%s' \"${REPRO_SENTINEL:-missing}\""]) + .await? + .expect_stdout(&declared) + .context(format!("REPRO_SENTINEL in exec (tty {tty})"))?; + + if *tty { + testkit::ensure!( + !field("home").is_empty() && field("home") != "/root", + "the TTY main command inherited the supervisor's HOME: {main:?}" + ); + testkit::ensure!( + !field("user").is_empty(), + "the TTY main command has no USER: {main:?}" + ); + testkit::ensure!( + !field("term").is_empty() && field("term") != "dumb", + "the TTY main command has no usable TERM: {main:?}" + ); + } + } + Ok(()) +} + +/// The `key=value` lines the main command wrote to [`MAIN_ENV_PATH`]. The +/// sandbox can be Ready before the main command writes them. +async fn main_environment( + sandbox: &Sandbox, +) -> testkit::Result> { + let output = sandbox + .poll_exec( + &format!( + "the main command of {} to record its environment", + sandbox.name() + ), + Duration::from_secs(30), + ["cat", MAIN_ENV_PATH], + |output| output.exit_code == 0 && output.stdout.contains("term="), + ) + .await?; + Ok(output + .stdout + .lines() + .filter_map(|line| line.split_once('=')) + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect()) +} + +/// How long a restart under the on-failure policy may take to be reported. +/// The runner kills a test after two minutes, so the test's deadlines must +/// add up to less. +const RESTART_TIMEOUT: Duration = Duration::from_secs(45); +/// How long a running main command may take to write its workspace marker. +const MARKER_TIMEOUT: Duration = Duration::from_secs(15); + +/// Under the on-failure policy, a main command that fails is replaced by a new +/// instance of it. The workspace survives the restart, the restart is counted, +/// and the policy is still reported afterwards. +#[testkit::test] +async fn on_failure_policy_restarts_main_and_keeps_workspace(cx: &mut Context) -> testkit::Result { + const SCRIPT: &str = r#" +marker=/sandbox/.openshell-restart-e2e +if [ -e "$marker" ]; then + printf 'replacement-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/replacement-run + sleep 300 +else + touch "$marker" + printf 'initial-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/initial-run + while [ ! -e /sandbox/.openshell-restart-release ]; do sleep 0.05; done + exit 17 +fi +"#; + let sandbox = cx + .sandbox("of") + .command(["sh", "-c", SCRIPT]) + .restart_policy(SandboxRestartPolicy::OnFailure) + .create_ready() + .await?; + sandbox + .poll_exec( + "the initial main command to write its workspace marker", + MARKER_TIMEOUT, + ["cat", "/sandbox/initial-run"], + |output| output.stdout.contains("initial-"), + ) + .await?; + + let initial = sandbox.get_json().await?; + let initial_instance = main_instance(&initial).to_string(); + testkit::ensure!( + !initial_instance.is_empty(), + "sandbox get reports no main process instance: {initial}" + ); + sandbox + .exec(["touch", "/sandbox/.openshell-restart-release"]) + .await? + .success()?; + + let restarted = poll_until( + "the on-failure restart", + RESTART_TIMEOUT, + Duration::from_millis(250), + async || { + let details = sandbox.get_json().await?; + let instance = main_instance(&details); + let replaced = !instance.is_empty() && instance != initial_instance; + Ok( + if details["phase"] == "Ready" && details["restart_count"] == 1 && replaced { + Poll::Ready(details) + } else { + Poll::Pending(format!( + "phase {}, restart count {}, main process instance {instance:?}", + details["phase"], details["restart_count"] + )) + }, + ) + }, + ) + .await?; + sandbox + .poll_exec( + "the replacement main command to find the workspace and write its marker", + MARKER_TIMEOUT, + ["cat", "/sandbox/initial-run", "/sandbox/replacement-run"], + |output| output.stdout.contains("initial-") && output.stdout.contains("replacement-"), + ) + .await?; + testkit::ensure!( + restarted["restart_policy"] == "on-failure", + "the restart policy after the restart is {}; expected \"on-failure\"", + restarted["restart_policy"] + ); + Ok(()) +} + +/// The ID of the sandbox's current main process instance, from +/// [`Sandbox::get_json`]. Empty if none is reported. +fn main_instance(details: &serde_json::Value) -> &str { + details["main_process_instance_id"] + .as_str() + .unwrap_or_default() +} + +/// How long the gateway may take to remove an ephemeral sandbox after its +/// main command exits. Both sandboxes are released before the test waits, so +/// their removals overlap. +const EPHEMERAL_CLEANUP_TIMEOUT: Duration = Duration::from_secs(60); + +/// A detached sandbox created with `--no-keep` is removed once its main +/// command exits, whether the command succeeds or fails. +#[testkit::test] +async fn detached_no_keep_sandbox_is_removed_after_main_exits(cx: &mut Context) -> testkit::Result { + let mut sandboxes = Vec::new(); + for (suffix, code) in [("ez", 0), ("ef", 17)] { + let builder = cx.cli_sandbox(suffix); + let release_path = format!("/sandbox/.ephemeral-release-{}", builder.name()); + let script = + format!("while [ ! -e '{release_path}' ]; do sleep 0.1; done; sleep 1; exit {code}"); + let sandbox = builder + .args(["--no-keep"]) + .command(["sh", "-c", &script]) + .create() + .await?; + sandboxes.push((sandbox, release_path, code)); + } + + for (sandbox, release_path, _) in &sandboxes { + sandbox.wait_ready().await?; + sandbox + .exec(["touch", release_path.as_str()]) + .await? + .success()?; + } + for (sandbox, _, code) in &sandboxes { + sandbox + .wait_absent(EPHEMERAL_CLEANUP_TIMEOUT) + .await + .context(format!("ephemeral sandbox exited {code}"))?; + } + Ok(()) } diff --git a/tests/suites/conformance/src/persistent_workspace.rs b/tests/suites/conformance/src/persistent_workspace.rs index de2940e571..1b123fe618 100644 --- a/tests/suites/conformance/src/persistent_workspace.rs +++ b/tests/suites/conformance/src/persistent_workspace.rs @@ -10,7 +10,7 @@ use std::time::Duration; -use testkit::{Context, Poll}; +use testkit::{Context, ResultExt as _}; /// Files in the workspace survive stop and start, and start runs the main /// command again. @@ -26,50 +26,37 @@ async fn persistent_workspace_survives_stop_start(cx: &mut Context) -> testkit:: let sandbox = cx .sandbox("kw") .command(["sh", "-c", main.as_str()]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let sentinel = format!("openshell-stop-start-{}", cx.run_id()); - let write = sandbox + sandbox .exec([ "sh", "-c", &format!("printf '%s\\n' '{sentinel}' > '{SENTINEL}' && sync"), ]) - .await?; - testkit::ensure!( - write.exit_code == 0, - "writing the sentinel exited {}; stderr: {}", - write.exit_code, - write.stderr - ); + .await? + .success() + .context("write the sentinel")?; sandbox.stop().await?; sandbox.start().await?; - let read = sandbox.exec(["cat", SENTINEL]).await?; - testkit::ensure!( - read.stdout == format!("{sentinel}\n"), - "sentinel after restart is {:?}; stderr: {}", - read.stdout, - read.stderr - ); + sandbox + .exec(["cat", SENTINEL]) + .await? + .expect_stdout(&format!("{sentinel}\n")) + .context("read the sentinel after restart")?; // The sandbox is Ready before the restarted main command has necessarily // rewritten the count, so wait for it to say it ran again. - testkit::poll_until( - "the main command to run again", - Duration::from_secs(30), - Duration::from_secs(1), - async || { - let runs = sandbox.exec(["cat", RUN_COUNT]).await?; - Ok(if runs.stdout == "2\n" { - Poll::Ready(()) - } else { - Poll::Pending(format!("run count {:?}", runs.stdout.trim())) - }) - }, - ) - .await?; + sandbox + .poll_exec( + "the main command to run again", + Duration::from_secs(30), + ["cat", RUN_COUNT], + |runs| runs.stdout == "2\n", + ) + .await?; Ok(()) } diff --git a/tests/suites/conformance/src/policy_advisor.rs b/tests/suites/conformance/src/policy_advisor.rs index 003917163c..82ff54cc09 100644 --- a/tests/suites/conformance/src/policy_advisor.rs +++ b/tests/suites/conformance/src/policy_advisor.rs @@ -13,7 +13,7 @@ use std::time::{Duration, Instant}; use serde_json::Value; use testkit::sdk::raw::proto; -use testkit::{Context, Error, Sandbox}; +use testkit::{Context, Error, ResultExt as _, Sandbox}; const READY_TIMEOUT: Duration = Duration::from_mins(4); const PROPOSAL_TIMEOUT: Duration = Duration::from_secs(90); @@ -161,12 +161,13 @@ async fn denied_address_produces_draft_rule(cx: &mut Context) -> testkit::Result "printf 'BINARY=%s\\n' \"$(readlink -f /proc/$$/exe)\"; \ if exec 3<>/dev/tcp/1.1.1.1/443; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi", ]) - .await?; + .await? + .success() + .context("the connection to 1.1.1.1:443 must be denied")?; testkit::ensure!( - probe.exit_code == 0 && probe.stdout.lines().any(|line| line == "DENIED"), - "the connection to 1.1.1.1:443 was not denied: stdout {:?}, stderr {:?}", - probe.stdout, - probe.stderr + probe.stdout.lines().any(|line| line == "DENIED"), + "the connection to 1.1.1.1:443 was not denied: stdout {:?}", + probe.stdout ); let binary = probe .stdout @@ -199,12 +200,13 @@ async fn denied_hostname_produces_draft_rule(cx: &mut Context) -> testkit::Resul "-c", "if exec 3<>/dev/tcp/pypi.org/80; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi", ]) - .await?; + .await? + .success() + .context("the connection to pypi.org:80 must be denied")?; testkit::ensure!( - probe.exit_code == 0 && probe.stdout.lines().any(|line| line == "DENIED"), - "the connection to pypi.org:80 was not denied: stdout {:?}, stderr {:?}", - probe.stdout, - probe.stderr + probe.stdout.lines().any(|line| line == "DENIED"), + "the connection to pypi.org:80 was not denied: stdout {:?}", + probe.stdout ); expect_draft( @@ -228,8 +230,7 @@ async fn create( if let Some(policy) = policy { builder = builder.policy(policy); } - let sandbox = builder.create().await?; - sandbox.wait_ready().await?; + let sandbox = builder.create_ready().await?; Ok(sandbox) } @@ -289,13 +290,13 @@ async fn bash_path(sandbox: &Sandbox) -> testkit::Result { "-c", "printf '%s\\n' \"$(readlink -f /proc/$$/exe)\"", ]) - .await?; + .await? + .success()?; let path = output.stdout.trim(); testkit::ensure!( - output.exit_code == 0 && is_bash_path(path), - "expected one absolute Bash path; got {:?}, stderr {:?}", - output.stdout, - output.stderr + is_bash_path(path), + "expected one absolute Bash path; got {:?}", + output.stdout ); Ok(path.to_string()) } @@ -326,13 +327,9 @@ async fn policy_local( path, body, ]) - .await?; - testkit::ensure!( - output.exit_code == 0, - "{method} http://policy.local{path} exited {}; stderr: {}", - output.exit_code, - output.stderr - ); + .await? + .success() + .context(format!("{method} http://policy.local{path}"))?; let (headers, json) = output.stdout.split_once("\r\n\r\n").ok_or_else(|| { Error::msg(format!( "{method} http://policy.local{path} returned no complete HTTP response: {:?}", diff --git a/tests/suites/conformance/src/process.rs b/tests/suites/conformance/src/process.rs new file mode 100644 index 0000000000..1f7e9d1eae --- /dev/null +++ b/tests/suites/conformance/src/process.rs @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox process suite: limits the sandbox applies to its processes. + +use testkit::{Context, ResultExt as _}; + +/// Sandbox processes can't write core dumps: both the soft and the hard core +/// file size limits are 0, so a process can't raise its own limit either. +#[testkit::test] +async fn core_dumps_are_disabled(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("cd") + .command(["sleep", "infinity"]) + .create_ready() + .await?; + + sandbox + .exec([ + "sh", + "-c", + "printf '%s %s' \"$(ulimit -c)\" \"$(ulimit -Hc)\"", + ]) + .await? + .expect_stdout("0 0") + .context("core file size limits (soft hard)")?; + Ok(()) +} diff --git a/tests/suites/conformance/src/settings.rs b/tests/suites/conformance/src/settings.rs new file mode 100644 index 0000000000..c667663d16 --- /dev/null +++ b/tests/suites/conformance/src/settings.rs @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Settings suite: sandbox-scoped settings and the gateway-global settings +//! that override them. +//! +//! The global test changes a setting for the whole gateway. That is safe +//! because the runner runs one test at a time; a family that runs tests +//! concurrently needs to give it a test group of its own. + +use std::time::Duration; + +use serde_json::Value; +use testkit::{Context, Poll, Sandbox, poll_until}; + +/// A registered boolean setting that does not change what these tests do. +const KEY: &str = "ocsf_json_enabled"; +/// How the CLI shows a setting with no value. +const UNSET: &str = ""; + +/// With no global value, a sandbox setting starts unset, reads back after it +/// is set, and is unset again after it is deleted. +#[testkit::test] +async fn sandbox_setting_round_trip(cx: &mut Context) -> testkit::Result { + // A global value would hide the sandbox's own, so clear it for the test. + restore_global_setting_on_cleanup(cx, KEY).await?; + delete_global_setting(&cx.cli_handle(), KEY).await?; + let sandbox = cx + .sandbox("sr") + .command(["sleep", "infinity"]) + .create_ready() + .await?; + expect_sandbox_setting(cx, &sandbox, UNSET, "unset").await?; + + set_sandbox_setting(cx, &sandbox, "true").await?.success()?; + expect_sandbox_setting(cx, &sandbox, "true", "sandbox").await?; + + cx.cli(["settings", "delete", sandbox.name(), "--key", KEY]) + .await? + .success()?; + expect_sandbox_setting(cx, &sandbox, UNSET, "unset").await +} + +/// A global setting overrides the sandbox's value and blocks sandbox writes +/// and deletes for that key. Once the global setting is deleted, the sandbox +/// controls the key again. +#[testkit::test] +async fn global_setting_overrides_sandbox_scope(cx: &mut Context) -> testkit::Result { + let cli = cx.cli_handle(); + restore_global_setting_on_cleanup(cx, KEY).await?; + delete_global_setting(&cli, KEY).await?; + + let sandbox = cx + .sandbox("sg") + .command(["sleep", "infinity"]) + .create_ready() + .await?; + set_sandbox_setting(cx, &sandbox, "true").await?.success()?; + expect_sandbox_setting(cx, &sandbox, "true", "sandbox").await?; + + set_global_setting(&cli, KEY, "false").await?; + testkit::ensure!( + read_global_setting(&cli, KEY).await? == "false", + "the global {KEY} does not read back as false" + ); + expect_sandbox_setting(cx, &sandbox, "false", "global").await?; + + let blocked = set_sandbox_setting(cx, &sandbox, "true").await?.failure()?; + testkit::ensure!( + blocked.message().contains("managed globally"), + "setting a globally managed key on the sandbox failed without saying the key is managed globally: {}", + blocked.message() + ); + cx.cli(["settings", "delete", sandbox.name(), "--key", KEY]) + .await? + .failure()?; + + delete_global_setting(&cli, KEY).await?; + let global = read_global_setting(&cli, KEY).await?; + testkit::ensure!( + global == UNSET, + "the global {KEY} is {global:?} after delete; expected {UNSET:?}" + ); + + set_sandbox_setting(cx, &sandbox, "false") + .await? + .success()?; + expect_sandbox_setting(cx, &sandbox, "false", "sandbox").await +} + +async fn set_sandbox_setting( + cx: &Context, + sandbox: &Sandbox, + value: &str, +) -> testkit::Result { + cx.cli([ + "settings", + "set", + sandbox.name(), + "--key", + KEY, + "--value", + value, + ]) + .await +} + +/// Wait until `settings get` for the sandbox shows [`KEY`] with `value` from +/// `scope`. +async fn expect_sandbox_setting( + cx: &Context, + sandbox: &Sandbox, + value: &str, + scope: &str, +) -> testkit::Result { + poll_until( + &format!( + "{KEY} on sandbox {} to be {value} ({scope})", + sandbox.name() + ), + Duration::from_secs(10), + Duration::from_secs(1), + async || { + let settings: Value = cx + .cli(["settings", "get", sandbox.name(), "--json"]) + .await? + .success()? + .json()?; + let setting = &settings["settings"][KEY]; + Ok(if setting["value"] == value && setting["scope"] == scope { + Poll::Ready(()) + } else { + Poll::Pending(format!("{KEY} is {setting}")) + }) + }, + ) + .await +} + +/// The gateway-global value of `key` as the CLI shows it, [`UNSET`] if none. +async fn read_global_setting(cli: &testkit::Cli, key: &str) -> testkit::Result { + let settings: Value = cli + .run(["settings", "get", "--global", "--json"]) + .await? + .success()? + .json()?; + Ok(settings["settings"][key] + .as_str() + .unwrap_or(UNSET) + .to_string()) +} + +async fn set_global_setting(cli: &testkit::Cli, key: &str, value: &str) -> testkit::Result { + cli.run([ + "settings", "set", "--global", "--key", key, "--value", value, "--yes", + ]) + .await? + .success()?; + Ok(()) +} + +async fn delete_global_setting(cli: &testkit::Cli, key: &str) -> testkit::Result { + cli.run(["settings", "delete", "--global", "--key", key, "--yes"]) + .await? + .success()?; + Ok(()) +} + +/// Record the gateway-global value of `key` and put it back after the test: +/// set it again if it had one, delete it otherwise. +async fn restore_global_setting_on_cleanup(cx: &Context, key: &'static str) -> testkit::Result { + let cli = cx.cli_handle(); + let prior = read_global_setting(&cli, key).await?; + testkit::ensure!( + !prior.starts_with(" testkit::Result { let sandbox = cx .sandbox("smoke") .command(["sleep", "infinity"]) - .create() + .create_ready() .await?; - sandbox.wait_ready().await?; let marker = format!("openshell-e2e-{}", cx.run_id()); - let output = sandbox.exec(["echo", marker.as_str()]).await?; - testkit::ensure!( - output.exit_code == 0, - "exec exited with {}; stderr: {}", - output.exit_code, - output.stderr - ); - testkit::ensure!( - output.stdout == format!("{marker}\n"), - "exec stdout was {:?}; expected {:?}", - output.stdout, - format!("{marker}\n") - ); + sandbox + .exec(["echo", marker.as_str()]) + .await? + .expect_stdout(&format!("{marker}\n"))?; sandbox.delete().await?; - cx.expect_absent(&sandbox).await + sandbox.expect_absent().await } diff --git a/tests/suites/conformance/src/ssh.rs b/tests/suites/conformance/src/ssh.rs new file mode 100644 index 0000000000..305d6c706a --- /dev/null +++ b/tests/suites/conformance/src/ssh.rs @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A sandbox's SSH host identity, through `sandbox ssh-config` and a plain +//! `ssh` client that pins the host key. +//! +//! Ported from `e2e/rust/tests/ssh_host_identity.rs`. The check across a +//! gateway restart needs gateway control, which testkit doesn't have yet. + +use std::path::Path; +use std::time::Duration; + +use testkit::sdk::{ListOptions, SandboxPhase}; +use testkit::{Context, ResultExt as _, Sandbox}; + +/// How long one `ssh` command may take. +const SSH_TIMEOUT: Duration = Duration::from_secs(30); +const SSH_OUTPUT: &str = "ssh-identity-ok"; + +/// A sandbox reports one SSH host key fingerprint in get and list, and that +/// key is the one `ssh` sees. The key stays the same across stop and start, +/// so a pinned `known_hosts` keeps working, and a new sandbox with the same +/// name gets a new key. The workload can't read the supervisor's +/// credentials. +/// +/// Needs a named gateway, which `sandbox ssh-config` puts in its +/// `ProxyCommand`, and an `ssh` client on the test host. +#[testkit::test] +async fn ssh_host_identity_stable_across_stop_start_and_new_on_recreate( + cx: &mut Context, +) -> testkit::Result { + let cli = cx.cli_handle(); + testkit::ensure!( + cli.gateway().name.is_some(), + "this test needs a named gateway for `sandbox ssh-config`; set OPENSHELL_GATEWAY" + ); + let sandbox = ready_sandbox(cx).await?; + let name = sandbox.name().to_string(); + let original = fingerprint(cx, &sandbox).await?; + + let directory = tempfile::tempdir().context("create a directory for the SSH config")?; + let config = directory.path().join("ssh_config"); + let known_hosts = directory.path().join("known_hosts"); + let rendered = cx + .cli(["sandbox", "ssh-config", &name]) + .await? + .success()? + .stdout; + let alias = rendered + .lines() + .find_map(|line| line.trim().strip_prefix("Host ")) + .ok_or_else(|| testkit::Error::msg(format!("no Host line in ssh-config: {rendered}")))? + .trim() + .to_string(); + std::fs::write(&config, &rendered).context("write the SSH config")?; + + let ssh = Ssh { + cli: &cli, + config: &config, + known_hosts: &known_hosts, + alias: &alias, + }; + ssh.run("accept-new").await?; + let key = tokio::process::Command::new("ssh-keygen") + .args(["-l", "-E", "sha256", "-f"]) + .arg(&known_hosts) + .output() + .await + .context("run ssh-keygen")?; + let listed = String::from_utf8_lossy(&key.stdout); + testkit::ensure!( + key.status.success() && listed.split_whitespace().any(|word| word == original), + "the host key ssh pinned is not {original}; ssh-keygen printed {:?}; stderr: {}", + listed.trim(), + String::from_utf8_lossy(&key.stderr).trim() + ); + + let read = sandbox + .exec(["sh", "-c", "test ! -r /.openshell/supervisor/auth.json"]) + .await?; + testkit::ensure!( + read.exit_code == 0, + "the workload can read the supervisor's auth.json" + ); + + cx.cli(["sandbox", "stop", &name]).await?.success()?; + sandbox + .wait_phase(SandboxPhase::Stopped, testkit::STOP_TIMEOUT) + .await?; + let stopped = fingerprint(cx, &sandbox).await?; + testkit::ensure!( + stopped == original, + "the fingerprint changed from {original} to {stopped} on stop" + ); + cx.cli(["sandbox", "start", &name]).await?.success()?; + sandbox.wait_ready().await?; + let started = fingerprint(cx, &sandbox).await?; + testkit::ensure!( + started == original, + "the fingerprint changed from {original} to {started} on start" + ); + ssh.run("yes") + .await + .context("ssh with the pinned host key after a restart")?; + + cx.cli(["sandbox", "delete", &name]).await?.success()?; + // The name stays reserved until the gateway has removed the sandbox. + cx.client() + .wait_deleted(&name, testkit::DELETE_TIMEOUT, Some(sandbox.id())) + .await + .context(format!("wait for sandbox {name} to be deleted"))?; + let replacement = ready_sandbox(cx).await?; + let replaced = fingerprint(cx, &replacement).await?; + testkit::ensure!( + replaced != original, + "a new sandbox named {name} has the deleted one's host key {original}" + ); + Ok(()) +} + +async fn ready_sandbox(cx: &Context) -> testkit::Result { + cx.sandbox("hk") + .command(["sleep", "infinity"]) + .create_ready() + .await +} + +/// The sandbox's host key fingerprint, the same in get and list. +async fn fingerprint(cx: &Context, sandbox: &Sandbox) -> testkit::Result { + let name = sandbox.name(); + let fingerprint = sandbox.get().await?.host_key_fingerprint.ok_or_else(|| { + testkit::Error::msg(format!("sandbox {name} has no host key fingerprint")) + })?; + testkit::ensure!( + fingerprint.starts_with("SHA256:"), + "sandbox {name} has fingerprint {fingerprint:?}; expected SHA256:..." + ); + let listed = cx + .client() + .list_all_sandboxes(ListOptions::default()) + .await + .context("list sandboxes")? + .into_iter() + .find(|listed| listed.name == name) + .ok_or_else(|| testkit::Error::msg(format!("sandbox {name} is not listed")))?; + testkit::ensure!( + listed.host_key_fingerprint.as_deref() == Some(fingerprint.as_str()), + "sandbox {name} is listed with fingerprint {:?}; get says {fingerprint}", + listed.host_key_fingerprint + ); + Ok(fingerprint) +} + +/// A plain `ssh` client using the config from `sandbox ssh-config` and its +/// own `known_hosts`. +struct Ssh<'a> { + cli: &'a testkit::Cli, + config: &'a Path, + known_hosts: &'a Path, + alias: &'a str, +} + +impl Ssh<'_> { + /// Run a command over SSH with `StrictHostKeyChecking` set to `strict`. + async fn run(&self, strict: &str) -> testkit::Result { + let mut ssh = tokio::process::Command::new("ssh"); + ssh.kill_on_drop(true) + .arg("-F") + .arg(self.config) + .arg("-o") + .arg(format!("StrictHostKeyChecking={strict}")) + .arg("-o") + .arg(format!("UserKnownHostsFile={}", self.known_hosts.display())) + .args(["-o", "BatchMode=yes", self.alias, "printf", SSH_OUTPUT]); + // The ProxyCommand runs the CLI, which must reach the same gateway. + self.cli.apply_env(&mut ssh); + let output = tokio::time::timeout(SSH_TIMEOUT, ssh.output()) + .await + .map_err(|_| testkit::Error::msg(format!("ssh timed out after {SSH_TIMEOUT:?}")))? + .context("run ssh; the test host needs an ssh client")?; + let stdout = String::from_utf8_lossy(&output.stdout); + testkit::ensure!( + output.status.success() && stdout == SSH_OUTPUT, + "ssh to {} exited with {:?} and printed {stdout:?}; stderr: {}", + self.alias, + output.status.code(), + String::from_utf8_lossy(&output.stderr).trim() + ); + Ok(()) + } +} diff --git a/tests/suites/conformance/src/templates.rs b/tests/suites/conformance/src/templates.rs new file mode 100644 index 0000000000..9e6e2196d0 --- /dev/null +++ b/tests/suites/conformance/src/templates.rs @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox templates suite: reusable workload templates and sandboxes created +//! from them. + +use std::collections::HashMap; + +use testkit::sdk::raw::proto::datamodel::v1::ObjectMeta; +use testkit::sdk::{ + DeleteOptions, SandboxResources, SandboxTemplateCreateSpec, SandboxTemplateListOptions, + SandboxWorkloadConfig, SandboxWorkloadTemplate, SandboxWorkloadTemplateSpec, SdkError, +}; +use testkit::{Context, ResultExt as _}; + +/// Create template `name`, and delete it after the test. The cleanup is +/// registered before the create, so it also covers a create the gateway +/// applied but reported as failed. +async fn create_template( + cx: &Context, + name: &str, + labels: HashMap, + workload: SandboxWorkloadConfig, +) -> Result { + let client = cx.client().clone(); + let owned = name.to_string(); + cx.on_cleanup(format!("delete sandbox template {name}"), async move || { + client + .delete_sandbox_template( + &owned, + DeleteOptions { + allow_missing: true, + }, + ) + .await?; + Ok(()) + }); + cx.client() + .create_sandbox_template(SandboxWorkloadTemplate { + metadata: Some(ObjectMeta { + name: name.to_string(), + labels, + ..ObjectMeta::default() + }), + spec: Some(SandboxWorkloadTemplateSpec { + workload: Some(workload), + ..SandboxWorkloadTemplateSpec::default() + }), + }) + .await +} + +/// A template stores its labels, environment and resources, is listed, and a +/// sandbox created from it runs with its environment and records which +/// template it came from. +#[testkit::test] +async fn sandbox_from_template_uses_its_workload(cx: &mut Context) -> testkit::Result { + let template_name = cx.name("tp"); + let labels = HashMap::from([("e2e".to_string(), "sandbox-template".to_string())]); + let workload = SandboxWorkloadConfig { + environment: HashMap::from([("FEATURE_FLAG".to_string(), "on".to_string())]), + resources: Some(SandboxResources { + cpu: "500m".to_string(), + memory: "512Mi".to_string(), + ..SandboxResources::default() + }), + ..SandboxWorkloadConfig::default() + }; + create_template(cx, &template_name, labels.clone(), workload.clone()).await?; + + let stored = cx.client().get_sandbox_template(&template_name).await?; + let meta = stored.metadata.clone().unwrap_or_default(); + testkit::ensure!( + meta.name == template_name && meta.labels.get("e2e") == labels.get("e2e"), + "template get returned name {:?} with labels {:?}; expected {template_name:?} with {labels:?}", + meta.name, + meta.labels + ); + let stored_workload = stored + .spec + .as_ref() + .and_then(|spec| spec.workload.clone()) + .unwrap_or_default(); + // Field by field: the gateway may fill in fields the template left unset. + let sent = workload.resources.clone().unwrap_or_default(); + let resources = stored_workload.resources.clone().unwrap_or_default(); + testkit::ensure!( + stored_workload.environment == workload.environment + && resources.cpu == sent.cpu + && resources.memory == sent.memory, + "template get returned workload {stored_workload:?}; expected environment {:?}, cpu {:?}, memory {:?}", + workload.environment, + sent.cpu, + sent.memory + ); + + let listed = cx + .client() + .list_all_sandbox_templates(SandboxTemplateListOptions::default()) + .await?; + testkit::ensure!( + listed.iter().any(|entry| entry + .metadata + .as_ref() + .is_some_and(|m| m.name == template_name)), + "template list does not include {template_name}" + ); + + let sandbox_name = cx.name("ts"); + cx.track(&sandbox_name); + cx.client() + .create_sandbox_from_template(SandboxTemplateCreateSpec { + name: Some(sandbox_name.clone()), + template_name: template_name.clone(), + command: vec!["sleep".to_string(), "infinity".to_string()], + ..SandboxTemplateCreateSpec::default() + }) + .await?; + let sandbox = cx.adopt(&sandbox_name).await?; + let ready = sandbox.wait_ready().await?; + let provenance = ready.created_from_workload_template.as_ref(); + testkit::ensure!( + provenance.is_some_and(|p| p.name == template_name), + "sandbox {sandbox_name} records template {provenance:?}; expected {template_name:?}" + ); + + sandbox + .exec(["sh", "-c", "printf '%s' \"${FEATURE_FLAG:-missing}\""]) + .await? + .expect_stdout(&workload.environment["FEATURE_FLAG"]) + .context("FEATURE_FLAG in a sandbox from the template")?; + Ok(()) +} + +/// Template operations fail with the right status: a duplicate name is +/// `AlreadyExists`, a deleted template is `NotFound`, and creating a sandbox +/// from a missing template is `NotFound`. +#[testkit::test] +async fn template_crud_errors(cx: &mut Context) -> testkit::Result { + let name = cx.name("te"); + let empty = + async || create_template(cx, &name, HashMap::new(), SandboxWorkloadConfig::default()).await; + + empty().await?; + let again = empty().await; + testkit::ensure!( + matches!(again, Err(SdkError::AlreadyExists { .. })), + "creating template {name} again returned {again:?}; expected AlreadyExists" + ); + + cx.client() + .delete_sandbox_template(&name, DeleteOptions::default()) + .await?; + let deleted = cx.client().get_sandbox_template(&name).await; + testkit::ensure!( + matches!(deleted, Err(SdkError::NotFound { .. })), + "getting deleted template {name} returned {deleted:?}; expected NotFound" + ); + + let sandbox_name = cx.name("tm"); + cx.track(&sandbox_name); + let missing = cx.name("tx"); + let from_missing = cx + .client() + .create_sandbox_from_template(SandboxTemplateCreateSpec { + name: Some(sandbox_name), + template_name: missing.clone(), + command: vec!["true".to_string()], + ..SandboxTemplateCreateSpec::default() + }) + .await; + testkit::ensure!( + matches!(from_missing, Err(SdkError::NotFound { .. })), + "creating a sandbox from missing template {missing} returned {from_missing:?}; expected NotFound" + ); + Ok(()) +} diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index 6633a3e703..6657ca1da6 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -493,6 +493,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -560,6 +570,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1137,6 +1153,25 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "rand 0.10.2", + "ring", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "url", +] + [[package]] name = "hkdf" version = "0.13.0" @@ -1517,6 +1552,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2020,6 +2104,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2108,6 +2196,19 @@ dependencies = [ "tower", ] +[[package]] +name = "openshell-policy" +version = "0.0.0" +dependencies = [ + "hickory-proto", + "miette", + "openshell-core", + "openshell-policy-schema", + "prost-types", + "serde", + "serde_json", +] + [[package]] name = "openshell-policy-schema" version = "0.0.0" @@ -2154,15 +2255,19 @@ dependencies = [ "libc", "libtest-mimic", "linkme", + "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", "serde", "serde_json", + "tempfile", "tokio", + "tonic", ] [[package]] @@ -2353,6 +2458,12 @@ dependencies = [ "openshell-testkit", ] +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.5" @@ -2771,6 +2882,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2853,6 +2973,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -3099,6 +3228,22 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -3967,6 +4112,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4114,6 +4269,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" diff --git a/tests/suites/drivers/podman/tests/oci_identity.rs b/tests/suites/drivers/podman/tests/oci_identity.rs new file mode 100644 index 0000000000..ae214ed283 --- /dev/null +++ b/tests/suites/drivers/podman/tests/oci_identity.rs @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Podman driver supervisor identity. +//! +//! Podman runs a sandbox's supervisor container as the workload's resolved +//! OCI identity with no effective capabilities; Docker runs it as a fixed +//! unprivileged user. The workload side of the same image is covered by the +//! `oci-image` feature suite in `tests/suites/features`. + +use container_engine_support::Engine; +use testkit::{Context, ResultExt as _}; + +const OCI_IDENTITY: &str = "2345:2346"; +/// A complete policy without a `process` section, so the sandbox identity +/// falls back to the image `USER`. +const IMAGE_IDENTITY_POLICY: &str = "version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /etc] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort + +network_policies: {} +"; + +/// The supervisor container runs as the image's numeric `USER`, not root or a +/// fixed user, and has no effective capabilities. +#[testkit::test] +async fn supervisor_runs_as_oci_identity_without_capabilities(cx: &mut Context) -> testkit::Result { + cx.expect_compute_driver("podman").await?; + let engine = Engine::from_env()?; + + let image = engine + .build_test_image( + cx, + "oci-identity", + &format!( + "FROM {} +USER 0:0 +RUN mkdir -p /home/app/project && \\ + chown {OCI_IDENTITY} /home/app /home/app/project && \\ + chmod 0700 /home/app /home/app/project +WORKDIR /home/app/project +USER {OCI_IDENTITY} +", + testkit::base_image() + ), + ) + .await?; + let image_user = engine + .output([ + "image", + "inspect", + "--format", + "{{.Config.User}}", + &image.tag, + ]) + .await?; + testkit::ensure!( + image_user == OCI_IDENTITY, + "the test image has USER {image_user:?}; expected {OCI_IDENTITY}" + ); + + let sandbox = cx + .cli_sandbox("oi") + .image(&image.tag) + .policy_yaml(IMAGE_IDENTITY_POLICY) + .command(["sleep", "infinity"]) + .create_ready() + .await + .context("sandbox starts from the OCI identity image")?; + + let supervisor = engine + .container_for_role(sandbox.name(), "supervisor") + .await?; + let user = engine + .inspect_container(&supervisor, "{{.Config.User}}") + .await?; + testkit::ensure!( + user == OCI_IDENTITY, + "the supervisor container runs as {user:?}; expected the image identity {OCI_IDENTITY}" + ); + let caps = engine + .inspect_container(&supervisor, "{{.EffectiveCaps}}") + .await?; + testkit::ensure!( + caps == "[]", + "the supervisor container has effective capabilities {caps}; expected none" + ); + Ok(()) +} diff --git a/tests/suites/drivers/podman/tests/userns.rs b/tests/suites/drivers/podman/tests/userns.rs index 5dff4a8a44..545554bfff 100644 --- a/tests/suites/drivers/podman/tests/userns.rs +++ b/tests/suites/drivers/podman/tests/userns.rs @@ -37,22 +37,11 @@ async fn configured_userns_matches_podman_reference(cx: &mut Context) -> testkit let image = testkit::base_image(); let reference = reference_uid_map(&engine, userns, &image).await?; - let name = cx.name("pu"); - cx.track(&name); let run = cx - .cli([ - "sandbox", - "create", - "--name", - &name, - "--from", - &image, - "--no-tty", - "--", - "sh", - "-c", - WORKSPACE_AND_UID_MAP_PROBE, - ]) + .cli_sandbox("pu") + .image(&image) + .command(["sh", "-c", WORKSPACE_AND_UID_MAP_PROBE]) + .run() .await? .success() .context("sandbox can write to its owned workspace and prints its UID map")?; diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index bb1a1427ee..624e43a830 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -472,6 +472,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cli-attach" +version = "0.0.0" +dependencies = [ + "nix", + "openshell-testkit", + "tokio", +] + [[package]] name = "cmake" version = "0.1.58" @@ -493,6 +502,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -560,6 +579,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1137,6 +1162,25 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "rand 0.10.2", + "ring", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "url", +] + [[package]] name = "hkdf" version = "0.13.0" @@ -1167,10 +1211,18 @@ dependencies = [ [[package]] name = "host-gateway" version = "0.0.0" +dependencies = [ + "host-server-support", + "openshell-testkit", +] + +[[package]] +name = "host-server-support" +version = "0.0.0" dependencies = [ "openshell-testkit", - "tempfile", "tokio", + "tungstenite", ] [[package]] @@ -1526,6 +1578,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2039,7 +2140,6 @@ version = "0.0.0" dependencies = [ "container-engine-support", "openshell-testkit", - "serde_json", "tempfile", ] @@ -2057,6 +2157,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2145,6 +2249,19 @@ dependencies = [ "tower", ] +[[package]] +name = "openshell-policy" +version = "0.0.0" +dependencies = [ + "hickory-proto", + "miette", + "openshell-core", + "openshell-policy-schema", + "prost-types", + "serde", + "serde_json", +] + [[package]] name = "openshell-policy-schema" version = "0.0.0" @@ -2203,15 +2320,19 @@ dependencies = [ "libc", "libtest-mimic", "linkme", + "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", "serde", "serde_json", + "tempfile", "tokio", + "tonic", ] [[package]] @@ -2385,6 +2506,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.5" @@ -2803,6 +2930,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2885,6 +3021,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -3131,6 +3276,22 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -3999,6 +4160,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4146,6 +4317,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index af907b4b15..35ff3237a2 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] +members = ["cli-attach", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] diff --git a/tests/suites/features/cli-attach/Cargo.toml b/tests/suites/features/cli-attach/Cargo.toml new file mode 100644 index 0000000000..99d3a919e0 --- /dev/null +++ b/tests/suites/features/cli-attach/Cargo.toml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "cli-attach" +description = "Features suite: `sandbox connect` recovers its SSH transport and forwards termination" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +openshell-testkit = { path = "../../../testkit/testkit" } +tokio = { version = "1.43", features = ["rt", "time"] } + +[target.'cfg(target_os = "linux")'.dependencies] +nix = { version = "0.29", features = ["process", "signal", "term"] } diff --git a/tests/suites/features/cli-attach/tests/cli_attach.rs b/tests/suites/features/cli-attach/tests/cli_attach.rs new file mode 100644 index 0000000000..cbe614913c --- /dev/null +++ b/tests/suites/features/cli-attach/tests/cli_attach.rs @@ -0,0 +1,220 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! How `sandbox connect` supervises the `ssh` process it attaches through: +//! it reconnects when the SSH transport is lost, and a SIGTERM sent to the +//! CLI alone ends the `ssh` child too. +//! +//! The tests find the CLI's `ssh` and `ssh-proxy` processes in the test +//! host's `/proc`, so they need Linux. Attach behavior every driver must +//! provide is in the conformance `attach` suite. +//! +//! CI runs them on Docker only. Through a Kubernetes port-forward, the +//! reconnect can attach read-only while the old SSH session still owns input +//! (#4445). +//! +//! Ported from `e2e/rust/tests/sandbox_lifecycle.rs`. + +use std::time::Duration; + +use testkit::Context; + +/// How long an attachment may take to show output or recover. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +const ATTACH_TIMEOUT: Duration = Duration::from_secs(30); + +/// Killing the `ssh-proxy` transport of an established `sandbox connect` +/// makes the CLI report the lost connection and reconnect to the same main +/// process, which then receives new input. The CLI keeps running. +#[testkit::test] +async fn connect_recovers_its_ssh_transport(cx: &mut Context) -> testkit::Result { + #[cfg(target_os = "linux")] + return linux::connect_recovers_its_ssh_transport(cx).await; + #[cfg(not(target_os = "linux"))] + return needs_linux(cx); +} + +/// A SIGTERM sent to only the `sandbox connect` process, with a terminal as +/// its stdin, is forwarded to its `ssh` child. The CLI exits 143 and the +/// `ssh` child is gone. +#[testkit::test] +async fn connect_forwards_pid_targeted_termination_and_reaps_ssh( + cx: &mut Context, +) -> testkit::Result { + #[cfg(target_os = "linux")] + return linux::connect_forwards_pid_targeted_termination_and_reaps_ssh(cx).await; + #[cfg(not(target_os = "linux"))] + return needs_linux(cx); +} + +#[cfg(not(target_os = "linux"))] +fn needs_linux(_cx: &Context) -> testkit::Result { + Err(testkit::Error::msg( + "this test reads the CLI's processes from /proc and needs a Linux test host", + )) +} + +#[cfg(target_os = "linux")] +mod linux { + use std::os::fd::OwnedFd; + use std::process::Stdio; + use std::time::Duration; + + use nix::sys::signal::{Signal, kill}; + use nix::unistd::Pid; + use testkit::{Context, ResultExt as _, Sandbox}; + + use super::ATTACH_TIMEOUT; + + /// How long the CLI treats a new attachment as still connecting. A lost + /// transport after that is one it recovers from. + const ESTABLISHED: Duration = Duration::from_secs(3); + + /// A main process that prints a numbered line every 0.2 seconds and + /// echoes each line of its input. + const ECHO_MAIN: &str = r#"trap 'kill "$writer" 2>/dev/null || true' EXIT; (n=1; while true; do printf 'transport_pid=%s sequence=%04d\n' "$$" "$n"; n=$((n + 1)); sleep 0.2; done) & writer=$!; while IFS= read -r line; do printf 'transport_pid=%s input=%s\n' "$$" "$line"; done"#; + + pub(super) async fn connect_recovers_its_ssh_transport(cx: &mut Context) -> testkit::Result { + let sandbox = ready_sandbox(cx, "tr", &["sh", "-lc", ECHO_MAIN]).await?; + let mut connect = cx.cli_spawn(["sandbox", "connect", sandbox.name()])?; + let first = connect.stdout_line(ATTACH_TIMEOUT).await?; + testkit::ensure!( + first.contains("transport_pid="), + "unexpected attachment output {first:?}" + ); + + // Recovery starts only for an established attachment. + tokio::time::sleep(ESTABLISHED).await; + let proxy = wait_for_process( + &format!("the ssh-proxy of sandbox {}", sandbox.name()), + || find_process(None, &["ssh-proxy", "--sandbox", sandbox.name()]), + ) + .await?; + send(proxy, Signal::SIGTERM).context("terminate the ssh-proxy transport")?; + connect + .stderr_until(ATTACH_TIMEOUT, |line| { + line.contains("Connection to sandbox lost; reconnecting") + }) + .await + .context("the CLI reports the lost transport")?; + + let token = format!("after-transport-recovery-{}", cx.run_id()); + connect + .write_stdin(format!("{token}\n").as_bytes()) + .await + .context("send input after recovery")?; + connect + .stdout_until(ATTACH_TIMEOUT, |line| { + line.contains(&format!("input={token}")) + }) + .await + .context("the reconnected attachment reaches the main process")?; + testkit::ensure!( + connect.try_wait()?.is_none(), + "`{}` exited after recovering", + connect.command() + ); + connect.kill().await + } + + pub(super) async fn connect_forwards_pid_targeted_termination_and_reaps_ssh( + cx: &mut Context, + ) -> testkit::Result { + let sandbox = ready_sandbox(cx, "term", &["sh", "-lc", "exec sleep infinity"]).await?; + let pty = nix::pty::openpty(None, None) + .map_err(testkit::Error::from_display) + .context("open a pseudo-terminal")?; + // Held until the end, so the terminal stays open. + let _controller: OwnedFd = pty.master; + let mut connect = cx.cli_spawn_with_stdin( + ["sandbox", "connect", sandbox.name()], + Stdio::from(pty.slave), + )?; + let connect_pid = connect.pid(); + + let ssh = wait_for_process(&format!("the ssh child of `{}`", connect.command()), || { + find_process(Some(connect_pid), &["-s", "sandbox", "openshell-main"]) + }) + .await?; + send(connect_pid, Signal::SIGTERM).context("send SIGTERM to only the CLI")?; + let code = connect.wait(Duration::from_secs(10)).await?; + testkit::ensure!( + code == 143, + "`{}` exited with {code} on SIGTERM; expected 143", + connect.command() + ); + testkit::poll_until( + &format!("ssh process {ssh} to be gone"), + Duration::from_secs(5), + Duration::from_millis(25), + async || { + Ok(if std::fs::metadata(format!("/proc/{ssh}")).is_ok() { + testkit::Poll::Pending(format!("/proc/{ssh} still exists")) + } else { + testkit::Poll::Ready(()) + }) + }, + ) + .await + } + + async fn ready_sandbox( + cx: &Context, + suffix: &str, + command: &[&str], + ) -> testkit::Result { + cx.sandbox(suffix) + .command(command.iter().copied()) + .create_ready() + .await + } + + fn send(pid: u32, signal: Signal) -> testkit::Result { + let pid = i32::try_from(pid).map_err(testkit::Error::from_display)?; + kill(Pid::from_raw(pid), signal).map_err(testkit::Error::from_display) + } + + async fn wait_for_process( + what: &str, + mut find: impl FnMut() -> Option, + ) -> testkit::Result { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + if let Some(pid) = find() { + return Ok(pid); + } + testkit::ensure!( + tokio::time::Instant::now() < deadline, + "{what} did not start" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + /// A process whose arguments include all of `args`, and whose parent is + /// `parent` if given. + fn find_process(parent: Option, args: &[&str]) -> Option { + for entry in std::fs::read_dir("/proc").ok()?.filter_map(Result::ok) { + let Ok(pid) = entry.file_name().to_string_lossy().parse::() else { + continue; + }; + if let Some(parent) = parent { + let status = + std::fs::read_to_string(entry.path().join("status")).unwrap_or_default(); + let ppid = status.lines().find_map(|line| { + line.strip_prefix("PPid:") + .and_then(|value| value.trim().parse::().ok()) + }); + if ppid != Some(parent) { + continue; + } + } + let cmdline = std::fs::read(entry.path().join("cmdline")).unwrap_or_default(); + let argv: Vec<&[u8]> = cmdline.split(|byte| *byte == 0).collect(); + if args.iter().all(|arg| argv.contains(&arg.as_bytes())) { + return Some(pid); + } + } + None + } +} diff --git a/tests/suites/features/host-gateway/Cargo.toml b/tests/suites/features/host-gateway/Cargo.toml index d81b6cafb2..0dfef50bc5 100644 --- a/tests/suites/features/host-gateway/Cargo.toml +++ b/tests/suites/features/host-gateway/Cargo.toml @@ -12,6 +12,5 @@ repository = "https://github.com/NVIDIA/OpenShell" publish = false [dependencies] +host-server-support = { path = "../../../support/host-server" } openshell-testkit = { path = "../../../testkit/testkit" } -tempfile = "3" -tokio = { version = "1.43", features = ["io-util", "net", "rt"] } diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index 024b1cd84b..7d285116e4 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -9,34 +9,25 @@ //! Python test image ([`testkit::python_image`]) that requests a page from that //! server through the name. -use testkit::{Context, Error, Result, ResultExt as _}; -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::task::JoinHandle; +use host_server_support::HttpRecorder; +use testkit::policy; +use testkit::sdk::raw::proto; +use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; -/// The image's system Python, which needs nothing beyond its standard library. -/// The policy names its real path, `/usr/bin/python3.*`, because a rule matches -/// the executable behind a symlink. -const PYTHON: &str = "/usr/bin/python3"; +/// The image's Python, which needs nothing beyond its standard library. The +/// policy allows [`PYTHON_BINARIES`], the real paths behind `python3` in +/// `testkit::python_image()` images, because a rule matches the executable +/// behind a symlink. +const PYTHON: &str = "python3"; /// Requests `/` from `host.openshell.internal` on the port in `argv[1]` and -/// prints the response. A failed connection is retried for 20 seconds, so a -/// slow start doesn't fail the test, and reported on stderr. +/// prints the response. The server listens before the sandbox is created, +/// and a Ready sandbox's network is up, so the client doesn't retry. const CLIENT: &str = r#" -import socket, sys, time +import socket, sys host, port = "host.openshell.internal", int(sys.argv[1]) -deadline = time.monotonic() + 20 -while True: - try: - conn = socket.create_connection((host, port), timeout=10) - break - except OSError as error: - print(f"connect {host}:{port}: {error}", file=sys.stderr, flush=True) - if time.monotonic() > deadline: - sys.exit(1) - time.sleep(1) -with conn: +with socket.create_connection((host, port), timeout=10) as conn: conn.sendall(f"GET / HTTP/1.1\r\nHost: {host}:{port}\r\nConnection: close\r\n\r\n".encode()) response = b"" while chunk := conn.recv(4096): @@ -51,9 +42,7 @@ print(response.decode(errors="replace")) #[testkit::test] async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result { let marker = format!("host-gateway-{}", cx.run_id()); - let server = HostServer::start(marker.clone()).await?; - let policy = PolicyFile::new(&policy(server.port))?; - let port = server.port.to_string(); + let server = HttpRecorder::start(marker.clone()).await?; let image = testkit::python_image(); // The client runs through `sandbox exec` in a Ready sandbox, not as the @@ -61,33 +50,17 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result // attach to a main process that already finished with "sandbox is not // ready" when the request reaches a replica that doesn't hold the // sandbox's session. - let name = cx.name("hg"); - cx.track(&name); - cx.cli([ - "sandbox", - "create", - "--name", - &name, - "--from", - &image, - "--policy", - policy.path()?, - "--detach", - "--no-tty", - "--", - "sh", - "-c", - "exec sleep infinity", - ]) - .await? - .success() - .context(format!("create a sandbox from {image}"))?; - cx.adopt(&name).await?.wait_ready().await?; - - let client = cx - .cli([ - "sandbox", "exec", "--name", &name, "--no-tty", "--", PYTHON, "-c", CLIENT, &port, - ]) + let sandbox = cx + .sandbox("hg") + .image(image.clone()) + .policy(policy(server.port)) + .command(["sh", "-c", "exec sleep infinity"]) + .create_ready() + .await + .context(format!("create a sandbox from {image}"))?; + + let client = sandbox + .exec([PYTHON, "-c", CLIENT, &server.port.to_string()]) .await? .success() .context("run the client in the sandbox")?; @@ -100,110 +73,18 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result Ok(()) } -/// A sandbox policy that allows the image's system Python ([`PYTHON`]) to +/// A sandbox policy that allows the image's Python ([`PYTHON`]) to /// connect to `host.openshell.internal` on `port`. The endpoint names only the /// host and port, as the policy docs describe, so the driver's address for the /// name is what makes it reachable, not an `allowed_ips` range. -fn policy(port: u16) -> String { - format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /etc] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - host_server: - name: host_server - endpoints: - - host: host.openshell.internal - port: {port} - binaries: - - path: /usr/bin/python3.* -"# - ) -} - -/// An HTTP server on the host that answers every request with `body`. It -/// stops when dropped. -struct HostServer { - port: u16, - task: JoinHandle<()>, -} - -impl HostServer { - /// Listen on every host address, since drivers map - /// `host.openshell.internal` to different ones. - async fn start(body: String) -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| Error::from_error(&error)) - .context("bind the host server")?; - let port = listener - .local_addr() - .map_err(|error| Error::from_error(&error)) - .context("read the host server's address")? - .port(); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - tokio::spawn(respond(stream, body.clone())); - } - }); - Ok(Self { port, task }) - } -} - -impl Drop for HostServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -/// Read one request's headers from `stream` and answer with `body`. -async fn respond(mut stream: TcpStream, body: String) { - let _ = stream.set_nodelay(true); - let mut request = Vec::new(); - let mut buf = [0_u8; 1024]; - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - match stream.read(&mut buf).await { - Ok(read) if read > 0 && request.len() < 64 * 1024 => { - request.extend_from_slice(&buf[..read]); - } - _ => return, - } - } - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() +fn policy(port: u16) -> proto::SandboxPolicy { + let mut policy = policy::base(); + policy::allow( + &mut policy, + "host_server", + "host.openshell.internal", + port, + PYTHON_BINARIES, ); - let _ = stream.write_all(response.as_bytes()).await; - let _ = stream.shutdown().await; -} - -/// A sandbox policy written to a temporary file for `sandbox create --policy`. -struct PolicyFile(tempfile::NamedTempFile); - -impl PolicyFile { - fn new(contents: &str) -> Result { - let file = tempfile::NamedTempFile::new() - .map_err(|error| Error::from_error(&error)) - .context("create policy file")?; - std::fs::write(file.path(), contents) - .map_err(|error| Error::from_error(&error)) - .context(format!("write {}", file.path().display()))?; - Ok(Self(file)) - } - - fn path(&self) -> Result<&str> { - let path = self.0.path(); - path.to_str() - .ok_or_else(|| Error::msg(format!("{} is not UTF-8", path.display()))) - } + policy } diff --git a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs index 8035d0694e..a617c24843 100644 --- a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs +++ b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs @@ -9,7 +9,7 @@ use std::io::Write as _; use keycloak_support::KeycloakContext; -use testkit::{CliOutput, Context, Error, ResultExt as _}; +use testkit::{CliOutput, Context, ResultExt as _}; const CREDENTIAL_KEY: &str = "KEYCLOAK_ACCESS_TOKEN"; @@ -172,11 +172,9 @@ binaries: let mut file = tempfile::Builder::new() .suffix(".yaml") .tempfile() - .map_err(|error| Error::from_error(&error)) .context("create provider profile")?; file.write_all(profile.as_bytes()) .and_then(|()| file.flush()) - .map_err(|error| Error::from_error(&error)) .context("write provider profile")?; Ok(file) } diff --git a/tests/suites/features/kubernetes-updates/tests/update.rs b/tests/suites/features/kubernetes-updates/tests/update.rs index 61ac57e9fa..ab71a028a1 100644 --- a/tests/suites/features/kubernetes-updates/tests/update.rs +++ b/tests/suites/features/kubernetes-updates/tests/update.rs @@ -458,7 +458,7 @@ async fn image_update_retains_workspace_and_identity(cx: &mut Context) -> testki let sandbox = cx // The testkit's unique prefix leaves five characters of the name limit. .sandbox("image") - .spec(|spec| spec.image = Some(ORIGINAL_IMAGE.to_string())) + .image(ORIGINAL_IMAGE.as_str()) .create() .await?; sandbox.wait_ready().await?; @@ -597,7 +597,7 @@ async fn terminal_workload_can_stop_update_and_restart(cx: &mut Context) -> test let sandbox = cx .sandbox("term") .restart_policy(testkit::sdk::SandboxRestartPolicy::Never) - .spec(|spec| spec.image = Some(ORIGINAL_IMAGE.to_string())) + .image(ORIGINAL_IMAGE.as_str()) .create() .await?; sandbox.wait_ready().await?; diff --git a/tests/suites/features/oci-image/Cargo.toml b/tests/suites/features/oci-image/Cargo.toml index b966d2c38d..4298cd0686 100644 --- a/tests/suites/features/oci-image/Cargo.toml +++ b/tests/suites/features/oci-image/Cargo.toml @@ -14,5 +14,4 @@ publish = false [dependencies] container-engine-support = { path = "../../../support/container-engine" } openshell-testkit = { path = "../../../testkit/testkit" } -serde_json = "1" tempfile = "3" diff --git a/tests/suites/features/oci-image/tests/oci_image.rs b/tests/suites/features/oci-image/tests/oci_image.rs index 4206c1ccb5..8e156eb8eb 100644 --- a/tests/suites/features/oci-image/tests/oci_image.rs +++ b/tests/suites/features/oci-image/tests/oci_image.rs @@ -11,7 +11,6 @@ //! driver created, through the same engine. use container_engine_support::{Engine, Image}; -use serde_json::Value; use testkit::{Context, Error, Result, ResultExt as _}; /// Sandbox condition reason for an image `WORKDIR` the sandbox identity @@ -87,10 +86,9 @@ USER 2345:2346 ), ) .await?; - let policy = PolicyFile::new(IMAGE_IDENTITY_POLICY)?; let checks = workspace_checks("2345:2346", "/home/app/project", true); let sandbox = oci - .create_sandbox("pu", &image, Some(policy.path()?), &checks) + .create_sandbox("pu", &image, Some(IMAGE_IDENTITY_POLICY), &checks) .await?; oci.container_checks(&sandbox, &image, "2345:2346").await }) @@ -106,13 +104,12 @@ async fn userless_image_uses_default_identity(cx: &mut Context) -> testkit::Resu let image = oci .build_image("userless", &format!("FROM {}\n", oci.base_image)) .await?; - let policy = PolicyFile::new(IMAGE_IDENTITY_POLICY)?; let checks = format!( "{} test \"$(id -G)\" = 1000;", workspace_checks("1000:1000", "/sandbox", false) ); let sandbox = oci - .create_sandbox("ud", &image, Some(policy.path()?), &checks) + .create_sandbox("ud", &image, Some(IMAGE_IDENTITY_POLICY), &checks) .await?; oci.container_checks(&sandbox, &image, "1000:1000").await }) @@ -137,19 +134,19 @@ USER root ), ) .await?; - let policy = PolicyFile::new(&format!( + let policy = format!( "{IMAGE_IDENTITY_POLICY} process: run_as_user: \"4234\" run_as_group: \"4235\" " - ))?; + ); let checks = format!( "{} test \"$(id -G)\" = 4235;", workspace_checks("4234:4235", "/sandbox", false) ); let sandbox = oci - .create_sandbox("pr", &image, Some(policy.path()?), &checks) + .create_sandbox("pr", &image, Some(&policy), &checks) .await?; oci.container_checks(&sandbox, &image, "4234:4235").await }) @@ -198,22 +195,12 @@ USER app ), ) .await?; - let name = oci.track(suffix); - let create = oci - .cx - .cli([ - "sandbox", - "create", - "--name", - &name, - "--from", - &image.tag, - "--no-tty", - "--", - "sh", - "-c", - "echo workspace-access-marker", - ]) + let builder = oci.cx.cli_sandbox(suffix); + let name = builder.name().to_string(); + let create = builder + .image(&image.tag) + .command(["sh", "-c", "echo workspace-access-marker"]) + .run() .await?; if writable { let create = create @@ -238,12 +225,7 @@ USER app ); // Require the condition reason, so another provisioning failure // cannot pass as a workspace rejection. - let sandbox: Value = oci - .cx - .cli(["sandbox", "get", &name, "--output", "json"]) - .await? - .success()? - .json()?; + let sandbox = oci.cx.adopt(&name).await?.get_json().await?; let reasons: Vec<&str> = sandbox["conditions"] .as_array() .into_iter() @@ -278,27 +260,12 @@ async fn run(cx: &Context, test: impl AsyncFnOnce(&Oci<'_>) -> Result) -> Result } impl Oci<'_> { - /// Build `containerfile` with a tag unique to this run, and remove the - /// image when the test ends. Cleanup runs last registered first, so the - /// sandboxes created from the image are deleted before it is removed. + /// Build `containerfile` as a test image named `oci-`, removed when + /// the test ends. async fn build_image(&self, name: &str, containerfile: &str) -> Result { - let tag = format!("localhost/openshell-test-oci-{name}:{}", self.cx.run_id()); - self.engine.build_image(&tag, containerfile).await?; - let (engine, built) = (self.engine.clone(), tag.clone()); - self.cx - .on_cleanup(format!("remove image {tag}"), async move || { - engine.remove_image(&built).await - }); - self.engine.image(&tag).await - } - - /// Name a sandbox for this test and register it for cleanup. Sandbox - /// names are limited to 19 characters on some drivers, so keep `short` - /// to a few characters. - fn track(&self, short: &str) -> String { - let name = self.cx.name(short); - self.cx.track(&name); - name + self.engine + .build_test_image(self.cx, &format!("oci-{name}"), containerfile) + .await } /// Create a detached sandbox whose main process runs `checks` and writes @@ -311,21 +278,21 @@ impl Oci<'_> { policy: Option<&str>, checks: &str, ) -> Result { - let name = self.track(short); let main = format!( "(set -eu; {checks} touch main-write) >/tmp/oci-main.log 2>&1; \ echo $? >/tmp/oci-main.status; exec sleep infinity" ); - let mut args = vec!["sandbox", "create", "--name", &name, "--from", &image.tag]; + let mut builder = self.cx.cli_sandbox(short).image(&image.tag); if let Some(policy) = policy { - args.extend(["--policy", policy]); + builder = builder.policy_yaml(policy); } - args.extend(["--detach", "--", "sh", "-c", &main]); - self.cx - .cli(args) - .await? - .success() - .context("sandbox starts from the test image")?; + let name = builder + .command(["sh", "-c", &main]) + .create() + .await + .context("sandbox starts from the test image")? + .name() + .to_string(); let script = format!( "set -eu; i=0; \ @@ -346,9 +313,7 @@ impl Oci<'_> { /// Upload and download default to paths relative to the image workspace. async fn file_transfer_uses_workspace(&self, sandbox: &str) -> Result { - let local = tempfile::tempdir() - .map_err(|error| Error::from_error(&error)) - .context("create temp dir")?; + let local = tempfile::tempdir().context("create temp dir")?; let upload = local.path().join("oci-transfer.txt"); write(&upload, "oci-transfer-ok")?; self.cx @@ -381,9 +346,7 @@ impl Oci<'_> { .await? .success() .context("download resolves relative paths in the workspace")?; - let downloaded = std::fs::read_to_string(&download) - .map_err(|error| Error::from_error(&error)) - .context("read download")?; + let downloaded = std::fs::read_to_string(&download).context("read download")?; testkit::ensure!( downloaded == "oci-transfer-ok", "downloaded file has unexpected content: {downloaded:?}" @@ -391,9 +354,7 @@ impl Oci<'_> { // A directory upload merges into an existing workspace directory. let merge = local.path().join("merge-upload"); - std::fs::create_dir(&merge) - .map_err(|error| Error::from_error(&error)) - .context("create merge dir")?; + std::fs::create_dir(&merge).context("create merge dir")?; write(&merge.join("conflict.txt"), "local-conflict")?; write(&merge.join("added.txt"), "local-added")?; self.exec( @@ -527,27 +488,8 @@ fn workspace_checks(identity: &str, workspace: &str, image_file: bool) -> String checks } -/// A sandbox policy written to a temporary file for `sandbox create --policy`. -struct PolicyFile(tempfile::NamedTempFile); - -impl PolicyFile { - fn new(contents: &str) -> Result { - let file = tempfile::NamedTempFile::new() - .map_err(|error| Error::from_error(&error)) - .context("create policy file")?; - write(file.path(), contents)?; - Ok(Self(file)) - } - - fn path(&self) -> Result<&str> { - utf8(self.0.path()) - } -} - fn write(path: &std::path::Path, contents: &str) -> Result { - std::fs::write(path, contents) - .map_err(|error| Error::from_error(&error)) - .context(format!("write {}", path.display())) + std::fs::write(path, contents).context(format!("write {}", path.display())) } fn utf8(path: &std::path::Path) -> Result<&str> { diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index f8b591438f..94cd21327d 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -493,6 +493,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -560,6 +570,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1137,6 +1153,25 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "rand 0.10.2", + "ring", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "url", +] + [[package]] name = "hkdf" version = "0.13.0" @@ -1164,6 +1199,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "host-server-support" +version = "0.0.0" +dependencies = [ + "openshell-testkit", + "tokio", + "tungstenite", +] + [[package]] name = "hostname" version = "0.4.2" @@ -1517,6 +1561,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2029,6 +2122,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2117,6 +2214,19 @@ dependencies = [ "tower", ] +[[package]] +name = "openshell-policy" +version = "0.0.0" +dependencies = [ + "hickory-proto", + "miette", + "openshell-core", + "openshell-policy-schema", + "prost-types", + "serde", + "serde_json", +] + [[package]] name = "openshell-policy-schema" version = "0.0.0" @@ -2163,15 +2273,19 @@ dependencies = [ "libc", "libtest-mimic", "linkme", + "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", "serde", "serde_json", + "tempfile", "tokio", + "tonic", ] [[package]] @@ -2353,6 +2467,12 @@ dependencies = [ "openshell-testkit", ] +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.5" @@ -2771,6 +2891,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2853,6 +2982,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -3099,6 +3237,22 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -3967,6 +4121,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4114,6 +4278,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" diff --git a/tests/support/Cargo.toml b/tests/support/Cargo.toml index f56ee275f5..4c7c495f97 100644 --- a/tests/support/Cargo.toml +++ b/tests/support/Cargo.toml @@ -5,4 +5,4 @@ # environment runs. Suites depend on them by path. [workspace] resolver = "2" -members = ["container-engine", "keycloak", "podman"] +members = ["container-engine", "host-server", "keycloak", "podman"] diff --git a/tests/support/container-engine/src/lib.rs b/tests/support/container-engine/src/lib.rs index f380a9815f..7d47f2d33f 100644 --- a/tests/support/container-engine/src/lib.rs +++ b/tests/support/container-engine/src/lib.rs @@ -20,7 +20,7 @@ use std::path::Path; -use testkit::{Error, Result, ResultExt as _}; +use testkit::{Context, Error, Result, ResultExt as _}; use tokio::process::Command; pub const ENGINE_ENV: &str = "OPENSHELL_TEST_CONTAINER_ENGINE"; @@ -64,17 +64,11 @@ impl Engine { Ok(Self { command }) } - /// Build `containerfile` as `tag`. The build context is empty. Remove - /// the image with [`Engine::remove_image`], for example in a cleanup - /// hook registered right after the build. - pub async fn build_image(&self, tag: &str, containerfile: &str) -> Result { - let context = tempfile::tempdir() - .map_err(|error| Error::from_error(&error)) - .context("create build context")?; + /// Build `containerfile` as `tag`. The build context is empty. + async fn build_image(&self, tag: &str, containerfile: &str) -> Result { + let context = tempfile::tempdir().context("create build context")?; let file = context.path().join("Containerfile"); - std::fs::write(&file, containerfile) - .map_err(|error| Error::from_error(&error)) - .context("write Containerfile")?; + std::fs::write(&file, containerfile).context("write Containerfile")?; self.output([ "build", "--file", @@ -87,6 +81,25 @@ impl Engine { .map(drop) } + /// Build `containerfile` as `localhost/openshell-test-:` + /// and remove the image when the test ends. Cleanup runs last registered + /// first, so sandboxes created from the image afterwards are deleted + /// before it is removed. + pub async fn build_test_image( + &self, + cx: &Context, + name: &str, + containerfile: &str, + ) -> Result { + let tag = format!("localhost/openshell-test-{name}:{}", cx.run_id()); + self.build_image(&tag, containerfile).await?; + let (engine, built) = (self.clone(), tag.clone()); + cx.on_cleanup(format!("remove image {tag}"), async move || { + engine.remove_image(&built).await + }); + self.image(&tag).await + } + /// Image `tag` and the ID it has now. pub async fn image(&self, tag: &str) -> Result { let id = self @@ -157,7 +170,6 @@ impl Engine { .kill_on_drop(true) .output() .await - .map_err(|error| Error::from_error(&error)) .context(format!("run {command}"))?; testkit::ensure!( output.status.success(), diff --git a/tests/support/host-server/Cargo.toml b/tests/support/host-server/Cargo.toml new file mode 100644 index 0000000000..e7db63f53d --- /dev/null +++ b/tests/support/host-server/Cargo.toml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "host-server-support" +description = "Servers on the test host that sandboxes reach through host.openshell.internal" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +openshell-testkit = { path = "../../testkit/testkit" } +tokio = { version = "1.43", features = ["io-util", "net", "rt", "time"] } +tungstenite = "0.26" + +[dev-dependencies] +tokio = { version = "1.43", features = ["macros", "rt"] } diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs new file mode 100644 index 0000000000..4d8ff8c812 --- /dev/null +++ b/tests/support/host-server/src/http.rs @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::{Arc, Mutex, PoisonError}; + +use testkit::{Poll, Result}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::TcpStream; +use tokio::task::JoinHandle; + +use crate::{MAX_REQUEST, READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, find, header}; + +/// A request an [`HttpRecorder`] read. +#[derive(Clone, Debug)] +pub struct Request { + /// The request line and headers. + pub head: String, + pub body: String, +} + +/// An HTTP server that answers every request with `200` and a fixed body, +/// and records each request it read. It reads a body by `Content-Length` +/// only, not a chunked one. +pub struct HttpRecorder { + pub port: u16, + requests: Arc>>, + task: JoinHandle<()>, +} + +impl HttpRecorder { + pub async fn start(body: impl Into) -> Result { + let (listener, port) = bind().await?; + let body: Arc = body.into().into(); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(respond( + stream, + Arc::clone(&body), + Arc::clone(&task_requests), + )); + } + }); + Ok(Self { + port, + requests, + task, + }) + } + + /// The requests the server has read, once there are at least `count`. + pub async fn wait_for_requests(&self, count: usize) -> Result> { + testkit::poll_until( + &format!( + "{count} request(s) at the host server on port {}", + self.port + ), + WAIT_TIMEOUT, + WAIT_INTERVAL, + async || { + let requests = self + .requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + Ok(if requests.len() >= count { + Poll::Ready(requests) + } else { + Poll::Pending(format!("{} request(s)", requests.len())) + }) + }, + ) + .await + } +} + +impl Drop for HttpRecorder { + fn drop(&mut self) { + self.task.abort(); + } +} + +/// Read one request, headers and `Content-Length` body, record it, and +/// answer with `body`. +async fn respond(mut stream: TcpStream, body: Arc, requests: Arc>>) { + let _ = stream.set_nodelay(true); + let Some(request) = read_request(&mut stream).await else { + return; + }; + requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(request); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(response.as_bytes()).await; + let _ = stream.shutdown().await; +} + +pub(crate) async fn read_request(stream: &mut TcpStream) -> Option { + let mut request = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + if let Some(end) = find(&request, b"\r\n\r\n") { + let head = String::from_utf8_lossy(&request[..end]).into_owned(); + let length = header(&head, "content-length") + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + if request.len() >= end + 4 + length { + let body = + String::from_utf8_lossy(&request[end + 4..end + 4 + length]).into_owned(); + return Some(Request { head, body }); + } + } + match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut buffer)).await { + Ok(Ok(read)) if read > 0 && request.len() < MAX_REQUEST => { + request.extend_from_slice(&buffer[..read]); + } + _ => return None, + } + } +} diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs new file mode 100644 index 0000000000..8ac8528570 --- /dev/null +++ b/tests/support/host-server/src/lib.rs @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Servers on the test host that sandboxes reach through +//! `host.openshell.internal`, and that record what arrived, so a test can +//! check what the supervisor forwarded upstream. +//! +//! The servers listen on every host address, since drivers map +//! `host.openshell.internal` to different ones, and stop when dropped. They +//! run where the tests run, so a gateway whose sandboxes can't reach the test +//! host, such as one on a remote cluster, can't run tests that use them. + +mod http; +mod websocket; + +use std::time::Duration; + +use testkit::{Result, ResultExt as _}; +use tokio::net::TcpListener; + +pub use http::{HttpRecorder, Request}; +pub use websocket::WebSocketRecorder; + +const WAIT_TIMEOUT: Duration = Duration::from_secs(10); +const WAIT_INTERVAL: Duration = Duration::from_millis(100); +const READ_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_REQUEST: usize = 1024 * 1024; + +async fn bind() -> Result<(TcpListener, u16)> { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .context("bind a host server")?; + let port = listener + .local_addr() + .context("read the host server's address")? + .port(); + Ok((listener, port)) +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +fn header(head: &str, name: &str) -> Option { + head.lines().find_map(|line| { + let (header, value) = line.split_once(':')?; + header + .trim() + .eq_ignore_ascii_case(name) + .then(|| value.trim().to_string()) + }) +} diff --git a/tests/support/host-server/src/websocket.rs b/tests/support/host-server/src/websocket.rs new file mode 100644 index 0000000000..28426438c7 --- /dev/null +++ b/tests/support/host-server/src/websocket.rs @@ -0,0 +1,219 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use testkit::{Poll, Result}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::TcpStream; +use tokio::task::JoinHandle; + +use crate::http::read_request; +use crate::{READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, header}; + +/// A WebSocket server that completes the handshake, then reads frames until +/// the connection closes, and records whether any was a binary frame. +pub struct WebSocketRecorder { + pub port: u16, + state: Arc, + task: JoinHandle<()>, +} + +#[derive(Default)] +struct WebSocketState { + handshake_seen: AtomicBool, + binary_seen: AtomicBool, + finished: AtomicBool, +} + +impl WebSocketRecorder { + pub async fn start() -> Result { + let (listener, port) = bind().await?; + let state = Arc::new(WebSocketState::default()); + let task_state = Arc::clone(&state); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(accept_websocket(stream, Arc::clone(&task_state))); + } + }); + Ok(Self { port, state, task }) + } + + /// Wait until a client completed the handshake. + pub async fn wait_for_handshake(&self) -> Result { + testkit::poll_until( + &format!( + "a WebSocket handshake at the host server on port {}", + self.port + ), + WAIT_TIMEOUT, + WAIT_INTERVAL, + async || { + Ok(if self.state.handshake_seen.load(Ordering::Acquire) { + Poll::Ready(()) + } else { + Poll::Pending("no handshake".to_string()) + }) + }, + ) + .await + } + + /// Wait until a connection that completed the handshake has ended, by a + /// close frame, the client closing it, or [`READ_TIMEOUT`] passing, and + /// return whether a binary frame arrived on it. + pub async fn wait_until_finished(&self) -> Result { + testkit::poll_until( + &format!( + "the WebSocket connection at the host server on port {} to end", + self.port + ), + READ_TIMEOUT + WAIT_TIMEOUT, + WAIT_INTERVAL, + async || { + Ok(if self.state.finished.load(Ordering::Acquire) { + Poll::Ready(self.state.binary_seen.load(Ordering::Acquire)) + } else { + Poll::Pending("the connection is open".to_string()) + }) + }, + ) + .await + } +} + +impl Drop for WebSocketRecorder { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn accept_websocket(mut stream: TcpStream, state: Arc) { + let _ = stream.set_nodelay(true); + let Some(request) = read_request(&mut stream).await else { + return; + }; + let Some(key) = header(&request.head, "sec-websocket-key") else { + return; + }; + // The supervisor checks Sec-WebSocket-Accept, so compute it properly. + let accept = tungstenite::handshake::derive_accept_key(key.as_bytes()); + let response = format!( + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept}\r\n\r\n" + ); + if stream.write_all(response.as_bytes()).await.is_err() { + return; + } + state.handshake_seen.store(true, Ordering::Release); + + let _ = tokio::time::timeout(READ_TIMEOUT, read_frames(&mut stream, &state)).await; + state.finished.store(true, Ordering::Release); +} + +/// Read frames until a close frame or the end of the stream, and record a +/// binary frame. +async fn read_frames(stream: &mut TcpStream, state: &WebSocketState) -> std::io::Result<()> { + loop { + let mut frame_header = [0_u8; 2]; + stream.read_exact(&mut frame_header).await?; + let opcode = frame_header[0] & 0x0f; + let length = match frame_header[1] & 0x7f { + 126 => u64::from(stream.read_u16().await?), + 127 => stream.read_u64().await?, + length => u64::from(length), + }; + if opcode == 0x02 { + state.binary_seen.store(true, Ordering::Release); + } + if opcode == 0x08 { + return Ok(()); + } + let mask = if frame_header[1] & 0x80 == 0 { 0 } else { 4 }; + let skipped = tokio::io::copy( + &mut (&mut *stream).take(mask + length), + &mut tokio::io::sink(), + ) + .await?; + if skipped < mask + length { + return Ok(()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::find; + + /// Complete a handshake with `recorder`, send `frames`, and close. + async fn send_frames(recorder: &WebSocketRecorder, frames: &[(u8, Vec)]) { + let mut stream = TcpStream::connect(("127.0.0.1", recorder.port)) + .await + .unwrap(); + stream.set_nodelay(true).unwrap(); + stream + .write_all( + b"GET / HTTP/1.1\r\nHost: test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\ + Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n", + ) + .await + .unwrap(); + let mut response = Vec::new(); + while find(&response, b"\r\n\r\n").is_none() { + let mut buffer = [0_u8; 1024]; + let read = stream.read(&mut buffer).await.unwrap(); + assert!(read > 0, "closed during the handshake"); + response.extend_from_slice(&buffer[..read]); + } + let response = String::from_utf8_lossy(&response); + assert!(response.starts_with("HTTP/1.1 101"), "{response}"); + // The RFC 6455 sample key's accept value. + assert!( + response.contains("Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo="), + "{response}" + ); + for (opcode, payload) in frames { + stream.write_all(&frame(*opcode, payload)).await.unwrap(); + } + stream.write_all(&frame(0x08, &[])).await.unwrap(); + } + + /// A final, masked client frame. + fn frame(opcode: u8, payload: &[u8]) -> Vec { + let mask = [1_u8, 2, 3, 4]; + let mut frame = vec![0x80 | opcode]; + match payload.len() { + length @ 0..=125 => frame.push(0x80 | u8::try_from(length).unwrap()), + length => { + frame.push(0x80 | 126); + frame.extend_from_slice(&u16::try_from(length).unwrap().to_be_bytes()); + } + } + frame.extend_from_slice(&mask); + frame.extend(payload.iter().enumerate().map(|(i, b)| b ^ mask[i % 4])); + frame + } + + #[tokio::test] + async fn websocket_recorder_sees_a_binary_frame_after_a_long_text_frame() { + let recorder = WebSocketRecorder::start().await.unwrap(); + send_frames( + &recorder, + &[(0x01, vec![b'x'; 300]), (0x02, b"binary".to_vec())], + ) + .await; + assert!(recorder.wait_until_finished().await.unwrap()); + } + + #[tokio::test] + async fn websocket_recorder_sees_no_binary_frame_in_text_frames() { + let recorder = WebSocketRecorder::start().await.unwrap(); + send_frames( + &recorder, + &[(0x01, vec![b'x'; 300]), (0x01, b"text".to_vec())], + ) + .await; + assert!(!recorder.wait_until_finished().await.unwrap()); + } +} diff --git a/tests/support/keycloak/src/lib.rs b/tests/support/keycloak/src/lib.rs index d5b0b58f31..7e96c5b41c 100644 --- a/tests/support/keycloak/src/lib.rs +++ b/tests/support/keycloak/src/lib.rs @@ -158,7 +158,6 @@ impl Keycloak { .kill_on_drop(true) .output() .await - .map_err(|error| Error::from_error(&error)) .context("run curl")?; Ok(if output.status.success() { Poll::Ready(()) @@ -253,20 +252,14 @@ async fn post_form(url: &str, fields: &[&str], secret: (&str, &str)) -> Result Self { + Self { + binary: PathBuf::from(binary), + gateway: Gateway::from_endpoint( + "http://127.0.0.1:1".to_string(), + "default".to_string(), + ), + } + } + /// Check that the CLI reaches the gateway: `openshell status` must report /// it connected. pub async fn check_connected(&self) -> Result { @@ -98,36 +111,11 @@ impl Cli { I: IntoIterator, S: AsRef, { - let args: Vec<_> = args - .into_iter() - .map(|arg| arg.as_ref().to_os_string()) - .collect(); - let command = std::iter::once(self.binary.as_os_str()) - .chain(args.iter().map(std::ffi::OsString::as_os_str)) - .map(|part| part.to_string_lossy()) - .collect::>() - .join(" "); - - let mut process = tokio::process::Command::new(&self.binary); - // Set first, so the gateway and workspace selection below wins over + let (command, mut process) = self.command(args); + // Set first, so the gateway and workspace selection wins over // anything the caller passes. - process - .args(&args) - .envs(env.iter().copied()) - .env("NO_COLOR", "1") - .env(WORKSPACE_ENV, &self.gateway.workspace) - .stdin(Stdio::null()) - .kill_on_drop(true); - // Select exactly the gateway the SDK connected to. - match &self.gateway.name { - Some(name) => process.env(GATEWAY_ENV, name), - None => process.env_remove(GATEWAY_ENV), - }; - if self.gateway.endpoint_from_env { - process.env(ENDPOINT_ENV, &self.gateway.endpoint); - } else { - process.env_remove(ENDPOINT_ENV); - } + process.envs(env.iter().copied()).stdin(Stdio::null()); + self.apply_env(&mut process); if let Some(dir) = dir { process.current_dir(dir); } @@ -136,26 +124,108 @@ impl Cli { .await .map_err(|_| Error::msg(format!("timed out after {CLI_TIMEOUT:?}"))) .context(format!("run {command}"))? - .map_err(|error| Error::from_error(&error)) .context(format!("run {command}"))?; Ok(CliOutput { command, - // A signal-terminated CLI has no code; report it as a failure. - exit_code: output.status.code().unwrap_or(-1), + exit_code: exit_code(output.status), stdout: String::from_utf8_lossy(&output.stdout).into_owned(), stderr: String::from_utf8_lossy(&output.stderr).into_owned(), }) } + + /// Start the CLI with `args` and keep it running, with stdin, stdout, + /// and stderr piped to the test. Use it for commands that stream or + /// read input, such as `sandbox connect`. + pub fn spawn(&self, args: I) -> Result + where + I: IntoIterator, + S: AsRef, + { + self.spawn_with_stdin(args, Stdio::piped()) + } + + /// Like [`Cli::spawn`], with `stdin` as the CLI's stdin, such as + /// [`Stdio::null`], a file, or a pseudo-terminal. Stdout and stderr are + /// still piped. + pub fn spawn_with_stdin(&self, args: I, stdin: impl Into) -> Result + where + I: IntoIterator, + S: AsRef, + { + let (command, mut process) = self.command(args); + process + .stdin(stdin) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + self.apply_env(&mut process); + CliChild::spawn(command, &mut process) + } + + /// The gateway and workspace the CLI is pointed at. + pub fn gateway(&self) -> &Gateway { + &self.gateway + } + + /// Point `process` at the CLI's gateway and workspace, as every CLI + /// invocation is. Use it for a program that runs the CLI itself, such as + /// `ssh` with the `ProxyCommand` from `sandbox ssh-config`. + pub fn apply_env(&self, process: &mut tokio::process::Command) { + process + .env("NO_COLOR", "1") + .env(WORKSPACE_ENV, &self.gateway.workspace); + // Select exactly the gateway the SDK connected to. + match &self.gateway.name { + Some(name) => process.env(GATEWAY_ENV, name), + None => process.env_remove(GATEWAY_ENV), + }; + if self.gateway.endpoint_from_env { + process.env(ENDPOINT_ENV, &self.gateway.endpoint); + } else { + process.env_remove(ENDPOINT_ENV); + } + } + + /// The command line for reports, and a process for it that is killed + /// when dropped. + fn command(&self, args: I) -> (String, tokio::process::Command) + where + I: IntoIterator, + S: AsRef, + { + let args: Vec<_> = args + .into_iter() + .map(|arg| arg.as_ref().to_os_string()) + .collect(); + let command = std::iter::once(self.binary.as_os_str()) + .chain(args.iter().map(std::ffi::OsString::as_os_str)) + .map(|part| part.to_string_lossy()) + .collect::>() + .join(" "); + let mut process = tokio::process::Command::new(&self.binary); + process.args(&args).kill_on_drop(true); + (command, process) + } } +/// The exit code of a finished CLI. A signal-terminated CLI has no code; +/// report it as a failure. +pub(crate) fn exit_code(status: std::process::ExitStatus) -> i32 { + status.code().unwrap_or(-1) +} + +/// The characters the CLI's error reports draw around wrapped lines: the +/// gutter, the error marker, and the box corners. +const REPORT_GUTTER: [char; 6] = ['│', '×', '╰', '─', '╭', '├']; + impl CliOutput { /// Fail unless the command exited 0. pub fn success(self) -> Result { crate::ensure!( self.exit_code == 0, - "`{}` exited with {}; stderr: {}", + "`{}` exited with {}; stdout: {}; stderr: {}", self.command, self.exit_code, + self.stdout.trim(), self.stderr.trim() ); Ok(self) @@ -173,6 +243,29 @@ impl CliOutput { Ok(self) } + /// Fail unless the command exited 0 and printed exactly `expected`. + pub fn expect_stdout(self, expected: &str) -> Result { + let output = self.success()?; + crate::ensure!( + output.stdout == expected, + "`{}` printed {:?}; expected {expected:?}", + output.command, + output.stdout + ); + Ok(output) + } + + /// Stdout and stderr as one line of words, for matching an error report. + /// The CLI wraps reports to the terminal width and draws a gutter, so + /// match against this rather than the raw output. + pub fn message(&self) -> String { + format!("{} {}", self.stdout, self.stderr) + .replace(REPORT_GUTTER, " ") + .split_whitespace() + .collect::>() + .join(" ") + } + /// Parse stdout as JSON, from a command run with `--output json`. pub fn json(&self) -> Result { serde_json::from_str(&self.stdout) diff --git a/tests/testkit/testkit/src/cli_child.rs b/tests/testkit/testkit/src/cli_child.rs new file mode 100644 index 0000000000..7ca7e6a09b --- /dev/null +++ b/tests/testkit/testkit/src/cli_child.rs @@ -0,0 +1,378 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A CLI process the test keeps running, for commands that stream output or +//! read input, such as `sandbox connect` or a piped `sandbox exec`. + +use std::process::ExitStatus; +use std::time::Duration; + +use tokio::io::{AsyncBufReadExt as _, AsyncRead, AsyncWriteExt as _, BufReader}; +use tokio::process::{Child, ChildStdin, Command}; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; + +use crate::cli::{CliOutput, exit_code}; +use crate::error::{Error, Result, ResultExt}; + +/// A running CLI started by [`crate::Cli::spawn`]. +/// +/// Stdout and stderr are read in the background as they arrive, so a CLI that +/// writes a lot never blocks on a full pipe while the test does something +/// else. Read them line by line with [`CliChild::stdout_line`] and +/// [`CliChild::stdout_until`] (and the `stderr` forms), or all at once with +/// [`CliChild::output`]. +/// +/// The process is killed when the handle is dropped, so a failing test +/// doesn't leave it running. +#[derive(Debug)] +pub struct CliChild { + command: String, + child: Child, + pid: u32, + stdin: Option, + stdout: Stream, + stderr: Stream, +} + +/// One output stream: the lines read so far and not yet taken. +#[derive(Debug)] +struct Stream { + name: &'static str, + lines: mpsc::UnboundedReceiver>, + reader: JoinHandle<()>, +} + +impl CliChild { + pub(crate) fn spawn(command: String, process: &mut Command) -> Result { + let mut child = process + .spawn() + .map_err(|error| Error::from_error(&error)) + .context(format!("start {command}"))?; + let pid = child + .id() + .ok_or_else(|| Error::msg(format!("{command} exited before it was watched")))?; + let stdout = Stream::read("stdout", child.stdout.take().expect("stdout is piped")); + let stderr = Stream::read("stderr", child.stderr.take().expect("stderr is piped")); + Ok(Self { + command, + stdin: child.stdin.take(), + child, + pid, + stdout, + stderr, + }) + } + + /// The command line, for messages. + pub fn command(&self) -> &str { + &self.command + } + + /// The process ID of the CLI. + pub fn pid(&self) -> u32 { + self.pid + } + + /// Write `bytes` to the CLI's stdin and flush it. The error is the + /// I/O error itself, so a test can accept `BrokenPipe` from a command that + /// exits before it reads all its input. + pub async fn write_stdin(&mut self, bytes: &[u8]) -> std::io::Result<()> { + let stdin = self.stdin.as_mut().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotConnected, + "stdin is closed or not piped", + ) + })?; + stdin.write_all(bytes).await?; + stdin.flush().await + } + + /// Close the CLI's stdin, so it reads end of file. + pub fn close_stdin(&mut self) { + self.stdin = None; + } + + /// The next line of stdout, without its line ending. Fails if none + /// arrives within `timeout` or stdout closes. + pub async fn stdout_line(&mut self, timeout: Duration) -> Result { + let command = &self.command; + self.stdout.next_line(command, timeout).await + } + + /// The next line of stderr, like [`CliChild::stdout_line`]. + pub async fn stderr_line(&mut self, timeout: Duration) -> Result { + let command = &self.command; + self.stderr.next_line(command, timeout).await + } + + /// Read stdout until a line matches `found`, and return the lines read, + /// the matching one last. Fails if no line matches within `timeout` or + /// stdout closes first. + pub async fn stdout_until( + &mut self, + timeout: Duration, + found: impl FnMut(&str) -> bool, + ) -> Result> { + let command = &self.command; + self.stdout.until(command, timeout, found).await + } + + /// Read stderr until a line matches `found`, like + /// [`CliChild::stdout_until`]. + pub async fn stderr_until( + &mut self, + timeout: Duration, + found: impl FnMut(&str) -> bool, + ) -> Result> { + let command = &self.command; + self.stderr.until(command, timeout, found).await + } + + /// Send a Unix signal, such as `libc::SIGTERM`, to the CLI process only. + #[cfg(unix)] + pub fn signal(&self, signal: i32) -> Result { + let pid = libc::pid_t::try_from(self.pid).map_err(Error::from_display)?; + // SAFETY: kill(2) takes plain integers and has no memory effects. + if unsafe { libc::kill(pid, signal) } != 0 { + let error = std::io::Error::last_os_error(); + return Err(Error::from_error(&error) + .context(format!("send signal {signal} to {}", self.command))); + } + Ok(()) + } + + /// Kill the CLI and wait for it to exit. + pub async fn kill(&mut self) -> Result { + self.child + .kill() + .await + .map_err(|error| Error::from_error(&error)) + .context(format!("kill {}", self.command)) + } + + /// The CLI's exit code if it has exited, without waiting. + pub fn try_wait(&mut self) -> Result> { + Ok(self + .child + .try_wait() + .map_err(|error| Error::from_error(&error)) + .context(format!("check {}", self.command))? + .map(exit_code)) + } + + /// Wait for the CLI to exit and return its exit code. Fails if it is + /// still running after `timeout`; it then keeps running until killed or + /// dropped. + pub async fn wait(&mut self, timeout: Duration) -> Result { + Ok(exit_code(self.wait_status(timeout).await?)) + } + + /// Close stdin, wait for the CLI to exit, and return its exit code with + /// the output not yet read as lines. Fails if the CLI, or a process that + /// inherited its stdout or stderr, is still running after `timeout`. + pub async fn output(mut self, timeout: Duration) -> Result { + self.close_stdin(); + let deadline = tokio::time::Instant::now() + timeout; + let status = self.wait_status(timeout).await?; + let stdout = self.stdout.rest(&self.command, deadline).await?; + let stderr = self.stderr.rest(&self.command, deadline).await?; + Ok(CliOutput { + command: self.command, + exit_code: exit_code(status), + stdout: String::from_utf8_lossy(&stdout).into_owned(), + stderr: String::from_utf8_lossy(&stderr).into_owned(), + }) + } + + async fn wait_status(&mut self, timeout: Duration) -> Result { + tokio::time::timeout(timeout, self.child.wait()) + .await + .map_err(|_| Error::msg(format!("still running after {timeout:?}"))) + .context(format!("wait for {}", self.command))? + .map_err(|error| Error::from_error(&error)) + .context(format!("wait for {}", self.command)) + } +} + +impl Stream { + fn read(name: &'static str, pipe: impl AsyncRead + Unpin + Send + 'static) -> Self { + let (sender, lines) = mpsc::unbounded_channel(); + let reader = tokio::spawn(async move { + let mut pipe = BufReader::new(pipe); + loop { + let mut line = Vec::new(); + match pipe.read_until(b'\n', &mut line).await { + Ok(0) | Err(_) => break, + Ok(_) => { + if sender.send(line).is_err() { + break; + } + } + } + } + }); + Self { + name, + lines, + reader, + } + } + + async fn next_line(&mut self, command: &str, timeout: Duration) -> Result { + match tokio::time::timeout(timeout, self.lines.recv()).await { + Ok(Some(line)) => Ok(trim_line_ending(&line)), + Ok(None) => Err(Error::msg(format!("{} of {command} closed", self.name))), + Err(_) => Err(Error::msg(format!( + "no line on {} of {command} within {timeout:?}", + self.name + ))), + } + } + + async fn until( + &mut self, + command: &str, + timeout: Duration, + mut found: impl FnMut(&str) -> bool, + ) -> Result> { + let mut read = Vec::new(); + let outcome = tokio::time::timeout(timeout, async { + while let Some(line) = self.lines.recv().await { + let line = trim_line_ending(&line); + let matched = found(&line); + read.push(line); + if matched { + return true; + } + } + false + }) + .await; + match outcome { + Ok(true) => Ok(read), + Ok(false) => Err(Error::msg(format!( + "{} of {command} closed before the expected line; read {read:?}", + self.name + ))), + Err(_) => Err(Error::msg(format!( + "no expected line on {} of {command} within {timeout:?}; read {read:?}", + self.name + ))), + } + } + + /// Everything not yet taken, once the pipe has closed. + async fn rest(&mut self, command: &str, deadline: tokio::time::Instant) -> Result> { + tokio::time::timeout_at(deadline, &mut self.reader) + .await + .map_err(|_| { + Error::msg(format!( + "{} of {command} is still open after the CLI exited", + self.name + )) + })? + .map_err(|error| Error::from_error(&error))?; + let mut bytes = Vec::new(); + while let Ok(line) = self.lines.try_recv() { + bytes.extend(line); + } + Ok(bytes) + } +} + +impl Drop for Stream { + fn drop(&mut self) { + self.reader.abort(); + } +} + +fn trim_line_ending(line: &[u8]) -> String { + let line = line.strip_suffix(b"\n").unwrap_or(line); + let line = line.strip_suffix(b"\r").unwrap_or(line); + String::from_utf8_lossy(line).into_owned() +} + +#[cfg(all(test, unix))] +mod tests { + use std::time::Duration; + + use crate::cli::Cli; + + const TIMEOUT: Duration = Duration::from_secs(10); + + /// `sh` in place of the CLI, so the tests need no gateway. + fn sh() -> Cli { + Cli::with_binary("/bin/sh") + } + + #[tokio::test] + async fn reads_lines_and_writes_stdin() { + let mut child = sh() + .spawn([ + "-c", + "echo ready; echo warning >&2; read line; echo \"got=$line\"; exit 3", + ]) + .unwrap(); + assert_eq!(child.stdout_line(TIMEOUT).await.unwrap(), "ready"); + assert_eq!(child.stderr_line(TIMEOUT).await.unwrap(), "warning"); + child.write_stdin(b"hello\n").await.unwrap(); + let read = child + .stdout_until(TIMEOUT, |line| line.starts_with("got=")) + .await + .unwrap(); + assert_eq!(read, ["got=hello"]); + assert_eq!(child.wait(TIMEOUT).await.unwrap(), 3); + } + + #[tokio::test] + async fn output_keeps_unread_bytes_exactly() { + let mut child = sh() + .spawn(["-c", "echo first; printf 'a\\r\\nb'; cat >&2"]) + .unwrap(); + assert_eq!(child.stdout_line(TIMEOUT).await.unwrap(), "first"); + child.write_stdin(b"input").await.unwrap(); + let output = child.output(TIMEOUT).await.unwrap(); + assert_eq!(output.exit_code, 0); + assert_eq!(output.stdout, "a\r\nb"); + assert_eq!(output.stderr, "input"); + } + + #[tokio::test] + async fn until_reports_what_it_read_when_the_stream_closes() { + let mut child = sh().spawn(["-c", "echo one; echo two"]).unwrap(); + let error = child + .stdout_until(TIMEOUT, |line| line == "three") + .await + .unwrap_err() + .to_string(); + assert!(error.contains("closed before the expected line"), "{error}"); + assert!(error.contains(r#"["one", "two"]"#), "{error}"); + } + + #[tokio::test] + async fn signal_reaches_the_process() { + let mut child = sh() + .spawn([ + "-c", + "trap 'exit 42' TERM; echo ready; while :; do sleep 0.1; done", + ]) + .unwrap(); + assert_eq!(child.stdout_line(TIMEOUT).await.unwrap(), "ready"); + assert_eq!(child.try_wait().unwrap(), None); + child.signal(libc::SIGTERM).unwrap(); + assert_eq!(child.wait(TIMEOUT).await.unwrap(), 42); + } + + #[tokio::test] + async fn wait_times_out_on_a_running_process() { + let mut child = sh().spawn(["-c", "sleep 30"]).unwrap(); + let error = child + .wait(Duration::from_millis(100)) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("still running"), "{error}"); + child.kill().await.unwrap(); + } +} diff --git a/tests/testkit/testkit/src/cli_sandbox.rs b/tests/testkit/testkit/src/cli_sandbox.rs new file mode 100644 index 0000000000..8d58968cd2 --- /dev/null +++ b/tests/testkit/testkit/src/cli_sandbox.rs @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandboxes created through `openshell sandbox create`, for tests of what +//! only the CLI offers, such as `--policy` files, `--no-keep`, or an attached +//! main command. + +use crate::cli::CliOutput; +use crate::context::{Context, Sandbox}; +use crate::error::{Result, ResultExt}; +use crate::yaml_file::YamlFile; + +/// A sandbox to create with `openshell sandbox create`. Start one with +/// [`Context::cli_sandbox`]. +/// +/// The sandbox is tracked for cleanup before the CLI runs, so a create the +/// gateway accepted but the CLI reported as failed is still cleaned up. The +/// CLI runs with `--no-tty`: tests never have a terminal. +pub struct CliSandboxBuilder<'a> { + cx: &'a Context, + name: String, + args: Vec, + policy: Option, + command: Vec, +} + +impl<'a> CliSandboxBuilder<'a> { + pub(crate) fn new(cx: &'a Context, name: String) -> Self { + Self { + cx, + name, + args: Vec::new(), + policy: None, + command: Vec::new(), + } + } + + /// The sandbox's name, for building the command before the create. + pub fn name(&self) -> &str { + &self.name + } + + /// Create the sandbox from `image` (`--from`). + #[must_use] + pub fn image(self, image: &str) -> Self { + self.args(["--from", image]) + } + + /// Create the sandbox with this policy YAML (`--policy`). It is written + /// to a temporary file that lives until the CLI exits. + #[must_use] + pub fn policy_yaml(mut self, yaml: &str) -> Self { + self.policy = Some(yaml.to_string()); + self + } + + /// Other `sandbox create` arguments, such as `--no-keep` or `--label`. + #[must_use] + pub fn args(mut self, args: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.args.extend(args.into_iter().map(Into::into)); + self + } + + /// The sandbox's main command. Empty selects the gateway's default. + #[must_use] + pub fn command(mut self, command: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.command = command.into_iter().map(Into::into).collect(); + self + } + + /// Create the sandbox with `--detach`, require the CLI to succeed, and + /// return the sandbox. It need not be Ready yet. + pub async fn create(mut self) -> Result { + self.args.push("--detach".to_string()); + self.invoke() + .await? + .success() + .context(format!("create sandbox {}", self.name))?; + self.cx.adopt(&self.name).await + } + + /// Create the sandbox and wait until it is Ready. + pub async fn create_ready(self) -> Result { + let sandbox = self.create().await?; + sandbox.wait_ready().await?; + Ok(sandbox) + } + + /// Create the sandbox attached to its main command and return the CLI's + /// output, whatever its exit code. For a test of the main command's + /// result, or of a create that must fail. + pub async fn run(self) -> Result { + self.invoke().await + } + + async fn invoke(&self) -> Result { + self.cx.track(&self.name); + let policy = self.policy.as_deref().map(YamlFile::new).transpose()?; + let mut args = vec!["sandbox", "create", "--name", &self.name, "--no-tty"]; + if let Some(policy) = &policy { + args.extend(["--policy", policy.path()?]); + } + args.extend(self.args.iter().map(String::as_str)); + if !self.command.is_empty() { + args.push("--"); + args.extend(self.command.iter().map(String::as_str)); + } + self.cx.cli(args).await + } +} diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index ea20878837..d491bf6725 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -14,6 +14,8 @@ use rand::Rng; use crate::cleanup::{CLEANUP_TIMEOUT, Cleanup}; use crate::cli::{Cli, CliOutput}; +use crate::cli_child::CliChild; +use crate::cli_sandbox::CliSandboxBuilder; use crate::error::{Error, Result, ResultExt}; use crate::poll::{Poll, poll_until}; @@ -78,6 +80,30 @@ impl Context { self.cli.run_with_env(None, env, args).await } + /// Start the CLI with `args` and keep it running, with stdin, stdout, + /// and stderr piped to the test. See [`CliChild`]. + pub fn cli_spawn(&self, args: I) -> Result + where + I: IntoIterator, + S: AsRef, + { + self.cli.spawn(args) + } + + /// Like [`Context::cli_spawn`], with `stdin` as the CLI's stdin, such + /// as [`std::process::Stdio::null`], a file, or a pseudo-terminal. + pub fn cli_spawn_with_stdin( + &self, + args: I, + stdin: impl Into, + ) -> Result + where + I: IntoIterator, + S: AsRef, + { + self.cli.spawn_with_stdin(args, stdin) + } + /// The CLI that [`Context::cli`] runs, for code that can't borrow the /// context, such as a cleanup hook. pub fn cli_handle(&self) -> Cli { @@ -104,6 +130,7 @@ impl Context { SandboxBuilder { client: self.client.clone(), grpc: self.grpc.clone(), + cli: self.cli.clone(), policy: None, cleanup: self.cleanup.clone(), spec: SandboxSpec { @@ -113,6 +140,14 @@ impl Context { } } + /// Start describing a sandbox named `cx.name(suffix)` that the + /// `openshell` CLI creates. Use it for what only `sandbox create` + /// offers, such as a `--policy` file or `--no-keep`; prefer + /// [`Context::sandbox`] otherwise. + pub fn cli_sandbox(&self, suffix: &str) -> CliSandboxBuilder<'_> { + CliSandboxBuilder::new(self, self.name(suffix)) + } + /// Track a sandbox the test is about to create through the CLI, so cleanup /// deletes it even if the test fails after the gateway accepted the create. /// Call this before the CLI command; use [`Context::adopt`] afterwards. @@ -173,37 +208,13 @@ impl Context { Ok(Sandbox { client: self.client.clone(), grpc: self.grpc.clone(), + cli: self.cli.clone(), cleanup: self.cleanup.clone(), name: name.to_string(), id: found.id, }) } - /// Fail unless the sandbox is gone from both get and list. - pub async fn expect_absent(&self, sandbox: &Sandbox) -> Result { - let name = sandbox.name(); - match self.client.get_sandbox(name).await { - Err(SdkError::NotFound { .. }) => {} - Err(error) => return Err(Error::from(error).context(format!("get sandbox {name}"))), - Ok(found) => { - return Err(Error::msg(format!( - "sandbox {name} still exists (id {}, phase {:?})", - found.id, found.phase - ))); - } - } - let listed = self - .client - .list_all_sandboxes(ListOptions::default()) - .await - .context("list sandboxes")?; - crate::ensure!( - !listed.iter().any(|entry| entry.name == name), - "sandbox {name} is absent from get but still listed" - ); - Ok(()) - } - /// Fail unless the gateway runs exactly one compute driver, `name`, as /// gateway info reports it. Driver suites check this first, so a suite /// pointed at another driver's gateway fails instead of testing it. @@ -214,7 +225,6 @@ impl Context { .await? .get_gateway_info(proto::GetGatewayInfoRequest {}) .await - .map_err(|status| Error::from_error(&status)) .context("get gateway info")? .into_inner(); let drivers: Vec<(&str, &str)> = info @@ -235,25 +245,6 @@ impl Context { Ok(()) } - /// Fail unless `list` shows the sandbox in `phase`. - pub async fn expect_listed(&self, sandbox: &Sandbox, phase: SandboxPhase) -> Result { - let name = sandbox.name(); - let listed = self - .client - .list_all_sandboxes(ListOptions::default()) - .await - .context("list sandboxes")?; - let entry = listed.iter().find(|entry| entry.name == name); - crate::ensure!(entry.is_some(), "sandbox {name} is not listed"); - let entry = entry.expect("checked above"); - crate::ensure!( - entry.phase == phase, - "sandbox {name} is listed in phase {:?}; expected {phase:?}", - entry.phase - ); - Ok(()) - } - /// Run the cleanup hooks, including deleting the sandboxes the test /// created and has not deleted. Returns the cleanup failures; they are /// reported separately from the test result. @@ -304,6 +295,7 @@ impl Grpc { pub struct SandboxBuilder { client: WorkspaceScopedClient, grpc: Grpc, + cli: Cli, policy: Option, cleanup: Cleanup, spec: SandboxSpec, @@ -321,6 +313,13 @@ impl SandboxBuilder { self } + /// Create the sandbox from `image` instead of the gateway's default. + #[must_use] + pub fn image(mut self, image: impl Into) -> Self { + self.spec.image = Some(image.into()); + self + } + #[must_use] pub fn restart_policy(mut self, policy: SandboxRestartPolicy) -> Self { self.spec.restart_policy = policy; @@ -341,6 +340,13 @@ impl SandboxBuilder { self } + /// Create the sandbox and wait until it is Ready. + pub async fn create_ready(self) -> Result { + let sandbox = self.create().await?; + sandbox.wait_ready().await?; + Ok(sandbox) + } + /// Create the sandbox. It is tracked for cleanup before the request is /// sent, so a create that fails after the gateway accepted it is still /// cleaned up. @@ -358,32 +364,15 @@ impl SandboxBuilder { // The SDK's spec has no policy field, so create through gRPC. Some(policy) => { crate::ensure!( - self.spec.image.is_none() - && self.spec.environment.is_empty() - && self.spec.providers.is_empty() - && !self.spec.gpu - && self.spec.service_exposures.is_empty(), - "testkit creates a sandbox with a policy from its name, labels, command, restart policy, and policy only" + !self.spec.gpu && self.spec.service_exposures.is_empty(), + "testkit doesn't create a sandbox with a policy and a GPU or service exposures" ); - let request = proto::CreateSandboxRequest { - name: name.clone(), - labels: self.spec.labels, - workspace_scope: self.grpc.scope(), - spec: Some(proto::SandboxSpec { - command: self.spec.command, - tty: self.spec.tty, - policy: Some(policy), - restart_policy: restart_policy(self.spec.restart_policy) as i32, - ..proto::SandboxSpec::default() - }), - ..proto::CreateSandboxRequest::default() - }; + let request = policy_create_request(self.spec, policy, self.grpc.scope()); self.grpc .client() .await? .create_sandbox(request) .await - .map_err(|status| Error::from_error(&status)) .context(format!("create sandbox {name}"))?; self.client .get_sandbox(&name) @@ -395,6 +384,7 @@ impl SandboxBuilder { Ok(Sandbox { client: self.client, grpc: self.grpc, + cli: self.cli, cleanup: self.cleanup, name, id, @@ -402,11 +392,47 @@ impl SandboxBuilder { } } -const fn restart_policy(policy: SandboxRestartPolicy) -> proto::SandboxRestartPolicy { - match policy { - SandboxRestartPolicy::Never => proto::SandboxRestartPolicy::Never, - SandboxRestartPolicy::OnFailure => proto::SandboxRestartPolicy::OnFailure, - SandboxRestartPolicy::Always => proto::SandboxRestartPolicy::Always, +/// The gRPC request for a sandbox created from `spec` with `policy`, built +/// as the SDK builds one from `spec` alone. The spec is destructured without +/// `..`, so a field added to the SDK's spec fails to compile here instead of +/// being dropped. [`SandboxBuilder::create`] rejects a GPU or service +/// exposures before calling this. A policy field on the SDK's spec would +/// replace this (#5410). +fn policy_create_request( + spec: SandboxSpec, + policy: proto::SandboxPolicy, + workspace_scope: Option, +) -> proto::CreateSandboxRequest { + let SandboxSpec { + name, + image, + labels, + environment, + providers, + gpu: _, + command, + tty, + service_exposures: _, + restart_policy, + } = spec; + proto::CreateSandboxRequest { + name: name.unwrap_or_default(), + labels, + workspace_scope, + spec: Some(proto::SandboxSpec { + environment, + template: image.map(|image| proto::SandboxTemplate { + image, + ..proto::SandboxTemplate::default() + }), + providers, + command, + tty, + policy: Some(policy), + restart_policy: proto::SandboxRestartPolicy::from(restart_policy) as i32, + ..proto::SandboxSpec::default() + }), + ..proto::CreateSandboxRequest::default() } } @@ -414,6 +440,7 @@ const fn restart_policy(policy: SandboxRestartPolicy) -> proto::SandboxRestartPo pub struct Sandbox { client: WorkspaceScopedClient, grpc: Grpc, + cli: Cli, cleanup: Cleanup, name: String, id: String, @@ -422,11 +449,52 @@ pub struct Sandbox { /// The buffered output of an exec. #[derive(Debug)] pub struct ExecOutput { + /// The command and the sandbox it ran in, for reports. + pub command: String, pub exit_code: i32, pub stdout: String, pub stderr: String, } +impl ExecOutput { + /// Fail unless the command exited 0. + pub fn success(self) -> Result { + crate::ensure!( + self.exit_code == 0, + "{} exited with {}; stdout: {}; stderr: {}", + self.command, + self.exit_code, + self.stdout.trim(), + self.stderr.trim() + ); + Ok(self) + } + + /// Fail unless the command exited non-zero, e.g. when a request must be + /// rejected. + pub fn failure(self) -> Result { + crate::ensure!( + self.exit_code != 0, + "{} succeeded but was expected to fail; stdout: {}", + self.command, + self.stdout.trim() + ); + Ok(self) + } + + /// Fail unless the command exited 0 and printed exactly `expected`. + pub fn expect_stdout(self, expected: &str) -> Result { + let output = self.success()?; + crate::ensure!( + output.stdout == expected, + "{} printed {:?}; expected {expected:?}", + output.command, + output.stdout + ); + Ok(output) + } +} + impl Sandbox { pub fn name(&self) -> &str { &self.name @@ -522,7 +590,6 @@ impl Sandbox { .await? .update_config(request) .await - .map_err(|status| Error::from_error(&status)) .context(format!("set {key} on sandbox {}", self.name))?; Ok(()) } @@ -539,7 +606,6 @@ impl Sandbox { .await? .get_sandbox_config(request) .await - .map_err(|status| Error::from_error(&status)) .context(format!("get config of sandbox {}", self.name))? .into_inner()) } @@ -557,7 +623,6 @@ impl Sandbox { .await? .get_draft_policy(request) .await - .map_err(|status| Error::from_error(&status)) .context(format!("get draft policy of sandbox {}", self.name))? .into_inner()) } @@ -577,6 +642,7 @@ impl Sandbox { S: Into, { let command: Vec = command.into_iter().map(Into::into).collect(); + let description = format!("exec {command:?} in sandbox {}", self.name); let options = ExecOptions { timeout: Some(EXEC_TIMEOUT), ..ExecOptions::default() @@ -585,14 +651,112 @@ impl Sandbox { .client .exec(&self.name, &command, options) .await - .context(format!("exec {command:?} in sandbox {}", self.name))?; + .context(&description)?; Ok(ExecOutput { + command: description, exit_code: result.exit_code, stdout: String::from_utf8_lossy(&result.stdout).into_owned(), stderr: String::from_utf8_lossy(&result.stderr).into_owned(), }) } + /// Run `command` every 250 ms until `ready` accepts its output, and return + /// that output. An exec error fails at once; a timeout reports the last + /// output. + pub async fn poll_exec( + &self, + what: &str, + timeout: Duration, + command: I, + ready: impl Fn(&ExecOutput) -> bool, + ) -> Result + where + I: IntoIterator, + S: Into, + { + let command: Vec = command.into_iter().map(Into::into).collect(); + poll_until(what, timeout, Duration::from_millis(250), async || { + let output = self.exec(command.iter().cloned()).await?; + Ok(if ready(&output) { + Poll::Ready(output) + } else { + Poll::Pending(format!( + "exited {} with stdout {:?}; stderr: {}", + output.exit_code, + output.stdout, + output.stderr.trim() + )) + }) + }) + .await + } + + /// `sandbox get --output json`. It reports fields the SDK does not + /// expose, such as `restart_count`, `restart_policy`, + /// `main_process_instance_id`, and `conditions`. + pub async fn get_json(&self) -> Result { + self.cli + .run(["sandbox", "get", &self.name, "--output", "json"]) + .await? + .success()? + .json() + } + + /// Fail unless `list` shows the sandbox in `phase`. + pub async fn expect_listed(&self, phase: SandboxPhase) -> Result { + let name = &self.name; + let listed = self + .client + .list_all_sandboxes(ListOptions::default()) + .await + .context("list sandboxes")?; + let entry = listed.iter().find(|entry| entry.name == *name); + crate::ensure!(entry.is_some(), "sandbox {name} is not listed"); + let entry = entry.expect("checked above"); + crate::ensure!( + entry.phase == phase, + "sandbox {name} is listed in phase {:?}; expected {phase:?}", + entry.phase + ); + Ok(()) + } + + /// Fail unless the sandbox is gone from both get and list. + pub async fn expect_absent(&self) -> Result { + let name = &self.name; + match self.client.get_sandbox(name).await { + Err(SdkError::NotFound { .. }) => {} + Err(error) => return Err(Error::from(error).context(format!("get sandbox {name}"))), + Ok(found) => { + return Err(Error::msg(format!( + "sandbox {name} still exists (id {}, phase {:?})", + found.id, found.phase + ))); + } + } + let listed = self + .client + .list_all_sandboxes(ListOptions::default()) + .await + .context("list sandboxes")?; + crate::ensure!( + !listed.iter().any(|entry| entry.name == *name), + "sandbox {name} is absent from get but still listed" + ); + Ok(()) + } + + /// Wait up to `timeout` for the gateway to remove the sandbox by itself, + /// for example after a `--no-keep` main command exits, then check it is + /// gone from both get and list. + pub async fn wait_absent(&self, timeout: Duration) -> Result { + self.client + .wait_deleted(&self.name, timeout, Some(&self.id)) + .await + .context(format!("wait for sandbox {} to be removed", self.name))?; + self.expect_absent().await + } + /// Delete the sandbox and wait until it is gone. pub async fn delete(&self) -> Result { self.client @@ -631,3 +795,26 @@ fn generate_run_id() -> String { .map(|_| char::from(ALPHABET[rng.random_range(0..ALPHABET.len())])) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn policy_create_request_keeps_the_spec_and_adds_the_policy() { + let spec = SandboxSpec { + image: Some("python:3".to_string()), + providers: vec!["provider".to_string()], + ..SandboxSpec::default() + }; + let policy = crate::policy::base(); + + let spec = policy_create_request(spec, policy.clone(), None) + .spec + .expect("spec"); + + assert_eq!(spec.template.expect("template").image, "python:3"); + assert_eq!(spec.providers, ["provider"]); + assert_eq!(spec.policy, Some(policy)); + } +} diff --git a/tests/testkit/testkit/src/error.rs b/tests/testkit/testkit/src/error.rs index 70a702ddc1..3434e3fb1b 100644 --- a/tests/testkit/testkit/src/error.rs +++ b/tests/testkit/testkit/src/error.rs @@ -7,10 +7,12 @@ use std::fmt; use openshell_sdk::SdkError; -/// A test failure: a message with the context it happened in. +/// A test failure: a message with the context it happened in, and the gRPC +/// status code when the gateway refused a request. #[derive(Debug)] pub struct Error { message: String, + code: Option, } /// The result of a test or test step. @@ -20,9 +22,15 @@ impl Error { pub fn msg(message: impl Into) -> Self { Self { message: message.into(), + code: None, } } + /// The gRPC status code, when the gateway refused a request. + pub fn grpc_code(&self) -> Option { + self.code + } + pub fn from_display(error: impl fmt::Display) -> Self { Self::msg(error.to_string()) } @@ -45,7 +53,10 @@ impl Error { #[must_use] pub fn context(self, context: impl fmt::Display) -> Self { - Self::msg(format!("{context}: {}", self.message)) + Self { + message: format!("{context}: {}", self.message), + code: self.code, + } } } @@ -59,6 +70,25 @@ impl std::error::Error for Error {} impl From for Error { fn from(error: SdkError) -> Self { + Self { + code: error.grpc_status().map(tonic::Status::code), + ..Self::from_error(&error) + } + } +} + +/// A gateway's refusal, keeping its status code for [`Error::grpc_code`]. +impl From for Error { + fn from(status: tonic::Status) -> Self { + Self { + code: Some(status.code()), + ..Self::from_error(&status) + } + } +} + +impl From for Error { + fn from(error: std::io::Error) -> Self { Self::from_error(&error) } } diff --git a/tests/testkit/testkit/src/image.rs b/tests/testkit/testkit/src/image.rs index 26bb2278fe..07a6e2d937 100644 --- a/tests/testkit/testkit/src/image.rs +++ b/tests/testkit/testkit/src/image.rs @@ -9,7 +9,9 @@ const BASE_IMAGE_ENV: &str = "OPENSHELL_TEST_BASE_IMAGE"; /// nvcr.io. const DEFAULT_BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; -/// Overrides [`DEFAULT_PYTHON_IMAGE`]. +/// Overrides [`DEFAULT_PYTHON_IMAGE`]. The image must run `python3` from one +/// of [`PYTHON_BINARIES`]; otherwise policies that allow those paths don't +/// match it, and its connections are denied. const PYTHON_IMAGE_ENV: &str = "OPENSHELL_TEST_PYTHON_IMAGE"; /// The workload image for tests that run `python3`, curl, git or openssl in a /// sandbox, built from `e2e/python/Dockerfile.workload`. It isn't published: @@ -17,6 +19,16 @@ const PYTHON_IMAGE_ENV: &str = "OPENSHELL_TEST_PYTHON_IMAGE"; /// `mise run e2e:workload:build` builds it locally. const DEFAULT_PYTHON_IMAGE: &str = "openshell/e2e-python:dev"; +/// The paths `python3` runs from in [`python_image`] images, as network +/// policy binaries: the system Python, the pullable uv image's Python, and the +/// workload image's uv-installed Python behind its virtualenv. Add a path +/// here for an image that installs Python elsewhere. +pub const PYTHON_BINARIES: &[&str] = &[ + "/usr/bin/python*", + "/usr/local/bin/python*", + "/sandbox/.uv/python/*/bin/python*", +]; + /// The image named by [`BASE_IMAGE_ENV`], or [`DEFAULT_BASE_IMAGE`]. pub fn base_image() -> String { image_from_env(BASE_IMAGE_ENV, DEFAULT_BASE_IMAGE) diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 834f28c1e7..7cd67cc4e7 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -28,32 +28,41 @@ mod cleanup; mod cli; +mod cli_child; +mod cli_sandbox; mod context; mod error; pub mod gateway; mod image; +pub mod policy; mod poll; pub mod program; +mod provider; mod ready; mod registry; mod run; +mod yaml_file; pub use cleanup::CLEANUP_TIMEOUT; pub use cli::{CLI_TIMEOUT, Cli, CliOutput}; +pub use cli_child::CliChild; +pub use cli_sandbox::CliSandboxBuilder; pub use context::{ Context, DELETE_TIMEOUT, EXEC_TIMEOUT, ExecOutput, READY_TIMEOUT, STOP_TIMEOUT, Sandbox, SandboxBuilder, }; pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; -pub use image::{base_image, python_image}; +pub use image::{PYTHON_BINARIES, base_image, python_image}; pub use openshell_sdk as sdk; -pub use poll::{Poll, poll_until}; +pub use poll::{Poll, poll_until, with_deadline}; pub use program::main; pub use ready::check as check_ready; pub use registry::Test; pub use run::{TestInfo, run}; pub use testkit_macros::test; +pub use tonic::Code as GrpcCode; +pub use yaml_file::YamlFile; #[doc(hidden)] pub mod __private { diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs new file mode 100644 index 0000000000..ab7be7cae3 --- /dev/null +++ b/tests/testkit/testkit/src/policy.rs @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox policies for tests: [`base`], read from `base-policy.yaml`, plus +//! [`allow`] and the [`Endpoint`] settings for the network access a test +//! needs. +//! +//! ```ignore +//! let mut policy = testkit::policy::base(); +//! testkit::policy::allow(&mut policy, "upstream", "host.openshell.internal", port, PYTHON_BINARIES) +//! .rest(Access::Full) +//! .credential_binding(&provider); +//! ``` + +use openshell_sdk::raw::proto; + +use crate::error::{Error, Result, ResultExt as _}; + +pub use proto::NetworkAccessPreset as Access; + +const BASE: &str = include_str!("base-policy.yaml"); + +/// The policy in `base-policy.yaml`. It allows no network access. +pub fn base() -> proto::SandboxPolicy { + openshell_policy::parse_sandbox_policy(BASE).expect("base-policy.yaml is a valid policy") +} + +/// Let `binaries` connect to `host:port` under the network policy `name`, +/// adding the policy if it doesn't exist. Returns the new endpoint, which +/// [`Endpoint`] configures further. +/// +/// A policy's binaries apply to all its endpoints, so calling this again with +/// the same `name` adds `binaries` to the existing ones, and every endpoint +/// under `name` allows all of them. Use a different `name` to keep them apart. +pub fn allow<'a>( + policy: &'a mut proto::SandboxPolicy, + name: &str, + host: &str, + port: u16, + binaries: &[&str], +) -> &'a mut proto::NetworkEndpoint { + let rule = policy + .network_policies + .entry(name.to_string()) + .or_insert_with(|| proto::NetworkPolicyRule { + name: name.to_string(), + ..proto::NetworkPolicyRule::default() + }); + for binary in binaries { + if !rule + .binaries + .iter() + .any(|existing| existing.path == *binary) + { + rule.binaries.push(proto::NetworkBinary { + path: (*binary).to_string(), + }); + } + } + rule.endpoints.push(proto::NetworkEndpoint { + host: host.to_string(), + port: port.into(), + ports: vec![port.into()], + ..proto::NetworkEndpoint::default() + }); + rule.endpoints + .last_mut() + .expect("an endpoint was just added") +} + +/// The policy as YAML, for CLI commands such as `policy set`. +pub fn to_yaml(policy: &proto::SandboxPolicy) -> Result { + openshell_policy::serialize_sandbox_policy(policy) + .map_err(|error| Error::msg(format!("{error:?}"))) + .context("serialize the policy") +} + +/// Settings for an endpoint from [`allow`], named as in policy YAML. +pub trait Endpoint { + /// `protocol: rest` with `access`. + fn rest(&mut self, access: Access) -> &mut Self; + /// `protocol: websocket` with `access`. + fn websocket(&mut self, access: Access) -> &mut Self; + /// `tls: skip`. + fn tls_skip(&mut self) -> &mut Self; + /// `request_body_credential_rewrite`. + fn request_body_credential_rewrite(&mut self, rewrite: bool) -> &mut Self; + /// `credential_binding: {provider: }`. + fn credential_binding(&mut self, provider: &str) -> &mut Self; + /// `allow_uninspected_credentials: true`. + fn allow_uninspected_credentials(&mut self) -> &mut Self; +} + +impl Endpoint for proto::NetworkEndpoint { + fn rest(&mut self, access: Access) -> &mut Self { + self.protocol = "rest".to_string(); + self.access = access as i32; + self + } + + fn websocket(&mut self, access: Access) -> &mut Self { + self.protocol = "websocket".to_string(); + self.access = access as i32; + self + } + + fn tls_skip(&mut self) -> &mut Self { + self.tls = proto::NetworkTlsMode::Skip as i32; + self + } + + fn request_body_credential_rewrite(&mut self, rewrite: bool) -> &mut Self { + self.request_body_credential_rewrite = rewrite; + self + } + + fn credential_binding(&mut self, provider: &str) -> &mut Self { + self.credential_binding = Some(proto::NetworkCredentialBinding { + provider: provider.to_string(), + }); + self + } + + fn allow_uninspected_credentials(&mut self) -> &mut Self { + self.allow_uninspected_credentials = true; + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Each helper builds what the YAML it is named after parses to, so the + /// gateway can't tell the two apart. + #[test] + fn helpers_match_the_yaml_they_name() { + let mut policy = base(); + allow( + &mut policy, + "upstream", + "example.test", + 8080, + &["/usr/bin/python*"], + ) + .rest(Access::Full) + .tls_skip() + .request_body_credential_rewrite(true) + .credential_binding("provider"); + allow( + &mut policy, + "upstream", + "example.test", + 8081, + &["/usr/bin/python*"], + ) + .websocket(Access::ReadWrite) + .allow_uninspected_credentials(); + + let yaml = format!( + "{BASE}network_policies: + upstream: + name: upstream + endpoints: + - host: example.test + port: 8080 + protocol: rest + access: full + tls: skip + request_body_credential_rewrite: true + credential_binding: {{provider: provider}} + - host: example.test + port: 8081 + protocol: websocket + access: read-write + allow_uninspected_credentials: true + binaries: + - path: /usr/bin/python* +" + ); + let expected = openshell_policy::parse_sandbox_policy(&yaml).expect("the YAML parses"); + assert_eq!(policy, expected); + let round_trip = + openshell_policy::parse_sandbox_policy(&to_yaml(&policy).expect("serializes")) + .expect("the serialized policy parses"); + assert_eq!(round_trip, expected); + } +} diff --git a/tests/testkit/testkit/src/poll.rs b/tests/testkit/testkit/src/poll.rs index 838d89526d..f65592be88 100644 --- a/tests/testkit/testkit/src/poll.rs +++ b/tests/testkit/testkit/src/poll.rs @@ -15,6 +15,18 @@ pub enum Poll { Pending(String), } +/// Fail unless `future` finishes within `timeout`. For a deadline shorter +/// than the step's own, such as one that must fit inside the runner's kill. +pub async fn with_deadline( + what: &str, + timeout: Duration, + future: impl Future>, +) -> Result { + tokio::time::timeout(timeout, future) + .await + .map_err(|_| Error::msg(format!("{what} did not finish within {timeout:?}")))? +} + /// Observe until the condition holds or `timeout` elapses. /// /// An error from `observe` fails immediately. On timeout, the error reports diff --git a/tests/testkit/testkit/src/provider.rs b/tests/testkit/testkit/src/provider.rs new file mode 100644 index 0000000000..c82901c74a --- /dev/null +++ b/tests/testkit/testkit/src/provider.rs @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use serde::Deserialize; + +use crate::context::Context; +use crate::error::{Error, Result, ResultExt as _}; +use crate::yaml_file::YamlFile; + +impl Context { + /// Import `profile`, a provider profile in YAML, and return its `id`, the + /// provider type it defines. Cleanup deletes the profile after the test's + /// sandboxes and the providers created after it. + pub async fn import_profile(&self, profile: &str) -> Result { + #[derive(Deserialize)] + struct Profile { + id: String, + } + let id = serde_yml::from_str::(profile) + .map_err(|error| Error::from_error(&error)) + .context("read the provider profile's id")? + .id; + // Deleting a missing profile succeeds, so register the delete before + // the import. + self.on_cleanup_delete("profile", &id); + let file = YamlFile::new(profile)?; + self.cli(["profile", "import", "--file", file.path()?]) + .await? + .success() + .context(format!("import profile {id}"))?; + Ok(id) + } + + /// Create a provider called `name` of `provider_type` with `credentials`, + /// as pairs of environment variable and value. Cleanup deletes the + /// provider after the test's sandboxes. + /// + /// The CLI reads the credential values from its environment, so they + /// aren't in the reported command. + pub async fn create_provider( + &self, + name: &str, + provider_type: &str, + credentials: &[(&str, &str)], + ) -> Result { + // Deleting a missing provider succeeds, so register the delete before + // the create. + self.on_cleanup_delete("provider", name); + let mut args = vec![ + "provider", + "create", + "--name", + name, + "--type", + provider_type, + ]; + for (key, _) in credentials { + args.extend(["--credential", key]); + } + self.cli_with_env(credentials, args) + .await? + .success() + .context(format!("create provider {name}"))?; + Ok(()) + } + + /// Import `profile` with [`Context::import_profile`] and create a provider + /// called `name` of its type with [`Context::create_provider`]. + pub async fn install_provider( + &self, + name: &str, + profile: &str, + credentials: &[(&str, &str)], + ) -> Result { + let provider_type = self.import_profile(profile).await?; + self.create_provider(name, &provider_type, credentials) + .await + } + + fn on_cleanup_delete(&self, kind: &'static str, name: &str) { + let cli = self.cli_handle(); + let owned = name.to_string(); + self.on_cleanup(format!("delete {kind} {name}"), async move || { + cli.run([kind, "delete", &owned]).await?.success()?; + Ok(()) + }); + } +} diff --git a/tests/testkit/testkit/src/yaml_file.rs b/tests/testkit/testkit/src/yaml_file.rs new file mode 100644 index 0000000000..4549c7e95b --- /dev/null +++ b/tests/testkit/testkit/src/yaml_file.rs @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::error::{Error, Result, ResultExt as _}; + +/// YAML in a temporary file, for CLI commands that read one, such as +/// `sandbox create --policy` or `profile import --file`. The file is removed +/// when dropped. +pub struct YamlFile(tempfile::NamedTempFile); + +impl YamlFile { + pub fn new(contents: &str) -> Result { + let file = tempfile::Builder::new() + .suffix(".yaml") + .tempfile() + .context("create a temporary YAML file")?; + std::fs::write(file.path(), contents) + .context(format!("write {}", file.path().display()))?; + Ok(Self(file)) + } + + /// The file's path, as a CLI argument. + pub fn path(&self) -> Result<&str> { + let path = self.0.path(); + path.to_str() + .ok_or_else(|| Error::msg(format!("{} is not UTF-8", path.display()))) + } +} diff --git a/tests/tmachine/artifacts.nix b/tests/tmachine/artifacts.nix index ef1be7ed98..465bc8b2fe 100644 --- a/tests/tmachine/artifacts.nix +++ b/tests/tmachine/artifacts.nix @@ -106,11 +106,6 @@ let # rootless guest; the other tests in the binary pass. Nextest archive # filters cannot select individual tests, so keep the complete binary here. "proxy_egress_pipeline" - # Conformance covers stop/start workspace preservation and deletion while - # stopped. The remaining canonical-main, TTY, attachment replay, and - # no-keep cases still need migration. Nextest archive filters cannot select - # individual tests, so keep the complete binary in the follow-up bucket. - "sandbox_lifecycle" # Needs a prebuilt musl DNS probe in guest artifact mode; tracked in #3009. "transparent_tcp" "workspace_lifecycle"