From 3ae31085bce542687ef574c1e83d0ac788a5f452 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 5 Nov 2026 14:04:46 -0800 Subject: [PATCH 01/16] feat(testkit): create a sandbox with a policy, an image, and providers Signed-off-by: Matthew Grossman --- tests/testkit/testkit/src/context.rs | 90 ++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 19 deletions(-) diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index ea20878837..995a6869d3 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -358,26 +358,10 @@ impl SandboxBuilder { // The SDK's spec has no policy field, so create through gRPC. Some(policy) => { crate::ensure!( - self.spec.image.is_none() - && self.spec.environment.is_empty() - && self.spec.providers.is_empty() - && !self.spec.gpu - && self.spec.service_exposures.is_empty(), - "testkit creates a sandbox with a policy from its name, labels, command, restart policy, and policy only" + !self.spec.gpu && self.spec.service_exposures.is_empty(), + "testkit doesn't create a sandbox with a policy and a GPU or service exposures" ); - let request = proto::CreateSandboxRequest { - name: name.clone(), - labels: self.spec.labels, - workspace_scope: self.grpc.scope(), - spec: Some(proto::SandboxSpec { - command: self.spec.command, - tty: self.spec.tty, - policy: Some(policy), - restart_policy: restart_policy(self.spec.restart_policy) as i32, - ..proto::SandboxSpec::default() - }), - ..proto::CreateSandboxRequest::default() - }; + let request = policy_create_request(self.spec, policy, self.grpc.scope()); self.grpc .client() .await? @@ -402,6 +386,34 @@ impl SandboxBuilder { } } +/// The gRPC request for a sandbox created from `spec` with `policy`, built +/// as the SDK builds one from `spec` alone. +fn policy_create_request( + spec: SandboxSpec, + policy: proto::SandboxPolicy, + workspace_scope: Option, +) -> proto::CreateSandboxRequest { + proto::CreateSandboxRequest { + name: spec.name.unwrap_or_default(), + labels: spec.labels, + workspace_scope, + spec: Some(proto::SandboxSpec { + environment: spec.environment, + template: spec.image.map(|image| proto::SandboxTemplate { + image, + ..proto::SandboxTemplate::default() + }), + providers: spec.providers, + command: spec.command, + tty: spec.tty, + policy: Some(policy), + restart_policy: restart_policy(spec.restart_policy) as i32, + ..proto::SandboxSpec::default() + }), + ..proto::CreateSandboxRequest::default() + } +} + const fn restart_policy(policy: SandboxRestartPolicy) -> proto::SandboxRestartPolicy { match policy { SandboxRestartPolicy::Never => proto::SandboxRestartPolicy::Never, @@ -631,3 +643,43 @@ fn generate_run_id() -> String { .map(|_| char::from(ALPHABET[rng.random_range(0..ALPHABET.len())])) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn policy_create_request_keeps_the_spec_and_adds_the_policy() { + let spec = SandboxSpec { + name: Some("test-sb".to_string()), + image: Some("python:3".to_string()), + labels: [("k".to_string(), "v".to_string())].into(), + environment: [("E".to_string(), "1".to_string())].into(), + providers: vec!["provider".to_string()], + command: vec!["sleep".to_string(), "infinity".to_string()], + restart_policy: SandboxRestartPolicy::Never, + ..SandboxSpec::default() + }; + let policy = proto::SandboxPolicy { + version: 1, + ..proto::SandboxPolicy::default() + }; + let scope = Some(proto::workspace_selector("ws")); + + let request = policy_create_request(spec, policy.clone(), scope.clone()); + + assert_eq!(request.name, "test-sb"); + assert_eq!(request.labels.get("k").map(String::as_str), Some("v")); + assert_eq!(request.workspace_scope, scope); + let spec = request.spec.expect("spec"); + assert_eq!(spec.template.expect("template").image, "python:3"); + assert_eq!(spec.environment.get("E").map(String::as_str), Some("1")); + assert_eq!(spec.providers, ["provider"]); + assert_eq!(spec.command, ["sleep", "infinity"]); + assert_eq!(spec.policy, Some(policy)); + assert_eq!( + spec.restart_policy, + proto::SandboxRestartPolicy::Never as i32 + ); + } +} From 7d8a7e78323fed9d67c534879787e80b433a30b0 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 5 Nov 2026 13:51:03 -0800 Subject: [PATCH 02/16] test(conformance): port the credential gating e2e tests Port the six credential gating scenarios from e2e/rust/tests/credential_gating.rs into a conformance suite. Each test imports its own provider profile and creates a provider under a run-unique name, creates sandboxes from the Python test image through the CLI, and reaches probe servers on the host through host.openshell.internal. Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 163 ++++++ tests/suites/conformance/Cargo.toml | 5 +- tests/suites/conformance/README.md | 7 +- .../src/credential_gating/client.rs | 123 +++++ .../conformance/src/credential_gating/mod.rs | 485 ++++++++++++++++++ .../src/credential_gating/probe.rs | 296 +++++++++++ tests/suites/conformance/src/lib.rs | 1 + 7 files changed, 1078 insertions(+), 2 deletions(-) create mode 100644 tests/suites/conformance/src/credential_gating/client.rs create mode 100644 tests/suites/conformance/src/credential_gating/mod.rs create mode 100644 tests/suites/conformance/src/credential_gating/probe.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index ea04cd150d..6e42410284 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -493,6 +493,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -506,8 +516,11 @@ dependencies = [ name = "conformance" version = "0.0.0" dependencies = [ + "base64 0.22.1", + "openshell-policy", "openshell-testkit", "serde_json", + "sha1 0.10.6", "tempfile", "tokio", ] @@ -561,6 +574,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1138,6 +1157,25 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "rand 0.10.2", + "ring", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "url", +] + [[package]] name = "hkdf" version = "0.13.0" @@ -1518,6 +1556,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2021,6 +2108,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2109,6 +2200,19 @@ dependencies = [ "tower", ] +[[package]] +name = "openshell-policy" +version = "0.0.0" +dependencies = [ + "hickory-proto", + "miette", + "openshell-core", + "openshell-policy-schema", + "prost-types", + "serde", + "serde_json", +] + [[package]] name = "openshell-policy-schema" version = "0.0.0" @@ -2337,6 +2441,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.5" @@ -2755,6 +2865,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2837,6 +2956,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -3083,6 +3211,22 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -3951,6 +4095,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4098,6 +4252,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index 99274d8dfa..bd6bc3f1bb 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -21,9 +21,12 @@ path = "src/main.rs" test = false [dependencies] +base64 = "0.22" +openshell-policy = { path = "../../../crates/openshell-policy" } openshell-testkit = { path = "../../testkit/testkit" } serde_json = "1" +sha1 = "0.10" tempfile = "3" -tokio = { version = "1", features = ["time"] } +tokio = { version = "1", features = ["io-util", "net", "rt", "time"] } [workspace] diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index c756107cc1..a8570604bb 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -14,6 +14,7 @@ nextest run the tests as the library's unit tests. The crate also builds the | `persistent-workspace` | Files in `/sandbox` survive stop and start | | `policy-advisor` | `policy.local` accepts a proposed rule; a denied address or hostname yields one draft rule | | `file-transfer` | `sandbox upload` and `download`: round trips, `.gitignore` filtering, workspace-escape and option-looking names | +| `credential-gating` | Provider credentials: admission of credentialed endpoints, REST body placeholders and rewriting, WebSocket binary frames, live policy binding | `smoke` is the smallest end-to-end check that the runner can run a real test in a tmachine environment on CI. The other suites are ports of the old @@ -21,7 +22,11 @@ tmachine environment on CI. The other suites are ports of the old (credited in each file). `file-transfer` runs `git` on the machine that runs the tests, so a test environment needs it installed (`tests/tmachine/ansible/playbooks/test-tools.yaml` does that for the tmachine -environments). +environments). `credential-gating`, ported from NVIDIA/OpenShell#5264, starts +servers on the machine that runs the tests and reaches them from sandboxes +created from `testkit::python_image()` through `host.openshell.internal`, so +it fails against a gateway whose sandboxes can't reach that machine, such as +a remote Kubernetes cluster. ## Running it diff --git a/tests/suites/conformance/src/credential_gating/client.rs b/tests/suites/conformance/src/credential_gating/client.rs new file mode 100644 index 0000000000..3ade29cc2a --- /dev/null +++ b/tests/suites/conformance/src/credential_gating/client.rs @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Commands the tests run in the sandbox. + +use super::{TEST_HOST, TOKEN_ENV}; + +/// Prints `TOKEN_ABSENT`, `TOKEN_PLACEHOLDER`, or `TOKEN_UNSAFE` for the +/// provider's environment variable, without printing its value. +pub(super) const KEY_PRESENCE_SCRIPT: &str = r#"if [ "${E2E_GATING_TOKEN+x}" != x ]; then + echo TOKEN_ABSENT +else + case "$E2E_GATING_TOKEN" in + openshell:resolve:env:*) echo TOKEN_PLACEHOLDER ;; + *) echo TOKEN_UNSAFE ;; + esac +fi"#; + +/// POSTs a body containing the provider's token to the HTTP probe and prints +/// the probe's verdict, or `BODY_DENIED` without one. +pub(super) fn body_client_script(port: u16) -> String { + format!( + r#" +import os +import socket + +host = {TEST_HOST:?} +port = {port} +token = os.environ[{TOKEN_ENV:?}] + +with socket.create_connection((host, port), timeout=10) as sock: + target = f"{{host}}:{{port}}" + body = ("prefix-" + token + "-suffix").encode("utf-8") + request = ( + f"POST /token HTTP/1.1\r\nHost: {{target}}\r\n" + f"Content-Type: text/plain\r\nContent-Length: {{len(body)}}\r\nConnection: close\r\n\r\n" + ).encode("ascii") + body + sock.sendall(request) + sock.settimeout(3) + response = b"" + while True: + try: + chunk = sock.recv(4096) + except socket.timeout: + break + if not chunk: + break + response += chunk + print("BODY_REWRITTEN" if b"BODY_REWRITTEN" in response else "BODY_TEXT" if b"BODY_TEXT" in response else "BODY_DENIED") +"# + ) +} + +/// Upgrades to WebSocket at the probe, sends one binary frame, and prints +/// `BINARY_DENIED` if the connection closed or answered with close code 1008, +/// or `BINARY_FORWARDED` otherwise. +pub(super) fn binary_websocket_client_script(port: u16) -> String { + format!( + r#" +import base64 +import os +import socket +import struct + +host = {TEST_HOST:?} +port = {port} + +def recv_until(sock, marker): + data = b"" + while marker not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + +def recv_exact(sock, size): + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + break + data += chunk + return data + +with socket.create_connection((host, port), timeout=10) as sock: + target = f"{{host}}:{{port}}" + key = base64.b64encode(os.urandom(16)).decode("ascii") + request = ( + f"GET /ws HTTP/1.1\r\nHost: {{target}}\r\n" + "Upgrade: websocket\r\nConnection: Upgrade\r\n" + f"Sec-WebSocket-Key: {{key}}\r\nSec-WebSocket-Version: 13\r\n\r\n" + ) + sock.sendall(request.encode("ascii")) + if not recv_until(sock, b"\r\n\r\n").startswith(b"HTTP/1.1 101"): + raise RuntimeError("upgrade failed") + payload = b"binary-credential-channel" + mask = os.urandom(4) + masked = bytes(byte ^ mask[index % 4] for index, byte in enumerate(payload)) + frame = bytes([0x82, 0x80 | len(payload)]) + mask + masked + sock.sendall(frame) + sock.settimeout(3) + denied = False + try: + header = recv_exact(sock, 2) + if not header: + denied = True + elif len(header) == 2: + opcode = header[0] & 0x0f + masked = bool(header[1] & 0x80) + payload_length = header[1] & 0x7f + if opcode == 0x08 and not masked and payload_length < 126: + close_payload = recv_exact(sock, payload_length) + denied = ( + len(close_payload) >= 2 + and struct.unpack("!H", close_payload[:2])[0] == 1008 + ) + except socket.timeout: + denied = True + print("BINARY_DENIED" if denied else "BINARY_FORWARDED") +"# + ) +} diff --git a/tests/suites/conformance/src/credential_gating/mod.rs b/tests/suites/conformance/src/credential_gating/mod.rs new file mode 100644 index 0000000000..fd9dd06077 --- /dev/null +++ b/tests/suites/conformance/src/credential_gating/mod.rs @@ -0,0 +1,485 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Credential gating: which endpoints a provider credential may reach, and +//! what the supervisor does with the credential in REST bodies and WebSocket +//! frames. +//! +//! Ported from `e2e/rust/tests/credential_gating.rs` (NVIDIA/OpenShell#5264). +//! Each test imports its own provider profile and creates a provider under a +//! name unique to the run; cleanup deletes them after the test's sandboxes. +//! The probe servers run where the tests run, which is the host the gateway +//! runs its sandboxes on, and the sandbox clients reach them through +//! `host.openshell.internal`, so a gateway on a remote cluster can't run this +//! suite. + +mod client; +mod probe; + +use std::time::Duration; + +use testkit::sdk::raw::proto; +use testkit::{Context, Error, Poll, Result, ResultExt as _, Sandbox}; + +use client::{KEY_PRESENCE_SCRIPT, binary_websocket_client_script, body_client_script}; +use probe::{BinaryWebSocketProbeServer, HttpProbeServer}; + +const TEST_HOST: &str = "host.openshell.internal"; +const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; +const TEST_SECRET: &str = "e2e-gating-secret-value"; +const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; + +/// A provider whose profile declares the REST and WebSocket probes as its +/// endpoints keeps the credential placeholder in a REST body. +#[testkit::test] +async fn profile_rest_endpoint_keeps_body_placeholder(cx: &mut Context) -> testkit::Result { + let server = HttpProbeServer::start().await?; + let websocket = BinaryWebSocketProbeServer::start().await?; + let provider = install_provider(cx, &profile(Some((server.port, websocket.port)))).await?; + + let policy = policy( + server.port, + &[ + "protocol: rest", + "access: full", + "request_body_credential_rewrite: false", + ], + ); + let sandbox = create_sandbox(cx, "rest", &policy, &provider).await?; + let output = run( + &sandbox, + &["python3", "-c", &body_client_script(server.port)], + ) + .await?; + testkit::ensure!(output.contains("BODY_TEXT"), "{output}"); + expect_one_body(&server, true, false).await +} + +/// A provider's WebSocket endpoint completes the handshake but doesn't +/// forward a binary frame, which the supervisor can't inspect for the +/// credential. +#[testkit::test] +async fn profile_websocket_endpoint_denies_binary_frames(cx: &mut Context) -> testkit::Result { + let server = HttpProbeServer::start().await?; + let websocket = BinaryWebSocketProbeServer::start().await?; + let provider = install_provider(cx, &profile(Some((server.port, websocket.port)))).await?; + + let policy = policy( + websocket.port, + &["protocol: websocket", "access: read-write"], + ); + let sandbox = create_sandbox(cx, "ws", &policy, &provider).await?; + let script = binary_websocket_client_script(websocket.port); + let output = run(&sandbox, &["python3", "-c", &script]).await?; + testkit::ensure!(output.contains("BINARY_DENIED"), "{output}"); + websocket + .wait_for_handshake() + .await + .context("upstream should receive the WebSocket handshake")?; + testkit::ensure!( + !websocket.binary_seen(), + "credentialed WebSocket binary frame reached upstream" + ); + Ok(()) +} + +/// The gateway rejects a policy that binds a provider to an L4-only +/// endpoint or to an endpoint with `tls: skip`, and accepts an L4 endpoint +/// that opts in to uninspected credentials. +#[testkit::test] +async fn policy_binding_requires_inspection_or_opt_in(cx: &mut Context) -> testkit::Result { + let server = HttpProbeServer::start().await?; + let provider = install_provider(cx, &profile(None)).await?; + let binding = credential_binding(&provider); + + let l4 = policy(server.port, &[&binding]); + let error = expect_rejected(cx, "l4", &l4, &provider).await?; + testkit::ensure!(error.contains("credentialed endpoint"), "{error}"); + testkit::ensure!(error.contains("L4-only"), "{error}"); + + let tls_skip = policy( + server.port, + &["protocol: rest", "access: full", "tls: skip", &binding], + ); + let error = expect_rejected(cx, "tls", &tls_skip, &provider).await?; + testkit::ensure!(error.contains("credentialed endpoint"), "{error}"); + testkit::ensure!(error.contains("tls:") && error.contains("skip"), "{error}"); + + let opt_in = policy( + server.port, + &["allow_uninspected_credentials: true", &binding], + ); + let sandbox = create_sandbox(cx, "optin", &opt_in, &provider).await?; + let output = run(&sandbox, &["echo", "ENDPOINTLESS_OPT_IN_ACCEPTED"]).await?; + testkit::ensure!(output.contains("ENDPOINTLESS_OPT_IN_ACCEPTED"), "{output}"); + Ok(()) +} + +/// A policy binding to a REST endpoint without body rewriting sends the +/// credential placeholder upstream, not the secret. +#[testkit::test] +async fn policy_binding_rest_body_keeps_placeholder_without_rewrite( + cx: &mut Context, +) -> testkit::Result { + let server = HttpProbeServer::start().await?; + let provider = install_provider(cx, &profile(None)).await?; + + let policy = policy( + server.port, + &[ + "protocol: rest", + "access: full", + "request_body_credential_rewrite: false", + &credential_binding(&provider), + ], + ); + let sandbox = create_sandbox(cx, "body", &policy, &provider).await?; + let output = run( + &sandbox, + &["python3", "-c", &body_client_script(server.port)], + ) + .await?; + testkit::ensure!(output.contains("BODY_TEXT"), "{output}"); + expect_one_body(&server, true, false).await +} + +/// A policy binding with `request_body_credential_rewrite: true` replaces +/// the placeholder in the REST body with the secret upstream, and the +/// secret doesn't reach the sandbox. +#[testkit::test] +async fn policy_binding_rest_body_rewrite_sends_secret_upstream( + cx: &mut Context, +) -> testkit::Result { + let server = HttpProbeServer::start().await?; + let provider = install_provider(cx, &profile(None)).await?; + + let policy = policy( + server.port, + &[ + "protocol: rest", + "access: full", + "request_body_credential_rewrite: true", + &credential_binding(&provider), + ], + ); + let sandbox = create_sandbox(cx, "rw", &policy, &provider).await?; + let output = run( + &sandbox, + &["python3", "-c", &body_client_script(server.port)], + ) + .await?; + testkit::ensure!( + output.contains("BODY_REWRITTEN"), + "the client did not report BODY_REWRITTEN" + ); + testkit::ensure!( + !output.contains(TEST_SECRET), + "the secret reached the sandbox" + ); + testkit::ensure!( + !output.contains(PLACEHOLDER_PREFIX), + "the client printed the placeholder" + ); + expect_one_body(&server, false, true).await +} + +/// Binding and unbinding a provider with `policy set --wait` adds and +/// removes its environment variable in a running sandbox. +#[testkit::test] +async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> testkit::Result { + let server = HttpProbeServer::start().await?; + let provider = install_provider(cx, &profile(None)).await?; + let unbound = policy( + server.port, + &[ + "protocol: rest", + "access: full", + "request_body_credential_rewrite: false", + ], + ); + let bound = policy( + server.port, + &[ + "protocol: rest", + "access: full", + "request_body_credential_rewrite: false", + &credential_binding(&provider), + ], + ); + + let sandbox = create_sandbox(cx, "live", &unbound, &provider).await?; + wait_for_token(&sandbox, "TOKEN_ABSENT").await?; + + set_policy(cx, &sandbox, &bound) + .await + .context("bind the provider")?; + wait_for_token(&sandbox, "TOKEN_PLACEHOLDER").await?; + + set_policy(cx, &sandbox, &unbound) + .await + .context("unbind the provider")?; + wait_for_token(&sandbox, "TOKEN_ABSENT").await +} + +/// A provider profile with one bearer credential in `TOKEN_ENV`. With +/// `endpoints`, it declares the REST and WebSocket probe ports as its +/// endpoints; without, a policy must bind the provider explicitly. +fn profile(endpoints: Option<(u16, u16)>) -> String { + let endpoints = endpoints.map_or_else(String::new, |(rest, websocket)| { + format!( + "endpoints: + - host: {TEST_HOST} + port: {rest} + protocol: rest + access: full + - host: {TEST_HOST} + port: {websocket} + protocol: websocket + access: read-write +" + ) + }); + format!( + "id: PROFILE_ID +display_name: E2E Credential Gating +category: other +credentials: + - name: token + env_vars: [{TOKEN_ENV}] + required: true + auth_style: bearer + header_name: authorization +{endpoints}binaries: + - path: /usr/bin/python* + - path: /usr/local/bin/python* + - path: /sandbox/.uv/python/*/bin/python* +" + ) +} + +/// A sandbox policy that lets the image's Python reach `TEST_HOST` on +/// `port`, with `endpoint` as extra fields of that endpoint. +fn policy(port: u16, endpoint: &[&str]) -> String { + let endpoint: String = endpoint + .iter() + .map(|field| format!(" {field}\n")) + .collect(); + format!( + "version: 1 +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +network_policies: + credential_gating: + name: credential_gating + endpoints: + - host: {TEST_HOST} + port: {port} +{endpoint} binaries: + - path: /usr/bin/python* + - path: /usr/local/bin/python* + - path: /sandbox/.uv/python/*/bin/python* +" + ) +} + +/// The endpoint field that binds `provider`'s credential in a policy. +fn credential_binding(provider: &str) -> String { + format!("credential_binding: {{provider: {provider}}}") +} + +/// Import `profile` under a run-unique ID and create a provider of that type +/// with the same name, holding `TEST_SECRET`. Returns the name. Cleanup +/// deletes the provider, then the profile, after the test's sandboxes. +async fn install_provider(cx: &Context, profile: &str) -> Result { + let name = cx.name("cg"); + // Both deletes succeed for a missing resource, so register them before + // the creates. Hooks run last registered first. + for kind in ["profile", "provider"] { + let cli = cx.cli_handle(); + let owned = name.clone(); + cx.on_cleanup(format!("delete {kind} {name}"), async move || { + cli.run([kind, "delete", &owned]).await?.success()?; + Ok(()) + }); + } + + let file = YamlFile::new(&profile.replace("PROFILE_ID", &name))?; + cx.cli(["profile", "import", "--file", file.path()?]) + .await? + .success() + .context(format!("import profile {name}"))?; + // `--credential KEY` reads the value from the CLI's environment, so the + // secret isn't in the reported command. + cx.cli_with_env( + &[(TOKEN_ENV, TEST_SECRET)], + [ + "provider", + "create", + "--name", + &name, + "--type", + &name, + "--credential", + TOKEN_ENV, + ], + ) + .await? + .success() + .context(format!("create provider {name}"))?; + Ok(name) +} + +fn parse_policy(yaml: &str) -> Result { + openshell_policy::parse_sandbox_policy(yaml) + .map_err(|error| Error::msg(format!("{error:?}"))) + .context("parse the test policy") +} + +fn sandbox_with( + cx: &Context, + suffix: &str, + policy: &str, + provider: &str, +) -> Result { + Ok(cx + .sandbox(suffix) + .policy(parse_policy(policy)?) + .spec(|spec| { + spec.image = Some(testkit::python_image()); + spec.providers = vec![provider.to_string()]; + }) + .command(["sleep", "infinity"])) +} + +/// Create a long-running sandbox from the Python image with `policy` and +/// `provider`, and wait until it is Ready. +async fn create_sandbox( + cx: &Context, + suffix: &str, + policy: &str, + provider: &str, +) -> Result { + let sandbox = sandbox_with(cx, suffix, policy, provider)?.create().await?; + sandbox.wait_ready().await?; + Ok(sandbox) +} + +/// Fail unless the gateway rejects the sandbox, and return the rejection. +async fn expect_rejected( + cx: &Context, + suffix: &str, + policy: &str, + provider: &str, +) -> Result { + match sandbox_with(cx, suffix, policy, provider)?.create().await { + Ok(sandbox) => Err(Error::msg(format!( + "the gateway accepted sandbox {}", + sandbox.name() + ))), + Err(error) => Ok(error.to_string()), + } +} + +/// Run `command` in `sandbox`; it must exit 0. Returns stdout and stderr. +async fn run(sandbox: &Sandbox, command: &[&str]) -> Result { + let output = sandbox.exec(command.iter().copied()).await?; + testkit::ensure!( + output.exit_code == 0, + "{command:?} exited with {}; stdout: {}; stderr: {}", + output.exit_code, + output.stdout.trim(), + output.stderr.trim() + ); + Ok(format!("{}{}", output.stdout, output.stderr)) +} + +/// Fail unless the HTTP probe saw exactly one request, with the placeholder +/// and the secret as expected. +async fn expect_one_body(server: &HttpProbeServer, placeholder: bool, secret: bool) -> Result { + let observations = server.wait_for_observations(1).await?; + testkit::ensure!( + observations.len() == 1, + "expected one request at the HTTP probe; observations: {observations:?}" + ); + testkit::ensure!( + observations[0].saw_placeholder == placeholder && observations[0].saw_secret == secret, + "expected placeholder={placeholder} secret={secret} upstream; saw {:?}", + observations[0] + ); + Ok(()) +} + +/// Wait until new exec sessions report `expected` from +/// [`KEY_PRESENCE_SCRIPT`]. +async fn wait_for_token(sandbox: &Sandbox, expected: &str) -> Result { + testkit::poll_until( + &format!("the provider environment to report {expected}"), + Duration::from_secs(30), + Duration::from_millis(500), + async || { + let output = sandbox.exec(["sh", "-lc", KEY_PRESENCE_SCRIPT]).await?; + Ok(if output.stdout.contains(expected) { + Poll::Ready(()) + } else { + Poll::Pending(format!( + "exit {}; stdout: {}; stderr: {}", + output.exit_code, + output.stdout.trim(), + output.stderr.trim() + )) + }) + }, + ) + .await +} + +/// Replace `sandbox`'s policy with `policy` and wait until it is applied. +async fn set_policy(cx: &Context, sandbox: &Sandbox, policy: &str) -> Result { + let file = YamlFile::new(policy)?; + cx.cli([ + "policy", + "set", + sandbox.name(), + "--policy", + file.path()?, + "--wait", + "--timeout", + "120", + ]) + .await? + .success() + .context("policy set")?; + Ok(()) +} + +/// YAML in a temporary file, for CLI commands that read one. The file is +/// removed when dropped. +struct YamlFile(tempfile::NamedTempFile); + +impl YamlFile { + fn new(contents: &str) -> Result { + let file = tempfile::Builder::new() + .suffix(".yaml") + .tempfile() + .map_err(|error| Error::from_error(&error)) + .context("create a temporary YAML file")?; + std::fs::write(file.path(), contents) + .map_err(|error| Error::from_error(&error)) + .context(format!("write {}", file.path().display()))?; + Ok(Self(file)) + } + + fn path(&self) -> Result<&str> { + self.0 + .path() + .to_str() + .ok_or_else(|| Error::msg(format!("{} is not UTF-8", self.0.path().display()))) + } +} diff --git a/tests/suites/conformance/src/credential_gating/probe.rs b/tests/suites/conformance/src/credential_gating/probe.rs new file mode 100644 index 0000000000..46a3974361 --- /dev/null +++ b/tests/suites/conformance/src/credential_gating/probe.rs @@ -0,0 +1,296 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Servers on the host that the sandbox clients reach through +//! `host.openshell.internal`, recording what arrived upstream. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use base64::Engine as _; +use sha1::{Digest as _, Sha1}; +use testkit::{Error, Poll, Result, ResultExt as _}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::JoinHandle; + +use super::{PLACEHOLDER_PREFIX, TEST_SECRET}; + +const WEBSOCKET_GUID: &str = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; +const OBSERVE_TIMEOUT: Duration = Duration::from_secs(10); +const OBSERVE_INTERVAL: Duration = Duration::from_millis(100); + +/// What one request body that reached the HTTP probe contained. +#[derive(Debug, Default, Clone, Copy)] +pub(super) struct BodyObservation { + pub(super) saw_placeholder: bool, + pub(super) saw_secret: bool, +} + +/// An HTTP server that records whether each request carried the credential +/// placeholder or the secret, and answers with `BODY_TEXT`, +/// `BODY_REWRITTEN`, or `BODY_BAD`. It stops when dropped. +pub(super) struct HttpProbeServer { + pub(super) port: u16, + observations: Arc>>, + task: JoinHandle<()>, +} + +impl HttpProbeServer { + pub(super) async fn start() -> Result { + let (listener, port) = bind("HTTP probe").await?; + let observations = Arc::new(Mutex::new(Vec::new())); + let task_observations = Arc::clone(&observations); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let observations = Arc::clone(&task_observations); + tokio::spawn(async move { + let _ = handle_http_probe(stream, observations).await; + }); + } + }); + Ok(Self { + port, + observations, + task, + }) + } + + /// The requests the server has seen, once there are at least `count`. + pub(super) async fn wait_for_observations(&self, count: usize) -> Result> { + testkit::poll_until( + &format!("{count} request(s) at the HTTP probe"), + OBSERVE_TIMEOUT, + OBSERVE_INTERVAL, + async || { + let observations = self + .observations + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + Ok(if observations.len() >= count { + Poll::Ready(observations) + } else { + Poll::Pending(format!("{observations:?}")) + }) + }, + ) + .await + } +} + +impl Drop for HttpProbeServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +/// A WebSocket server that completes the handshake and records whether a +/// binary frame followed. It stops when dropped. +pub(super) struct BinaryWebSocketProbeServer { + pub(super) port: u16, + handshake_seen: Arc, + binary_seen: Arc, + task: JoinHandle<()>, +} + +impl BinaryWebSocketProbeServer { + pub(super) async fn start() -> Result { + let (listener, port) = bind("WebSocket probe").await?; + let handshake_seen = Arc::new(AtomicBool::new(false)); + let binary_seen = Arc::new(AtomicBool::new(false)); + let task_handshake_seen = Arc::clone(&handshake_seen); + let task_binary_seen = Arc::clone(&binary_seen); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let handshake_seen = Arc::clone(&task_handshake_seen); + let binary_seen = Arc::clone(&task_binary_seen); + tokio::spawn(async move { + let _ = + handle_binary_websocket_probe(stream, handshake_seen, binary_seen).await; + }); + } + }); + Ok(Self { + port, + handshake_seen, + binary_seen, + task, + }) + } + + /// Wait until a client completed the WebSocket handshake. + pub(super) async fn wait_for_handshake(&self) -> Result { + testkit::poll_until( + "the WebSocket handshake at the probe", + OBSERVE_TIMEOUT, + OBSERVE_INTERVAL, + async || { + Ok(if self.handshake_seen.load(Ordering::Acquire) { + Poll::Ready(()) + } else { + Poll::Pending("no handshake".to_string()) + }) + }, + ) + .await + } + + pub(super) fn binary_seen(&self) -> bool { + self.binary_seen.load(Ordering::Acquire) + } +} + +impl Drop for BinaryWebSocketProbeServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +/// Listen on every host address, since drivers map +/// `host.openshell.internal` to different ones. +async fn bind(what: &str) -> Result<(TcpListener, u16)> { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| Error::from_error(&error)) + .context(format!("bind the {what}"))?; + let port = listener + .local_addr() + .map_err(|error| Error::from_error(&error)) + .context(format!("read the {what}'s address"))? + .port(); + Ok((listener, port)) +} + +async fn recv_until(stream: &mut TcpStream, marker: &[u8]) -> std::io::Result> { + let mut received = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let read = stream.read(&mut buffer).await?; + if read == 0 { + return Ok(received); + } + received.extend_from_slice(&buffer[..read]); + if received + .windows(marker.len()) + .any(|window| window == marker) + { + return Ok(received); + } + } +} + +fn header_value(request: &str, name: &str) -> Option { + request.lines().find_map(|line| { + let (header, value) = line.split_once(':')?; + header + .trim() + .eq_ignore_ascii_case(name) + .then(|| value.trim().to_string()) + }) +} + +/// The `Sec-WebSocket-Accept` value for `key`, which the supervisor checks. +fn websocket_accept(key: &str) -> String { + let mut hasher = Sha1::new(); + hasher.update(key.as_bytes()); + hasher.update(WEBSOCKET_GUID.as_bytes()); + base64::engine::general_purpose::STANDARD.encode(hasher.finalize()) +} + +async fn handle_binary_websocket_probe( + mut stream: TcpStream, + handshake_seen: Arc, + binary_seen: Arc, +) -> std::io::Result<()> { + let _ = stream.set_nodelay(true); + let request_bytes = recv_until(&mut stream, b"\r\n\r\n").await?; + let request = String::from_utf8_lossy(&request_bytes); + let key = header_value(&request, "Sec-WebSocket-Key").ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "missing WebSocket key") + })?; + let accept = websocket_accept(&key); + let response = format!( + "HTTP/1.1 101 Switching Protocols\r\n\ + Upgrade: websocket\r\n\ + Connection: Upgrade\r\n\ + Sec-WebSocket-Accept: {accept}\r\n\ + \r\n" + ); + stream.write_all(response.as_bytes()).await?; + handshake_seen.store(true, Ordering::Release); + + let mut header = [0_u8; 2]; + if tokio::time::timeout(Duration::from_secs(5), stream.read_exact(&mut header)) + .await + .is_ok_and(|result| result.is_ok()) + && header[0] & 0x0f == 0x02 + { + binary_seen.store(true, Ordering::Release); + } + Ok(()) +} + +async fn handle_http_probe( + mut stream: TcpStream, + observations: Arc>>, +) -> std::io::Result<()> { + let _ = stream.set_nodelay(true); + let mut received = Vec::new(); + let mut buffer = [0_u8; 4096]; + let mut expected_total = None; + loop { + let read = tokio::time::timeout(Duration::from_secs(10), stream.read(&mut buffer)).await; + let Ok(Ok(read)) = read else { + break; + }; + if read == 0 { + break; + } + received.extend_from_slice(&buffer[..read]); + if expected_total.is_none() + && let Some(header_end) = received.windows(4).position(|window| window == b"\r\n\r\n") + { + let header_end = header_end + 4; + let headers = String::from_utf8_lossy(&received[..header_end]); + let content_length = header_value(&headers, "content-length") + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + expected_total = Some(header_end + content_length); + } + if expected_total.is_some_and(|expected| received.len() >= expected) { + break; + } + } + + let observation = BodyObservation { + saw_placeholder: contains(&received, PLACEHOLDER_PREFIX), + saw_secret: contains(&received, TEST_SECRET), + }; + observations + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(observation); + if expected_total.is_some_and(|expected| received.len() >= expected) { + let result = if observation.saw_secret && !observation.saw_placeholder { + "BODY_REWRITTEN" + } else if observation.saw_placeholder && !observation.saw_secret { + "BODY_TEXT" + } else { + "BODY_BAD" + }; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{result}", + result.len() + ); + stream.write_all(response.as_bytes()).await?; + } + Ok(()) +} + +fn contains(haystack: &[u8], needle: &str) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle.as_bytes()) +} diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 00a4d45027..98b916092f 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -9,6 +9,7 @@ //! program in `src/main.rs` runs them by linking this library. mod cli; +mod credential_gating; mod file_transfer; mod lifecycle; mod persistent_workspace; From 15fd553c8f255718faedb189c001a74d8fa2f3d2 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 5 Nov 2026 13:51:03 -0800 Subject: [PATCH 03/16] test(e2e): provide the Python test image to host-lane conformance The conformance suite now creates sandboxes from testkit::python_image(). Docker and Podman runners build openshell/e2e-python:dev also in e2e-conformance-only mode, and the Kubernetes and VM runners default OPENSHELL_TEST_PYTHON_IMAGE to the pullable uv Python image. Signed-off-by: Matthew Grossman --- TESTING.md | 15 ++++++++++----- e2e/rust/e2e-docker.sh | 6 +++--- e2e/rust/e2e-kubernetes.sh | 1 + e2e/rust/e2e-podman.sh | 6 +++--- e2e/rust/e2e-vm.sh | 3 ++- e2e/support/conformance.sh | 6 ++++++ tasks/test.toml | 1 + 7 files changed, 26 insertions(+), 12 deletions(-) diff --git a/TESTING.md b/TESTING.md index a3936ad649..d150c6afcc 100644 --- a/TESTING.md +++ b/TESTING.md @@ -424,11 +424,14 @@ Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. - Conformance (`tests/suites/conformance`) - the testkit `conformance` crate: - lifecycle, the CLI, persistent workspaces, file transfer, and the policy - advisor. Driver E2E runs it with `openshell-conformance` against the lane's - gateway and writes a JUnit report under `results/`; set - `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program, as Branch E2E does - with the static build it makes once per architecture. The + lifecycle, the CLI, persistent workspaces, file transfer, the policy + advisor, and credential gating. Driver E2E runs it with + `openshell-conformance` against the lane's gateway and writes a JUnit report + under `results/`; set `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program, + as Branch E2E does with the static build it makes once per architecture. + The Docker and Podman runners build `openshell/e2e-python:dev` for it; the + Kubernetes and VM runners set `OPENSHELL_TEST_PYTHON_IMAGE` to a pullable + Python image unless it is already set. The installed-artifact conformance lanes run the same crate through `openshell-test-runner` (see above). - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or @@ -656,6 +659,8 @@ explains the isolation backend separation behind the capability-free runtime. On a **remote** cluster, drop the `e2e-host-gateway` feature. Those tests rely on the sandbox-side `host.openshell.internal` alias reaching the machine running the tests, which is unreachable from pods on a remote cluster, so they fail. +The conformance suite's `credential_gating` tests rely on the alias too and +fail on a remote cluster; the feature selection doesn't skip them. Left enabled, the `host_gateway_alias` suite fails because `host.openshell.internal` does not resolve inside the pod, so the gateway SSRF-denies the request (`DNS resolution failed` / `ssrf_denied`) — a networking diff --git a/e2e/rust/e2e-docker.sh b/e2e/rust/e2e-docker.sh index 8db220ca1b..68d48e7370 100755 --- a/e2e/rust/e2e-docker.sh +++ b/e2e/rust/e2e-docker.sh @@ -54,9 +54,9 @@ if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then exit 0 fi -if [ "${E2E_FEATURES}" != "e2e-conformance-only" ]; then - CONTAINER_ENGINE=docker bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" -fi +# The conformance suite runs on every lane, including e2e-conformance-only, +# and creates sandboxes from this image (testkit::python_image()). +CONTAINER_ENGINE=docker bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" exec "${ROOT}/e2e/with-docker-gateway.sh" \ bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index fc90a827ee..7ef5576eab 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -80,6 +80,7 @@ run_conformance() { fi OPENSHELL_CONFORMANCE_JUNIT="${OPENSHELL_CONFORMANCE_JUNIT:-${ROOT}/results/${OPENSHELL_E2E_REPORT_NAME:-e2e-kubernetes}-conformance.xml}" \ + OPENSHELL_TEST_PYTHON_IMAGE="${OPENSHELL_TEST_PYTHON_IMAGE:-${E2E_PULLABLE_PYTHON_IMAGE}}" \ e2e_run_openshell_conformance "Kubernetes" } diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index 0d6cd5008c..fff2c2bf7f 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -117,9 +117,9 @@ if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ] && [ -z "${OPENSHELL_BIN:-}" ]; then export OPENSHELL_BIN="${ROOT}/target/debug/openshell" fi -if [ "${E2E_FEATURES}" != "e2e-conformance-only" ]; then - CONTAINER_ENGINE=podman bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" -fi +# The conformance suite runs on every lane, including e2e-conformance-only, +# and creates sandboxes from this image (testkit::python_image()). +CONTAINER_ENGINE=podman bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" exec "${ROOT}/e2e/with-podman-gateway.sh" \ bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 0cb2c46858..bafaadbc71 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -463,7 +463,8 @@ e2e_export_gateway_restart_metadata \ # preparation; allow 180s for slower CI runners. export OPENSHELL_PROVISION_TIMEOUT="${SANDBOX_PROVISION_TIMEOUT}" -e2e_run_openshell_conformance "VM" +OPENSHELL_TEST_PYTHON_IMAGE="${OPENSHELL_TEST_PYTHON_IMAGE:-${E2E_PULLABLE_PYTHON_IMAGE}}" \ + e2e_run_openshell_conformance "VM" if [ "${E2E_FEATURES}" = "e2e-conformance-only" ]; then echo "::notice::Skipping Rust e2e tests (e2e-conformance-only)" exit 0 diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index 56ce5a50f1..eef41ecb29 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -23,6 +23,12 @@ e2e_require_openshell_bin() { export OPENSHELL_BIN="${binary_dir}/$(basename "${OPENSHELL_BIN}")" } +# A pullable image for testkit::python_image() on lanes that can't load the +# locally built openshell/e2e-python:dev into their compute platform. It +# matches the Rust harness's E2E_WORKLOAD_IMAGE for those lanes. +# shellcheck disable=SC2034 # Used by the scripts that source this file. +E2E_PULLABLE_PYTHON_IMAGE="ghcr.io/astral-sh/uv:0.12.17-python3.12-trixie-slim@sha256:9a59bb7206905ccaae4f7dab222fbac47c125a21e5fc16f43f427cd6c940ade3" + # Print the path of the openshell-conformance program: OPENSHELL_CONFORMANCE_BIN # if set, else a debug build from this checkout. e2e_openshell_conformance_bin() { diff --git a/tasks/test.toml b/tasks/test.toml index 2839c1cf0c..30c31ec753 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -177,6 +177,7 @@ run = "OPENSHELL_TEST_CONTAINER_ENGINE=podman e2e/with-podman-gateway.sh cargo t ["e2e:testkit"] description = "Run the testkit suites in tests/suites/conformance against a Docker-backed gateway" +depends = ["e2e:workload:build"] run = "e2e/with-docker-gateway.sh cargo nextest run --locked --manifest-path tests/suites/conformance/Cargo.toml --workspace --test-threads 1 --no-fail-fast" ["e2e:rust:push"] From 38842bb8af3f5b75fd70a79f2c4b8544fef8ae49 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 5 Nov 2026 15:09:57 -0800 Subject: [PATCH 04/16] test(e2e): remove the credential gating tests ported to conformance Signed-off-by: Matthew Grossman --- e2e/rust/Cargo.toml | 5 - e2e/rust/e2e-podman.sh | 1 - e2e/rust/tests/credential_gating.rs | 1057 ----------------- .../conformance/src/credential_gating/mod.rs | 2 +- 4 files changed, 1 insertion(+), 1064 deletions(-) delete mode 100644 e2e/rust/tests/credential_gating.rs diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index b0938d9625..2af824018c 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -204,11 +204,6 @@ name = "websocket_conformance" path = "tests/websocket_conformance.rs" required-features = ["e2e-host-gateway"] -[[test]] -name = "credential_gating" -path = "tests/credential_gating.rs" -required-features = ["e2e-host-gateway"] - [[test]] name = "user_namespaces" path = "tests/user_namespaces.rs" diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index fff2c2bf7f..e718f0afea 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -25,7 +25,6 @@ PODMAN_CI_TESTS=( bypass_detection ephemeral_cleanup core_dump_hardening - credential_gating default_image driver_config_volume edge_tunnel_e2e diff --git a/e2e/rust/tests/credential_gating.rs b/e2e/rust/tests/credential_gating.rs deleted file mode 100644 index 110e02cb1e..0000000000 --- a/e2e/rust/tests/credential_gating.rs +++ /dev/null @@ -1,1057 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for credentialed endpoint admission and REST body backstops. -//! -//! Each test installs its own provider and profile under a unique name, so the -//! tests can run in parallel. Each test prints its steps to stderr with the -//! elapsed time, so a timeout shows which step was slow. - -use std::fmt::Display; -use std::io::Write; -use std::process::Stdio; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; - -use base64::Engine as _; -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::sandbox::SandboxGuard; -use sha1::{Digest, Sha1}; -use tempfile::NamedTempFile; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::task::JoinHandle; - -const TEST_HOST: &str = "host.openshell.internal"; -const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; -const TEST_SECRET: &str = "e2e-gating-secret-value"; -const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; -const KEY_PRESENCE_SCRIPT: &str = r#"if [ "${E2E_GATING_TOKEN+x}" != x ]; then - echo TOKEN_ABSENT -else - case "$E2E_GATING_TOKEN" in - openshell:resolve:env:*) echo TOKEN_PLACEHOLDER ;; - *) echo TOKEN_UNSAFE ;; - esac -fi"#; -const WEBSOCKET_GUID: &str = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; - -async fn run_cli(args: &[&str]) -> (bool, String) { - let mut command = openshell_cmd(); - command - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let output = command.output().await.expect("spawn openshell CLI"); - ( - output.status.success(), - format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ), - ) -} - -/// Prints a test's steps to stderr with the time since the test started. -#[derive(Clone, Copy)] -struct Progress { - scenario: &'static str, - started: Instant, -} - -impl Progress { - fn start(scenario: &'static str) -> Self { - let progress = Self { - scenario, - started: Instant::now(), - }; - progress.step("start"); - progress - } - - fn step(self, message: impl Display) { - eprintln!( - "credential_gating[{}] +{:.1}s {message}", - self.scenario, - self.started.elapsed().as_secs_f64() - ); - } -} - -fn utf8_path(file: &NamedTempFile) -> Result<&str, String> { - file.path() - .to_str() - .ok_or_else(|| format!("{} is not UTF-8", file.path().display())) -} - -/// Retry a delete until it takes effect. -/// -/// The gateway refuses to delete a provider or a profile that a sandbox still -/// references, and sandbox teardown drains asynchronously, so a single attempt -/// can be rejected right after a sandbox was removed. This runs from `Drop`, -/// so it blocks instead of awaiting. -fn delete_until_gone(args: &[&str]) -> Result<(), String> { - const ATTEMPTS: u32 = 40; - let binary = openshell_bin(); - let mut last_output = String::new(); - for _ in 0..ATTEMPTS { - match std::process::Command::new(&binary) - .args(args) - .stdin(Stdio::null()) - .output() - { - Ok(output) => { - let combined = format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - if output.status.success() || combined.to_lowercase().contains("not found") { - return Ok(()); - } - last_output = combined; - } - Err(error) => last_output = format!("spawn openshell CLI: {error}"), - } - std::thread::sleep(Duration::from_millis(500)); - } - Err(format!( - "'{}' still failing after {ATTEMPTS} attempts:\n{last_output}", - args.join(" ") - )) -} - -/// A provider and the profile it uses, both under one name unique to the -/// test. Dropping it deletes both, also when the test fails. -struct GatingProvider { - name: String, - progress: Progress, -} - -impl GatingProvider { - async fn install_with_endpoints( - progress: Progress, - rest_port: u16, - websocket_port: u16, - ) -> Result { - let name = unique_name(progress.scenario); - let profile = write_provider_profile(&name, rest_port, websocket_port)?; - Self::install(progress, name, &profile).await - } - - async fn install_endpointless(progress: Progress) -> Result { - let name = unique_name(progress.scenario); - let profile = write_endpointless_provider_profile(&name)?; - Self::install(progress, name, &profile).await - } - - async fn install( - progress: Progress, - name: String, - profile: &NamedTempFile, - ) -> Result { - progress.step(format_args!("importing profile {name}")); - let (imported, output) = - run_cli(&["profile", "import", "--file", utf8_path(profile)?]).await; - if !imported { - return Err(format!("profile import failed:\n{output}")); - } - let provider = Self { name, progress }; - progress.step(format_args!("creating provider {}", provider.name)); - let credential = format!("{TOKEN_ENV}={TEST_SECRET}"); - let (created, output) = run_cli(&[ - "provider", - "create", - "--name", - &provider.name, - "--type", - &provider.name, - "--credential", - &credential, - ]) - .await; - if !created { - return Err(format!("provider create failed:\n{output}")); - } - Ok(provider) - } -} - -impl Drop for GatingProvider { - fn drop(&mut self) { - self.progress - .step(format_args!("deleting provider and profile {}", self.name)); - for kind in ["provider", "profile"] { - if let Err(error) = delete_until_gone(&[kind, "delete", &self.name]) { - self.progress - .step(format_args!("{kind} cleanup did not settle: {error}")); - } - } - self.progress.step("cleanup finished"); - } -} - -/// A provider and profile name that does not collide with parallel tests or -/// earlier runs against the same gateway. -fn unique_name(scenario: &str) -> String { - format!( - "e2e-cg-{scenario}-{}-{:08x}", - std::process::id(), - rand::random::() - ) -} - -fn write_provider_profile( - profile_id: &str, - rest_port: u16, - websocket_port: u16, -) -> Result { - let mut file = tempfile::Builder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| format!("create profile: {error}"))?; - let profile = format!( - r"id: {profile_id} -display_name: E2E Credential Gating -category: other -credentials: - - name: token - env_vars: [{TOKEN_ENV}] - required: true - auth_style: bearer - header_name: authorization -endpoints: - - host: {TEST_HOST} - port: {rest_port} - protocol: rest - access: full - - host: {TEST_HOST} - port: {websocket_port} - protocol: websocket - access: read-write -binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -", - ); - file.write_all(profile.as_bytes()) - .map_err(|error| format!("write profile: {error}"))?; - file.flush() - .map_err(|error| format!("flush profile: {error}"))?; - Ok(file) -} - -fn write_endpointless_provider_profile(profile_id: &str) -> Result { - let mut file = tempfile::Builder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| format!("create endpointless profile: {error}"))?; - let profile = format!( - r"id: {profile_id} -display_name: E2E Endpointless Credential Gating -category: other -credentials: - - name: token - env_vars: [{TOKEN_ENV}] - required: true - auth_style: bearer - header_name: authorization -binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -", - ); - file.write_all(profile.as_bytes()) - .map_err(|error| format!("write endpointless profile: {error}"))?; - file.flush() - .map_err(|error| format!("flush endpointless profile: {error}"))?; - Ok(file) -} - -#[derive(Clone, Copy)] -enum EndpointMode { - L4, - TlsSkip, - L4OptIn, - RestBody { rewrite: bool }, - WebSocket, -} - -#[derive(Clone, Copy)] -enum CredentialSource { - ProviderProfile, - PolicyBinding, -} - -fn write_policy( - port: u16, - mode: EndpointMode, - credential_source: CredentialSource, - provider: &str, -) -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let endpoint_options = match mode { - EndpointMode::L4 => String::new(), - EndpointMode::TlsSkip => { - " protocol: rest\n access: full\n tls: skip\n".to_string() - } - EndpointMode::L4OptIn => " allow_uninspected_credentials: true\n".to_string(), - EndpointMode::RestBody { rewrite } => format!( - " protocol: rest\n access: full\n request_body_credential_rewrite: {rewrite}\n" - ), - EndpointMode::WebSocket => { - " protocol: websocket\n access: read-write\n".to_string() - } - }; - let credential_binding = match credential_source { - CredentialSource::ProviderProfile => String::new(), - CredentialSource::PolicyBinding => { - format!(" credential_binding:\n provider: {provider}\n") - } - }; - let policy = format!( - r"version: 1 -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox -network_policies: - credential_gating: - name: credential_gating - endpoints: - - host: {TEST_HOST} - port: {port} -{endpoint_options}{credential_binding} binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -", - ); - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -#[derive(Debug, Default, Clone, Copy)] -struct BodyObservation { - saw_placeholder: bool, - saw_secret: bool, -} - -struct HttpProbeServer { - port: u16, - observations: Arc>>, - task: JoinHandle<()>, -} - -impl HttpProbeServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind HTTP probe: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read HTTP probe address: {error}"))? - .port(); - let observations = Arc::new(Mutex::new(Vec::new())); - let task_observations = Arc::clone(&observations); - let task = tokio::spawn(async move { - loop { - let Ok((stream, _)) = listener.accept().await else { - break; - }; - let observations = Arc::clone(&task_observations); - tokio::spawn(async move { - let _ = handle_http_probe(stream, observations).await; - }); - } - }); - Ok(Self { - port, - observations, - task, - }) - } - - async fn wait_for_observations(&self, count: usize) -> Vec { - for _ in 0..100 { - let observations = self.observations.lock().unwrap().clone(); - if observations.len() >= count { - return observations; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - self.observations.lock().unwrap().clone() - } -} - -impl Drop for HttpProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -struct BinaryWebSocketProbeServer { - port: u16, - handshake_seen: Arc, - binary_seen: Arc, - task: JoinHandle<()>, -} - -impl BinaryWebSocketProbeServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind WebSocket probe: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read WebSocket probe address: {error}"))? - .port(); - let handshake_seen = Arc::new(AtomicBool::new(false)); - let binary_seen = Arc::new(AtomicBool::new(false)); - let task_handshake_seen = Arc::clone(&handshake_seen); - let task_binary_seen = Arc::clone(&binary_seen); - let task = tokio::spawn(async move { - loop { - let Ok((stream, _)) = listener.accept().await else { - break; - }; - let handshake_seen = Arc::clone(&task_handshake_seen); - let binary_seen = Arc::clone(&task_binary_seen); - tokio::spawn(async move { - let _ = - handle_binary_websocket_probe(stream, handshake_seen, binary_seen).await; - }); - } - }); - Ok(Self { - port, - handshake_seen, - binary_seen, - task, - }) - } - - async fn wait_for_handshake(&self) -> bool { - for _ in 0..100 { - if self.handshake_seen.load(Ordering::Acquire) { - return true; - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - false - } -} - -impl Drop for BinaryWebSocketProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn recv_until(stream: &mut TcpStream, marker: &[u8]) -> std::io::Result> { - let mut received = Vec::new(); - let mut buffer = [0_u8; 1024]; - loop { - let read = stream.read(&mut buffer).await?; - if read == 0 { - return Ok(received); - } - received.extend_from_slice(&buffer[..read]); - if received - .windows(marker.len()) - .any(|window| window == marker) - { - return Ok(received); - } - } -} - -fn websocket_header_value(request: &str, name: &str) -> Option { - request.lines().find_map(|line| { - let (header, value) = line.split_once(':')?; - header - .trim() - .eq_ignore_ascii_case(name) - .then(|| value.trim().to_string()) - }) -} - -fn websocket_accept(key: &str) -> String { - let mut hasher = Sha1::new(); - hasher.update(key.as_bytes()); - hasher.update(WEBSOCKET_GUID.as_bytes()); - base64::engine::general_purpose::STANDARD.encode(hasher.finalize()) -} - -async fn handle_binary_websocket_probe( - mut stream: TcpStream, - handshake_seen: Arc, - binary_seen: Arc, -) -> std::io::Result<()> { - let request_bytes = recv_until(&mut stream, b"\r\n\r\n").await?; - let request = String::from_utf8_lossy(&request_bytes); - let key = websocket_header_value(&request, "Sec-WebSocket-Key").ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::InvalidData, "missing WebSocket key") - })?; - let accept = websocket_accept(&key); - let response = format!( - "HTTP/1.1 101 Switching Protocols\r\n\ - Upgrade: websocket\r\n\ - Connection: Upgrade\r\n\ - Sec-WebSocket-Accept: {accept}\r\n\ - \r\n" - ); - stream.write_all(response.as_bytes()).await?; - handshake_seen.store(true, Ordering::Release); - - let mut header = [0_u8; 2]; - if tokio::time::timeout(Duration::from_secs(5), stream.read_exact(&mut header)) - .await - .is_ok_and(|result| result.is_ok()) - && header[0] & 0x0f == 0x02 - { - binary_seen.store(true, Ordering::Release); - } - Ok(()) -} - -async fn handle_http_probe( - mut stream: TcpStream, - observations: Arc>>, -) -> std::io::Result<()> { - let mut received = Vec::new(); - let mut buffer = [0_u8; 4096]; - let mut expected_total = None; - loop { - let read = tokio::time::timeout(Duration::from_secs(10), stream.read(&mut buffer)).await; - let Ok(Ok(read)) = read else { - break; - }; - if read == 0 { - break; - } - received.extend_from_slice(&buffer[..read]); - if expected_total.is_none() - && let Some(header_end) = received.windows(4).position(|window| window == b"\r\n\r\n") - { - let header_end = header_end + 4; - let headers = String::from_utf8_lossy(&received[..header_end]); - let content_length = headers - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - expected_total = Some(header_end + content_length); - } - if expected_total.is_some_and(|expected| received.len() >= expected) { - break; - } - } - - let observation = BodyObservation { - saw_placeholder: received - .windows(PLACEHOLDER_PREFIX.len()) - .any(|window| window == PLACEHOLDER_PREFIX.as_bytes()), - saw_secret: received - .windows(TEST_SECRET.len()) - .any(|window| window == TEST_SECRET.as_bytes()), - }; - observations.lock().unwrap().push(observation); - if expected_total.is_some_and(|expected| received.len() >= expected) { - let result = if observation.saw_secret && !observation.saw_placeholder { - "BODY_REWRITTEN" - } else if observation.saw_placeholder && !observation.saw_secret { - "BODY_TEXT" - } else { - "BODY_BAD" - }; - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{result}", - result.len() - ); - stream.write_all(response.as_bytes()).await?; - } - Ok(()) -} - -fn body_client_script(port: u16) -> String { - format!( - r#" -import os -import socket - -host = {TEST_HOST:?} -port = {port} -token = os.environ[{TOKEN_ENV:?}] - -with socket.create_connection((host, port), timeout=10) as sock: - target = f"{{host}}:{{port}}" - body = ("prefix-" + token + "-suffix").encode("utf-8") - request = ( - f"POST /token HTTP/1.1\r\nHost: {{target}}\r\n" - f"Content-Type: text/plain\r\nContent-Length: {{len(body)}}\r\nConnection: close\r\n\r\n" - ).encode("ascii") + body - sock.sendall(request) - sock.settimeout(3) - response = b"" - while True: - try: - chunk = sock.recv(4096) - except socket.timeout: - break - if not chunk: - break - response += chunk - print("BODY_REWRITTEN" if b"BODY_REWRITTEN" in response else "BODY_TEXT" if b"BODY_TEXT" in response else "BODY_DENIED") -"# - ) -} - -fn binary_websocket_client_script(port: u16) -> String { - format!( - r#" -import base64 -import os -import socket -import struct - -host = {TEST_HOST:?} -port = {port} - -def recv_until(sock, marker): - data = b"" - while marker not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def recv_exact(sock, size): - data = b"" - while len(data) < size: - chunk = sock.recv(size - len(data)) - if not chunk: - break - data += chunk - return data - -with socket.create_connection((host, port), timeout=10) as sock: - target = f"{{host}}:{{port}}" - key = base64.b64encode(os.urandom(16)).decode("ascii") - request = ( - f"GET /ws HTTP/1.1\r\nHost: {{target}}\r\n" - "Upgrade: websocket\r\nConnection: Upgrade\r\n" - f"Sec-WebSocket-Key: {{key}}\r\nSec-WebSocket-Version: 13\r\n\r\n" - ) - sock.sendall(request.encode("ascii")) - if not recv_until(sock, b"\r\n\r\n").startswith(b"HTTP/1.1 101"): - raise RuntimeError("upgrade failed") - payload = b"binary-credential-channel" - mask = os.urandom(4) - masked = bytes(byte ^ mask[index % 4] for index, byte in enumerate(payload)) - frame = bytes([0x82, 0x80 | len(payload)]) + mask + masked - sock.sendall(frame) - sock.settimeout(3) - denied = False - try: - header = recv_exact(sock, 2) - if not header: - denied = True - elif len(header) == 2: - opcode = header[0] & 0x0f - masked = bool(header[1] & 0x80) - payload_length = header[1] & 0x7f - if opcode == 0x08 and not masked and payload_length < 126: - close_payload = recv_exact(sock, payload_length) - denied = ( - len(close_payload) >= 2 - and struct.unpack("!H", close_payload[:2])[0] == 1008 - ) - except socket.timeout: - denied = True - print("BINARY_DENIED" if denied else "BINARY_FORWARDED") -"# - ) -} - -async fn sandbox_create_failure( - progress: Progress, - label: &str, - provider: &str, - policy: &NamedTempFile, -) -> String { - progress.step(format_args!( - "{label}: creating sandbox, expecting rejection" - )); - let policy_path = policy.path().to_str().expect("policy path is UTF-8"); - let (success, output) = run_cli(&[ - "sandbox", - "create", - "--policy", - policy_path, - "--provider", - provider, - "--", - "echo", - "must-not-run", - ]) - .await; - assert!(!success, "sandbox create unexpectedly succeeded:\n{output}"); - progress.step(format_args!("{label}: rejected")); - output -} - -/// Create a sandbox, run one command in it, and delete it. -/// -/// Returns the create output followed by the command output. -async fn run_in_sandbox( - progress: Progress, - label: &str, - create_args: &[&str], - command: &[&str], -) -> Result { - progress.step(format_args!("{label}: creating sandbox")); - let mut sandbox = SandboxGuard::create(create_args).await?; - progress.step(format_args!( - "{label}: sandbox {} ready, running command", - sandbox.name - )); - let output = sandbox.exec(command).await; - progress.step(format_args!("{label}: deleting sandbox {}", sandbox.name)); - sandbox.cleanup().await; - progress.step(format_args!("{label}: sandbox deleted")); - Ok(format!("{}{}", sandbox.create_output, output?)) -} - -async fn assert_gateway_admission( - progress: Progress, - provider: &str, - port: u16, - credential_source: CredentialSource, -) -> Result<(), String> { - let l4 = write_policy(port, EndpointMode::L4, credential_source, provider)?; - let l4_error = sandbox_create_failure(progress, "L4-only policy", provider, &l4).await; - assert!(l4_error.contains("credentialed endpoint"), "{l4_error}"); - assert!(l4_error.contains("L4-only"), "{l4_error}"); - - let tls_skip = write_policy(port, EndpointMode::TlsSkip, credential_source, provider)?; - let tls_error = sandbox_create_failure(progress, "tls: skip policy", provider, &tls_skip).await; - assert!(tls_error.contains("credentialed endpoint"), "{tls_error}"); - assert!( - tls_error.contains("tls:") && tls_error.contains("skip"), - "{tls_error}" - ); - - let opt_in = write_policy(port, EndpointMode::L4OptIn, credential_source, provider)?; - let accepted_marker = match credential_source { - CredentialSource::ProviderProfile => "OPT_IN_ACCEPTED", - CredentialSource::PolicyBinding => "ENDPOINTLESS_OPT_IN_ACCEPTED", - }; - let output = run_in_sandbox( - progress, - "opt-in policy", - &["--policy", utf8_path(&opt_in)?, "--provider", provider], - &["echo", accepted_marker], - ) - .await?; - assert!(output.contains(accepted_marker), "{output}"); - Ok(()) -} - -async fn wait_for_provider_key_presence( - sandbox: &SandboxGuard, - expected_present: bool, -) -> Result<(), String> { - let expected = if expected_present { - "TOKEN_PLACEHOLDER" - } else { - "TOKEN_ABSENT" - }; - let mut last_output = String::new(); - for _ in 0..60 { - last_output = sandbox.exec(&["sh", "-lc", KEY_PRESENCE_SCRIPT]).await?; - if last_output.contains(expected) { - return Ok(()); - } - tokio::time::sleep(Duration::from_millis(500)).await; - } - Err(format!( - "provider environment did not converge to {expected}; last key-presence output:\n{last_output}" - )) -} - -async fn set_policy(progress: Progress, sandbox: &str, policy_path: &str) -> Result<(), String> { - let (success, output) = run_cli(&[ - "policy", - "set", - sandbox, - "--policy", - policy_path, - "--wait", - "--timeout", - "120", - ]) - .await; - if !success { - return Err(format!("policy update failed:\n{output}")); - } - progress.step("policy applied"); - Ok(()) -} - -async fn assert_endpointless_provider_env_live_update( - progress: Progress, - provider: &str, - port: u16, -) -> Result<(), String> { - let unbound = write_policy( - port, - EndpointMode::RestBody { rewrite: false }, - CredentialSource::ProviderProfile, - provider, - )?; - let bound = write_policy( - port, - EndpointMode::RestBody { rewrite: false }, - CredentialSource::PolicyBinding, - provider, - )?; - let unbound_path = utf8_path(&unbound)?; - let bound_path = utf8_path(&bound)?; - - progress.step("creating long-running sandbox with the unbound policy"); - let mut sandbox = SandboxGuard::create_keep_with_args( - &["--policy", unbound_path, "--provider", provider, "--no-tty"], - &["sh", "-c", "echo PROVIDER_UPDATE_READY && sleep infinity"], - "PROVIDER_UPDATE_READY", - ) - .await?; - - let result = async { - progress.step(format_args!( - "sandbox {} ready, waiting for the token to be absent", - sandbox.name - )); - wait_for_provider_key_presence(&sandbox, false).await?; - - progress.step("binding the provider with policy set --wait"); - set_policy(progress, &sandbox.name, bound_path) - .await - .map_err(|error| format!("binding {error}"))?; - progress.step("waiting for the token placeholder"); - wait_for_provider_key_presence(&sandbox, true).await?; - - progress.step("unbinding the provider with policy set --wait"); - set_policy(progress, &sandbox.name, unbound_path) - .await - .map_err(|error| format!("unbinding {error}"))?; - progress.step("waiting for the token to be absent"); - wait_for_provider_key_presence(&sandbox, false).await - } - .await; - - progress.step(format_args!("deleting sandbox {}", sandbox.name)); - sandbox.cleanup().await; - progress.step("sandbox deleted"); - result -} - -async fn run_body_sandbox( - progress: Progress, - provider: &str, - port: u16, - mode: EndpointMode, - credential_source: CredentialSource, -) -> Result { - let policy = write_policy(port, mode, credential_source, provider)?; - let script = body_client_script(port); - run_in_sandbox( - progress, - "REST body client", - &["--policy", utf8_path(&policy)?, "--provider", provider], - &["python3", "-c", &script], - ) - .await -} - -async fn assert_websocket_binary_denied( - progress: Progress, - provider: &str, - server: &BinaryWebSocketProbeServer, -) -> Result<(), String> { - let policy = write_policy( - server.port, - EndpointMode::WebSocket, - CredentialSource::ProviderProfile, - provider, - )?; - let script = binary_websocket_client_script(server.port); - let output = run_in_sandbox( - progress, - "WebSocket client", - &["--policy", utf8_path(&policy)?, "--provider", provider], - &["python3", "-c", &script], - ) - .await?; - assert!(output.contains("BINARY_DENIED"), "{output}"); - assert!( - server.wait_for_handshake().await, - "upstream should receive the WebSocket handshake" - ); - assert!( - !server.binary_seen.load(Ordering::Acquire), - "credentialed WebSocket binary frame reached upstream" - ); - Ok(()) -} - -/// Start both probe servers and install a provider whose profile declares -/// their endpoints. -async fn profile_endpoint_fixture( - progress: Progress, -) -> (HttpProbeServer, BinaryWebSocketProbeServer, GatingProvider) { - let server = HttpProbeServer::start().await.expect("start HTTP probe"); - let websocket_server = BinaryWebSocketProbeServer::start() - .await - .expect("start WebSocket probe"); - let provider = - GatingProvider::install_with_endpoints(progress, server.port, websocket_server.port) - .await - .expect("install credentialed provider"); - (server, websocket_server, provider) -} - -/// Start the HTTP probe server and install a provider whose profile declares -/// no endpoints, so policies must bind it explicitly. -async fn endpointless_fixture(progress: Progress) -> (HttpProbeServer, GatingProvider) { - let server = HttpProbeServer::start().await.expect("start HTTP probe"); - let provider = GatingProvider::install_endpointless(progress) - .await - .expect("install endpointless provider"); - (server, provider) -} - -#[tokio::test] -async fn profile_rest_endpoint_keeps_body_placeholder() { - let progress = Progress::start("profile-rest"); - let (server, _websocket_server, provider) = profile_endpoint_fixture(progress).await; - - let output = run_body_sandbox( - progress, - &provider.name, - server.port, - EndpointMode::RestBody { rewrite: false }, - CredentialSource::ProviderProfile, - ) - .await - .expect("run REST body client"); - assert!(output.contains("BODY_TEXT"), "{output}"); - let observations = server.wait_for_observations(1).await; - assert_eq!(observations.len(), 1, "observations: {observations:?}"); - assert!(observations[0].saw_placeholder); - assert!(!observations[0].saw_secret); - progress.step("checks passed"); -} - -#[tokio::test] -async fn profile_websocket_endpoint_denies_binary_frames() { - let progress = Progress::start("profile-websocket"); - let (_server, websocket_server, provider) = profile_endpoint_fixture(progress).await; - - assert_websocket_binary_denied(progress, &provider.name, &websocket_server) - .await - .expect("run WebSocket client"); - progress.step("checks passed"); -} - -#[tokio::test] -async fn policy_binding_requires_inspection_or_opt_in() { - let progress = Progress::start("binding-admission"); - let (server, provider) = endpointless_fixture(progress).await; - - assert_gateway_admission( - progress, - &provider.name, - server.port, - CredentialSource::PolicyBinding, - ) - .await - .expect("gateway admission"); - progress.step("checks passed"); -} - -#[tokio::test] -async fn policy_binding_rest_body_keeps_placeholder_without_rewrite() { - let progress = Progress::start("binding-body"); - let (server, provider) = endpointless_fixture(progress).await; - - let literal = run_body_sandbox( - progress, - &provider.name, - server.port, - EndpointMode::RestBody { rewrite: false }, - CredentialSource::PolicyBinding, - ) - .await - .expect("run REST body client"); - assert!(literal.contains("BODY_TEXT"), "{literal}"); - let observations = server.wait_for_observations(1).await; - assert_eq!(observations.len(), 1, "observations: {observations:?}"); - assert!(observations[0].saw_placeholder); - assert!(!observations[0].saw_secret); - progress.step("checks passed"); -} - -#[tokio::test] -async fn policy_binding_rest_body_rewrite_sends_secret_upstream() { - let progress = Progress::start("binding-rewrite"); - let (server, provider) = endpointless_fixture(progress).await; - - let rewritten = run_body_sandbox( - progress, - &provider.name, - server.port, - EndpointMode::RestBody { rewrite: true }, - CredentialSource::PolicyBinding, - ) - .await - .expect("run REST body client"); - assert!(rewritten.contains("BODY_REWRITTEN")); - assert!(!rewritten.contains(TEST_SECRET)); - assert!(!rewritten.contains(PLACEHOLDER_PREFIX)); - let observations = server.wait_for_observations(1).await; - assert_eq!(observations.len(), 1, "observations: {observations:?}"); - assert!(!observations[0].saw_placeholder); - assert!(observations[0].saw_secret); - progress.step("checks passed"); -} - -#[tokio::test] -async fn policy_binding_updates_provider_environment_live() { - let progress = Progress::start("binding-live-update"); - let (server, provider) = endpointless_fixture(progress).await; - - assert_endpointless_provider_env_live_update(progress, &provider.name, server.port) - .await - .expect("provider environment live update"); - progress.step("checks passed"); -} diff --git a/tests/suites/conformance/src/credential_gating/mod.rs b/tests/suites/conformance/src/credential_gating/mod.rs index fd9dd06077..170ef19c65 100644 --- a/tests/suites/conformance/src/credential_gating/mod.rs +++ b/tests/suites/conformance/src/credential_gating/mod.rs @@ -5,7 +5,7 @@ //! what the supervisor does with the credential in REST bodies and WebSocket //! frames. //! -//! Ported from `e2e/rust/tests/credential_gating.rs` (NVIDIA/OpenShell#5264). +//! Ported from the e2e tests added in NVIDIA/OpenShell#5264. //! Each test imports its own provider profile and creates a provider under a //! name unique to the run; cleanup deletes them after the test's sandboxes. //! The probe servers run where the tests run, which is the host the gateway From 7a70c12838e9e281ee0ccd0d734cc6bdf9c133ad Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 6 Nov 2026 10:56:51 -0800 Subject: [PATCH 05/16] test(testkit): add policy helpers, provider install, and YAML files Add testkit::policy (a base policy from base-policy.yaml, allow(), and Endpoint settings), install_provider(), YamlFile, ExecOutput::success(), and PYTHON_BINARIES, so suites build policies and providers the same way. Signed-off-by: Matthew Grossman --- .../mod.rs => credential_gating.rs} | 0 .../src/credential_gating/client.rs | 123 -------- .../src/credential_gating/probe.rs | 296 ------------------ tests/suites/drivers/podman/Cargo.lock | 162 ++++++++++ tests/testkit/Cargo.lock | 161 ++++++++++ tests/testkit/README.md | 13 +- tests/testkit/testkit/Cargo.toml | 3 +- tests/testkit/testkit/src/base-policy.yaml | 17 + tests/testkit/testkit/src/context.rs | 39 ++- tests/testkit/testkit/src/image.rs | 9 + tests/testkit/testkit/src/lib.rs | 7 +- tests/testkit/testkit/src/policy.rs | 184 +++++++++++ tests/testkit/testkit/src/provider.rs | 46 +++ tests/testkit/testkit/src/yaml_file.rs | 30 ++ 14 files changed, 647 insertions(+), 443 deletions(-) rename tests/suites/conformance/src/{credential_gating/mod.rs => credential_gating.rs} (100%) delete mode 100644 tests/suites/conformance/src/credential_gating/client.rs delete mode 100644 tests/suites/conformance/src/credential_gating/probe.rs create mode 100644 tests/testkit/testkit/src/base-policy.yaml create mode 100644 tests/testkit/testkit/src/policy.rs create mode 100644 tests/testkit/testkit/src/provider.rs create mode 100644 tests/testkit/testkit/src/yaml_file.rs diff --git a/tests/suites/conformance/src/credential_gating/mod.rs b/tests/suites/conformance/src/credential_gating.rs similarity index 100% rename from tests/suites/conformance/src/credential_gating/mod.rs rename to tests/suites/conformance/src/credential_gating.rs diff --git a/tests/suites/conformance/src/credential_gating/client.rs b/tests/suites/conformance/src/credential_gating/client.rs deleted file mode 100644 index 3ade29cc2a..0000000000 --- a/tests/suites/conformance/src/credential_gating/client.rs +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Commands the tests run in the sandbox. - -use super::{TEST_HOST, TOKEN_ENV}; - -/// Prints `TOKEN_ABSENT`, `TOKEN_PLACEHOLDER`, or `TOKEN_UNSAFE` for the -/// provider's environment variable, without printing its value. -pub(super) const KEY_PRESENCE_SCRIPT: &str = r#"if [ "${E2E_GATING_TOKEN+x}" != x ]; then - echo TOKEN_ABSENT -else - case "$E2E_GATING_TOKEN" in - openshell:resolve:env:*) echo TOKEN_PLACEHOLDER ;; - *) echo TOKEN_UNSAFE ;; - esac -fi"#; - -/// POSTs a body containing the provider's token to the HTTP probe and prints -/// the probe's verdict, or `BODY_DENIED` without one. -pub(super) fn body_client_script(port: u16) -> String { - format!( - r#" -import os -import socket - -host = {TEST_HOST:?} -port = {port} -token = os.environ[{TOKEN_ENV:?}] - -with socket.create_connection((host, port), timeout=10) as sock: - target = f"{{host}}:{{port}}" - body = ("prefix-" + token + "-suffix").encode("utf-8") - request = ( - f"POST /token HTTP/1.1\r\nHost: {{target}}\r\n" - f"Content-Type: text/plain\r\nContent-Length: {{len(body)}}\r\nConnection: close\r\n\r\n" - ).encode("ascii") + body - sock.sendall(request) - sock.settimeout(3) - response = b"" - while True: - try: - chunk = sock.recv(4096) - except socket.timeout: - break - if not chunk: - break - response += chunk - print("BODY_REWRITTEN" if b"BODY_REWRITTEN" in response else "BODY_TEXT" if b"BODY_TEXT" in response else "BODY_DENIED") -"# - ) -} - -/// Upgrades to WebSocket at the probe, sends one binary frame, and prints -/// `BINARY_DENIED` if the connection closed or answered with close code 1008, -/// or `BINARY_FORWARDED` otherwise. -pub(super) fn binary_websocket_client_script(port: u16) -> String { - format!( - r#" -import base64 -import os -import socket -import struct - -host = {TEST_HOST:?} -port = {port} - -def recv_until(sock, marker): - data = b"" - while marker not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def recv_exact(sock, size): - data = b"" - while len(data) < size: - chunk = sock.recv(size - len(data)) - if not chunk: - break - data += chunk - return data - -with socket.create_connection((host, port), timeout=10) as sock: - target = f"{{host}}:{{port}}" - key = base64.b64encode(os.urandom(16)).decode("ascii") - request = ( - f"GET /ws HTTP/1.1\r\nHost: {{target}}\r\n" - "Upgrade: websocket\r\nConnection: Upgrade\r\n" - f"Sec-WebSocket-Key: {{key}}\r\nSec-WebSocket-Version: 13\r\n\r\n" - ) - sock.sendall(request.encode("ascii")) - if not recv_until(sock, b"\r\n\r\n").startswith(b"HTTP/1.1 101"): - raise RuntimeError("upgrade failed") - payload = b"binary-credential-channel" - mask = os.urandom(4) - masked = bytes(byte ^ mask[index % 4] for index, byte in enumerate(payload)) - frame = bytes([0x82, 0x80 | len(payload)]) + mask + masked - sock.sendall(frame) - sock.settimeout(3) - denied = False - try: - header = recv_exact(sock, 2) - if not header: - denied = True - elif len(header) == 2: - opcode = header[0] & 0x0f - masked = bool(header[1] & 0x80) - payload_length = header[1] & 0x7f - if opcode == 0x08 and not masked and payload_length < 126: - close_payload = recv_exact(sock, payload_length) - denied = ( - len(close_payload) >= 2 - and struct.unpack("!H", close_payload[:2])[0] == 1008 - ) - except socket.timeout: - denied = True - print("BINARY_DENIED" if denied else "BINARY_FORWARDED") -"# - ) -} diff --git a/tests/suites/conformance/src/credential_gating/probe.rs b/tests/suites/conformance/src/credential_gating/probe.rs deleted file mode 100644 index 46a3974361..0000000000 --- a/tests/suites/conformance/src/credential_gating/probe.rs +++ /dev/null @@ -1,296 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Servers on the host that the sandbox clients reach through -//! `host.openshell.internal`, recording what arrived upstream. - -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, PoisonError}; -use std::time::Duration; - -use base64::Engine as _; -use sha1::{Digest as _, Sha1}; -use testkit::{Error, Poll, Result, ResultExt as _}; -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::task::JoinHandle; - -use super::{PLACEHOLDER_PREFIX, TEST_SECRET}; - -const WEBSOCKET_GUID: &str = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; -const OBSERVE_TIMEOUT: Duration = Duration::from_secs(10); -const OBSERVE_INTERVAL: Duration = Duration::from_millis(100); - -/// What one request body that reached the HTTP probe contained. -#[derive(Debug, Default, Clone, Copy)] -pub(super) struct BodyObservation { - pub(super) saw_placeholder: bool, - pub(super) saw_secret: bool, -} - -/// An HTTP server that records whether each request carried the credential -/// placeholder or the secret, and answers with `BODY_TEXT`, -/// `BODY_REWRITTEN`, or `BODY_BAD`. It stops when dropped. -pub(super) struct HttpProbeServer { - pub(super) port: u16, - observations: Arc>>, - task: JoinHandle<()>, -} - -impl HttpProbeServer { - pub(super) async fn start() -> Result { - let (listener, port) = bind("HTTP probe").await?; - let observations = Arc::new(Mutex::new(Vec::new())); - let task_observations = Arc::clone(&observations); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - let observations = Arc::clone(&task_observations); - tokio::spawn(async move { - let _ = handle_http_probe(stream, observations).await; - }); - } - }); - Ok(Self { - port, - observations, - task, - }) - } - - /// The requests the server has seen, once there are at least `count`. - pub(super) async fn wait_for_observations(&self, count: usize) -> Result> { - testkit::poll_until( - &format!("{count} request(s) at the HTTP probe"), - OBSERVE_TIMEOUT, - OBSERVE_INTERVAL, - async || { - let observations = self - .observations - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone(); - Ok(if observations.len() >= count { - Poll::Ready(observations) - } else { - Poll::Pending(format!("{observations:?}")) - }) - }, - ) - .await - } -} - -impl Drop for HttpProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -/// A WebSocket server that completes the handshake and records whether a -/// binary frame followed. It stops when dropped. -pub(super) struct BinaryWebSocketProbeServer { - pub(super) port: u16, - handshake_seen: Arc, - binary_seen: Arc, - task: JoinHandle<()>, -} - -impl BinaryWebSocketProbeServer { - pub(super) async fn start() -> Result { - let (listener, port) = bind("WebSocket probe").await?; - let handshake_seen = Arc::new(AtomicBool::new(false)); - let binary_seen = Arc::new(AtomicBool::new(false)); - let task_handshake_seen = Arc::clone(&handshake_seen); - let task_binary_seen = Arc::clone(&binary_seen); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - let handshake_seen = Arc::clone(&task_handshake_seen); - let binary_seen = Arc::clone(&task_binary_seen); - tokio::spawn(async move { - let _ = - handle_binary_websocket_probe(stream, handshake_seen, binary_seen).await; - }); - } - }); - Ok(Self { - port, - handshake_seen, - binary_seen, - task, - }) - } - - /// Wait until a client completed the WebSocket handshake. - pub(super) async fn wait_for_handshake(&self) -> Result { - testkit::poll_until( - "the WebSocket handshake at the probe", - OBSERVE_TIMEOUT, - OBSERVE_INTERVAL, - async || { - Ok(if self.handshake_seen.load(Ordering::Acquire) { - Poll::Ready(()) - } else { - Poll::Pending("no handshake".to_string()) - }) - }, - ) - .await - } - - pub(super) fn binary_seen(&self) -> bool { - self.binary_seen.load(Ordering::Acquire) - } -} - -impl Drop for BinaryWebSocketProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -/// Listen on every host address, since drivers map -/// `host.openshell.internal` to different ones. -async fn bind(what: &str) -> Result<(TcpListener, u16)> { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| Error::from_error(&error)) - .context(format!("bind the {what}"))?; - let port = listener - .local_addr() - .map_err(|error| Error::from_error(&error)) - .context(format!("read the {what}'s address"))? - .port(); - Ok((listener, port)) -} - -async fn recv_until(stream: &mut TcpStream, marker: &[u8]) -> std::io::Result> { - let mut received = Vec::new(); - let mut buffer = [0_u8; 1024]; - loop { - let read = stream.read(&mut buffer).await?; - if read == 0 { - return Ok(received); - } - received.extend_from_slice(&buffer[..read]); - if received - .windows(marker.len()) - .any(|window| window == marker) - { - return Ok(received); - } - } -} - -fn header_value(request: &str, name: &str) -> Option { - request.lines().find_map(|line| { - let (header, value) = line.split_once(':')?; - header - .trim() - .eq_ignore_ascii_case(name) - .then(|| value.trim().to_string()) - }) -} - -/// The `Sec-WebSocket-Accept` value for `key`, which the supervisor checks. -fn websocket_accept(key: &str) -> String { - let mut hasher = Sha1::new(); - hasher.update(key.as_bytes()); - hasher.update(WEBSOCKET_GUID.as_bytes()); - base64::engine::general_purpose::STANDARD.encode(hasher.finalize()) -} - -async fn handle_binary_websocket_probe( - mut stream: TcpStream, - handshake_seen: Arc, - binary_seen: Arc, -) -> std::io::Result<()> { - let _ = stream.set_nodelay(true); - let request_bytes = recv_until(&mut stream, b"\r\n\r\n").await?; - let request = String::from_utf8_lossy(&request_bytes); - let key = header_value(&request, "Sec-WebSocket-Key").ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::InvalidData, "missing WebSocket key") - })?; - let accept = websocket_accept(&key); - let response = format!( - "HTTP/1.1 101 Switching Protocols\r\n\ - Upgrade: websocket\r\n\ - Connection: Upgrade\r\n\ - Sec-WebSocket-Accept: {accept}\r\n\ - \r\n" - ); - stream.write_all(response.as_bytes()).await?; - handshake_seen.store(true, Ordering::Release); - - let mut header = [0_u8; 2]; - if tokio::time::timeout(Duration::from_secs(5), stream.read_exact(&mut header)) - .await - .is_ok_and(|result| result.is_ok()) - && header[0] & 0x0f == 0x02 - { - binary_seen.store(true, Ordering::Release); - } - Ok(()) -} - -async fn handle_http_probe( - mut stream: TcpStream, - observations: Arc>>, -) -> std::io::Result<()> { - let _ = stream.set_nodelay(true); - let mut received = Vec::new(); - let mut buffer = [0_u8; 4096]; - let mut expected_total = None; - loop { - let read = tokio::time::timeout(Duration::from_secs(10), stream.read(&mut buffer)).await; - let Ok(Ok(read)) = read else { - break; - }; - if read == 0 { - break; - } - received.extend_from_slice(&buffer[..read]); - if expected_total.is_none() - && let Some(header_end) = received.windows(4).position(|window| window == b"\r\n\r\n") - { - let header_end = header_end + 4; - let headers = String::from_utf8_lossy(&received[..header_end]); - let content_length = header_value(&headers, "content-length") - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); - expected_total = Some(header_end + content_length); - } - if expected_total.is_some_and(|expected| received.len() >= expected) { - break; - } - } - - let observation = BodyObservation { - saw_placeholder: contains(&received, PLACEHOLDER_PREFIX), - saw_secret: contains(&received, TEST_SECRET), - }; - observations - .lock() - .unwrap_or_else(PoisonError::into_inner) - .push(observation); - if expected_total.is_some_and(|expected| received.len() >= expected) { - let result = if observation.saw_secret && !observation.saw_placeholder { - "BODY_REWRITTEN" - } else if observation.saw_placeholder && !observation.saw_secret { - "BODY_TEXT" - } else { - "BODY_BAD" - }; - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{result}", - result.len() - ); - stream.write_all(response.as_bytes()).await?; - } - Ok(()) -} - -fn contains(haystack: &[u8], needle: &str) -> bool { - haystack - .windows(needle.len()) - .any(|window| window == needle.as_bytes()) -} diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index 6633a3e703..d062100826 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -493,6 +493,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -560,6 +570,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1137,6 +1153,25 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "rand 0.10.2", + "ring", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "url", +] + [[package]] name = "hkdf" version = "0.13.0" @@ -1517,6 +1552,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2020,6 +2104,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2108,6 +2196,19 @@ dependencies = [ "tower", ] +[[package]] +name = "openshell-policy" +version = "0.0.0" +dependencies = [ + "hickory-proto", + "miette", + "openshell-core", + "openshell-policy-schema", + "prost-types", + "serde", + "serde_json", +] + [[package]] name = "openshell-policy-schema" version = "0.0.0" @@ -2156,12 +2257,14 @@ dependencies = [ "linkme", "openshell-bootstrap", "openshell-core", + "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", "serde", "serde_json", + "tempfile", "tokio", ] @@ -2353,6 +2456,12 @@ dependencies = [ "openshell-testkit", ] +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.5" @@ -2771,6 +2880,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2853,6 +2971,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -3099,6 +3226,22 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -3967,6 +4110,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4114,6 +4267,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index db2e79b6ca..383f9f2454 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -493,6 +493,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -551,6 +561,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1128,6 +1144,25 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "rand 0.10.2", + "ring", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "url", +] + [[package]] name = "hkdf" version = "0.13.0" @@ -1508,6 +1543,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2011,6 +2095,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2099,6 +2187,19 @@ dependencies = [ "tower", ] +[[package]] +name = "openshell-policy" +version = "0.0.0" +dependencies = [ + "hickory-proto", + "miette", + "openshell-core", + "openshell-policy-schema", + "prost-types", + "serde", + "serde_json", +] + [[package]] name = "openshell-policy-schema" version = "0.0.0" @@ -2147,6 +2248,7 @@ dependencies = [ "linkme", "openshell-bootstrap", "openshell-core", + "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", @@ -2329,6 +2431,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.5" @@ -2747,6 +2855,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2829,6 +2946,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -3075,6 +3201,22 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -3943,6 +4085,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4090,6 +4242,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" diff --git a/tests/testkit/README.md b/tests/testkit/README.md index c02ad01c25..ae8b25f95c 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -65,7 +65,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `cx.sandbox(suffix)` to build and create a sandbox with a run-unique name. The returned `Sandbox` can wait for ready or finished, stop, start, exec, - read its config and draft policy, set a setting, and delete. + read its config and draft policy, set a setting, and delete. `exec` output + has `success()`. - `cx.track(name)` to register a sandbox created some other way, such as through the CLI, for cleanup. Call it before the create, so a create that fails after the gateway accepted it is still cleaned @@ -83,6 +84,16 @@ fails. A killed test process skips cleanup, so names are unique to the run. `success()`, `failure()`, and `json()` on the output. `cx.cli_handle()` returns that CLI for code that can't borrow `cx`, such as a cleanup hook. - `cx.on_cleanup(name, hook)` to remove something when the test ends. +- `testkit::policy` for sandbox policies: `policy::base()` reads + `testkit/src/base-policy.yaml`, which allows no network access, and + `policy::allow(&mut policy, name, host, port, binaries)` adds an endpoint + that `policy::Endpoint` configures (`.rest(Access::Full)`, + `.credential_binding(provider)`, ...). Pass the result to + `cx.sandbox(..).policy(policy)`, or `policy::to_yaml` for the CLI. + `testkit::PYTHON_BINARIES` names `python3` in `python_image()` images. +- `testkit::install_provider(cx, name, profile, credentials)` to import a + provider profile and create a provider of it, with cleanup for both. +- `testkit::YamlFile` for YAML a CLI command reads from a file. - `cx.client()` for SDK calls testkit does not wrap. - `poll_until` for waiting on other state. diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index 47ff56c13c..288e11990d 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -17,6 +17,7 @@ name = "testkit" [dependencies] openshell-bootstrap = { path = "../../../crates/openshell-bootstrap" } openshell-core = { path = "../../../crates/openshell-core", default-features = false } +openshell-policy = { path = "../../../crates/openshell-policy" } openshell-sdk = { path = "../../../crates/openshell-sdk" } openshell-testkit-macros = { path = "../testkit-macros" } clap = { version = "4", features = ["derive"] } @@ -27,13 +28,13 @@ quick-junit = "0.8" rand = "0.9" serde = "1" serde_json = "1" +tempfile = "3" tokio = { version = "1.43", features = ["macros", "process", "rt", "time"] } [target.'cfg(unix)'.dependencies] libc = "0.2" [dev-dependencies] -tempfile = "3" tonic = "0.14" # A test program, run by `cargo test` and nextest through `testkit::main`'s diff --git a/tests/testkit/testkit/src/base-policy.yaml b/tests/testkit/testkit/src/base-policy.yaml new file mode 100644 index 0000000000..0935dbaa9a --- /dev/null +++ b/tests/testkit/testkit/src/base-policy.yaml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The sandbox policy tests start from (testkit::policy::base()): the image's +# system directories read-only, the sandbox's home and /tmp writable, the +# sandbox user, and no network access. Tests add the network access they need +# with testkit::policy::allow(). +version: 1 +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 995a6869d3..8709db8a18 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -439,6 +439,20 @@ pub struct ExecOutput { pub stderr: String, } +impl ExecOutput { + /// Fail unless the command exited 0. + pub fn success(self) -> Result { + crate::ensure!( + self.exit_code == 0, + "exec exited with {}; stdout: {}; stderr: {}", + self.exit_code, + self.stdout.trim(), + self.stderr.trim() + ); + Ok(self) + } +} + impl Sandbox { pub fn name(&self) -> &str { &self.name @@ -651,35 +665,18 @@ mod tests { #[test] fn policy_create_request_keeps_the_spec_and_adds_the_policy() { let spec = SandboxSpec { - name: Some("test-sb".to_string()), image: Some("python:3".to_string()), - labels: [("k".to_string(), "v".to_string())].into(), - environment: [("E".to_string(), "1".to_string())].into(), providers: vec!["provider".to_string()], - command: vec!["sleep".to_string(), "infinity".to_string()], - restart_policy: SandboxRestartPolicy::Never, ..SandboxSpec::default() }; - let policy = proto::SandboxPolicy { - version: 1, - ..proto::SandboxPolicy::default() - }; - let scope = Some(proto::workspace_selector("ws")); + let policy = crate::policy::base(); - let request = policy_create_request(spec, policy.clone(), scope.clone()); + let spec = policy_create_request(spec, policy.clone(), None) + .spec + .expect("spec"); - assert_eq!(request.name, "test-sb"); - assert_eq!(request.labels.get("k").map(String::as_str), Some("v")); - assert_eq!(request.workspace_scope, scope); - let spec = request.spec.expect("spec"); assert_eq!(spec.template.expect("template").image, "python:3"); - assert_eq!(spec.environment.get("E").map(String::as_str), Some("1")); assert_eq!(spec.providers, ["provider"]); - assert_eq!(spec.command, ["sleep", "infinity"]); assert_eq!(spec.policy, Some(policy)); - assert_eq!( - spec.restart_policy, - proto::SandboxRestartPolicy::Never as i32 - ); } } diff --git a/tests/testkit/testkit/src/image.rs b/tests/testkit/testkit/src/image.rs index 26bb2278fe..08e13825a0 100644 --- a/tests/testkit/testkit/src/image.rs +++ b/tests/testkit/testkit/src/image.rs @@ -17,6 +17,15 @@ const PYTHON_IMAGE_ENV: &str = "OPENSHELL_TEST_PYTHON_IMAGE"; /// `mise run e2e:workload:build` builds it locally. const DEFAULT_PYTHON_IMAGE: &str = "openshell/e2e-python:dev"; +/// The paths `python3` runs from in [`python_image`] images, as network +/// policy binaries: the system Python, the pullable uv image's Python, and the +/// workload image's uv-installed Python behind its virtualenv. +pub const PYTHON_BINARIES: &[&str] = &[ + "/usr/bin/python*", + "/usr/local/bin/python*", + "/sandbox/.uv/python/*/bin/python*", +]; + /// The image named by [`BASE_IMAGE_ENV`], or [`DEFAULT_BASE_IMAGE`]. pub fn base_image() -> String { image_from_env(BASE_IMAGE_ENV, DEFAULT_BASE_IMAGE) diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 834f28c1e7..2786668d9b 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -32,11 +32,14 @@ mod context; mod error; pub mod gateway; mod image; +pub mod policy; mod poll; pub mod program; +mod provider; mod ready; mod registry; mod run; +mod yaml_file; pub use cleanup::CLEANUP_TIMEOUT; pub use cli::{CLI_TIMEOUT, Cli, CliOutput}; @@ -46,14 +49,16 @@ pub use context::{ }; pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; -pub use image::{base_image, python_image}; +pub use image::{PYTHON_BINARIES, base_image, python_image}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until}; pub use program::main; +pub use provider::install_provider; pub use ready::check as check_ready; pub use registry::Test; pub use run::{TestInfo, run}; pub use testkit_macros::test; +pub use yaml_file::YamlFile; #[doc(hidden)] pub mod __private { diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs new file mode 100644 index 0000000000..454cc4fb52 --- /dev/null +++ b/tests/testkit/testkit/src/policy.rs @@ -0,0 +1,184 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox policies for tests: [`base`], read from `base-policy.yaml`, plus +//! [`allow`] and the [`Endpoint`] settings for the network access a test +//! needs. +//! +//! ```ignore +//! let mut policy = testkit::policy::base(); +//! testkit::policy::allow(&mut policy, "upstream", "host.openshell.internal", port, PYTHON_BINARIES) +//! .rest(Access::Full) +//! .credential_binding(&provider); +//! ``` + +use openshell_sdk::raw::proto; + +use crate::error::{Error, Result, ResultExt as _}; + +pub use proto::NetworkAccessPreset as Access; + +const BASE: &str = include_str!("base-policy.yaml"); + +/// The policy in `base-policy.yaml`. It allows no network access. +pub fn base() -> proto::SandboxPolicy { + openshell_policy::parse_sandbox_policy(BASE).expect("base-policy.yaml is a valid policy") +} + +/// Let `binaries` connect to `host:port` under the network policy `name`, +/// adding the policy if it doesn't exist. Returns the new endpoint, which +/// [`Endpoint`] configures further. +pub fn allow<'a>( + policy: &'a mut proto::SandboxPolicy, + name: &str, + host: &str, + port: u16, + binaries: &[&str], +) -> &'a mut proto::NetworkEndpoint { + let rule = policy + .network_policies + .entry(name.to_string()) + .or_insert_with(|| proto::NetworkPolicyRule { + name: name.to_string(), + ..proto::NetworkPolicyRule::default() + }); + for binary in binaries { + if !rule + .binaries + .iter() + .any(|existing| existing.path == *binary) + { + rule.binaries.push(proto::NetworkBinary { + path: (*binary).to_string(), + }); + } + } + rule.endpoints.push(proto::NetworkEndpoint { + host: host.to_string(), + port: port.into(), + ports: vec![port.into()], + ..proto::NetworkEndpoint::default() + }); + rule.endpoints + .last_mut() + .expect("an endpoint was just added") +} + +/// The policy as YAML, for CLI commands such as `policy set`. +pub fn to_yaml(policy: &proto::SandboxPolicy) -> Result { + openshell_policy::serialize_sandbox_policy(policy) + .map_err(|error| Error::msg(format!("{error:?}"))) + .context("serialize the policy") +} + +/// Settings for an endpoint from [`allow`], named as in policy YAML. +pub trait Endpoint { + /// `protocol: rest` with `access`. + fn rest(&mut self, access: Access) -> &mut Self; + /// `protocol: websocket` with `access`. + fn websocket(&mut self, access: Access) -> &mut Self; + /// `tls: skip`. + fn tls_skip(&mut self) -> &mut Self; + /// `request_body_credential_rewrite`. + fn request_body_credential_rewrite(&mut self, rewrite: bool) -> &mut Self; + /// `credential_binding: {provider: }`. + fn credential_binding(&mut self, provider: &str) -> &mut Self; + /// `allow_uninspected_credentials: true`. + fn allow_uninspected_credentials(&mut self) -> &mut Self; +} + +impl Endpoint for proto::NetworkEndpoint { + fn rest(&mut self, access: Access) -> &mut Self { + self.protocol = "rest".to_string(); + self.access = access as i32; + self + } + + fn websocket(&mut self, access: Access) -> &mut Self { + self.protocol = "websocket".to_string(); + self.access = access as i32; + self + } + + fn tls_skip(&mut self) -> &mut Self { + self.tls = proto::NetworkTlsMode::Skip as i32; + self + } + + fn request_body_credential_rewrite(&mut self, rewrite: bool) -> &mut Self { + self.request_body_credential_rewrite = rewrite; + self + } + + fn credential_binding(&mut self, provider: &str) -> &mut Self { + self.credential_binding = Some(proto::NetworkCredentialBinding { + provider: provider.to_string(), + }); + self + } + + fn allow_uninspected_credentials(&mut self) -> &mut Self { + self.allow_uninspected_credentials = true; + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Each helper builds what the YAML it is named after parses to, so the + /// gateway can't tell the two apart. + #[test] + fn helpers_match_the_yaml_they_name() { + let mut policy = base(); + allow( + &mut policy, + "upstream", + "example.test", + 8080, + &["/usr/bin/python*"], + ) + .rest(Access::Full) + .tls_skip() + .request_body_credential_rewrite(true) + .credential_binding("provider"); + allow( + &mut policy, + "upstream", + "example.test", + 8081, + &["/usr/bin/python*"], + ) + .websocket(Access::ReadWrite) + .allow_uninspected_credentials(); + + let yaml = format!( + "{BASE}network_policies: + upstream: + name: upstream + endpoints: + - host: example.test + port: 8080 + protocol: rest + access: full + tls: skip + request_body_credential_rewrite: true + credential_binding: {{provider: provider}} + - host: example.test + port: 8081 + protocol: websocket + access: read-write + allow_uninspected_credentials: true + binaries: + - path: /usr/bin/python* +" + ); + let expected = openshell_policy::parse_sandbox_policy(&yaml).expect("the YAML parses"); + assert_eq!(policy, expected); + let round_trip = + openshell_policy::parse_sandbox_policy(&to_yaml(&policy).expect("serializes")) + .expect("the serialized policy parses"); + assert_eq!(round_trip, expected); + } +} diff --git a/tests/testkit/testkit/src/provider.rs b/tests/testkit/testkit/src/provider.rs new file mode 100644 index 0000000000..c87fc6c51c --- /dev/null +++ b/tests/testkit/testkit/src/provider.rs @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::context::Context; +use crate::error::{Result, ResultExt as _}; +use crate::yaml_file::YamlFile; + +/// Import `profile`, a provider profile whose `id` is `name`, and create a +/// provider called `name` of that type with `credentials`, as pairs of +/// environment variable and value. Cleanup deletes the provider, then the +/// profile, after the test's sandboxes. +/// +/// The CLI reads the credential values from its environment, so they aren't +/// in the reported command. +pub async fn install_provider( + cx: &Context, + name: &str, + profile: &str, + credentials: &[(&str, &str)], +) -> Result { + // Both deletes succeed for a missing resource, so register them before + // the creates. Hooks run last registered first. + for kind in ["profile", "provider"] { + let cli = cx.cli_handle(); + let owned = name.to_string(); + cx.on_cleanup(format!("delete {kind} {name}"), async move || { + cli.run([kind, "delete", &owned]).await?.success()?; + Ok(()) + }); + } + + let file = YamlFile::new(profile)?; + cx.cli(["profile", "import", "--file", file.path()?]) + .await? + .success() + .context(format!("import profile {name}"))?; + let mut args = vec!["provider", "create", "--name", name, "--type", name]; + for (key, _) in credentials { + args.extend(["--credential", key]); + } + cx.cli_with_env(credentials, args) + .await? + .success() + .context(format!("create provider {name}"))?; + Ok(()) +} diff --git a/tests/testkit/testkit/src/yaml_file.rs b/tests/testkit/testkit/src/yaml_file.rs new file mode 100644 index 0000000000..04ca3d815d --- /dev/null +++ b/tests/testkit/testkit/src/yaml_file.rs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::error::{Error, Result, ResultExt as _}; + +/// YAML in a temporary file, for CLI commands that read one, such as +/// `sandbox create --policy` or `profile import --file`. The file is removed +/// when dropped. +pub struct YamlFile(tempfile::NamedTempFile); + +impl YamlFile { + pub fn new(contents: &str) -> Result { + let file = tempfile::Builder::new() + .suffix(".yaml") + .tempfile() + .map_err(|error| Error::from_error(&error)) + .context("create a temporary YAML file")?; + std::fs::write(file.path(), contents) + .map_err(|error| Error::from_error(&error)) + .context(format!("write {}", file.path().display()))?; + Ok(Self(file)) + } + + /// The file's path, as a CLI argument. + pub fn path(&self) -> Result<&str> { + let path = self.0.path(); + path.to_str() + .ok_or_else(|| Error::msg(format!("{} is not UTF-8", path.display()))) + } +} From f0d5287120bd280cc60cf40dc24c9555e930f7a8 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 6 Nov 2026 10:56:52 -0800 Subject: [PATCH 06/16] test(support): add host servers that record what sandboxes send Add host-server-support with HttpRecorder and WebSocketRecorder, servers on the test host that sandboxes reach through host.openshell.internal. Move the host-gateway feature test onto them and onto testkit's base policy. Signed-off-by: Matthew Grossman --- tests/suites/features/Cargo.lock | 173 +++++++++++- tests/suites/features/host-gateway/Cargo.toml | 3 +- .../host-gateway/tests/host_gateway.rs | 121 +-------- tests/support/Cargo.lock | 172 ++++++++++++ tests/support/Cargo.toml | 2 +- tests/support/host-server/Cargo.toml | 18 ++ tests/support/host-server/src/lib.rs | 257 ++++++++++++++++++ 7 files changed, 635 insertions(+), 111 deletions(-) create mode 100644 tests/support/host-server/Cargo.toml create mode 100644 tests/support/host-server/src/lib.rs diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index bb1a1427ee..6fbc0b4836 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -493,6 +493,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -560,6 +570,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1137,6 +1153,25 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "rand 0.10.2", + "ring", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "url", +] + [[package]] name = "hkdf" version = "0.13.0" @@ -1168,8 +1203,17 @@ dependencies = [ name = "host-gateway" version = "0.0.0" dependencies = [ + "host-server-support", "openshell-testkit", - "tempfile", +] + +[[package]] +name = "host-server-support" +version = "0.0.0" +dependencies = [ + "base64 0.22.1", + "openshell-testkit", + "sha1 0.10.6", "tokio", ] @@ -1526,6 +1570,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2057,6 +2150,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2145,6 +2242,19 @@ dependencies = [ "tower", ] +[[package]] +name = "openshell-policy" +version = "0.0.0" +dependencies = [ + "hickory-proto", + "miette", + "openshell-core", + "openshell-policy-schema", + "prost-types", + "serde", + "serde_json", +] + [[package]] name = "openshell-policy-schema" version = "0.0.0" @@ -2205,12 +2315,14 @@ dependencies = [ "linkme", "openshell-bootstrap", "openshell-core", + "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", "serde", "serde_json", + "tempfile", "tokio", ] @@ -2385,6 +2497,12 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.5" @@ -2803,6 +2921,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2885,6 +3012,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -3131,6 +3267,22 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -3999,6 +4151,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4146,6 +4308,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" diff --git a/tests/suites/features/host-gateway/Cargo.toml b/tests/suites/features/host-gateway/Cargo.toml index d81b6cafb2..0dfef50bc5 100644 --- a/tests/suites/features/host-gateway/Cargo.toml +++ b/tests/suites/features/host-gateway/Cargo.toml @@ -12,6 +12,5 @@ repository = "https://github.com/NVIDIA/OpenShell" publish = false [dependencies] +host-server-support = { path = "../../../support/host-server" } openshell-testkit = { path = "../../../testkit/testkit" } -tempfile = "3" -tokio = { version = "1.43", features = ["io-util", "net", "rt"] } diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index 024b1cd84b..a5c9c0d435 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -9,10 +9,9 @@ //! Python test image ([`testkit::python_image`]) that requests a page from that //! server through the name. -use testkit::{Context, Error, Result, ResultExt as _}; -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::task::JoinHandle; +use host_server_support::HttpRecorder; +use testkit::policy; +use testkit::{Context, Result, ResultExt as _, YamlFile}; /// The image's system Python, which needs nothing beyond its standard library. /// The policy names its real path, `/usr/bin/python3.*`, because a rule matches @@ -51,8 +50,8 @@ print(response.decode(errors="replace")) #[testkit::test] async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result { let marker = format!("host-gateway-{}", cx.run_id()); - let server = HostServer::start(marker.clone()).await?; - let policy = PolicyFile::new(&policy(server.port))?; + let server = HttpRecorder::start(marker.clone()).await?; + let policy = YamlFile::new(&policy(server.port)?)?; let port = server.port.to_string(); let image = testkit::python_image(); @@ -104,106 +103,14 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result /// connect to `host.openshell.internal` on `port`. The endpoint names only the /// host and port, as the policy docs describe, so the driver's address for the /// name is what makes it reachable, not an `allowed_ips` range. -fn policy(port: u16) -> String { - format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /etc] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - host_server: - name: host_server - endpoints: - - host: host.openshell.internal - port: {port} - binaries: - - path: /usr/bin/python3.* -"# - ) -} - -/// An HTTP server on the host that answers every request with `body`. It -/// stops when dropped. -struct HostServer { - port: u16, - task: JoinHandle<()>, -} - -impl HostServer { - /// Listen on every host address, since drivers map - /// `host.openshell.internal` to different ones. - async fn start(body: String) -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| Error::from_error(&error)) - .context("bind the host server")?; - let port = listener - .local_addr() - .map_err(|error| Error::from_error(&error)) - .context("read the host server's address")? - .port(); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - tokio::spawn(respond(stream, body.clone())); - } - }); - Ok(Self { port, task }) - } -} - -impl Drop for HostServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -/// Read one request's headers from `stream` and answer with `body`. -async fn respond(mut stream: TcpStream, body: String) { - let _ = stream.set_nodelay(true); - let mut request = Vec::new(); - let mut buf = [0_u8; 1024]; - while !request.windows(4).any(|window| window == b"\r\n\r\n") { - match stream.read(&mut buf).await { - Ok(read) if read > 0 && request.len() < 64 * 1024 => { - request.extend_from_slice(&buf[..read]); - } - _ => return, - } - } - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() +fn policy(port: u16) -> Result { + let mut policy = policy::base(); + policy::allow( + &mut policy, + "host_server", + "host.openshell.internal", + port, + &["/usr/bin/python3.*"], ); - let _ = stream.write_all(response.as_bytes()).await; - let _ = stream.shutdown().await; -} - -/// A sandbox policy written to a temporary file for `sandbox create --policy`. -struct PolicyFile(tempfile::NamedTempFile); - -impl PolicyFile { - fn new(contents: &str) -> Result { - let file = tempfile::NamedTempFile::new() - .map_err(|error| Error::from_error(&error)) - .context("create policy file")?; - std::fs::write(file.path(), contents) - .map_err(|error| Error::from_error(&error)) - .context(format!("write {}", file.path().display()))?; - Ok(Self(file)) - } - - fn path(&self) -> Result<&str> { - let path = self.0.path(); - path.to_str() - .ok_or_else(|| Error::msg(format!("{} is not UTF-8", path.display()))) - } + policy::to_yaml(&policy) } diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index f8b591438f..cd15939875 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -493,6 +493,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -560,6 +570,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1137,6 +1153,25 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "rand 0.10.2", + "ring", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "url", +] + [[package]] name = "hkdf" version = "0.13.0" @@ -1164,6 +1199,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "host-server-support" +version = "0.0.0" +dependencies = [ + "base64 0.22.1", + "openshell-testkit", + "sha1 0.10.6", + "tokio", +] + [[package]] name = "hostname" version = "0.4.2" @@ -1517,6 +1562,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2029,6 +2123,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2117,6 +2215,19 @@ dependencies = [ "tower", ] +[[package]] +name = "openshell-policy" +version = "0.0.0" +dependencies = [ + "hickory-proto", + "miette", + "openshell-core", + "openshell-policy-schema", + "prost-types", + "serde", + "serde_json", +] + [[package]] name = "openshell-policy-schema" version = "0.0.0" @@ -2165,12 +2276,14 @@ dependencies = [ "linkme", "openshell-bootstrap", "openshell-core", + "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", "serde", "serde_json", + "tempfile", "tokio", ] @@ -2353,6 +2466,12 @@ dependencies = [ "openshell-testkit", ] +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + [[package]] name = "potential_utf" version = "0.1.5" @@ -2771,6 +2890,15 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2853,6 +2981,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -3099,6 +3236,22 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -3967,6 +4120,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4114,6 +4277,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" diff --git a/tests/support/Cargo.toml b/tests/support/Cargo.toml index f56ee275f5..4c7c495f97 100644 --- a/tests/support/Cargo.toml +++ b/tests/support/Cargo.toml @@ -5,4 +5,4 @@ # environment runs. Suites depend on them by path. [workspace] resolver = "2" -members = ["container-engine", "keycloak", "podman"] +members = ["container-engine", "host-server", "keycloak", "podman"] diff --git a/tests/support/host-server/Cargo.toml b/tests/support/host-server/Cargo.toml new file mode 100644 index 0000000000..aa964e782d --- /dev/null +++ b/tests/support/host-server/Cargo.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "host-server-support" +description = "Servers on the test host that sandboxes reach through host.openshell.internal" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +base64 = "0.22" +openshell-testkit = { path = "../../testkit/testkit" } +sha1 = "0.10" +tokio = { version = "1.43", features = ["io-util", "net", "rt", "time"] } diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs new file mode 100644 index 0000000000..f5d1f6522f --- /dev/null +++ b/tests/support/host-server/src/lib.rs @@ -0,0 +1,257 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Servers on the test host that sandboxes reach through +//! `host.openshell.internal`, and that record what arrived, so a test can +//! check what the supervisor forwarded upstream. +//! +//! The servers listen on every host address, since drivers map +//! `host.openshell.internal` to different ones, and stop when dropped. They +//! run where the tests run, so a gateway whose sandboxes can't reach the test +//! host, such as one on a remote cluster, can't run tests that use them. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use base64::Engine as _; +use sha1::{Digest as _, Sha1}; +use testkit::{Error, Poll, Result, ResultExt as _}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::JoinHandle; + +const WAIT_TIMEOUT: Duration = Duration::from_secs(10); +const WAIT_INTERVAL: Duration = Duration::from_millis(100); +const READ_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_REQUEST: usize = 1024 * 1024; + +/// An HTTP server that answers every request with `200` and a fixed body, +/// and records each request it read. +pub struct HttpRecorder { + pub port: u16, + requests: Arc>>, + task: JoinHandle<()>, +} + +impl HttpRecorder { + pub async fn start(body: impl Into) -> Result { + let (listener, port) = bind().await?; + let body: Arc = body.into().into(); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(respond( + stream, + Arc::clone(&body), + Arc::clone(&task_requests), + )); + } + }); + Ok(Self { + port, + requests, + task, + }) + } + + /// The requests the server has read, headers and body, once there are at + /// least `count`. + pub async fn wait_for_requests(&self, count: usize) -> Result> { + testkit::poll_until( + &format!( + "{count} request(s) at the host server on port {}", + self.port + ), + WAIT_TIMEOUT, + WAIT_INTERVAL, + async || { + let requests = self + .requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + Ok(if requests.len() >= count { + Poll::Ready(requests) + } else { + Poll::Pending(format!("{} request(s)", requests.len())) + }) + }, + ) + .await + } +} + +impl Drop for HttpRecorder { + fn drop(&mut self) { + self.task.abort(); + } +} + +/// A WebSocket server that completes the handshake and records whether a +/// binary frame followed. +pub struct WebSocketRecorder { + pub port: u16, + handshake_seen: Arc, + binary_seen: Arc, + task: JoinHandle<()>, +} + +impl WebSocketRecorder { + pub async fn start() -> Result { + let (listener, port) = bind().await?; + let handshake_seen = Arc::new(AtomicBool::new(false)); + let binary_seen = Arc::new(AtomicBool::new(false)); + let (task_handshake, task_binary) = (Arc::clone(&handshake_seen), Arc::clone(&binary_seen)); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(accept_websocket( + stream, + Arc::clone(&task_handshake), + Arc::clone(&task_binary), + )); + } + }); + Ok(Self { + port, + handshake_seen, + binary_seen, + task, + }) + } + + /// Wait until a client completed the handshake. + pub async fn wait_for_handshake(&self) -> Result { + testkit::poll_until( + &format!( + "a WebSocket handshake at the host server on port {}", + self.port + ), + WAIT_TIMEOUT, + WAIT_INTERVAL, + async || { + Ok(if self.handshake_seen.load(Ordering::Acquire) { + Poll::Ready(()) + } else { + Poll::Pending("no handshake".to_string()) + }) + }, + ) + .await + } + + /// Whether a binary frame arrived after a handshake. + pub fn binary_seen(&self) -> bool { + self.binary_seen.load(Ordering::Acquire) + } +} + +impl Drop for WebSocketRecorder { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn bind() -> Result<(TcpListener, u16)> { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| Error::from_error(&error)) + .context("bind a host server")?; + let port = listener + .local_addr() + .map_err(|error| Error::from_error(&error)) + .context("read the host server's address")? + .port(); + Ok((listener, port)) +} + +/// Read one request, headers and `Content-Length` body, record it, and +/// answer with `body`. +async fn respond(mut stream: TcpStream, body: Arc, requests: Arc>>) { + let _ = stream.set_nodelay(true); + let Some(request) = read_request(&mut stream).await else { + return; + }; + requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(String::from_utf8_lossy(&request).into_owned()); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(response.as_bytes()).await; + let _ = stream.shutdown().await; +} + +async fn read_request(stream: &mut TcpStream) -> Option> { + let mut request = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + if let Some(end) = find(&request, b"\r\n\r\n") { + let length = header(&String::from_utf8_lossy(&request[..end]), "content-length") + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + if request.len() >= end + 4 + length { + return Some(request); + } + } + match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut buffer)).await { + Ok(Ok(read)) if read > 0 && request.len() < MAX_REQUEST => { + request.extend_from_slice(&buffer[..read]); + } + _ => return None, + } + } +} + +async fn accept_websocket( + mut stream: TcpStream, + handshake_seen: Arc, + binary_seen: Arc, +) { + let _ = stream.set_nodelay(true); + let Some(request) = read_request(&mut stream).await else { + return; + }; + let Some(key) = header(&String::from_utf8_lossy(&request), "sec-websocket-key") else { + return; + }; + // The supervisor checks Sec-WebSocket-Accept, so compute it properly. + let mut hasher = Sha1::new(); + hasher.update(key.as_bytes()); + hasher.update(b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11"); + let accept = base64::engine::general_purpose::STANDARD.encode(hasher.finalize()); + let response = format!( + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept}\r\n\r\n" + ); + if stream.write_all(response.as_bytes()).await.is_err() { + return; + } + handshake_seen.store(true, Ordering::Release); + + let mut frame_header = [0_u8; 2]; + if let Ok(Ok(_)) = + tokio::time::timeout(READ_TIMEOUT, stream.read_exact(&mut frame_header)).await + && frame_header[0] & 0x0f == 0x02 + { + binary_seen.store(true, Ordering::Release); + } +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +fn header(head: &str, name: &str) -> Option { + head.lines().find_map(|line| { + let (header, value) = line.split_once(':')?; + header + .trim() + .eq_ignore_ascii_case(name) + .then(|| value.trim().to_string()) + }) +} From 566b39d81b67ede236190af2d8dbef02f87a197d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 6 Nov 2026 10:56:52 -0800 Subject: [PATCH 07/16] test(conformance): build credential gating on shared test support Use testkit's policy helpers and install_provider and host-server-support's recorders, and assert on the requests that reached the host server instead of verdicts the sandbox clients print. Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 16 +- tests/suites/conformance/Cargo.toml | 6 +- tests/suites/conformance/README.md | 2 +- .../conformance/src/credential_gating.rs | 478 +++++++----------- 4 files changed, 200 insertions(+), 302 deletions(-) diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 6e42410284..29a6acdb75 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -516,11 +516,9 @@ dependencies = [ name = "conformance" version = "0.0.0" dependencies = [ - "base64 0.22.1", - "openshell-policy", + "host-server-support", "openshell-testkit", "serde_json", - "sha1 0.10.6", "tempfile", "tokio", ] @@ -1203,6 +1201,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "host-server-support" +version = "0.0.0" +dependencies = [ + "base64 0.22.1", + "openshell-testkit", + "sha1 0.10.6", + "tokio", +] + [[package]] name = "hostname" version = "0.4.2" @@ -2261,12 +2269,14 @@ dependencies = [ "linkme", "openshell-bootstrap", "openshell-core", + "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", "serde", "serde_json", + "tempfile", "tokio", ] diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index bd6bc3f1bb..7201953c9d 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -21,12 +21,10 @@ path = "src/main.rs" test = false [dependencies] -base64 = "0.22" -openshell-policy = { path = "../../../crates/openshell-policy" } +host-server-support = { path = "../../support/host-server" } openshell-testkit = { path = "../../testkit/testkit" } serde_json = "1" -sha1 = "0.10" tempfile = "3" -tokio = { version = "1", features = ["io-util", "net", "rt", "time"] } +tokio = { version = "1", features = ["time"] } [workspace] diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index a8570604bb..33cf17058e 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -23,7 +23,7 @@ tmachine environment on CI. The other suites are ports of the old tests, so a test environment needs it installed (`tests/tmachine/ansible/playbooks/test-tools.yaml` does that for the tmachine environments). `credential-gating`, ported from NVIDIA/OpenShell#5264, starts -servers on the machine that runs the tests and reaches them from sandboxes +servers (`tests/support/host-server`) on the machine that runs the tests and reaches them from sandboxes created from `testkit::python_image()` through `host.openshell.internal`, so it fails against a gateway whose sandboxes can't reach that machine, such as a remote Kubernetes cluster. diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 170ef19c65..616eed1374 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -2,57 +2,84 @@ // SPDX-License-Identifier: Apache-2.0 //! Credential gating: which endpoints a provider credential may reach, and -//! what the supervisor does with the credential in REST bodies and WebSocket -//! frames. +//! what the supervisor forwards upstream in REST bodies and WebSocket frames. //! -//! Ported from the e2e tests added in NVIDIA/OpenShell#5264. -//! Each test imports its own provider profile and creates a provider under a -//! name unique to the run; cleanup deletes them after the test's sandboxes. -//! The probe servers run where the tests run, which is the host the gateway -//! runs its sandboxes on, and the sandbox clients reach them through -//! `host.openshell.internal`, so a gateway on a remote cluster can't run this -//! suite. - -mod client; -mod probe; +//! Ported from the e2e tests added in NVIDIA/OpenShell#5264. The upstream +//! servers run on the test host ([`host_server_support`]), so a gateway on a +//! remote cluster can't run this suite. use std::time::Duration; +use host_server_support::{HttpRecorder, WebSocketRecorder}; +use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; -use testkit::{Context, Error, Poll, Result, ResultExt as _, Sandbox}; - -use client::{KEY_PRESENCE_SCRIPT, binary_websocket_client_script, body_client_script}; -use probe::{BinaryWebSocketProbeServer, HttpProbeServer}; +use testkit::{Context, Error, PYTHON_BINARIES, Poll, Result, ResultExt as _, Sandbox, YamlFile}; const TEST_HOST: &str = "host.openshell.internal"; const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; const TEST_SECRET: &str = "e2e-gating-secret-value"; const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; -/// A provider whose profile declares the REST and WebSocket probes as its +/// POSTs a body containing the provider's token to `TEST_HOST` on the port +/// in `argv[1]`, directly rather than through a proxy. +const BODY_CLIENT: &str = r#" +import os, sys, urllib.request +body = ("prefix-" + os.environ["E2E_GATING_TOKEN"] + "-suffix").encode() +request = urllib.request.Request(f"http://host.openshell.internal:{sys.argv[1]}/token", data=body, headers={"Content-Type": "text/plain"}) +urllib.request.build_opener(urllib.request.ProxyHandler({})).open(request, timeout=10).read() +"#; + +/// Upgrades to WebSocket at `TEST_HOST` on the port in `argv[1]`, sends one +/// binary frame, and waits briefly for the connection to be closed. +const WEBSOCKET_CLIENT: &str = r#" +import base64, os, socket, sys +host, port = "host.openshell.internal", int(sys.argv[1]) +with socket.create_connection((host, port), timeout=10) as sock: + key = base64.b64encode(os.urandom(16)).decode() + sock.sendall(f"GET /ws HTTP/1.1\r\nHost: {host}:{port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode()) + response = b"" + while b"\r\n\r\n" not in response: + chunk = sock.recv(4096) + if not chunk: + sys.exit(f"closed during the upgrade: {response!r}") + response += chunk + if not response.startswith(b"HTTP/1.1 101"): + sys.exit(f"upgrade failed: {response!r}") + payload, mask = b"binary-credential-channel", os.urandom(4) + sock.sendall(bytes([0x82, 0x80 | len(payload)]) + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))) + sock.settimeout(3) + try: + sock.recv(4096) + except (TimeoutError, ConnectionError): + pass +"#; + +/// Prints `TOKEN_ABSENT`, `TOKEN_PLACEHOLDER`, or `TOKEN_UNSAFE` for the +/// provider's environment variable, without printing its value. +const TOKEN_PRESENCE: &str = r#"if [ "${E2E_GATING_TOKEN+x}" != x ]; then + echo TOKEN_ABSENT +else + case "$E2E_GATING_TOKEN" in + openshell:resolve:env:*) echo TOKEN_PLACEHOLDER ;; + *) echo TOKEN_UNSAFE ;; + esac +fi"#; + +/// A provider whose profile declares the REST and WebSocket servers as its /// endpoints keeps the credential placeholder in a REST body. #[testkit::test] async fn profile_rest_endpoint_keeps_body_placeholder(cx: &mut Context) -> testkit::Result { - let server = HttpProbeServer::start().await?; - let websocket = BinaryWebSocketProbeServer::start().await?; - let provider = install_provider(cx, &profile(Some((server.port, websocket.port)))).await?; - - let policy = policy( - server.port, - &[ - "protocol: rest", - "access: full", - "request_body_credential_rewrite: false", - ], - ); - let sandbox = create_sandbox(cx, "rest", &policy, &provider).await?; - let output = run( - &sandbox, - &["python3", "-c", &body_client_script(server.port)], - ) - .await?; - testkit::ensure!(output.contains("BODY_TEXT"), "{output}"); - expect_one_body(&server, true, false).await + let server = HttpRecorder::start("ok").await?; + let websocket = WebSocketRecorder::start().await?; + let provider = install_provider(cx, Some((server.port, websocket.port))).await?; + + let mut policy = policy::base(); + allow(&mut policy, server.port) + .rest(Access::Full) + .request_body_credential_rewrite(false); + let sandbox = create_sandbox(cx, "rest", policy, &provider).await?; + run_python(&sandbox, BODY_CLIENT, server.port).await?; + expect_one_body(&server, PLACEHOLDER_PREFIX, TEST_SECRET).await } /// A provider's WebSocket endpoint completes the handshake but doesn't @@ -60,18 +87,14 @@ async fn profile_rest_endpoint_keeps_body_placeholder(cx: &mut Context) -> testk /// credential. #[testkit::test] async fn profile_websocket_endpoint_denies_binary_frames(cx: &mut Context) -> testkit::Result { - let server = HttpProbeServer::start().await?; - let websocket = BinaryWebSocketProbeServer::start().await?; - let provider = install_provider(cx, &profile(Some((server.port, websocket.port)))).await?; - - let policy = policy( - websocket.port, - &["protocol: websocket", "access: read-write"], - ); - let sandbox = create_sandbox(cx, "ws", &policy, &provider).await?; - let script = binary_websocket_client_script(websocket.port); - let output = run(&sandbox, &["python3", "-c", &script]).await?; - testkit::ensure!(output.contains("BINARY_DENIED"), "{output}"); + let server = HttpRecorder::start("ok").await?; + let websocket = WebSocketRecorder::start().await?; + let provider = install_provider(cx, Some((server.port, websocket.port))).await?; + + let mut policy = policy::base(); + allow(&mut policy, websocket.port).websocket(Access::ReadWrite); + let sandbox = create_sandbox(cx, "ws", policy, &provider).await?; + run_python(&sandbox, WEBSOCKET_CLIENT, websocket.port).await?; websocket .wait_for_handshake() .await @@ -88,30 +111,30 @@ async fn profile_websocket_endpoint_denies_binary_frames(cx: &mut Context) -> te /// that opts in to uninspected credentials. #[testkit::test] async fn policy_binding_requires_inspection_or_opt_in(cx: &mut Context) -> testkit::Result { - let server = HttpProbeServer::start().await?; - let provider = install_provider(cx, &profile(None)).await?; - let binding = credential_binding(&provider); + let server = HttpRecorder::start("ok").await?; + let provider = install_provider(cx, None).await?; - let l4 = policy(server.port, &[&binding]); - let error = expect_rejected(cx, "l4", &l4, &provider).await?; + let mut l4 = policy::base(); + allow(&mut l4, server.port).credential_binding(&provider); + let error = expect_rejected(cx, "l4", l4, &provider).await?; testkit::ensure!(error.contains("credentialed endpoint"), "{error}"); testkit::ensure!(error.contains("L4-only"), "{error}"); - let tls_skip = policy( - server.port, - &["protocol: rest", "access: full", "tls: skip", &binding], - ); - let error = expect_rejected(cx, "tls", &tls_skip, &provider).await?; + let mut tls_skip = policy::base(); + allow(&mut tls_skip, server.port) + .rest(Access::Full) + .tls_skip() + .credential_binding(&provider); + let error = expect_rejected(cx, "tls", tls_skip, &provider).await?; testkit::ensure!(error.contains("credentialed endpoint"), "{error}"); testkit::ensure!(error.contains("tls:") && error.contains("skip"), "{error}"); - let opt_in = policy( - server.port, - &["allow_uninspected_credentials: true", &binding], - ); - let sandbox = create_sandbox(cx, "optin", &opt_in, &provider).await?; - let output = run(&sandbox, &["echo", "ENDPOINTLESS_OPT_IN_ACCEPTED"]).await?; - testkit::ensure!(output.contains("ENDPOINTLESS_OPT_IN_ACCEPTED"), "{output}"); + let mut opt_in = policy::base(); + allow(&mut opt_in, server.port) + .allow_uninspected_credentials() + .credential_binding(&provider); + let sandbox = create_sandbox(cx, "optin", opt_in, &provider).await?; + sandbox.exec(["true"]).await?.success()?; Ok(()) } @@ -121,93 +144,58 @@ async fn policy_binding_requires_inspection_or_opt_in(cx: &mut Context) -> testk async fn policy_binding_rest_body_keeps_placeholder_without_rewrite( cx: &mut Context, ) -> testkit::Result { - let server = HttpProbeServer::start().await?; - let provider = install_provider(cx, &profile(None)).await?; - - let policy = policy( - server.port, - &[ - "protocol: rest", - "access: full", - "request_body_credential_rewrite: false", - &credential_binding(&provider), - ], - ); - let sandbox = create_sandbox(cx, "body", &policy, &provider).await?; - let output = run( - &sandbox, - &["python3", "-c", &body_client_script(server.port)], - ) - .await?; - testkit::ensure!(output.contains("BODY_TEXT"), "{output}"); - expect_one_body(&server, true, false).await + let server = HttpRecorder::start("ok").await?; + let provider = install_provider(cx, None).await?; + + let mut policy = policy::base(); + allow(&mut policy, server.port) + .rest(Access::Full) + .request_body_credential_rewrite(false) + .credential_binding(&provider); + let sandbox = create_sandbox(cx, "body", policy, &provider).await?; + run_python(&sandbox, BODY_CLIENT, server.port).await?; + expect_one_body(&server, PLACEHOLDER_PREFIX, TEST_SECRET).await } /// A policy binding with `request_body_credential_rewrite: true` replaces -/// the placeholder in the REST body with the secret upstream, and the -/// secret doesn't reach the sandbox. +/// the placeholder in the REST body with the secret upstream. #[testkit::test] async fn policy_binding_rest_body_rewrite_sends_secret_upstream( cx: &mut Context, ) -> testkit::Result { - let server = HttpProbeServer::start().await?; - let provider = install_provider(cx, &profile(None)).await?; - - let policy = policy( - server.port, - &[ - "protocol: rest", - "access: full", - "request_body_credential_rewrite: true", - &credential_binding(&provider), - ], - ); - let sandbox = create_sandbox(cx, "rw", &policy, &provider).await?; - let output = run( - &sandbox, - &["python3", "-c", &body_client_script(server.port)], - ) - .await?; - testkit::ensure!( - output.contains("BODY_REWRITTEN"), - "the client did not report BODY_REWRITTEN" - ); - testkit::ensure!( - !output.contains(TEST_SECRET), - "the secret reached the sandbox" - ); - testkit::ensure!( - !output.contains(PLACEHOLDER_PREFIX), - "the client printed the placeholder" - ); - expect_one_body(&server, false, true).await + let server = HttpRecorder::start("ok").await?; + let provider = install_provider(cx, None).await?; + + let mut policy = policy::base(); + allow(&mut policy, server.port) + .rest(Access::Full) + .request_body_credential_rewrite(true) + .credential_binding(&provider); + let sandbox = create_sandbox(cx, "rw", policy, &provider).await?; + run_python(&sandbox, BODY_CLIENT, server.port).await?; + expect_one_body(&server, TEST_SECRET, PLACEHOLDER_PREFIX).await } /// Binding and unbinding a provider with `policy set --wait` adds and /// removes its environment variable in a running sandbox. #[testkit::test] async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> testkit::Result { - let server = HttpProbeServer::start().await?; - let provider = install_provider(cx, &profile(None)).await?; - let unbound = policy( - server.port, - &[ - "protocol: rest", - "access: full", - "request_body_credential_rewrite: false", - ], - ); - let bound = policy( - server.port, - &[ - "protocol: rest", - "access: full", - "request_body_credential_rewrite: false", - &credential_binding(&provider), - ], - ); - - let sandbox = create_sandbox(cx, "live", &unbound, &provider).await?; + let server = HttpRecorder::start("ok").await?; + let provider = install_provider(cx, None).await?; + + let policy_binding = |bind: bool| { + let mut policy = policy::base(); + let endpoint = allow(&mut policy, server.port) + .rest(Access::Full) + .request_body_credential_rewrite(false); + if bind { + endpoint.credential_binding(&provider); + } + policy + }; + let (unbound, bound) = (policy_binding(false), policy_binding(true)); + + let sandbox = create_sandbox(cx, "live", unbound.clone(), &provider).await?; wait_for_token(&sandbox, "TOKEN_ABSENT").await?; set_policy(cx, &sandbox, &bound) @@ -221,10 +209,22 @@ async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> t wait_for_token(&sandbox, "TOKEN_ABSENT").await } -/// A provider profile with one bearer credential in `TOKEN_ENV`. With -/// `endpoints`, it declares the REST and WebSocket probe ports as its +/// Let the image's Python reach `TEST_HOST` on `port`. +fn allow(policy: &mut proto::SandboxPolicy, port: u16) -> &mut proto::NetworkEndpoint { + policy::allow( + policy, + "credential_gating", + TEST_HOST, + port, + PYTHON_BINARIES, + ) +} + +/// Install a run-unique provider holding `TEST_SECRET` in `TOKEN_ENV`. With +/// `endpoints`, its profile declares the REST and WebSocket ports as its /// endpoints; without, a policy must bind the provider explicitly. -fn profile(endpoints: Option<(u16, u16)>) -> String { +async fn install_provider(cx: &Context, endpoints: Option<(u16, u16)>) -> Result { + let name = cx.name("cg"); let endpoints = endpoints.map_or_else(String::new, |(rest, websocket)| { format!( "endpoints: @@ -239,8 +239,12 @@ fn profile(endpoints: Option<(u16, u16)>) -> String { " ) }); - format!( - "id: PROFILE_ID + let binaries: String = PYTHON_BINARIES + .iter() + .map(|path| format!(" - path: {path}\n")) + .collect(); + let profile = format!( + "id: {name} display_name: E2E Credential Gating category: other credentials: @@ -250,112 +254,25 @@ credentials: auth_style: bearer header_name: authorization {endpoints}binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -" - ) -} - -/// A sandbox policy that lets the image's Python reach `TEST_HOST` on -/// `port`, with `endpoint` as extra fields of that endpoint. -fn policy(port: u16, endpoint: &[&str]) -> String { - let endpoint: String = endpoint - .iter() - .map(|field| format!(" {field}\n")) - .collect(); - format!( - "version: 1 -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox -network_policies: - credential_gating: - name: credential_gating - endpoints: - - host: {TEST_HOST} - port: {port} -{endpoint} binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -" - ) -} - -/// The endpoint field that binds `provider`'s credential in a policy. -fn credential_binding(provider: &str) -> String { - format!("credential_binding: {{provider: {provider}}}") -} - -/// Import `profile` under a run-unique ID and create a provider of that type -/// with the same name, holding `TEST_SECRET`. Returns the name. Cleanup -/// deletes the provider, then the profile, after the test's sandboxes. -async fn install_provider(cx: &Context, profile: &str) -> Result { - let name = cx.name("cg"); - // Both deletes succeed for a missing resource, so register them before - // the creates. Hooks run last registered first. - for kind in ["profile", "provider"] { - let cli = cx.cli_handle(); - let owned = name.clone(); - cx.on_cleanup(format!("delete {kind} {name}"), async move || { - cli.run([kind, "delete", &owned]).await?.success()?; - Ok(()) - }); - } - - let file = YamlFile::new(&profile.replace("PROFILE_ID", &name))?; - cx.cli(["profile", "import", "--file", file.path()?]) - .await? - .success() - .context(format!("import profile {name}"))?; - // `--credential KEY` reads the value from the CLI's environment, so the - // secret isn't in the reported command. - cx.cli_with_env( - &[(TOKEN_ENV, TEST_SECRET)], - [ - "provider", - "create", - "--name", - &name, - "--type", - &name, - "--credential", - TOKEN_ENV, - ], - ) - .await? - .success() - .context(format!("create provider {name}"))?; +{binaries}" + ); + testkit::install_provider(cx, &name, &profile, &[(TOKEN_ENV, TEST_SECRET)]).await?; Ok(name) } -fn parse_policy(yaml: &str) -> Result { - openshell_policy::parse_sandbox_policy(yaml) - .map_err(|error| Error::msg(format!("{error:?}"))) - .context("parse the test policy") -} - fn sandbox_with( cx: &Context, suffix: &str, - policy: &str, + policy: proto::SandboxPolicy, provider: &str, -) -> Result { - Ok(cx - .sandbox(suffix) - .policy(parse_policy(policy)?) +) -> testkit::SandboxBuilder { + cx.sandbox(suffix) + .policy(policy) .spec(|spec| { spec.image = Some(testkit::python_image()); spec.providers = vec![provider.to_string()]; }) - .command(["sleep", "infinity"])) + .command(["sleep", "infinity"]) } /// Create a long-running sandbox from the Python image with `policy` and @@ -363,10 +280,10 @@ fn sandbox_with( async fn create_sandbox( cx: &Context, suffix: &str, - policy: &str, + policy: proto::SandboxPolicy, provider: &str, ) -> Result { - let sandbox = sandbox_with(cx, suffix, policy, provider)?.create().await?; + let sandbox = sandbox_with(cx, suffix, policy, provider).create().await?; sandbox.wait_ready().await?; Ok(sandbox) } @@ -375,10 +292,10 @@ async fn create_sandbox( async fn expect_rejected( cx: &Context, suffix: &str, - policy: &str, + policy: proto::SandboxPolicy, provider: &str, ) -> Result { - match sandbox_with(cx, suffix, policy, provider)?.create().await { + match sandbox_with(cx, suffix, policy, provider).create().await { Ok(sandbox) => Err(Error::msg(format!( "the gateway accepted sandbox {}", sandbox.name() @@ -387,44 +304,42 @@ async fn expect_rejected( } } -/// Run `command` in `sandbox`; it must exit 0. Returns stdout and stderr. -async fn run(sandbox: &Sandbox, command: &[&str]) -> Result { - let output = sandbox.exec(command.iter().copied()).await?; - testkit::ensure!( - output.exit_code == 0, - "{command:?} exited with {}; stdout: {}; stderr: {}", - output.exit_code, - output.stdout.trim(), - output.stderr.trim() - ); - Ok(format!("{}{}", output.stdout, output.stderr)) +/// Run `script` with the image's Python and `port` as its argument. +async fn run_python(sandbox: &Sandbox, script: &str, port: u16) -> Result { + let port = port.to_string(); + sandbox + .exec(["python3", "-c", script, &port]) + .await? + .success() + .context("run the client in the sandbox")?; + Ok(()) } -/// Fail unless the HTTP probe saw exactly one request, with the placeholder -/// and the secret as expected. -async fn expect_one_body(server: &HttpProbeServer, placeholder: bool, secret: bool) -> Result { - let observations = server.wait_for_observations(1).await?; +/// Fail unless `server` saw exactly one request, containing `present` and +/// not `absent`. +async fn expect_one_body(server: &HttpRecorder, present: &str, absent: &str) -> Result { + let requests = server.wait_for_requests(1).await?; testkit::ensure!( - observations.len() == 1, - "expected one request at the HTTP probe; observations: {observations:?}" + requests.len() == 1, + "expected one request upstream; got {}", + requests.len() ); + let request = &requests[0]; testkit::ensure!( - observations[0].saw_placeholder == placeholder && observations[0].saw_secret == secret, - "expected placeholder={placeholder} secret={secret} upstream; saw {:?}", - observations[0] + request.contains(present) && !request.contains(absent), + "expected {present:?} and not {absent:?} upstream" ); Ok(()) } -/// Wait until new exec sessions report `expected` from -/// [`KEY_PRESENCE_SCRIPT`]. +/// Wait until new exec sessions report `expected` from [`TOKEN_PRESENCE`]. async fn wait_for_token(sandbox: &Sandbox, expected: &str) -> Result { testkit::poll_until( &format!("the provider environment to report {expected}"), Duration::from_secs(30), Duration::from_millis(500), async || { - let output = sandbox.exec(["sh", "-lc", KEY_PRESENCE_SCRIPT]).await?; + let output = sandbox.exec(["sh", "-lc", TOKEN_PRESENCE]).await?; Ok(if output.stdout.contains(expected) { Poll::Ready(()) } else { @@ -441,8 +356,8 @@ async fn wait_for_token(sandbox: &Sandbox, expected: &str) -> Result { } /// Replace `sandbox`'s policy with `policy` and wait until it is applied. -async fn set_policy(cx: &Context, sandbox: &Sandbox, policy: &str) -> Result { - let file = YamlFile::new(policy)?; +async fn set_policy(cx: &Context, sandbox: &Sandbox, policy: &proto::SandboxPolicy) -> Result { + let file = YamlFile::new(&policy::to_yaml(policy)?)?; cx.cli([ "policy", "set", @@ -458,28 +373,3 @@ async fn set_policy(cx: &Context, sandbox: &Sandbox, policy: &str) -> Result { .context("policy set")?; Ok(()) } - -/// YAML in a temporary file, for CLI commands that read one. The file is -/// removed when dropped. -struct YamlFile(tempfile::NamedTempFile); - -impl YamlFile { - fn new(contents: &str) -> Result { - let file = tempfile::Builder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| Error::from_error(&error)) - .context("create a temporary YAML file")?; - std::fs::write(file.path(), contents) - .map_err(|error| Error::from_error(&error)) - .context(format!("write {}", file.path().display()))?; - Ok(Self(file)) - } - - fn path(&self) -> Result<&str> { - self.0 - .path() - .to_str() - .ok_or_else(|| Error::msg(format!("{} is not UTF-8", self.0.path().display()))) - } -} From 99bbed46ce04e8cdcb3fba08c40e9f3f087d826a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 6 Nov 2026 13:11:28 -0800 Subject: [PATCH 08/16] fix(e2e): pass the host gateway IP to the external Kubernetes driver The external-driver lane renders server.hostGatewayIP, but only the in-process driver reads it from the gateway TOML. The standalone driver takes OPENSHELL_HOST_GATEWAY_IP, which the post-renderer never set, so sandbox policy DNS refused host.openshell.internal on that lane. Copy the chart's rendered value into the driver container when present. Signed-off-by: Matthew Grossman --- .../host-gateway/driver-env-patch.yaml | 15 ++++++++++ .../host-gateway/kustomization.yaml | 30 +++++++++++++++++++ .../post-renderer.sh | 7 +++++ 3 files changed, 52 insertions(+) create mode 100644 e2e/helm-plugins/openshell-external-compute-driver/host-gateway/driver-env-patch.yaml create mode 100644 e2e/helm-plugins/openshell-external-compute-driver/host-gateway/kustomization.yaml diff --git a/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/driver-env-patch.yaml b/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/driver-env-patch.yaml new file mode 100644 index 0000000000..3c829a6cb8 --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/driver-env-patch.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openshell +spec: + template: + spec: + containers: + - name: kubernetes-compute-driver + env: + - name: OPENSHELL_HOST_GATEWAY_IP + value: replaced-by-kustomize diff --git a/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/kustomization.yaml b/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/kustomization.yaml new file mode 100644 index 0000000000..2afb0e5262 --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/host-gateway/kustomization.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The in-process driver reads server.hostGatewayIP from the rendered gateway +# TOML. The external driver reads it from its own environment, so copy the +# chart's rendered value into that container. +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +patches: + - path: driver-env-patch.yaml + target: + group: apps + version: v1 + kind: StatefulSet + name: openshell +replacements: + - source: + group: apps + version: v1 + kind: StatefulSet + name: openshell + fieldPath: spec.template.spec.hostAliases.0.ip + targets: + - select: + group: apps + version: v1 + kind: StatefulSet + name: openshell + fieldPaths: + - spec.template.spec.containers.[name=kubernetes-compute-driver].env.[name=OPENSHELL_HOST_GATEWAY_IP].value diff --git a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh index 43ee52140a..c05b6b553f 100755 --- a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh +++ b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh @@ -16,4 +16,11 @@ cp "${plugin_dir}/kustomization.yaml" "${work_dir}/kustomization.yaml" cp "${plugin_dir}/workload-patch.yaml" "${work_dir}/workload-patch.yaml" tee "${work_dir}/rendered.yaml" >/dev/null +# server.hostGatewayIP renders the gateway pod's hostAliases entry. +if grep -q 'host\.openshell\.internal' "${work_dir}/rendered.yaml" \ + && grep -q '^ *hostAliases:' "${work_dir}/rendered.yaml"; then + cp -R "${plugin_dir}/host-gateway" "${work_dir}/host-gateway" + printf 'components:\n - host-gateway\n' >>"${work_dir}/kustomization.yaml" +fi + kubectl kustomize "${work_dir}" From 61504a338c66626d0a72af7eb414727f614db653 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 6 Nov 2026 15:09:56 -0800 Subject: [PATCH 09/16] test(testkit): default python_image() to a public pinned image Conformance has to run against any gateway, and the unpublished openshell/e2e-python:dev image only exists where a lane built or loaded it. Default to the pinned uv Python image, which any gateway can pull, and keep OPENSHELL_TEST_PYTHON_IMAGE as the override. Host lanes no longer build the workload image for conformance. The base test policy names no workload user, so drivers use a default non-root user on images without one, and host_gateway runs python3 from PATH. Signed-off-by: Matthew Grossman --- TESTING.md | 4 +--- e2e/rust/e2e-docker.sh | 6 +++--- e2e/rust/e2e-kubernetes.sh | 1 - e2e/rust/e2e-podman.sh | 6 +++--- e2e/rust/e2e-vm.sh | 3 +-- e2e/support/conformance.sh | 6 ------ tasks/test.toml | 1 - .../features/host-gateway/tests/host_gateway.rs | 15 ++++++++------- tests/testkit/README.md | 8 ++++---- tests/testkit/testkit/src/base-policy.yaml | 10 ++++------ tests/testkit/testkit/src/image.rs | 9 ++++----- 11 files changed, 28 insertions(+), 41 deletions(-) diff --git a/TESTING.md b/TESTING.md index d150c6afcc..902d8c19ad 100644 --- a/TESTING.md +++ b/TESTING.md @@ -429,9 +429,7 @@ Suites: `openshell-conformance` against the lane's gateway and writes a JUnit report under `results/`; set `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program, as Branch E2E does with the static build it makes once per architecture. - The Docker and Podman runners build `openshell/e2e-python:dev` for it; the - Kubernetes and VM runners set `OPENSHELL_TEST_PYTHON_IMAGE` to a pullable - Python image unless it is already set. The + The installed-artifact conformance lanes run the same crate through `openshell-test-runner` (see above). - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or diff --git a/e2e/rust/e2e-docker.sh b/e2e/rust/e2e-docker.sh index 68d48e7370..8db220ca1b 100755 --- a/e2e/rust/e2e-docker.sh +++ b/e2e/rust/e2e-docker.sh @@ -54,9 +54,9 @@ if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then exit 0 fi -# The conformance suite runs on every lane, including e2e-conformance-only, -# and creates sandboxes from this image (testkit::python_image()). -CONTAINER_ENGINE=docker bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" +if [ "${E2E_FEATURES}" != "e2e-conformance-only" ]; then + CONTAINER_ENGINE=docker bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" +fi exec "${ROOT}/e2e/with-docker-gateway.sh" \ bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index 7ef5576eab..fc90a827ee 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -80,7 +80,6 @@ run_conformance() { fi OPENSHELL_CONFORMANCE_JUNIT="${OPENSHELL_CONFORMANCE_JUNIT:-${ROOT}/results/${OPENSHELL_E2E_REPORT_NAME:-e2e-kubernetes}-conformance.xml}" \ - OPENSHELL_TEST_PYTHON_IMAGE="${OPENSHELL_TEST_PYTHON_IMAGE:-${E2E_PULLABLE_PYTHON_IMAGE}}" \ e2e_run_openshell_conformance "Kubernetes" } diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index e718f0afea..495863fede 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -116,9 +116,9 @@ if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ] && [ -z "${OPENSHELL_BIN:-}" ]; then export OPENSHELL_BIN="${ROOT}/target/debug/openshell" fi -# The conformance suite runs on every lane, including e2e-conformance-only, -# and creates sandboxes from this image (testkit::python_image()). -CONTAINER_ENGINE=podman bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" +if [ "${E2E_FEATURES}" != "e2e-conformance-only" ]; then + CONTAINER_ENGINE=podman bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" +fi exec "${ROOT}/e2e/with-podman-gateway.sh" \ bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index bafaadbc71..0cb2c46858 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -463,8 +463,7 @@ e2e_export_gateway_restart_metadata \ # preparation; allow 180s for slower CI runners. export OPENSHELL_PROVISION_TIMEOUT="${SANDBOX_PROVISION_TIMEOUT}" -OPENSHELL_TEST_PYTHON_IMAGE="${OPENSHELL_TEST_PYTHON_IMAGE:-${E2E_PULLABLE_PYTHON_IMAGE}}" \ - e2e_run_openshell_conformance "VM" +e2e_run_openshell_conformance "VM" if [ "${E2E_FEATURES}" = "e2e-conformance-only" ]; then echo "::notice::Skipping Rust e2e tests (e2e-conformance-only)" exit 0 diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index eef41ecb29..56ce5a50f1 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -23,12 +23,6 @@ e2e_require_openshell_bin() { export OPENSHELL_BIN="${binary_dir}/$(basename "${OPENSHELL_BIN}")" } -# A pullable image for testkit::python_image() on lanes that can't load the -# locally built openshell/e2e-python:dev into their compute platform. It -# matches the Rust harness's E2E_WORKLOAD_IMAGE for those lanes. -# shellcheck disable=SC2034 # Used by the scripts that source this file. -E2E_PULLABLE_PYTHON_IMAGE="ghcr.io/astral-sh/uv:0.12.17-python3.12-trixie-slim@sha256:9a59bb7206905ccaae4f7dab222fbac47c125a21e5fc16f43f427cd6c940ade3" - # Print the path of the openshell-conformance program: OPENSHELL_CONFORMANCE_BIN # if set, else a debug build from this checkout. e2e_openshell_conformance_bin() { diff --git a/tasks/test.toml b/tasks/test.toml index 30c31ec753..2839c1cf0c 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -177,7 +177,6 @@ run = "OPENSHELL_TEST_CONTAINER_ENGINE=podman e2e/with-podman-gateway.sh cargo t ["e2e:testkit"] description = "Run the testkit suites in tests/suites/conformance against a Docker-backed gateway" -depends = ["e2e:workload:build"] run = "e2e/with-docker-gateway.sh cargo nextest run --locked --manifest-path tests/suites/conformance/Cargo.toml --workspace --test-threads 1 --no-fail-fast" ["e2e:rust:push"] diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index a5c9c0d435..33269aa095 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -11,12 +11,13 @@ use host_server_support::HttpRecorder; use testkit::policy; -use testkit::{Context, Result, ResultExt as _, YamlFile}; +use testkit::{Context, PYTHON_BINARIES, Result, ResultExt as _, YamlFile}; -/// The image's system Python, which needs nothing beyond its standard library. -/// The policy names its real path, `/usr/bin/python3.*`, because a rule matches -/// the executable behind a symlink. -const PYTHON: &str = "/usr/bin/python3"; +/// The image's Python, which needs nothing beyond its standard library. The +/// policy allows [`PYTHON_BINARIES`], the real paths behind `python3` in +/// `testkit::python_image()` images, because a rule matches the executable +/// behind a symlink. +const PYTHON: &str = "python3"; /// Requests `/` from `host.openshell.internal` on the port in `argv[1]` and /// prints the response. A failed connection is retried for 20 seconds, so a @@ -99,7 +100,7 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result Ok(()) } -/// A sandbox policy that allows the image's system Python ([`PYTHON`]) to +/// A sandbox policy that allows the image's Python ([`PYTHON`]) to /// connect to `host.openshell.internal` on `port`. The endpoint names only the /// host and port, as the policy docs describe, so the driver's address for the /// name is what makes it reachable, not an `allowed_ips` range. @@ -110,7 +111,7 @@ fn policy(port: u16) -> Result { "host_server", "host.openshell.internal", port, - &["/usr/bin/python3.*"], + PYTHON_BINARIES, ); policy::to_yaml(&policy) } diff --git a/tests/testkit/README.md b/tests/testkit/README.md index ae8b25f95c..b89a726cdf 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -76,10 +76,10 @@ fails. A killed test process skips cleanup, so names are unique to the run. compute driver. Driver suites call it first. - `testkit::base_image()` for the image to create sandboxes from or build on: `OPENSHELL_TEST_BASE_IMAGE`, or OpenShell's default sandbox image. -- `testkit::python_image()` for sandboxes that run `python3`, curl, git, or - openssl: `OPENSHELL_TEST_PYTHON_IMAGE`, or `openshell/e2e-python:dev` - (`e2e/python/Dockerfile.workload`). The tmachine installers load it into - every environment; `mise run e2e:workload:build` builds it locally. +- `testkit::python_image()` for sandboxes that run `python3`: + `OPENSHELL_TEST_PYTHON_IMAGE`, or a pinned public uv Python image that any + gateway can pull. It has the standard library only, not curl, git, or + openssl. - `cx.cli([...])` to run the installed CLI against the same gateway, with `success()`, `failure()`, and `json()` on the output. `cx.cli_handle()` returns that CLI for code that can't borrow `cx`, such as a cleanup hook. diff --git a/tests/testkit/testkit/src/base-policy.yaml b/tests/testkit/testkit/src/base-policy.yaml index 0935dbaa9a..19f77d006f 100644 --- a/tests/testkit/testkit/src/base-policy.yaml +++ b/tests/testkit/testkit/src/base-policy.yaml @@ -2,9 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 # The sandbox policy tests start from (testkit::policy::base()): the image's -# system directories read-only, the sandbox's home and /tmp writable, the -# sandbox user, and no network access. Tests add the network access they need -# with testkit::policy::allow(). +# system directories read-only, the sandbox's home and /tmp writable, and no +# network access. Tests add the network access they need with +# testkit::policy::allow(). It names no workload user, so the driver runs the +# image's non-root user, or a default non-root user when the image has none. version: 1 filesystem_policy: include_workdir: true @@ -12,6 +13,3 @@ filesystem_policy: read_write: [/sandbox, /tmp, /dev/null] landlock: compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox diff --git a/tests/testkit/testkit/src/image.rs b/tests/testkit/testkit/src/image.rs index 08e13825a0..67f6a78ae0 100644 --- a/tests/testkit/testkit/src/image.rs +++ b/tests/testkit/testkit/src/image.rs @@ -11,11 +11,10 @@ const DEFAULT_BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; /// Overrides [`DEFAULT_PYTHON_IMAGE`]. const PYTHON_IMAGE_ENV: &str = "OPENSHELL_TEST_PYTHON_IMAGE"; -/// The workload image for tests that run `python3`, curl, git or openssl in a -/// sandbox, built from `e2e/python/Dockerfile.workload`. It isn't published: -/// the tmachine installers load it into every environment, and -/// `mise run e2e:workload:build` builds it locally. -const DEFAULT_PYTHON_IMAGE: &str = "openshell/e2e-python:dev"; +/// The image for tests that run `python3` in a sandbox. It is public and +/// pinned by digest so any gateway can pull it, and has the Python standard +/// library only: no curl, git or openssl. +const DEFAULT_PYTHON_IMAGE: &str = "ghcr.io/astral-sh/uv:0.12.17-python3.12-trixie-slim@sha256:9a59bb7206905ccaae4f7dab222fbac47c125a21e5fc16f43f427cd6c940ade3"; /// The paths `python3` runs from in [`python_image`] images, as network /// policy binaries: the system Python, the pullable uv image's Python, and the From 3a3122064f58e030d3c86711a6a625ba85d1f09b Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 6 Nov 2026 16:25:17 -0800 Subject: [PATCH 10/16] test(conformance): tighten credential gating checks from review - The WebSocket recorder reads frames until the connection ends, and the test waits for that before checking that no binary frame arrived. - The HTTP recorder keeps the request head and body apart. The REST tests compare the body exactly and check the secret appears nowhere, and failure messages redact it. - testkit errors keep the gateway's gRPC status code, and the admission test checks for FailedPrecondition. - The policy create request destructures the SDK spec, so a new field fails to compile instead of being dropped. - host_gateway no longer retries its connection. - Docs state that conformance requires sandboxes to reach the test machine as host.openshell.internal. Signed-off-by: Matthew Grossman --- TESTING.md | 4 +- .../post-renderer.sh | 7 +- tests/suites/conformance/Cargo.lock | 1 + tests/suites/conformance/README.md | 11 +- .../conformance/src/credential_gating.rs | 78 ++++++---- tests/suites/drivers/podman/Cargo.lock | 1 + tests/suites/features/Cargo.lock | 1 + .../host-gateway/tests/host_gateway.rs | 18 +-- tests/support/Cargo.lock | 1 + tests/support/host-server/src/lib.rs | 136 ++++++++++++------ tests/testkit/README.md | 5 +- tests/testkit/testkit/Cargo.toml | 4 +- tests/testkit/testkit/src/context.rs | 43 +++--- tests/testkit/testkit/src/error.rs | 28 +++- tests/testkit/testkit/src/lib.rs | 1 + tests/testkit/testkit/src/provider.rs | 6 + .../ansible/playbooks/openshell-k3s.yaml | 2 +- .../roles/openshell_gateway/tasks/docker.yaml | 4 +- .../roles/openshell_gateway/tasks/podman.yaml | 4 +- 19 files changed, 232 insertions(+), 123 deletions(-) diff --git a/TESTING.md b/TESTING.md index 902d8c19ad..9810e104d2 100644 --- a/TESTING.md +++ b/TESTING.md @@ -657,8 +657,8 @@ explains the isolation backend separation behind the capability-free runtime. On a **remote** cluster, drop the `e2e-host-gateway` feature. Those tests rely on the sandbox-side `host.openshell.internal` alias reaching the machine running the tests, which is unreachable from pods on a remote cluster, so they fail. -The conformance suite's `credential_gating` tests rely on the alias too and -fail on a remote cluster; the feature selection doesn't skip them. +The conformance suite requires the alias to reach that machine too, for its +`credential_gating` tests; the feature selection doesn't skip them. Left enabled, the `host_gateway_alias` suite fails because `host.openshell.internal` does not resolve inside the pod, so the gateway SSRF-denies the request (`DNS resolution failed` / `ssrf_denied`) — a networking diff --git a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh index c05b6b553f..b8142c72e4 100755 --- a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh +++ b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh @@ -16,9 +16,10 @@ cp "${plugin_dir}/kustomization.yaml" "${work_dir}/kustomization.yaml" cp "${plugin_dir}/workload-patch.yaml" "${work_dir}/workload-patch.yaml" tee "${work_dir}/rendered.yaml" >/dev/null -# server.hostGatewayIP renders the gateway pod's hostAliases entry. -if grep -q 'host\.openshell\.internal' "${work_dir}/rendered.yaml" \ - && grep -q '^ *hostAliases:' "${work_dir}/rendered.yaml"; then +# server.hostGatewayIP renders the gateway pod's only hostAliases entry +# (deploy/helm/openshell/templates/_gateway-workload.tpl), which the +# host-gateway component copies from hostAliases.0.ip. +if grep -q '^ *hostAliases:' "${work_dir}/rendered.yaml"; then cp -R "${plugin_dir}/host-gateway" "${work_dir}/host-gateway" printf 'components:\n - host-gateway\n' >>"${work_dir}/kustomization.yaml" fi diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 29a6acdb75..43ec90afd7 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -2278,6 +2278,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "tonic", ] [[package]] diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index 33cf17058e..466fea9af6 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -23,10 +23,9 @@ tmachine environment on CI. The other suites are ports of the old tests, so a test environment needs it installed (`tests/tmachine/ansible/playbooks/test-tools.yaml` does that for the tmachine environments). `credential-gating`, ported from NVIDIA/OpenShell#5264, starts -servers (`tests/support/host-server`) on the machine that runs the tests and reaches them from sandboxes -created from `testkit::python_image()` through `host.openshell.internal`, so -it fails against a gateway whose sandboxes can't reach that machine, such as -a remote Kubernetes cluster. +servers (`tests/support/host-server`) on the machine that runs the tests and +reaches them from sandboxes created from `testkit::python_image()` through +`host.openshell.internal`. ## Running it @@ -47,7 +46,9 @@ tests/suites/conformance/target/debug/openshell-conformance run --junit conforma ``` The program runs only this suite: passing it means a gateway meets the -conformance contract below. Feature and driver suites are not part of it. The +conformance contract below. The machine that runs it must be reachable from +sandboxes as `host.openshell.internal`, which rules out a gateway on a remote +Kubernetes cluster. Feature and driver suites are not part of it. The [testkit README](../../testkit/README.md#test-programs) describes the commands and exit codes. `tasks/scripts/check-conformance-program.sh` checks that it lists the same tests as nextest. diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 616eed1374..f6700b9f12 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -5,15 +5,17 @@ //! what the supervisor forwards upstream in REST bodies and WebSocket frames. //! //! Ported from the e2e tests added in NVIDIA/OpenShell#5264. The upstream -//! servers run on the test host ([`host_server_support`]), so a gateway on a -//! remote cluster can't run this suite. +//! servers run on the test host ([`host_server_support`]), so the suite +//! requires sandboxes to reach the test host as `host.openshell.internal`. use std::time::Duration; -use host_server_support::{HttpRecorder, WebSocketRecorder}; +use host_server_support::{HttpRecorder, Request, WebSocketRecorder}; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; -use testkit::{Context, Error, PYTHON_BINARIES, Poll, Result, ResultExt as _, Sandbox, YamlFile}; +use testkit::{ + Context, Error, GrpcCode, PYTHON_BINARIES, Poll, Result, ResultExt as _, Sandbox, YamlFile, +}; const TEST_HOST: &str = "host.openshell.internal"; const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; @@ -71,7 +73,7 @@ fi"#; async fn profile_rest_endpoint_keeps_body_placeholder(cx: &mut Context) -> testkit::Result { let server = HttpRecorder::start("ok").await?; let websocket = WebSocketRecorder::start().await?; - let provider = install_provider(cx, Some((server.port, websocket.port))).await?; + let provider = install_gating_provider(cx, Some((server.port, websocket.port))).await?; let mut policy = policy::base(); allow(&mut policy, server.port) @@ -79,7 +81,7 @@ async fn profile_rest_endpoint_keeps_body_placeholder(cx: &mut Context) -> testk .request_body_credential_rewrite(false); let sandbox = create_sandbox(cx, "rest", policy, &provider).await?; run_python(&sandbox, BODY_CLIENT, server.port).await?; - expect_one_body(&server, PLACEHOLDER_PREFIX, TEST_SECRET).await + expect_placeholder_body(&server).await } /// A provider's WebSocket endpoint completes the handshake but doesn't @@ -89,7 +91,7 @@ async fn profile_rest_endpoint_keeps_body_placeholder(cx: &mut Context) -> testk async fn profile_websocket_endpoint_denies_binary_frames(cx: &mut Context) -> testkit::Result { let server = HttpRecorder::start("ok").await?; let websocket = WebSocketRecorder::start().await?; - let provider = install_provider(cx, Some((server.port, websocket.port))).await?; + let provider = install_gating_provider(cx, Some((server.port, websocket.port))).await?; let mut policy = policy::base(); allow(&mut policy, websocket.port).websocket(Access::ReadWrite); @@ -99,8 +101,9 @@ async fn profile_websocket_endpoint_denies_binary_frames(cx: &mut Context) -> te .wait_for_handshake() .await .context("upstream should receive the WebSocket handshake")?; + let binary_seen = websocket.wait_until_finished().await?; testkit::ensure!( - !websocket.binary_seen(), + !binary_seen, "credentialed WebSocket binary frame reached upstream" ); Ok(()) @@ -112,10 +115,11 @@ async fn profile_websocket_endpoint_denies_binary_frames(cx: &mut Context) -> te #[testkit::test] async fn policy_binding_requires_inspection_or_opt_in(cx: &mut Context) -> testkit::Result { let server = HttpRecorder::start("ok").await?; - let provider = install_provider(cx, None).await?; + let provider = install_gating_provider(cx, None).await?; let mut l4 = policy::base(); allow(&mut l4, server.port).credential_binding(&provider); + // The wording checks follow the #5264 tests. let error = expect_rejected(cx, "l4", l4, &provider).await?; testkit::ensure!(error.contains("credentialed endpoint"), "{error}"); testkit::ensure!(error.contains("L4-only"), "{error}"); @@ -145,7 +149,7 @@ async fn policy_binding_rest_body_keeps_placeholder_without_rewrite( cx: &mut Context, ) -> testkit::Result { let server = HttpRecorder::start("ok").await?; - let provider = install_provider(cx, None).await?; + let provider = install_gating_provider(cx, None).await?; let mut policy = policy::base(); allow(&mut policy, server.port) @@ -154,7 +158,7 @@ async fn policy_binding_rest_body_keeps_placeholder_without_rewrite( .credential_binding(&provider); let sandbox = create_sandbox(cx, "body", policy, &provider).await?; run_python(&sandbox, BODY_CLIENT, server.port).await?; - expect_one_body(&server, PLACEHOLDER_PREFIX, TEST_SECRET).await + expect_placeholder_body(&server).await } /// A policy binding with `request_body_credential_rewrite: true` replaces @@ -164,7 +168,7 @@ async fn policy_binding_rest_body_rewrite_sends_secret_upstream( cx: &mut Context, ) -> testkit::Result { let server = HttpRecorder::start("ok").await?; - let provider = install_provider(cx, None).await?; + let provider = install_gating_provider(cx, None).await?; let mut policy = policy::base(); allow(&mut policy, server.port) @@ -173,7 +177,13 @@ async fn policy_binding_rest_body_rewrite_sends_secret_upstream( .credential_binding(&provider); let sandbox = create_sandbox(cx, "rw", policy, &provider).await?; run_python(&sandbox, BODY_CLIENT, server.port).await?; - expect_one_body(&server, TEST_SECRET, PLACEHOLDER_PREFIX).await + let request = one_request(&server).await?; + testkit::ensure!( + request.body == format!("prefix-{TEST_SECRET}-suffix"), + "expected the secret in the body upstream; got {}", + redact(&request) + ); + Ok(()) } /// Binding and unbinding a provider with `policy set --wait` adds and @@ -181,7 +191,7 @@ async fn policy_binding_rest_body_rewrite_sends_secret_upstream( #[testkit::test] async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> testkit::Result { let server = HttpRecorder::start("ok").await?; - let provider = install_provider(cx, None).await?; + let provider = install_gating_provider(cx, None).await?; let policy_binding = |bind: bool| { let mut policy = policy::base(); @@ -223,7 +233,7 @@ fn allow(policy: &mut proto::SandboxPolicy, port: u16) -> &mut proto::NetworkEnd /// Install a run-unique provider holding `TEST_SECRET` in `TOKEN_ENV`. With /// `endpoints`, its profile declares the REST and WebSocket ports as its /// endpoints; without, a policy must bind the provider explicitly. -async fn install_provider(cx: &Context, endpoints: Option<(u16, u16)>) -> Result { +async fn install_gating_provider(cx: &Context, endpoints: Option<(u16, u16)>) -> Result { let name = cx.name("cg"); let endpoints = endpoints.map_or_else(String::new, |(rest, websocket)| { format!( @@ -288,7 +298,8 @@ async fn create_sandbox( Ok(sandbox) } -/// Fail unless the gateway rejects the sandbox, and return the rejection. +/// Fail unless the gateway rejects the sandbox as a failed precondition, and +/// return the rejection. async fn expect_rejected( cx: &Context, suffix: &str, @@ -300,7 +311,10 @@ async fn expect_rejected( "the gateway accepted sandbox {}", sandbox.name() ))), - Err(error) => Ok(error.to_string()), + Err(error) if error.grpc_code() == Some(GrpcCode::FailedPrecondition) => { + Ok(error.to_string()) + } + Err(error) => Err(error.context("expected a failed precondition")), } } @@ -315,23 +329,39 @@ async fn run_python(sandbox: &Sandbox, script: &str, port: u16) -> Result { Ok(()) } -/// Fail unless `server` saw exactly one request, containing `present` and -/// not `absent`. -async fn expect_one_body(server: &HttpRecorder, present: &str, absent: &str) -> Result { - let requests = server.wait_for_requests(1).await?; +/// The one request `server` saw. +async fn one_request(server: &HttpRecorder) -> Result { + let mut requests = server.wait_for_requests(1).await?; testkit::ensure!( requests.len() == 1, "expected one request upstream; got {}", requests.len() ); - let request = &requests[0]; + Ok(requests.remove(0)) +} + +/// Fail unless `server` saw one request whose body has the credential +/// placeholder in place of the token, and the secret appears nowhere in it. +async fn expect_placeholder_body(server: &HttpRecorder) -> Result { + let request = one_request(server).await?; testkit::ensure!( - request.contains(present) && !request.contains(absent), - "expected {present:?} and not {absent:?} upstream" + request + .body + .starts_with(&format!("prefix-{PLACEHOLDER_PREFIX}")) + && request.body.ends_with("-suffix") + && !request.head.contains(TEST_SECRET) + && !request.body.contains(TEST_SECRET), + "expected the placeholder in the body upstream and no secret; got {}", + redact(&request) ); Ok(()) } +/// `request` for a failure message, with the secret replaced. +fn redact(request: &Request) -> String { + format!("{}\n\n{}", request.head, request.body).replace(TEST_SECRET, "") +} + /// Wait until new exec sessions report `expected` from [`TOKEN_PRESENCE`]. async fn wait_for_token(sandbox: &Sandbox, expected: &str) -> Result { testkit::poll_until( diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index d062100826..67042d4d88 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2266,6 +2266,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "tonic", ] [[package]] diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 6fbc0b4836..9023943a1e 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2324,6 +2324,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "tonic", ] [[package]] diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index 33269aa095..472197dfd1 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -20,23 +20,13 @@ use testkit::{Context, PYTHON_BINARIES, Result, ResultExt as _, YamlFile}; const PYTHON: &str = "python3"; /// Requests `/` from `host.openshell.internal` on the port in `argv[1]` and -/// prints the response. A failed connection is retried for 20 seconds, so a -/// slow start doesn't fail the test, and reported on stderr. +/// prints the response. The server listens before the sandbox is created, +/// and a Ready sandbox's network is up, so the client doesn't retry. const CLIENT: &str = r#" -import socket, sys, time +import socket, sys host, port = "host.openshell.internal", int(sys.argv[1]) -deadline = time.monotonic() + 20 -while True: - try: - conn = socket.create_connection((host, port), timeout=10) - break - except OSError as error: - print(f"connect {host}:{port}: {error}", file=sys.stderr, flush=True) - if time.monotonic() > deadline: - sys.exit(1) - time.sleep(1) -with conn: +with socket.create_connection((host, port), timeout=10) as conn: conn.sendall(f"GET / HTTP/1.1\r\nHost: {host}:{port}\r\nConnection: close\r\n\r\n".encode()) response = b"" while chunk := conn.recv(4096): diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index cd15939875..9f889d6bb7 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -2285,6 +2285,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "tonic", ] [[package]] diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index f5d1f6522f..0860622054 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -26,11 +26,20 @@ const WAIT_INTERVAL: Duration = Duration::from_millis(100); const READ_TIMEOUT: Duration = Duration::from_secs(10); const MAX_REQUEST: usize = 1024 * 1024; +/// A request an [`HttpRecorder`] read. +#[derive(Clone, Debug)] +pub struct Request { + /// The request line and headers. + pub head: String, + pub body: String, +} + /// An HTTP server that answers every request with `200` and a fixed body, -/// and records each request it read. +/// and records each request it read. It reads a body by `Content-Length` +/// only, not a chunked one. pub struct HttpRecorder { pub port: u16, - requests: Arc>>, + requests: Arc>>, task: JoinHandle<()>, } @@ -56,9 +65,8 @@ impl HttpRecorder { }) } - /// The requests the server has read, headers and body, once there are at - /// least `count`. - pub async fn wait_for_requests(&self, count: usize) -> Result> { + /// The requests the server has read, once there are at least `count`. + pub async fn wait_for_requests(&self, count: usize) -> Result> { testkit::poll_until( &format!( "{count} request(s) at the host server on port {}", @@ -89,36 +97,32 @@ impl Drop for HttpRecorder { } } -/// A WebSocket server that completes the handshake and records whether a -/// binary frame followed. +/// A WebSocket server that completes the handshake, then reads frames until +/// the connection closes, and records whether any was a binary frame. pub struct WebSocketRecorder { pub port: u16, - handshake_seen: Arc, - binary_seen: Arc, + state: Arc, task: JoinHandle<()>, } +#[derive(Default)] +struct WebSocketState { + handshake_seen: AtomicBool, + binary_seen: AtomicBool, + finished: AtomicBool, +} + impl WebSocketRecorder { pub async fn start() -> Result { let (listener, port) = bind().await?; - let handshake_seen = Arc::new(AtomicBool::new(false)); - let binary_seen = Arc::new(AtomicBool::new(false)); - let (task_handshake, task_binary) = (Arc::clone(&handshake_seen), Arc::clone(&binary_seen)); + let state = Arc::new(WebSocketState::default()); + let task_state = Arc::clone(&state); let task = tokio::spawn(async move { while let Ok((stream, _)) = listener.accept().await { - tokio::spawn(accept_websocket( - stream, - Arc::clone(&task_handshake), - Arc::clone(&task_binary), - )); + tokio::spawn(accept_websocket(stream, Arc::clone(&task_state))); } }); - Ok(Self { - port, - handshake_seen, - binary_seen, - task, - }) + Ok(Self { port, state, task }) } /// Wait until a client completed the handshake. @@ -131,7 +135,7 @@ impl WebSocketRecorder { WAIT_TIMEOUT, WAIT_INTERVAL, async || { - Ok(if self.handshake_seen.load(Ordering::Acquire) { + Ok(if self.state.handshake_seen.load(Ordering::Acquire) { Poll::Ready(()) } else { Poll::Pending("no handshake".to_string()) @@ -141,9 +145,26 @@ impl WebSocketRecorder { .await } - /// Whether a binary frame arrived after a handshake. - pub fn binary_seen(&self) -> bool { - self.binary_seen.load(Ordering::Acquire) + /// Wait until a connection that completed the handshake has ended, by a + /// close frame, the client closing it, or [`READ_TIMEOUT`] passing, and + /// return whether a binary frame arrived on it. + pub async fn wait_until_finished(&self) -> Result { + testkit::poll_until( + &format!( + "the WebSocket connection at the host server on port {} to end", + self.port + ), + READ_TIMEOUT + WAIT_TIMEOUT, + WAIT_INTERVAL, + async || { + Ok(if self.state.finished.load(Ordering::Acquire) { + Poll::Ready(self.state.binary_seen.load(Ordering::Acquire)) + } else { + Poll::Pending("the connection is open".to_string()) + }) + }, + ) + .await } } @@ -168,7 +189,7 @@ async fn bind() -> Result<(TcpListener, u16)> { /// Read one request, headers and `Content-Length` body, record it, and /// answer with `body`. -async fn respond(mut stream: TcpStream, body: Arc, requests: Arc>>) { +async fn respond(mut stream: TcpStream, body: Arc, requests: Arc>>) { let _ = stream.set_nodelay(true); let Some(request) = read_request(&mut stream).await else { return; @@ -176,7 +197,7 @@ async fn respond(mut stream: TcpStream, body: Arc, requests: Arc, requests: Arc Option> { +async fn read_request(stream: &mut TcpStream) -> Option { let mut request = Vec::new(); let mut buffer = [0_u8; 4096]; loop { if let Some(end) = find(&request, b"\r\n\r\n") { - let length = header(&String::from_utf8_lossy(&request[..end]), "content-length") + let head = String::from_utf8_lossy(&request[..end]).into_owned(); + let length = header(&head, "content-length") .and_then(|value| value.parse::().ok()) .unwrap_or(0); if request.len() >= end + 4 + length { - return Some(request); + let body = + String::from_utf8_lossy(&request[end + 4..end + 4 + length]).into_owned(); + return Some(Request { head, body }); } } match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut buffer)).await { @@ -206,19 +230,16 @@ async fn read_request(stream: &mut TcpStream) -> Option> { } } -async fn accept_websocket( - mut stream: TcpStream, - handshake_seen: Arc, - binary_seen: Arc, -) { +async fn accept_websocket(mut stream: TcpStream, state: Arc) { let _ = stream.set_nodelay(true); let Some(request) = read_request(&mut stream).await else { return; }; - let Some(key) = header(&String::from_utf8_lossy(&request), "sec-websocket-key") else { + let Some(key) = header(&request.head, "sec-websocket-key") else { return; }; // The supervisor checks Sec-WebSocket-Accept, so compute it properly. + // SHA-1 is what RFC 6455 specifies for it; it protects nothing here. let mut hasher = Sha1::new(); hasher.update(key.as_bytes()); hasher.update(b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11"); @@ -229,14 +250,39 @@ async fn accept_websocket( if stream.write_all(response.as_bytes()).await.is_err() { return; } - handshake_seen.store(true, Ordering::Release); + state.handshake_seen.store(true, Ordering::Release); - let mut frame_header = [0_u8; 2]; - if let Ok(Ok(_)) = - tokio::time::timeout(READ_TIMEOUT, stream.read_exact(&mut frame_header)).await - && frame_header[0] & 0x0f == 0x02 - { - binary_seen.store(true, Ordering::Release); + let _ = tokio::time::timeout(READ_TIMEOUT, read_frames(&mut stream, &state)).await; + state.finished.store(true, Ordering::Release); +} + +/// Read frames until a close frame or the end of the stream, and record a +/// binary frame. +async fn read_frames(stream: &mut TcpStream, state: &WebSocketState) -> std::io::Result<()> { + loop { + let mut frame_header = [0_u8; 2]; + stream.read_exact(&mut frame_header).await?; + let opcode = frame_header[0] & 0x0f; + let length = match frame_header[1] & 0x7f { + 126 => u64::from(stream.read_u16().await?), + 127 => stream.read_u64().await?, + length => u64::from(length), + }; + if opcode == 0x02 { + state.binary_seen.store(true, Ordering::Release); + } + if opcode == 0x08 { + return Ok(()); + } + let mask = if frame_header[1] & 0x80 == 0 { 0 } else { 4 }; + let skipped = tokio::io::copy( + &mut (&mut *stream).take(mask + length), + &mut tokio::io::sink(), + ) + .await?; + if skipped < mask + length { + return Ok(()); + } } } diff --git a/tests/testkit/README.md b/tests/testkit/README.md index b89a726cdf..6256ea92e5 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -92,9 +92,12 @@ fails. A killed test process skips cleanup, so names are unique to the run. `cx.sandbox(..).policy(policy)`, or `policy::to_yaml` for the CLI. `testkit::PYTHON_BINARIES` names `python3` in `python_image()` images. - `testkit::install_provider(cx, name, profile, credentials)` to import a - provider profile and create a provider of it, with cleanup for both. + provider profile, whose `id` must be `name`, and create a provider of it, + with cleanup for both. - `testkit::YamlFile` for YAML a CLI command reads from a file. - `cx.client()` for SDK calls testkit does not wrap. +- `error.grpc_code()` for the gRPC status code (`testkit::GrpcCode`) of a + request the gateway refused, such as a sandbox create. - `poll_until` for waiting on other state. ## Running against a gateway diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index 288e11990d..ae8ad5c9ad 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -30,13 +30,11 @@ serde = "1" serde_json = "1" tempfile = "3" tokio = { version = "1.43", features = ["macros", "process", "rt", "time"] } +tonic = "0.14" [target.'cfg(unix)'.dependencies] libc = "0.2" -[dev-dependencies] -tonic = "0.14" - # A test program, run by `cargo test` and nextest through `testkit::main`'s # libtest interface, that checks the program's own subcommands. [[test]] diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 8709db8a18..d5e6c7703e 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -367,7 +367,7 @@ impl SandboxBuilder { .await? .create_sandbox(request) .await - .map_err(|status| Error::from_error(&status)) + .map_err(|status| Error::from_status(&status)) .context(format!("create sandbox {name}"))?; self.client .get_sandbox(&name) @@ -387,41 +387,48 @@ impl SandboxBuilder { } /// The gRPC request for a sandbox created from `spec` with `policy`, built -/// as the SDK builds one from `spec` alone. +/// as the SDK builds one from `spec` alone. The spec is destructured without +/// `..`, so a field added to the SDK's spec fails to compile here instead of +/// being dropped. [`SandboxBuilder::create`] rejects a GPU or service +/// exposures before calling this. fn policy_create_request( spec: SandboxSpec, policy: proto::SandboxPolicy, workspace_scope: Option, ) -> proto::CreateSandboxRequest { + let SandboxSpec { + name, + image, + labels, + environment, + providers, + gpu: _, + command, + tty, + service_exposures: _, + restart_policy, + } = spec; proto::CreateSandboxRequest { - name: spec.name.unwrap_or_default(), - labels: spec.labels, + name: name.unwrap_or_default(), + labels, workspace_scope, spec: Some(proto::SandboxSpec { - environment: spec.environment, - template: spec.image.map(|image| proto::SandboxTemplate { + environment, + template: image.map(|image| proto::SandboxTemplate { image, ..proto::SandboxTemplate::default() }), - providers: spec.providers, - command: spec.command, - tty: spec.tty, + providers, + command, + tty, policy: Some(policy), - restart_policy: restart_policy(spec.restart_policy) as i32, + restart_policy: proto::SandboxRestartPolicy::from(restart_policy) as i32, ..proto::SandboxSpec::default() }), ..proto::CreateSandboxRequest::default() } } -const fn restart_policy(policy: SandboxRestartPolicy) -> proto::SandboxRestartPolicy { - match policy { - SandboxRestartPolicy::Never => proto::SandboxRestartPolicy::Never, - SandboxRestartPolicy::OnFailure => proto::SandboxRestartPolicy::OnFailure, - SandboxRestartPolicy::Always => proto::SandboxRestartPolicy::Always, - } -} - /// A sandbox created by the test. pub struct Sandbox { client: WorkspaceScopedClient, diff --git a/tests/testkit/testkit/src/error.rs b/tests/testkit/testkit/src/error.rs index 70a702ddc1..51d2fc3904 100644 --- a/tests/testkit/testkit/src/error.rs +++ b/tests/testkit/testkit/src/error.rs @@ -7,10 +7,12 @@ use std::fmt; use openshell_sdk::SdkError; -/// A test failure: a message with the context it happened in. +/// A test failure: a message with the context it happened in, and the gRPC +/// status code when the gateway refused a request. #[derive(Debug)] pub struct Error { message: String, + code: Option, } /// The result of a test or test step. @@ -20,9 +22,23 @@ impl Error { pub fn msg(message: impl Into) -> Self { Self { message: message.into(), + code: None, } } + /// A gateway's refusal, keeping its status code for [`Error::grpc_code`]. + pub fn from_status(status: &tonic::Status) -> Self { + Self { + code: Some(status.code()), + ..Self::from_error(status) + } + } + + /// The gRPC status code, when the gateway refused a request. + pub fn grpc_code(&self) -> Option { + self.code + } + pub fn from_display(error: impl fmt::Display) -> Self { Self::msg(error.to_string()) } @@ -45,7 +61,10 @@ impl Error { #[must_use] pub fn context(self, context: impl fmt::Display) -> Self { - Self::msg(format!("{context}: {}", self.message)) + Self { + message: format!("{context}: {}", self.message), + code: self.code, + } } } @@ -59,7 +78,10 @@ impl std::error::Error for Error {} impl From for Error { fn from(error: SdkError) -> Self { - Self::from_error(&error) + Self { + code: error.grpc_status().map(tonic::Status::code), + ..Self::from_error(&error) + } } } diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 2786668d9b..6a57c71420 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -58,6 +58,7 @@ pub use ready::check as check_ready; pub use registry::Test; pub use run::{TestInfo, run}; pub use testkit_macros::test; +pub use tonic::Code as GrpcCode; pub use yaml_file::YamlFile; #[doc(hidden)] diff --git a/tests/testkit/testkit/src/provider.rs b/tests/testkit/testkit/src/provider.rs index c87fc6c51c..0e6f3a12bd 100644 --- a/tests/testkit/testkit/src/provider.rs +++ b/tests/testkit/testkit/src/provider.rs @@ -18,6 +18,12 @@ pub async fn install_provider( profile: &str, credentials: &[(&str, &str)], ) -> Result { + crate::ensure!( + profile + .lines() + .any(|line| line.trim_end() == format!("id: {name}")), + "the profile for provider {name} must have `id: {name}`" + ); // Both deletes succeed for a missing resource, so register them before // the creates. Hooks run last registered first. for kind in ["profile", "provider"] { diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml index 5b68bc0067..e9d810b2a1 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml @@ -34,7 +34,7 @@ src: "{{ openshell_sandbox_image }}" - name: supervisor src: "{{ openshell_supervisor_image }}" - # The testkit's python_image() sandbox workload. + # The e2e-python sandbox workload image some tests use. - name: e2e-python src: "{{ openshell_e2e_python_image }}" diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml index 11b033972a..5d980c2c7f 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 --- -# Tests run sandboxes from the e2e-python workload image (the testkit's -# python_image()), so load it next to the runtime images. +# Some tests run sandboxes from the e2e-python workload image, so load it next +# to the runtime images. - name: Copy OpenShell runtime and test workload images become: true ansible.builtin.copy: diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml index b85361b769..40d70adf24 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 --- -# Tests run sandboxes from the e2e-python workload image (the testkit's -# python_image()), so load it next to the runtime images. +# Some tests run sandboxes from the e2e-python workload image, so load it next +# to the runtime images. - name: Copy OpenShell runtime and test workload images become: true ansible.builtin.copy: From 9ab1be143bb31329d38fb5916e22187187341c60 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 08:44:23 -0800 Subject: [PATCH 11/16] test(testkit): restore openshell/e2e-python:dev as the default test image The tmachine installers load it and the Docker and Podman lanes build it, so those lanes don't pull an image inside a test's timeout or lose curl, git and openssl. The Kubernetes and VM lanes keep using the pinned public image through OPENSHELL_TEST_PYTHON_IMAGE, and the base test policy names the sandbox user again. Running conformance against any gateway without an override is left to a follow-up. Signed-off-by: Matthew Grossman --- TESTING.md | 4 +++- e2e/rust/e2e-docker.sh | 6 +++--- e2e/rust/e2e-kubernetes.sh | 1 + e2e/rust/e2e-podman.sh | 6 +++--- e2e/rust/e2e-vm.sh | 3 ++- e2e/support/conformance.sh | 6 ++++++ tasks/test.toml | 1 + tests/testkit/README.md | 8 ++++---- tests/testkit/testkit/src/base-policy.yaml | 10 ++++++---- tests/testkit/testkit/src/image.rs | 9 +++++---- tests/tmachine/ansible/playbooks/openshell-k3s.yaml | 2 +- .../ansible/roles/openshell_gateway/tasks/docker.yaml | 4 ++-- .../ansible/roles/openshell_gateway/tasks/podman.yaml | 4 ++-- 13 files changed, 39 insertions(+), 25 deletions(-) diff --git a/TESTING.md b/TESTING.md index 9810e104d2..0d4eb3f68d 100644 --- a/TESTING.md +++ b/TESTING.md @@ -429,7 +429,9 @@ Suites: `openshell-conformance` against the lane's gateway and writes a JUnit report under `results/`; set `OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program, as Branch E2E does with the static build it makes once per architecture. - The + The Docker and Podman runners build `openshell/e2e-python:dev` for it; the + Kubernetes and VM runners set `OPENSHELL_TEST_PYTHON_IMAGE` to a pullable + Python image unless it is already set. The installed-artifact conformance lanes run the same crate through `openshell-test-runner` (see above). - Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or diff --git a/e2e/rust/e2e-docker.sh b/e2e/rust/e2e-docker.sh index 8db220ca1b..68d48e7370 100755 --- a/e2e/rust/e2e-docker.sh +++ b/e2e/rust/e2e-docker.sh @@ -54,9 +54,9 @@ if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then exit 0 fi -if [ "${E2E_FEATURES}" != "e2e-conformance-only" ]; then - CONTAINER_ENGINE=docker bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" -fi +# The conformance suite runs on every lane, including e2e-conformance-only, +# and creates sandboxes from this image (testkit::python_image()). +CONTAINER_ENGINE=docker bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" exec "${ROOT}/e2e/with-docker-gateway.sh" \ bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index fc90a827ee..7ef5576eab 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -80,6 +80,7 @@ run_conformance() { fi OPENSHELL_CONFORMANCE_JUNIT="${OPENSHELL_CONFORMANCE_JUNIT:-${ROOT}/results/${OPENSHELL_E2E_REPORT_NAME:-e2e-kubernetes}-conformance.xml}" \ + OPENSHELL_TEST_PYTHON_IMAGE="${OPENSHELL_TEST_PYTHON_IMAGE:-${E2E_PULLABLE_PYTHON_IMAGE}}" \ e2e_run_openshell_conformance "Kubernetes" } diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index 495863fede..e718f0afea 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -116,9 +116,9 @@ if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ] && [ -z "${OPENSHELL_BIN:-}" ]; then export OPENSHELL_BIN="${ROOT}/target/debug/openshell" fi -if [ "${E2E_FEATURES}" != "e2e-conformance-only" ]; then - CONTAINER_ENGINE=podman bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" -fi +# The conformance suite runs on every lane, including e2e-conformance-only, +# and creates sandboxes from this image (testkit::python_image()). +CONTAINER_ENGINE=podman bash "${ROOT}/tasks/scripts/e2e-build-workload.sh" exec "${ROOT}/e2e/with-podman-gateway.sh" \ bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 0cb2c46858..bafaadbc71 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -463,7 +463,8 @@ e2e_export_gateway_restart_metadata \ # preparation; allow 180s for slower CI runners. export OPENSHELL_PROVISION_TIMEOUT="${SANDBOX_PROVISION_TIMEOUT}" -e2e_run_openshell_conformance "VM" +OPENSHELL_TEST_PYTHON_IMAGE="${OPENSHELL_TEST_PYTHON_IMAGE:-${E2E_PULLABLE_PYTHON_IMAGE}}" \ + e2e_run_openshell_conformance "VM" if [ "${E2E_FEATURES}" = "e2e-conformance-only" ]; then echo "::notice::Skipping Rust e2e tests (e2e-conformance-only)" exit 0 diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh index 56ce5a50f1..eef41ecb29 100644 --- a/e2e/support/conformance.sh +++ b/e2e/support/conformance.sh @@ -23,6 +23,12 @@ e2e_require_openshell_bin() { export OPENSHELL_BIN="${binary_dir}/$(basename "${OPENSHELL_BIN}")" } +# A pullable image for testkit::python_image() on lanes that can't load the +# locally built openshell/e2e-python:dev into their compute platform. It +# matches the Rust harness's E2E_WORKLOAD_IMAGE for those lanes. +# shellcheck disable=SC2034 # Used by the scripts that source this file. +E2E_PULLABLE_PYTHON_IMAGE="ghcr.io/astral-sh/uv:0.12.17-python3.12-trixie-slim@sha256:9a59bb7206905ccaae4f7dab222fbac47c125a21e5fc16f43f427cd6c940ade3" + # Print the path of the openshell-conformance program: OPENSHELL_CONFORMANCE_BIN # if set, else a debug build from this checkout. e2e_openshell_conformance_bin() { diff --git a/tasks/test.toml b/tasks/test.toml index 2839c1cf0c..30c31ec753 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -177,6 +177,7 @@ run = "OPENSHELL_TEST_CONTAINER_ENGINE=podman e2e/with-podman-gateway.sh cargo t ["e2e:testkit"] description = "Run the testkit suites in tests/suites/conformance against a Docker-backed gateway" +depends = ["e2e:workload:build"] run = "e2e/with-docker-gateway.sh cargo nextest run --locked --manifest-path tests/suites/conformance/Cargo.toml --workspace --test-threads 1 --no-fail-fast" ["e2e:rust:push"] diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 6256ea92e5..d68a1557f8 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -76,10 +76,10 @@ fails. A killed test process skips cleanup, so names are unique to the run. compute driver. Driver suites call it first. - `testkit::base_image()` for the image to create sandboxes from or build on: `OPENSHELL_TEST_BASE_IMAGE`, or OpenShell's default sandbox image. -- `testkit::python_image()` for sandboxes that run `python3`: - `OPENSHELL_TEST_PYTHON_IMAGE`, or a pinned public uv Python image that any - gateway can pull. It has the standard library only, not curl, git, or - openssl. +- `testkit::python_image()` for sandboxes that run `python3`, curl, git, or + openssl: `OPENSHELL_TEST_PYTHON_IMAGE`, or `openshell/e2e-python:dev` + (`e2e/python/Dockerfile.workload`). The tmachine installers load it into + every environment; `mise run e2e:workload:build` builds it locally. - `cx.cli([...])` to run the installed CLI against the same gateway, with `success()`, `failure()`, and `json()` on the output. `cx.cli_handle()` returns that CLI for code that can't borrow `cx`, such as a cleanup hook. diff --git a/tests/testkit/testkit/src/base-policy.yaml b/tests/testkit/testkit/src/base-policy.yaml index 19f77d006f..0935dbaa9a 100644 --- a/tests/testkit/testkit/src/base-policy.yaml +++ b/tests/testkit/testkit/src/base-policy.yaml @@ -2,10 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # The sandbox policy tests start from (testkit::policy::base()): the image's -# system directories read-only, the sandbox's home and /tmp writable, and no -# network access. Tests add the network access they need with -# testkit::policy::allow(). It names no workload user, so the driver runs the -# image's non-root user, or a default non-root user when the image has none. +# system directories read-only, the sandbox's home and /tmp writable, the +# sandbox user, and no network access. Tests add the network access they need +# with testkit::policy::allow(). version: 1 filesystem_policy: include_workdir: true @@ -13,3 +12,6 @@ filesystem_policy: read_write: [/sandbox, /tmp, /dev/null] landlock: compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox diff --git a/tests/testkit/testkit/src/image.rs b/tests/testkit/testkit/src/image.rs index 67f6a78ae0..08e13825a0 100644 --- a/tests/testkit/testkit/src/image.rs +++ b/tests/testkit/testkit/src/image.rs @@ -11,10 +11,11 @@ const DEFAULT_BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; /// Overrides [`DEFAULT_PYTHON_IMAGE`]. const PYTHON_IMAGE_ENV: &str = "OPENSHELL_TEST_PYTHON_IMAGE"; -/// The image for tests that run `python3` in a sandbox. It is public and -/// pinned by digest so any gateway can pull it, and has the Python standard -/// library only: no curl, git or openssl. -const DEFAULT_PYTHON_IMAGE: &str = "ghcr.io/astral-sh/uv:0.12.17-python3.12-trixie-slim@sha256:9a59bb7206905ccaae4f7dab222fbac47c125a21e5fc16f43f427cd6c940ade3"; +/// The workload image for tests that run `python3`, curl, git or openssl in a +/// sandbox, built from `e2e/python/Dockerfile.workload`. It isn't published: +/// the tmachine installers load it into every environment, and +/// `mise run e2e:workload:build` builds it locally. +const DEFAULT_PYTHON_IMAGE: &str = "openshell/e2e-python:dev"; /// The paths `python3` runs from in [`python_image`] images, as network /// policy binaries: the system Python, the pullable uv image's Python, and the diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml index e9d810b2a1..5b68bc0067 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s.yaml @@ -34,7 +34,7 @@ src: "{{ openshell_sandbox_image }}" - name: supervisor src: "{{ openshell_supervisor_image }}" - # The e2e-python sandbox workload image some tests use. + # The testkit's python_image() sandbox workload. - name: e2e-python src: "{{ openshell_e2e_python_image }}" diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml index 5d980c2c7f..11b033972a 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/docker.yaml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 --- -# Some tests run sandboxes from the e2e-python workload image, so load it next -# to the runtime images. +# Tests run sandboxes from the e2e-python workload image (the testkit's +# python_image()), so load it next to the runtime images. - name: Copy OpenShell runtime and test workload images become: true ansible.builtin.copy: diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml index 40d70adf24..b85361b769 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/podman.yaml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 --- -# Some tests run sandboxes from the e2e-python workload image, so load it next -# to the runtime images. +# Tests run sandboxes from the e2e-python workload image (the testkit's +# python_image()), so load it next to the runtime images. - name: Copy OpenShell runtime and test workload images become: true ansible.builtin.copy: From 4ff621be1b3083c3ae3ff71096b5c4e8c28fbbfa Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 09:00:39 -0800 Subject: [PATCH 12/16] test(conformance): add a positive control to the WebSocket denial test The test passed whenever no binary frame reached upstream, whatever the reason. It now first sends the same frame from a sandbox without the provider and requires that it arrives, so the denial check can fail. Unit tests check that WebSocketRecorder sees a binary frame after a long text frame, and none among text frames. Signed-off-by: Matthew Grossman --- .../conformance/src/credential_gating.rs | 61 ++++++++++----- tests/support/host-server/Cargo.toml | 3 + tests/support/host-server/src/lib.rs | 76 +++++++++++++++++++ 3 files changed, 119 insertions(+), 21 deletions(-) diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index f6700b9f12..d794d51e30 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -79,29 +79,29 @@ async fn profile_rest_endpoint_keeps_body_placeholder(cx: &mut Context) -> testk allow(&mut policy, server.port) .rest(Access::Full) .request_body_credential_rewrite(false); - let sandbox = create_sandbox(cx, "rest", policy, &provider).await?; + let sandbox = create_sandbox(cx, "rest", policy, &[&provider]).await?; run_python(&sandbox, BODY_CLIENT, server.port).await?; expect_placeholder_body(&server).await } /// A provider's WebSocket endpoint completes the handshake but doesn't /// forward a binary frame, which the supervisor can't inspect for the -/// credential. +/// credential. The same client in a sandbox without the provider gets its +/// binary frame through, which shows the denial comes from the credential. #[testkit::test] async fn profile_websocket_endpoint_denies_binary_frames(cx: &mut Context) -> testkit::Result { let server = HttpRecorder::start("ok").await?; let websocket = WebSocketRecorder::start().await?; let provider = install_gating_provider(cx, Some((server.port, websocket.port))).await?; - let mut policy = policy::base(); - allow(&mut policy, websocket.port).websocket(Access::ReadWrite); - let sandbox = create_sandbox(cx, "ws", policy, &provider).await?; - run_python(&sandbox, WEBSOCKET_CLIENT, websocket.port).await?; - websocket - .wait_for_handshake() - .await - .context("upstream should receive the WebSocket handshake")?; - let binary_seen = websocket.wait_until_finished().await?; + let control = WebSocketRecorder::start().await?; + let binary_seen = send_binary_frame(cx, "wsctl", &control, &[]).await?; + testkit::ensure!( + binary_seen, + "the binary frame didn't reach upstream from a sandbox without the provider" + ); + + let binary_seen = send_binary_frame(cx, "ws", &websocket, &[&provider]).await?; testkit::ensure!( !binary_seen, "credentialed WebSocket binary frame reached upstream" @@ -137,7 +137,7 @@ async fn policy_binding_requires_inspection_or_opt_in(cx: &mut Context) -> testk allow(&mut opt_in, server.port) .allow_uninspected_credentials() .credential_binding(&provider); - let sandbox = create_sandbox(cx, "optin", opt_in, &provider).await?; + let sandbox = create_sandbox(cx, "optin", opt_in, &[&provider]).await?; sandbox.exec(["true"]).await?.success()?; Ok(()) } @@ -156,7 +156,7 @@ async fn policy_binding_rest_body_keeps_placeholder_without_rewrite( .rest(Access::Full) .request_body_credential_rewrite(false) .credential_binding(&provider); - let sandbox = create_sandbox(cx, "body", policy, &provider).await?; + let sandbox = create_sandbox(cx, "body", policy, &[&provider]).await?; run_python(&sandbox, BODY_CLIENT, server.port).await?; expect_placeholder_body(&server).await } @@ -175,7 +175,7 @@ async fn policy_binding_rest_body_rewrite_sends_secret_upstream( .rest(Access::Full) .request_body_credential_rewrite(true) .credential_binding(&provider); - let sandbox = create_sandbox(cx, "rw", policy, &provider).await?; + let sandbox = create_sandbox(cx, "rw", policy, &[&provider]).await?; run_python(&sandbox, BODY_CLIENT, server.port).await?; let request = one_request(&server).await?; testkit::ensure!( @@ -205,7 +205,7 @@ async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> t }; let (unbound, bound) = (policy_binding(false), policy_binding(true)); - let sandbox = create_sandbox(cx, "live", unbound.clone(), &provider).await?; + let sandbox = create_sandbox(cx, "live", unbound.clone(), &[&provider]).await?; wait_for_token(&sandbox, "TOKEN_ABSENT").await?; set_policy(cx, &sandbox, &bound) @@ -219,6 +219,25 @@ async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> t wait_for_token(&sandbox, "TOKEN_ABSENT").await } +/// Run [`WEBSOCKET_CLIENT`] against `server` in a sandbox with `providers`, +/// and return whether its binary frame reached `server`. +async fn send_binary_frame( + cx: &Context, + suffix: &str, + server: &WebSocketRecorder, + providers: &[&str], +) -> Result { + let mut policy = policy::base(); + allow(&mut policy, server.port).websocket(Access::ReadWrite); + let sandbox = create_sandbox(cx, suffix, policy, providers).await?; + run_python(&sandbox, WEBSOCKET_CLIENT, server.port).await?; + server + .wait_for_handshake() + .await + .context("upstream should receive the WebSocket handshake")?; + server.wait_until_finished().await +} + /// Let the image's Python reach `TEST_HOST` on `port`. fn allow(policy: &mut proto::SandboxPolicy, port: u16) -> &mut proto::NetworkEndpoint { policy::allow( @@ -274,26 +293,26 @@ fn sandbox_with( cx: &Context, suffix: &str, policy: proto::SandboxPolicy, - provider: &str, + providers: &[&str], ) -> testkit::SandboxBuilder { cx.sandbox(suffix) .policy(policy) .spec(|spec| { spec.image = Some(testkit::python_image()); - spec.providers = vec![provider.to_string()]; + spec.providers = providers.iter().map(ToString::to_string).collect(); }) .command(["sleep", "infinity"]) } /// Create a long-running sandbox from the Python image with `policy` and -/// `provider`, and wait until it is Ready. +/// `providers`, and wait until it is Ready. async fn create_sandbox( cx: &Context, suffix: &str, policy: proto::SandboxPolicy, - provider: &str, + providers: &[&str], ) -> Result { - let sandbox = sandbox_with(cx, suffix, policy, provider).create().await?; + let sandbox = sandbox_with(cx, suffix, policy, providers).create().await?; sandbox.wait_ready().await?; Ok(sandbox) } @@ -306,7 +325,7 @@ async fn expect_rejected( policy: proto::SandboxPolicy, provider: &str, ) -> Result { - match sandbox_with(cx, suffix, policy, provider).create().await { + match sandbox_with(cx, suffix, policy, &[provider]).create().await { Ok(sandbox) => Err(Error::msg(format!( "the gateway accepted sandbox {}", sandbox.name() diff --git a/tests/support/host-server/Cargo.toml b/tests/support/host-server/Cargo.toml index aa964e782d..ad59276395 100644 --- a/tests/support/host-server/Cargo.toml +++ b/tests/support/host-server/Cargo.toml @@ -16,3 +16,6 @@ base64 = "0.22" openshell-testkit = { path = "../../testkit/testkit" } sha1 = "0.10" tokio = { version = "1.43", features = ["io-util", "net", "rt", "time"] } + +[dev-dependencies] +tokio = { version = "1.43", features = ["macros", "rt"] } diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 0860622054..5de1af3efa 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -301,3 +301,79 @@ fn header(head: &str, name: &str) -> Option { .then(|| value.trim().to_string()) }) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Complete a handshake with `recorder`, send `frames`, and close. + async fn send_frames(recorder: &WebSocketRecorder, frames: &[(u8, Vec)]) { + let mut stream = TcpStream::connect(("127.0.0.1", recorder.port)) + .await + .unwrap(); + stream.set_nodelay(true).unwrap(); + stream + .write_all( + b"GET / HTTP/1.1\r\nHost: test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\ + Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n", + ) + .await + .unwrap(); + let mut response = Vec::new(); + while find(&response, b"\r\n\r\n").is_none() { + let mut buffer = [0_u8; 1024]; + let read = stream.read(&mut buffer).await.unwrap(); + assert!(read > 0, "closed during the handshake"); + response.extend_from_slice(&buffer[..read]); + } + let response = String::from_utf8_lossy(&response); + assert!(response.starts_with("HTTP/1.1 101"), "{response}"); + // The RFC 6455 sample key's accept value. + assert!( + response.contains("Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo="), + "{response}" + ); + for (opcode, payload) in frames { + stream.write_all(&frame(*opcode, payload)).await.unwrap(); + } + stream.write_all(&frame(0x08, &[])).await.unwrap(); + } + + /// A final, masked client frame. + fn frame(opcode: u8, payload: &[u8]) -> Vec { + let mask = [1_u8, 2, 3, 4]; + let mut frame = vec![0x80 | opcode]; + match payload.len() { + length @ 0..=125 => frame.push(0x80 | u8::try_from(length).unwrap()), + length => { + frame.push(0x80 | 126); + frame.extend_from_slice(&u16::try_from(length).unwrap().to_be_bytes()); + } + } + frame.extend_from_slice(&mask); + frame.extend(payload.iter().enumerate().map(|(i, b)| b ^ mask[i % 4])); + frame + } + + #[tokio::test] + async fn websocket_recorder_sees_a_binary_frame_after_a_long_text_frame() { + let recorder = WebSocketRecorder::start().await.unwrap(); + send_frames( + &recorder, + &[(0x01, vec![b'x'; 300]), (0x02, b"binary".to_vec())], + ) + .await; + assert!(recorder.wait_until_finished().await.unwrap()); + } + + #[tokio::test] + async fn websocket_recorder_sees_no_binary_frame_in_text_frames() { + let recorder = WebSocketRecorder::start().await.unwrap(); + send_frames( + &recorder, + &[(0x01, vec![b'x'; 300]), (0x01, b"text".to_vec())], + ) + .await; + assert!(!recorder.wait_until_finished().await.unwrap()); + } +} From e5e72bf28671d587c189402ab8a1321cff95d37d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 10:19:16 -0800 Subject: [PATCH 13/16] docs(testing): link the remote-cluster follow-up for credential gating Signed-off-by: Matthew Grossman --- TESTING.md | 6 ++++-- tests/suites/conformance/README.md | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/TESTING.md b/TESTING.md index 0d4eb3f68d..a11a608b13 100644 --- a/TESTING.md +++ b/TESTING.md @@ -659,8 +659,10 @@ explains the isolation backend separation behind the capability-free runtime. On a **remote** cluster, drop the `e2e-host-gateway` feature. Those tests rely on the sandbox-side `host.openshell.internal` alias reaching the machine running the tests, which is unreachable from pods on a remote cluster, so they fail. -The conformance suite requires the alias to reach that machine too, for its -`credential_gating` tests; the feature selection doesn't skip them. +The conformance suite's `credential_gating` tests need the alias to reach that +machine too, and the feature selection doesn't skip them, so they fail on a +remote cluster. [#5409](https://github.com/NVIDIA/OpenShell/issues/5409) +tracks handling that. Left enabled, the `host_gateway_alias` suite fails because `host.openshell.internal` does not resolve inside the pod, so the gateway SSRF-denies the request (`DNS resolution failed` / `ssrf_denied`) — a networking diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index 466fea9af6..c51df37c16 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -48,7 +48,8 @@ tests/suites/conformance/target/debug/openshell-conformance run --junit conforma The program runs only this suite: passing it means a gateway meets the conformance contract below. The machine that runs it must be reachable from sandboxes as `host.openshell.internal`, which rules out a gateway on a remote -Kubernetes cluster. Feature and driver suites are not part of it. The +Kubernetes cluster ([#5409](https://github.com/NVIDIA/OpenShell/issues/5409)). +Feature and driver suites are not part of it. The [testkit README](../../testkit/README.md#test-programs) describes the commands and exit codes. `tasks/scripts/check-conformance-program.sh` checks that it lists the same tests as nextest. From eb3561b54af51897d45a3ea7644bf681aff55e9c Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 10:32:40 -0800 Subject: [PATCH 14/16] refactor(test-support): derive the WebSocket accept key with tungstenite Use tungstenite's derive_accept_key instead of hashing with sha1 and base64 directly, and split host-server into http.rs and websocket.rs. Signed-off-by: Matthew Grossman --- tests/support/host-server/Cargo.toml | 3 +- tests/support/host-server/src/http.rs | 125 ++++++++ tests/support/host-server/src/lib.rs | 339 +-------------------- tests/support/host-server/src/websocket.rs | 219 +++++++++++++ 4 files changed, 353 insertions(+), 333 deletions(-) create mode 100644 tests/support/host-server/src/http.rs create mode 100644 tests/support/host-server/src/websocket.rs diff --git a/tests/support/host-server/Cargo.toml b/tests/support/host-server/Cargo.toml index ad59276395..e7db63f53d 100644 --- a/tests/support/host-server/Cargo.toml +++ b/tests/support/host-server/Cargo.toml @@ -12,10 +12,9 @@ repository = "https://github.com/NVIDIA/OpenShell" publish = false [dependencies] -base64 = "0.22" openshell-testkit = { path = "../../testkit/testkit" } -sha1 = "0.10" tokio = { version = "1.43", features = ["io-util", "net", "rt", "time"] } +tungstenite = "0.26" [dev-dependencies] tokio = { version = "1.43", features = ["macros", "rt"] } diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs new file mode 100644 index 0000000000..4d8ff8c812 --- /dev/null +++ b/tests/support/host-server/src/http.rs @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::{Arc, Mutex, PoisonError}; + +use testkit::{Poll, Result}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::TcpStream; +use tokio::task::JoinHandle; + +use crate::{MAX_REQUEST, READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, find, header}; + +/// A request an [`HttpRecorder`] read. +#[derive(Clone, Debug)] +pub struct Request { + /// The request line and headers. + pub head: String, + pub body: String, +} + +/// An HTTP server that answers every request with `200` and a fixed body, +/// and records each request it read. It reads a body by `Content-Length` +/// only, not a chunked one. +pub struct HttpRecorder { + pub port: u16, + requests: Arc>>, + task: JoinHandle<()>, +} + +impl HttpRecorder { + pub async fn start(body: impl Into) -> Result { + let (listener, port) = bind().await?; + let body: Arc = body.into().into(); + let requests = Arc::new(Mutex::new(Vec::new())); + let task_requests = Arc::clone(&requests); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(respond( + stream, + Arc::clone(&body), + Arc::clone(&task_requests), + )); + } + }); + Ok(Self { + port, + requests, + task, + }) + } + + /// The requests the server has read, once there are at least `count`. + pub async fn wait_for_requests(&self, count: usize) -> Result> { + testkit::poll_until( + &format!( + "{count} request(s) at the host server on port {}", + self.port + ), + WAIT_TIMEOUT, + WAIT_INTERVAL, + async || { + let requests = self + .requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + Ok(if requests.len() >= count { + Poll::Ready(requests) + } else { + Poll::Pending(format!("{} request(s)", requests.len())) + }) + }, + ) + .await + } +} + +impl Drop for HttpRecorder { + fn drop(&mut self) { + self.task.abort(); + } +} + +/// Read one request, headers and `Content-Length` body, record it, and +/// answer with `body`. +async fn respond(mut stream: TcpStream, body: Arc, requests: Arc>>) { + let _ = stream.set_nodelay(true); + let Some(request) = read_request(&mut stream).await else { + return; + }; + requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(request); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(response.as_bytes()).await; + let _ = stream.shutdown().await; +} + +pub(crate) async fn read_request(stream: &mut TcpStream) -> Option { + let mut request = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + if let Some(end) = find(&request, b"\r\n\r\n") { + let head = String::from_utf8_lossy(&request[..end]).into_owned(); + let length = header(&head, "content-length") + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + if request.len() >= end + 4 + length { + let body = + String::from_utf8_lossy(&request[end + 4..end + 4 + length]).into_owned(); + return Some(Request { head, body }); + } + } + match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut buffer)).await { + Ok(Ok(read)) if read > 0 && request.len() < MAX_REQUEST => { + request.extend_from_slice(&buffer[..read]); + } + _ => return None, + } + } +} diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 5de1af3efa..0c11e4ceca 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -10,170 +10,22 @@ //! run where the tests run, so a gateway whose sandboxes can't reach the test //! host, such as one on a remote cluster, can't run tests that use them. -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, PoisonError}; +mod http; +mod websocket; + use std::time::Duration; -use base64::Engine as _; -use sha1::{Digest as _, Sha1}; -use testkit::{Error, Poll, Result, ResultExt as _}; -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::task::JoinHandle; +use testkit::{Error, Result, ResultExt as _}; +use tokio::net::TcpListener; + +pub use http::{HttpRecorder, Request}; +pub use websocket::WebSocketRecorder; const WAIT_TIMEOUT: Duration = Duration::from_secs(10); const WAIT_INTERVAL: Duration = Duration::from_millis(100); const READ_TIMEOUT: Duration = Duration::from_secs(10); const MAX_REQUEST: usize = 1024 * 1024; -/// A request an [`HttpRecorder`] read. -#[derive(Clone, Debug)] -pub struct Request { - /// The request line and headers. - pub head: String, - pub body: String, -} - -/// An HTTP server that answers every request with `200` and a fixed body, -/// and records each request it read. It reads a body by `Content-Length` -/// only, not a chunked one. -pub struct HttpRecorder { - pub port: u16, - requests: Arc>>, - task: JoinHandle<()>, -} - -impl HttpRecorder { - pub async fn start(body: impl Into) -> Result { - let (listener, port) = bind().await?; - let body: Arc = body.into().into(); - let requests = Arc::new(Mutex::new(Vec::new())); - let task_requests = Arc::clone(&requests); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - tokio::spawn(respond( - stream, - Arc::clone(&body), - Arc::clone(&task_requests), - )); - } - }); - Ok(Self { - port, - requests, - task, - }) - } - - /// The requests the server has read, once there are at least `count`. - pub async fn wait_for_requests(&self, count: usize) -> Result> { - testkit::poll_until( - &format!( - "{count} request(s) at the host server on port {}", - self.port - ), - WAIT_TIMEOUT, - WAIT_INTERVAL, - async || { - let requests = self - .requests - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone(); - Ok(if requests.len() >= count { - Poll::Ready(requests) - } else { - Poll::Pending(format!("{} request(s)", requests.len())) - }) - }, - ) - .await - } -} - -impl Drop for HttpRecorder { - fn drop(&mut self) { - self.task.abort(); - } -} - -/// A WebSocket server that completes the handshake, then reads frames until -/// the connection closes, and records whether any was a binary frame. -pub struct WebSocketRecorder { - pub port: u16, - state: Arc, - task: JoinHandle<()>, -} - -#[derive(Default)] -struct WebSocketState { - handshake_seen: AtomicBool, - binary_seen: AtomicBool, - finished: AtomicBool, -} - -impl WebSocketRecorder { - pub async fn start() -> Result { - let (listener, port) = bind().await?; - let state = Arc::new(WebSocketState::default()); - let task_state = Arc::clone(&state); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - tokio::spawn(accept_websocket(stream, Arc::clone(&task_state))); - } - }); - Ok(Self { port, state, task }) - } - - /// Wait until a client completed the handshake. - pub async fn wait_for_handshake(&self) -> Result { - testkit::poll_until( - &format!( - "a WebSocket handshake at the host server on port {}", - self.port - ), - WAIT_TIMEOUT, - WAIT_INTERVAL, - async || { - Ok(if self.state.handshake_seen.load(Ordering::Acquire) { - Poll::Ready(()) - } else { - Poll::Pending("no handshake".to_string()) - }) - }, - ) - .await - } - - /// Wait until a connection that completed the handshake has ended, by a - /// close frame, the client closing it, or [`READ_TIMEOUT`] passing, and - /// return whether a binary frame arrived on it. - pub async fn wait_until_finished(&self) -> Result { - testkit::poll_until( - &format!( - "the WebSocket connection at the host server on port {} to end", - self.port - ), - READ_TIMEOUT + WAIT_TIMEOUT, - WAIT_INTERVAL, - async || { - Ok(if self.state.finished.load(Ordering::Acquire) { - Poll::Ready(self.state.binary_seen.load(Ordering::Acquire)) - } else { - Poll::Pending("the connection is open".to_string()) - }) - }, - ) - .await - } -} - -impl Drop for WebSocketRecorder { - fn drop(&mut self) { - self.task.abort(); - } -} - async fn bind() -> Result<(TcpListener, u16)> { let listener = TcpListener::bind(("0.0.0.0", 0)) .await @@ -187,105 +39,6 @@ async fn bind() -> Result<(TcpListener, u16)> { Ok((listener, port)) } -/// Read one request, headers and `Content-Length` body, record it, and -/// answer with `body`. -async fn respond(mut stream: TcpStream, body: Arc, requests: Arc>>) { - let _ = stream.set_nodelay(true); - let Some(request) = read_request(&mut stream).await else { - return; - }; - requests - .lock() - .unwrap_or_else(PoisonError::into_inner) - .push(request); - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - let _ = stream.write_all(response.as_bytes()).await; - let _ = stream.shutdown().await; -} - -async fn read_request(stream: &mut TcpStream) -> Option { - let mut request = Vec::new(); - let mut buffer = [0_u8; 4096]; - loop { - if let Some(end) = find(&request, b"\r\n\r\n") { - let head = String::from_utf8_lossy(&request[..end]).into_owned(); - let length = header(&head, "content-length") - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); - if request.len() >= end + 4 + length { - let body = - String::from_utf8_lossy(&request[end + 4..end + 4 + length]).into_owned(); - return Some(Request { head, body }); - } - } - match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut buffer)).await { - Ok(Ok(read)) if read > 0 && request.len() < MAX_REQUEST => { - request.extend_from_slice(&buffer[..read]); - } - _ => return None, - } - } -} - -async fn accept_websocket(mut stream: TcpStream, state: Arc) { - let _ = stream.set_nodelay(true); - let Some(request) = read_request(&mut stream).await else { - return; - }; - let Some(key) = header(&request.head, "sec-websocket-key") else { - return; - }; - // The supervisor checks Sec-WebSocket-Accept, so compute it properly. - // SHA-1 is what RFC 6455 specifies for it; it protects nothing here. - let mut hasher = Sha1::new(); - hasher.update(key.as_bytes()); - hasher.update(b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11"); - let accept = base64::engine::general_purpose::STANDARD.encode(hasher.finalize()); - let response = format!( - "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept}\r\n\r\n" - ); - if stream.write_all(response.as_bytes()).await.is_err() { - return; - } - state.handshake_seen.store(true, Ordering::Release); - - let _ = tokio::time::timeout(READ_TIMEOUT, read_frames(&mut stream, &state)).await; - state.finished.store(true, Ordering::Release); -} - -/// Read frames until a close frame or the end of the stream, and record a -/// binary frame. -async fn read_frames(stream: &mut TcpStream, state: &WebSocketState) -> std::io::Result<()> { - loop { - let mut frame_header = [0_u8; 2]; - stream.read_exact(&mut frame_header).await?; - let opcode = frame_header[0] & 0x0f; - let length = match frame_header[1] & 0x7f { - 126 => u64::from(stream.read_u16().await?), - 127 => stream.read_u64().await?, - length => u64::from(length), - }; - if opcode == 0x02 { - state.binary_seen.store(true, Ordering::Release); - } - if opcode == 0x08 { - return Ok(()); - } - let mask = if frame_header[1] & 0x80 == 0 { 0 } else { 4 }; - let skipped = tokio::io::copy( - &mut (&mut *stream).take(mask + length), - &mut tokio::io::sink(), - ) - .await?; - if skipped < mask + length { - return Ok(()); - } - } -} - fn find(haystack: &[u8], needle: &[u8]) -> Option { haystack .windows(needle.len()) @@ -301,79 +54,3 @@ fn header(head: &str, name: &str) -> Option { .then(|| value.trim().to_string()) }) } - -#[cfg(test)] -mod tests { - use super::*; - - /// Complete a handshake with `recorder`, send `frames`, and close. - async fn send_frames(recorder: &WebSocketRecorder, frames: &[(u8, Vec)]) { - let mut stream = TcpStream::connect(("127.0.0.1", recorder.port)) - .await - .unwrap(); - stream.set_nodelay(true).unwrap(); - stream - .write_all( - b"GET / HTTP/1.1\r\nHost: test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\ - Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n", - ) - .await - .unwrap(); - let mut response = Vec::new(); - while find(&response, b"\r\n\r\n").is_none() { - let mut buffer = [0_u8; 1024]; - let read = stream.read(&mut buffer).await.unwrap(); - assert!(read > 0, "closed during the handshake"); - response.extend_from_slice(&buffer[..read]); - } - let response = String::from_utf8_lossy(&response); - assert!(response.starts_with("HTTP/1.1 101"), "{response}"); - // The RFC 6455 sample key's accept value. - assert!( - response.contains("Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo="), - "{response}" - ); - for (opcode, payload) in frames { - stream.write_all(&frame(*opcode, payload)).await.unwrap(); - } - stream.write_all(&frame(0x08, &[])).await.unwrap(); - } - - /// A final, masked client frame. - fn frame(opcode: u8, payload: &[u8]) -> Vec { - let mask = [1_u8, 2, 3, 4]; - let mut frame = vec![0x80 | opcode]; - match payload.len() { - length @ 0..=125 => frame.push(0x80 | u8::try_from(length).unwrap()), - length => { - frame.push(0x80 | 126); - frame.extend_from_slice(&u16::try_from(length).unwrap().to_be_bytes()); - } - } - frame.extend_from_slice(&mask); - frame.extend(payload.iter().enumerate().map(|(i, b)| b ^ mask[i % 4])); - frame - } - - #[tokio::test] - async fn websocket_recorder_sees_a_binary_frame_after_a_long_text_frame() { - let recorder = WebSocketRecorder::start().await.unwrap(); - send_frames( - &recorder, - &[(0x01, vec![b'x'; 300]), (0x02, b"binary".to_vec())], - ) - .await; - assert!(recorder.wait_until_finished().await.unwrap()); - } - - #[tokio::test] - async fn websocket_recorder_sees_no_binary_frame_in_text_frames() { - let recorder = WebSocketRecorder::start().await.unwrap(); - send_frames( - &recorder, - &[(0x01, vec![b'x'; 300]), (0x01, b"text".to_vec())], - ) - .await; - assert!(!recorder.wait_until_finished().await.unwrap()); - } -} diff --git a/tests/support/host-server/src/websocket.rs b/tests/support/host-server/src/websocket.rs new file mode 100644 index 0000000000..28426438c7 --- /dev/null +++ b/tests/support/host-server/src/websocket.rs @@ -0,0 +1,219 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use testkit::{Poll, Result}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::TcpStream; +use tokio::task::JoinHandle; + +use crate::http::read_request; +use crate::{READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, header}; + +/// A WebSocket server that completes the handshake, then reads frames until +/// the connection closes, and records whether any was a binary frame. +pub struct WebSocketRecorder { + pub port: u16, + state: Arc, + task: JoinHandle<()>, +} + +#[derive(Default)] +struct WebSocketState { + handshake_seen: AtomicBool, + binary_seen: AtomicBool, + finished: AtomicBool, +} + +impl WebSocketRecorder { + pub async fn start() -> Result { + let (listener, port) = bind().await?; + let state = Arc::new(WebSocketState::default()); + let task_state = Arc::clone(&state); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(accept_websocket(stream, Arc::clone(&task_state))); + } + }); + Ok(Self { port, state, task }) + } + + /// Wait until a client completed the handshake. + pub async fn wait_for_handshake(&self) -> Result { + testkit::poll_until( + &format!( + "a WebSocket handshake at the host server on port {}", + self.port + ), + WAIT_TIMEOUT, + WAIT_INTERVAL, + async || { + Ok(if self.state.handshake_seen.load(Ordering::Acquire) { + Poll::Ready(()) + } else { + Poll::Pending("no handshake".to_string()) + }) + }, + ) + .await + } + + /// Wait until a connection that completed the handshake has ended, by a + /// close frame, the client closing it, or [`READ_TIMEOUT`] passing, and + /// return whether a binary frame arrived on it. + pub async fn wait_until_finished(&self) -> Result { + testkit::poll_until( + &format!( + "the WebSocket connection at the host server on port {} to end", + self.port + ), + READ_TIMEOUT + WAIT_TIMEOUT, + WAIT_INTERVAL, + async || { + Ok(if self.state.finished.load(Ordering::Acquire) { + Poll::Ready(self.state.binary_seen.load(Ordering::Acquire)) + } else { + Poll::Pending("the connection is open".to_string()) + }) + }, + ) + .await + } +} + +impl Drop for WebSocketRecorder { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn accept_websocket(mut stream: TcpStream, state: Arc) { + let _ = stream.set_nodelay(true); + let Some(request) = read_request(&mut stream).await else { + return; + }; + let Some(key) = header(&request.head, "sec-websocket-key") else { + return; + }; + // The supervisor checks Sec-WebSocket-Accept, so compute it properly. + let accept = tungstenite::handshake::derive_accept_key(key.as_bytes()); + let response = format!( + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept}\r\n\r\n" + ); + if stream.write_all(response.as_bytes()).await.is_err() { + return; + } + state.handshake_seen.store(true, Ordering::Release); + + let _ = tokio::time::timeout(READ_TIMEOUT, read_frames(&mut stream, &state)).await; + state.finished.store(true, Ordering::Release); +} + +/// Read frames until a close frame or the end of the stream, and record a +/// binary frame. +async fn read_frames(stream: &mut TcpStream, state: &WebSocketState) -> std::io::Result<()> { + loop { + let mut frame_header = [0_u8; 2]; + stream.read_exact(&mut frame_header).await?; + let opcode = frame_header[0] & 0x0f; + let length = match frame_header[1] & 0x7f { + 126 => u64::from(stream.read_u16().await?), + 127 => stream.read_u64().await?, + length => u64::from(length), + }; + if opcode == 0x02 { + state.binary_seen.store(true, Ordering::Release); + } + if opcode == 0x08 { + return Ok(()); + } + let mask = if frame_header[1] & 0x80 == 0 { 0 } else { 4 }; + let skipped = tokio::io::copy( + &mut (&mut *stream).take(mask + length), + &mut tokio::io::sink(), + ) + .await?; + if skipped < mask + length { + return Ok(()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::find; + + /// Complete a handshake with `recorder`, send `frames`, and close. + async fn send_frames(recorder: &WebSocketRecorder, frames: &[(u8, Vec)]) { + let mut stream = TcpStream::connect(("127.0.0.1", recorder.port)) + .await + .unwrap(); + stream.set_nodelay(true).unwrap(); + stream + .write_all( + b"GET / HTTP/1.1\r\nHost: test\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\ + Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n", + ) + .await + .unwrap(); + let mut response = Vec::new(); + while find(&response, b"\r\n\r\n").is_none() { + let mut buffer = [0_u8; 1024]; + let read = stream.read(&mut buffer).await.unwrap(); + assert!(read > 0, "closed during the handshake"); + response.extend_from_slice(&buffer[..read]); + } + let response = String::from_utf8_lossy(&response); + assert!(response.starts_with("HTTP/1.1 101"), "{response}"); + // The RFC 6455 sample key's accept value. + assert!( + response.contains("Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo="), + "{response}" + ); + for (opcode, payload) in frames { + stream.write_all(&frame(*opcode, payload)).await.unwrap(); + } + stream.write_all(&frame(0x08, &[])).await.unwrap(); + } + + /// A final, masked client frame. + fn frame(opcode: u8, payload: &[u8]) -> Vec { + let mask = [1_u8, 2, 3, 4]; + let mut frame = vec![0x80 | opcode]; + match payload.len() { + length @ 0..=125 => frame.push(0x80 | u8::try_from(length).unwrap()), + length => { + frame.push(0x80 | 126); + frame.extend_from_slice(&u16::try_from(length).unwrap().to_be_bytes()); + } + } + frame.extend_from_slice(&mask); + frame.extend(payload.iter().enumerate().map(|(i, b)| b ^ mask[i % 4])); + frame + } + + #[tokio::test] + async fn websocket_recorder_sees_a_binary_frame_after_a_long_text_frame() { + let recorder = WebSocketRecorder::start().await.unwrap(); + send_frames( + &recorder, + &[(0x01, vec![b'x'; 300]), (0x02, b"binary".to_vec())], + ) + .await; + assert!(recorder.wait_until_finished().await.unwrap()); + } + + #[tokio::test] + async fn websocket_recorder_sees_no_binary_frame_in_text_frames() { + let recorder = WebSocketRecorder::start().await.unwrap(); + send_frames( + &recorder, + &[(0x01, vec![b'x'; 300]), (0x01, b"text".to_vec())], + ) + .await; + assert!(!recorder.wait_until_finished().await.unwrap()); + } +} From d24635532759feedd7a50e932159f52642805bf3 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 10:32:40 -0800 Subject: [PATCH 15/16] fix(e2e): fail the external-driver render when the host gateway alias is missing The post-renderer copied the host gateway IP whenever the chart rendered any hostAliases. It now matches the host.openshell.internal entry, and fails if the lane set server.hostGatewayIP but no such entry rendered. Signed-off-by: Matthew Grossman --- .../openshell-external-compute-driver/post-renderer.sh | 10 ++++++++-- e2e/with-kube-gateway.sh | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh index b8142c72e4..24faf26b62 100755 --- a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh +++ b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh @@ -18,10 +18,16 @@ tee "${work_dir}/rendered.yaml" >/dev/null # server.hostGatewayIP renders the gateway pod's only hostAliases entry # (deploy/helm/openshell/templates/_gateway-workload.tpl), which the -# host-gateway component copies from hostAliases.0.ip. -if grep -q '^ *hostAliases:' "${work_dir}/rendered.yaml"; then +# host-gateway component copies from hostAliases.0.ip. The lane exports +# OPENSHELL_E2E_EXPECTED_HOST_GATEWAY_IP when it sets server.hostGatewayIP, so +# a chart change that moves the alias fails here instead of leaving the +# external driver without the IP. +if grep -q '^ *- host\.openshell\.internal$' "${work_dir}/rendered.yaml"; then cp -R "${plugin_dir}/host-gateway" "${work_dir}/host-gateway" printf 'components:\n - host-gateway\n' >>"${work_dir}/kustomization.yaml" +elif [ -n "${OPENSHELL_E2E_EXPECTED_HOST_GATEWAY_IP:-}" ]; then + echo "ERROR: server.hostGatewayIP is set, but the rendered chart has no host.openshell.internal hostAliases entry." >&2 + exit 1 fi kubectl kustomize "${work_dir}" diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 31c0771dc8..b85e72b579 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -1331,6 +1331,7 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then exit 2 fi export HELM_PLUGINS="${ROOT}/e2e/helm-plugins" + export OPENSHELL_E2E_EXPECTED_HOST_GATEWAY_IP="${HOST_GATEWAY_IP}" helm_post_renderer_args+=( --post-renderer openshell-external-compute-driver ) From e22087fdb6512cd2fe0d26d43525e71489194d17 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 9 Nov 2026 10:32:40 -0800 Subject: [PATCH 16/16] test(testkit): split provider install into profile import and provider create import_profile reads the profile's id from its YAML and returns it, and create_provider takes the provider name and type separately, so the provider name no longer has to match the profile id. install_provider does both. Document that policy::allow merges binaries under one name and that an override Python image must use a PYTHON_BINARIES path. Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 4 +- tests/suites/drivers/podman/Cargo.lock | 1 + tests/suites/features/Cargo.lock | 4 +- tests/support/Cargo.lock | 4 +- tests/testkit/Cargo.lock | 1 + tests/testkit/README.md | 17 +++-- tests/testkit/testkit/Cargo.toml | 3 +- tests/testkit/testkit/src/context.rs | 3 +- tests/testkit/testkit/src/image.rs | 7 +- tests/testkit/testkit/src/lib.rs | 2 +- tests/testkit/testkit/src/policy.rs | 4 ++ tests/testkit/testkit/src/provider.rs | 93 +++++++++++++++++--------- 12 files changed, 95 insertions(+), 48 deletions(-) diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 43ec90afd7..0d4ab62225 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -1205,10 +1205,9 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ - "base64 0.22.1", "openshell-testkit", - "sha1 0.10.6", "tokio", + "tungstenite", ] [[package]] @@ -2267,6 +2266,7 @@ dependencies = [ "libc", "libtest-mimic", "linkme", + "noyalib", "openshell-bootstrap", "openshell-core", "openshell-policy", diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index 67042d4d88..6657ca1da6 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2255,6 +2255,7 @@ dependencies = [ "libc", "libtest-mimic", "linkme", + "noyalib", "openshell-bootstrap", "openshell-core", "openshell-policy", diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 9023943a1e..bcc428d57d 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1211,10 +1211,9 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ - "base64 0.22.1", "openshell-testkit", - "sha1 0.10.6", "tokio", + "tungstenite", ] [[package]] @@ -2313,6 +2312,7 @@ dependencies = [ "libc", "libtest-mimic", "linkme", + "noyalib", "openshell-bootstrap", "openshell-core", "openshell-policy", diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 9f889d6bb7..94cd21327d 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -1203,10 +1203,9 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ - "base64 0.22.1", "openshell-testkit", - "sha1 0.10.6", "tokio", + "tungstenite", ] [[package]] @@ -2274,6 +2273,7 @@ dependencies = [ "libc", "libtest-mimic", "linkme", + "noyalib", "openshell-bootstrap", "openshell-core", "openshell-policy", diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index 383f9f2454..a02c314451 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -2246,6 +2246,7 @@ dependencies = [ "libc", "libtest-mimic", "linkme", + "noyalib", "openshell-bootstrap", "openshell-core", "openshell-policy", diff --git a/tests/testkit/README.md b/tests/testkit/README.md index d68a1557f8..d6ea00f73b 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -79,7 +79,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `testkit::python_image()` for sandboxes that run `python3`, curl, git, or openssl: `OPENSHELL_TEST_PYTHON_IMAGE`, or `openshell/e2e-python:dev` (`e2e/python/Dockerfile.workload`). The tmachine installers load it into - every environment; `mise run e2e:workload:build` builds it locally. + every environment; `mise run e2e:workload:build` builds it locally. An + override image must run `python3` from one of `testkit::PYTHON_BINARIES`. - `cx.cli([...])` to run the installed CLI against the same gateway, with `success()`, `failure()`, and `json()` on the output. `cx.cli_handle()` returns that CLI for code that can't borrow `cx`, such as a cleanup hook. @@ -87,13 +88,15 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `testkit::policy` for sandbox policies: `policy::base()` reads `testkit/src/base-policy.yaml`, which allows no network access, and `policy::allow(&mut policy, name, host, port, binaries)` adds an endpoint - that `policy::Endpoint` configures (`.rest(Access::Full)`, - `.credential_binding(provider)`, ...). Pass the result to - `cx.sandbox(..).policy(policy)`, or `policy::to_yaml` for the CLI. + (binaries apply to every endpoint under `name`) that `policy::Endpoint` + configures (`.rest(Access::Full)`, `.credential_binding(provider)`, ...). + Pass the result to `cx.sandbox(..).policy(policy)`, or `policy::to_yaml` + for the CLI. `testkit::PYTHON_BINARIES` names `python3` in `python_image()` images. -- `testkit::install_provider(cx, name, profile, credentials)` to import a - provider profile, whose `id` must be `name`, and create a provider of it, - with cleanup for both. +- `testkit::import_profile(cx, profile)` to import a provider profile and + return its `id`, and `testkit::create_provider(cx, name, type, credentials)` + to create a provider, each with cleanup. `testkit::install_provider(cx, name, + profile, credentials)` does both. - `testkit::YamlFile` for YAML a CLI command reads from a file. - `cx.client()` for SDK calls testkit does not wrap. - `error.grpc_code()` for the gRPC status code (`testkit::GrpcCode`) of a diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index ae8ad5c9ad..abfa8cf97b 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -26,8 +26,9 @@ libtest-mimic = "0.8" linkme = "0.3" quick-junit = "0.8" rand = "0.9" -serde = "1" +serde = { version = "1", features = ["derive"] } serde_json = "1" +serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } tempfile = "3" tokio = { version = "1.43", features = ["macros", "process", "rt", "time"] } tonic = "0.14" diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index d5e6c7703e..3ebde90d1e 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -390,7 +390,8 @@ impl SandboxBuilder { /// as the SDK builds one from `spec` alone. The spec is destructured without /// `..`, so a field added to the SDK's spec fails to compile here instead of /// being dropped. [`SandboxBuilder::create`] rejects a GPU or service -/// exposures before calling this. +/// exposures before calling this. A policy field on the SDK's spec would +/// replace this (#5410). fn policy_create_request( spec: SandboxSpec, policy: proto::SandboxPolicy, diff --git a/tests/testkit/testkit/src/image.rs b/tests/testkit/testkit/src/image.rs index 08e13825a0..07a6e2d937 100644 --- a/tests/testkit/testkit/src/image.rs +++ b/tests/testkit/testkit/src/image.rs @@ -9,7 +9,9 @@ const BASE_IMAGE_ENV: &str = "OPENSHELL_TEST_BASE_IMAGE"; /// nvcr.io. const DEFAULT_BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; -/// Overrides [`DEFAULT_PYTHON_IMAGE`]. +/// Overrides [`DEFAULT_PYTHON_IMAGE`]. The image must run `python3` from one +/// of [`PYTHON_BINARIES`]; otherwise policies that allow those paths don't +/// match it, and its connections are denied. const PYTHON_IMAGE_ENV: &str = "OPENSHELL_TEST_PYTHON_IMAGE"; /// The workload image for tests that run `python3`, curl, git or openssl in a /// sandbox, built from `e2e/python/Dockerfile.workload`. It isn't published: @@ -19,7 +21,8 @@ const DEFAULT_PYTHON_IMAGE: &str = "openshell/e2e-python:dev"; /// The paths `python3` runs from in [`python_image`] images, as network /// policy binaries: the system Python, the pullable uv image's Python, and the -/// workload image's uv-installed Python behind its virtualenv. +/// workload image's uv-installed Python behind its virtualenv. Add a path +/// here for an image that installs Python elsewhere. pub const PYTHON_BINARIES: &[&str] = &[ "/usr/bin/python*", "/usr/local/bin/python*", diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 6a57c71420..0dc52e6846 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -53,7 +53,7 @@ pub use image::{PYTHON_BINARIES, base_image, python_image}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until}; pub use program::main; -pub use provider::install_provider; +pub use provider::{create_provider, import_profile, install_provider}; pub use ready::check as check_ready; pub use registry::Test; pub use run::{TestInfo, run}; diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs index 454cc4fb52..ab7be7cae3 100644 --- a/tests/testkit/testkit/src/policy.rs +++ b/tests/testkit/testkit/src/policy.rs @@ -28,6 +28,10 @@ pub fn base() -> proto::SandboxPolicy { /// Let `binaries` connect to `host:port` under the network policy `name`, /// adding the policy if it doesn't exist. Returns the new endpoint, which /// [`Endpoint`] configures further. +/// +/// A policy's binaries apply to all its endpoints, so calling this again with +/// the same `name` adds `binaries` to the existing ones, and every endpoint +/// under `name` allows all of them. Use a different `name` to keep them apart. pub fn allow<'a>( policy: &'a mut proto::SandboxPolicy, name: &str, diff --git a/tests/testkit/testkit/src/provider.rs b/tests/testkit/testkit/src/provider.rs index 0e6f3a12bd..062642fb6a 100644 --- a/tests/testkit/testkit/src/provider.rs +++ b/tests/testkit/testkit/src/provider.rs @@ -1,46 +1,58 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use serde::Deserialize; + use crate::context::Context; -use crate::error::{Result, ResultExt as _}; +use crate::error::{Error, Result, ResultExt as _}; use crate::yaml_file::YamlFile; -/// Import `profile`, a provider profile whose `id` is `name`, and create a -/// provider called `name` of that type with `credentials`, as pairs of -/// environment variable and value. Cleanup deletes the provider, then the -/// profile, after the test's sandboxes. +/// Import `profile`, a provider profile in YAML, and return its `id`, the +/// provider type it defines. Cleanup deletes the profile after the test's +/// sandboxes and the providers created after it. +pub async fn import_profile(cx: &Context, profile: &str) -> Result { + #[derive(Deserialize)] + struct Profile { + id: String, + } + let id = serde_yml::from_str::(profile) + .map_err(|error| Error::from_error(&error)) + .context("read the provider profile's id")? + .id; + // Deleting a missing profile succeeds, so register the delete before the + // import. + on_cleanup_delete(cx, "profile", &id); + let file = YamlFile::new(profile)?; + cx.cli(["profile", "import", "--file", file.path()?]) + .await? + .success() + .context(format!("import profile {id}"))?; + Ok(id) +} + +/// Create a provider called `name` of `provider_type` with `credentials`, as +/// pairs of environment variable and value. Cleanup deletes the provider +/// after the test's sandboxes. /// /// The CLI reads the credential values from its environment, so they aren't /// in the reported command. -pub async fn install_provider( +pub async fn create_provider( cx: &Context, name: &str, - profile: &str, + provider_type: &str, credentials: &[(&str, &str)], ) -> Result { - crate::ensure!( - profile - .lines() - .any(|line| line.trim_end() == format!("id: {name}")), - "the profile for provider {name} must have `id: {name}`" - ); - // Both deletes succeed for a missing resource, so register them before - // the creates. Hooks run last registered first. - for kind in ["profile", "provider"] { - let cli = cx.cli_handle(); - let owned = name.to_string(); - cx.on_cleanup(format!("delete {kind} {name}"), async move || { - cli.run([kind, "delete", &owned]).await?.success()?; - Ok(()) - }); - } - - let file = YamlFile::new(profile)?; - cx.cli(["profile", "import", "--file", file.path()?]) - .await? - .success() - .context(format!("import profile {name}"))?; - let mut args = vec!["provider", "create", "--name", name, "--type", name]; + // Deleting a missing provider succeeds, so register the delete before the + // create. + on_cleanup_delete(cx, "provider", name); + let mut args = vec![ + "provider", + "create", + "--name", + name, + "--type", + provider_type, + ]; for (key, _) in credentials { args.extend(["--credential", key]); } @@ -50,3 +62,24 @@ pub async fn install_provider( .context(format!("create provider {name}"))?; Ok(()) } + +/// Import `profile` with [`import_profile`] and create a provider called +/// `name` of its type with [`create_provider`]. +pub async fn install_provider( + cx: &Context, + name: &str, + profile: &str, + credentials: &[(&str, &str)], +) -> Result { + let provider_type = import_profile(cx, profile).await?; + create_provider(cx, name, &provider_type, credentials).await +} + +fn on_cleanup_delete(cx: &Context, kind: &'static str, name: &str) { + let cli = cx.cli_handle(); + let owned = name.to_string(); + cx.on_cleanup(format!("delete {kind} {name}"), async move || { + cli.run([kind, "delete", &owned]).await?.success()?; + Ok(()) + }); +}