From 2df7b6fe62f121a5586bbcb38f5943dcf56906dd Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 3 Nov 2026 17:11:08 +0530 Subject: [PATCH 1/5] fix(supervisor): reject unsupported live filesystem policy changes Compare reloads with the filesystem grants actually applied at startup. Reject changed grants before replacing the running policy, retain the active policy and report the failed load through existing status handling. Preserve network-only reloads and equivalent path ordering. Signed-off-by: Shiju --- .../src/config_runtime.rs | 228 +++++++++++++++--- .../how-it-works/policies/manage-policies.mdx | 2 +- 2 files changed, 201 insertions(+), 29 deletions(-) diff --git a/crates/openshell-supervisor/src/config_runtime.rs b/crates/openshell-supervisor/src/config_runtime.rs index ecda1a7194..a4b3bd539d 100644 --- a/crates/openshell-supervisor/src/config_runtime.rs +++ b/crates/openshell-supervisor/src/config_runtime.rs @@ -26,12 +26,12 @@ use tracing::{debug, info, warn}; use crate::provider_readiness::EnvironmentIdentity; use crate::{ CapturedProviderEnvironment, FailedRuntimeRevision, GatewayRuntimeFailureDisposition, - InitialPollDisposition, LoadedPolicyOrigin, MiddlewareAuthentication, - MiddlewareRegistryReconciliation, MiddlewareRegistryStatus, MiddlewareReloadContext, - PolicyGatewayClient, PolicyPollLoopContext, PolicyStatusUpdate, RejectedPolicyGeneration, - apply_gateway_runtime_reload_failure, apply_policy_validation_failure, - emit_policy_validation_failure, endpoint_status, enqueue_policy_status, - gateway_policy_runtime_needs_reconciliation, initial_poll_disposition, + GatewayRuntimeReloadError, InitialPollDisposition, LoadedPolicyOrigin, + MiddlewareAuthentication, MiddlewareRegistryReconciliation, MiddlewareRegistryStatus, + MiddlewareReloadContext, PolicyGatewayClient, PolicyPollLoopContext, PolicyStatusUpdate, + RejectedPolicyGeneration, apply_gateway_runtime_reload_failure, + apply_policy_validation_failure, emit_policy_validation_failure, endpoint_status, + enqueue_policy_status, gateway_policy_runtime_needs_reconciliation, initial_poll_disposition, middleware_registry_needs_rebuild, next_poll_delay, ocsf_ctx, provider_environment_is_installable, reconcile_middleware_registry, reload_gateway_configuration_runtime, retain_extension_credentials, run_policy_status_reporter, @@ -209,6 +209,34 @@ impl ConfigRuntime { } } + fn validate_live_filesystem_policy( + &self, + candidate: Option<&openshell_core::proto::SandboxPolicy>, + ) -> Result<()> { + let (Some(applied), Some(candidate)) = (&self.endpoint_policy, candidate) else { + return Ok(()); + }; + let same_paths = |left: &[String], right: &[String]| { + left.iter().collect::>() == right.iter().collect() + }; + let unchanged = match (&applied.filesystem, &candidate.filesystem) { + (Some(applied), Some(candidate)) => { + applied.include_workdir == candidate.include_workdir + && same_paths(&applied.read_only, &candidate.read_only) + && same_paths(&applied.read_write, &candidate.read_write) + } + (None, None) => true, + _ => false, + }; + if !unchanged { + // Reloading the network engine cannot change installed filesystem permissions. + miette::bail!( + "filesystem policy cannot be changed on a running sandbox; recreate the sandbox to apply filesystem changes" + ); + } + Ok(()) + } + /// Bring the runtime to one sandbox configuration snapshot. /// /// The caller reports the outcome, then records an applied snapshot with @@ -430,28 +458,34 @@ impl ConfigRuntime { let mut runtime_error = String::new(); if policy_runtime_changed { let pid = ctx.entrypoint_pid.load(Ordering::Acquire); - let runtime_result = reload_gateway_configuration_runtime( - &ctx.opa_engine, - result.policy.as_ref(), - pid, - MiddlewareReloadContext { - desired_services: &result.supervisor_middleware_services, - authentication: &MiddlewareAuthentication { - credentials: middleware_credentials.clone(), - enabled: result.extension_authentication_enabled, - }, - registry_changed: middleware_registry_changed, - connector: &ctx.middleware_connector, - }, - ctx.vm_identity, - || { - if let Some(prepared) = prepared_provider.as_ref() { - ctx.provider_credentials - .install_prepared(&prepared.credentials); - } - }, - ) - .await; + let runtime_result = match self.validate_live_filesystem_policy(result.policy.as_ref()) + { + Ok(()) => { + reload_gateway_configuration_runtime( + &ctx.opa_engine, + result.policy.as_ref(), + pid, + MiddlewareReloadContext { + desired_services: &result.supervisor_middleware_services, + authentication: &MiddlewareAuthentication { + credentials: middleware_credentials.clone(), + enabled: result.extension_authentication_enabled, + }, + registry_changed: middleware_registry_changed, + connector: &ctx.middleware_connector, + }, + ctx.vm_identity, + || { + if let Some(prepared) = prepared_provider.as_ref() { + ctx.provider_credentials + .install_prepared(&prepared.credentials); + } + }, + ) + .await + } + Err(error) => Err(GatewayRuntimeReloadError::PolicyValidation(error)), + }; match runtime_result { Ok(generation) => { @@ -3371,6 +3405,144 @@ mod tests { } } + #[tokio::test] + async fn live_filesystem_changes_reject_streamed_policy_without_advancing_active_revision() { + use openshell_core::proto::{ConfigurationAdmissionState, FilesystemPolicy, PolicySource}; + + for source in [PolicySource::Sandbox, PolicySource::Global] { + for change in ["read_only", "read_write", "from_none"] { + let mut policy = proto_policy_fixture(); + if change == "from_none" { + policy.filesystem = None; + } else { + // Startup may already have added the proxy's filesystem prerequisites. + policy + .filesystem + .as_mut() + .unwrap() + .read_only + .push("/proxy-baseline".into()); + } + let initial = settings_poll_result(Some(policy.clone()), 1, source); + let engine = Arc::new(OpaEngine::from_proto(&policy).unwrap()); + let mut ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + ctx.endpoint_policy = Some(policy.clone()); + let (client, _polls, _reports) = scripted_policy_gateway(); + let mut runtime = started_runtime(&ctx, &initial); + let original_generation = engine.current_generation(); + let original_revision = runtime.applied_revision.clone(); + let filesystem = policy + .filesystem + .get_or_insert_with(FilesystemPolicy::default); + if change == "read_write" { + filesystem.read_write.push("/extra-readwrite".into()); + } else { + filesystem.read_only.push("/extra-readonly".into()); + } + let mut candidate = settings_poll_result(Some(policy), 2, source); + let fail_closed = source == PolicySource::Global && change == "read_only"; + candidate.policy_validation_failure_mode = if fail_closed { + PolicyValidationFailureMode::FailClosed + } else { + PolicyValidationFailureMode::RetainLastValid + }; + let result = apply_with_provider(&mut runtime, &ctx, &client, &candidate).await; + + assert_eq!( + result.outcome, + if fail_closed { + ConfigApplyOutcome::FailedClosed + } else { + ConfigApplyOutcome::FailedRetainedLastKnownGood + }, + "{source:?}: {change}" + ); + assert_eq!( + result.admission.state, + i32::from(ConfigurationAdmissionState::Rejected) + ); + assert!( + result + .failure + .as_ref() + .unwrap() + .message + .contains("filesystem") + ); + if source == PolicySource::Sandbox { + let status = result.policy_load.unwrap(); + assert_eq!(status.version, 2); + assert!(!status.loaded); + assert!(status.error.contains("filesystem")); + } + assert_eq!(runtime.applied_revision, original_revision); + assert_eq!(runtime.policy_version, 1); + assert_eq!(runtime.policy_hash, initial.policy_hash); + assert_eq!(runtime.endpoint_policy, initial.policy); + assert_eq!(engine.fail_closed_reason().is_some(), fail_closed); + assert_eq!( + engine.current_generation(), + original_generation + u64::from(fail_closed) + ); + } + } + } + + #[tokio::test] + async fn live_filesystem_change_poll_reports_failure_then_accepts_network_only_repair() { + use openshell_core::proto::PolicySource; + + let policy = proto_policy_fixture(); + let initial = settings_poll_result(Some(policy.clone()), 1, PolicySource::Sandbox); + let mut rejected = policy.clone(); + rejected + .filesystem + .as_mut() + .unwrap() + .read_only + .push("/extra-readonly".into()); + let mut rejected = settings_poll_result(Some(rejected), 2, PolicySource::Sandbox); + rejected.policy_validation_failure_mode = PolicyValidationFailureMode::RetainLastValid; + let mut repaired = policy.clone(); + repaired.network_policies = proto_tcp_policy_fixture().network_policies; + repaired.filesystem.as_mut().unwrap().read_only.reverse(); + let repaired = settings_poll_result(Some(repaired), 3, PolicySource::Sandbox); + let engine = Arc::new(OpaEngine::from_proto(&policy).unwrap()); + let mut ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + ctx.endpoint_policy = Some(policy); + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(initial).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + polls.send(rejected).unwrap(); + let (version, loaded, error) = timeout(Duration::from_secs(1), reports.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(version, 2); + assert!(!loaded); + assert!(error.contains("filesystem")); + assert_eq!(engine.current_generation(), 0); + polls.send(repaired).unwrap(); + expect_policy_report(&mut reports, 3).await; + assert_eq!(engine.current_generation(), 1); + handle.abort(); + } + #[tokio::test] async fn rejected_stream_snapshot_preserves_admitted_settings() { use openshell_core::proto::{ConfigApplyOutcome, PolicySource}; diff --git a/docs/how-it-works/policies/manage-policies.mdx b/docs/how-it-works/policies/manage-policies.mdx index c20af7b67b..479e93b469 100644 --- a/docs/how-it-works/policies/manage-policies.mdx +++ b/docs/how-it-works/policies/manage-policies.mdx @@ -191,7 +191,7 @@ running sandbox. The startup sections do not: | Network rules | The sandbox loads the new rules. Connections opened under the previous rules close. | Apply the change, then retry the request. | | Middleware in `network_middlewares` | You can add, remove, or reconfigure built-in middleware and external services already registered with the gateway. | Apply the change with `openshell policy set`. | | External middleware registration | Policy changes cannot register a service or change its gateway connection settings. | Update the gateway configuration and restart the gateway. | -| Added filesystem paths | The saved policy can change, but the running workload keeps its existing filesystem permissions. | Recreate the sandbox. | +| Added filesystem paths | The running sandbox rejects the revision because its filesystem permissions are fixed at startup. | Recreate the sandbox. | | Removed filesystem paths, or changed `include_workdir`, Landlock, or process settings | OpenShell rejects the change after the workload starts. | Recreate the sandbox. | When new network rules take effect, OpenShell closes connections that were From b14b5eba1b94aa927d9ae046a64792e1a4bc555a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 3 Nov 2026 23:14:00 +0000 Subject: [PATCH 2/5] fix(tests): pin the k3s installer to a release (#5223) get.k3s.io serves the latest install.sh from k3s master, so the pinned checksum broke when upstream changed the script. Download install.sh from the release tag matching k3s_version instead and pin its checksum. Signed-off-by: Matthew Grossman Signed-off-by: Shiju --- tests/tmachine/ansible/playbooks/k3s.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/tmachine/ansible/playbooks/k3s.yaml b/tests/tmachine/ansible/playbooks/k3s.yaml index 8666b2ac6f..d27c6c487b 100644 --- a/tests/tmachine/ansible/playbooks/k3s.yaml +++ b/tests/tmachine/ansible/playbooks/k3s.yaml @@ -21,10 +21,12 @@ state: present update_cache: true + # get.k3s.io serves the latest script, so pin the one from the k3s_version + # release tag. Update the checksum whenever k3s_version changes. - name: Download K3s installer ansible.builtin.get_url: - url: https://get.k3s.io - checksum: sha256:e5cc3b3d9dfc1662c2d9be6da5abc9a4cd317d6abc3a5ffc02e3dd3248207fee + url: "https://raw.githubusercontent.com/k3s-io/k3s/v{{ k3s_version | urlencode }}/install.sh" + checksum: sha256:3ce239d57d43b2d836d2b561043433e6decae8b9dc41f5d13908c0fafb0340cd dest: /tmp/install-k3s.sh mode: "0755" From 84933181a25444c18d752ed8146f57339e682dac Mon Sep 17 00:00:00 2001 From: Shiju Date: Wed, 4 Nov 2026 12:42:06 +0530 Subject: [PATCH 3/5] docs: clarify rejected policy revision behavior Signed-off-by: Shiju --- docs/how-it-works/policies/manage-policies.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-it-works/policies/manage-policies.mdx b/docs/how-it-works/policies/manage-policies.mdx index 479e93b469..6c5e04ce2e 100644 --- a/docs/how-it-works/policies/manage-policies.mdx +++ b/docs/how-it-works/policies/manage-policies.mdx @@ -191,7 +191,7 @@ running sandbox. The startup sections do not: | Network rules | The sandbox loads the new rules. Connections opened under the previous rules close. | Apply the change, then retry the request. | | Middleware in `network_middlewares` | You can add, remove, or reconfigure built-in middleware and external services already registered with the gateway. | Apply the change with `openshell policy set`. | | External middleware registration | Policy changes cannot register a service or change its gateway connection settings. | Update the gateway configuration and restart the gateway. | -| Added filesystem paths | The running sandbox rejects the revision because its filesystem permissions are fixed at startup. | Recreate the sandbox. | +| Added filesystem paths | The sandbox rejects the entire revision, including network changes. With the default `fail_closed` failure mode, it blocks network traffic until a valid revision loads. | Recreate the sandbox. | | Removed filesystem paths, or changed `include_workdir`, Landlock, or process settings | OpenShell rejects the change after the workload starts. | Recreate the sandbox. | When new network rules take effect, OpenShell closes connections that were From dbf59381c8f2394ab4985161ef1fc796586fab22 Mon Sep 17 00:00:00 2001 From: Shiju Date: Wed, 11 Nov 2026 20:01:11 +0530 Subject: [PATCH 4/5] fix(supervisor): preserve startup filesystem policy on retries Signed-off-by: Shiju --- .../src/config_runtime.rs | 106 +++++++++++++++++- crates/openshell-supervisor/src/lib.rs | 4 +- 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/crates/openshell-supervisor/src/config_runtime.rs b/crates/openshell-supervisor/src/config_runtime.rs index a4b3bd539d..ca7ea4e2fa 100644 --- a/crates/openshell-supervisor/src/config_runtime.rs +++ b/crates/openshell-supervisor/src/config_runtime.rs @@ -210,12 +210,27 @@ impl ConfigRuntime { } fn validate_live_filesystem_policy( - &self, + installed: Option<&openshell_core::proto::SandboxPolicy>, candidate: Option<&openshell_core::proto::SandboxPolicy>, ) -> Result<()> { - let (Some(applied), Some(candidate)) = (&self.endpoint_policy, candidate) else { + let (Some(applied), Some(candidate)) = (installed, candidate) else { return Ok(()); }; + let mut candidate = candidate.clone(); + if !applied.network_policies.is_empty() { + let (ro, rw) = crate::proxy_baseline_paths(); + // Global policies retain their raw gateway form. Restore only the + // proxy prerequisites that startup actually installed. + crate::enrich_proto_baseline_paths_with(&mut candidate, &ro, &rw, |path| { + applied.filesystem.as_ref().is_some_and(|filesystem| { + filesystem + .read_only + .iter() + .chain(&filesystem.read_write) + .any(|installed| installed == path) + }) + }); + } let same_paths = |left: &[String], right: &[String]| { left.iter().collect::>() == right.iter().collect() }; @@ -458,8 +473,10 @@ impl ConfigRuntime { let mut runtime_error = String::new(); if policy_runtime_changed { let pid = ctx.entrypoint_pid.load(Ordering::Acquire); - let runtime_result = match self.validate_live_filesystem_policy(result.policy.as_ref()) - { + let runtime_result = match Self::validate_live_filesystem_policy( + ctx.endpoint_policy.as_ref(), + result.policy.as_ref(), + ) { Ok(()) => { reload_gateway_configuration_runtime( &ctx.opa_engine, @@ -3495,6 +3512,87 @@ mod tests { } } + #[tokio::test] + async fn global_policy_retry_uses_installed_filesystem_after_missing_initial_acknowledgement() { + use openshell_core::proto::{ConfigApplyOutcome, FilesystemPolicy, PolicySource}; + + for acknowledged in [false, true] { + let mut policy = proto_tcp_policy_fixture(); + policy.filesystem = Some(FilesystemPolicy { + include_workdir: true, + read_only: vec!["/usr".into(), "/tmp".into()], + ..Default::default() + }); + let initial = settings_poll_result(Some(policy.clone()), 1, PolicySource::Global); + let mut installed = policy.clone(); + let (ro, rw) = proxy_baseline_paths(); + assert!(enrich_proto_baseline_paths_with( + &mut installed, + &ro, + &rw, + |path| matches!(path, "/proc" | "/dev/null"), + )); + let engine = Arc::new(OpaEngine::from_proto(&installed).unwrap()); + let mut ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + ctx.endpoint_policy = Some(installed.clone()); + let (client, _polls, _reports) = scripted_policy_gateway(); + // A failed first fetch leaves the runtime unacknowledged; a normal + // acknowledgement instead replaces its endpoint policy with the raw one. + let mut runtime = if acknowledged { + let mut runtime = started_runtime(&ctx, &initial); + runtime.endpoint_policy.clone_from(&initial.policy); + runtime + } else { + ConfigRuntime::new(&ctx) + }; + + let result = apply_with_provider(&mut runtime, &ctx, &client, &initial).await; + assert!(matches!( + result.outcome, + ConfigApplyOutcome::Applied | ConfigApplyOutcome::IgnoredDuplicate + )); + assert!(engine.fail_closed_reason().is_none()); + + let mut network_only = policy; + network_only + .network_policies + .values_mut() + .next() + .unwrap() + .endpoints[0] + .port = 6380; + let network_only = settings_poll_result(Some(network_only), 2, PolicySource::Global); + let result = apply_with_provider(&mut runtime, &ctx, &client, &network_only).await; + assert_eq!(result.outcome, ConfigApplyOutcome::Applied); + assert_eq!(runtime.endpoint_policy, network_only.policy); + assert_eq!(ctx.endpoint_policy, Some(installed)); + assert!(engine.fail_closed_reason().is_none()); + + let mut changed = network_only.policy.clone().unwrap(); + changed + .filesystem + .as_mut() + .unwrap() + .read_only + .push("/app".into()); + let changed = settings_poll_result(Some(changed), 3, PolicySource::Global); + let result = apply_with_provider(&mut runtime, &ctx, &client, &changed).await; + assert_eq!(result.outcome, ConfigApplyOutcome::FailedClosed); + assert_eq!( + runtime.applied_revision, + Some(sandbox_config_revision(&network_only)) + ); + assert!(engine.fail_closed_reason().is_some()); + } + } + #[tokio::test] async fn live_filesystem_change_poll_reports_failure_then_accepts_network_only_repair() { use openshell_core::proto::PolicySource; diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 398c4bef77..b727783752 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -4265,7 +4265,9 @@ struct PolicyPollLoopContext { endpoint_observation_tx: Option, /// Single FIFO consumed by the endpoint status reporter. endpoint_status_rx: Option, - /// Gateway policy currently installed in OPA, absent for local overrides. + /// Policy installed at startup, including its filesystem prerequisites. + /// Kept unchanged when gateway updates replace the policy in OPA. + /// Absent for local overrides. endpoint_policy: Option, /// Session accepted by `ConnectSupervisor`; `None` suspends endpoint reports. supervisor_session_id: tokio::sync::watch::Receiver>, From cdf8e84b2ea1596dcc25a9106cec8da3cbe6c2fc Mon Sep 17 00:00:00 2001 From: Shiju Date: Wed, 11 Nov 2026 20:51:14 +0530 Subject: [PATCH 5/5] test(e2e): isolate credential-gating scenarios Reuse the merged test split from #5264 to avoid the sequential credential test hitting the five-minute Kubernetes timeout. Preserve all scenarios and the existing timeout. Cherry-picked from ef83ee264007c5373bd9fcaa567fd0f26762e69b. Signed-off-by: Shiju --- e2e/rust/tests/credential_gating.rs | 653 ++++++++++++++++------------ 1 file changed, 385 insertions(+), 268 deletions(-) diff --git a/e2e/rust/tests/credential_gating.rs b/e2e/rust/tests/credential_gating.rs index bf0d6ae9b5..110e02cb1e 100644 --- a/e2e/rust/tests/credential_gating.rs +++ b/e2e/rust/tests/credential_gating.rs @@ -4,15 +4,20 @@ #![cfg(feature = "e2e")] //! E2E coverage for credentialed endpoint admission and REST body backstops. +//! +//! Each test installs its own provider and profile under a unique name, so the +//! tests can run in parallel. Each test prints its steps to stderr with the +//! elapsed time, so a timeout shows which step was slow. +use std::fmt::Display; use std::io::Write; use std::process::Stdio; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant}; use base64::Engine as _; -use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; use openshell_e2e::harness::sandbox::SandboxGuard; use sha1::{Digest, Sha1}; use tempfile::NamedTempFile; @@ -20,8 +25,6 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::task::JoinHandle; -const PROFILE_ID: &str = "e2e-credential-gating"; -const PROVIDER_NAME: &str = "e2e-credential-gating"; const TEST_HOST: &str = "host.openshell.internal"; const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; const TEST_SECRET: &str = "e2e-gating-secret-value"; @@ -53,21 +56,68 @@ async fn run_cli(args: &[&str]) -> (bool, String) { ) } +/// Prints a test's steps to stderr with the time since the test started. +#[derive(Clone, Copy)] +struct Progress { + scenario: &'static str, + started: Instant, +} + +impl Progress { + fn start(scenario: &'static str) -> Self { + let progress = Self { + scenario, + started: Instant::now(), + }; + progress.step("start"); + progress + } + + fn step(self, message: impl Display) { + eprintln!( + "credential_gating[{}] +{:.1}s {message}", + self.scenario, + self.started.elapsed().as_secs_f64() + ); + } +} + +fn utf8_path(file: &NamedTempFile) -> Result<&str, String> { + file.path() + .to_str() + .ok_or_else(|| format!("{} is not UTF-8", file.path().display())) +} + /// Retry a delete until it takes effect. /// /// The gateway refuses to delete a provider or a profile that a sandbox still /// references, and sandbox teardown drains asynchronously, so a single attempt -/// can be rejected right after a sandbox was removed. -async fn delete_until_gone(args: &[&str]) -> Result<(), String> { +/// can be rejected right after a sandbox was removed. This runs from `Drop`, +/// so it blocks instead of awaiting. +fn delete_until_gone(args: &[&str]) -> Result<(), String> { const ATTEMPTS: u32 = 40; + let binary = openshell_bin(); let mut last_output = String::new(); for _ in 0..ATTEMPTS { - let (deleted, output) = run_cli(args).await; - if deleted || output.to_lowercase().contains("not found") { - return Ok(()); + match std::process::Command::new(&binary) + .args(args) + .stdin(Stdio::null()) + .output() + { + Ok(output) => { + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + if output.status.success() || combined.to_lowercase().contains("not found") { + return Ok(()); + } + last_output = combined; + } + Err(error) => last_output = format!("spawn openshell CLI: {error}"), } - last_output = output; - tokio::time::sleep(Duration::from_millis(500)).await; + std::thread::sleep(Duration::from_millis(500)); } Err(format!( "'{}' still failing after {ATTEMPTS} attempts:\n{last_output}", @@ -75,28 +125,97 @@ async fn delete_until_gone(args: &[&str]) -> Result<(), String> { )) } -/// Strict teardown for the install paths: every test resource must be gone -/// before it is recreated, otherwise creation fails with "already exists". -async fn ensure_provider_resources_absent() -> Result<(), String> { - delete_until_gone(&["provider", "delete", PROVIDER_NAME]).await?; - delete_until_gone(&["profile", "delete", PROFILE_ID]).await +/// A provider and the profile it uses, both under one name unique to the +/// test. Dropping it deletes both, also when the test fails. +struct GatingProvider { + name: String, + progress: Progress, } -/// Best-effort teardown. Never fails the test: it also runs on the failure -/// path, where the original assertion is the interesting one. -async fn cleanup_provider_resources() { - if let Err(error) = ensure_provider_resources_absent().await { - eprintln!("provider cleanup did not settle: {error}"); +impl GatingProvider { + async fn install_with_endpoints( + progress: Progress, + rest_port: u16, + websocket_port: u16, + ) -> Result { + let name = unique_name(progress.scenario); + let profile = write_provider_profile(&name, rest_port, websocket_port)?; + Self::install(progress, name, &profile).await + } + + async fn install_endpointless(progress: Progress) -> Result { + let name = unique_name(progress.scenario); + let profile = write_endpointless_provider_profile(&name)?; + Self::install(progress, name, &profile).await } + + async fn install( + progress: Progress, + name: String, + profile: &NamedTempFile, + ) -> Result { + progress.step(format_args!("importing profile {name}")); + let (imported, output) = + run_cli(&["profile", "import", "--file", utf8_path(profile)?]).await; + if !imported { + return Err(format!("profile import failed:\n{output}")); + } + let provider = Self { name, progress }; + progress.step(format_args!("creating provider {}", provider.name)); + let credential = format!("{TOKEN_ENV}={TEST_SECRET}"); + let (created, output) = run_cli(&[ + "provider", + "create", + "--name", + &provider.name, + "--type", + &provider.name, + "--credential", + &credential, + ]) + .await; + if !created { + return Err(format!("provider create failed:\n{output}")); + } + Ok(provider) + } +} + +impl Drop for GatingProvider { + fn drop(&mut self) { + self.progress + .step(format_args!("deleting provider and profile {}", self.name)); + for kind in ["provider", "profile"] { + if let Err(error) = delete_until_gone(&[kind, "delete", &self.name]) { + self.progress + .step(format_args!("{kind} cleanup did not settle: {error}")); + } + } + self.progress.step("cleanup finished"); + } +} + +/// A provider and profile name that does not collide with parallel tests or +/// earlier runs against the same gateway. +fn unique_name(scenario: &str) -> String { + format!( + "e2e-cg-{scenario}-{}-{:08x}", + std::process::id(), + rand::random::() + ) } -fn write_provider_profile(rest_port: u16, websocket_port: u16) -> Result { +fn write_provider_profile( + profile_id: &str, + rest_port: u16, + websocket_port: u16, +) -> Result { let mut file = tempfile::Builder::new() .suffix(".yaml") .tempfile() .map_err(|error| format!("create profile: {error}"))?; let profile = format!( - r"id: {PROFILE_ID} + r"id: {profile_id} display_name: E2E Credential Gating category: other credentials: @@ -127,42 +246,13 @@ binaries: Ok(file) } -async fn install_provider(rest_port: u16, websocket_port: u16) -> Result<(), String> { - ensure_provider_resources_absent().await?; - let profile = write_provider_profile(rest_port, websocket_port)?; - let profile_path = profile - .path() - .to_str() - .ok_or_else(|| "profile path is not UTF-8".to_string())?; - let (imported, output) = run_cli(&["profile", "import", "--file", profile_path]).await; - if !imported { - return Err(format!("profile import failed:\n{output}")); - } - let credential = format!("{TOKEN_ENV}={TEST_SECRET}"); - let (created, output) = run_cli(&[ - "provider", - "create", - "--name", - PROVIDER_NAME, - "--type", - PROFILE_ID, - "--credential", - &credential, - ]) - .await; - if !created { - return Err(format!("provider create failed:\n{output}")); - } - Ok(()) -} - -fn write_endpointless_provider_profile() -> Result { +fn write_endpointless_provider_profile(profile_id: &str) -> Result { let mut file = tempfile::Builder::new() .suffix(".yaml") .tempfile() .map_err(|error| format!("create endpointless profile: {error}"))?; let profile = format!( - r"id: {PROFILE_ID} + r"id: {profile_id} display_name: E2E Endpointless Credential Gating category: other credentials: @@ -184,35 +274,6 @@ binaries: Ok(file) } -async fn install_endpointless_provider() -> Result<(), String> { - ensure_provider_resources_absent().await?; - let profile = write_endpointless_provider_profile()?; - let profile_path = profile - .path() - .to_str() - .ok_or_else(|| "endpointless profile path is not UTF-8".to_string())?; - let (imported, output) = run_cli(&["profile", "import", "--file", profile_path]).await; - if !imported { - return Err(format!("endpointless profile import failed:\n{output}")); - } - let credential = format!("{TOKEN_ENV}={TEST_SECRET}"); - let (created, output) = run_cli(&[ - "provider", - "create", - "--name", - PROVIDER_NAME, - "--type", - PROFILE_ID, - "--credential", - &credential, - ]) - .await; - if !created { - return Err(format!("endpointless provider create failed:\n{output}")); - } - Ok(()) -} - #[derive(Clone, Copy)] enum EndpointMode { L4, @@ -232,6 +293,7 @@ fn write_policy( port: u16, mode: EndpointMode, credential_source: CredentialSource, + provider: &str, ) -> Result { let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; let endpoint_options = match mode { @@ -250,11 +312,11 @@ fn write_policy( let credential_binding = match credential_source { CredentialSource::ProviderProfile => String::new(), CredentialSource::PolicyBinding => { - format!(" credential_binding:\n provider: {PROVIDER_NAME}\n") + format!(" credential_binding:\n provider: {provider}\n") } }; let policy = format!( - r#"version: 1 + r"version: 1 filesystem_policy: include_workdir: true read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] @@ -274,7 +336,7 @@ network_policies: - path: /usr/bin/python* - path: /usr/local/bin/python* - path: /sandbox/.uv/python/*/bin/python* -"#, +", ); file.write_all(policy.as_bytes()) .map_err(|error| format!("write policy: {error}"))?; @@ -631,7 +693,15 @@ with socket.create_connection((host, port), timeout=10) as sock: ) } -async fn sandbox_create_failure(policy: &NamedTempFile) -> String { +async fn sandbox_create_failure( + progress: Progress, + label: &str, + provider: &str, + policy: &NamedTempFile, +) -> String { + progress.step(format_args!( + "{label}: creating sandbox, expecting rejection" + )); let policy_path = policy.path().to_str().expect("policy path is UTF-8"); let (success, output) = run_cli(&[ "sandbox", @@ -639,54 +709,71 @@ async fn sandbox_create_failure(policy: &NamedTempFile) -> String { "--policy", policy_path, "--provider", - PROVIDER_NAME, + provider, "--", "echo", "must-not-run", ]) .await; assert!(!success, "sandbox create unexpectedly succeeded:\n{output}"); + progress.step(format_args!("{label}: rejected")); output } +/// Create a sandbox, run one command in it, and delete it. +/// +/// Returns the create output followed by the command output. +async fn run_in_sandbox( + progress: Progress, + label: &str, + create_args: &[&str], + command: &[&str], +) -> Result { + progress.step(format_args!("{label}: creating sandbox")); + let mut sandbox = SandboxGuard::create(create_args).await?; + progress.step(format_args!( + "{label}: sandbox {} ready, running command", + sandbox.name + )); + let output = sandbox.exec(command).await; + progress.step(format_args!("{label}: deleting sandbox {}", sandbox.name)); + sandbox.cleanup().await; + progress.step(format_args!("{label}: sandbox deleted")); + Ok(format!("{}{}", sandbox.create_output, output?)) +} + async fn assert_gateway_admission( + progress: Progress, + provider: &str, port: u16, credential_source: CredentialSource, ) -> Result<(), String> { - let l4 = write_policy(port, EndpointMode::L4, credential_source)?; - let l4_error = sandbox_create_failure(&l4).await; + let l4 = write_policy(port, EndpointMode::L4, credential_source, provider)?; + let l4_error = sandbox_create_failure(progress, "L4-only policy", provider, &l4).await; assert!(l4_error.contains("credentialed endpoint"), "{l4_error}"); assert!(l4_error.contains("L4-only"), "{l4_error}"); - let tls_skip = write_policy(port, EndpointMode::TlsSkip, credential_source)?; - let tls_error = sandbox_create_failure(&tls_skip).await; + let tls_skip = write_policy(port, EndpointMode::TlsSkip, credential_source, provider)?; + let tls_error = sandbox_create_failure(progress, "tls: skip policy", provider, &tls_skip).await; assert!(tls_error.contains("credentialed endpoint"), "{tls_error}"); assert!( tls_error.contains("tls:") && tls_error.contains("skip"), "{tls_error}" ); - let opt_in = write_policy(port, EndpointMode::L4OptIn, credential_source)?; - let opt_in_path = opt_in - .path() - .to_str() - .ok_or_else(|| "opt-in policy path is not UTF-8".to_string())?; + let opt_in = write_policy(port, EndpointMode::L4OptIn, credential_source, provider)?; let accepted_marker = match credential_source { CredentialSource::ProviderProfile => "OPT_IN_ACCEPTED", CredentialSource::PolicyBinding => "ENDPOINTLESS_OPT_IN_ACCEPTED", }; - let mut sandbox = SandboxGuard::create(&[ - "--policy", - opt_in_path, - "--provider", - PROVIDER_NAME, - "--", - "echo", - accepted_marker, - ]) + let output = run_in_sandbox( + progress, + "opt-in policy", + &["--policy", utf8_path(&opt_in)?, "--provider", provider], + &["echo", accepted_marker], + ) .await?; - assert!(sandbox.create_output.contains(accepted_marker)); - sandbox.cleanup().await; + assert!(output.contains(accepted_marker), "{output}"); Ok(()) } @@ -712,168 +799,120 @@ async fn wait_for_provider_key_presence( )) } -async fn assert_endpointless_provider_env_live_update(port: u16) -> Result<(), String> { +async fn set_policy(progress: Progress, sandbox: &str, policy_path: &str) -> Result<(), String> { + let (success, output) = run_cli(&[ + "policy", + "set", + sandbox, + "--policy", + policy_path, + "--wait", + "--timeout", + "120", + ]) + .await; + if !success { + return Err(format!("policy update failed:\n{output}")); + } + progress.step("policy applied"); + Ok(()) +} + +async fn assert_endpointless_provider_env_live_update( + progress: Progress, + provider: &str, + port: u16, +) -> Result<(), String> { let unbound = write_policy( port, EndpointMode::RestBody { rewrite: false }, CredentialSource::ProviderProfile, + provider, )?; let bound = write_policy( port, EndpointMode::RestBody { rewrite: false }, CredentialSource::PolicyBinding, + provider, )?; - let unbound_path = unbound - .path() - .to_str() - .ok_or_else(|| "unbound policy path is not UTF-8".to_string())?; - let bound_path = bound - .path() - .to_str() - .ok_or_else(|| "bound policy path is not UTF-8".to_string())?; + let unbound_path = utf8_path(&unbound)?; + let bound_path = utf8_path(&bound)?; + progress.step("creating long-running sandbox with the unbound policy"); let mut sandbox = SandboxGuard::create_keep_with_args( - &[ - "--policy", - unbound_path, - "--provider", - PROVIDER_NAME, - "--no-tty", - ], + &["--policy", unbound_path, "--provider", provider, "--no-tty"], &["sh", "-c", "echo PROVIDER_UPDATE_READY && sleep infinity"], "PROVIDER_UPDATE_READY", ) .await?; let result = async { + progress.step(format_args!( + "sandbox {} ready, waiting for the token to be absent", + sandbox.name + )); wait_for_provider_key_presence(&sandbox, false).await?; - let (success, output) = run_cli(&[ - "policy", - "set", - &sandbox.name, - "--policy", - bound_path, - "--wait", - "--timeout", - "120", - ]) - .await; - if !success { - return Err(format!("binding policy update failed:\n{output}")); - } + progress.step("binding the provider with policy set --wait"); + set_policy(progress, &sandbox.name, bound_path) + .await + .map_err(|error| format!("binding {error}"))?; + progress.step("waiting for the token placeholder"); wait_for_provider_key_presence(&sandbox, true).await?; - let (success, output) = run_cli(&[ - "policy", - "set", - &sandbox.name, - "--policy", - unbound_path, - "--wait", - "--timeout", - "120", - ]) - .await; - if !success { - return Err(format!("unbinding policy update failed:\n{output}")); - } + progress.step("unbinding the provider with policy set --wait"); + set_policy(progress, &sandbox.name, unbound_path) + .await + .map_err(|error| format!("unbinding {error}"))?; + progress.step("waiting for the token to be absent"); wait_for_provider_key_presence(&sandbox, false).await } .await; + progress.step(format_args!("deleting sandbox {}", sandbox.name)); sandbox.cleanup().await; + progress.step("sandbox deleted"); result } async fn run_body_sandbox( + progress: Progress, + provider: &str, port: u16, mode: EndpointMode, credential_source: CredentialSource, ) -> Result { - let policy = write_policy(port, mode, credential_source)?; - let policy_path = policy - .path() - .to_str() - .ok_or_else(|| "body policy path is not UTF-8".to_string())?; - let script = body_client_script(port); - let mut sandbox = SandboxGuard::create(&[ - "--policy", - policy_path, - "--provider", - PROVIDER_NAME, - "--", - "python3", - "-c", - &script, - ]) - .await?; - let output = sandbox.create_output.clone(); - sandbox.cleanup().await; - Ok(output) -} - -async fn run_profile_body_sandbox(port: u16) -> Result { - let policy = write_policy( - port, - EndpointMode::RestBody { rewrite: false }, - CredentialSource::ProviderProfile, - )?; - let policy_path = policy - .path() - .to_str() - .ok_or_else(|| "body policy path is not UTF-8".to_string())?; + let policy = write_policy(port, mode, credential_source, provider)?; let script = body_client_script(port); - let mut sandbox = SandboxGuard::create(&[ - "--policy", - policy_path, - "--provider", - PROVIDER_NAME, - "--", - "python3", - "-c", - &script, - ]) - .await?; - let output = sandbox.create_output.clone(); - sandbox.cleanup().await; - Ok(output) -} - -async fn assert_rest_body_preserves_placeholder(server: &HttpProbeServer) -> Result<(), String> { - let output = run_profile_body_sandbox(server.port).await?; - assert!(output.contains("BODY_TEXT")); - let observations = server.wait_for_observations(1).await; - assert_eq!(observations.len(), 1, "observations: {observations:?}"); - assert!(observations[0].saw_placeholder); - assert!(!observations[0].saw_secret); - Ok(()) + run_in_sandbox( + progress, + "REST body client", + &["--policy", utf8_path(&policy)?, "--provider", provider], + &["python3", "-c", &script], + ) + .await } -async fn assert_websocket_binary_denied(server: &BinaryWebSocketProbeServer) -> Result<(), String> { +async fn assert_websocket_binary_denied( + progress: Progress, + provider: &str, + server: &BinaryWebSocketProbeServer, +) -> Result<(), String> { let policy = write_policy( server.port, EndpointMode::WebSocket, CredentialSource::ProviderProfile, + provider, )?; - let policy_path = policy - .path() - .to_str() - .ok_or_else(|| "WebSocket policy path is not UTF-8".to_string())?; let script = binary_websocket_client_script(server.port); - let mut sandbox = SandboxGuard::create(&[ - "--policy", - policy_path, - "--provider", - PROVIDER_NAME, - "--", - "python3", - "-c", - &script, - ]) + let output = run_in_sandbox( + progress, + "WebSocket client", + &["--policy", utf8_path(&policy)?, "--provider", provider], + &["python3", "-c", &script], + ) .await?; - assert!(sandbox.create_output.contains("BINARY_DENIED")); - sandbox.cleanup().await; + assert!(output.contains("BINARY_DENIED"), "{output}"); assert!( server.wait_for_handshake().await, "upstream should receive the WebSocket handshake" @@ -885,56 +924,134 @@ async fn assert_websocket_binary_denied(server: &BinaryWebSocketProbeServer) -> Ok(()) } -#[tokio::test] -async fn credentialed_endpoint_gates_work_end_to_end() { +/// Start both probe servers and install a provider whose profile declares +/// their endpoints. +async fn profile_endpoint_fixture( + progress: Progress, +) -> (HttpProbeServer, BinaryWebSocketProbeServer, GatingProvider) { let server = HttpProbeServer::start().await.expect("start HTTP probe"); let websocket_server = BinaryWebSocketProbeServer::start() .await .expect("start WebSocket probe"); - install_provider(server.port, websocket_server.port) + let provider = + GatingProvider::install_with_endpoints(progress, server.port, websocket_server.port) + .await + .expect("install credentialed provider"); + (server, websocket_server, provider) +} + +/// Start the HTTP probe server and install a provider whose profile declares +/// no endpoints, so policies must bind it explicitly. +async fn endpointless_fixture(progress: Progress) -> (HttpProbeServer, GatingProvider) { + let server = HttpProbeServer::start().await.expect("start HTTP probe"); + let provider = GatingProvider::install_endpointless(progress) .await - .expect("install credentialed provider"); + .expect("install endpointless provider"); + (server, provider) +} - let result = async { - assert_rest_body_preserves_placeholder(&server).await?; - assert_websocket_binary_denied(&websocket_server).await - } - .await; +#[tokio::test] +async fn profile_rest_endpoint_keeps_body_placeholder() { + let progress = Progress::start("profile-rest"); + let (server, _websocket_server, provider) = profile_endpoint_fixture(progress).await; - cleanup_provider_resources().await; - result.expect("credential gating E2E"); + let output = run_body_sandbox( + progress, + &provider.name, + server.port, + EndpointMode::RestBody { rewrite: false }, + CredentialSource::ProviderProfile, + ) + .await + .expect("run REST body client"); + assert!(output.contains("BODY_TEXT"), "{output}"); + let observations = server.wait_for_observations(1).await; + assert_eq!(observations.len(), 1, "observations: {observations:?}"); + assert!(observations[0].saw_placeholder); + assert!(!observations[0].saw_secret); + progress.step("checks passed"); +} - install_endpointless_provider() +#[tokio::test] +async fn profile_websocket_endpoint_denies_binary_frames() { + let progress = Progress::start("profile-websocket"); + let (_server, websocket_server, provider) = profile_endpoint_fixture(progress).await; + + assert_websocket_binary_denied(progress, &provider.name, &websocket_server) .await - .expect("install endpointless provider"); - let endpointless_result = async { - assert_gateway_admission(server.port, CredentialSource::PolicyBinding).await?; - let literal = run_body_sandbox( - server.port, - EndpointMode::RestBody { rewrite: false }, - CredentialSource::PolicyBinding, - ) - .await?; - assert!(literal.contains("BODY_TEXT")); - let rewritten = run_body_sandbox( - server.port, - EndpointMode::RestBody { rewrite: true }, - CredentialSource::PolicyBinding, - ) - .await?; - assert!(rewritten.contains("BODY_REWRITTEN")); - assert!(!rewritten.contains(TEST_SECRET)); - assert!(!rewritten.contains(PLACEHOLDER_PREFIX)); - let observations = server.wait_for_observations(3).await; - assert_eq!(observations.len(), 3, "observations: {observations:?}"); - assert!(observations[1].saw_placeholder); - assert!(!observations[1].saw_secret); - assert!(!observations[2].saw_placeholder); - assert!(observations[2].saw_secret); - assert_endpointless_provider_env_live_update(server.port).await?; - Ok::<(), String>(()) - } - .await; - cleanup_provider_resources().await; - endpointless_result.expect("endpointless provider credential gating E2E"); + .expect("run WebSocket client"); + progress.step("checks passed"); +} + +#[tokio::test] +async fn policy_binding_requires_inspection_or_opt_in() { + let progress = Progress::start("binding-admission"); + let (server, provider) = endpointless_fixture(progress).await; + + assert_gateway_admission( + progress, + &provider.name, + server.port, + CredentialSource::PolicyBinding, + ) + .await + .expect("gateway admission"); + progress.step("checks passed"); +} + +#[tokio::test] +async fn policy_binding_rest_body_keeps_placeholder_without_rewrite() { + let progress = Progress::start("binding-body"); + let (server, provider) = endpointless_fixture(progress).await; + + let literal = run_body_sandbox( + progress, + &provider.name, + server.port, + EndpointMode::RestBody { rewrite: false }, + CredentialSource::PolicyBinding, + ) + .await + .expect("run REST body client"); + assert!(literal.contains("BODY_TEXT"), "{literal}"); + let observations = server.wait_for_observations(1).await; + assert_eq!(observations.len(), 1, "observations: {observations:?}"); + assert!(observations[0].saw_placeholder); + assert!(!observations[0].saw_secret); + progress.step("checks passed"); +} + +#[tokio::test] +async fn policy_binding_rest_body_rewrite_sends_secret_upstream() { + let progress = Progress::start("binding-rewrite"); + let (server, provider) = endpointless_fixture(progress).await; + + let rewritten = run_body_sandbox( + progress, + &provider.name, + server.port, + EndpointMode::RestBody { rewrite: true }, + CredentialSource::PolicyBinding, + ) + .await + .expect("run REST body client"); + assert!(rewritten.contains("BODY_REWRITTEN")); + assert!(!rewritten.contains(TEST_SECRET)); + assert!(!rewritten.contains(PLACEHOLDER_PREFIX)); + let observations = server.wait_for_observations(1).await; + assert_eq!(observations.len(), 1, "observations: {observations:?}"); + assert!(!observations[0].saw_placeholder); + assert!(observations[0].saw_secret); + progress.step("checks passed"); +} + +#[tokio::test] +async fn policy_binding_updates_provider_environment_live() { + let progress = Progress::start("binding-live-update"); + let (server, provider) = endpointless_fixture(progress).await; + + assert_endpointless_provider_env_live_update(progress, &provider.name, server.port) + .await + .expect("provider environment live update"); + progress.step("checks passed"); }