diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 20f6a334b1..b59d104dcf 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -328,6 +328,10 @@ pub struct Config { /// sandbox attempts, in seconds. Must be between 1 and 86400, inclusive. pub image_preparation_timeout_seconds: u32, + /// Window to reach Ready after the supervisor first reports, and after each + /// repairable configuration change, in seconds. Must be between 1 and 86400. + pub admission_repair_window_seconds: u32, + /// Maximum gRPC requests allowed per rate-limit window. /// /// When paired with [`Self::grpc_rate_limit_window_secs`], positive values @@ -1048,6 +1052,7 @@ impl Config { default_credential_driver: None, ssh_session_ttl_secs: default_ssh_session_ttl_secs(), image_preparation_timeout_seconds: 1800, + admission_repair_window_seconds: 300, grpc_rate_limit_requests: None, grpc_rate_limit_window_secs: None, service_routing: ServiceRoutingConfig::default(), diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index c691642904..99fb64d673 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -58,6 +58,9 @@ pub const OTLP_AGENT_PROTOCOL: &str = "OPENSHELL_OTLP_AGENT_PROTOCOL"; /// `openshell.gateway.name` on supervisor telemetry. pub const GATEWAY_NAME: &str = "OPENSHELL_GATEWAY_NAME"; +/// Seconds the supervisor waits for its first boundary attachment. +pub const ATTACH_REQUEST_TIMEOUT_SECS: &str = "OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECS"; + /// OTLP transport, named with the `OTEL_EXPORTER_OTLP_PROTOCOL` values. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub enum OtlpProtocol { diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 28d3161f42..b58a3914f6 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -122,12 +122,17 @@ pub struct KubernetesSandboxRuntimeConfig { /// TCP port of the Sandbox Protocol listener: the workload boundary for /// supervisor-initiated sandboxes, the supervisor for runtime-initiated ones. pub boundary_port: u16, + /// How long a sandbox runtime bootstrap may take before the driver reaps + /// it, in seconds. The supervisor waits as long for its first attachment. + /// Raise it for slow isolation backends such as peer-pods. + pub bootstrap_grace_seconds: u32, } impl Default for KubernetesSandboxRuntimeConfig { fn default() -> Self { Self { boundary_port: 5500, + bootstrap_grace_seconds: 300, app_listener: KubernetesAppListenerConfig::default(), } } @@ -157,6 +162,11 @@ impl KubernetesSandboxRuntimeConfig { if self.boundary_port < 1024 { return Err("sandbox_runtime.boundary_port must be at least 1024".to_string()); } + if !(1..=86_400).contains(&self.bootstrap_grace_seconds) { + return Err( + "sandbox_runtime.bootstrap_grace_seconds must be between 1 and 86400".to_string(), + ); + } Ok(()) } } @@ -2115,6 +2125,19 @@ mod tests { cfg.validate_workspace_mode().unwrap(); } + #[test] + fn sandbox_runtime_bootstrap_grace_must_be_bounded() { + let mut runtime = KubernetesSandboxRuntimeConfig::default(); + assert_eq!(runtime.bootstrap_grace_seconds, 300); + runtime.validate().unwrap(); + for seconds in [0, 86_401] { + runtime.bootstrap_grace_seconds = seconds; + assert!(runtime.validate().is_err()); + } + runtime.bootstrap_grace_seconds = 900; + runtime.validate().unwrap(); + } + #[test] fn validate_workspace_mode_operator_requires_discovery() { let cfg = KubernetesComputeConfig { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index c08a567eed..fff3562e3a 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -612,9 +612,6 @@ fn admission_error(error: tonic::Status) -> KubernetesDriverError { } } const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30); -/// Unsupervised shared sandbox instances removed at once by one reconcile pass. -/// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded. -const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5); fn random_sandbox_runtime_token() -> Result { use std::fmt::Write as _; @@ -1122,6 +1119,27 @@ impl KubernetesComputeDriver { environment } + /// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded. + fn bootstrap_grace(&self) -> Duration { + Duration::from_secs(u64::from( + self.config.sandbox_runtime.bootstrap_grace_seconds, + )) + } + + /// Supervisor Pod environment beyond the fixed list: telemetry, and an + /// attach wait that matches the driver's bootstrap grace. + fn supervisor_environment(&self) -> Vec<(&'static str, String)> { + let mut environment = self.supervisor_otlp_environment(); + environment.push(( + openshell_core::sandbox_env::ATTACH_REQUEST_TIMEOUT_SECS, + self.config + .sandbox_runtime + .bootstrap_grace_seconds + .to_string(), + )); + environment + } + #[cfg(test)] pub(crate) fn new_for_test(config: KubernetesComputeConfig) -> Self { let service = tower::service_fn(|_request: http::Request| async { @@ -3582,7 +3600,7 @@ impl KubernetesComputeDriver { .as_ref() .map(|generation| (generation.name.as_str(), generation.digest.as_str())), owner, - &self.supervisor_otlp_environment(), + &self.supervisor_environment(), ) .map_err(KubernetesDriverError::Message)?, &self.config.sandbox_runtime.app_listener, @@ -4102,6 +4120,7 @@ impl KubernetesComputeDriver { sandbox_api.api.clone(), cr_name.to_string(), Some(cr_uid.to_string()), + self.bootstrap_grace(), ); Ok(()) } @@ -5540,11 +5559,7 @@ impl KubernetesComputeDriver { lookup_api: &AgentSandboxApi, object: &DynamicObject, ) { - if !sandbox_runtime_bootstrap_is_stale( - object, - SystemTime::now(), - SANDBOX_RUNTIME_BOOTSTRAP_GRACE, - ) { + if !sandbox_runtime_bootstrap_is_stale(object, SystemTime::now(), self.bootstrap_grace()) { return; } if sandbox_runtime_bootstrap_operation(object) != Some("create") { @@ -5986,9 +6001,10 @@ fn spawn_sandbox_runtime_bootstrap_completion( sandboxes: Api, sandbox_name: String, expected_sandbox_uid: Option, + grace: Duration, ) { tokio::spawn(async move { - let deadline = tokio::time::Instant::now() + SANDBOX_RUNTIME_BOOTSTRAP_GRACE; + let deadline = tokio::time::Instant::now() + grace; match tokio::time::timeout_at(deadline, control.wait_available()).await { Ok(true) => {} Ok(false) | Err(_) => return, @@ -9189,6 +9205,21 @@ mod tests { ); } + #[tokio::test] + async fn supervisor_environment_matches_bootstrap_grace() { + let mut config = KubernetesComputeConfig::default(); + config.sandbox_runtime.bootstrap_grace_seconds = 900; + let driver = KubernetesComputeDriver::new_for_test(config); + assert_eq!(driver.bootstrap_grace(), Duration::from_mins(15)); + assert_eq!( + driver.supervisor_environment(), + [( + openshell_core::sandbox_env::ATTACH_REQUEST_TIMEOUT_SECS, + "900".to_string() + )] + ); + } + #[tokio::test] async fn supervisor_otlp_environment_carries_only_configured_signals() { use openshell_core::sandbox_env::{ @@ -10533,15 +10564,16 @@ mod tests { Some("restart") ); let started_at = SystemTime::UNIX_EPOCH + started; + let grace = Duration::from_mins(5); assert!(!sandbox_runtime_bootstrap_is_stale( &object, - started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE - Duration::from_secs(1), - SANDBOX_RUNTIME_BOOTSTRAP_GRACE + started_at + grace - Duration::from_secs(1), + grace )); assert!(sandbox_runtime_bootstrap_is_stale( &object, - started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE, - SANDBOX_RUNTIME_BOOTSTRAP_GRACE + started_at + grace, + grace )); object .metadata @@ -10550,9 +10582,7 @@ mod tests { .unwrap() .remove(ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT); assert!(sandbox_runtime_bootstrap_is_stale( - &object, - started_at, - SANDBOX_RUNTIME_BOOTSTRAP_GRACE + &object, started_at, grace )); object .metadata diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 69ffc69ccb..c275c9e21b 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -285,6 +285,7 @@ async fn main() -> Result<()> { app_listener: openshell_driver_kubernetes::config::KubernetesAppListenerConfig::default(), boundary_port: args.sandbox_runtime_boundary_port, + ..Default::default() }, https_proxy: args.https_proxy, no_proxy: args.no_proxy, diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index 9ea72d396c..0acd5235ae 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -47,15 +47,21 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); /// Initial attachment may include runtime image pulls and trusted bootstrap /// work before the boundary begins listening. Keep this aligned with the /// driver bootstrap grace period rather than the normal operation timeout. -const ATTACH_REQUEST_TIMEOUT: Duration = Duration::from_mins(5); +/// Slow isolation backends raise it with `OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECS`. +fn initial_attach_timeout() -> Duration { + static TIMEOUT: std::sync::OnceLock = std::sync::OnceLock::new(); + *TIMEOUT.get_or_init(|| { + std::env::var(openshell_core::sandbox_env::ATTACH_REQUEST_TIMEOUT_SECS) + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|seconds| *seconds > 0) + .map_or(Duration::from_mins(5), Duration::from_secs) + }) +} /// How long one control call keeps retrying boundary connect attempts. Boot-time /// callers retry whole calls above this; past boot, exhausting this window /// means the remote boundary (or its launcher) is gone rather than still starting. const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_secs(30); -/// Bound on a routed sandbox instance's first connection. The instance is -/// registered before its workload Pod is scheduled and its image pulled; the -/// driver reaps a bootstrap that is still incomplete after the same interval. -const ROUTED_FIRST_CONNECT_TIMEOUT: Duration = Duration::from_mins(5); /// Bounds the liveness check that decides whether a stream failure lost the /// whole connection. const CONNECTION_PROBE_TIMEOUT: Duration = Duration::from_secs(5); @@ -176,7 +182,9 @@ impl BoundaryTransport { let window = if registration.has_connected() { CONNECT_RETRY_TIMEOUT } else { - ROUTED_FIRST_CONNECT_TIMEOUT + // Registered before the workload Pod is scheduled; the driver + // reaps an incomplete bootstrap after the same interval. + initial_attach_timeout() }; return tokio::time::timeout(window, registration.accept(&descriptor.tls)) .await @@ -1871,7 +1879,7 @@ impl BoundaryClient { let remember_attach = matches!(request, Request::Attach { .. }); let remember_confirm = matches!(request, Request::Confirm); let timeout = if remember_attach || matches!(request, Request::DiscoverPolicy) { - ATTACH_REQUEST_TIMEOUT + initial_attach_timeout() } else { REQUEST_TIMEOUT }; diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 1f0f215881..6160bd8349 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -705,6 +705,13 @@ fn prepare_server_config_with_drivers( config.image_preparation_timeout_seconds = seconds; } + if let Some(seconds) = file + .as_ref() + .and_then(|f| f.openshell.gateway.admission_repair_window_seconds) + { + config.admission_repair_window_seconds = seconds; + } + if let Some(issuer) = args.oidc_issuer.clone() { config = config.with_oidc(openshell_core::OidcConfig { issuer, @@ -4869,6 +4876,7 @@ version = 2 [openshell.gateway] policy_validation_failure_mode = "retain_last_valid" image_preparation_timeout_seconds = 2400 +admission_repair_window_seconds = 900 [openshell.drivers.docker] unknown_docker_key = true @@ -4895,6 +4903,7 @@ mem_mib = "not-a-number" assert_eq!(prepared.config.compute_driver.as_deref(), Some("podman")); assert_eq!(prepared.config.image_preparation_timeout_seconds, 2400); + assert_eq!(prepared.config.admission_repair_window_seconds, 900); assert_eq!( prepared.config.policy_validation_failure_mode, openshell_core::PolicyValidationFailureMode::RetainLastValid diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index a6768040e6..4a58ba91c8 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -99,6 +99,14 @@ pub type SharedComputeDriverConfigurationUpdates = Arc< + Sync, >; +fn validate_deadline_seconds(name: &str, seconds: u32) -> Result<(), String> { + if (1..=86_400).contains(&seconds) { + Ok(()) + } else { + Err(format!("{name} must be between 1 and 86400")) + } +} + fn supports_isolation_registration(capabilities: &[String]) -> bool { capabilities.iter().any(|capability| { capability == openshell_core::extension_protocol::COMPUTE_ISOLATION_BACKEND @@ -1335,13 +1343,18 @@ impl ComputeRuntime { /// Validate the budget once at startup. Persisted attempts keep their /// original deadline even if the operator changes this value on restart. pub(crate) fn with_image_preparation_timeout(mut self, seconds: u32) -> Result { - if !(1..=86_400).contains(&seconds) { - return Err("image_preparation_timeout_seconds must be between 1 and 86400".into()); - } + validate_deadline_seconds("image_preparation_timeout_seconds", seconds)?; self.image_preparation_timeout_seconds = seconds; Ok(self) } + /// Validate the repair window once at startup. It applies process-wide. + pub(crate) fn with_admission_repair_window(self, seconds: u32) -> Result { + validate_deadline_seconds("admission_repair_window_seconds", seconds)?; + provisioning_deadline::set_repair_window_seconds(seconds); + Ok(self) + } + #[must_use] pub fn driver_info_snapshots(&self) -> &[ComputeDriverInfoSnapshot] { std::slice::from_ref(&self.driver_info) diff --git a/crates/openshell-server/src/compute/provisioning_deadline.rs b/crates/openshell-server/src/compute/provisioning_deadline.rs index cb351e64d0..e10b4da10e 100644 --- a/crates/openshell-server/src/compute/provisioning_deadline.rs +++ b/crates/openshell-server/src/compute/provisioning_deadline.rs @@ -11,8 +11,18 @@ use openshell_core::proto::SandboxProvisioning; use openshell_core::time::{timestamp_from_millis, timestamp_to_millis}; use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicI64, Ordering}; -const REPAIR_WINDOW_MS: i64 = 300_000; +static REPAIR_WINDOW_MS: AtomicI64 = AtomicI64::new(300_000); + +/// Set once at startup. Armed attempts keep the deadline they already persisted. +pub(super) fn set_repair_window_seconds(seconds: u32) { + REPAIR_WINDOW_MS.store(i64::from(seconds) * 1_000, Ordering::Relaxed); +} + +fn repair_window_ms() -> i64 { + REPAIR_WINDOW_MS.load(Ordering::Relaxed) +} /// How long one scan pass waits, in total, for sandboxes' local keys. A sandbox /// whose keys are still busy is skipped until the next pass. Keys are taken in @@ -144,7 +154,7 @@ impl ProvisioningDeadline { preparation_deadline_at_ms: None, admission_start_at_ms: None, state: DeadlineState::Armed { - deadline_at_ms: now_ms.saturating_add(REPAIR_WINDOW_MS), + deadline_at_ms: now_ms.saturating_add(repair_window_ms()), }, } } @@ -168,7 +178,7 @@ impl ProvisioningDeadline { } self.admission_start_at_ms = Some(now_ms); self.state = DeadlineState::Armed { - deadline_at_ms: now_ms.saturating_add(REPAIR_WINDOW_MS), + deadline_at_ms: now_ms.saturating_add(repair_window_ms()), }; true } @@ -196,7 +206,7 @@ impl ProvisioningDeadline { if !self.is_preparing() { self.state = DeadlineState::Armed { deadline_at_ms: deadline_at_ms - .max(change.committed_at_ms.saturating_add(REPAIR_WINDOW_MS)), + .max(change.committed_at_ms.saturating_add(repair_window_ms())), }; } self.change = change; @@ -220,7 +230,7 @@ impl ProvisioningDeadline { } self.first_rejection_at_ms = Some(now_ms); self.state = DeadlineState::Armed { - deadline_at_ms: deadline_at_ms.max(now_ms.saturating_add(REPAIR_WINDOW_MS)), + deadline_at_ms: deadline_at_ms.max(now_ms.saturating_add(repair_window_ms())), }; true } diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 865f6b1ed1..50665fcd87 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -149,6 +149,9 @@ pub struct GatewayFileSection { /// Absolute preparation budget for new attempts; existing deadlines persist. #[serde(default)] pub image_preparation_timeout_seconds: Option, + /// Window to reach Ready after the supervisor first reports. + #[serde(default)] + pub admission_repair_window_seconds: Option, /// Explicit activation only after the compatible deployment rollout ends. #[serde(default)] pub enable_sandbox_updates: Option, diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 02c1ba16b3..c68f7e1703 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -2284,6 +2284,9 @@ async fn build_compute_runtime( .and_then(|runtime| { runtime.with_image_preparation_timeout(config.image_preparation_timeout_seconds) }) + .and_then(|runtime| { + runtime.with_admission_repair_window(config.admission_repair_window_seconds) + }) .map_err(Error::config)?; Ok(runtime.with_telemetry_compute_driver(telemetry_compute_driver)) } diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 175836ccb4..c81f64d50d 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -282,6 +282,10 @@ ssh_session_ttl_secs = 3600 # Applies to new attempts only; allowed values are 1 through 86400 seconds. image_preparation_timeout_seconds = 1800 +# Time to reach Ready after the supervisor's first configuration report. +# Raise it for slow isolation backends. Allowed values are 1 through 86400. +admission_repair_window_seconds = 300 + # Reject invalid policy generations securely by default. Set # "retain_last_valid" only when availability takes priority. policy_validation_failure_mode = "fail_closed" @@ -1379,6 +1383,10 @@ gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes [openshell.drivers.kubernetes.sandbox_runtime] # TLS-protected boundary listener reached only by the paired control pod. boundary_port = 5500 +# Seconds before the driver reaps an incomplete runtime bootstrap. The +# supervisor waits as long for its first attachment. Raise it for slow +# isolation backends. Allowed values are 1 through 86400. +bootstrap_grace_seconds = 300 ``` The Kubernetes driver always creates the workload `NetworkPolicy` before diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 2ac6a4f69d..ed46c015d3 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -1171,7 +1171,7 @@ stopped sandbox repeats configuration admission before launching its workload. Image preparation and initial supervisor startup have an absolute deadline of 1800 seconds by default, independently of the CLI wait timeout. Operators can set `image_preparation_timeout_seconds` in `[openshell.gateway]` to any value from 1 through 86400. Each new create or explicit start stores its deadline. Progress events, configuration edits, and gateway or driver restarts cannot extend it. Changing the gateway setting affects new attempts only. -The first authenticated configuration report from the current supervisor ends preparation and starts a separate 300-second admission repair window. A slow image download therefore does not consume time reserved for fixing policy. During admission, an effective policy, settings, provider, profile, or attachment change resets the window from its stored change time. The first failed configuration load for that change grants another full window. Repeated failures, duplicate reports, and reconnects do not extend it; reaching `Ready` clears it. +The first authenticated configuration report from the current supervisor ends preparation and starts a separate admission repair window of 300 seconds by default. Operators can set `admission_repair_window_seconds` in `[openshell.gateway]` to any value from 1 through 86400. A slow image download therefore does not consume time reserved for fixing policy. During admission, an effective policy, settings, provider, profile, or attachment change resets the window from its stored change time. The first failed configuration load for that change grants another full window. Repeated failures, duplicate reports, and reconnects do not extend it; reaching `Ready` clears it. Preparation expiry sets the sandbox to `Error` with reason `ImagePreparationTimedOut`; admission repair expiry uses `ProvisioningTimedOut`. The gateway stops its workload and supervisor compute, retaining the sandbox record, diagnostic, and restartable storage. Cleanup can remain pending if the backend is unavailable; the gateway retries it. Inspect `provisioning` in JSON output for the active `phase`, `deadline`, original `preparation_deadline`, `admission_start_time`, and cleanup timestamps. TUI NOTES identifies preparation timeout and distinguishes pending cleanup from reclaimed compute.