feat(compute): configure slow backend deadlines

crates/openshell-core/src/config.rs

@@ -328,6 +328,10 @@ pub struct Config {
     /// sandbox attempts, in seconds. Must be between 1 and 86400, inclusive.
     pub image_preparation_timeout_seconds: u32,
 
+    /// Window to reach Ready after the supervisor first reports, and after each
+    /// repairable configuration change, in seconds. Must be between 1 and 86400.
+    pub admission_repair_window_seconds: u32,
+
     /// Maximum gRPC requests allowed per rate-limit window.
     ///
     /// When paired with [`Self::grpc_rate_limit_window_secs`], positive values
@@ -1048,6 +1052,7 @@ impl Config {
             default_credential_driver: None,
             ssh_session_ttl_secs: default_ssh_session_ttl_secs(),
             image_preparation_timeout_seconds: 1800,
+            admission_repair_window_seconds: 300,
             grpc_rate_limit_requests: None,
             grpc_rate_limit_window_secs: None,
             service_routing: ServiceRoutingConfig::default(),

crates/openshell-core/src/sandbox_env.rs

@@ -58,6 +58,9 @@ pub const OTLP_AGENT_PROTOCOL: &str = "OPENSHELL_OTLP_AGENT_PROTOCOL";
 /// `openshell.gateway.name` on supervisor telemetry.
 pub const GATEWAY_NAME: &str = "OPENSHELL_GATEWAY_NAME";
 
+/// Seconds the supervisor waits for its first boundary attachment.
+pub const ATTACH_REQUEST_TIMEOUT_SECS: &str = "OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECS";
+
 /// OTLP transport, named with the `OTEL_EXPORTER_OTLP_PROTOCOL` values.
 #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
 pub enum OtlpProtocol {

crates/openshell-driver-kubernetes/src/config.rs

@@ -122,12 +122,17 @@ pub struct KubernetesSandboxRuntimeConfig {
     /// TCP port of the Sandbox Protocol listener: the workload boundary for
     /// supervisor-initiated sandboxes, the supervisor for runtime-initiated ones.
     pub boundary_port: u16,
+    /// How long a sandbox runtime bootstrap may take before the driver reaps
+    /// it, in seconds. The supervisor waits as long for its first attachment.
+    /// Raise it for slow isolation backends such as peer-pods.
+    pub bootstrap_grace_seconds: u32,
 }
 
 impl Default for KubernetesSandboxRuntimeConfig {
     fn default() -> Self {
         Self {
             boundary_port: 5500,
+            bootstrap_grace_seconds: 300,
             app_listener: KubernetesAppListenerConfig::default(),
         }
     }
@@ -157,6 +162,11 @@ impl KubernetesSandboxRuntimeConfig {
         if self.boundary_port < 1024 {
             return Err("sandbox_runtime.boundary_port must be at least 1024".to_string());
         }
+        if !(1..=86_400).contains(&self.bootstrap_grace_seconds) {
+            return Err(
+                "sandbox_runtime.bootstrap_grace_seconds must be between 1 and 86400".to_string(),
+            );
+        }
         Ok(())
     }
 }
@@ -2115,6 +2125,19 @@ mod tests {
         cfg.validate_workspace_mode().unwrap();
     }
 
+    #[test]
+    fn sandbox_runtime_bootstrap_grace_must_be_bounded() {
+        let mut runtime = KubernetesSandboxRuntimeConfig::default();
+        assert_eq!(runtime.bootstrap_grace_seconds, 300);
+        runtime.validate().unwrap();
+        for seconds in [0, 86_401] {
+            runtime.bootstrap_grace_seconds = seconds;
+            assert!(runtime.validate().is_err());
+        }
+        runtime.bootstrap_grace_seconds = 900;
+        runtime.validate().unwrap();
+    }
+
     #[test]
     fn validate_workspace_mode_operator_requires_discovery() {
         let cfg = KubernetesComputeConfig {

crates/openshell-driver-kubernetes/src/driver.rs

@@ -612,9 +612,6 @@ fn admission_error(error: tonic::Status) -> KubernetesDriverError {
     }
 }
 const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30);
-/// Unsupervised shared sandbox instances removed at once by one reconcile pass.
-/// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded.
-const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5);
 
 fn random_sandbox_runtime_token() -> Result<String, KubernetesDriverError> {
     use std::fmt::Write as _;
@@ -1122,6 +1119,27 @@ impl KubernetesComputeDriver {
         environment
     }
 
+    /// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded.
+    fn bootstrap_grace(&self) -> Duration {
+        Duration::from_secs(u64::from(
+            self.config.sandbox_runtime.bootstrap_grace_seconds,
+        ))
+    }
+
+    /// Supervisor Pod environment beyond the fixed list: telemetry, and an
+    /// attach wait that matches the driver's bootstrap grace.
+    fn supervisor_environment(&self) -> Vec<(&'static str, String)> {
+        let mut environment = self.supervisor_otlp_environment();
+        environment.push((
+            openshell_core::sandbox_env::ATTACH_REQUEST_TIMEOUT_SECS,
+            self.config
+                .sandbox_runtime
+                .bootstrap_grace_seconds
+                .to_string(),
+        ));
+        environment
+    }
+
     #[cfg(test)]
     pub(crate) fn new_for_test(config: KubernetesComputeConfig) -> Self {
         let service = tower::service_fn(|_request: http::Request<kube::client::Body>| async {
@@ -3582,7 +3600,7 @@ impl KubernetesComputeDriver {
                         .as_ref()
                         .map(|generation| (generation.name.as_str(), generation.digest.as_str())),
                     owner,
-                    &self.supervisor_otlp_environment(),
+                    &self.supervisor_environment(),
                 )
                 .map_err(KubernetesDriverError::Message)?,
                 &self.config.sandbox_runtime.app_listener,
@@ -4102,6 +4120,7 @@ impl KubernetesComputeDriver {
             sandbox_api.api.clone(),
             cr_name.to_string(),
             Some(cr_uid.to_string()),
+            self.bootstrap_grace(),
         );
         Ok(())
     }
@@ -5540,11 +5559,7 @@ impl KubernetesComputeDriver {
         lookup_api: &AgentSandboxApi,
         object: &DynamicObject,
     ) {
-        if !sandbox_runtime_bootstrap_is_stale(
-            object,
-            SystemTime::now(),
-            SANDBOX_RUNTIME_BOOTSTRAP_GRACE,
-        ) {
+        if !sandbox_runtime_bootstrap_is_stale(object, SystemTime::now(), self.bootstrap_grace()) {
             return;
         }
         if sandbox_runtime_bootstrap_operation(object) != Some("create") {
@@ -5986,9 +6001,10 @@ fn spawn_sandbox_runtime_bootstrap_completion(
     sandboxes: Api<DynamicObject>,
     sandbox_name: String,
     expected_sandbox_uid: Option<String>,
+    grace: Duration,
 ) {
     tokio::spawn(async move {
-        let deadline = tokio::time::Instant::now() + SANDBOX_RUNTIME_BOOTSTRAP_GRACE;
+        let deadline = tokio::time::Instant::now() + grace;
         match tokio::time::timeout_at(deadline, control.wait_available()).await {
             Ok(true) => {}
             Ok(false) | Err(_) => return,
@@ -9189,6 +9205,21 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn supervisor_environment_matches_bootstrap_grace() {
+        let mut config = KubernetesComputeConfig::default();
+        config.sandbox_runtime.bootstrap_grace_seconds = 900;
+        let driver = KubernetesComputeDriver::new_for_test(config);
+        assert_eq!(driver.bootstrap_grace(), Duration::from_mins(15));
+        assert_eq!(
+            driver.supervisor_environment(),
+            [(
+                openshell_core::sandbox_env::ATTACH_REQUEST_TIMEOUT_SECS,
+                "900".to_string()
+            )]
+        );
+    }
+
     #[tokio::test]
     async fn supervisor_otlp_environment_carries_only_configured_signals() {
         use openshell_core::sandbox_env::{
@@ -10533,15 +10564,16 @@ mod tests {
             Some("restart")
         );
         let started_at = SystemTime::UNIX_EPOCH + started;
+        let grace = Duration::from_mins(5);
         assert!(!sandbox_runtime_bootstrap_is_stale(
             &object,
-            started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE - Duration::from_secs(1),
-            SANDBOX_RUNTIME_BOOTSTRAP_GRACE
+            started_at + grace - Duration::from_secs(1),
+            grace
         ));
         assert!(sandbox_runtime_bootstrap_is_stale(
             &object,
-            started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE,
-            SANDBOX_RUNTIME_BOOTSTRAP_GRACE
+            started_at + grace,
+            grace
         ));
         object
             .metadata
@@ -10550,9 +10582,7 @@ mod tests {
             .unwrap()
             .remove(ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT);
         assert!(sandbox_runtime_bootstrap_is_stale(
-            &object,
-            started_at,
-            SANDBOX_RUNTIME_BOOTSTRAP_GRACE
+            &object, started_at, grace
         ));
         object
             .metadata

crates/openshell-driver-kubernetes/src/main.rs

@@ -285,6 +285,7 @@ async fn main() -> Result<()> {
                 app_listener:
                     openshell_driver_kubernetes::config::KubernetesAppListenerConfig::default(),
                 boundary_port: args.sandbox_runtime_boundary_port,
+                ..Default::default()
             },
             https_proxy: args.https_proxy,
             no_proxy: args.no_proxy,

crates/openshell-sandbox-backend/src/runtime.rs

@@ -47,15 +47,21 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
 /// Initial attachment may include runtime image pulls and trusted bootstrap
 /// work before the boundary begins listening. Keep this aligned with the
 /// driver bootstrap grace period rather than the normal operation timeout.
-const ATTACH_REQUEST_TIMEOUT: Duration = Duration::from_mins(5);
+/// Slow isolation backends raise it with `OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECS`.
+fn initial_attach_timeout() -> Duration {
+    static TIMEOUT: std::sync::OnceLock<Duration> = std::sync::OnceLock::new();
+    *TIMEOUT.get_or_init(|| {
+        std::env::var(openshell_core::sandbox_env::ATTACH_REQUEST_TIMEOUT_SECS)
+            .ok()
+            .and_then(|value| value.parse::<u64>().ok())
+            .filter(|seconds| *seconds > 0)
+            .map_or(Duration::from_mins(5), Duration::from_secs)
+    })
+}
 /// How long one control call keeps retrying boundary connect attempts. Boot-time
 /// callers retry whole calls above this; past boot, exhausting this window
 /// means the remote boundary (or its launcher) is gone rather than still starting.
 const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_secs(30);
-/// Bound on a routed sandbox instance's first connection. The instance is
-/// registered before its workload Pod is scheduled and its image pulled; the
-/// driver reaps a bootstrap that is still incomplete after the same interval.
-const ROUTED_FIRST_CONNECT_TIMEOUT: Duration = Duration::from_mins(5);
 /// Bounds the liveness check that decides whether a stream failure lost the
 /// whole connection.
 const CONNECTION_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
@@ -176,7 +182,9 @@ impl BoundaryTransport {
                 let window = if registration.has_connected() {
                     CONNECT_RETRY_TIMEOUT
                 } else {
-                    ROUTED_FIRST_CONNECT_TIMEOUT
+                    // Registered before the workload Pod is scheduled; the driver
+                    // reaps an incomplete bootstrap after the same interval.
+                    initial_attach_timeout()
                 };
                 return tokio::time::timeout(window, registration.accept(&descriptor.tls))
                     .await
@@ -1871,7 +1879,7 @@ impl BoundaryClient {
         let remember_attach = matches!(request, Request::Attach { .. });
         let remember_confirm = matches!(request, Request::Confirm);
         let timeout = if remember_attach || matches!(request, Request::DiscoverPolicy) {
-            ATTACH_REQUEST_TIMEOUT
+            initial_attach_timeout()
         } else {
             REQUEST_TIMEOUT
         };

crates/openshell-server/src/cli.rs

@@ -705,6 +705,13 @@ fn prepare_server_config_with_drivers(
         config.image_preparation_timeout_seconds = seconds;
     }
 
+    if let Some(seconds) = file
+        .as_ref()
+        .and_then(|f| f.openshell.gateway.admission_repair_window_seconds)
+    {
+        config.admission_repair_window_seconds = seconds;
+    }
+
     if let Some(issuer) = args.oidc_issuer.clone() {
         config = config.with_oidc(openshell_core::OidcConfig {
             issuer,
@@ -4869,6 +4876,7 @@ version = 2
 [openshell.gateway]
 policy_validation_failure_mode = "retain_last_valid"
 image_preparation_timeout_seconds = 2400
+admission_repair_window_seconds = 900
 
 [openshell.drivers.docker]
 unknown_docker_key = true
@@ -4895,6 +4903,7 @@ mem_mib = "not-a-number"
 
                 assert_eq!(prepared.config.compute_driver.as_deref(), Some("podman"));
                 assert_eq!(prepared.config.image_preparation_timeout_seconds, 2400);
+                assert_eq!(prepared.config.admission_repair_window_seconds, 900);
                 assert_eq!(
                     prepared.config.policy_validation_failure_mode,
                     openshell_core::PolicyValidationFailureMode::RetainLastValid

crates/openshell-server/src/compute/mod.rs

@@ -99,6 +99,14 @@ pub type SharedComputeDriverConfigurationUpdates = Arc<
         + Sync,
 >;
 
+fn validate_deadline_seconds(name: &str, seconds: u32) -> Result<(), String> {
+    if (1..=86_400).contains(&seconds) {
+        Ok(())
+    } else {
+        Err(format!("{name} must be between 1 and 86400"))
+    }
+}
+
 fn supports_isolation_registration(capabilities: &[String]) -> bool {
     capabilities.iter().any(|capability| {
         capability == openshell_core::extension_protocol::COMPUTE_ISOLATION_BACKEND
@@ -1335,13 +1343,18 @@ impl ComputeRuntime {
     /// Validate the budget once at startup. Persisted attempts keep their
     /// original deadline even if the operator changes this value on restart.
     pub(crate) fn with_image_preparation_timeout(mut self, seconds: u32) -> Result<Self, String> {
-        if !(1..=86_400).contains(&seconds) {
-            return Err("image_preparation_timeout_seconds must be between 1 and 86400".into());
-        }
+        validate_deadline_seconds("image_preparation_timeout_seconds", seconds)?;
         self.image_preparation_timeout_seconds = seconds;
         Ok(self)
     }
 
+    /// Validate the repair window once at startup. It applies process-wide.
+    pub(crate) fn with_admission_repair_window(self, seconds: u32) -> Result<Self, String> {
+        validate_deadline_seconds("admission_repair_window_seconds", seconds)?;
+        provisioning_deadline::set_repair_window_seconds(seconds);
+        Ok(self)
+    }
+
     #[must_use]
     pub fn driver_info_snapshots(&self) -> &[ComputeDriverInfoSnapshot] {
         std::slice::from_ref(&self.driver_info)

crates/openshell-server/src/compute/provisioning_deadline.rs

@@ -11,8 +11,18 @@
 use openshell_core::proto::SandboxProvisioning;
 use openshell_core::time::{timestamp_from_millis, timestamp_to_millis};
 use serde::{Deserialize, Serialize};
+use std::sync::atomic::{AtomicI64, Ordering};
 
-const REPAIR_WINDOW_MS: i64 = 300_000;
+static REPAIR_WINDOW_MS: AtomicI64 = AtomicI64::new(300_000);
+
+/// Set once at startup. Armed attempts keep the deadline they already persisted.
+pub(super) fn set_repair_window_seconds(seconds: u32) {
+    REPAIR_WINDOW_MS.store(i64::from(seconds) * 1_000, Ordering::Relaxed);
+}
+
+fn repair_window_ms() -> i64 {
+    REPAIR_WINDOW_MS.load(Ordering::Relaxed)
+}
 
 /// How long one scan pass waits, in total, for sandboxes' local keys. A sandbox
 /// whose keys are still busy is skipped until the next pass. Keys are taken in
@@ -144,7 +154,7 @@ impl ProvisioningDeadline {
             preparation_deadline_at_ms: None,
             admission_start_at_ms: None,
             state: DeadlineState::Armed {
-                deadline_at_ms: now_ms.saturating_add(REPAIR_WINDOW_MS),
+                deadline_at_ms: now_ms.saturating_add(repair_window_ms()),
             },
         }
     }
@@ -168,7 +178,7 @@ impl ProvisioningDeadline {
         }
         self.admission_start_at_ms = Some(now_ms);
         self.state = DeadlineState::Armed {
-            deadline_at_ms: now_ms.saturating_add(REPAIR_WINDOW_MS),
+            deadline_at_ms: now_ms.saturating_add(repair_window_ms()),
         };
         true
     }
@@ -196,7 +206,7 @@ impl ProvisioningDeadline {
         if !self.is_preparing() {
             self.state = DeadlineState::Armed {
                 deadline_at_ms: deadline_at_ms
-                    .max(change.committed_at_ms.saturating_add(REPAIR_WINDOW_MS)),
+                    .max(change.committed_at_ms.saturating_add(repair_window_ms())),
             };
         }
         self.change = change;
@@ -220,7 +230,7 @@ impl ProvisioningDeadline {
         }
         self.first_rejection_at_ms = Some(now_ms);
         self.state = DeadlineState::Armed {
-            deadline_at_ms: deadline_at_ms.max(now_ms.saturating_add(REPAIR_WINDOW_MS)),
+            deadline_at_ms: deadline_at_ms.max(now_ms.saturating_add(repair_window_ms())),
         };
         true
     }