Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 116 additions & 0 deletions crates/openshell-driver-kubernetes/src/driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,9 @@ const SANDBOX_VERSIONS: &[&str] = &[SANDBOX_VERSION_V1BETA1, SANDBOX_VERSION_V1A
pub const SANDBOX_KIND: &str = "Sandbox";
const SANDBOX_POD_NAME_ANNOTATION: &str = "agents.x-k8s.io/pod-name";
const SANDBOX_SUSPENDED_CONDITION: &str = "Suspended";
const POD_SCHEDULED_CONDITION: &str = "PodScheduled";
/// User-facing text; the raw scheduler message exposes cluster details.
const WAITING_FOR_CAPACITY_MESSAGE: &str = "Waiting for cluster capacity or a compatible node";
const SANDBOX_SUSPENDED_POD_NOT_OWNED_REASON: &str = "PodNotOwned";
const SANDBOX_TOKEN_AUDIENCE: &str = "openshell-gateway";
const POD_NAME_EXTRA: &str = "authentication.kubernetes.io/pod-name";
Expand Down Expand Up @@ -6625,6 +6628,48 @@ async fn sandbox_runtime_workload_generation_matches(
}
}

/// The scheduler's verdict for a pod it cannot place, as a sandbox condition.
/// The raw scheduler message stays in gateway logs.
fn pod_unschedulable_condition(pod: &Pod) -> Option<SandboxCondition> {
let condition = pod
.status
.as_ref()?
.conditions
.as_ref()?
.iter()
.find(|condition| {
condition.type_ == POD_SCHEDULED_CONDITION
&& condition.status == "False"
&& condition.reason.as_deref() == Some("Unschedulable")
})?;
debug!(
message = condition.message.as_deref().unwrap_or_default(),
"Pod is unschedulable"
);
Some(SandboxCondition {
r#type: POD_SCHEDULED_CONDITION.to_string(),
status: "False".to_string(),
reason: "Unschedulable".to_string(),
message: WAITING_FOR_CAPACITY_MESSAGE.to_string(),
transition_time: condition.last_transition_time.as_ref().and_then(|time| {
openshell_core::time::timestamp_from_millis(time.0.timestamp_millis()).ok()
}),
})
}

/// Best-effort: a failed read only omits the scheduling hint.
async fn workload_unschedulable_condition(
client: &Client,
namespace: &str,
pod_name: &str,
) -> Option<SandboxCondition> {
let pods = Api::<Pod>::namespaced(client.clone(), namespace);
match tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(pod_name)).await {
Ok(Ok(Some(pod))) => pod_unschedulable_condition(&pod),
_ => None,
}
}

async fn sandbox_runtime_supervisor_generation_matches(
client: &Client,
namespace: &str,
Expand Down Expand Up @@ -6720,6 +6765,18 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness(
mark_sandbox_runtime_control_unavailable(&mut sandbox);
}
}
if let Some(status) = sandbox.status.as_mut()
&& !status.conditions.iter().any(|condition| {
condition.r#type == "Ready" && condition.status.eq_ignore_ascii_case("true")
})
&& let Some(condition) =
workload_unschedulable_condition(client, &object_namespace, &name).await
{
status
.conditions
.retain(|condition| condition.r#type != POD_SCHEDULED_CONDITION);
status.conditions.push(condition);
}
Ok((name, sandbox))
}

Expand Down Expand Up @@ -6882,6 +6939,13 @@ fn attach_kube_progress_metadata(
mark_progress_complete(metadata, PROGRESS_STEP_PULLING_IMAGE, label);
mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX);
}
"FailedScheduling" => {
info!(message, "Pod failed scheduling");
mark_progress_detail(
metadata,
format!("Waiting to be scheduled: {WAITING_FOR_CAPACITY_MESSAGE}"),
);
}
_ => {}
}
}
Expand Down Expand Up @@ -11295,6 +11359,58 @@ mod tests {
);
}

#[test]
fn kube_failed_scheduling_event_adds_waiting_detail() {
let mut metadata = std::collections::HashMap::new();

attach_kube_progress_metadata(
&mut metadata,
"FailedScheduling",
"0/3 nodes are available: 3 Insufficient memory.",
);

assert_eq!(
metadata.get(PROGRESS_ACTIVE_DETAIL_KEY).map(String::as_str),
Some("Waiting to be scheduled: Waiting for cluster capacity or a compatible node")
);
assert!(!metadata.contains_key(PROGRESS_COMPLETE_STEP_KEY));
assert!(!metadata.contains_key(PROGRESS_ACTIVE_STEP_KEY));
}

#[test]
fn unschedulable_pod_reports_generic_message() {
let pod_with = |reason: &str, status: &str| Pod {
status: Some(
serde_json::from_value(serde_json::json!({
"conditions": [{
"type": "PodScheduled",
"status": status,
"reason": reason,
"message": "0/3 nodes are available: 3 Insufficient memory.",
"lastTransitionTime": "2026-10-24T08:00:00Z"
}]
}))
.expect("valid Pod status"),
),
..Pod::default()
};

let condition = pod_unschedulable_condition(&pod_with("Unschedulable", "False"))
.expect("unschedulable pod");
assert_eq!(condition.r#type, "PodScheduled");
assert_eq!(condition.status, "False");
assert_eq!(condition.reason, "Unschedulable");
assert_eq!(
condition.message,
"Waiting for cluster capacity or a compatible node"
);
assert!(condition.transition_time.is_some());

assert!(pod_unschedulable_condition(&pod_with("SchedulingGated", "False")).is_none());
assert!(pod_unschedulable_condition(&pod_with("", "True")).is_none());
assert!(pod_unschedulable_condition(&Pod::default()).is_none());
}

#[test]
fn sandbox_runtime_renders_credential_free_boundary_workload() {
let params = SandboxPodParams {
Expand Down
8 changes: 8 additions & 0 deletions docs/how-it-works/sandboxes/overview.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1077,6 +1077,14 @@ After a gateway restart, an existing sandbox can return to `Provisioning`
temporarily while its supervisor reconnects. Wait for the phase to return to
`Ready` before you connect to the sandbox or execute commands.

On the Kubernetes driver, a sandbox pod that the scheduler cannot place keeps
the sandbox in `Provisioning` with a `PodScheduled=False` condition. Its reason
is `Unschedulable` and its message is the generic `Waiting for cluster capacity
or a compatible node`. `openshell sandbox create` shows the same text while it
waits, and `openshell sandbox get` reports the condition. The condition clears
once the pod is scheduled. The scheduler's raw message, which can reveal node
counts and resource constraints, is available to operators in gateway logs.

The gateway records a successful canonical main-process exit as `Ready=False`
with reason `MainProcessCompleted`. Nonzero and signal-normalized results use
`MainProcessFailed` and the `Error` phase. It also sets `status.exit_code`;
Expand Down
Loading