Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/openshell-gateway/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ tracing = { workspace = true, optional = true }

[target.'cfg(target_os = "windows")'.dependencies]
openshell-driver-mxc = { path = "../openshell-driver-mxc", optional = true }
windows = { workspace = true, features = ["Win32_System_JobObjects"] }

[features]
default = ["telemetry", "in-tree-compute-drivers"]
Expand Down
2 changes: 2 additions & 0 deletions crates/openshell-gateway/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ compile_error!(
build a telemetry-free gateway with `--no-default-features --features defaults-without-telemetry`"
);

pub mod process_lifetime;

#[cfg(all(not(target_os = "windows"), feature = "compute-driver-vm"))]
mod vm;

Expand Down
6 changes: 6 additions & 0 deletions crates/openshell-gateway/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@

#[tokio::main]
async fn main() -> miette::Result<()> {
// Must run before anything is spawned so every sandbox process joins the job.
if let Err(error) = openshell_gateway::process_lifetime::kill_children_on_exit() {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we have the MXC driver own the jobs, ideally one per sandbox generation? This works while the driver runs inside the gateway, but the split in https://github.com/NVIDIA/OpenShell-Windows/issues/20 starts the external driver through an operator or launcher. That driver and its children would not automatically join this gateway-owned job, so the cleanup guarantee would disappear after extraction. Keeping containment with the driver would let us carry the fix across the split. If gateway loss must also stop an independently running driver’s workloads, we should define that behavior across the authenticated connection rather than rely on the process tree.

eprintln!(
"warning: could not tie sandbox processes to the gateway lifetime; they will outlive it: {error}"
);
}
openshell_server::cli::run_cli_with_compute_drivers(
openshell_gateway::install_default_compute_drivers(),
)
Expand Down
114 changes: 114 additions & 0 deletions crates/openshell-gateway/src/process_lifetime.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Tie the lifetime of every sandbox process to the gateway process.
//!
//! With the MXC driver built into the gateway, the gateway starts `wxc-exec`,
//! `openshell-supervisor` and `openshell-windows-sandbox` for each sandbox. If
//! the gateway is killed (crash, `taskkill`, logoff), those processes used to
//! survive with no owner and, because they inherited the gateway's listening
//! socket, kept its port bound so a new gateway could not start.
//!
//! A Windows *job object* with `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE` fixes the
//! first half: the gateway puts itself in the job, every child it spawns
//! inherits membership, and Windows ends all members when the last handle to
//! the job closes, which happens when the gateway process ends for any reason.
//! (The listening sockets are also made non-inheritable; see
//! `openshell_server::gateway_listener`.)
/// Put the current process into a new kill-on-close job object.
///
/// Call this first thing in `main`, before anything is spawned. The job handle
/// is deliberately never closed: it must stay open for as long as the process
/// lives, and closing it (which only happens when the process exits) is what
/// triggers the clean-up. On other platforms this does nothing.
///
/// # Errors
///
/// Returns a description of the failing Win32 call. The caller decides whether
/// that is fatal; nested jobs are supported on Windows 8 and later, so this
/// normally only fails on unusual hosts.
pub fn kill_children_on_exit() -> Result<(), String> {
#[cfg(windows)]
{
windows_impl::kill_children_on_exit()
}
#[cfg(not(windows))]
{
Ok(())
}
}

#[cfg(windows)]
#[allow(unsafe_code)] // Win32 job object calls.
mod windows_impl {
use std::mem::size_of;

use windows::Win32::System::JobObjects::{
AssignProcessToJobObject, CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JobObjectExtendedLimitInformation,
SetInformationJobObject,
};
use windows::Win32::System::Threading::GetCurrentProcess;
use windows::core::PCWSTR;

pub(super) fn kill_children_on_exit() -> Result<(), String> {
// SAFETY: plain Win32 calls with valid arguments; the info struct
// outlives the call and `size_of` matches its type.
unsafe {
let job = CreateJobObjectW(None, PCWSTR::null())
.map_err(|e| format!("CreateJobObjectW failed: {e}"))?;
let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
SetInformationJobObject(
job,
JobObjectExtendedLimitInformation,
(&raw const info).cast(),
u32::try_from(size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>())
.map_err(|e| format!("job info size: {e}"))?,
)
.map_err(|e| format!("SetInformationJobObject failed: {e}"))?;
AssignProcessToJobObject(job, GetCurrentProcess())
.map_err(|e| format!("AssignProcessToJobObject failed: {e}"))?;
// `job` is intentionally leaked (see the module docs).
let _ = job;
}
Ok(())
}
}

#[cfg(all(test, windows))]
#[allow(unsafe_code)] // Win32 job object query.
mod tests {
use std::mem::size_of;

use windows::Win32::System::JobObjects::{
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
JobObjectExtendedLimitInformation, QueryInformationJobObject,
};

#[test]
fn puts_the_process_in_a_kill_on_close_job() {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we add a graceful gateway shutdown/restart test with persisted state, alongside the hard-kill coverage, and run the lifecycle checks on ARM64 too? This test confirms the job flags, but not what happens to a running workload or its sandbox record. Docker, Podman, and VM use a stop-and-recover lifecycle on graceful gateway restart; MXC still has no restart recovery. We do not need to add recovery in this PR, but the Windows docs and build guidance should make clear that gateway exit terminates workloads without providing the same recovery behavior.

super::kill_children_on_exit().expect("job object");

// A null job handle asks about the job the calling process is in; with
// nested jobs that is the innermost one, i.e. the one just created.
let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
// SAFETY: `info` outlives the call and the size matches its type.
unsafe {
QueryInformationJobObject(
None,
JobObjectExtendedLimitInformation,
(&raw mut info).cast(),
u32::try_from(size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>()).unwrap(),
None,
)
}
.expect("QueryInformationJobObject");
assert_ne!(
info.BasicLimitInformation.LimitFlags.0 & JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE.0,
0,
"the process job must kill its members when closed"
);
}
}
4 changes: 4 additions & 0 deletions crates/openshell-server/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -146,5 +146,9 @@ wiremock = "0.6"
# `testing` provides InMemorySpanExporter, so span assertions do not need a
# collector, a network hop, or a flush barrier.
opentelemetry_sdk = { workspace = true, features = ["testing"] }
# Windows-only tests read the inherit flag of the gateway's listening socket.
[target.'cfg(target_os = "windows")'.dev-dependencies]
windows = { workspace = true }

[lints]
workspace = true
46 changes: 46 additions & 0 deletions crates/openshell-server/src/gateway_listener.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,49 @@ pub async fn bind_gateway_listener(address: SocketAddr) -> Result<BoundGatewayLi
address: local_addr,
})
}

/// Windows only: `mio` 1.2.1 and later create sockets with
/// `WSA_FLAG_NO_HANDLE_INHERIT`, so a sandbox process the gateway spawns cannot
/// keep a copy of the listening socket and hold its port after a gateway crash.
/// Older `mio` versions (1.2.0 and earlier) used plain `socket()`, which is
/// inheritable. These tests fail if the dependency ever regresses to that.
#[cfg(all(test, windows))]
#[allow(unsafe_code)] // Win32 handle flags, test-only.
mod tests {
use std::ffi::c_void;
use std::os::windows::io::AsRawSocket;

use windows::Win32::Foundation::{GetHandleInformation, HANDLE, HANDLE_FLAG_INHERIT};

use super::*;

fn is_inheritable(listener: &TcpListener) -> bool {
let handle = HANDLE(listener.as_raw_socket() as *mut c_void);
let mut flags = 0u32;
// SAFETY: `listener` owns a live socket handle and `flags` outlives the call.
unsafe { GetHandleInformation(handle, &raw mut flags) }.expect("GetHandleInformation");
flags & HANDLE_FLAG_INHERIT.0 != 0
}

#[tokio::test]
async fn bound_gateway_listener_is_not_inheritable() {
let bound = bind_gateway_listener("127.0.0.1:0".parse().unwrap())
.await
.unwrap();
assert!(
!is_inheritable(&bound.listener),
"child processes would inherit the gateway's listening socket"
);
}

#[tokio::test]
async fn listener_accepts_connections() {
let bound = bind_gateway_listener("127.0.0.1:0".parse().unwrap())
.await
.unwrap();
let client = tokio::net::TcpStream::connect(bound.address);
let (accepted, connected) = tokio::join!(bound.listener.accept(), client);
accepted.expect("accept");
connected.expect("connect");
}
}
Loading