diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index cb9433d08e..e9394df901 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -14,6 +14,47 @@ binaries. The driver embeds libkrun, libkrunfw, the guest OCI unpacker, the portable guest sandbox, and the custom kernel runtime. Each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir. +Guest init creates missing `/dev/fd` and standard-stream links after mounting +procfs and devtmpfs, before using Bash process substitution. Existing links to +the expected procfs paths, or entries that resolve to the same files, stay +intact. Conflicting entries and unrelated dangling links fail boot with a +diagnostic rather than being replaced. + +## Lifecycle extension requirements + +Lifecycle extensions declare requirements through `LaunchPlan::require_backend` +and `require_backend_feature`. An explicit backend stays selected; incompatible +requirements fail before the workload launcher starts. Without an explicit +requirement, the driver chooses a backend that supports the requested features. + +These are VM-driver-local backend capabilities. The driver resolves the final +plan after operator-installed extensions contribute their requirements, then +checks it against the selected backend. The gateway's `GetCapabilities` +snapshot describes the compute-driver protocol and supported resource request +forms; it does not describe this per-launch VM plan. + +| Feature | libkrun | QEMU | +|---|---|---| +| Guest init drop-ins | Supported | Supported | +| External kernel image | Unsupported | Supported for CPU-only and GPU VMs | +| PCI passthrough | Unsupported | Requires an assigned GPU device | +| Host directory mounts | Supported through virtiofs | Unsupported | + +For example, an operator-installed extension can set `plan.kernel_image` and +require `BackendFeature::ExternalKernelImage` for a CPU-only sandbox. The +resolver selects QEMU with a vsock CID and ordinary CPU sizing, without +allocating a GPU. Requiring libkrun for that same plan fails with an +external-kernel diagnostic. Kernel paths remain extension-owned; sandbox +requests cannot supply them. + +The current QEMU launcher requires a Linux x86_64 host with KVM. Other host +targets fail preflight before the workload launcher starts. External kernels are checked for a +64-bit little-endian x86_64 ELF header or an x86_64 Linux bzImage header. The +VMM validates the complete image. Kernel security and boot-driver requirements +still apply, and authenticated runtime admission confirms enforcement before +workload code runs. Both workload and image-preparation launchers use the same +kernel preflight entry point; preparation retains its libkrun bootstrap backend. + ## How it fits together Caller driver config is disabled by default, including upload workflows diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 37950675b0..97a99e1078 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -31,6 +31,31 @@ mount_initial_fs() { mount -t tmpfs tmpfs /tmp 2>/dev/null || true mount -t tmpfs tmpfs /run 2>/dev/null || true mount -t devtmpfs devtmpfs /dev 2>/dev/null || true + setup_fd_links /dev +} + +setup_fd_links() { + local dev_dir="$1" + local name target + for name in fd stdin stdout stderr; do + case "$name" in + fd) target=/proc/self/fd ;; + stdin) target=/proc/self/fd/0 ;; + stdout) target=/proc/self/fd/1 ;; + stderr) target=/proc/self/fd/2 ;; + esac + if [[ -L "$dev_dir/$name" ]] && [[ "$(readlink "$dev_dir/$name")" == "$target" ]]; then + continue + fi + if [[ -e "$dev_dir/$name" || -L "$dev_dir/$name" ]]; then + if [[ ! "$dev_dir/$name" -ef "$target" ]]; then + printf 'guest init: %s must refer to %s\n' "$dev_dir/$name" "$target" >&2 + return 1 + fi + else + ln -s "$target" "$dev_dir/$name" + fi + done } bind_mount_into_newroot() { diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index f7946c34c2..2938a22245 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -1994,9 +1994,11 @@ impl VmDriver { } if let Err(err) = Self::validate_launch_plan_backend(is_gpu, &plan).and_then(|()| { - // Cannot fail today: `validate_sandbox` already ran and non-GPU - // QEMU is blocked above. Keeps virtiofs mounts off QEMU once that - // guard is lifted. + plan.backend + .validate_host() + .map_err(Status::failed_precondition)?; + // Extensions can select QEMU after request validation, so check + // the selected backend's mount support again before launch. validate_vm_driver_mounts( &driver_config.mounts, &self.config, @@ -3003,13 +3005,30 @@ impl VmDriver { plan.require_backend_feature(BackendFeature::GuestInitDropins); } - if plan.required_backends.contains(&VmBackend::Qemu) - || plan.backend == VmBackend::Qemu - || plan + if let Some(required) = plan.required_backends.first() { + if plan + .required_backends + .iter() + .any(|backend| backend != required) + { + return Err(Status::failed_precondition( + "VM lifecycle extensions requested conflicting VM backends", + )); + } + plan.backend = required.clone(); + } else if plan + .required_backend_features + .iter() + .any(|feature| !feature.is_supported_by(&plan.backend)) + && plan .required_backend_features .iter() - .any(|feature| feature.requires_qemu()) + .all(|feature| feature.is_supported_by(&VmBackend::Qemu)) { + plan.backend = VmBackend::Qemu; + } + + if plan.backend == VmBackend::Qemu { self.configure_qemu_launch_plan(sandbox_id, is_gpu, gpu_bdf, plan)?; } @@ -3043,33 +3062,21 @@ impl VmDriver { #[allow(clippy::result_large_err)] fn validate_launch_plan_backend(is_gpu: bool, plan: &LaunchPlan) -> Result<(), Status> { - // NOTE: this guard exists because the non-GPU QEMU launch path - // (PCI device transport, VFIO root port wiring) has not landed - // yet. Until then, even though the resolver will happily promote - // a plan to QEMU when an extension requires `PciPassthrough` or - // `ExternalKernelImage`, the launch itself is blocked here so we - // don't spawn a QEMU instance with no concrete device backing. - // Remove this guard once the non-GPU QEMU launch path supports - // emitting `pcie-root-port` + `vfio-pci` for arbitrary device - // descriptors. - if plan.backend == VmBackend::Qemu && !is_gpu { - let offending_feature = plan - .required_backend_features - .iter() - .find(|feature| feature.requires_qemu()) - .map_or("(explicit QEMU backend requirement)", |feature| { - feature.as_str() - }); - return Err(Status::failed_precondition(format!( - "vm lifecycle extension required '{offending_feature}', which resolves to the QEMU backend, \ - but non-GPU QEMU launch is not yet supported (pending PCI device transport)" - ))); - } if plan.backend != VmBackend::Qemu && is_gpu { return Err(Status::failed_precondition( "GPU sandbox launch requires the QEMU backend", )); } + if plan.gpu_bdf.is_some() && !is_gpu { + return Err(Status::failed_precondition( + "GPU passthrough requires a GPU sandbox request", + )); + } + if is_gpu && plan.gpu_bdf.is_none() { + return Err(Status::failed_precondition( + "GPU sandbox launch is missing its assigned GPU device", + )); + } if plan.required_backends.contains(&VmBackend::Libkrun) && plan.required_backends.contains(&VmBackend::Qemu) { @@ -3089,29 +3096,30 @@ impl VmDriver { "VM lifecycle extension requires the QEMU backend", )); } - if plan.backend != VmBackend::Qemu - && let Some(feature) = plan - .required_backend_features - .iter() - .find(|feature| feature.requires_qemu()) + if let Some(feature) = plan + .required_backend_features + .iter() + .find(|feature| !feature.is_supported_by(&plan.backend)) { return Err(Status::failed_precondition(format!( - "VM backend feature '{}' requires a VM backend with PCI-style launch support", + "VM backend {:?} does not support feature '{}'", + plan.backend, feature.as_str() ))); } - if plan.kernel_image.is_some() && plan.backend != VmBackend::Qemu { + if plan + .required_backend_features + .contains(&BackendFeature::PciPassthrough) + && plan.gpu_bdf.is_none() + { return Err(Status::failed_precondition( - "selected kernel image requires a VM backend that supports external kernel images", + "PCI passthrough requires an assigned GPU device; arbitrary PCI devices are not supported", )); } - if let Some(kernel_image) = &plan.kernel_image - && !kernel_image.is_file() - { - return Err(Status::failed_precondition(format!( - "selected kernel image does not exist: {}", - kernel_image.display() - ))); + if let Some(kernel_image) = &plan.kernel_image { + plan.backend + .validate_kernel_image(kernel_image) + .map_err(Status::failed_precondition)?; } Ok(()) } @@ -13693,10 +13701,28 @@ while :; do sleep 0.05; done"#, let base = unique_temp_dir(); std::fs::create_dir_all(&base).unwrap(); let kernel = base.join("vmlinux"); - std::fs::write(&kernel, b"kernel").unwrap(); - plan.backend = VmBackend::Qemu; + let mut header = [0u8; 64]; + header[..6].copy_from_slice(b"\x7fELF\x02\x01"); + header[18] = 62; // EM_X86_64; a header fixture, not a bootable guest. + std::fs::write(&kernel, header).unwrap(); plan.kernel_image = Some(kernel); - VmDriver::validate_launch_plan_backend(true, &plan).expect("existing kernel is accepted"); + let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); + plan.require_backend(VmBackend::Libkrun); + driver + .resolve_launch_plan_backend("kernel-check", false, None, &mut plan) + .unwrap(); + assert_eq!(plan.backend, VmBackend::Libkrun); + assert!(VmDriver::validate_launch_plan_backend(false, &plan).is_err()); + + plan.required_backends.clear(); + driver + .resolve_launch_plan_backend("kernel-check", false, None, &mut plan) + .unwrap(); + assert_eq!(plan.backend, VmBackend::Qemu); + assert!(plan.gpu_bdf.is_none()); + assert!(plan.vsock_cid.is_some()); + VmDriver::validate_launch_plan_backend(false, &plan) + .expect("CPU-only external kernel is accepted without PCI passthrough"); let _ = std::fs::remove_dir_all(base); } @@ -13714,6 +13740,9 @@ while :; do sleep 0.05; done"#, assert_eq!(plan.backend, VmBackend::Qemu); assert!(plan.vsock_cid.is_some()); + let error = VmDriver::validate_launch_plan_backend(false, &plan) + .expect_err("a PCI feature needs an assigned device"); + assert!(error.message().contains("assigned GPU device")); } #[test] @@ -13730,6 +13759,23 @@ while :; do sleep 0.05; done"#, assert_eq!(plan.backend, VmBackend::Qemu); assert!(plan.vsock_cid.is_some()); + VmDriver::validate_launch_plan_backend(false, &plan).expect("CPU-only QEMU is supported"); + } + + #[test] + fn conflicting_backend_requirements_fail_before_allocating_a_cid() { + let driver = test_driver_with_extensions(LifecycleExtensionRegistry::new()); + let mut plan = driver + .build_vm_launch_plan("conflict", false, false, None) + .unwrap(); + plan.require_backend(VmBackend::Libkrun); + plan.require_backend(VmBackend::Qemu); + let error = driver + .resolve_launch_plan_backend("conflict", false, None, &mut plan) + .expect_err("conflicting runtimes must fail"); + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("conflicting VM backends")); + assert!(plan.vsock_cid.is_none()); } #[test] diff --git a/crates/openshell-driver-vm/src/lifecycle.rs b/crates/openshell-driver-vm/src/lifecycle.rs index f92082d99e..6ce929f6a4 100644 --- a/crates/openshell-driver-vm/src/lifecycle.rs +++ b/crates/openshell-driver-vm/src/lifecycle.rs @@ -82,20 +82,9 @@ pub type LifecycleResult = Result; /// /// Extensions declare features they need (e.g. PCI passthrough or an /// external kernel image) and the VM driver resolves a concrete -/// [`VmBackend`] that can satisfy them. The mapping from feature to -/// backend lives in [`BackendFeature::requires_qemu`] for now; once a -/// third backend exists this should evolve into a per-backend capability -/// table that the resolver intersects against feature requirements. -/// -/// # Current limitations -/// -/// Until the non-GPU QEMU launch path (PCI device transport / VFIO root -/// port wiring) lands, the driver still rejects launches where the -/// resolved backend is QEMU but the sandbox has no GPU. As a result, -/// declaring [`Self::PciPassthrough`] or [`Self::ExternalKernelImage`] on -/// a non-GPU sandbox is accepted by [`LifecycleExtensionRegistry::validate`] -/// at registration time but will fail provisioning with a -/// `FailedPrecondition` status. +/// [`VmBackend`] that can satisfy them. [`Self::is_supported_by`] describes +/// each backend independently. Explicit backend requirements are preserved; +/// an unsupported feature fails before the launcher starts. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BackendFeature { /// Extension supplies its own kernel image via @@ -105,8 +94,8 @@ pub enum BackendFeature { /// [`LaunchPlan::guest_init_dropins`]. Supported by all backends. GuestInitDropins, /// Extension needs PCI device passthrough on the guest. Currently - /// QEMU-only and currently rejected for non-GPU sandboxes pending the - /// non-GPU QEMU launch path landing. + /// QEMU-only. The current device model requires an assigned GPU BDF; + /// arbitrary PCI device descriptors are not supported. PciPassthrough, } @@ -120,13 +109,15 @@ impl BackendFeature { } } - /// Returns true when satisfying this feature requires the QEMU backend - /// today. This is the simplest possible resolver and is expected to be - /// replaced with a per-backend capability table once a third backend - /// exists. + /// Whether the backend implements this feature. Device assignment and + /// kernel compatibility are validated separately for the concrete plan. #[must_use] - pub fn requires_qemu(self) -> bool { - matches!(self, Self::ExternalKernelImage | Self::PciPassthrough) + pub fn is_supported_by(self, backend: &VmBackend) -> bool { + match (backend, self) { + (VmBackend::Libkrun, Self::ExternalKernelImage | Self::PciPassthrough) => false, + (VmBackend::Libkrun | VmBackend::Qemu, Self::GuestInitDropins) + | (VmBackend::Qemu, Self::ExternalKernelImage | Self::PciPassthrough) => true, + } } } @@ -349,8 +340,11 @@ pub trait LifecycleExtension: std::fmt::Debug + Send + Sync { /// inject additional guest environment variables, and MAY return an /// error to abort the launch. Implementations MUST NOT change /// [`LaunchPlan::backend`], [`LaunchPlan::required_backends`], or - /// [`LaunchPlan::required_backend_features`]; those changes are - /// ignored by the driver once `before_launch` is reached. + /// [`LaunchPlan::required_backend_features`]; the driver has already + /// resolved and validated them and does not repeat those steps after + /// this hook. + /// The driver-owned [`LaunchPlan::gpu_bdf`] must remain unchanged; + /// the registry rejects hooks that clear or replace that assignment. /// /// If this hook performs allocations that must be released on failure /// or delete, implement @@ -604,8 +598,15 @@ impl LifecycleExtensionRegistry { plan: &mut LaunchPlan, ) -> LifecycleResult<()> { let span_status = openshell_otel::ErrorStatusGuard::current(); + let assigned_gpu = plan.gpu_bdf.clone(); for ext in self.active_for(sandbox) { ext.before_launch(sandbox, state_dir, plan).await?; + if plan.gpu_bdf != assigned_gpu { + return Err(LifecycleError::new(format!( + "extension '{}' changed the assigned GPU device in before_launch", + ext.name() + ))); + } } span_status.finish(Ok(())) } @@ -978,6 +979,41 @@ mod tests { extension.clone() } + #[derive(Debug)] + struct GpuMutatingExtension(Option); + + #[tonic::async_trait] + impl LifecycleExtension for GpuMutatingExtension { + fn name(&self) -> &'static str { + "gpu-mutator" + } + + async fn before_launch( + &self, + _sandbox: &Sandbox, + _state_dir: &Path, + plan: &mut LaunchPlan, + ) -> LifecycleResult<()> { + plan.gpu_bdf.clone_from(&self.0); + Ok(()) + } + } + + #[tokio::test] + async fn before_launch_rejects_clearing_or_replacing_the_assigned_gpu() { + for replacement in [None, Some("0000:02:00.0".to_string())] { + let registry = + LifecycleExtensionRegistry::with(vec![Arc::new(GpuMutatingExtension(replacement))]); + let mut plan = sample_plan(VmBackend::Qemu); + plan.gpu_bdf = Some("0000:01:00.0".to_string()); + let error = registry + .before_launch(&sample_sandbox(), Path::new("/tmp/state"), &mut plan) + .await + .expect_err("a requested GPU must retain its assignment after hooks"); + assert!(error.message().contains("changed the assigned GPU device")); + } + } + #[tokio::test] async fn configure_launch_runs_each_extension_in_order() { let ext_a = RecordingExtension::new("a"); diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index a292e81994..67846c10ac 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -1357,6 +1357,93 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn guest_init_creates_fd_links_and_preserves_existing_aliases() { + use std::os::unix::fs::MetadataExt as _; + + let dev = tempfile::tempdir().expect("temporary guest dev"); + let run = || run_guest_setup_fd_links(dev.path()); + let output = run(); + assert!(output.status.success(), "{output:?}"); + let aliases = [ + ("fd", "/proc/self/fd"), + ("stdin", "/proc/self/fd/0"), + ("stdout", "/proc/self/fd/1"), + ("stderr", "/proc/self/fd/2"), + ]; + let mut inodes = Vec::new(); + for (name, target) in aliases { + let path = dev.path().join(name); + assert_eq!(fs::read_link(&path).expect("read alias"), Path::new(target)); + inodes.push(fs::symlink_metadata(path).expect("stat alias").ino()); + } + let output = run(); + assert!(output.status.success(), "{output:?}"); + for ((name, _), inode) in aliases.into_iter().zip(inodes) { + assert_eq!( + fs::symlink_metadata(dev.path().join(name)) + .expect("stat retained alias") + .ino(), + inode, + "existing {name} alias stays intact" + ); + } + #[cfg(target_os = "linux")] + { + let stdin = dev.path().join("stdin"); + fs::remove_file(&stdin).expect("replace fixture alias"); + std::os::unix::fs::symlink("fd/0", &stdin).expect("equivalent relative alias"); + let output = run(); + assert!(output.status.success(), "{output:?}"); + assert_eq!( + fs::read_link(stdin).expect("retained relative alias"), + Path::new("fd/0") + ); + } + } + + #[test] + fn guest_init_rejects_conflicting_fd_entries_without_replacing_them() { + let dev = tempfile::tempdir().expect("temporary guest dev"); + let fd = dev.path().join("fd"); + fs::write(&fd, b"keep this entry").expect("conflicting file"); + let output = run_guest_setup_fd_links(dev.path()); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("must refer to /proc/self/fd")); + assert_eq!(fs::read(&fd).expect("retained file"), b"keep this entry"); + + fs::remove_file(&fd).expect("remove fixture file"); + std::os::unix::fs::symlink("missing-target", &fd).expect("dangling link"); + let output = run_guest_setup_fd_links(dev.path()); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("must refer to /proc/self/fd")); + assert_eq!( + fs::read_link(fd).expect("retained link"), + Path::new("missing-target") + ); + } + + fn run_guest_setup_fd_links(dev: &Path) -> std::process::Output { + let script = include_str!("../scripts/openshell-vm-sandbox-init.sh"); + let start = script + .find("setup_fd_links() {\n") + .expect("fd setup function"); + let tail = &script[start..]; + let end = tail.find("\n}\n").expect("fd setup function end") + "\n}\n".len(); + Command::new("bash") + .args([ + "-c", + &format!( + "set -euo pipefail\n{}\nsetup_fd_links \"$1\"\n", + &tail[..end] + ), + "test", + ]) + .arg(dev) + .output() + .expect("run shipped fd setup") + } + /// Runs the shipped guest init's `configure_hosts` against `root`. /// /// The function is cut out of the real script so the test exercises the diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index ad300d98cf..ad98899c8e 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -4,6 +4,7 @@ #![allow(unsafe_code)] use std::ffi::CString; +use std::io::Read as _; use std::path::{Path, PathBuf}; use std::process::{Command as StdCommand, Stdio}; use std::ptr; @@ -25,6 +26,82 @@ pub enum VmBackend { Qemu, } +impl VmBackend { + pub(crate) fn validate_host(&self) -> Result<(), String> { + match self { + Self::Libkrun => Ok(()), + Self::Qemu if cfg!(all(target_os = "linux", target_arch = "x86_64")) => Ok(()), + Self::Qemu => Err("QEMU requires a Linux x86_64 host with KVM".to_string()), + } + } + + pub(crate) fn validate_kernel_image(&self, path: &Path) -> Result<(), String> { + match self { + Self::Libkrun => Err(format!( + "libkrun does not support external kernel images: {}", + path.display() + )), + Self::Qemu => validate_qemu_kernel_image(path), + } + } +} + +/// Preflight the formats accepted by the `x86_64` QEMU launcher. The VMM still +/// validates the complete image; runtime admission confirms enforcement. +fn validate_qemu_kernel_image(path: &Path) -> Result<(), String> { + if !path.is_file() { + return Err(format!( + "selected kernel image is not a regular file: {}", + path.display() + )); + } + let file = std::fs::File::open(path).map_err(|error| { + format!( + "cannot open selected kernel image {}: {error}", + path.display() + ) + })?; + let mut header = Vec::new(); + file.take(0x238).read_to_end(&mut header).map_err(|error| { + format!( + "cannot read selected kernel image {}: {error}", + path.display() + ) + })?; + if header.starts_with(b"\x7fELF") { + // ELFCLASS64, ELFDATA2LSB and EM_X86_64. + if header.len() >= 64 + && header[4] == 2 + && header[5] == 1 + && u16::from_le_bytes([header[18], header[19]]) == 62 + { + return Ok(()); + } + return Err(format!( + "QEMU kernel must be a 64-bit little-endian x86_64 ELF image: {}", + path.display() + )); + } + // Linux/x86 boot protocol: boot_flag, HdrS, version and XLF_KERNEL_64. + // https://www.kernel.org/doc/html/latest/arch/x86/boot.html + if header.len() == 0x238 + && header[0x1fe..0x200] == [0x55, 0xaa] + && &header[0x202..0x206] == b"HdrS" + { + if u16::from_le_bytes([header[0x206], header[0x207]]) >= 0x020c && header[0x236] & 1 != 0 { + return Ok(()); + } + return Err(format!( + "QEMU kernel bzImage must target x86_64: {}", + path.display() + )); + } + Err(format!( + "unsupported QEMU kernel format in {}; expected x86_64 ELF or bzImage", + path.display() + )) +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct VsockPortMap { pub guest_port: u32, @@ -61,6 +138,11 @@ pub struct VmLaunchConfig { } pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { + // Both workload and image-preparation launchers enter here. Reject + // incompatible external kernels before touching disks or starting the VMM. + if let Some(kernel_image) = &config.kernel_image { + config.backend.validate_kernel_image(kernel_image)?; + } match config.backend { VmBackend::Qemu => run_qemu_vm(config), VmBackend::Libkrun => run_libkrun_vm(config), @@ -71,13 +153,12 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { if !config.mounts.is_empty() { return Err("virtiofs mounts are not yet supported with the QEMU backend".to_string()); } - let gpu_bdf = config - .gpu_bdf - .as_deref() - .ok_or("gpu_bdf is required for QEMU backend")?; - let vsock_cid = config - .vsock_cid - .ok_or("vsock_cid is required for QEMU backend")?; + if config.vsock_port_map.is_some() { + return Err( + "QEMU uses host AF_VSOCK; Unix-socket vsock port maps require libkrun".to_string(), + ); + } + VmBackend::Qemu.validate_host()?; if !config.root_disk.is_file() { return Err(format!( @@ -97,16 +178,6 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { return Err(format!("image disk not found: {}", image_disk.display())); } - if let Err(err) = procguard::die_with_parent_cleanup(procguard_kill_children) { - return Err(format!("procguard arm failed: {err}")); - } - - #[cfg(target_os = "linux")] - check_kvm_access()?; - - let guest_env = qemu_guest_env_vars(config); - write_guest_env_file(&config.overlay_disk, &guest_env)?; - let vmlinux = if let Some(kernel_image) = &config.kernel_image { kernel_image.clone() } else { @@ -115,43 +186,18 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { if !vmlinux.is_file() { return Err(format!("VM kernel not found: {}", vmlinux.display())); } + validate_qemu_kernel_image(&vmlinux)?; + let mut qemu_cmd = qemu_command(config, &vmlinux)?; - let kernel_cmdline = build_kernel_cmdline(config); + if let Err(err) = procguard::die_with_parent_cleanup(procguard_kill_children) { + return Err(format!("procguard arm failed: {err}")); + } - let mut qemu_cmd = StdCommand::new("qemu-system-x86_64"); - qemu_cmd - .arg("-machine") - .arg("q35,accel=kvm") - .arg("-cpu") - .arg("host") - .arg("-smp") - .arg(config.vcpus.to_string()) - .arg("-m") - .arg(format!("{}M", config.mem_mib)) - .arg("-nographic") - .arg("-no-reboot") - .args(qemu_network_args()) - .arg("-kernel") - .arg(&vmlinux) - .arg("-append") - .arg(&kernel_cmdline) - .args(qemu_disk_args(config)) - .arg("-device") - .arg("pcie-root-port,id=vsock_root,slot=1") - .arg("-device") - .arg(format!( - "vhost-vsock-pci,guest-cid={vsock_cid},bus=vsock_root" - )) - .arg("-device") - .arg("pcie-root-port,id=gpu_root,slot=2") - .arg("-device") - .arg(format!("vfio-pci,host={gpu_bdf},bus=gpu_root")) - .arg("-serial") - .arg(format!("file:{}", config.console_output.display())); + #[cfg(target_os = "linux")] + check_kvm_access()?; - qemu_cmd.stdin(Stdio::null()); - qemu_cmd.stdout(Stdio::inherit()); - qemu_cmd.stderr(Stdio::inherit()); + let guest_env = qemu_guest_env_vars(config); + write_guest_env_file(&config.overlay_disk, &guest_env)?; #[cfg(target_os = "linux")] { @@ -185,6 +231,55 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { } } +fn qemu_command(config: &VmLaunchConfig, kernel: &Path) -> Result { + let vsock_cid = config + .vsock_cid + .ok_or("vsock_cid is required for QEMU backend")?; + if vsock_cid < 3 { + return Err("QEMU guest vsock CID must be at least 3".to_string()); + } + let mut qemu_cmd = StdCommand::new("qemu-system-x86_64"); + qemu_cmd + .arg("-machine") + .arg("q35,accel=kvm") + .arg("-cpu") + .arg("host") + .arg("-smp") + .arg(config.vcpus.to_string()) + .arg("-m") + .arg(format!("{}M", config.mem_mib)) + .arg("-nographic") + .arg("-no-reboot") + .args(qemu_network_args()) + .arg("-kernel") + .arg(kernel) + .arg("-append") + .arg(build_kernel_cmdline(config)) + .args(qemu_disk_args(config)) + .arg("-device") + .arg("pcie-root-port,id=vsock_root,slot=1") + .arg("-device") + .arg(format!( + "vhost-vsock-pci,guest-cid={vsock_cid},bus=vsock_root" + )) + .arg("-serial") + .arg(format!("file:{}", config.console_output.display())); + + if let Some(gpu_bdf) = &config.gpu_bdf { + qemu_cmd + .arg("-device") + .arg("pcie-root-port,id=gpu_root,slot=2") + .arg("-device") + .arg(format!("vfio-pci,host={gpu_bdf},bus=gpu_root")); + } + + qemu_cmd.stdin(Stdio::null()); + qemu_cmd.stdout(Stdio::inherit()); + qemu_cmd.stderr(Stdio::inherit()); + + Ok(qemu_cmd) +} + fn qemu_network_args() -> [&'static str; 2] { ["-nic", "none"] } @@ -825,6 +920,129 @@ mod tests { } } + #[test] + fn qemu_command_keeps_vsock_and_only_adds_vfio_for_an_assigned_gpu() { + let mut config = qemu_config(); + for gpu in [None, Some("0000:01:00.0".to_string())] { + config.gpu_bdf = gpu.clone(); + let command = qemu_command(&config, Path::new("/vmlinux")).expect("QEMU command"); + let args: Vec<_> = command + .get_args() + .map(|arg| arg.to_string_lossy()) + .collect(); + assert!( + args.iter() + .any(|arg| arg == "vhost-vsock-pci,guest-cid=4,bus=vsock_root") + ); + assert!(args.windows(2).any(|pair| pair == ["-nic", "none"])); + assert_eq!( + args.iter().any(|arg| arg.starts_with("vfio-pci,")), + gpu.is_some() + ); + assert_eq!( + args.iter().any(|arg| arg.contains("gpu_root,slot=2")), + gpu.is_some() + ); + assert_eq!( + qemu_guest_env_vars(&config) + .iter() + .any(|env| env == "GPU_ENABLED=true"), + gpu.is_some() + ); + } + for cid in [None, Some(2)] { + config.vsock_cid = cid; + assert!(qemu_command(&config, Path::new("/vmlinux")).is_err()); + } + } + + #[cfg(not(all(target_os = "linux", target_arch = "x86_64")))] + #[test] + fn qemu_rejects_unsupported_hosts_before_disk_setup() { + let mut config = qemu_config(); + config.gpu_bdf = None; + assert_eq!( + run_vm(&config).unwrap_err(), + "QEMU requires a Linux x86_64 host with KVM" + ); + } + + #[test] + fn kernel_preflight_checks_format_and_guest_architecture() { + let file = tempfile::NamedTempFile::new().unwrap(); + let mut elf = vec![0u8; 64]; + elf[..6].copy_from_slice(b"\x7fELF\x02\x01"); + elf[18] = 62; // EM_X86_64. + std::fs::write(file.path(), &elf).unwrap(); + VmBackend::Qemu + .validate_kernel_image(file.path()) + .expect("x86_64 ELF header"); + elf[18] = 183; // EM_AARCH64. + std::fs::write(file.path(), &elf).unwrap(); + assert!( + VmBackend::Qemu + .validate_kernel_image(file.path()) + .unwrap_err() + .contains("x86_64 ELF") + ); + + let mut bzimage = vec![0u8; 0x238]; + bzimage[0x1fe..0x200].copy_from_slice(&[0x55, 0xaa]); + bzimage[0x202..0x206].copy_from_slice(b"HdrS"); + bzimage[0x206..0x208].copy_from_slice(&0x020cu16.to_le_bytes()); + bzimage[0x236] = 1; + std::fs::write(file.path(), &bzimage).unwrap(); + VmBackend::Qemu + .validate_kernel_image(file.path()) + .expect("x86_64 bzImage header"); + bzimage[0x236] = 0; + std::fs::write(file.path(), &bzimage).unwrap(); + assert!( + VmBackend::Qemu + .validate_kernel_image(file.path()) + .unwrap_err() + .contains("bzImage must target x86_64") + ); + for invalid in [b"".as_slice(), b"\x7fELF", b"not a kernel"] { + std::fs::write(file.path(), invalid).unwrap(); + assert!(VmBackend::Qemu.validate_kernel_image(file.path()).is_err()); + } + } + + #[test] + fn workload_and_preparation_entry_reject_incompatible_kernels_before_disk_setup() { + let file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(file.path(), b"not a kernel").unwrap(); + for mode in ["sandbox", "image-prep"] { + let mut config = qemu_config(); + config.kernel_image = Some(file.path().to_path_buf()); + config.gpu_bdf = None; + config.env = vec![format!("OPENSHELL_VM_INIT_MODE={mode}")]; + assert!( + run_vm(&config) + .unwrap_err() + .contains("unsupported QEMU kernel format") + ); + config.backend = VmBackend::Libkrun; + assert!( + run_vm(&config) + .unwrap_err() + .contains("libkrun does not support external kernel images") + ); + } + let mut config = qemu_config(); + config.vsock_port_map = Some(VsockPortMap { + guest_port: 1234, + host_socket: "/tmp/unused.sock".into(), + host_initiated: false, + }); + assert!( + run_vm(&config) + .unwrap_err() + .contains("Unix-socket vsock port maps require libkrun") + ); + } + #[test] fn qemu_guest_env_vars_omit_network_metadata() { let env = qemu_guest_env_vars(&qemu_config()); diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 70f6891ce1..30ae5b32d5 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -214,6 +214,12 @@ Podman also supports `image` mounts, which mount a container image read-only at The MicroVM driver runs each sandbox in its own lightweight VM. It requires host virtualization: Apple Hypervisor on macOS or KVM on Linux. +CPU-only VMs use libkrun by default. On Linux x86_64 hosts with KVM, +operator-installed lifecycle extensions can select QEMU when they need an +external x86_64 kernel, without requesting a GPU. +QEMU VM launches don't support virtiofs mounts. Kernel selection remains under +operator control; the guest must still pass authenticated runtime admission. + The VM driver is opt-in. Enable it in the gateway TOML file: ```toml