diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34e9af9d..1bc4542c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,37 @@ jobs: export DBUS_SESSION_BUS_ADDRESS="unix:path=$runtime_dir/bus" TINYBOX_REQUIRE_NATIVE_RESOURCE_TESTS=1 cargo test -j1 -p tinybox-module native_ TINYBOX_REQUIRE_NATIVE_RESOURCE_TESTS=1 cargo test -j1 -p tinybox-module resource_aware_bus_exec_cancellation_reports_events_and_reaps_descendants + bwrap-pasta-live: + name: Bubblewrap private network (pasta) reachability + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + submodules: recursive + persist-credentials: false + - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 + - name: Install bubblewrap, passt (pasta) and util-linux (nsenter) + run: | + set -euo pipefail + sudo apt-get install -y --no-install-recommends bubblewrap passt util-linux + # Ubuntu 24.04 denies unprivileged user namespaces to programs without + # an AppArmor profile for them, which is every bwrap jail here; the + # native-resource job lifts it the same way + # (scripts/ci/prepare-rootless-native-resources.sh). + if sysctl -n kernel.apparmor_restrict_unprivileged_userns >/dev/null 2>&1; then + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + fi + bwrap --version + pasta --version | head -1 + - name: Prove outbound works and host loopback, gateway and abstract sockets do not + env: + TINYBOX_REQUIRE_BWRAP: "1" + TINYBOX_REQUIRE_PASTA: "1" + run: | + set -euo pipefail + cargo test --locked -p tinybox-jail --test bwrap_pasta_live -- --nocapture + cargo test --locked -p tinybox-jail --test bwrap_isolation -- --nocapture + cargo test --locked -p tinybox-jail --test bwrap_fd_leak -- --nocapture docker-network-live: name: Docker Egress and denied network connectivity runs-on: ubuntu-22.04 diff --git a/Cargo.lock b/Cargo.lock index d4957cd0..20c63a72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,6 +285,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "command-fds" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b60b5124979fccd9addd89d8b97a1d6eebb4950694520c75ddd722535ea443f" +dependencies = [ + "nix 0.31.3", + "thiserror", +] + [[package]] name = "command-group" version = "5.0.1" @@ -1541,6 +1551,15 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "seccompiler" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae55de56877481d112a559bbc12667635fdaf5e005712fd4e2b2fa50ffc884" +dependencies = [ + "libc", +] + [[package]] name = "serde" version = "1.0.229" @@ -1907,9 +1926,13 @@ dependencies = [ name = "tinybox-jail" version = "0.2.6" dependencies = [ + "command-fds", "landlock", + "libc", "log", + "nix 0.31.3", "rustix", + "seccompiler", "serde", "serde_json", "tempfile", diff --git a/crates/tinybox-jail/Cargo.toml b/crates/tinybox-jail/Cargo.toml index 2be21df6..b483caa9 100644 --- a/crates/tinybox-jail/Cargo.toml +++ b/crates/tinybox-jail/Cargo.toml @@ -33,6 +33,19 @@ landlock = { version = "0.4", optional = true } # `NamespaceProbe` locates `bwrap`; reused so there is one definition of how the # Linux backends find it. tinybox-linux.workspace = true +# Hands the bwrap launcher the grant descriptors it binds (`--bind-fd`), mapped +# only in the child after fork; std cannot without `unsafe`, which the +# workspace forbids. Pinned: it runs in the jail's spawn path. +command-fds = "=0.3.3" +# `openat2(RESOLVE_NO_SYMLINKS)` for those descriptors, safely. Already in the +# lockfile, and the version command-fds builds on. +nix = { version = "0.31", default-features = false, features = ["fs"] } +# Compiles the seccomp deny list bwrap installs on every jail (`bwrap/seccomp.rs`); +# its only dependency is libc. Default features off: no JSON frontend. +seccompiler = { version = "0.5", default-features = false } +# The per-architecture syscall numbers that deny list names. Already in the +# lockfile through several dependencies. +libc = "0.2" [target.'cfg(windows)'.dependencies] # AppContainer profile APIs, ACL editing, STARTUPINFOEXW process spawn, and the diff --git a/crates/tinybox-jail/src/bwrap/args.rs b/crates/tinybox-jail/src/bwrap/args.rs index 1444d4fe..518286c3 100644 --- a/crates/tinybox-jail/src/bwrap/args.rs +++ b/crates/tinybox-jail/src/bwrap/args.rs @@ -14,6 +14,7 @@ use std::ffi::OsString; use std::io; +use std::os::fd::{OwnedFd, RawFd}; use std::path::{Path, PathBuf}; use crate::jail::Jail; @@ -44,12 +45,30 @@ pub(super) const ETC_ENTRIES: &[&str] = &[ ]; /// Switches that depend on the `bwrap` build or the host, decided by the probe. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Default)] pub(super) struct Options { /// Pass `--disable-userns` (bubblewrap 0.8+): the child cannot create /// nested user namespaces, which closes the usual escape route to a /// capability-bearing namespace. pub disable_userns: bool, + /// Bind the jail's grants from descriptors (`--bind-fd`/`--ro-bind-fd`) + /// opened here, instead of by path (see [`jail_args_with_fds`]). + pub bind_fds: bool, + /// Leave the host's `/etc/resolv.conf` out of the baseline: the jail runs + /// in a private network whose resolver is supplied separately. + pub private_resolv: bool, +} + +/// The first descriptor a grant is handed to `bwrap` on; the launcher keeps +/// this one and the ones after it open. +pub(super) const FIRST_GRANT_FD: RawFd = 10; + +/// A `bwrap` argument list, plus the descriptors it binds grants from, each +/// with the number the child must see it on. +#[derive(Debug, Default)] +pub(super) struct JailArgs { + pub argv: Vec, + pub fds: Vec<(OwnedFd, RawFd)>, } #[derive(Debug)] @@ -65,6 +84,8 @@ enum Kind { struct Mount { path: PathBuf, kind: Kind, + /// One of the jail's own grants, not a baseline system path. + grant: bool, } impl Mount { @@ -117,8 +138,8 @@ fn replaces_private_mount(path: &Path) -> bool { } /// The baseline mounts: system directories (or their symlinks) and the short -/// `/etc` allow-list. -fn baseline_mounts() -> Vec { +/// `/etc` allow-list, without `/etc/resolv.conf` when `private_resolv`. +fn baseline_mounts(private_resolv: bool) -> Vec { let mut mounts: Vec = Vec::new(); for dir in SYSTEM_DIRS { let path = PathBuf::from(dir); @@ -126,17 +147,23 @@ fn baseline_mounts() -> Vec { Ok(target) => Mount { path, kind: Kind::Symlink(target), + grant: false, }, Err(_) => Mount { path, kind: Kind::Bind { writable: false }, + grant: false, }, }); } for entry in ETC_ENTRIES { + if private_resolv && *entry == "/etc/resolv.conf" { + continue; + } mounts.push(Mount { path: PathBuf::from(entry), kind: Kind::Bind { writable: false }, + grant: false, }); } @@ -166,29 +193,37 @@ pub(super) fn jail_args( cwd: Option<&Path>, options: Options, ) -> io::Result> { - for path in std::iter::once(&jail.root) - .chain(&jail.read_write) - .chain(&jail.read_only) - .chain(&jail.deny_read) - .map(PathBuf::as_path) - .chain(cwd) - { - require_clean(path)?; - } - // A grant inside a writable one must not reach its target through a - // symlink the jailed program could have planted. - let writable: Vec<&Path> = std::iter::once(&jail.root) - .chain(&jail.read_write) - .map(PathBuf::as_path) - .collect(); - let nested: Vec<&Path> = jail - .read_only - .iter() - .chain(&jail.read_write) - .map(PathBuf::as_path) - .collect(); - crate::jail::reject_symlinked_grants(&writable, &nested)?; - let mut mounts = baseline_mounts(); + let paths_only = Options { + bind_fds: false, + ..options + }; + jail_args_with_fds(jail, cwd, paths_only).map(|args| args.argv) +} + +/// [`jail_args`], with the grants bound from descriptors when +/// `options.bind_fds` is set. +/// +/// A path check followed by `bwrap --bind PATH` leaves a window in which a +/// program with write access to a grant's parent can swap a directory for a +/// symlink. Here each grant is opened once, as an `O_PATH` descriptor, and +/// `bwrap` binds that descriptor, so what was checked is what is mounted. A +/// grant inside an already-opened grant is opened relative to it with +/// `RESOLVE_NO_SYMLINKS | RESOLVE_BENEATH`, so no symlink on that stretch is +/// followed (another sandbox may be able to write there even when this one +/// cannot); any other grant is opened by its declared path, whose ancestors +/// the host controls. A missing read-only grant is skipped, as `--ro-bind-try` +/// skips it. +/// +/// # Errors +/// As [`jail_args`], plus the error from opening a grant (`ELOOP` when a +/// symlink was planted inside a writable grant). +pub(super) fn jail_args_with_fds( + jail: &Jail, + cwd: Option<&Path>, + options: Options, +) -> io::Result { + check_paths(jail, cwd)?; + let mut mounts = baseline_mounts(options.private_resolv); for (path, writable) in std::iter::once((&jail.root, true)) .chain(jail.read_write.iter().map(|p| (p, true))) .chain(jail.read_only.iter().map(|p| (p, false))) @@ -211,6 +246,7 @@ pub(super) fn jail_args( mounts.push(Mount { path: path.clone(), kind: Kind::Bind { writable }, + grant: true, }); } @@ -240,6 +276,7 @@ pub(super) fn jail_args( argv.extend([os(flag), os(path)]); } + let mut fds = GrantFds::default(); for mount in &mounts { match &mount.kind { Kind::Symlink(target) => argv.extend([ @@ -247,6 +284,20 @@ pub(super) fn jail_args( target.clone().into_os_string(), mount.path.clone().into_os_string(), ]), + Kind::Bind { writable } if options.bind_fds && mount.grant => { + let Some(fd) = fds.open(&mount.path, *writable)? else { + continue; + }; + argv.extend([ + os(if *writable { + "--bind-fd" + } else { + "--ro-bind-fd" + }), + fd.to_string().into(), + mount.path.clone().into_os_string(), + ]); + } Kind::Bind { writable } => argv.extend([ os(if *writable { "--bind" } else { "--ro-bind-try" }), mount.path.clone().into_os_string(), @@ -261,7 +312,89 @@ pub(super) fn jail_args( os("--chdir"), cwd.unwrap_or(&jail.root).as_os_str().to_os_string(), ]); - Ok(argv) + Ok(JailArgs { + argv, + fds: fds.into_mappings(), + }) +} + +/// Every path is absolute and clean, and no grant inside a writable one +/// reaches its target through a symlink the jailed program could have planted. +fn check_paths(jail: &Jail, cwd: Option<&Path>) -> io::Result<()> { + for path in std::iter::once(&jail.root) + .chain(&jail.read_write) + .chain(&jail.read_only) + .chain(&jail.deny_read) + .map(PathBuf::as_path) + .chain(cwd) + { + require_clean(path)?; + } + let writable: Vec<&Path> = std::iter::once(&jail.root) + .chain(&jail.read_write) + .map(PathBuf::as_path) + .collect(); + let nested: Vec<&Path> = jail + .read_only + .iter() + .chain(&jail.read_write) + .map(PathBuf::as_path) + .collect(); + crate::jail::reject_symlinked_grants(&writable, &nested) +} + +/// The grant descriptors opened so far, shallowest first, numbered from +/// [`FIRST_GRANT_FD`]. +#[derive(Default)] +struct GrantFds { + opened: Vec<(PathBuf, OwnedFd)>, +} + +impl GrantFds { + /// Opens `path` and returns the number the child will see it on, or + /// `None` for a missing read-only grant. + fn open(&mut self, path: &Path, writable: bool) -> io::Result> { + use nix::fcntl::{OFlag, OpenHow, ResolveFlag}; + let flags = OFlag::O_PATH | OFlag::O_CLOEXEC; + // The deepest grant already opened that contains this one. + let parent = self + .opened + .iter() + .filter(|(dir, _)| path.starts_with(dir) && path != dir) + .max_by_key(|(dir, _)| dir.components().count()); + let opened = match parent { + Some((dir, fd)) => nix::fcntl::openat2( + fd, + path.strip_prefix(dir).map_err(io::Error::other)?, + OpenHow::new() + .flags(flags) + .resolve(ResolveFlag::RESOLVE_NO_SYMLINKS | ResolveFlag::RESOLVE_BENEATH), + ), + None => nix::fcntl::openat2(nix::fcntl::AT_FDCWD, path, OpenHow::new().flags(flags)), + }; + let fd = match opened { + Ok(fd) => fd, + Err(nix::errno::Errno::ENOENT) if !writable => return Ok(None), + Err(errno) => { + return Err(io::Error::new( + io::Error::from(errno).kind(), + format!("cannot open jail grant {}: {errno}", path.display()), + )); + } + }; + let number = + FIRST_GRANT_FD + RawFd::try_from(self.opened.len()).map_err(io::Error::other)?; + self.opened.push((path.to_path_buf(), fd)); + Ok(Some(number)) + } + + fn into_mappings(self) -> Vec<(OwnedFd, RawFd)> { + self.opened + .into_iter() + .zip(FIRST_GRANT_FD..) + .map(|((_, fd), number)| (fd, number)) + .collect() + } } /// `deny_read` plus, for each entry under a baseline symlink, the same path diff --git a/crates/tinybox-jail/src/bwrap/args_tests.rs b/crates/tinybox-jail/src/bwrap/args_tests.rs index c52adde5..68ba022d 100644 --- a/crates/tinybox-jail/src/bwrap/args_tests.rs +++ b/crates/tinybox-jail/src/bwrap/args_tests.rs @@ -4,6 +4,8 @@ use super::*; const OPTS: Options = Options { disable_userns: true, + bind_fds: false, + private_resolv: false, }; fn build(jail: &Jail) -> io::Result> { @@ -55,6 +57,8 @@ fn disable_userns_is_omitted_when_the_probe_found_no_support() -> io::Result<()> None, Options { disable_userns: false, + bind_fds: false, + private_resolv: false, }, )?; assert!(!argv.iter().any(|a| a == "--disable-userns")); @@ -348,3 +352,110 @@ fn a_denial_through_a_baseline_symlink_also_covers_the_link_target() -> io::Resu ); Ok(()) } + +const FD_OPTS: Options = Options { + disable_userns: true, + bind_fds: true, + private_resolv: false, +}; + +fn strings(argv: &[OsString]) -> Vec { + argv.iter() + .map(|a| a.to_string_lossy().into_owned()) + .collect() +} + +#[test] +fn grants_are_bound_from_descriptors_and_baseline_paths_by_path() -> io::Result<()> { + let root = tempfile::tempdir()?; + let nested = root.path().join("vendor"); + std::fs::create_dir(&nested)?; + let j = jail(root.path()).add_read_only(&nested); + + let built = jail_args_with_fds(&j, None, FD_OPTS)?; + let argv = strings(&built.argv); + + let numbers: Vec = built.fds.iter().map(|(_, number)| *number).collect(); + assert_eq!(numbers, vec![FIRST_GRANT_FD, FIRST_GRANT_FD + 1]); + let root_at = argv + .windows(3) + .position(|w| w[0] == "--bind-fd" && w[1] == "10" && Path::new(&w[2]) == root.path()); + let nested_at = argv + .windows(3) + .position(|w| w[0] == "--ro-bind-fd" && w[1] == "11" && Path::new(&w[2]) == nested); + assert!(root_at.is_some() && nested_at.is_some(), "{argv:?}"); + assert!( + root_at < nested_at, + "the nested grant must land on top: {argv:?}" + ); + // Neither grant is also bound by path, which bwrap would re-resolve. + assert_eq!(triple(&argv, "--bind", root.path()), None); + assert_eq!(triple(&argv, "--ro-bind-try", &nested), None); + // Baseline system paths are still bound by path. + assert!(argv.iter().any(|a| a == "--ro-bind-try"), "{argv:?}"); + Ok(()) +} + +#[test] +fn a_symlink_swapped_in_after_the_check_is_not_followed() -> io::Result<()> { + let root = tempfile::tempdir()?; + let outside = tempfile::tempdir()?; + let nested = root.path().join("vendor"); + let mut fds = GrantFds::default(); + assert_eq!(fds.open(root.path(), true)?, Some(FIRST_GRANT_FD)); + + // What `reject_symlinked_grants` checked was a directory; by the time the + // grant is opened another program has replaced it with a link out. + std::os::unix::fs::symlink(outside.path(), &nested)?; + let error = fds + .open(&nested, false) + .err() + .ok_or_else(|| io::Error::other("a symlinked nested grant was opened"))?; + + assert!(error.to_string().contains("vendor"), "{error}"); + assert!( + error.to_string().contains("ELOOP") || error.to_string().contains("symbolic links"), + "{error}" + ); + Ok(()) +} + +#[test] +fn a_symlink_above_a_grant_is_left_to_the_host() -> io::Result<()> { + // A top-level grant spelled through a host symlink (`/var/run` style) + // still opens: only the stretch inside another grant is held to no links. + let real = tempfile::tempdir()?; + let links = tempfile::tempdir()?; + let link = links.path().join("work"); + std::os::unix::fs::symlink(real.path(), &link)?; + + let mut fds = GrantFds::default(); + assert_eq!(fds.open(&link, true)?, Some(FIRST_GRANT_FD)); + Ok(()) +} + +#[test] +fn a_missing_read_only_grant_is_skipped_as_ro_bind_try_skips_it() -> io::Result<()> { + let root = tempfile::tempdir()?; + let missing = root.path().join("not-there"); + let j = jail(root.path()).add_read_only(&missing); + + let args = jail_args_with_fds(&j, None, FD_OPTS)?; + + assert_eq!(args.fds.len(), 1, "only the root is opened"); + assert!( + !strings(&args.argv).iter().any(|a| Path::new(a) == missing), + "a missing grant reaches bwrap" + ); + Ok(()) +} + +#[test] +fn without_descriptor_support_grants_are_bound_by_path() -> io::Result<()> { + let root = tempfile::tempdir()?; + let args = jail_args_with_fds(&jail(root.path()), None, OPTS)?; + + assert!(args.fds.is_empty()); + assert!(triple(&strings(&args.argv), "--bind", root.path()).is_some()); + Ok(()) +} diff --git a/crates/tinybox-jail/src/bwrap/launcher.rs b/crates/tinybox-jail/src/bwrap/launcher.rs index 2d56330f..dc7bf474 100644 --- a/crates/tinybox-jail/src/bwrap/launcher.rs +++ b/crates/tinybox-jail/src/bwrap/launcher.rs @@ -27,13 +27,15 @@ use std::path::{Path, PathBuf}; use std::process::Command; /// The script: open and delete the args file, close every other inherited -/// descriptor above 2, then `exec bwrap --args 9 -- COMMAND...`. `$0` is -/// `bwrap`, `$1` the args file, the rest the command and its arguments (bwrap +/// descriptor above 2 except the `$2` grant descriptors from 10 up (see +/// [`super::args::FIRST_GRANT_FD`]), then `exec bwrap --args 9 -- COMMAND...`. +/// `$0` is `bwrap`, `$1` the args file, the rest the command and its arguments (bwrap /// reads options from the file but the command itself must be on its argv; a /// command line is visible to every local user anyway, in the sandbox as on /// any host). const SCRIPT: &str = r#"f=$1 -shift +k=$2 +shift 2 exec 9<"$f" || exit 126 command -p rm -f -- "$f" command -p rmdir -- "${f%/*}" 2>/dev/null @@ -42,7 +44,9 @@ for fd in /proc/self/fd/*; do n=${fd##*/} case $n in 0|1|2|9) ;; - *) eval "exec $n>&-" || exit 126 ;; + *) if [ "$n" -ge 10 ] && [ "$n" -lt $((10 + k)) ]; then :; else + eval "exec $n>&-" || exit 126 + fi ;; esac done exec "$0" --args 9 -- "$@" @@ -157,22 +161,48 @@ impl ArgsFile { } } +/// Namespaces the launcher joins before it runs: `nsenter` enters the user +/// and network namespace open on these child descriptors (by descriptor, not +/// by pid, so a reused pid cannot redirect it), and keeps every other +/// descriptor. The script then closes these two, as it closes everything +/// outside its keep list. +#[derive(Debug, Clone, Copy)] +pub(super) struct Enter<'a> { + pub nsenter: &'a Path, + pub user_fd: i32, + pub net_fd: i32, +} + /// The launcher command: an empty environment, the script, `bwrap`, the file, -/// then the command to run inside the sandbox and its arguments. +/// how many grant descriptors to keep, then the command to run inside the +/// sandbox and its arguments. With `enter`, the shell runs under `nsenter`. pub(super) fn command( + enter: Option>, shell: &str, bwrap: &Path, args: &ArgsFile, + grant_fds: usize, program: &std::ffi::OsStr, program_args: &[OsString], ) -> Command { - let mut launcher = Command::new(shell); + let mut launcher = match enter { + Some(enter) => { + let mut nsenter = Command::new(enter.nsenter); + nsenter + .arg(format!("--user=/proc/self/fd/{}", enter.user_fd)) + .arg(format!("--net=/proc/self/fd/{}", enter.net_fd)) + .args(["--preserve-credentials", "--", shell]); + nsenter + } + None => Command::new(shell), + }; launcher .env_clear() .arg("-c") .arg(SCRIPT) .arg(bwrap) .arg(args.path()) + .arg(grant_fds.to_string()) .arg(program) .args(program_args); launcher diff --git a/crates/tinybox-jail/src/bwrap/launcher_tests.rs b/crates/tinybox-jail/src/bwrap/launcher_tests.rs index 2eddea54..7219704c 100644 --- a/crates/tinybox-jail/src/bwrap/launcher_tests.rs +++ b/crates/tinybox-jail/src/bwrap/launcher_tests.rs @@ -26,7 +26,16 @@ fn run(shell: &str, body: &str, items: &[&str]) -> io::Result { let bwrap = stand_in(dir.path(), body)?; let argv: Vec = items.iter().map(OsString::from).collect(); let file = ArgsFile::create(&argv, &tmp())?; - let out = command(shell, &bwrap, &file, "prog".as_ref(), &["a".into()]).output()?; + let out = command( + None, + shell, + &bwrap, + &file, + 0, + "prog".as_ref(), + &["a".into()], + ) + .output()?; file.discard(); Ok(String::from_utf8_lossy(&out.stdout).into_owned()) } @@ -81,7 +90,7 @@ fn the_args_file_is_gone_once_the_launcher_has_started_bwrap() -> io::Result<()> let file = ArgsFile::create(&["x".into()], &tmp())?; let path = file.path().to_path_buf(); assert!( - command(shell(), &bwrap, &file, "prog".as_ref(), &[]) + command(None, shell(), &bwrap, &file, 0, "prog".as_ref(), &[]) .status()? .success() ); @@ -94,7 +103,7 @@ fn the_launcher_runs_with_an_empty_environment() -> io::Result<()> { let dir = tempfile::tempdir()?; let bwrap = stand_in(dir.path(), "env")?; let file = ArgsFile::create(&[], &tmp())?; - let out = command(shell(), &bwrap, &file, "prog".as_ref(), &[]).output()?; + let out = command(None, shell(), &bwrap, &file, 0, "prog".as_ref(), &[]).output()?; file.discard(); let text = String::from_utf8_lossy(&out.stdout); assert!( @@ -137,7 +146,7 @@ fn a_launcher_that_cannot_open_the_args_file_fails_closed_without_running_bwrap( dir: dir.path().join("gone"), file: dir.path().join("gone").join("args"), }; - let status = command(shell(), &bwrap, &missing, "prog".as_ref(), &[]).status()?; + let status = command(None, shell(), &bwrap, &missing, 0, "prog".as_ref(), &[]).status()?; assert_eq!(status.code(), Some(126)); assert!(!marker.exists(), "bwrap must not run without its arguments"); Ok(()) @@ -195,3 +204,94 @@ fn an_args_directory_reached_through_a_symlink_into_a_grant_is_refused() -> io:: assert_eq!(chosen, temp.path().canonicalize()?); Ok(()) } + +#[test] +fn the_grant_descriptors_reach_bwrap_and_nothing_else_does() -> io::Result<()> { + let shell = shell(); + if !shell.ends_with("bash") { + eprintln!("skipped: no bash"); + return Ok(()); + } + let dir = tempfile::tempdir()?; + let granted = tempfile::tempdir()?; + let leaked = tempfile::NamedTempFile::new()?; + // An inheritable descriptor the launcher must still close. + let stray = std::fs::File::open(leaked.path())?; + rustix::io::fcntl_setfd(&stray, rustix::io::FdFlags::empty())?; + let bwrap = stand_in(dir.path(), "ls -l /proc/self/fd")?; + let file = ArgsFile::create(&[], &tmp())?; + let mut launcher = command(None, shell, &bwrap, &file, 2, "prog".as_ref(), &[]); + let mappings = vec![ + command_fds::FdMapping { + parent_fd: std::fs::File::open(granted.path())?.into(), + child_fd: 10, + }, + command_fds::FdMapping { + parent_fd: std::fs::File::open(granted.path())?.into(), + child_fd: 11, + }, + ]; + command_fds::CommandFdExt::fd_mappings(&mut launcher, mappings).map_err(io::Error::other)?; + + let out = launcher.output()?; + file.discard(); + drop(stray); + let text = String::from_utf8_lossy(&out.stdout); + let granted = granted.path().display().to_string(); + for fd in ["10 ->", "11 ->"] { + assert!( + text.lines().any(|l| l.contains(fd) && l.contains(&granted)), + "{fd} {text}" + ); + } + assert!( + !text.contains(&leaked.path().display().to_string()), + "{text}" + ); + Ok(()) +} + +#[test] +fn entering_namespaces_puts_nsenter_in_front_by_descriptor() -> io::Result<()> { + let file = ArgsFile::create(&[], &tmp())?; + let enter = Enter { + nsenter: Path::new("/usr/bin/nsenter"), + user_fd: 14, + net_fd: 15, + }; + let launcher = command( + Some(enter), + "/bin/bash", + Path::new("/usr/bin/bwrap"), + &file, + 4, + "prog".as_ref(), + &[], + ); + file.discard(); + assert_eq!(launcher.get_program(), "/usr/bin/nsenter"); + assert_eq!( + launcher.get_envs().count(), + 0, + "no environment reaches nsenter" + ); + let argv: Vec = launcher + .get_args() + .map(|a| a.to_string_lossy().into_owned()) + .collect(); + assert_eq!( + &argv[..6], + [ + "--user=/proc/self/fd/14", + "--net=/proc/self/fd/15", + "--preserve-credentials", + "--", + "/bin/bash", + "-c", + ] + ); + // The launcher keeps only the 4 descriptors from 10, so the namespace + // descriptors on 14 and 15 are closed before bwrap runs. + assert_eq!(argv.get(9).map(String::as_str), Some("4")); + Ok(()) +} diff --git a/crates/tinybox-jail/src/bwrap/mod.rs b/crates/tinybox-jail/src/bwrap/mod.rs index dab0b501..f2ba11f8 100644 --- a/crates/tinybox-jail/src/bwrap/mod.rs +++ b/crates/tinybox-jail/src/bwrap/mod.rs @@ -1,8 +1,10 @@ //! Linux bubblewrap backend: namespace isolation for untrusted code. //! //! [`BubblewrapBackend`] runs the command under `bwrap` with fresh user, pid, -//! ipc and uts namespaces (and a net namespace unless the jail allows -//! network). The child sees a private mount tree holding only the baseline +//! ipc and uts namespaces, and a network namespace of its own: an empty one +//! when the jail denies network, or one with user-mode networking from `pasta` +//! when it allows network and `pasta` works here (see "Private network"). +//! The child sees a private mount tree holding only the baseline //! system paths plus the jail's grants, so it cannot read an ungranted path, //! cannot see host processes, cannot create nested user namespaces //! (`--disable-userns`), and dies with its parent. @@ -36,34 +38,61 @@ //! [`JailBackend::isolation`] is the **minimum over every configuration the //! backend accepts**. It is reported as `Process`, the closest defined level //! below `Kernel`, but note what it means here: pid, mount, user and ipc -//! namespaces and a private filesystem view, with the host *network* shared and -//! no seccomp. That is more than the plain `Process` doc describes and less than +//! namespaces, a private filesystem view and the seccomp deny list, with the +//! host *network* shared (an `allow_net` jail where `pasta` is unavailable). That is more than the plain `Process` doc describes and less than //! `Kernel`; do not read it as "same process table". `Process` with `--disable-userns` support (bubblewrap //! 0.8+), `None` without it. [`JailBackend::isolation_for`] reaches -//! [`IsolationLevel::Kernel`] only for a jail that denies network (a new net -//! namespace). With `allow_net` the host network namespace is shared, so host -//! loopback services and abstract sockets are reachable. Hosts must ask +//! [`IsolationLevel::Kernel`] for a jail that denies network (an empty net +//! namespace), and for one that allows it when +//! [`BubblewrapBackend::networks_privately`] (a `pasta` network). Otherwise an +//! `allow_net` jail shares the host network namespace, so host loopback +//! services and abstract sockets are reachable, and it stays `Process`. Hosts must ask //! [`JailBackend::is_suitable_for_untrusted_jail`]; the per-backend //! `is_suitable_for_untrusted_code` is false by construction. `Kernel` here -//! means separate pid, mount, user, ipc and uts (and net) namespaces: it does -//! not imply seccomp or resource limits. +//! means separate pid, mount, user, ipc and uts (and net) namespaces plus that +//! deny list: it does not imply resource limits. +//! +//! # Private network +//! +//! When the jail allows network and a trusted `pasta` and `nsenter` are found +//! (at fixed root-owned paths, never on `PATH`) and pass a probe, `pasta` +//! starts a holder that owns a fresh user and network namespace, and the +//! launcher enters it with `nsenter` before running `bwrap` (see `pasta.rs`). +//! `pasta` is started only with no port forwarding, no gateway mapping and DNS +//! through one forwarded address, and the jail gets its own +//! `/etc/resolv.conf` naming it. So outbound connections work, while host +//! loopback services, the host behind the gateway address and the host's +//! abstract sockets are unreachable. The holder lives exactly as long as +//! `bwrap` (`--sync-fd`), so no `pasta` outlives a jail, even one that is +//! killed. `slirp4netns` is not used: it only attaches to an existing +//! namespace, which Ubuntu's `AppArmor` user-namespace restriction refuses. +//! +//! # Seccomp +//! +//! Every jail starts under a seccomp deny list (`seccomp.rs`): kernel keyrings, +//! other processes' memory, `bpf`/`perf`/`userfaultfd`/`io_uring`, every mount +//! call, `setns` and namespace-creating `clone`/`unshare`, file handles, +//! modules, `kexec`, reboot, host clock, swap and syslog, terminal injection +//! ioctls, and the x86 legacy calls. Denied calls fail with `EPERM` (`clone3` +//! with `ENOSYS`, so the C library falls back to `clone`). `ptrace` stays +//! allowed on Linux 4.8 and later, confined to the jail's pid namespace. A +//! `bwrap` without `--add-seccomp-fd`, or an architecture other than `x86_64` +//! and `aarch64`, leaves the backend unavailable rather than unfiltered. //! //! # Known gaps (not closed in this backend) //! -//! - No seccomp filter: `keyctl` (the user's kernel keyrings), `ptrace`, -//! `bpf`, `io_uring` and similar syscalls remain reachable. A filter needs a -//! file descriptor handed to `bwrap --seccomp`; the launcher could carry one, -//! but a vetted BPF program is not part of this change. //! - No CPU, memory or pid limits (`tinybox-linux` offers them through a //! `systemd-run` scope; not wired here). -//! - Check-then-use gap on nested grants: the symlink check on a grant inside a -//! writable directory (`reject_symlinked_grants`) runs before `bwrap` binds -//! the path, so a program that is already running in another session could -//! swap a directory for a symlink in between. Closing it needs `O_PATH` -//! descriptors opened with `openat2(RESOLVE_NO_SYMLINKS)` and -//! `--bind-fd`/`--ro-bind-fd` (bubblewrap 0.10+); that is follow-up work. -//! The window is only open while another sandbox with write access to the -//! same tree is running. +//! - Check-then-use gap on nested grants, **only where `bwrap` lacks +//! `--bind-fd` or the kernel lacks `openat2` (before Linux 5.6)**. Then the +//! symlink check on a grant inside a writable directory +//! (`reject_symlinked_grants`) runs before `bwrap` binds the path, so a +//! program already running in another session could swap a directory for a +//! symlink in between. Otherwise each grant is opened once as an `O_PATH` +//! descriptor, a nested one beneath its parent's with +//! `RESOLVE_NO_SYMLINKS | RESOLVE_BENEATH`, and bound with +//! `--bind-fd`/`--ro-bind-fd`, so what was checked is what is mounted. +//! [`BubblewrapBackend::binds_by_descriptor`] reports which is in effect. //! - `deny_read` paths must exist when the jail starts: a path created later //! under a mounted grant is **not** protected. One nested more than a level //! inside a writable grant is refused. @@ -96,6 +125,8 @@ use crate::jail::{Jail, JailBackend}; mod args; mod launcher; +mod pasta; +mod seccomp; /// Backend name reported by [`BubblewrapBackend`]. pub const BWRAP_BACKEND_NAME: &str = "bubblewrap"; @@ -111,6 +142,14 @@ static DEFAULT_PROBE: OnceLock> = OnceLock::new(); struct Verified { program: PathBuf, disable_userns: bool, + /// `bwrap` lists `--bind-fd` and the kernel has `openat2`: grants are + /// bound from descriptors rather than re-resolved by path. + bind_fds: bool, + /// The compiled seccomp filters every jail starts under (see `seccomp`). + seccomp: Vec>, + /// A working `pasta` and `nsenter`: an `allow_net` jail then gets a + /// private network instead of the host's (see `pasta`). + pasta: Option, } /// Bubblewrap (`bwrap`) namespace jail backend. @@ -166,6 +205,33 @@ impl BubblewrapBackend { self.verified.as_ref().map(|v| v.program.as_path()) } + /// Whether grants are bound from descriptors opened at spawn + /// (`--bind-fd`, bubblewrap 0.9+ on Linux 5.6+), which closes the + /// check-then-use gap on nested grants. When false they are bound by path + /// and the gap stays open (see the module docs). + #[must_use] + pub fn binds_by_descriptor(&self) -> bool { + self.verified.as_ref().is_some_and(|v| v.bind_fds) + } + + /// Whether a jail that allows network runs in a private network namespace + /// with user-mode networking from `pasta`, rather than sharing the host's. + /// Then host loopback services and abstract sockets are out of its reach + /// and [`JailBackend::isolation_for`] reports `Kernel` for it too. + #[must_use] + pub fn networks_privately(&self) -> bool { + self.verified.as_ref().is_some_and(|v| v.pasta.is_some()) + } + + /// Whether jails start under the seccomp deny list. True whenever the + /// backend is available: a `bwrap` that cannot load it is not used. + #[must_use] + pub fn installs_seccomp(&self) -> bool { + self.verified + .as_ref() + .is_some_and(|v| !v.seccomp.is_empty()) + } + /// Test constructor that skips the probe, for fake launchers. #[cfg(test)] fn unverified(program: impl Into, disable_userns: bool) -> Self { @@ -173,6 +239,9 @@ impl BubblewrapBackend { verified: Some(Verified { program: program.into(), disable_userns, + bind_fds: false, + seccomp: Vec::new(), + pasta: None, }), } } @@ -188,18 +257,96 @@ fn is_trusted_binary(path: &Path) -> bool { /// Runs throwaway sandboxes to learn whether `program` really isolates, and /// whether it supports `--disable-userns`. +/// +/// A `bwrap` that cannot install the seccomp filters (no `--add-seccomp-fd`, +/// or no filter for this architecture) is not used: a jail is refused rather +/// than started unfiltered. fn verify(program: &Path) -> Option { + let seccomp = match seccomp_filters(program) { + Ok(filters) => filters, + Err(error) => { + log::debug!("[cwd_jail:bwrap] unavailable: {error}"); + return None; + } + }; for disable_userns in [true, false] { if probe(program, disable_userns) { return Some(Verified { program: program.to_path_buf(), disable_userns, + bind_fds: supports_fd_binds(program), + seccomp, + pasta: private_network(program), }); } } None } +/// A `pasta` that really brings up a private network, when there is one and +/// `program` can hold it open (`--sync-fd`) and mount the jail's resolver +/// (`--ro-bind-data`). Without one, an `allow_net` jail shares the host network. +fn private_network(program: &Path) -> Option { + let found = pasta::Pasta::find(is_trusted_binary)?; + let flags = help_lists(program, "--sync-fd") && help_lists(program, "--ro-bind-data"); + let works = flags && found.start().is_ok(); + log::debug!("[cwd_jail:bwrap] pasta private network flags={flags} works={works}"); + works.then_some(found) +} + +/// The filters for this kernel and architecture, if `program` can load them. +fn seccomp_filters(program: &Path) -> io::Result>> { + if !help_lists(program, "--add-seccomp-fd") { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "{} cannot load seccomp filters (no --add-seccomp-fd); jails are refused", + program.display() + ), + )); + } + let release = std::fs::read_to_string("/proc/sys/kernel/osrelease")?; + let kernel = seccomp::kernel_version(&release).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("unreadable kernel release {release:?}"), + ) + })?; + seccomp::filters(kernel) +} + +/// Whether `program --help` names `flag` as an option. +fn help_lists(program: &Path, flag: &str) -> bool { + Command::new(program) + .arg("--help") + .env_clear() + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .output() + .is_ok_and(|out| { + String::from_utf8_lossy(&out.stdout) + .split_whitespace() + .any(|word| word == flag) + }) +} + +/// Whether `program --help` lists `--bind-fd` (bubblewrap 0.9 does) and +/// `openat2` works on this kernel (Linux 5.6+). +fn supports_fd_binds(program: &Path) -> bool { + let listed = help_lists(program, "--bind-fd"); + let openat2 = nix::fcntl::openat2( + nix::fcntl::AT_FDCWD, + "/", + nix::fcntl::OpenHow::new().flags(nix::fcntl::OFlag::O_PATH | nix::fcntl::OFlag::O_CLOEXEC), + ) + .is_ok(); + log::debug!( + "[cwd_jail:bwrap] {} --bind-fd listed={listed} openat2={openat2}", + program.display() + ); + listed && openat2 +} + fn probe(program: &Path, disable_userns: bool) -> bool { // A fresh, private, randomly named directory: never a predictable path // another user could pre-create or symlink. @@ -241,7 +388,11 @@ fn run_probe(program: &Path, disable_userns: bool, root: &Path, host_only: &Path return false; }; let jail = Jail::new(root, "bwrap-probe").deny_net(); - let Ok(argv) = args::jail_args(&jail, None, args::Options { disable_userns }) else { + let options = args::Options { + disable_userns, + ..args::Options::default() + }; + let Ok(argv) = args::jail_args(&jail, None, options) else { return false; }; let check: Vec = [ @@ -263,7 +414,7 @@ fn run_probe(program: &Path, disable_userns: bool, root: &Path, host_only: &Path let shell = launcher::pick_shell(|path| Path::new(path).is_file()); // Positive evidence is required: a launcher that exits 0 without running // the check (`true`) must not pass. - let out = launcher::command(shell, program, &file, "/bin/sh".as_ref(), &check) + let out = launcher::command(None, shell, program, &file, 0, "/bin/sh".as_ref(), &check) .stdin(Stdio::null()) .stderr(Stdio::null()) .output(); @@ -289,14 +440,36 @@ struct Prepared { /// /// The launcher command carries no environment and its own argv holds only /// paths; the caller's variables and arguments live in the private args file. -fn prepare_command(verified: &Verified, jail: &Jail, cmd: &Command) -> io::Result { - let mut argv = args::jail_args( +fn prepare_command( + verified: &Verified, + jail: &Jail, + cmd: &Command, + network: Option<&pasta::Network>, +) -> io::Result { + let args::JailArgs { mut argv, mut fds } = args::jail_args_with_fds( jail, cmd.get_current_dir(), args::Options { disable_userns: verified.disable_userns, + bind_fds: verified.bind_fds, + private_resolv: network.is_some(), }, )?; + // Each filter on its own pipe, numbered after the grants. A filter is a + // few kilobytes, well inside a pipe's buffer, so writing it before bwrap + // reads cannot block. + for filter in &verified.seccomp { + let (reader, mut writer) = std::io::pipe()?; + io::Write::write_all(&mut writer, filter)?; + drop(writer); + let number = args::FIRST_GRANT_FD + i32::try_from(fds.len()).map_err(io::Error::other)?; + argv.extend(["--add-seccomp-fd".into(), number.to_string().into()]); + fds.push((reader.into(), number)); + } + let namespaces = match network { + Some(network) => Some(join_network(network, &mut argv, &mut fds)?), + None => None, + }; argv.push("--clearenv".into()); for (key, value) in cmd.get_envs() { if let Some(value) = value { @@ -313,19 +486,72 @@ fn prepare_command(verified: &Verified, jail: &Jail, cmd: &Command) -> io::Resul let file = launcher::ArgsFile::create(&argv, &base)?; let shell = launcher::pick_shell(|path| Path::new(path).is_file()); let program_args: Vec = cmd.get_args().map(OsString::from).collect(); - let launcher = launcher::command( + // The namespace descriptors are numbered after the kept ones, so the + // launcher closes them once `nsenter` has entered. + let keep = fds.len(); + let enter = match (namespaces, verified.pasta.as_ref()) { + (Some((user, net)), Some(pasta)) => { + let user_fd = args::FIRST_GRANT_FD + i32::try_from(keep).map_err(io::Error::other)?; + fds.push((user, user_fd)); + fds.push((net, user_fd + 1)); + Some(launcher::Enter { + nsenter: pasta.nsenter(), + user_fd, + net_fd: user_fd + 1, + }) + } + _ => None, + }; + let mut launcher = launcher::command( + enter, shell, &verified.program, &file, + keep, cmd.get_program(), &program_args, ); + // Mapped onto 10, 11, … in the child only, after fork: never inheritable + // in this process, where another thread could spawn concurrently. + let mappings = fds + .into_iter() + .map(|(parent_fd, child_fd)| command_fds::FdMapping { + parent_fd, + child_fd, + }) + .collect(); + command_fds::CommandFdExt::fd_mappings(&mut launcher, mappings) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; Ok(Prepared { launcher, args: file, }) } +/// Wires a pasta network into the `bwrap` arguments: the jail's resolver on a +/// pipe (`--ro-bind-data`) and the holder's writer kept by `bwrap` +/// (`--sync-fd`), both numbered after the descriptors already in `fds`. +/// Returns the user and network namespace descriptors for `nsenter`. +fn join_network( + network: &pasta::Network, + argv: &mut Vec, + fds: &mut Vec<(std::os::fd::OwnedFd, i32)>, +) -> io::Result<(std::os::fd::OwnedFd, std::os::fd::OwnedFd)> { + let (user, net, writer) = network.descriptors()?; + let (reader, mut resolv) = std::io::pipe()?; + io::Write::write_all(&mut resolv, pasta::resolv_conf().as_bytes())?; + drop(resolv); + for (fd, flag) in [(reader.into(), "--ro-bind-data"), (writer, "--sync-fd")] { + let number = args::FIRST_GRANT_FD + i32::try_from(fds.len()).map_err(io::Error::other)?; + argv.extend([flag.into(), number.to_string().into()]); + if flag == "--ro-bind-data" { + argv.push("/etc/resolv.conf".into()); + } + fds.push((fd, number)); + } + Ok((user, net)) +} + impl JailBackend for BubblewrapBackend { fn carves_denials(&self) -> bool { true @@ -361,7 +587,9 @@ impl JailBackend for BubblewrapBackend { fn isolation_for(&self, jail: &Jail) -> IsolationLevel { match self.isolation() { - IsolationLevel::Process if !jail.allow_net => IsolationLevel::Kernel, + IsolationLevel::Process if !jail.allow_net || self.networks_privately() => { + IsolationLevel::Kernel + } level => level, } } @@ -389,10 +617,16 @@ impl BubblewrapBackend { .require(self.name(), &jail.constraints()) .map_err(|error| io::Error::new(io::ErrorKind::Unsupported, error))?; log::debug!("[cwd_jail:bwrap] spawning label={}", jail.label); + // An `allow_net` jail gets a private network when `pasta` works; + // dropping `network` on any error below kills `pasta` again. + let network = match &verified.pasta { + Some(pasta) if jail.allow_net => Some(pasta.start()?), + _ => None, + }; let Prepared { launcher: mut wrapper, args, - } = prepare_command(verified, jail, cmd)?; + } = prepare_command(verified, jail, cmd, network.as_ref())?; if captured { wrapper .stdin(Stdio::piped()) @@ -402,7 +636,14 @@ impl BubblewrapBackend { // can signal the whole sandbox. .process_group(0); } - wrapper.spawn().inspect_err(|_| args.discard()) + let child = wrapper.spawn().inspect_err(|_| args.discard())?; + // `bwrap` now holds the holder's writer; drop ours so the network ends + // when the jail does. + drop(wrapper); + if let Some(network) = network { + network.release(); + } + Ok(child) } } diff --git a/crates/tinybox-jail/src/bwrap/mod_tests.rs b/crates/tinybox-jail/src/bwrap/mod_tests.rs index e1b04c27..2546dedb 100644 --- a/crates/tinybox-jail/src/bwrap/mod_tests.rs +++ b/crates/tinybox-jail/src/bwrap/mod_tests.rs @@ -15,6 +15,9 @@ fn verified(program: &str) -> Verified { Verified { program: program.into(), disable_userns: true, + bind_fds: false, + seccomp: Vec::new(), + pasta: None, } } @@ -81,7 +84,7 @@ fn secrets_stay_out_of_argv_and_the_launcher_environment() -> io::Result<()> { .env("A", "1") .env_remove("B"); let jail = Jail::new(root.path(), "t"); - let prepared = prepare_command(&verified("/usr/bin/bwrap"), &jail, &cmd)?; + let prepared = prepare_command(&verified("/usr/bin/bwrap"), &jail, &cmd, None)?; // The host-side process: empty environment, argv of paths only. assert_eq!(prepared.launcher.get_envs().count(), 0); let argv = argv_of(&prepared.launcher); @@ -192,6 +195,20 @@ fn a_launcher_that_passes_the_probe_is_available_with_process_isolation_minimum( ); } +#[test] +fn a_bwrap_that_cannot_load_seccomp_filters_is_not_used() { + // It passes the isolation probe, but a jail would start unfiltered. + let backend = BubblewrapBackend::with_program(fixture("fake-bwrap-noseccomp.sh")); + assert!(!backend.is_available()); + assert!(!backend.installs_seccomp()); +} + +#[test] +fn a_launcher_that_passes_the_probe_installs_the_seccomp_filters() { + let backend = BubblewrapBackend::with_program(fixture("fake-bwrap-probe.sh")); + assert!(backend.installs_seccomp()); +} + #[test] fn a_bwrap_without_disable_userns_is_available_but_claims_no_isolation() { let backend = BubblewrapBackend::with_program(fixture("fake-bwrap-old.sh")); @@ -212,3 +229,78 @@ fn the_default_constructor_probes_only_trusted_paths() { backend.is_available() ); } + +#[test] +fn a_private_network_jail_enters_the_holder_and_brings_its_own_resolver() -> io::Result<()> { + let root = tempfile::tempdir()?; + let jail = Jail::new(root.path(), "t"); + let mut backend_state = verified("/usr/bin/bwrap"); + backend_state.pasta = pasta::Pasta::find(|_| true); + let (_reader, writer) = std::io::pipe()?; + let network = pasta::Network::fake( + std::fs::File::open("/proc/self/ns/user")?, + std::fs::File::open("/proc/self/ns/net")?, + writer.into(), + ); + let prepared = prepare_command(&backend_state, &jail, &Command::new("true"), Some(&network))?; + assert_eq!(prepared.launcher.get_program(), "/usr/bin/nsenter"); + let argv = argv_of(&prepared.launcher); + assert!(argv.iter().any(|a| a == "--preserve-credentials")); + let content = std::fs::read(prepared.args.path())?; + let fields: Vec<&[u8]> = content.split(|b| *b == 0).collect(); + // The resolver arrives by descriptor and the holder's writer is kept by + // bwrap; the host's resolv.conf is not bound, and the net stays shared + // with the holder (no --unshare-net). + assert!( + fields + .windows(3) + .any(|w| w[0] == b"--ro-bind-data" && w[2] == b"/etc/resolv.conf") + ); + assert!(fields.iter().any(|f| *f == b"--sync-fd")); + assert!( + !fields + .windows(2) + .any(|w| w[0] == b"--ro-bind-try" && w[1] == b"/etc/resolv.conf") + ); + assert!(!fields.iter().any(|f| *f == b"--unshare-net")); + prepared.args.discard(); + network.release(); + Ok(()) +} + +#[test] +fn only_a_working_pasta_makes_an_allow_net_jail_kernel_isolated() { + let root = std::env::temp_dir(); + let open = Jail::new(&root, "t"); + let without = BubblewrapBackend { + verified: Some(verified("/usr/bin/bwrap")), + }; + assert!(!without.networks_privately()); + assert_eq!(without.isolation_for(&open), IsolationLevel::Process); + let mut with_pasta = verified("/usr/bin/bwrap"); + with_pasta.pasta = pasta::Pasta::find(|_| true); + let with = BubblewrapBackend { + verified: Some(with_pasta), + }; + assert!(with.networks_privately()); + assert_eq!(with.isolation_for(&open), IsolationLevel::Kernel); + // The backend-wide minimum is unchanged: it never promises Kernel. + assert_eq!(with.isolation(), IsolationLevel::Process); +} + +#[test] +fn a_pasta_that_fails_to_start_refuses_the_jail_instead_of_sharing_the_host_network() { + let root = std::env::temp_dir(); + let mut state = verified(&fake_bwrap().to_string_lossy()); + state.pasta = Some(pasta::Pasta::at("/nonexistent/pasta", "/usr/bin/nsenter")); + let backend = BubblewrapBackend { + verified: Some(state), + }; + // The claim is made for a backend whose probe found a working pasta, so + // a launch where pasta then fails must not run the jail at all. + let error = backend + .spawn(&Jail::new(&root, "t"), Command::new("true")) + .err() + .map(|e| e.kind()); + assert_eq!(error, Some(io::ErrorKind::NotFound)); +} diff --git a/crates/tinybox-jail/src/bwrap/pasta.rs b/crates/tinybox-jail/src/bwrap/pasta.rs new file mode 100644 index 00000000..57ed6db3 --- /dev/null +++ b/crates/tinybox-jail/src/bwrap/pasta.rs @@ -0,0 +1,295 @@ +//! A private network for an `allow_net` jail, from `pasta` (passt). +//! +//! Without this, a jail that allows network shares the host's network +//! namespace, so host loopback services and abstract unix sockets are +//! reachable from it. Here `pasta` creates a fresh user and network namespace +//! with user-mode networking out to the internet, and the jail is entered into +//! it. +//! +//! # Why a holder, not `pasta -- bwrap` +//! +//! `pasta` closes every inherited descriptor above 2 when it starts, and the +//! `bwrap` launch depends on inherited descriptors (the argument file, the +//! grant descriptors and the seccomp filters). So `pasta` runs only a holder +//! (`sh -c 'read _'`) that owns the namespaces; the launcher enters them with +//! `nsenter --user --net` by namespace descriptor, which keeps every other +//! descriptor, and then execs `bwrap`. Attaching `pasta` to namespaces `bwrap` +//! created is not an option on Ubuntu: its `AppArmor` user-namespace restriction +//! refuses the `setns`. +//! +//! # What the jail can reach +//! +//! `pasta` is only ever started with [`PASTA_ARGS`]: no port forwarding in +//! either direction (`-T none -U none`, so the jail cannot reach a service +//! bound to the host's loopback), no gateway mapping to the host +//! (`--no-map-gw`), and DNS only through [`DNS_ADDR`], which `pasta` forwards +//! to the host's resolver. The jail gets its own `/etc/resolv.conf` naming +//! that address. Abstract unix sockets belong to the network namespace, so the +//! host's are out of reach too. +//! +//! # Lifetime +//! +//! The holder reads a pipe whose only writer, once the jail has started, is +//! `bwrap` itself (`--sync-fd`). When `bwrap` exits, however it exits, the +//! holder sees end-of-file and exits, and `pasta` exits with it. Before the +//! hand-over the writer is held here, and [`Network`]'s `Drop` kills `pasta`. + +use std::fs::File; +use std::io; +use std::os::fd::OwnedFd; +use std::os::unix::fs::MetadataExt as _; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +/// Where a trusted `pasta` may live. +const TRUSTED_PASTA: &[&str] = &["/usr/bin/pasta", "/usr/local/bin/pasta", "/bin/pasta"]; + +/// Where a trusted `nsenter` may live. +const TRUSTED_NSENTER: &[&str] = &["/usr/bin/nsenter", "/bin/nsenter", "/usr/local/bin/nsenter"]; + +/// The only way `pasta` is started: no port forwarding, no gateway mapping to +/// the host, DNS forwarded through [`DNS_ADDR`] only. +pub(super) const PASTA_ARGS: &[&str] = &[ + "--config-net", + "--quiet", + "--no-map-gw", + "-T", + "none", + "-U", + "none", + "--dns-forward", + DNS_ADDR, +]; + +/// The jail's resolver: `pasta` forwards DNS sent here to the host's. +pub(super) const DNS_ADDR: &str = "169.254.1.1"; + +/// How long the jail's network may take to come up. +const READY_TIMEOUT: Duration = Duration::from_secs(5); + +/// A `pasta` and an `nsenter` found at trusted paths. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Pasta { + program: PathBuf, + nsenter: PathBuf, +} + +/// The jail's `/etc/resolv.conf`. +pub(super) fn resolv_conf() -> String { + format!("nameserver {DNS_ADDR}\n") +} + +impl Pasta { + /// `pasta` and `nsenter` at trusted paths, when both are there. + pub(super) fn find(trusted: impl Fn(&Path) -> bool) -> Option { + let first = |paths: &[&str]| paths.iter().map(PathBuf::from).find(|path| trusted(path)); + Some(Self { + program: first(TRUSTED_PASTA)?, + nsenter: first(TRUSTED_NSENTER)?, + }) + } + + /// The `nsenter` the launcher enters the namespaces with. + pub(super) fn nsenter(&self) -> &Path { + &self.nsenter + } + + /// Starts `pasta` with a holder and waits until its network is up. + /// + /// # Errors + /// The spawn error; `InvalidData` when the holder cannot be identified as + /// `pasta`'s child in a network namespace of its own; `TimedOut` when the + /// network has no default route within [`READY_TIMEOUT`]. + pub(super) fn start(&self) -> io::Result { + let (reader, writer) = io::pipe()?; + let child = Command::new(&self.program) + .args(PASTA_ARGS) + .args(["--", "/bin/sh", "-c", "read _"]) + .env_clear() + .stdin(Stdio::from(OwnedFd::from(reader))) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + let mut network = Network { + pasta: Some(child), + user: None, + net: None, + writer: Some(writer.into()), + }; + let pasta = network.pasta_pid()?; + let deadline = Instant::now() + READY_TIMEOUT; + let holder = wait_for(deadline, || holder_of(pasta))?; + let (user, net) = open_namespaces(pasta, holder)?; + network.user = Some(user); + network.net = Some(net); + wait_for(deadline, || has_default_route(holder).then_some(()))?; + Ok(network) + } +} + +/// A running `pasta` and the namespaces of its holder. +#[derive(Debug)] +pub(super) struct Network { + pasta: Option, + user: Option, + net: Option, + /// The holder's only writer until it is handed to `bwrap`. + writer: Option, +} + +impl Network { + fn pasta_pid(&self) -> io::Result { + self.pasta + .as_ref() + .map(Child::id) + .ok_or_else(|| io::Error::other("pasta already released")) + } + + /// The descriptors the launcher needs: the holder's user and network + /// namespaces, then the writer `bwrap` keeps with `--sync-fd`. Each is a + /// duplicate; the originals stay here until [`Network::release`]. + /// + /// # Errors + /// The error from duplicating a descriptor. + pub(super) fn descriptors(&self) -> io::Result<(OwnedFd, OwnedFd, OwnedFd)> { + let missing = || io::Error::other("pasta network not started"); + let user = self.user.as_ref().ok_or_else(missing)?.try_clone()?; + let net = self.net.as_ref().ok_or_else(missing)?.try_clone()?; + let writer = self.writer.as_ref().ok_or_else(missing)?.try_clone()?; + Ok((user.into(), net.into(), writer)) + } + + /// After the jail has started: drop this side's copies, so `bwrap` holds + /// the holder's only writer, and reap `pasta` in the background when it + /// exits. + pub(super) fn release(mut self) { + self.writer = None; + self.user = None; + self.net = None; + if let Some(mut pasta) = self.pasta.take() { + std::thread::spawn(move || { + let _ = pasta.wait(); + }); + } + } +} + +#[cfg(test)] +impl Pasta { + /// A `pasta` and `nsenter` at the given paths, without the trust check. + pub(super) fn at(program: impl Into, nsenter: impl Into) -> Self { + Self { + program: program.into(), + nsenter: nsenter.into(), + } + } +} + +#[cfg(test)] +impl Network { + /// A network over the given descriptors with no `pasta` behind it, for + /// tests of how a launch is wired. + pub(super) fn fake(user: File, net: File, writer: OwnedFd) -> Self { + Self { + pasta: None, + user: Some(user), + net: Some(net), + writer: Some(writer), + } + } +} + +impl Drop for Network { + fn drop(&mut self) { + if let Some(mut pasta) = self.pasta.take() { + let _ = pasta.kill(); + let _ = pasta.wait(); + } + } +} + +/// Polls `check` until it returns a value or `deadline` passes. +fn wait_for(deadline: Instant, mut check: impl FnMut() -> Option) -> io::Result { + loop { + if let Some(value) = check() { + return Ok(value); + } + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "pasta did not bring up the jail's network within {} s", + READY_TIMEOUT.as_secs() + ), + )); + } + std::thread::sleep(Duration::from_millis(10)); + } +} + +/// The first child of `pasta`, read from `/proc`. +fn holder_of(pasta: u32) -> Option { + let children = std::fs::read_to_string(format!("/proc/{pasta}/task/{pasta}/children")).ok()?; + first_pid(&children) +} + +/// The first pid in a `/proc//task//children` listing. +pub(super) fn first_pid(children: &str) -> Option { + children.split_whitespace().next()?.parse().ok() +} + +/// The parent pid from a `/proc//status` file. +pub(super) fn parent_pid(status: &str) -> Option { + status + .lines() + .find_map(|line| line.strip_prefix("PPid:")) + .and_then(|value| value.trim().parse().ok()) +} + +/// Whether a `/proc//net/route` table has a default route. +pub(super) fn default_route(table: &str) -> bool { + table + .lines() + .skip(1) + .any(|line| line.split_whitespace().nth(1) == Some("00000000")) +} + +fn has_default_route(holder: u32) -> bool { + std::fs::read_to_string(format!("/proc/{holder}/net/route")).is_ok_and(|t| default_route(&t)) +} + +fn is_child_of(pasta: u32, holder: u32) -> bool { + std::fs::read_to_string(format!("/proc/{holder}/status")) + .is_ok_and(|status| parent_pid(&status) == Some(pasta)) +} + +/// Opens the holder's user and network namespaces, checking before and after +/// that the pid still belongs to `pasta`'s child, and that the network +/// namespace is not this process's own. +fn open_namespaces(pasta: u32, holder: u32) -> io::Result<(File, File)> { + let wrong = |what: &str| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("pasta holder {holder}: {what}"), + ) + }; + if !is_child_of(pasta, holder) { + return Err(wrong("not pasta's child")); + } + let user = File::open(format!("/proc/{holder}/ns/user"))?; + let net = File::open(format!("/proc/{holder}/ns/net"))?; + if !is_child_of(pasta, holder) { + return Err(wrong("exited while its namespaces were opened")); + } + let own = std::fs::metadata("/proc/self/ns/net")?; + let held = net.metadata()?; + if (held.dev(), held.ino()) == (own.dev(), own.ino()) { + return Err(wrong("shares this process's network namespace")); + } + Ok((user, net)) +} + +#[cfg(test)] +#[path = "pasta_tests.rs"] +mod tests; diff --git a/crates/tinybox-jail/src/bwrap/pasta_tests.rs b/crates/tinybox-jail/src/bwrap/pasta_tests.rs new file mode 100644 index 00000000..2f5f4d2b --- /dev/null +++ b/crates/tinybox-jail/src/bwrap/pasta_tests.rs @@ -0,0 +1,141 @@ +//! Unit tests for the pasta private network: the closed-down flags, the +//! `/proc` parsing, and the failure paths that need no `pasta`. + +use super::*; + +#[test] +fn pasta_is_only_ever_started_with_forwarding_and_the_gateway_closed() { + let args = PASTA_ARGS.join(" "); + for required in [ + "--no-map-gw", + "-T none", + "-U none", + "--dns-forward 169.254.1.1", + ] { + assert!(args.contains(required), "missing {required} in {args}"); + } + // Nothing that would open a path back to the host. + for forbidden in [ + "-t", + "-u", + "--map-host-loopback", + "--map-guest-addr", + "auto", + ] { + assert!( + !PASTA_ARGS.contains(&forbidden), + "{forbidden} must never be passed: {args}" + ); + } +} + +#[test] +fn the_jail_resolver_is_the_forwarded_address_only() { + assert_eq!(resolv_conf(), "nameserver 169.254.1.1\n"); +} + +#[test] +fn pasta_and_nsenter_come_only_from_trusted_paths() { + let found = Pasta::find(|path| { + path == Path::new("/usr/bin/pasta") || path == Path::new("/usr/bin/nsenter") + }); + assert_eq!( + found, + Some(Pasta { + program: "/usr/bin/pasta".into(), + nsenter: "/usr/bin/nsenter".into(), + }) + ); + assert_eq!( + found.as_ref().map(Pasta::nsenter), + Some(Path::new("/usr/bin/nsenter")) + ); + // Either one missing means no private network. + assert!(Pasta::find(|path| path.ends_with("pasta")).is_none()); + assert!(Pasta::find(|path| path.ends_with("nsenter")).is_none()); +} + +#[test] +fn proc_listings_are_parsed() { + assert_eq!(first_pid("4182 4190 \n"), Some(4182)); + assert_eq!(first_pid(""), None); + assert_eq!(first_pid("x"), None); + assert_eq!( + parent_pid("Name:\tsh\nState:\tS\nPPid:\t4181\nTracerPid:\t0\n"), + Some(4181) + ); + assert_eq!(parent_pid("Name:\tsh\n"), None); +} + +#[test] +fn only_a_default_route_counts_as_the_network_being_up() { + let header = "Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\n"; + let link = "eth0\t0000010A\t00000000\t0001\t0\t0\t100\t00F0FFFF\n"; + let default = "eth0\t00000000\t0100010A\t0003\t0\t0\t100\t00000000\n"; + assert!(!default_route(header)); + assert!(!default_route(&format!("{header}{link}"))); + assert!(default_route(&format!("{header}{link}{default}"))); + // The header line never counts, whatever it says. + assert!(!default_route("x\t00000000\n")); +} + +#[test] +fn waiting_reports_a_timeout_with_a_clear_error() { + let error = wait_for(Instant::now(), || None::<()>).err(); + assert_eq!( + error.as_ref().map(io::Error::kind), + Some(io::ErrorKind::TimedOut) + ); + assert!( + error.is_some_and(|e| e + .to_string() + .contains("did not bring up the jail's network")), + "the error names what failed" + ); + assert_eq!(wait_for(Instant::now(), || Some(7)).ok(), Some(7)); +} + +#[test] +fn a_pid_that_is_not_pastas_child_is_refused() -> io::Result<()> { + // This process is not a child of pid 1's first child, nor of pid 1 here. + let me = std::process::id(); + let error = open_namespaces(1, me).err(); + assert_eq!(error.map(|e| e.kind()), Some(io::ErrorKind::InvalidData)); + // A real parent/child pair whose namespace is this process's own is + // refused too: the holder must have a network of its own. + let mut child = Command::new("sleep").arg("5").spawn()?; + let refused = open_namespaces(me, child.id()).err(); + child.kill()?; + child.wait()?; + assert_eq!(refused.map(|e| e.kind()), Some(io::ErrorKind::InvalidData)); + Ok(()) +} + +#[test] +fn a_fake_network_hands_out_duplicates_and_releases_without_a_process() -> io::Result<()> { + let (_reader, writer) = io::pipe()?; + let network = Network::fake( + File::open("/proc/self/ns/user")?, + File::open("/proc/self/ns/net")?, + writer.into(), + ); + assert!(network.pasta_pid().is_err()); + let (user, net, _writer) = network.descriptors()?; + let own = std::fs::metadata("/proc/self/ns/net")?; + let dup = File::from(net).metadata()?; + assert_eq!((dup.dev(), dup.ino()), (own.dev(), own.ino())); + drop(user); + network.release(); + Ok(()) +} + +#[test] +fn an_unstarted_network_has_no_descriptors() { + let network = Network { + pasta: None, + user: None, + net: None, + writer: None, + }; + assert!(network.descriptors().is_err()); +} diff --git a/crates/tinybox-jail/src/bwrap/seccomp.rs b/crates/tinybox-jail/src/bwrap/seccomp.rs new file mode 100644 index 00000000..8a2a55e7 --- /dev/null +++ b/crates/tinybox-jail/src/bwrap/seccomp.rs @@ -0,0 +1,241 @@ +//! The seccomp filters a bubblewrap jail starts under. +//! +//! A deny list, not an allow list: jailed programs are arbitrary (MCP servers on +//! Node, Python or Go, shell commands), so the filter removes kernel attack +//! surface and escape primitives no ordinary tool needs rather than guessing +//! what every runtime calls. A denied call fails with `EPERM`; `clone3` fails +//! with `ENOSYS` instead, because its flags live in memory a filter cannot read +//! and the C library only falls back to `clone`, whose flags it can, on +//! `ENOSYS`. Seccomp takes one action per filter, so that is a second filter. +//! +//! `bwrap --add-seccomp-fd` installs them on the child after its own mount and +//! namespace setup. A program built for another architecture (an i386 binary on +//! `x86_64`) is killed by the architecture check every filter starts with. +//! +//! Left reachable on purpose: `ptrace` on Linux 4.8 and later, where seccomp is +//! re-checked after a tracer changes a call and the jail's pid namespace limits +//! it to the jail's own processes (on older kernels it is denied); `unshare` +//! and `clone` without a `CLONE_NEW*` flag; `personality`; and socket families, +//! which the network namespace already confines. + +use std::collections::BTreeMap; +use std::io; + +use seccompiler::{ + BpfProgram, SeccompAction, SeccompCmpArgLen, SeccompCmpOp, SeccompCondition, SeccompFilter, + SeccompRule, TargetArch, +}; + +/// The namespace-creating `clone`/`unshare` flags. `CLONE_NEWTIME` is +/// `unshare`-only: in `clone` its bit is part of the exit signal. +const CLONE_NEW: [u64; 7] = [ + 0x0002_0000, // CLONE_NEWNS + 0x0200_0000, // CLONE_NEWCGROUP + 0x0400_0000, // CLONE_NEWUTS + 0x0800_0000, // CLONE_NEWIPC + 0x1000_0000, // CLONE_NEWUSER + 0x2000_0000, // CLONE_NEWPID + 0x4000_0000, // CLONE_NEWNET +]; +const CLONE_NEWTIME: u64 = 0x80; + +/// `ioctl` requests that type into the controlling terminal. +const TIOCSTI: u64 = 0x5412; +const TIOCLINUX: u64 = 0x541C; + +/// The x32 ABI's syscall numbers on `x86_64`: same numbers with this bit set, +/// under the same architecture value, so each denial is repeated with it. +#[cfg(target_arch = "x86_64")] +const X32_SYSCALL_BIT: i64 = 0x4000_0000; + +/// Syscalls denied whatever their arguments. +fn always_denied(kernel: (u32, u32)) -> Vec { + let mut denied = vec![ + // The user's kernel keyrings hold credentials. + libc::SYS_keyctl, + libc::SYS_add_key, + libc::SYS_request_key, + // Other processes' memory. + libc::SYS_process_vm_readv, + libc::SYS_process_vm_writev, + // Kernel programs, perf and fault handling: privilege-escalation surface. + libc::SYS_bpf, + libc::SYS_perf_event_open, + libc::SYS_userfaultfd, + libc::SYS_io_uring_setup, + libc::SYS_io_uring_enter, + libc::SYS_io_uring_register, + // The jail's view is fixed by bwrap; nothing inside remounts it. + libc::SYS_mount, + libc::SYS_umount2, + libc::SYS_pivot_root, + libc::SYS_move_mount, + libc::SYS_open_tree, + libc::SYS_fsopen, + libc::SYS_fsconfig, + libc::SYS_fsmount, + libc::SYS_fspick, + libc::SYS_mount_setattr, + libc::SYS_setns, + // File handles open any file on a mount, granted or not. + libc::SYS_open_by_handle_at, + libc::SYS_name_to_handle_at, + // Host-wide or privileged state. + libc::SYS_init_module, + libc::SYS_finit_module, + libc::SYS_delete_module, + libc::SYS_kexec_load, + libc::SYS_kexec_file_load, + libc::SYS_reboot, + libc::SYS_swapon, + libc::SYS_swapoff, + libc::SYS_acct, + libc::SYS_settimeofday, + libc::SYS_clock_settime, + libc::SYS_clock_adjtime, + libc::SYS_adjtimex, + libc::SYS_syslog, + libc::SYS_quotactl, + libc::SYS_vhangup, + ]; + #[cfg(target_arch = "x86_64")] + denied.extend([ + libc::SYS_iopl, + libc::SYS_ioperm, + libc::SYS_modify_ldt, + libc::SYS_uselib, + libc::SYS__sysctl, + ]); + // Before 4.8 a tracer could rewrite a call after seccomp had allowed it. + if kernel < (4, 8) { + denied.push(libc::SYS_ptrace); + } + denied +} + +/// One rule per flag: the call matches when any of them is set in argument 0. +fn any_flag(flags: &[u64]) -> io::Result> { + flags + .iter() + .map(|&flag| { + condition(0, SeccompCmpOp::MaskedEq(flag), flag) + .and_then(|condition| rule(vec![condition])) + }) + .collect() +} + +fn condition(arg: u8, op: SeccompCmpOp, value: u64) -> io::Result { + SeccompCondition::new(arg, SeccompCmpArgLen::Dword, op, value).map_err(|error| invalid(&error)) +} + +fn rule(conditions: Vec) -> io::Result { + SeccompRule::new(conditions).map_err(|error| invalid(&error)) +} + +fn invalid(error: &seccompiler::BackendError) -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, error.to_string()) +} + +/// The seccomp target for an architecture name (`std::env::consts::ARCH`), +/// if a filter is defined for it. +fn target_arch(name: &str) -> io::Result { + match name { + "x86_64" => Ok(TargetArch::x86_64), + "aarch64" => Ok(TargetArch::aarch64), + other => Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("no seccomp filter is defined for {other}; the jail is refused"), + )), + } +} + +/// `major.minor` of a `uname -r` style release (`6.8.0-45-generic`). +pub(super) fn kernel_version(release: &str) -> Option<(u32, u32)> { + let mut parts = release + .split(|c: char| !c.is_ascii_digit()) + .filter(|part| !part.is_empty()); + Some((parts.next()?.parse().ok()?, parts.next()?.parse().ok()?)) +} + +/// Every rule keyed by syscall, with the x32 twin of each on `x86_64`. +fn with_x32(rules: BTreeMap>) -> BTreeMap> { + #[cfg(target_arch = "x86_64")] + { + let mut rules = rules; + let twins: Vec<(i64, Vec)> = rules + .iter() + .filter(|(nr, _)| **nr & X32_SYSCALL_BIT == 0) + .map(|(nr, chain)| (nr | X32_SYSCALL_BIT, chain.clone())) + .collect(); + rules.extend(twins); + rules + } + #[cfg(not(target_arch = "x86_64"))] + rules +} + +/// The two filters, compiled for this process's architecture and serialized +/// as `bwrap` reads them: the `EPERM` deny list, then `clone3` → `ENOSYS`. +/// +/// # Errors +/// `Unsupported` on an architecture without a filter; `InvalidInput` if the +/// filter does not compile. +pub(super) fn filters(kernel: (u32, u32)) -> io::Result>> { + let arch = target_arch(std::env::consts::ARCH)?; + let mut denied: BTreeMap> = always_denied(kernel) + .into_iter() + .map(|nr| (nr, Vec::new())) + .collect(); + denied.insert(libc::SYS_clone, any_flag(&CLONE_NEW)?); + let mut unshare = CLONE_NEW.to_vec(); + unshare.push(CLONE_NEWTIME); + denied.insert(libc::SYS_unshare, any_flag(&unshare)?); + denied.insert( + libc::SYS_ioctl, + [TIOCSTI, TIOCLINUX] + .into_iter() + .map(|request| condition(1, SeccompCmpOp::Eq, request).and_then(|c| rule(vec![c]))) + .collect::>()?, + ); + let eperm = errno(libc::EPERM)?; + let enosys = errno(libc::ENOSYS)?; + [ + (with_x32(denied), eperm), + ( + with_x32(BTreeMap::from([(libc::SYS_clone3, Vec::new())])), + enosys, + ), + ] + .into_iter() + .map(|(rules, action)| { + let filter = SeccompFilter::new(rules, SeccompAction::Allow, action, arch) + .map_err(|error| invalid(&error))?; + BpfProgram::try_from(filter) + .map(|program| serialize(&program)) + .map_err(|error| invalid(&error)) + }) + .collect() +} + +fn errno(value: i32) -> io::Result { + u32::try_from(value) + .map(SeccompAction::Errno) + .map_err(io::Error::other) +} + +/// `struct sock_filter` as the kernel lays it out: `u16 code, u8 jt, u8 jf, +/// u32 k`, native endian. +fn serialize(program: &BpfProgram) -> Vec { + let mut bytes = Vec::with_capacity(program.len() * 8); + for instruction in program { + bytes.extend(instruction.code.to_ne_bytes()); + bytes.push(instruction.jt); + bytes.push(instruction.jf); + bytes.extend(instruction.k.to_ne_bytes()); + } + bytes +} + +#[cfg(test)] +#[path = "seccomp_tests.rs"] +mod tests; diff --git a/crates/tinybox-jail/src/bwrap/seccomp_tests.rs b/crates/tinybox-jail/src/bwrap/seccomp_tests.rs new file mode 100644 index 00000000..03e8fb98 --- /dev/null +++ b/crates/tinybox-jail/src/bwrap/seccomp_tests.rs @@ -0,0 +1,70 @@ +//! The seccomp deny list: what it holds and that it compiles per architecture. + +use super::*; + +#[test] +fn a_kernel_release_reads_as_major_and_minor() { + assert_eq!(kernel_version("6.8.0-45-generic\n"), Some((6, 8))); + assert_eq!(kernel_version("4.4.0"), Some((4, 4))); + assert_eq!( + kernel_version("5.15.167.4-microsoft-standard-WSL2"), + Some((5, 15)) + ); + assert_eq!(kernel_version("garbage"), None); +} + +#[test] +fn ptrace_is_denied_only_before_linux_4_8() { + assert!(always_denied((4, 4)).contains(&libc::SYS_ptrace)); + assert!(!always_denied((4, 8)).contains(&libc::SYS_ptrace)); + assert!(!always_denied((6, 8)).contains(&libc::SYS_ptrace)); +} + +#[test] +fn the_escape_and_attack_surface_calls_are_denied() { + let denied = always_denied((6, 8)); + for nr in [ + libc::SYS_keyctl, + libc::SYS_bpf, + libc::SYS_io_uring_setup, + libc::SYS_mount, + libc::SYS_setns, + libc::SYS_open_by_handle_at, + libc::SYS_process_vm_readv, + ] { + assert!(denied.contains(&nr), "syscall {nr} is not denied"); + } +} + +#[test] +fn an_architecture_without_a_filter_is_refused() { + let kind = target_arch("riscv64").err().map(|error| error.kind()); + assert_eq!(kind, Some(io::ErrorKind::Unsupported)); + assert!(target_arch("x86_64").is_ok() && target_arch("aarch64").is_ok()); +} + +#[test] +fn both_filters_compile_to_whole_bpf_instructions() -> io::Result<()> { + let filters = filters((6, 8))?; + + assert_eq!( + filters.len(), + 2, + "the EPERM list and the clone3 ENOSYS filter" + ); + for filter in &filters { + assert!( + !filter.is_empty() && filter.len() % 8 == 0, + "{}", + filter.len() + ); + } + Ok(()) +} + +#[test] +fn namespace_flags_are_matched_one_rule_each() -> io::Result<()> { + // Any one flag must match on its own: rules in a chain are alternatives. + assert_eq!(any_flag(&CLONE_NEW)?.len(), CLONE_NEW.len()); + Ok(()) +} diff --git a/crates/tinybox-jail/tests/bwrap_isolation.rs b/crates/tinybox-jail/tests/bwrap_isolation.rs index 80c75a73..d557bc58 100644 --- a/crates/tinybox-jail/tests/bwrap_isolation.rs +++ b/crates/tinybox-jail/tests/bwrap_isolation.rs @@ -27,8 +27,13 @@ fn only_a_jail_with_a_private_network_is_suitable_for_untrusted_code() -> io::Re }; let start = Instant::now(); let root = tempfile::tempdir()?; - // tinymcp's default configuration shares the host network: not suitable. - assert!(!backend.is_suitable_for_untrusted_jail(&jail(root.path()))); + // tinymcp's default configuration allows network. It is suitable only + // when that network is private (pasta); a shared host network is not. + assert_eq!( + backend.is_suitable_for_untrusted_jail(&jail(root.path())), + backend.networks_privately() + && backend.isolation() >= tinybox_core::IsolationLevel::Process + ); // The per-backend answer is the worst case, so it is never true. assert!(!backend.is_suitable_for_untrusted_code()); // With a private network it is suitable, given `--disable-userns` support. @@ -109,6 +114,27 @@ fn a_read_only_grant_nested_in_the_root_stays_read_only() -> io::Result<()> { Ok(()) } +#[test] +fn grants_are_bound_from_descriptors_whenever_bwrap_supports_it() -> io::Result<()> { + let Some(backend) = backend()? else { + return Ok(()); + }; + let start = Instant::now(); + let program = backend + .program() + .ok_or_else(|| io::Error::other("no program"))?; + let help = Command::new(program).arg("--help").output()?; + let listed = String::from_utf8_lossy(&help.stdout) + .split_whitespace() + .any(|word| word == "--bind-fd"); + // openat2 is present on every kernel CI runs (5.6+), so the listing alone + // decides; the nested-grant tests above then ran through descriptors. + assert_eq!(backend.binds_by_descriptor(), listed); + eprintln!("[bwrap-real] binds_by_descriptor={listed}"); + done("bind-fd", start); + Ok(()) +} + #[test] fn a_read_only_grant_nested_in_a_read_write_grant_stays_read_only() -> io::Result<()> { let Some(backend) = backend()? else { @@ -344,12 +370,17 @@ fn network_is_unreachable_when_denied_and_reachable_when_shared() -> io::Result< let (connected, _) = started(&run(&backend, &jail(root.path()).deny_net(), cmd)?)?; assert!(!connected, "connect must fail under deny_net"); - // Control: with the host network shared the same connect succeeds, which - // is exactly the exposure `allow_net` carries. + // With network allowed: under a private (pasta) network the host's + // loopback listener stays unreachable; with the host network shared the + // same connect succeeds, which is exactly the exposure that carries. let mut cmd = Command::new("/usr/bin/env"); cmd.args(["bash", "-c", &connect]); let (connected, _) = started(&run(&backend, &jail(root.path()), cmd)?)?; - assert!(connected, "control connect should reach the host listener"); + assert_eq!( + connected, + !backend.networks_privately(), + "allow_net reaches host loopback only when the host network is shared" + ); done("network", start); Ok(()) } @@ -507,3 +538,79 @@ fn nested_user_namespaces_are_blocked_on_a_hardened_backend() -> io::Result<()> done("disable-userns", start); Ok(()) } + +/// Runs `script` with the jail's `python3`, or `None` when the host has none. +fn python( + backend: &tinybox_jail::BubblewrapBackend, + jail: &tinybox_jail::Jail, + script: &str, +) -> io::Result> { + if !Path::new("/usr/bin/python3").exists() { + eprintln!("skipped: no /usr/bin/python3 to issue raw syscalls"); + return Ok(None); + } + let (_, out) = sh(backend, jail, &format!("/usr/bin/python3 -c '{script}'"))?; + Ok(Some(out)) +} + +/// Raw syscalls from inside the jail: keyctl (denied), getpid (allowed) and a +/// ptrace attach to the jail's own child (allowed on Linux 4.8+). +const SYSCALL_PROBE: &str = r#" +import ctypes, errno, os, platform, time +libc = ctypes.CDLL(None, use_errno=True) +keyctl, getpid = {"x86_64": (250, 39), "aarch64": (219, 172)}[platform.machine()] +r = libc.syscall(keyctl, 0, -3, 0) +print("keyctl", r, errno.errorcode.get(ctypes.get_errno(), "none")) +print("getpid", libc.syscall(getpid) == os.getpid()) +pid = os.fork() +if pid == 0: + time.sleep(5) + os._exit(0) +r = libc.ptrace(16, pid, 0, 0) +print("ptrace", r, errno.errorcode.get(ctypes.get_errno(), "none") if r else "ok") +os.kill(pid, 9) +os.waitpid(pid, 0) +"#; + +#[test] +fn a_denied_syscall_fails_with_eperm_and_allowed_ones_still_work() -> io::Result<()> { + let Some(backend) = backend()? else { + return Ok(()); + }; + let start = Instant::now(); + assert!(backend.installs_seccomp()); + let root = tempfile::tempdir()?; + let Some(out) = python(&backend, &jail(root.path()), SYSCALL_PROBE)? else { + return Ok(()); + }; + + assert!( + out.contains("keyctl -1 EPERM"), + "keyctl was not denied: {out}" + ); + assert!( + out.contains("getpid True"), + "an allowed syscall failed: {out}" + ); + done("seccomp", start); + Ok(()) +} + +#[test] +fn a_jailed_process_can_trace_its_own_child() -> io::Result<()> { + let Some(backend) = backend()? else { + return Ok(()); + }; + let start = Instant::now(); + let root = tempfile::tempdir()?; + let Some(out) = python(&backend, &jail(root.path()), SYSCALL_PROBE)? else { + return Ok(()); + }; + + assert!( + out.contains("ptrace 0 ok"), + "ptrace of a jail child failed: {out}" + ); + done("seccomp-ptrace", start); + Ok(()) +} diff --git a/crates/tinybox-jail/tests/bwrap_pasta_live.rs b/crates/tinybox-jail/tests/bwrap_pasta_live.rs new file mode 100644 index 00000000..ad217095 --- /dev/null +++ b/crates/tinybox-jail/tests/bwrap_pasta_live.rs @@ -0,0 +1,187 @@ +//! Linux-only, live: an `allow_net` jail on a private `pasta` network reaches +//! the internet but not the host, and leaves no `pasta` behind. +//! +//! These need `bwrap`, `pasta` and `nsenter` and outbound network access, so +//! each skips (with a note) unless the backend reports +//! [`tinybox_jail::BubblewrapBackend::networks_privately`]. With +//! `TINYBOX_REQUIRE_PASTA=1` a missing private network is a failure instead; +//! CI sets it on the job that installs `passt`. +#![cfg(target_os = "linux")] + +use std::io; +use std::net::TcpListener; +use std::os::linux::net::SocketAddrExt as _; +use std::os::unix::fs::MetadataExt as _; +use std::os::unix::net::{SocketAddr, UnixListener}; +use std::path::Path; +use std::process::Command; +use std::time::{Duration, Instant}; + +use tinybox_jail::{BubblewrapBackend, JailBackend, spawn_captured_with}; + +#[allow(dead_code)] +mod support; + +use support::*; + +/// The backend when it networks privately, `None` to skip, or an error when +/// `TINYBOX_REQUIRE_PASTA=1` and it does not. +fn private() -> io::Result> { + let Some(backend) = backend()? else { + return Ok(None); + }; + if backend.networks_privately() { + return Ok(Some(backend)); + } + if std::env::var_os("TINYBOX_REQUIRE_PASTA").is_some_and(|v| v == "1") { + return Err(io::Error::other( + "TINYBOX_REQUIRE_PASTA=1 but bwrap has no private network (install passt; \ + pasta and nsenter must be root-owned at /usr/bin)", + )); + } + eprintln!("skipped: no working pasta private network"); + Ok(None) +} + +#[test] +fn live_a_private_network_reaches_out_but_not_the_host() -> io::Result<()> { + let Some(backend) = private()? else { + return Ok(()); + }; + let start = Instant::now(); + let root = tempfile::tempdir()?; + let loopback = TcpListener::bind("127.0.0.1:0")?; + let port = loopback.local_addr()?.port(); + let name = format!("tinybox-pasta-live-{}", std::process::id()); + let _abstract = UnixListener::bind_addr(&SocketAddr::from_abstract_name(name.as_bytes())?)?; + let script = format!( + r#"echo started +reach() {{ timeout 5 bash -c 'exec 3<>"/dev/tcp/$0/$1"' "$1" "$2" 2>/dev/null && echo "$3=reached" || echo "$3=refused"; }} +reach example.com 443 outbound +reach 127.0.0.1 {port} loopback +gw=$(ip route 2>/dev/null | awk '/default/{{print $3; exit}}') +reach "${{gw:-0.0.0.0}}" {port} gateway +python3 -c 'import socket,sys; socket.socket(socket.AF_UNIX).connect("\0"+sys.argv[1])' {name} 2>/dev/null \ + && echo abstract=reached || echo abstract=refused +# Only stdio may be open here: the holder's namespace and writer descriptors +# were closed before bwrap ran (3 is this listing's own directory handle). +echo "fds=$(cd /proc/self/fd && ls | sort -n | tr '\n' ' ')""# + ); + let mut cmd = Command::new("/usr/bin/env"); + cmd.args(["bash", "-c", &script]); + let (_, out) = started(&run(&backend, &jail(root.path()), cmd)?)?; + for expected in [ + "outbound=reached", + "loopback=refused", + "gateway=refused", + "abstract=refused", + ] { + assert!( + out.lines().any(|line| line == expected), + "expected {expected} in {out:?}" + ); + } + let fds = out + .lines() + .find_map(|line| line.strip_prefix("fds=")) + .ok_or_else(|| io::Error::other(format!("no fd listing in {out:?}")))?; + assert!( + fds.split_whitespace() + .all(|fd| matches!(fd, "0" | "1" | "2" | "3")), + "only stdio may be open in the jail, got {fds}" + ); + assert!(backend.is_suitable_for_untrusted_jail(&jail(root.path()))); + done("pasta reachability", start); + Ok(()) +} + +#[test] +fn live_killing_a_jail_leaves_no_pasta_behind() -> io::Result<()> { + let Some(backend) = private()? else { + return Ok(()); + }; + let start = Instant::now(); + let root = tempfile::tempdir()?; + let mut cmd = Command::new("/bin/sh"); + cmd.args(["-c", "sleep 30"]); + let jail = jail(root.path()); + backend + .require(&jail) + .map_err(|error| io::Error::new(io::ErrorKind::Unsupported, error))?; + let mut child = spawn_captured_with(&backend, &jail, cmd)?; + let pasta = eventually(|| pasta_of(child.id())) + .ok_or_else(|| io::Error::other("no pasta serves the jail's network"))?; + child.kill()?; + child.wait()?; + assert!( + eventually(|| (!Path::new(&format!("/proc/{pasta}")).exists()).then_some(())).is_some(), + "pasta {pasta} outlived the killed jail" + ); + done("pasta after kill", start); + Ok(()) +} + +#[test] +fn live_a_jail_that_exits_leaves_no_pasta_behind() -> io::Result<()> { + let Some(backend) = private()? else { + return Ok(()); + }; + let start = Instant::now(); + let root = tempfile::tempdir()?; + // Long enough to find its pasta, then a normal exit. + let mut cmd = Command::new("/bin/sh"); + cmd.args(["-c", "sleep 1"]); + let jail = jail(root.path()); + backend + .require(&jail) + .map_err(|error| io::Error::new(io::ErrorKind::Unsupported, error))?; + let child = spawn_captured_with(&backend, &jail, cmd)?; + let pasta = eventually(|| pasta_of(child.id())) + .ok_or_else(|| io::Error::other("no pasta serves the jail's network"))?; + assert!(child.wait_with_output()?.status.success()); + assert!( + eventually(|| (!Path::new(&format!("/proc/{pasta}")).exists()).then_some(())).is_some(), + "pasta {pasta} outlived the jail" + ); + done("pasta after exit", start); + Ok(()) +} + +/// Retries `check` for up to five seconds. +fn eventually(mut check: impl FnMut() -> Option) -> Option { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if let Some(value) = check() { + return Some(value); + } + if Instant::now() >= deadline { + return None; + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +/// The `pasta` whose holder shares the network namespace of process `jail`. +/// Tests run in parallel, so matching by namespace keeps each to its own. +fn pasta_of(jail: u32) -> Option { + let net = std::fs::metadata(format!("/proc/{jail}/ns/net")).ok()?; + let own = std::fs::metadata("/proc/self/ns/net").ok()?; + if net.ino() == own.ino() { + return None; + } + std::fs::read_dir("/proc") + .ok()? + .filter_map(|entry| entry.ok()?.file_name().to_str()?.parse::().ok()) + .filter(|pid| { + // On x86 with AVX2, pasta re-executes itself as `pasta.avx2`. + std::fs::read_to_string(format!("/proc/{pid}/comm")) + .is_ok_and(|c| c.trim().starts_with("pasta")) + }) + .find(|pid| { + std::fs::read_to_string(format!("/proc/{pid}/task/{pid}/children")) + .ok() + .and_then(|c| c.split_whitespace().next()?.parse::().ok()) + .and_then(|holder| std::fs::metadata(format!("/proc/{holder}/ns/net")).ok()) + .is_some_and(|holder| holder.ino() == net.ino() && holder.dev() == net.dev()) + }) +} diff --git a/crates/tinybox-jail/tests/fixtures/fake-bwrap-noseccomp.sh b/crates/tinybox-jail/tests/fixtures/fake-bwrap-noseccomp.sh new file mode 100755 index 00000000..5b123bf1 --- /dev/null +++ b/crates/tinybox-jail/tests/fixtures/fake-bwrap-noseccomp.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Stand-in for bwrap in unit tests: reads the NUL-separated argument list from +# `--args 9` like the real thing, applies --setenv, and runs the command +# unconfined. Behaviour depends on the file name: +# fake-bwrap.sh passes nothing (the isolation probe must reject it) +# fake-bwrap-probe.sh also answers the isolation probe +# fake-bwrap-old.sh like -probe, but rejects --disable-userns +# fake-bwrap-noseccomp.sh like -probe, but its --help lacks --add-seccomp-fd +name=$(basename "$0") +if [ "$1" = "--help" ]; then + [ "$name" = "fake-bwrap-noseccomp.sh" ] || echo " --add-seccomp-fd FD Load and use seccomp rules from FD (repeatable)" + exit 0 +fi +[ "$1" = "--args" ] && [ "$3" = "--" ] || { echo "fake bwrap: expected --args N --" >&2; exit 2; } +shift 3 +mode=opts +key= +while IFS= read -r -d '' line <&9; do + case $mode in + opts) + case $line in + --setenv) mode=setenv_k ;; + --disable-userns) + if [ "$name" = "fake-bwrap-old.sh" ]; then + echo "bwrap: Unknown option --disable-userns" >&2 + exit 1 + fi ;; + esac ;; + setenv_k) key=$line; mode=setenv_v ;; + setenv_v) export "$key=$line"; mode=opts ;; + esac +done +if [ "$name" != "fake-bwrap.sh" ] && [ "$1" = "/bin/sh" ]; then + case $3 in *"echo isolated"*) echo isolated; exit 0 ;; esac +fi +exec "$@" diff --git a/crates/tinybox-jail/tests/fixtures/fake-bwrap-old.sh b/crates/tinybox-jail/tests/fixtures/fake-bwrap-old.sh index b64f5b00..5b123bf1 100755 --- a/crates/tinybox-jail/tests/fixtures/fake-bwrap-old.sh +++ b/crates/tinybox-jail/tests/fixtures/fake-bwrap-old.sh @@ -5,7 +5,12 @@ # fake-bwrap.sh passes nothing (the isolation probe must reject it) # fake-bwrap-probe.sh also answers the isolation probe # fake-bwrap-old.sh like -probe, but rejects --disable-userns +# fake-bwrap-noseccomp.sh like -probe, but its --help lacks --add-seccomp-fd name=$(basename "$0") +if [ "$1" = "--help" ]; then + [ "$name" = "fake-bwrap-noseccomp.sh" ] || echo " --add-seccomp-fd FD Load and use seccomp rules from FD (repeatable)" + exit 0 +fi [ "$1" = "--args" ] && [ "$3" = "--" ] || { echo "fake bwrap: expected --args N --" >&2; exit 2; } shift 3 mode=opts diff --git a/crates/tinybox-jail/tests/fixtures/fake-bwrap-probe.sh b/crates/tinybox-jail/tests/fixtures/fake-bwrap-probe.sh index b64f5b00..5b123bf1 100755 --- a/crates/tinybox-jail/tests/fixtures/fake-bwrap-probe.sh +++ b/crates/tinybox-jail/tests/fixtures/fake-bwrap-probe.sh @@ -5,7 +5,12 @@ # fake-bwrap.sh passes nothing (the isolation probe must reject it) # fake-bwrap-probe.sh also answers the isolation probe # fake-bwrap-old.sh like -probe, but rejects --disable-userns +# fake-bwrap-noseccomp.sh like -probe, but its --help lacks --add-seccomp-fd name=$(basename "$0") +if [ "$1" = "--help" ]; then + [ "$name" = "fake-bwrap-noseccomp.sh" ] || echo " --add-seccomp-fd FD Load and use seccomp rules from FD (repeatable)" + exit 0 +fi [ "$1" = "--args" ] && [ "$3" = "--" ] || { echo "fake bwrap: expected --args N --" >&2; exit 2; } shift 3 mode=opts diff --git a/crates/tinybox-jail/tests/fixtures/fake-bwrap.sh b/crates/tinybox-jail/tests/fixtures/fake-bwrap.sh index b64f5b00..5b123bf1 100755 --- a/crates/tinybox-jail/tests/fixtures/fake-bwrap.sh +++ b/crates/tinybox-jail/tests/fixtures/fake-bwrap.sh @@ -5,7 +5,12 @@ # fake-bwrap.sh passes nothing (the isolation probe must reject it) # fake-bwrap-probe.sh also answers the isolation probe # fake-bwrap-old.sh like -probe, but rejects --disable-userns +# fake-bwrap-noseccomp.sh like -probe, but its --help lacks --add-seccomp-fd name=$(basename "$0") +if [ "$1" = "--help" ]; then + [ "$name" = "fake-bwrap-noseccomp.sh" ] || echo " --add-seccomp-fd FD Load and use seccomp rules from FD (repeatable)" + exit 0 +fi [ "$1" = "--args" ] && [ "$3" = "--" ] || { echo "fake bwrap: expected --args N --" >&2; exit 2; } shift 3 mode=opts