diff --git a/crates/buzz-cli/src/commands/sandbox.rs b/crates/buzz-cli/src/commands/sandbox.rs index 5dbc233c503..f8314586833 100644 --- a/crates/buzz-cli/src/commands/sandbox.rs +++ b/crates/buzz-cli/src/commands/sandbox.rs @@ -21,6 +21,12 @@ mod tests; /// Owner approvals and interactive sandbox terminals. #[derive(Subcommand)] pub enum SandboxCmd { + /// Validate a local captured bundle before any Git import (internal worker seam) + #[command(hide = true)] + InspectBundle { + /// Captured bundle file + file: std::path::PathBuf, + }, /// Attach a local TTY; omitted targets are selected from BUZZ_PASEO_AGENTS Attach { /// Channel UUID containing the sandbox thread @@ -56,6 +62,17 @@ pub enum SandboxCmd { }, } +/// Print bounded local preflight metadata, without reading credentials or connecting. +pub(crate) fn inspect_bundle_cmd(file: &std::path::Path) -> Result<(), CliError> { + let summary = crate::sandbox_archive::inspect_bundle_file(file).map_err(failure)?; + let encoded = serde_json::to_string(&summary).map_err(failure)?; + if encoded.len() > 16 * 1024 * 1024 { + return Err(failure("bundle metadata exceeded budget")); + } + println!("{encoded}"); + Ok(()) +} + fn failure(message: impl ToString) -> CliError { CliError::Other(message.to_string()) } @@ -168,6 +185,7 @@ pub async fn dispatch(command: SandboxCmd, client: &BuzzClient) -> Result<(), Cl )); } match command { + SandboxCmd::InspectBundle { file } => inspect_bundle_cmd(&file), SandboxCmd::Authorize { request, minutes } => { let event = one(client, json!({"ids":[request],"kinds":[30624],"limit":1})).await?; if event.id != request { diff --git a/crates/buzz-cli/src/lib.rs b/crates/buzz-cli/src/lib.rs index 0aad53dccd8..9c92f57c8e9 100644 --- a/crates/buzz-cli/src/lib.rs +++ b/crates/buzz-cli/src/lib.rs @@ -4,6 +4,8 @@ mod commands; mod error; mod help_tree; mod links; +/// Safe local snapshot archive inspection and restoration. +pub mod sandbox_archive; mod validate; use clap::{CommandFactory, FromArgMatches, Parser, Subcommand, ValueEnum}; @@ -2303,6 +2305,10 @@ async fn run(cli: Cli) -> Result<(), CliError> { }; } + if let Cmd::Sandbox(commands::sandbox::SandboxCmd::InspectBundle { ref file }) = cli.command { + return commands::sandbox::inspect_bundle_cmd(file); + } + // Auth: private key is required for all relay operations. // The keypair IS the identity — no tokens, no other auth. let private_key_str = cli.private_key.ok_or_else(|| { diff --git a/crates/buzz-cli/src/sandbox_archive.rs b/crates/buzz-cli/src/sandbox_archive.rs new file mode 100644 index 00000000000..d21e45b77eb --- /dev/null +++ b/crates/buzz-cli/src/sandbox_archive.rs @@ -0,0 +1,99 @@ +//! Local-only bounded archive preflight. No identity, relay or Git process is used. +use std::fs::OpenOptions; +use std::io::Read; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; + +use buzz_sdk::sandbox_attach::snapshot::{ + archive::inspect_bundle, MAX_BLOB_BYTES, MAX_SOURCE_BYTES, +}; +use serde::{Deserialize, Serialize}; + +/// Local archive refusal; file contents and scanner diagnostics are never included. +#[derive(Debug, thiserror::Error)] +pub enum ArchiveError { + /// Filesystem access failed. + #[error("archive file access failed: {0}")] + Io(#[from] std::io::Error), + /// A hard file/type or metadata bound was exceeded. + #[error("{0}")] + Invalid(&'static str), + /// The SDK rejected malformed or oversized bundle contents. + #[error("{0}")] + Bundle(#[from] buzz_sdk::SdkError), +} + +/// Bounded admission metadata for the exact captured bundle. +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct BundleSummary { + /// Local helper format version. + pub version: u8, + /// Number of packed objects, checked before import. + pub objects: u32, + /// Sum of reconstructed object bytes, including delta targets. + pub expanded_bytes: u64, + /// Object-ID length in bytes (20 or 32). + pub hash_bytes: usize, + /// Object ID/ref name pairs; callers account for ref/index/block costs. + pub refs: Vec<(String, String)>, +} + +/// Inspect a regular file without allocating from unchecked file metadata. +/// The bounded descriptor read also catches growth after the initial metadata check. +pub fn inspect_bundle_file(path: &Path) -> Result { + if !std::fs::symlink_metadata(path)?.is_file() { + return Err(ArchiveError::Invalid("bundle must be a regular file")); + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + options.custom_flags(nix::libc::O_NONBLOCK | nix::libc::O_NOFOLLOW); + let file = options.open(path)?; + let metadata = file.metadata()?; + if !metadata.is_file() || metadata.len() > MAX_BLOB_BYTES as u64 { + return Err(ArchiveError::Invalid( + "bundle exceeds regular-file byte budget", + )); + } + let mut bytes = Vec::new(); + file.take(MAX_BLOB_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > MAX_BLOB_BYTES { + return Err(ArchiveError::Invalid("bundle grew beyond byte budget")); + } + let inventory = inspect_bundle(&bytes, MAX_SOURCE_BYTES)?; + Ok(BundleSummary { + version: 1, + objects: inventory.objects, + expanded_bytes: inventory.expanded_bytes, + hash_bytes: inventory.hash_bytes, + refs: inventory.refs, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn inspector_refuses_oversized_files_before_reading_and_nonregular_inputs() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("sparse.bundle"); + std::fs::File::create(&path) + .unwrap() + .set_len(MAX_BLOB_BYTES as u64 + 1) + .unwrap(); + assert!(matches!( + inspect_bundle_file(&path), + Err(ArchiveError::Invalid(_)) + )); + assert!(inspect_bundle_file(temp.path()).is_err()); + #[cfg(unix)] + { + let link = temp.path().join("link"); + std::os::unix::fs::symlink(&path, &link).unwrap(); + assert!(inspect_bundle_file(&link).is_err()); + } + } +} diff --git a/docs/sandbox-snapshot-producer.md b/docs/sandbox-snapshot-producer.md new file mode 100644 index 00000000000..0581ba948a8 --- /dev/null +++ b/docs/sandbox-snapshot-producer.md @@ -0,0 +1,61 @@ +# Snapshot archive producer + +`buzz-agent-snapshot` builds a fully scanned archive inside an agent sandbox. +This layer packages a local artifact only. The ACP request handler, encrypted Blossom transfer, Rust extraction, and user-facing CLI follow separately. +The caller must hold a live owner-approved snapshot grant and run the helper under the isolated worker's containment and cancellation rules. +No terminal approval or global gateway delegation alone permits export. + +```sh +buzz-agent-snapshot --workspace /workspace --output /tmp/private-job/archive.tar.gz --work-bytes 536870912 +``` + +The output must be new. It is mode 0600 and is removed if creation fails. +Stdout contains only version, scan status, repository count, and byte count. +Failures report only `scan`, `budget`, or `archive`; scanner output and file names are suppressed. +The helper never uploads, fetches from a forge, signs relay events, or falls back to quarantine. + +Every repository gets a self-contained `git bundle --all HEAD` plus its tracked and non-ignored untracked working files. +Nested repositories have separate records and bundles; Git's directory markers are not copied as files. +The manifest records `version: 1`, `snapshot: true`, `scanned: true`, and repository path, head, branch, explicit shallow boundaries, file list, exclusions, and deletions. +Every record uses `base: bundle`; there is no origin URL or clone prerequisite. +A workspace that is itself a repository uses the output path `workspace`. + +The tar layout reuses the checkpoint structure: + +```text +snapshot/manifest.json +snapshot/repositories/0/history.bundle +snapshot/repositories/0/files/ +``` + +The user chose full locally available scanned history. +Historical versioned content, including a historical `.env.example`, remains in the bundle. +The working copy excludes checkpoint private/runtime paths and `.env*` files. +The next extractor must mark intentionally omitted tracked paths skip-worktree so they do not appear as accidental deletions; genuine source deletions remain absent. +Shallow clones retain their explicit local boundaries, without unshallow or network fetch. +Unborn HEADs are refused. + +Before packing, the helper inventories source objects and byte/count budgets. +After capture, the required local `buzz sandbox inspect-bundle` helper validates the actual bundle with the SDK before any Git import, independently of credentials or relay configuration. +The producer reserves the captured expansion, object/ref/index/block costs and aggregate counts, then verifies the neutral inventory matches before writing raw review bodies. +A source commit between preflight and capture therefore cannot inherit stale scratch admission; missing or malformed inspector output fails closed. +It imports the bundle into a neutral Git repository, verifies its object graph, and scans that exact history with immutable gitleaks rules. +It also scans the copied working files, manifest, ref names, and every archived raw object, including merge-only blobs and annotated-tag messages. +Global/system configuration, replacement refs, grafts, lazy fetch, hooks, fsmonitor, repository ignore files, and `GITLEAKS_*` overrides cannot weaken the scan. +Repository and file reads use open directory descriptors and refuse source path redirection through symlinks. +Internal relative symlinks are preserved; external links and special files are refused. + +Limits are 256 MiB source/expanded object bytes, 200,000 objects, 50,000 files/references, 64 repositories, and 180 seconds total creation time. +Subprocess output and time are bounded, and Git object bodies use one streamed batch. +Bundle and gzip output are admitted before writes; the final archive leaves space for the encrypted blob prefix/tag within 100 MiB. +The local work budget defaults to 512 MiB and may be set from 1 MiB through 2 GiB; filesystem free space can lower it. +The budget counts object review files, bundles/imports, copied files, block/index overhead, compressed output, and an operational reserve. +`statvfs` cannot reveal Kubernetes emptyDir sizeLimit: configure the work budget below the actual scratch quota after accounting for other agent data. +The separate attaching-machine extractor also needs its own store/work budget and capacity checks. + +Tar ownership and timestamps are normalized. UTF-8 path/link metadata is capped at 16 KiB per header and 1 MiB total; the next extractor must apply bounds before extension allocation. +The archive is validated and scanned after copying; source work can continue during capture, so it is a bounded working view rather than a filesystem transaction. + +Run `python3 voltage/images/buzz-agent/test-snapshot.py ` to exercise packaged code and the actual scanner with no network, a read-only root, UID 10001, dropped capabilities, and a 1 GiB memory cap. +The tests cover SHA-1/SHA-256, local branches, nested repositories, dirty/deleted files, executable modes, relative links, long PAX names, shallow boundaries, source/capacity refusal, and current/deleted/merge/tag secret refusal with ignore bypass attempts. +The shared policy fixture is checked against checkpoint constants and will also bind the Rust extractor. diff --git a/voltage/images/buzz-agent/Dockerfile b/voltage/images/buzz-agent/Dockerfile index 86a91eb7be0..05c0d0653a1 100644 --- a/voltage/images/buzz-agent/Dockerfile +++ b/voltage/images/buzz-agent/Dockerfile @@ -83,6 +83,7 @@ COPY --chmod=0755 voltage/images/buzz-agent/account.py /usr/local/bin/buzz-agent COPY --chmod=0755 voltage/images/buzz-agent/profile.py /usr/local/bin/buzz-agent-profile COPY --chmod=0755 voltage/images/buzz-agent/cache.py /usr/local/bin/buzz-agent-cache COPY --chmod=0755 voltage/images/buzz-agent/checkpoint.py /usr/local/bin/buzz-agent-checkpoint +COPY --chmod=0755 voltage/images/buzz-agent/snapshot.py /usr/local/bin/buzz-agent-snapshot COPY --chmod=0755 voltage/images/buzz-agent/review.py /usr/local/bin/buzz-agent-review COPY --chmod=0755 voltage/images/buzz-agent/heavy.py /usr/local/bin/buzz-heavy COPY --chmod=0755 voltage/images/buzz-agent/worker.py /usr/local/bin/buzz-agent-worker diff --git a/voltage/images/buzz-agent/Dockerfile.dockerignore b/voltage/images/buzz-agent/Dockerfile.dockerignore index a021be5b0b5..210d6c43de9 100644 --- a/voltage/images/buzz-agent/Dockerfile.dockerignore +++ b/voltage/images/buzz-agent/Dockerfile.dockerignore @@ -89,6 +89,7 @@ voltage/ !voltage/images/buzz-agent/profile.py !voltage/images/buzz-agent/cache.py !voltage/images/buzz-agent/checkpoint.py +!voltage/images/buzz-agent/snapshot.py !voltage/images/buzz-agent/review.py !voltage/images/buzz-agent/checkpoint-gitleaks.toml !voltage/images/buzz-agent/gh.sh diff --git a/voltage/images/buzz-agent/snapshot-policy.json b/voltage/images/buzz-agent/snapshot-policy.json new file mode 100644 index 00000000000..09826a5c3a6 --- /dev/null +++ b/voltage/images/buzz-agent/snapshot-policy.json @@ -0,0 +1,170 @@ +[ + [ + ".aws/nested", + true + ], + [ + ".cache/nested", + true + ], + [ + ".cargo/nested", + true + ], + [ + ".claude/nested", + true + ], + [ + ".codex/nested", + true + ], + [ + ".config/nested", + true + ], + [ + ".git/nested", + true + ], + [ + ".gnupg/nested", + true + ], + [ + ".local/nested", + true + ], + [ + ".npm/nested", + true + ], + [ + ".rustup/nested", + true + ], + [ + ".ssh/nested", + true + ], + [ + ".terraform/nested", + true + ], + [ + ".venv/nested", + true + ], + [ + "__pycache__/nested", + true + ], + [ + "build/nested", + true + ], + [ + "dist/nested", + true + ], + [ + "node_modules/nested", + true + ], + [ + "target/nested", + true + ], + [ + "venv/nested", + true + ], + [ + "repo/.git-credentials", + true + ], + [ + "repo/.netrc", + true + ], + [ + "repo/.npmrc", + true + ], + [ + "repo/.pypirc", + true + ], + [ + "repo/auth.json", + true + ], + [ + "repo/credentials", + true + ], + [ + "repo/credentials.json", + true + ], + [ + "repo/id_ed25519", + true + ], + [ + "repo/id_rsa", + true + ], + [ + ".env", + true + ], + [ + ".env.example", + true + ], + [ + "repo/.env.local", + true + ], + [ + "repo/key.pem", + true + ], + [ + "repo/key.key", + true + ], + [ + "repo/key.p12", + true + ], + [ + "repo/key.pfx", + true + ], + [ + "repo/state.tfstate", + true + ], + [ + "repo/state.tfstate.backup", + true + ], + [ + "src/main.rs", + false + ], + [ + "README.md", + false + ], + [ + "repo/environment.txt", + false + ], + [ + "repo/id_ed25519.pub", + false + ] +] diff --git a/voltage/images/buzz-agent/snapshot.py b/voltage/images/buzz-agent/snapshot.py new file mode 100644 index 00000000000..81b7124a4ab --- /dev/null +++ b/voltage/images/buzz-agent/snapshot.py @@ -0,0 +1,532 @@ +#!/usr/bin/python3 -I +"""Pack fully scanned, self-contained workspace snapshots. No network or upload.""" +import argparse +from contextlib import ExitStack, contextmanager +import errno +import importlib.machinery +import importlib.util +import json +import os +from pathlib import Path, PurePosixPath +import selectors +import signal +import stat +import subprocess +import sys +import tarfile +import tempfile +import time + +MAX_BLOB = 100 * 1024 * 1024 - 26 # BUZZSNAP1 NUL + AEAD tag are added later. +MAX_OBJECTS = 200000 +MAX_REPOS = 64 +MAX_META = 32 * 1024 * 1024 +MAX_PAX = 1024 * 1024 +DEADLINE = None +PHASE = 'archive' +SAFE_ENV = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'LC_ALL': 'C', + 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_SYSTEM': '/dev/null', + 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_NO_REPLACE_OBJECTS': '1', + 'GIT_GRAFT_FILE': '/dev/null', 'GIT_NO_LAZY_FETCH': '1', + 'GIT_ATTR_NOSYSTEM': '1', 'GIT_TEMPLATE_DIR': ''} + + +class Budget(ValueError): + """Refusal without exposing a file name or subprocess output.""" + + +def remaining(seconds=60): + value = min(seconds, DEADLINE - time.monotonic()) + if value <= 0: + raise Budget('Snapshot deadline exceeded') + return value + + +def run(*args, cwd=None, timeout=60, env=None, input=None, output=None, + consume=None, limit=MAX_META): + # Source Git config cannot execute hooks, start fsmonitor, fetch objects, + # or select an unbounded pack compression worker pool. + if args and args[0] == 'git': + args = (args[0], '-c', 'protocol.allow=never', '-c', 'core.hooksPath=/dev/null', + '-c', 'core.fsmonitor=false', '-c', 'pack.threads=1', + '-c', 'pack.windowMemory=16m', '-c', 'core.logAllRefUpdates=false', *args[1:]) + expires = time.monotonic() + remaining(timeout) + with ExitStack() as stack: + sink = stack.enter_context(open(output, 'xb')) if output is not None else None + process = subprocess.Popen(args, cwd=cwd, env=SAFE_ENV, + stdin=subprocess.PIPE if input is not None else subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + start_new_session=True) + data = bytearray() + written = total = 0 + try: + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + if input is not None: + os.set_blocking(process.stdin.fileno(), False) + selector.register(process.stdin, selectors.EVENT_WRITE) + while selector.get_map(): + left = expires - time.monotonic() + if left <= 0: + raise Budget('Snapshot command timed out') + for key, _ in selector.select(left): + if key.fileobj is process.stdin: + count = os.write(process.stdin.fileno(), input[written:written + 65536]) + written += count + if written == len(input): + selector.unregister(process.stdin) + process.stdin.close() + else: + chunk = os.read(process.stdout.fileno(), 65536) + if not chunk: + selector.unregister(process.stdout) + continue + total += len(chunk) + if total > limit: + raise Budget('Snapshot output exceeds budget') + if consume is not None: + consume(chunk) + elif sink is not None: + sink.write(chunk) + else: + data.extend(chunk) + if process.wait(timeout=max(.001, expires - time.monotonic())): + raise ValueError('Snapshot command refused; private output suppressed') + return bytes(data) + finally: + if process.returncode is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + process.stdout.close() + if process.stdin is not None and not process.stdin.closed: + process.stdin.close() + + +def checkpoint(): + installed = Path(__file__).name == 'buzz-agent-snapshot' + path = Path(__file__).resolve().with_name('buzz-agent-checkpoint' if installed else 'checkpoint.py') + loader = importlib.machinery.SourceFileLoader('snapshot_checkpoint', str(path)) + module = importlib.util.module_from_spec(importlib.util.spec_from_loader(loader.name, loader)) + loader.exec_module(module) + module.run = run + module.history_check = lambda repo: (0, run('git', '-C', str(repo), 'fsck', + '--full', '--no-reflogs', '--no-dangling', limit=1024 * 1024)) + return module + + +def path(value): + candidate = PurePosixPath(value) + if (not value or candidate.is_absolute() or str(candidate) != value + or len(value.encode('utf-8')) > 4096 or len(candidate.parts) > 128 + or any(part in {'.', '..'} for part in candidate.parts) + or any(c in value for c in '\\\0:') or any(ord(c) < 32 or ord(c) == 127 for c in value)): + raise ValueError('Snapshot contains an unsupported path') + return candidate + + +def inventory(repo, cp): + lines = cp.git(repo, 'cat-file', '--batch-all-objects', + '--batch-check=%(objectname) %(objecttype) %(objectsize)').decode().splitlines() + total = 0 + if len(lines) > MAX_OBJECTS: + raise Budget('Snapshot Git object count exceeds budget') + for line in lines: + oid, kind, size = line.split(' ') + if (len(oid) not in (40, 64) or any(c not in '0123456789abcdef' for c in oid) + or kind not in {'blob', 'tree', 'commit', 'tag'}): + raise ValueError('Snapshot Git inventory is invalid') + total += int(size) + if total > cp.MAX_BYTES: + raise Budget('Snapshot Git expansion exceeds budget') + return lines + + +def inspect_captured(bundle, cp): + # No fallback: source preflight cannot authorize costs of a different bundle + # captured after the source commits. The native SDK validates the exact pack. + result = json.loads(run('/usr/local/bin/buzz', 'sandbox', 'inspect-bundle', + str(bundle), limit=16 * 1024 * 1024)) + if (not isinstance(result, dict) + or set(result) != {'version', 'objects', 'expanded_bytes', 'hash_bytes', 'refs'} + or type(result['version']) is not int or result['version'] != 1 + or type(result['objects']) is not int or not 0 < result['objects'] <= MAX_OBJECTS + or type(result['expanded_bytes']) is not int or not 0 <= result['expanded_bytes'] <= cp.MAX_BYTES + or type(result['hash_bytes']) is not int or result['hash_bytes'] not in (20, 32) + or not isinstance(result['refs'], list) or not 0 < len(result['refs']) <= cp.MAX_FILES): + raise ValueError('Snapshot captured-bundle metadata is invalid') + parents = set() + for ref in result['refs']: + if (not isinstance(ref, list) or len(ref) != 2 or not all(isinstance(v, str) for v in ref) + or len(ref[0]) != result['hash_bytes'] * 2 + or any(c not in '0123456789abcdef' for c in ref[0]) + or not (ref[1] == 'HEAD' or ref[1].startswith('refs/'))): + raise ValueError('Snapshot captured reference is invalid') + parents.update(path(ref[1]).parents) + return result, len(parents) + + +class ObjectStream: + """Split one bounded cat-file batch without materializing object bodies.""" + def __init__(self, lines, directory): + self.lines = lines + self.directory = directory + self.index = 0 + self.header = bytearray() + self.sink = None + self.left = None + self.separator = False + + def feed(self, chunk): + offset = 0 + while offset < len(chunk): + if self.separator: + if chunk[offset] != 10: + raise ValueError('Snapshot object framing is invalid') + offset += 1 + self.separator = False + self.left = None + self.index += 1 + elif self.left is None: + end = chunk.find(b'\n', offset) + stop = len(chunk) if end < 0 else end + self.header.extend(chunk[offset:stop]) + if len(self.header) > 160: + raise Budget('Snapshot object header exceeds budget') + offset = stop + if end < 0: + continue + if self.index >= len(self.lines) or self.header.decode() != self.lines[self.index]: + raise ValueError('Snapshot object inventory changed') + oid, _, size = self.lines[self.index].split(' ') + self.sink = (self.directory / oid).open('xb') + self.left = int(size) + self.header.clear() + offset += 1 + if self.left == 0: + self.sink.close() + self.sink = None + self.separator = True + else: + count = min(self.left, len(chunk) - offset) + self.sink.write(chunk[offset:offset + count]) + self.left -= count + offset += count + if self.left == 0: + self.sink.close() + self.sink = None + self.separator = True + + def finish(self): + if self.index != len(self.lines) or self.left is not None or self.header: + raise ValueError('Snapshot object stream is incomplete') + + def close(self): + if self.sink is not None: + self.sink.close() + + +class Work: + """Both a configured scratch limit and actual filesystem capacity are required. + + statvfs cannot discover kubelet emptyDir sizeLimit. Operators must configure + this budget below that limit after reserving space for the running agent. + """ + def __init__(self, directory, limit): + if not 1024 * 1024 <= limit <= 2 * 1024 * 1024 * 1024: + raise Budget('Snapshot work budget is invalid') + info = os.statvfs(directory) + self.block = max(512, info.f_frsize) + self.limit = min(limit, info.f_bavail * info.f_frsize) + self.used = 0 + self.directories = set() + self.reserve(16 * 1024 * 1024) # Metadata/index overhead and operational reserve. + + def reserve(self, size): + self.used += size + if self.used > self.limit: + raise Budget('Snapshot scratch capacity exceeds budget') + + def file(self, path): + missing = [parent for parent in path.parents if parent not in self.directories] + self.reserve((len(missing) + 1) * self.block) + self.directories.update(missing) + + +class ArchiveSink: + """Admit compressed output before writing, rather than checking after close.""" + def __init__(self, sink, work): + self.sink = sink + self.work = work + self.size = 0 + + def write(self, data): + if self.size + len(data) > MAX_BLOB: + raise Budget('Snapshot archive exceeds blob budget') + self.work.reserve(len(data)) + count = self.sink.write(data) + self.size += count + return count + + def tell(self): + return self.size + + def flush(self): + self.sink.flush() + + +@contextmanager +def opened_repo(workspace, relative): + descriptor = os.open(workspace, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + for part in relative.parts: + child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=descriptor) + os.close(descriptor) + descriptor = child + # Git follows an already-open directory, not a mutable workspace path. + yield Path(f'/proc/{os.getpid()}/fd/{descriptor}'), descriptor + finally: + os.close(descriptor) + + +def copy_file(repo, descriptor, relative, destination, cp, limit): + # Walk each component with O_NOFOLLOW; source directory replacement cannot + # turn a scan/copy race into a read of an unrelated runtime credential. + parent = os.dup(descriptor) + try: + for component in relative.parts[:-1]: + child = os.open(component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) + os.close(parent) + parent = child + info = os.stat(relative.name, dir_fd=parent, follow_symlinks=False) + destination.parent.mkdir(parents=True, exist_ok=True) + if stat.S_ISLNK(info.st_mode): + target = os.readlink(relative.name, dir_fd=parent) + if (len(target.encode('utf-8')) > 4096 or Path(target).is_absolute() + or any(c in target for c in '\\\0:') + or any(ord(c) < 32 or ord(c) == 127 for c in target)): + raise ValueError('Snapshot symlink is unsafe') + resolved = (repo / relative.parent / target).resolve() + if not resolved.is_relative_to(repo.resolve()): + raise ValueError('Snapshot symlink leaves its repository') + size = len(target.encode('utf-8')) + if size > limit: + raise Budget('Snapshot source exceeds budget') + destination.symlink_to(target) + return size + if not stat.S_ISREG(info.st_mode): + raise ValueError('Snapshot refuses special files') + source = os.open(relative.name, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW, dir_fd=parent) + with os.fdopen(source, 'rb') as stream, destination.open('xb') as sink: + actual = os.fstat(stream.fileno()) + if not stat.S_ISREG(actual.st_mode) or actual.st_size > limit: + raise Budget('Snapshot source exceeds budget') + total = 0 + while chunk := stream.read(65536): + remaining() + total += len(chunk) + if total > limit: + raise Budget('Snapshot source exceeds budget') + sink.write(chunk) + destination.chmod(0o755 if actual.st_mode & 0o100 else 0o644) + return total + finally: + os.close(parent) + + +def discover(workspace, cp): + repos = [] + pending = [workspace] + entries = 0 + while pending: + remaining() + root = pending.pop() + with os.scandir(root) as children: + for child in children: + entries += 1 + if entries > cp.MAX_FILES * 2: + raise Budget('Snapshot discovery exceeds budget') + if child.name == '.git': + repos.append(root) + if len(repos) > MAX_REPOS: + raise Budget('Snapshot repository count exceeds budget') + elif (child.is_dir(follow_symlinks=False) + and not cp.excluded(PurePosixPath(child.name))): + pending.append(Path(child.path)) + return repos + + +def pack(workspace, output, work_bytes=512 * 1024 * 1024): + global DEADLINE, PHASE + DEADLINE = time.monotonic() + 180 + PHASE = 'archive' + os.environ.clear() + os.environ.update(SAFE_ENV) + cp = checkpoint() + workspace = Path(workspace).resolve(strict=True) + output = Path(output) + if output.exists() or output.is_symlink(): + raise ValueError('Snapshot output already exists') + repos = discover(workspace, cp) + if not repos or len(repos) > MAX_REPOS: + raise Budget('Snapshot requires between one and 64 repositories') + total = expanded = objects_count = refs_count = 0 + count = 0 + manifest = {'version': 1, 'snapshot': True, 'scanned': True, 'repositories': []} + with tempfile.TemporaryDirectory(prefix='buzz-snapshot-') as temp: + work = Work(temp, work_bytes) + stage = Path(temp) / 'snapshot' + review = Path(temp) / 'review' + stage.mkdir() + review.mkdir() + for index, repo in enumerate(sorted(repos)): + relative = repo.relative_to(workspace).as_posix() if repo != workspace else 'workspace' + path(relative) + original_relative = repo.relative_to(workspace) + with opened_repo(workspace, original_relative) as (repo, descriptor): + # Inventory before packing is conservative: even unreachable source + # objects count, so no large source object is decoded before a bound. + source_objects = inventory(repo, cp) + source_expanded = sum(int(line.rsplit(' ', 1)[1]) for line in source_objects) + expanded += source_expanded + objects_count += len(source_objects) + if expanded > cp.MAX_BYTES or objects_count > MAX_OBJECTS: + raise Budget('Snapshot aggregate Git expansion exceeds budget') + head = cp.git(repo, 'rev-parse', '--verify', 'HEAD').decode().strip() + record = {'path': relative, 'head': head, 'base': 'bundle', + 'branch': cp.git(repo, 'branch', '--show-current').decode().strip(), + 'shallow': cp.shallow_commits(repo), 'files': [], 'excluded': [], 'deleted': []} + target = stage / 'repositories' / str(index) + (target / 'files').mkdir(parents=True) + bundle = target / 'history.bundle' + source_cost = source_expanded + len(source_objects) * (work.block + 64) + work.reserve(source_cost) + allowance = min(MAX_BLOB, cp.MAX_BYTES - total, (work.limit - work.used) // 2) + if allowance <= 0: + raise Budget('Snapshot bundle capacity exceeds budget') + cp.git(repo, 'bundle', 'create', '-', '--all', 'HEAD', output=bundle, limit=allowance) + bundle_size = bundle.stat().st_size + total += bundle_size + work.reserve(2 * bundle_size) + if total > cp.MAX_BYTES or bundle_size > MAX_BLOB: + raise Budget('Snapshot source exceeds budget') + packed, ref_directories = inspect_captured(bundle, cp) + expanded += max(0, packed['expanded_bytes'] - source_expanded) + objects_count += max(0, packed['objects'] - len(source_objects)) + refs_count += len(packed['refs']) + if expanded > cp.MAX_BYTES or objects_count > MAX_OBJECTS or refs_count > cp.MAX_FILES: + raise Budget('Snapshot captured aggregate inventory exceeds budget') + actual_cost = (packed['expanded_bytes'] + packed['objects'] * (work.block + 64) + + (len(packed['refs']) + ref_directories) * work.block) + work.reserve(max(0, actual_cost - source_cost)) + neutral = Path(temp) / f'history-{index}' + cp.restore_history(bundle, record, neutral) + objects = inventory(neutral, cp) + actual_expanded = sum(int(line.rsplit(' ', 1)[1]) for line in objects) + if actual_expanded != packed['expanded_bytes'] or len(objects) != packed['objects']: + raise ValueError('Snapshot neutral inventory differs from the captured bundle') + object_review = review / str(index) + object_review.mkdir() + writer = ObjectStream(objects, object_review) + try: + names = ''.join(line.split(' ', 1)[0] + '\n' for line in objects).encode() + limit = sum(int(line.rsplit(' ', 1)[1]) for line in objects) + len(objects) * 160 + run('git', '-C', str(neutral), 'cat-file', '--batch', input=names, + consume=writer.feed, limit=limit) + writer.finish() + finally: + writer.close() + # Include ref names and shallow metadata, not just commit diffs. + header = bundle.open('rb') + with header: + value = header.read(8 * 1024 * 1024) + stop = value.find(b'\n\n') + if stop < 0: + raise Budget('Snapshot bundle header exceeds budget') + (object_review / 'references').write_bytes(value[:stop]) + names = cp.git(repo, 'ls-files', '-z', '--cached', '--others', '--exclude-standard') + for raw in sorted(set(names.split(b'\0')) - {b''}): + name = os.fsdecode(raw) + directory_marker = name.endswith('/') + candidate = path(name[:-1] if directory_marker else name) + if cp.excluded(candidate): + record['excluded'].append(str(candidate)) + continue + try: + info = (repo / candidate).lstat() + except FileNotFoundError: + continue # A working-tree deletion remains absent. + if stat.S_ISDIR(info.st_mode): + continue # Git emits untracked nested repos as directory/. + if directory_marker: + raise ValueError('Snapshot directory changed during capture') + destination = target / 'files' / candidate + work.file(destination) + added = copy_file(repo, descriptor, candidate, destination, cp, min(cp.MAX_BYTES - total, work.limit - work.used)) + work.reserve(added) + total += added + count += 1 + if total > cp.MAX_BYTES or count > cp.MAX_FILES: + raise Budget('Snapshot source exceeds budget') + record['files'].append(str(candidate)) + manifest['repositories'].append(record) + PHASE = 'scan' + cp.scan(neutral, history=True, revisions=('--all', 'HEAD', '--no-ext-diff', '--no-textconv')) + PHASE = 'archive' + encoded = json.dumps(manifest).encode() + total += len(encoded) + if total > cp.MAX_BYTES or len(encoded) > 8 * 1024 * 1024: + raise Budget('Snapshot manifest exceeds budget') + (stage / 'manifest.json').write_bytes(encoded) + # Scan the exact copied files and manifest, plus every archived object + # (including merge-only blobs and annotated-tag/commit messages). + PHASE = 'scan' + cp.scan(stage) + cp.scan(review) + PHASE = 'archive' + pax = [0] + def normalized(info): + info.uid = info.gid = info.mtime = 0 + info.uname = info.gname = '' + info.pax_headers = {} + header_bytes = len(info.tobuf(format=tarfile.PAX_FORMAT)) - 512 + pax[0] += header_bytes + if header_bytes > 16384 or pax[0] > MAX_PAX: + raise Budget('Snapshot path metadata exceeds budget') + return info + created = False + try: + with output.open('xb') as sink: + created = True + os.chmod(output, 0o600) + work.file(output) + with tarfile.open(fileobj=ArchiveSink(sink, work), mode='w:gz', dereference=False) as archive: + archive.add(stage, arcname='snapshot', filter=normalized) + if sink.tell() > MAX_BLOB: + raise Budget('Snapshot archive exceeds blob budget') + except BaseException: + if created: + output.unlink() + raise + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--workspace', required=True) + parser.add_argument('--output', required=True) + parser.add_argument('--work-bytes', type=int, default=512 * 1024 * 1024) + args = parser.parse_args() + try: + manifest = pack(args.workspace, args.output, args.work_bytes) + print(json.dumps({'version': 1, 'scanned': True, + 'repositories': len(manifest['repositories']), + 'bytes': Path(args.output).stat().st_size})) + return 0 + except Exception as error: + category = 'budget' if isinstance(error, Budget) or getattr(error, 'errno', None) in (errno.ENOSPC, errno.EDQUOT) else PHASE + print(json.dumps({'error': category}), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/voltage/images/buzz-agent/test-image.sh b/voltage/images/buzz-agent/test-image.sh index 595e3cedb4b..9f3233237c5 100755 --- a/voltage/images/buzz-agent/test-image.sh +++ b/voltage/images/buzz-agent/test-image.sh @@ -52,4 +52,5 @@ docker run --rm --tmpfs /home/agent:uid=10001,gid=10001,mode=0700 \ command -v codex-acp claude-agent-acp buzz-dev-mcp >/dev/null ' python3 "$(dirname "$0")/test-review.py" "$IMAGE" +python3 "$(dirname "$0")/test-snapshot.py" "$IMAGE" printf 'PASS: agent adapter packaging and unauthenticated ACP initialization (%s)\n' "$IMAGE" diff --git a/voltage/images/buzz-agent/test-snapshot.py b/voltage/images/buzz-agent/test-snapshot.py new file mode 100644 index 00000000000..a6c2ae30df2 --- /dev/null +++ b/voltage/images/buzz-agent/test-snapshot.py @@ -0,0 +1,278 @@ +"""Exercise the packaged scanner and self-contained archives without network.""" +import subprocess +import sys +from pathlib import Path + +SCRIPT = r''' +import importlib.machinery, importlib.util, json, os, subprocess, tarfile, tempfile, time +from pathlib import Path + +def load(name, path): + loader=importlib.machinery.SourceFileLoader(name,path) + module=importlib.util.module_from_spec(importlib.util.spec_from_loader(name,loader)) + loader.exec_module(module) + return module +snapshot=load('snapshot','/usr/local/bin/buzz-agent-snapshot') +cp=snapshot.checkpoint() +policy=json.loads(Path('/tmp/policy.json').read_text()) +for value,expected in policy: + assert cp.excluded(Path(value))==expected,(value,expected) +assert all(any(value==name+'/nested' and expected for value,expected in policy) for name in cp.EXCLUDED) +assert all(any(value=='repo/'+name and expected for value,expected in policy) for name in cp.PRIVATE_NAMES) + +def git(repo,*args): + result=subprocess.run(['git','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','-C',str(repo),*args],capture_output=True) + assert result.returncode==0,'fixture Git command failed' + return result.stdout.decode().strip() + +def repository(root,format='sha1'): + repo=root/'workspace'/'example' + repo.mkdir(parents=True) + git(repo,'init','--object-format='+format) + git(repo,'config','user.name','Snapshot Fixture') + git(repo,'config','user.email','fixture@example.invalid') + return repo + +def expect_refusal(workspace,output,category=None,**options): + try: + snapshot.pack(workspace,output,**options) + raise AssertionError('unsafe snapshot was accepted') + except (ValueError,OSError,subprocess.SubprocessError): + assert not output.exists(),'failed archive remained on disk' + if category=='scan': assert snapshot.PHASE=='scan' + +with tempfile.TemporaryDirectory() as temp: + root=Path(temp) + for format in ('sha1','sha256'): + case=root/format + repo=repository(case,format) + (repo/'source.rs').write_text('initial\n') + (repo/'deleted.txt').write_text('tracked deletion\n') + (repo/'nonowner-exec.txt').write_text('clean non-executable\n') + (repo/'.env.example').write_text('PUBLIC_URL=https://example.invalid\n') + git(repo,'add','.') + git(repo,'commit','-m','initial') + git(repo,'branch','local-only') + (repo/'nonowner-exec.txt').chmod(0o654) + assert git(repo,'diff','--name-only','--','nonowner-exec.txt')=='' + (repo/'source.rs').write_text('edited\n') + (repo/'deleted.txt').unlink() + (repo/'new.txt').write_text('untracked\n') + (repo/'.env').write_text('PRIVATE-LOOSE-FILE') + (repo/'credentials.json').write_text('PRIVATE-LOOSE-FILE') + (repo/'tool.sh').write_text('echo fixture\n') + (repo/'tool.sh').chmod(0o755) + (repo/'link').symlink_to('source.rs') + # Valid own PAX output must remain bounded and round-trip. + long=repo/('long-'+'x'*110) + long.write_text('long path\n') + archive=case/'snapshot.tar.gz' + manifest=snapshot.pack(case/'workspace',archive) + assert manifest['snapshot'] is True and manifest['scanned'] is True + record=manifest['repositories'][0] + assert record['base']=='bundle' and record['shallow']==[] + assert 'deleted.txt' not in record['files'] + assert '.env.example' in record['excluded'] + assert 'source.rs' in record['files'] and 'new.txt' in record['files'] + extracted=case/'extracted' + with tarfile.open(archive) as package: + names=package.getnames() + assert not any(name.endswith(('/.env','/credentials.json','/.env.example')) for name in names) + assert package.getmember('snapshot/repositories/0/files/link').issym() + assert package.getmember('snapshot/repositories/0/files/tool.sh').mode&0o100 + assert package.getmember('snapshot/repositories/0/files/nonowner-exec.txt').mode&0o111==0,'non-owner execute bits invented an executable Git file' + assert all(member.mtime==0 for member in package) + package.extractall(extracted,filter='data') + history=case/'history' + snapshot.DEADLINE=time.monotonic()+60 + cp=snapshot.checkpoint() + cp.restore_history(extracted/'snapshot/repositories/0/history.bundle',record,history) + assert git(history,'show','HEAD:.env.example').startswith('PUBLIC_URL=') + assert 'refs/heads/local-only' in git(history,'show-ref') + assert (extracted/'snapshot/repositories/0/files'/long.name).read_text()=='long path\n' + print('PASS: full history, exclusions, dirty files, Git owner-execute semantics, links, PAX and '+format) + + # Shallow bundle restores with its explicit local boundary, without origin. + source=root/'sha1/workspace/example' + (source/'source.rs').write_text('committed second\n') + git(source,'add','source.rs') + git(source,'commit','-m','second') + shallow=root/'shallow/workspace/example' + shallow.parent.mkdir(parents=True) + git(source,'clone','--depth=1',source.as_uri(),str(shallow)) + shallow_archive=root/'shallow/snapshot.tar.gz' + record=snapshot.pack(shallow.parent,shallow_archive)['repositories'][0] + assert record['shallow'] + staging=root/'shallow-extracted' + with tarfile.open(shallow_archive) as package: package.extractall(staging,filter='data') + snapshot.DEADLINE=time.monotonic()+60 + cp=snapshot.checkpoint() + bundle=staging/'snapshot/repositories/0/history.bundle' + cp.restore_history(bundle,record,root/'shallow-good') + broken=dict(record,shallow=[]) + try: + cp.restore_history(bundle,broken,root/'shallow-bad') + raise AssertionError('undeclared missing parents were accepted') + except ValueError: pass + print('PASS: explicit shallow boundaries; undeclared missing parents refused') + + case=root/'nested';outer=repository(case) + (outer/'outer.txt').write_text('outer\n');git(outer,'add','.');git(outer,'commit','-m','outer') + nested=outer/'nested';git(outer,'init',str(nested)) + git(nested,'config','user.name','Snapshot Fixture');git(nested,'config','user.email','fixture@example.invalid') + (nested/'inner.txt').write_text('inner\n');git(nested,'add','.');git(nested,'commit','-m','inner') + manifest=snapshot.pack(outer.parent,case/'nested.tar.gz') + assert [record['path'] for record in manifest['repositories']]==['example','example/nested'] + with tarfile.open(case/'nested.tar.gz') as package: + assert package.extractfile('snapshot/repositories/0/files/outer.txt').read()==b'outer\n' + assert package.extractfile('snapshot/repositories/1/files/inner.txt').read()==b'inner\n' + print('PASS: real outer plus untracked nested Git repository have separate complete bundles') + + # Real commits between source preflight and capture must be charged BEFORE import. + for succeeds,budget in ((False,20*1024*1024),(True,128*1024*1024)): + case=root/('racing-control' if succeeds else 'racing-refusal');repo=repository(case) + (repo/'safe').write_text('safe\n');git(repo,'add','.');git(repo,'commit','-m','initial') + cp=snapshot.checkpoint();original_git=cp.git;original_restore=cp.restore_history + seen={'changed':False,'imported':False} + def racing(directory,*args,**options): + if not seen['changed'] and args==('rev-parse','--verify','HEAD'): + seen['changed']=True + with (repo/'late-history').open('wb') as stream: stream.truncate(8*1024*1024) + git(repo,'add','late-history');git(repo,'commit','-m','concurrent source') + (repo/'late-history').unlink() + return original_git(directory,*args,**options) + def restoring(*args,**options): + seen['imported']=True + return original_restore(*args,**options) + cp.git=racing;cp.restore_history=restoring + factory=snapshot.checkpoint;snapshot.checkpoint=lambda:cp + output=case/'snapshot.tar.gz' + try: + if succeeds: + snapshot.pack(repo.parent,output,work_bytes=budget) + assert output.exists() and seen['imported'] + else: + try: + snapshot.pack(repo.parent,output,work_bytes=budget) + raise AssertionError('captured history escaped work accounting') + except snapshot.Budget: + assert not seen['imported'] and not output.exists() + assert seen['changed'] + finally: snapshot.checkpoint=factory + print('PASS: concurrent source growth refused before import at20MiB and accepted at128MiB control') + + # The packaged native inspector is required; invalid metadata cannot fall back. + case=root/'inspector-required';repo=repository(case) + (repo/'safe').write_text('safe\n');git(repo,'add','.');git(repo,'commit','-m','initial') + original_run=snapshot.run + for missing in (False,True): + def invalid_inspector(*args,**options): + if args[0]=='/usr/local/bin/buzz': + if missing: raise FileNotFoundError('required inspector is unavailable') + return b'{}' + return original_run(*args,**options) + snapshot.run=invalid_inspector + try: expect_refusal(repo.parent,case/'refused.tar.gz') + finally: snapshot.run=original_run + print('PASS: missing or invalid native inspector has no source-inventory fallback') + + # No current or historical credential may be hidden by repo/global ignores. + secret='sk-ant-oat01-'+'Z'*90 + for historical in (False,True): + case=root/('old-secret' if historical else 'new-secret') + repo=repository(case) + (repo/'safe').write_text('safe\n') + git(repo,'add','safe');git(repo,'commit','-m','initial') + (repo/'payload.txt').write_text(secret) + if historical: + git(repo,'add','payload.txt');git(repo,'commit','-m','old secret') + (repo/'payload.txt').unlink();git(repo,'commit','-am','remove secret') + (repo/'.gitleaksignore').write_text('*\n') + (repo/'.gitleaks.toml').write_text('[allowlist]\nregexes=[".*"]\n') + os.environ['GITLEAKS_CONFIG']=str(repo/'.gitleaks.toml') + expect_refusal(case/'workspace',case/'refused.tar.gz',category='scan') + print('PASS: staged working-tree and deleted historical secret refusal ignores bypass attempts') + + # A secret introduced only by a merge and an annotated tag still gets scanned. + case=root/'metadata-secret';repo=repository(case) + (repo/'safe').write_text('safe\n');git(repo,'add','safe');git(repo,'commit','-m','initial') + git(repo,'tag','-a','secret-tag','-m',secret) + expect_refusal(case/'workspace',case/'refused.tar.gz',category='scan') + print('PASS: annotated-tag secret is scanned as an archived object') + + case=root/'merge-secret';repo=repository(case) + (repo/'safe').write_text('safe\n');git(repo,'add','safe');git(repo,'commit','-m','initial') + base=git(repo,'branch','--show-current') + git(repo,'checkout','-b','side') + (repo/'side').write_text('side\n');git(repo,'add','side');git(repo,'commit','-m','side') + git(repo,'checkout',base) + (repo/'main').write_text('main\n');git(repo,'add','main');git(repo,'commit','-m','main') + git(repo,'merge','--no-commit','side') + (repo/'payload.txt').write_text(secret);git(repo,'add','payload.txt');git(repo,'commit','-m','merge') + (repo/'payload.txt').unlink();git(repo,'commit','-am','remove merge secret') + expect_refusal(case/'workspace',case/'refused.tar.gz',category='scan') + print('PASS: deleted merge-only secret is scanned from the exact bundled object') + + case=root/'bounds';repo=repository(case) + (repo/'safe').write_text('safe\n');git(repo,'add','safe');git(repo,'commit','-m','initial') + expect_refusal(case/'workspace',case/'small-budget.tar.gz',work_bytes=1024*1024) + (repo/'escape').symlink_to('/etc/passwd') + expect_refusal(case/'workspace',case/'external-link.tar.gz') + (repo/'escape').unlink() + (repo/'bad').write_bytes(b'x'*1024) + cp=snapshot.checkpoint();cp.MAX_BYTES=64 + original=snapshot.checkpoint;snapshot.checkpoint=lambda:cp + expect_refusal(case/'workspace',case/'scaled-source.tar.gz') + snapshot.checkpoint=original + output=case/'preserve-existing';output.write_text('keep me') + try: snapshot.pack(case/'workspace',output);raise AssertionError('existing output overwritten') + except ValueError: assert output.read_text()=='keep me' + unborn=repository(root/'unborn') + expect_refusal(unborn.parent,root/'unborn/refused.tar.gz') + print('PASS: capacity/source budgets, external links, existing files and unborn HEAD refused') + + # Bound both generic subprocess output and compressed archive writes BEFORE writes. + snapshot.DEADLINE=time.monotonic()+60 + limited=root/'limited-output' + try: + snapshot.run('/usr/bin/python3','-c','import sys;sys.stdout.buffer.write(b"x"*8192)',output=limited,limit=64) + raise AssertionError('subprocess output bound was bypassed') + except snapshot.Budget: assert limited.stat().st_size<=64 + import io + sink=io.BytesIO();work=snapshot.Work(root,512*1024*1024) + previous=snapshot.MAX_BLOB;snapshot.MAX_BLOB=8 + try: + snapshot.ArchiveSink(sink,work).write(b'x'*9) + raise AssertionError('compressed output was written before admission') + except snapshot.Budget: assert sink.getvalue()==b'' + finally: snapshot.MAX_BLOB=previous + # Intermediate source symlinks cannot redirect a copy outside the open repo. + cp=snapshot.checkpoint();snapshot.DEADLINE=time.monotonic()+60 + (repo/'sub').mkdir();outside=root/'outside';outside.mkdir() + (outside/'value').write_text('OUTSIDE-WORKSPACE') + (repo/'sub').rmdir();(repo/'sub').symlink_to(outside) + with snapshot.opened_repo(repo.parent,Path('example')) as (opened,descriptor): + try: + snapshot.copy_file(opened,descriptor,snapshot.path('sub/value'),root/'copy-refused',cp,100) + raise AssertionError('intermediate symlink redirected a source read') + except OSError: assert not (root/'copy-refused').exists() + small=root/'discovery-bound';small.mkdir() + for number in range(3): (small/str(number)).write_text('x') + previous=cp.MAX_FILES;cp.MAX_FILES=1 + try: + snapshot.discover(small,cp) + raise AssertionError('streamed discovery budget was bypassed') + except snapshot.Budget: pass + finally: cp.MAX_FILES=previous + print('PASS: streaming output admission and descriptor-relative source containment') +''' + +policy = Path(__file__).with_name('snapshot-policy.json').resolve() +result = subprocess.run(['docker','run','--rm','-i','--network','none','--read-only', + '--tmpfs','/tmp:rw,nosuid,nodev,size=1g,uid=10001,gid=10001,mode=0700', + '--memory','1g','--cap-drop','ALL','--security-opt','no-new-privileges', + '-v',f'{policy}:/tmp/policy.json:ro','--entrypoint','python3',sys.argv[1],'-'], + input=SCRIPT,text=True,capture_output=True,timeout=240) +print(result.stdout,end='') +print(result.stderr,end='',file=sys.stderr) +sys.exit(result.returncode)