From 6504ed331f64ba622cf005958f2387a5b2b45130 Mon Sep 17 00:00:00 2001 From: Tee8z Date: Tue, 10 Nov 2026 16:24:15 -0500 Subject: [PATCH 1/4] feat(agents): package fully scanned self-contained snapshots Signed-off-by: Tee8z --- docs/sandbox-snapshot-producer.md | 58 ++ voltage/images/buzz-agent/Dockerfile | 1 + .../images/buzz-agent/snapshot-policy.json | 170 ++++++ voltage/images/buzz-agent/snapshot.py | 498 ++++++++++++++++++ voltage/images/buzz-agent/test-image.sh | 1 + voltage/images/buzz-agent/test-snapshot.py | 226 ++++++++ 6 files changed, 954 insertions(+) create mode 100644 docs/sandbox-snapshot-producer.md create mode 100644 voltage/images/buzz-agent/snapshot-policy.json create mode 100644 voltage/images/buzz-agent/snapshot.py create mode 100644 voltage/images/buzz-agent/test-snapshot.py diff --git a/docs/sandbox-snapshot-producer.md b/docs/sandbox-snapshot-producer.md new file mode 100644 index 00000000000..6bc79a90fe9 --- /dev/null +++ b/docs/sandbox-snapshot-producer.md @@ -0,0 +1,58 @@ +# Snapshot archive producer + +`buzz-agent-snapshot` builds a fully scanned archive inside an agent sandbox. +This layer packages a local artifact only. The ACP request handler, encrypted Blossom transfer, Rust extraction, and user-facing CLI follow separately. +The caller must hold a live owner-approved snapshot grant and run the helper under the isolated worker's containment and cancellation rules. +No terminal approval or global gateway delegation alone permits export. + +```sh +buzz-agent-snapshot --workspace /workspace --output /tmp/private-job/archive.tar.gz --work-bytes 536870912 +``` + +The output must be new. It is mode 0600 and is removed if creation fails. +Stdout contains only version, scan status, repository count, and byte count. +Failures report only `scan`, `budget`, or `archive`; scanner output and file names are suppressed. +The helper never uploads, fetches from a forge, signs relay events, or falls back to quarantine. + +Every repository gets a self-contained `git bundle --all HEAD` plus its tracked and non-ignored untracked working files. +Nested repositories have separate records and bundles; Git's directory markers are not copied as files. +The manifest records `version: 1`, `snapshot: true`, `scanned: true`, and repository path, head, branch, explicit shallow boundaries, file list, exclusions, and deletions. +Every record uses `base: bundle`; there is no origin URL or clone prerequisite. +A workspace that is itself a repository uses the output path `workspace`. + +The tar layout reuses the checkpoint structure: + +```text +snapshot/manifest.json +snapshot/repositories/0/history.bundle +snapshot/repositories/0/files/ +``` + +The user chose full locally available scanned history. +Historical versioned content, including a historical `.env.example`, remains in the bundle. +The working copy excludes checkpoint private/runtime paths and `.env*` files. +The next extractor must mark intentionally omitted tracked paths skip-worktree so they do not appear as accidental deletions; genuine source deletions remain absent. +Shallow clones retain their explicit local boundaries, without unshallow or network fetch. +Unborn HEADs are refused. + +Before packing, the helper inventories source objects and byte/count budgets. +It imports the bundle into a neutral Git repository, verifies its object graph, and scans that exact history with immutable gitleaks rules. +It also scans the copied working files, manifest, ref names, and every archived raw object, including merge-only blobs and annotated-tag messages. +Global/system configuration, replacement refs, grafts, lazy fetch, hooks, fsmonitor, repository ignore files, and `GITLEAKS_*` overrides cannot weaken the scan. +Repository and file reads use open directory descriptors and refuse source path redirection through symlinks. +Internal relative symlinks are preserved; external links and special files are refused. + +Limits are 256 MiB source/expanded object bytes, 200,000 objects, 50,000 files/references, 64 repositories, and 180 seconds total creation time. +Subprocess output and time are bounded, and Git object bodies use one streamed batch. +Bundle and gzip output are admitted before writes; the final archive leaves space for the encrypted blob prefix/tag within 100 MiB. +The local work budget defaults to 512 MiB and may be set from 1 MiB through 2 GiB; filesystem free space can lower it. +The budget counts object review files, bundles/imports, copied files, block/index overhead, compressed output, and an operational reserve. +`statvfs` cannot reveal Kubernetes emptyDir sizeLimit: configure the work budget below the actual scratch quota after accounting for other agent data. +The separate attaching-machine extractor also needs its own store/work budget and capacity checks. + +Tar ownership and timestamps are normalized. UTF-8 path/link metadata is capped at 16 KiB per header and 1 MiB total; the next extractor must apply bounds before extension allocation. +The archive is validated and scanned after copying; source work can continue during capture, so it is a bounded working view rather than a filesystem transaction. + +Run `python3 voltage/images/buzz-agent/test-snapshot.py ` to exercise packaged code and the actual scanner with no network, a read-only root, UID 10001, dropped capabilities, and a 1 GiB memory cap. +The tests cover SHA-1/SHA-256, local branches, nested repositories, dirty/deleted files, executable modes, relative links, long PAX names, shallow boundaries, source/capacity refusal, and current/deleted/merge/tag secret refusal with ignore bypass attempts. +The shared policy fixture is checked against checkpoint constants and will also bind the Rust extractor. diff --git a/voltage/images/buzz-agent/Dockerfile b/voltage/images/buzz-agent/Dockerfile index 86a91eb7be0..05c0d0653a1 100644 --- a/voltage/images/buzz-agent/Dockerfile +++ b/voltage/images/buzz-agent/Dockerfile @@ -83,6 +83,7 @@ COPY --chmod=0755 voltage/images/buzz-agent/account.py /usr/local/bin/buzz-agent COPY --chmod=0755 voltage/images/buzz-agent/profile.py /usr/local/bin/buzz-agent-profile COPY --chmod=0755 voltage/images/buzz-agent/cache.py /usr/local/bin/buzz-agent-cache COPY --chmod=0755 voltage/images/buzz-agent/checkpoint.py /usr/local/bin/buzz-agent-checkpoint +COPY --chmod=0755 voltage/images/buzz-agent/snapshot.py /usr/local/bin/buzz-agent-snapshot COPY --chmod=0755 voltage/images/buzz-agent/review.py /usr/local/bin/buzz-agent-review COPY --chmod=0755 voltage/images/buzz-agent/heavy.py /usr/local/bin/buzz-heavy COPY --chmod=0755 voltage/images/buzz-agent/worker.py /usr/local/bin/buzz-agent-worker diff --git a/voltage/images/buzz-agent/snapshot-policy.json b/voltage/images/buzz-agent/snapshot-policy.json new file mode 100644 index 00000000000..09826a5c3a6 --- /dev/null +++ b/voltage/images/buzz-agent/snapshot-policy.json @@ -0,0 +1,170 @@ +[ + [ + ".aws/nested", + true + ], + [ + ".cache/nested", + true + ], + [ + ".cargo/nested", + true + ], + [ + ".claude/nested", + true + ], + [ + ".codex/nested", + true + ], + [ + ".config/nested", + true + ], + [ + ".git/nested", + true + ], + [ + ".gnupg/nested", + true + ], + [ + ".local/nested", + true + ], + [ + ".npm/nested", + true + ], + [ + ".rustup/nested", + true + ], + [ + ".ssh/nested", + true + ], + [ + ".terraform/nested", + true + ], + [ + ".venv/nested", + true + ], + [ + "__pycache__/nested", + true + ], + [ + "build/nested", + true + ], + [ + "dist/nested", + true + ], + [ + "node_modules/nested", + true + ], + [ + "target/nested", + true + ], + [ + "venv/nested", + true + ], + [ + "repo/.git-credentials", + true + ], + [ + "repo/.netrc", + true + ], + [ + "repo/.npmrc", + true + ], + [ + "repo/.pypirc", + true + ], + [ + "repo/auth.json", + true + ], + [ + "repo/credentials", + true + ], + [ + "repo/credentials.json", + true + ], + [ + "repo/id_ed25519", + true + ], + [ + "repo/id_rsa", + true + ], + [ + ".env", + true + ], + [ + ".env.example", + true + ], + [ + "repo/.env.local", + true + ], + [ + "repo/key.pem", + true + ], + [ + "repo/key.key", + true + ], + [ + "repo/key.p12", + true + ], + [ + "repo/key.pfx", + true + ], + [ + "repo/state.tfstate", + true + ], + [ + "repo/state.tfstate.backup", + true + ], + [ + "src/main.rs", + false + ], + [ + "README.md", + false + ], + [ + "repo/environment.txt", + false + ], + [ + "repo/id_ed25519.pub", + false + ] +] diff --git a/voltage/images/buzz-agent/snapshot.py b/voltage/images/buzz-agent/snapshot.py new file mode 100644 index 00000000000..9c50c81ecd7 --- /dev/null +++ b/voltage/images/buzz-agent/snapshot.py @@ -0,0 +1,498 @@ +#!/usr/bin/python3 -I +"""Pack fully scanned, self-contained workspace snapshots. No network or upload.""" +import argparse +from contextlib import ExitStack, contextmanager +import errno +import importlib.machinery +import importlib.util +import json +import os +from pathlib import Path, PurePosixPath +import selectors +import signal +import stat +import subprocess +import sys +import tarfile +import tempfile +import time + +MAX_BLOB = 100 * 1024 * 1024 - 26 # BUZZSNAP1 NUL + AEAD tag are added later. +MAX_OBJECTS = 200000 +MAX_REPOS = 64 +MAX_META = 32 * 1024 * 1024 +MAX_PAX = 1024 * 1024 +DEADLINE = None +PHASE = 'archive' +SAFE_ENV = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'LC_ALL': 'C', + 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_CONFIG_SYSTEM': '/dev/null', + 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_NO_REPLACE_OBJECTS': '1', + 'GIT_GRAFT_FILE': '/dev/null', 'GIT_NO_LAZY_FETCH': '1', + 'GIT_ATTR_NOSYSTEM': '1', 'GIT_TEMPLATE_DIR': ''} + + +class Budget(ValueError): + """Refusal without exposing a file name or subprocess output.""" + + +def remaining(seconds=60): + value = min(seconds, DEADLINE - time.monotonic()) + if value <= 0: + raise Budget('Snapshot deadline exceeded') + return value + + +def run(*args, cwd=None, timeout=60, env=None, input=None, output=None, + consume=None, limit=MAX_META): + # Source Git config cannot execute hooks, start fsmonitor, fetch objects, + # or select an unbounded pack compression worker pool. + if args and args[0] == 'git': + args = (args[0], '-c', 'protocol.allow=never', '-c', 'core.hooksPath=/dev/null', + '-c', 'core.fsmonitor=false', '-c', 'pack.threads=1', + '-c', 'pack.windowMemory=16m', *args[1:]) + expires = time.monotonic() + remaining(timeout) + with ExitStack() as stack: + sink = stack.enter_context(open(output, 'xb')) if output is not None else None + process = subprocess.Popen(args, cwd=cwd, env=SAFE_ENV, + stdin=subprocess.PIPE if input is not None else subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + start_new_session=True) + data = bytearray() + written = total = 0 + try: + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + if input is not None: + os.set_blocking(process.stdin.fileno(), False) + selector.register(process.stdin, selectors.EVENT_WRITE) + while selector.get_map(): + left = expires - time.monotonic() + if left <= 0: + raise Budget('Snapshot command timed out') + for key, _ in selector.select(left): + if key.fileobj is process.stdin: + count = os.write(process.stdin.fileno(), input[written:written + 65536]) + written += count + if written == len(input): + selector.unregister(process.stdin) + process.stdin.close() + else: + chunk = os.read(process.stdout.fileno(), 65536) + if not chunk: + selector.unregister(process.stdout) + continue + total += len(chunk) + if total > limit: + raise Budget('Snapshot output exceeds budget') + if consume is not None: + consume(chunk) + elif sink is not None: + sink.write(chunk) + else: + data.extend(chunk) + if process.wait(timeout=max(.001, expires - time.monotonic())): + raise ValueError('Snapshot command refused; private output suppressed') + return bytes(data) + finally: + if process.returncode is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + process.stdout.close() + if process.stdin is not None and not process.stdin.closed: + process.stdin.close() + + +def checkpoint(): + installed = Path(__file__).name == 'buzz-agent-snapshot' + path = Path(__file__).resolve().with_name('buzz-agent-checkpoint' if installed else 'checkpoint.py') + loader = importlib.machinery.SourceFileLoader('snapshot_checkpoint', str(path)) + module = importlib.util.module_from_spec(importlib.util.spec_from_loader(loader.name, loader)) + loader.exec_module(module) + module.run = run + module.history_check = lambda repo: (0, run('git', '-C', str(repo), 'fsck', + '--full', '--no-reflogs', '--no-dangling', limit=1024 * 1024)) + return module + + +def path(value): + candidate = PurePosixPath(value) + if (not value or candidate.is_absolute() or str(candidate) != value + or len(value.encode('utf-8')) > 4096 or len(candidate.parts) > 128 + or any(part in {'.', '..'} for part in candidate.parts) + or any(c in value for c in '\\\0:') or any(ord(c) < 32 or ord(c) == 127 for c in value)): + raise ValueError('Snapshot contains an unsupported path') + return candidate + + +def inventory(repo, cp): + lines = cp.git(repo, 'cat-file', '--batch-all-objects', + '--batch-check=%(objectname) %(objecttype) %(objectsize)').decode().splitlines() + total = 0 + if len(lines) > MAX_OBJECTS: + raise Budget('Snapshot Git object count exceeds budget') + for line in lines: + oid, kind, size = line.split(' ') + if (len(oid) not in (40, 64) or any(c not in '0123456789abcdef' for c in oid) + or kind not in {'blob', 'tree', 'commit', 'tag'}): + raise ValueError('Snapshot Git inventory is invalid') + total += int(size) + if total > cp.MAX_BYTES: + raise Budget('Snapshot Git expansion exceeds budget') + return lines + + +class ObjectStream: + """Split one bounded cat-file batch without materializing object bodies.""" + def __init__(self, lines, directory): + self.lines = lines + self.directory = directory + self.index = 0 + self.header = bytearray() + self.sink = None + self.left = None + self.separator = False + + def feed(self, chunk): + offset = 0 + while offset < len(chunk): + if self.separator: + if chunk[offset] != 10: + raise ValueError('Snapshot object framing is invalid') + offset += 1 + self.separator = False + self.left = None + self.index += 1 + elif self.left is None: + end = chunk.find(b'\n', offset) + stop = len(chunk) if end < 0 else end + self.header.extend(chunk[offset:stop]) + if len(self.header) > 160: + raise Budget('Snapshot object header exceeds budget') + offset = stop + if end < 0: + continue + if self.index >= len(self.lines) or self.header.decode() != self.lines[self.index]: + raise ValueError('Snapshot object inventory changed') + oid, _, size = self.lines[self.index].split(' ') + self.sink = (self.directory / oid).open('xb') + self.left = int(size) + self.header.clear() + offset += 1 + if self.left == 0: + self.sink.close() + self.sink = None + self.separator = True + else: + count = min(self.left, len(chunk) - offset) + self.sink.write(chunk[offset:offset + count]) + self.left -= count + offset += count + if self.left == 0: + self.sink.close() + self.sink = None + self.separator = True + + def finish(self): + if self.index != len(self.lines) or self.left is not None or self.header: + raise ValueError('Snapshot object stream is incomplete') + + def close(self): + if self.sink is not None: + self.sink.close() + + +class Work: + """Both a configured scratch limit and actual filesystem capacity are required. + + statvfs cannot discover kubelet emptyDir sizeLimit. Operators must configure + this budget below that limit after reserving space for the running agent. + """ + def __init__(self, directory, limit): + if not 1024 * 1024 <= limit <= 2 * 1024 * 1024 * 1024: + raise Budget('Snapshot work budget is invalid') + info = os.statvfs(directory) + self.block = max(512, info.f_frsize) + self.limit = min(limit, info.f_bavail * info.f_frsize) + self.used = 0 + self.directories = set() + self.reserve(16 * 1024 * 1024) # Metadata/index overhead and operational reserve. + + def reserve(self, size): + self.used += size + if self.used > self.limit: + raise Budget('Snapshot scratch capacity exceeds budget') + + def file(self, path): + missing = [parent for parent in path.parents if parent not in self.directories] + self.reserve((len(missing) + 1) * self.block) + self.directories.update(missing) + + +class ArchiveSink: + """Admit compressed output before writing, rather than checking after close.""" + def __init__(self, sink, work): + self.sink = sink + self.work = work + self.size = 0 + + def write(self, data): + if self.size + len(data) > MAX_BLOB: + raise Budget('Snapshot archive exceeds blob budget') + self.work.reserve(len(data)) + count = self.sink.write(data) + self.size += count + return count + + def tell(self): + return self.size + + def flush(self): + self.sink.flush() + + +@contextmanager +def opened_repo(workspace, relative): + descriptor = os.open(workspace, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + for part in relative.parts: + child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=descriptor) + os.close(descriptor) + descriptor = child + # Git follows an already-open directory, not a mutable workspace path. + yield Path(f'/proc/{os.getpid()}/fd/{descriptor}'), descriptor + finally: + os.close(descriptor) + + +def copy_file(repo, descriptor, relative, destination, cp, limit): + # Walk each component with O_NOFOLLOW; source directory replacement cannot + # turn a scan/copy race into a read of an unrelated runtime credential. + parent = os.dup(descriptor) + try: + for component in relative.parts[:-1]: + child = os.open(component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) + os.close(parent) + parent = child + info = os.stat(relative.name, dir_fd=parent, follow_symlinks=False) + destination.parent.mkdir(parents=True, exist_ok=True) + if stat.S_ISLNK(info.st_mode): + target = os.readlink(relative.name, dir_fd=parent) + if (len(target.encode('utf-8')) > 4096 or Path(target).is_absolute() + or any(c in target for c in '\\\0:') + or any(ord(c) < 32 or ord(c) == 127 for c in target)): + raise ValueError('Snapshot symlink is unsafe') + resolved = (repo / relative.parent / target).resolve() + if not resolved.is_relative_to(repo.resolve()): + raise ValueError('Snapshot symlink leaves its repository') + size = len(target.encode('utf-8')) + if size > limit: + raise Budget('Snapshot source exceeds budget') + destination.symlink_to(target) + return size + if not stat.S_ISREG(info.st_mode): + raise ValueError('Snapshot refuses special files') + source = os.open(relative.name, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW, dir_fd=parent) + with os.fdopen(source, 'rb') as stream, destination.open('xb') as sink: + actual = os.fstat(stream.fileno()) + if not stat.S_ISREG(actual.st_mode) or actual.st_size > limit: + raise Budget('Snapshot source exceeds budget') + total = 0 + while chunk := stream.read(65536): + remaining() + total += len(chunk) + if total > limit: + raise Budget('Snapshot source exceeds budget') + sink.write(chunk) + destination.chmod(0o755 if actual.st_mode & 0o111 else 0o644) + return total + finally: + os.close(parent) + + +def discover(workspace, cp): + repos = [] + pending = [workspace] + entries = 0 + while pending: + remaining() + root = pending.pop() + with os.scandir(root) as children: + for child in children: + entries += 1 + if entries > cp.MAX_FILES * 2: + raise Budget('Snapshot discovery exceeds budget') + if child.name == '.git': + repos.append(root) + if len(repos) > MAX_REPOS: + raise Budget('Snapshot repository count exceeds budget') + elif (child.is_dir(follow_symlinks=False) + and not cp.excluded(PurePosixPath(child.name))): + pending.append(Path(child.path)) + return repos + + +def pack(workspace, output, work_bytes=512 * 1024 * 1024): + global DEADLINE, PHASE + DEADLINE = time.monotonic() + 180 + PHASE = 'archive' + os.environ.clear() + os.environ.update(SAFE_ENV) + cp = checkpoint() + workspace = Path(workspace).resolve(strict=True) + output = Path(output) + if output.exists() or output.is_symlink(): + raise ValueError('Snapshot output already exists') + repos = discover(workspace, cp) + if not repos or len(repos) > MAX_REPOS: + raise Budget('Snapshot requires between one and 64 repositories') + total = expanded = objects_count = refs_count = 0 + count = 0 + manifest = {'version': 1, 'snapshot': True, 'scanned': True, 'repositories': []} + with tempfile.TemporaryDirectory(prefix='buzz-snapshot-') as temp: + work = Work(temp, work_bytes) + stage = Path(temp) / 'snapshot' + review = Path(temp) / 'review' + stage.mkdir() + review.mkdir() + for index, repo in enumerate(sorted(repos)): + relative = repo.relative_to(workspace).as_posix() if repo != workspace else 'workspace' + path(relative) + original_relative = repo.relative_to(workspace) + with opened_repo(workspace, original_relative) as (repo, descriptor): + # Inventory before packing is conservative: even unreachable source + # objects count, so no large source object is decoded before a bound. + source_objects = inventory(repo, cp) + expanded += sum(int(line.rsplit(' ', 1)[1]) for line in source_objects) + objects_count += len(source_objects) + if expanded > cp.MAX_BYTES or objects_count > MAX_OBJECTS: + raise Budget('Snapshot aggregate Git expansion exceeds budget') + head = cp.git(repo, 'rev-parse', '--verify', 'HEAD').decode().strip() + record = {'path': relative, 'head': head, 'base': 'bundle', + 'branch': cp.git(repo, 'branch', '--show-current').decode().strip(), + 'shallow': cp.shallow_commits(repo), 'files': [], 'excluded': [], 'deleted': []} + target = stage / 'repositories' / str(index) + (target / 'files').mkdir(parents=True) + bundle = target / 'history.bundle' + work.reserve(sum(int(line.rsplit(' ', 1)[1]) for line in source_objects) + + len(source_objects) * (work.block + 64)) + allowance = min(MAX_BLOB, cp.MAX_BYTES - total, (work.limit - work.used) // 2) + if allowance <= 0: + raise Budget('Snapshot bundle capacity exceeds budget') + cp.git(repo, 'bundle', 'create', '-', '--all', 'HEAD', output=bundle, limit=allowance) + bundle_size = bundle.stat().st_size + total += bundle_size + work.reserve(2 * bundle_size) + if total > cp.MAX_BYTES or bundle_size > MAX_BLOB: + raise Budget('Snapshot source exceeds budget') + neutral = Path(temp) / f'history-{index}' + cp.restore_history(bundle, record, neutral) + objects = inventory(neutral, cp) + object_review = review / str(index) + object_review.mkdir() + writer = ObjectStream(objects, object_review) + try: + names = ''.join(line.split(' ', 1)[0] + '\n' for line in objects).encode() + limit = sum(int(line.rsplit(' ', 1)[1]) for line in objects) + len(objects) * 160 + run('git', '-C', str(neutral), 'cat-file', '--batch', input=names, + consume=writer.feed, limit=limit) + writer.finish() + finally: + writer.close() + # Include ref names and shallow metadata, not just commit diffs. + header = bundle.open('rb') + with header: + value = header.read(8 * 1024 * 1024) + stop = value.find(b'\n\n') + if stop < 0: + raise Budget('Snapshot bundle header exceeds budget') + refs_count += sum(1 for line in value[:stop].splitlines() if not line.startswith((b'#', b'@', b'-'))) + if refs_count > cp.MAX_FILES: + raise Budget('Snapshot reference count exceeds budget') + (object_review / 'references').write_bytes(value[:stop]) + names = cp.git(repo, 'ls-files', '-z', '--cached', '--others', '--exclude-standard') + for raw in sorted(set(names.split(b'\0')) - {b''}): + name = os.fsdecode(raw) + directory_marker = name.endswith('/') + candidate = path(name[:-1] if directory_marker else name) + if cp.excluded(candidate): + record['excluded'].append(str(candidate)) + continue + try: + info = (repo / candidate).lstat() + except FileNotFoundError: + continue # A working-tree deletion remains absent. + if stat.S_ISDIR(info.st_mode): + continue # Git emits untracked nested repos as directory/. + if directory_marker: + raise ValueError('Snapshot directory changed during capture') + destination = target / 'files' / candidate + work.file(destination) + added = copy_file(repo, descriptor, candidate, destination, cp, min(cp.MAX_BYTES - total, work.limit - work.used)) + work.reserve(added) + total += added + count += 1 + if total > cp.MAX_BYTES or count > cp.MAX_FILES: + raise Budget('Snapshot source exceeds budget') + record['files'].append(str(candidate)) + manifest['repositories'].append(record) + PHASE = 'scan' + cp.scan(neutral, history=True, revisions=('--all', 'HEAD', '--no-ext-diff', '--no-textconv')) + PHASE = 'archive' + encoded = json.dumps(manifest).encode() + total += len(encoded) + if total > cp.MAX_BYTES or len(encoded) > 8 * 1024 * 1024: + raise Budget('Snapshot manifest exceeds budget') + (stage / 'manifest.json').write_bytes(encoded) + # Scan the exact copied files and manifest, plus every archived object + # (including merge-only blobs and annotated-tag/commit messages). + PHASE = 'scan' + cp.scan(stage) + cp.scan(review) + PHASE = 'archive' + pax = [0] + def normalized(info): + info.uid = info.gid = info.mtime = 0 + info.uname = info.gname = '' + info.pax_headers = {} + header_bytes = len(info.tobuf(format=tarfile.PAX_FORMAT)) - 512 + pax[0] += header_bytes + if header_bytes > 16384 or pax[0] > MAX_PAX: + raise Budget('Snapshot path metadata exceeds budget') + return info + created = False + try: + with output.open('xb') as sink: + created = True + os.chmod(output, 0o600) + work.file(output) + with tarfile.open(fileobj=ArchiveSink(sink, work), mode='w:gz', dereference=False) as archive: + archive.add(stage, arcname='snapshot', filter=normalized) + if sink.tell() > MAX_BLOB: + raise Budget('Snapshot archive exceeds blob budget') + except BaseException: + if created: + output.unlink() + raise + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--workspace', required=True) + parser.add_argument('--output', required=True) + parser.add_argument('--work-bytes', type=int, default=512 * 1024 * 1024) + args = parser.parse_args() + try: + manifest = pack(args.workspace, args.output, args.work_bytes) + print(json.dumps({'version': 1, 'scanned': True, + 'repositories': len(manifest['repositories']), + 'bytes': Path(args.output).stat().st_size})) + return 0 + except Exception as error: + category = 'budget' if isinstance(error, Budget) or getattr(error, 'errno', None) in (errno.ENOSPC, errno.EDQUOT) else PHASE + print(json.dumps({'error': category}), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/voltage/images/buzz-agent/test-image.sh b/voltage/images/buzz-agent/test-image.sh index 595e3cedb4b..9f3233237c5 100755 --- a/voltage/images/buzz-agent/test-image.sh +++ b/voltage/images/buzz-agent/test-image.sh @@ -52,4 +52,5 @@ docker run --rm --tmpfs /home/agent:uid=10001,gid=10001,mode=0700 \ command -v codex-acp claude-agent-acp buzz-dev-mcp >/dev/null ' python3 "$(dirname "$0")/test-review.py" "$IMAGE" +python3 "$(dirname "$0")/test-snapshot.py" "$IMAGE" printf 'PASS: agent adapter packaging and unauthenticated ACP initialization (%s)\n' "$IMAGE" diff --git a/voltage/images/buzz-agent/test-snapshot.py b/voltage/images/buzz-agent/test-snapshot.py new file mode 100644 index 00000000000..d5ecfdf14f0 --- /dev/null +++ b/voltage/images/buzz-agent/test-snapshot.py @@ -0,0 +1,226 @@ +"""Exercise the packaged scanner and self-contained archives without network.""" +import subprocess +import sys +from pathlib import Path + +SCRIPT = r''' +import importlib.machinery, importlib.util, json, os, subprocess, tarfile, tempfile, time +from pathlib import Path + +def load(name, path): + loader=importlib.machinery.SourceFileLoader(name,path) + module=importlib.util.module_from_spec(importlib.util.spec_from_loader(name,loader)) + loader.exec_module(module) + return module +snapshot=load('snapshot','/usr/local/bin/buzz-agent-snapshot') +cp=snapshot.checkpoint() +policy=json.loads(Path('/tmp/policy.json').read_text()) +for value,expected in policy: + assert cp.excluded(Path(value))==expected,(value,expected) +assert all(any(value==name+'/nested' and expected for value,expected in policy) for name in cp.EXCLUDED) +assert all(any(value=='repo/'+name and expected for value,expected in policy) for name in cp.PRIVATE_NAMES) + +def git(repo,*args): + result=subprocess.run(['git','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','-C',str(repo),*args],capture_output=True) + assert result.returncode==0,'fixture Git command failed' + return result.stdout.decode().strip() + +def repository(root,format='sha1'): + repo=root/'workspace'/'example' + repo.mkdir(parents=True) + git(repo,'init','--object-format='+format) + git(repo,'config','user.name','Snapshot Fixture') + git(repo,'config','user.email','fixture@example.invalid') + return repo + +def expect_refusal(workspace,output,category=None,**options): + try: + snapshot.pack(workspace,output,**options) + raise AssertionError('unsafe snapshot was accepted') + except (ValueError,OSError,subprocess.SubprocessError): + assert not output.exists(),'failed archive remained on disk' + if category=='scan': assert snapshot.PHASE=='scan' + +with tempfile.TemporaryDirectory() as temp: + root=Path(temp) + for format in ('sha1','sha256'): + case=root/format + repo=repository(case,format) + (repo/'source.rs').write_text('initial\n') + (repo/'deleted.txt').write_text('tracked deletion\n') + (repo/'.env.example').write_text('PUBLIC_URL=https://example.invalid\n') + git(repo,'add','.') + git(repo,'commit','-m','initial') + git(repo,'branch','local-only') + (repo/'source.rs').write_text('edited\n') + (repo/'deleted.txt').unlink() + (repo/'new.txt').write_text('untracked\n') + (repo/'.env').write_text('PRIVATE-LOOSE-FILE') + (repo/'credentials.json').write_text('PRIVATE-LOOSE-FILE') + (repo/'tool.sh').write_text('echo fixture\n') + (repo/'tool.sh').chmod(0o755) + (repo/'link').symlink_to('source.rs') + # Valid own PAX output must remain bounded and round-trip. + long=repo/('long-'+'x'*110) + long.write_text('long path\n') + archive=case/'snapshot.tar.gz' + manifest=snapshot.pack(case/'workspace',archive) + assert manifest['snapshot'] is True and manifest['scanned'] is True + record=manifest['repositories'][0] + assert record['base']=='bundle' and record['shallow']==[] + assert 'deleted.txt' not in record['files'] + assert '.env.example' in record['excluded'] + assert 'source.rs' in record['files'] and 'new.txt' in record['files'] + extracted=case/'extracted' + with tarfile.open(archive) as package: + names=package.getnames() + assert not any(name.endswith(('/.env','/credentials.json','/.env.example')) for name in names) + assert package.getmember('snapshot/repositories/0/files/link').issym() + assert package.getmember('snapshot/repositories/0/files/tool.sh').mode&0o111 + assert all(member.mtime==0 for member in package) + package.extractall(extracted,filter='data') + history=case/'history' + snapshot.DEADLINE=time.monotonic()+60 + cp=snapshot.checkpoint() + cp.restore_history(extracted/'snapshot/repositories/0/history.bundle',record,history) + assert git(history,'show','HEAD:.env.example').startswith('PUBLIC_URL=') + assert 'refs/heads/local-only' in git(history,'show-ref') + assert (extracted/'snapshot/repositories/0/files'/long.name).read_text()=='long path\n' + print('PASS: full history, exclusions, dirty files, links, PAX and '+format) + + # Shallow bundle restores with its explicit local boundary, without origin. + source=root/'sha1/workspace/example' + (source/'source.rs').write_text('committed second\n') + git(source,'add','source.rs') + git(source,'commit','-m','second') + shallow=root/'shallow/workspace/example' + shallow.parent.mkdir(parents=True) + git(source,'clone','--depth=1',source.as_uri(),str(shallow)) + shallow_archive=root/'shallow/snapshot.tar.gz' + record=snapshot.pack(shallow.parent,shallow_archive)['repositories'][0] + assert record['shallow'] + staging=root/'shallow-extracted' + with tarfile.open(shallow_archive) as package: package.extractall(staging,filter='data') + snapshot.DEADLINE=time.monotonic()+60 + cp=snapshot.checkpoint() + bundle=staging/'snapshot/repositories/0/history.bundle' + cp.restore_history(bundle,record,root/'shallow-good') + broken=dict(record,shallow=[]) + try: + cp.restore_history(bundle,broken,root/'shallow-bad') + raise AssertionError('undeclared missing parents were accepted') + except ValueError: pass + print('PASS: explicit shallow boundaries; undeclared missing parents refused') + + case=root/'nested';outer=repository(case) + (outer/'outer.txt').write_text('outer\n');git(outer,'add','.');git(outer,'commit','-m','outer') + nested=outer/'nested';git(outer,'init',str(nested)) + git(nested,'config','user.name','Snapshot Fixture');git(nested,'config','user.email','fixture@example.invalid') + (nested/'inner.txt').write_text('inner\n');git(nested,'add','.');git(nested,'commit','-m','inner') + manifest=snapshot.pack(outer.parent,case/'nested.tar.gz') + assert [record['path'] for record in manifest['repositories']]==['example','example/nested'] + with tarfile.open(case/'nested.tar.gz') as package: + assert package.extractfile('snapshot/repositories/0/files/outer.txt').read()==b'outer\n' + assert package.extractfile('snapshot/repositories/1/files/inner.txt').read()==b'inner\n' + print('PASS: real outer plus untracked nested Git repository have separate complete bundles') + + # No current or historical credential may be hidden by repo/global ignores. + secret='sk-ant-oat01-'+'Z'*90 + for historical in (False,True): + case=root/('old-secret' if historical else 'new-secret') + repo=repository(case) + (repo/'safe').write_text('safe\n') + git(repo,'add','safe');git(repo,'commit','-m','initial') + (repo/'payload.txt').write_text(secret) + if historical: + git(repo,'add','payload.txt');git(repo,'commit','-m','old secret') + (repo/'payload.txt').unlink();git(repo,'commit','-am','remove secret') + (repo/'.gitleaksignore').write_text('*\n') + (repo/'.gitleaks.toml').write_text('[allowlist]\nregexes=[".*"]\n') + os.environ['GITLEAKS_CONFIG']=str(repo/'.gitleaks.toml') + expect_refusal(case/'workspace',case/'refused.tar.gz',category='scan') + print('PASS: staged working-tree and deleted historical secret refusal ignores bypass attempts') + + # A secret introduced only by a merge and an annotated tag still gets scanned. + case=root/'metadata-secret';repo=repository(case) + (repo/'safe').write_text('safe\n');git(repo,'add','safe');git(repo,'commit','-m','initial') + git(repo,'tag','-a','secret-tag','-m',secret) + expect_refusal(case/'workspace',case/'refused.tar.gz',category='scan') + print('PASS: annotated-tag secret is scanned as an archived object') + + case=root/'merge-secret';repo=repository(case) + (repo/'safe').write_text('safe\n');git(repo,'add','safe');git(repo,'commit','-m','initial') + base=git(repo,'branch','--show-current') + git(repo,'checkout','-b','side') + (repo/'side').write_text('side\n');git(repo,'add','side');git(repo,'commit','-m','side') + git(repo,'checkout',base) + (repo/'main').write_text('main\n');git(repo,'add','main');git(repo,'commit','-m','main') + git(repo,'merge','--no-commit','side') + (repo/'payload.txt').write_text(secret);git(repo,'add','payload.txt');git(repo,'commit','-m','merge') + (repo/'payload.txt').unlink();git(repo,'commit','-am','remove merge secret') + expect_refusal(case/'workspace',case/'refused.tar.gz',category='scan') + print('PASS: deleted merge-only secret is scanned from the exact bundled object') + + case=root/'bounds';repo=repository(case) + (repo/'safe').write_text('safe\n');git(repo,'add','safe');git(repo,'commit','-m','initial') + expect_refusal(case/'workspace',case/'small-budget.tar.gz',work_bytes=1024*1024) + (repo/'escape').symlink_to('/etc/passwd') + expect_refusal(case/'workspace',case/'external-link.tar.gz') + (repo/'escape').unlink() + (repo/'bad').write_bytes(b'x'*1024) + cp=snapshot.checkpoint();cp.MAX_BYTES=64 + original=snapshot.checkpoint;snapshot.checkpoint=lambda:cp + expect_refusal(case/'workspace',case/'scaled-source.tar.gz') + snapshot.checkpoint=original + output=case/'preserve-existing';output.write_text('keep me') + try: snapshot.pack(case/'workspace',output);raise AssertionError('existing output overwritten') + except ValueError: assert output.read_text()=='keep me' + unborn=repository(root/'unborn') + expect_refusal(unborn.parent,root/'unborn/refused.tar.gz') + print('PASS: capacity/source budgets, external links, existing files and unborn HEAD refused') + + # Bound both generic subprocess output and compressed archive writes BEFORE writes. + snapshot.DEADLINE=time.monotonic()+60 + limited=root/'limited-output' + try: + snapshot.run('/usr/bin/python3','-c','import sys;sys.stdout.buffer.write(b"x"*8192)',output=limited,limit=64) + raise AssertionError('subprocess output bound was bypassed') + except snapshot.Budget: assert limited.stat().st_size<=64 + import io + sink=io.BytesIO();work=snapshot.Work(root,512*1024*1024) + previous=snapshot.MAX_BLOB;snapshot.MAX_BLOB=8 + try: + snapshot.ArchiveSink(sink,work).write(b'x'*9) + raise AssertionError('compressed output was written before admission') + except snapshot.Budget: assert sink.getvalue()==b'' + finally: snapshot.MAX_BLOB=previous + # Intermediate source symlinks cannot redirect a copy outside the open repo. + cp=snapshot.checkpoint();snapshot.DEADLINE=time.monotonic()+60 + (repo/'sub').mkdir();outside=root/'outside';outside.mkdir() + (outside/'value').write_text('OUTSIDE-WORKSPACE') + (repo/'sub').rmdir();(repo/'sub').symlink_to(outside) + with snapshot.opened_repo(repo.parent,Path('example')) as (opened,descriptor): + try: + snapshot.copy_file(opened,descriptor,snapshot.path('sub/value'),root/'copy-refused',cp,100) + raise AssertionError('intermediate symlink redirected a source read') + except OSError: assert not (root/'copy-refused').exists() + small=root/'discovery-bound';small.mkdir() + for number in range(3): (small/str(number)).write_text('x') + previous=cp.MAX_FILES;cp.MAX_FILES=1 + try: + snapshot.discover(small,cp) + raise AssertionError('streamed discovery budget was bypassed') + except snapshot.Budget: pass + finally: cp.MAX_FILES=previous + print('PASS: streaming output admission and descriptor-relative source containment') +''' + +policy = Path(__file__).with_name('snapshot-policy.json').resolve() +result = subprocess.run(['docker','run','--rm','-i','--network','none','--read-only', + '--tmpfs','/tmp:rw,nosuid,nodev,size=1g,uid=10001,gid=10001,mode=0700', + '--memory','1g','--cap-drop','ALL','--security-opt','no-new-privileges', + '-v',f'{policy}:/tmp/policy.json:ro','--entrypoint','python3',sys.argv[1],'-'], + input=SCRIPT,text=True,capture_output=True,timeout=240) +print(result.stdout,end='') +print(result.stderr,end='',file=sys.stderr) +sys.exit(result.returncode) From 6e7ceba132bbc99952aa41a9c322f10c56b94250 Mon Sep 17 00:00:00 2001 From: Tee8z Date: Tue, 10 Nov 2026 17:01:42 -0500 Subject: [PATCH 2/4] fix(snapshots): admit exact captured bundle costs before import Signed-off-by: Tee8z --- crates/buzz-cli/src/commands/sandbox.rs | 18 ++++ crates/buzz-cli/src/lib.rs | 6 ++ crates/buzz-cli/src/sandbox_archive.rs | 99 ++++++++++++++++++++++ docs/sandbox-snapshot-producer.md | 3 + voltage/images/buzz-agent/snapshot.py | 48 +++++++++-- voltage/images/buzz-agent/test-snapshot.py | 48 +++++++++++ 6 files changed, 215 insertions(+), 7 deletions(-) create mode 100644 crates/buzz-cli/src/sandbox_archive.rs diff --git a/crates/buzz-cli/src/commands/sandbox.rs b/crates/buzz-cli/src/commands/sandbox.rs index 5dbc233c503..f8314586833 100644 --- a/crates/buzz-cli/src/commands/sandbox.rs +++ b/crates/buzz-cli/src/commands/sandbox.rs @@ -21,6 +21,12 @@ mod tests; /// Owner approvals and interactive sandbox terminals. #[derive(Subcommand)] pub enum SandboxCmd { + /// Validate a local captured bundle before any Git import (internal worker seam) + #[command(hide = true)] + InspectBundle { + /// Captured bundle file + file: std::path::PathBuf, + }, /// Attach a local TTY; omitted targets are selected from BUZZ_PASEO_AGENTS Attach { /// Channel UUID containing the sandbox thread @@ -56,6 +62,17 @@ pub enum SandboxCmd { }, } +/// Print bounded local preflight metadata, without reading credentials or connecting. +pub(crate) fn inspect_bundle_cmd(file: &std::path::Path) -> Result<(), CliError> { + let summary = crate::sandbox_archive::inspect_bundle_file(file).map_err(failure)?; + let encoded = serde_json::to_string(&summary).map_err(failure)?; + if encoded.len() > 16 * 1024 * 1024 { + return Err(failure("bundle metadata exceeded budget")); + } + println!("{encoded}"); + Ok(()) +} + fn failure(message: impl ToString) -> CliError { CliError::Other(message.to_string()) } @@ -168,6 +185,7 @@ pub async fn dispatch(command: SandboxCmd, client: &BuzzClient) -> Result<(), Cl )); } match command { + SandboxCmd::InspectBundle { file } => inspect_bundle_cmd(&file), SandboxCmd::Authorize { request, minutes } => { let event = one(client, json!({"ids":[request],"kinds":[30624],"limit":1})).await?; if event.id != request { diff --git a/crates/buzz-cli/src/lib.rs b/crates/buzz-cli/src/lib.rs index 0aad53dccd8..9c92f57c8e9 100644 --- a/crates/buzz-cli/src/lib.rs +++ b/crates/buzz-cli/src/lib.rs @@ -4,6 +4,8 @@ mod commands; mod error; mod help_tree; mod links; +/// Safe local snapshot archive inspection and restoration. +pub mod sandbox_archive; mod validate; use clap::{CommandFactory, FromArgMatches, Parser, Subcommand, ValueEnum}; @@ -2303,6 +2305,10 @@ async fn run(cli: Cli) -> Result<(), CliError> { }; } + if let Cmd::Sandbox(commands::sandbox::SandboxCmd::InspectBundle { ref file }) = cli.command { + return commands::sandbox::inspect_bundle_cmd(file); + } + // Auth: private key is required for all relay operations. // The keypair IS the identity — no tokens, no other auth. let private_key_str = cli.private_key.ok_or_else(|| { diff --git a/crates/buzz-cli/src/sandbox_archive.rs b/crates/buzz-cli/src/sandbox_archive.rs new file mode 100644 index 00000000000..d21e45b77eb --- /dev/null +++ b/crates/buzz-cli/src/sandbox_archive.rs @@ -0,0 +1,99 @@ +//! Local-only bounded archive preflight. No identity, relay or Git process is used. +use std::fs::OpenOptions; +use std::io::Read; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; + +use buzz_sdk::sandbox_attach::snapshot::{ + archive::inspect_bundle, MAX_BLOB_BYTES, MAX_SOURCE_BYTES, +}; +use serde::{Deserialize, Serialize}; + +/// Local archive refusal; file contents and scanner diagnostics are never included. +#[derive(Debug, thiserror::Error)] +pub enum ArchiveError { + /// Filesystem access failed. + #[error("archive file access failed: {0}")] + Io(#[from] std::io::Error), + /// A hard file/type or metadata bound was exceeded. + #[error("{0}")] + Invalid(&'static str), + /// The SDK rejected malformed or oversized bundle contents. + #[error("{0}")] + Bundle(#[from] buzz_sdk::SdkError), +} + +/// Bounded admission metadata for the exact captured bundle. +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct BundleSummary { + /// Local helper format version. + pub version: u8, + /// Number of packed objects, checked before import. + pub objects: u32, + /// Sum of reconstructed object bytes, including delta targets. + pub expanded_bytes: u64, + /// Object-ID length in bytes (20 or 32). + pub hash_bytes: usize, + /// Object ID/ref name pairs; callers account for ref/index/block costs. + pub refs: Vec<(String, String)>, +} + +/// Inspect a regular file without allocating from unchecked file metadata. +/// The bounded descriptor read also catches growth after the initial metadata check. +pub fn inspect_bundle_file(path: &Path) -> Result { + if !std::fs::symlink_metadata(path)?.is_file() { + return Err(ArchiveError::Invalid("bundle must be a regular file")); + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + options.custom_flags(nix::libc::O_NONBLOCK | nix::libc::O_NOFOLLOW); + let file = options.open(path)?; + let metadata = file.metadata()?; + if !metadata.is_file() || metadata.len() > MAX_BLOB_BYTES as u64 { + return Err(ArchiveError::Invalid( + "bundle exceeds regular-file byte budget", + )); + } + let mut bytes = Vec::new(); + file.take(MAX_BLOB_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > MAX_BLOB_BYTES { + return Err(ArchiveError::Invalid("bundle grew beyond byte budget")); + } + let inventory = inspect_bundle(&bytes, MAX_SOURCE_BYTES)?; + Ok(BundleSummary { + version: 1, + objects: inventory.objects, + expanded_bytes: inventory.expanded_bytes, + hash_bytes: inventory.hash_bytes, + refs: inventory.refs, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn inspector_refuses_oversized_files_before_reading_and_nonregular_inputs() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("sparse.bundle"); + std::fs::File::create(&path) + .unwrap() + .set_len(MAX_BLOB_BYTES as u64 + 1) + .unwrap(); + assert!(matches!( + inspect_bundle_file(&path), + Err(ArchiveError::Invalid(_)) + )); + assert!(inspect_bundle_file(temp.path()).is_err()); + #[cfg(unix)] + { + let link = temp.path().join("link"); + std::os::unix::fs::symlink(&path, &link).unwrap(); + assert!(inspect_bundle_file(&link).is_err()); + } + } +} diff --git a/docs/sandbox-snapshot-producer.md b/docs/sandbox-snapshot-producer.md index 6bc79a90fe9..0581ba948a8 100644 --- a/docs/sandbox-snapshot-producer.md +++ b/docs/sandbox-snapshot-producer.md @@ -36,6 +36,9 @@ Shallow clones retain their explicit local boundaries, without unshallow or netw Unborn HEADs are refused. Before packing, the helper inventories source objects and byte/count budgets. +After capture, the required local `buzz sandbox inspect-bundle` helper validates the actual bundle with the SDK before any Git import, independently of credentials or relay configuration. +The producer reserves the captured expansion, object/ref/index/block costs and aggregate counts, then verifies the neutral inventory matches before writing raw review bodies. +A source commit between preflight and capture therefore cannot inherit stale scratch admission; missing or malformed inspector output fails closed. It imports the bundle into a neutral Git repository, verifies its object graph, and scans that exact history with immutable gitleaks rules. It also scans the copied working files, manifest, ref names, and every archived raw object, including merge-only blobs and annotated-tag messages. Global/system configuration, replacement refs, grafts, lazy fetch, hooks, fsmonitor, repository ignore files, and `GITLEAKS_*` overrides cannot weaken the scan. diff --git a/voltage/images/buzz-agent/snapshot.py b/voltage/images/buzz-agent/snapshot.py index 9c50c81ecd7..855ba3b2ba9 100644 --- a/voltage/images/buzz-agent/snapshot.py +++ b/voltage/images/buzz-agent/snapshot.py @@ -49,7 +49,7 @@ def run(*args, cwd=None, timeout=60, env=None, input=None, output=None, if args and args[0] == 'git': args = (args[0], '-c', 'protocol.allow=never', '-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', '-c', 'pack.threads=1', - '-c', 'pack.windowMemory=16m', *args[1:]) + '-c', 'pack.windowMemory=16m', '-c', 'core.logAllRefUpdates=false', *args[1:]) expires = time.monotonic() + remaining(timeout) with ExitStack() as stack: sink = stack.enter_context(open(output, 'xb')) if output is not None else None @@ -141,6 +141,30 @@ def inventory(repo, cp): return lines +def inspect_captured(bundle, cp): + # No fallback: source preflight cannot authorize costs of a different bundle + # captured after the source commits. The native SDK validates the exact pack. + result = json.loads(run('/usr/local/bin/buzz', 'sandbox', 'inspect-bundle', + str(bundle), limit=16 * 1024 * 1024)) + if (not isinstance(result, dict) + or set(result) != {'version', 'objects', 'expanded_bytes', 'hash_bytes', 'refs'} + or type(result['version']) is not int or result['version'] != 1 + or type(result['objects']) is not int or not 0 < result['objects'] <= MAX_OBJECTS + or type(result['expanded_bytes']) is not int or not 0 <= result['expanded_bytes'] <= cp.MAX_BYTES + or type(result['hash_bytes']) is not int or result['hash_bytes'] not in (20, 32) + or not isinstance(result['refs'], list) or not 0 < len(result['refs']) <= cp.MAX_FILES): + raise ValueError('Snapshot captured-bundle metadata is invalid') + parents = set() + for ref in result['refs']: + if (not isinstance(ref, list) or len(ref) != 2 or not all(isinstance(v, str) for v in ref) + or len(ref[0]) != result['hash_bytes'] * 2 + or any(c not in '0123456789abcdef' for c in ref[0]) + or not (ref[1] == 'HEAD' or ref[1].startswith('refs/'))): + raise ValueError('Snapshot captured reference is invalid') + parents.update(path(ref[1]).parents) + return result, len(parents) + + class ObjectStream: """Split one bounded cat-file batch without materializing object bodies.""" def __init__(self, lines, directory): @@ -362,7 +386,8 @@ def pack(workspace, output, work_bytes=512 * 1024 * 1024): # Inventory before packing is conservative: even unreachable source # objects count, so no large source object is decoded before a bound. source_objects = inventory(repo, cp) - expanded += sum(int(line.rsplit(' ', 1)[1]) for line in source_objects) + source_expanded = sum(int(line.rsplit(' ', 1)[1]) for line in source_objects) + expanded += source_expanded objects_count += len(source_objects) if expanded > cp.MAX_BYTES or objects_count > MAX_OBJECTS: raise Budget('Snapshot aggregate Git expansion exceeds budget') @@ -373,8 +398,8 @@ def pack(workspace, output, work_bytes=512 * 1024 * 1024): target = stage / 'repositories' / str(index) (target / 'files').mkdir(parents=True) bundle = target / 'history.bundle' - work.reserve(sum(int(line.rsplit(' ', 1)[1]) for line in source_objects) - + len(source_objects) * (work.block + 64)) + source_cost = source_expanded + len(source_objects) * (work.block + 64) + work.reserve(source_cost) allowance = min(MAX_BLOB, cp.MAX_BYTES - total, (work.limit - work.used) // 2) if allowance <= 0: raise Budget('Snapshot bundle capacity exceeds budget') @@ -384,9 +409,21 @@ def pack(workspace, output, work_bytes=512 * 1024 * 1024): work.reserve(2 * bundle_size) if total > cp.MAX_BYTES or bundle_size > MAX_BLOB: raise Budget('Snapshot source exceeds budget') + packed, ref_directories = inspect_captured(bundle, cp) + expanded += max(0, packed['expanded_bytes'] - source_expanded) + objects_count += max(0, packed['objects'] - len(source_objects)) + refs_count += len(packed['refs']) + if expanded > cp.MAX_BYTES or objects_count > MAX_OBJECTS or refs_count > cp.MAX_FILES: + raise Budget('Snapshot captured aggregate inventory exceeds budget') + actual_cost = (packed['expanded_bytes'] + packed['objects'] * (work.block + 64) + + (len(packed['refs']) + ref_directories) * work.block) + work.reserve(max(0, actual_cost - source_cost)) neutral = Path(temp) / f'history-{index}' cp.restore_history(bundle, record, neutral) objects = inventory(neutral, cp) + actual_expanded = sum(int(line.rsplit(' ', 1)[1]) for line in objects) + if actual_expanded != packed['expanded_bytes'] or len(objects) != packed['objects']: + raise ValueError('Snapshot neutral inventory differs from the captured bundle') object_review = review / str(index) object_review.mkdir() writer = ObjectStream(objects, object_review) @@ -405,9 +442,6 @@ def pack(workspace, output, work_bytes=512 * 1024 * 1024): stop = value.find(b'\n\n') if stop < 0: raise Budget('Snapshot bundle header exceeds budget') - refs_count += sum(1 for line in value[:stop].splitlines() if not line.startswith((b'#', b'@', b'-'))) - if refs_count > cp.MAX_FILES: - raise Budget('Snapshot reference count exceeds budget') (object_review / 'references').write_bytes(value[:stop]) names = cp.git(repo, 'ls-files', '-z', '--cached', '--others', '--exclude-standard') for raw in sorted(set(names.split(b'\0')) - {b''}): diff --git a/voltage/images/buzz-agent/test-snapshot.py b/voltage/images/buzz-agent/test-snapshot.py index d5ecfdf14f0..6417333d832 100644 --- a/voltage/images/buzz-agent/test-snapshot.py +++ b/voltage/images/buzz-agent/test-snapshot.py @@ -124,6 +124,54 @@ def expect_refusal(workspace,output,category=None,**options): assert package.extractfile('snapshot/repositories/1/files/inner.txt').read()==b'inner\n' print('PASS: real outer plus untracked nested Git repository have separate complete bundles') + # Real commits between source preflight and capture must be charged BEFORE import. + for succeeds,budget in ((False,20*1024*1024),(True,128*1024*1024)): + case=root/('racing-control' if succeeds else 'racing-refusal');repo=repository(case) + (repo/'safe').write_text('safe\n');git(repo,'add','.');git(repo,'commit','-m','initial') + cp=snapshot.checkpoint();original_git=cp.git;original_restore=cp.restore_history + seen={'changed':False,'imported':False} + def racing(directory,*args,**options): + if not seen['changed'] and args==('rev-parse','--verify','HEAD'): + seen['changed']=True + with (repo/'late-history').open('wb') as stream: stream.truncate(8*1024*1024) + git(repo,'add','late-history');git(repo,'commit','-m','concurrent source') + (repo/'late-history').unlink() + return original_git(directory,*args,**options) + def restoring(*args,**options): + seen['imported']=True + return original_restore(*args,**options) + cp.git=racing;cp.restore_history=restoring + factory=snapshot.checkpoint;snapshot.checkpoint=lambda:cp + output=case/'snapshot.tar.gz' + try: + if succeeds: + snapshot.pack(repo.parent,output,work_bytes=budget) + assert output.exists() and seen['imported'] + else: + try: + snapshot.pack(repo.parent,output,work_bytes=budget) + raise AssertionError('captured history escaped work accounting') + except snapshot.Budget: + assert not seen['imported'] and not output.exists() + assert seen['changed'] + finally: snapshot.checkpoint=factory + print('PASS: concurrent source growth refused before import at20MiB and accepted at128MiB control') + + # The packaged native inspector is required; invalid metadata cannot fall back. + case=root/'inspector-required';repo=repository(case) + (repo/'safe').write_text('safe\n');git(repo,'add','.');git(repo,'commit','-m','initial') + original_run=snapshot.run + for missing in (False,True): + def invalid_inspector(*args,**options): + if args[0]=='/usr/local/bin/buzz': + if missing: raise FileNotFoundError('required inspector is unavailable') + return b'{}' + return original_run(*args,**options) + snapshot.run=invalid_inspector + try: expect_refusal(repo.parent,case/'refused.tar.gz') + finally: snapshot.run=original_run + print('PASS: missing or invalid native inspector has no source-inventory fallback') + # No current or historical credential may be hidden by repo/global ignores. secret='sk-ant-oat01-'+'Z'*90 for historical in (False,True): From 1a1d563de937b226824d5afe2ba98f1cead7e95d Mon Sep 17 00:00:00 2001 From: Tee8z Date: Tue, 10 Nov 2026 17:49:20 -0500 Subject: [PATCH 3/4] fix(images): include snapshot helper in agent build context Signed-off-by: Tee8z --- voltage/images/buzz-agent/Dockerfile.dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/voltage/images/buzz-agent/Dockerfile.dockerignore b/voltage/images/buzz-agent/Dockerfile.dockerignore index a021be5b0b5..210d6c43de9 100644 --- a/voltage/images/buzz-agent/Dockerfile.dockerignore +++ b/voltage/images/buzz-agent/Dockerfile.dockerignore @@ -89,6 +89,7 @@ voltage/ !voltage/images/buzz-agent/profile.py !voltage/images/buzz-agent/cache.py !voltage/images/buzz-agent/checkpoint.py +!voltage/images/buzz-agent/snapshot.py !voltage/images/buzz-agent/review.py !voltage/images/buzz-agent/checkpoint-gitleaks.toml !voltage/images/buzz-agent/gh.sh From e094c955edfb59744d81db382b0df20e271a13a7 Mon Sep 17 00:00:00 2001 From: Tee8z Date: Tue, 10 Nov 2026 18:32:02 -0500 Subject: [PATCH 4/4] fix(agents): preserve Git executable modes in snapshots Signed-off-by: Tee8z --- voltage/images/buzz-agent/snapshot.py | 2 +- voltage/images/buzz-agent/test-snapshot.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/voltage/images/buzz-agent/snapshot.py b/voltage/images/buzz-agent/snapshot.py index 855ba3b2ba9..81b7124a4ab 100644 --- a/voltage/images/buzz-agent/snapshot.py +++ b/voltage/images/buzz-agent/snapshot.py @@ -327,7 +327,7 @@ def copy_file(repo, descriptor, relative, destination, cp, limit): if total > limit: raise Budget('Snapshot source exceeds budget') sink.write(chunk) - destination.chmod(0o755 if actual.st_mode & 0o111 else 0o644) + destination.chmod(0o755 if actual.st_mode & 0o100 else 0o644) return total finally: os.close(parent) diff --git a/voltage/images/buzz-agent/test-snapshot.py b/voltage/images/buzz-agent/test-snapshot.py index 6417333d832..a6c2ae30df2 100644 --- a/voltage/images/buzz-agent/test-snapshot.py +++ b/voltage/images/buzz-agent/test-snapshot.py @@ -48,10 +48,13 @@ def expect_refusal(workspace,output,category=None,**options): repo=repository(case,format) (repo/'source.rs').write_text('initial\n') (repo/'deleted.txt').write_text('tracked deletion\n') + (repo/'nonowner-exec.txt').write_text('clean non-executable\n') (repo/'.env.example').write_text('PUBLIC_URL=https://example.invalid\n') git(repo,'add','.') git(repo,'commit','-m','initial') git(repo,'branch','local-only') + (repo/'nonowner-exec.txt').chmod(0o654) + assert git(repo,'diff','--name-only','--','nonowner-exec.txt')=='' (repo/'source.rs').write_text('edited\n') (repo/'deleted.txt').unlink() (repo/'new.txt').write_text('untracked\n') @@ -76,7 +79,8 @@ def expect_refusal(workspace,output,category=None,**options): names=package.getnames() assert not any(name.endswith(('/.env','/credentials.json','/.env.example')) for name in names) assert package.getmember('snapshot/repositories/0/files/link').issym() - assert package.getmember('snapshot/repositories/0/files/tool.sh').mode&0o111 + assert package.getmember('snapshot/repositories/0/files/tool.sh').mode&0o100 + assert package.getmember('snapshot/repositories/0/files/nonowner-exec.txt').mode&0o111==0,'non-owner execute bits invented an executable Git file' assert all(member.mtime==0 for member in package) package.extractall(extracted,filter='data') history=case/'history' @@ -86,7 +90,7 @@ def expect_refusal(workspace,output,category=None,**options): assert git(history,'show','HEAD:.env.example').startswith('PUBLIC_URL=') assert 'refs/heads/local-only' in git(history,'show-ref') assert (extracted/'snapshot/repositories/0/files'/long.name).read_text()=='long path\n' - print('PASS: full history, exclusions, dirty files, links, PAX and '+format) + print('PASS: full history, exclusions, dirty files, Git owner-execute semantics, links, PAX and '+format) # Shallow bundle restores with its explicit local boundary, without origin. source=root/'sha1/workspace/example'