From c96fc72d777cc138c6acc17abfe75696e848565a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 22:34:32 -0800 Subject: [PATCH 1/4] feat(local): a supervisor can terminate the sandboxes of a run it killed A run killed outright tears nothing down. A local sandbox's work folder is made before anything it runs, under the root AGENT_ENV_LOCAL_SANDBOX_DIR names, and its teardown rebuilt from that folder reaches its containers, networks, compose stack and processes. So a supervisor that gives a run a root of its own can terminate each sandbox found there afterwards: local_sandbox_ids() lists them. Co-Authored-By: Claude Opus 5.5 --- .env.example | 5 +++ src/agent_env/bundle/run.py | 3 +- .../local_docker/provider.py | 12 +++++++ tst/integration/cli/run_bundle_test.py | 34 +++++++++++++++++++ .../sandbox_providers/local_sandbox_test.py | 4 ++- 5 files changed, 56 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 64cf5c726..676974f84 100644 --- a/.env.example +++ b/.env.example @@ -26,6 +26,11 @@ AGENT_ENV_ENVIRONMENT=dev # - a hosted registry (ECR): configure a [stores.image] table in config.toml (no built-in coordinates) # AGENT_ENV_IMAGE_STORE=local +# Where local sandboxes keep their work folders (default ~/.agent-env-sandboxes; on macOS, a path Docker Desktop +# shares). A sandbox's folder is made before anything it runs, so a supervisor that gives a run a folder of its own +# can clean up after killing it: terminate each id local_sandbox_ids() lists, then remove its folder. +# AGENT_ENV_LOCAL_SANDBOX_DIR=~/.agent-env-sandboxes + # Secret store backend (what secret: references in config resolve through): # - "local" (or unset): process env vars, or a flat YAML/JSON file via [stores.secret] in config.toml # - a hosted secret manager (AWS Secrets Manager, Google Cloud Secret Manager): configure a [stores.secret] table (no built-in coordinates) diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 0ad85b9f4..7efbd9008 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -240,7 +240,8 @@ def run_bundle( Each run's sandboxes are torn down as it ends, so the contexts returned record sandboxes that are gone. ``keep`` leaves them up instead, for the caller to use and then remove with ``BundleRun.teardown()``. Ctrl-C or SIGTERM cancels the runs and tears down every one, kept or not, then raises ``RunInterrupted`` - with the result. + with the result. A process killed outright tears down nothing: a supervisor that may have to kill it gives it an + ``AGENT_ENV_LOCAL_SANDBOX_DIR`` of its own, then terminates what ``local_sandbox_ids`` finds there. It runs its own event loop, and turns namespace routing on for the whole process while it runs, so it suits the CLI and scripts, not a service. From async code, call it in a thread; there it can't take over diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index ab4efc79e..81541e4de 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -627,6 +627,18 @@ def _copy_into_container(source: Path, container: str, destination: str) -> None raise RuntimeError(f"Could not copy {source} into container {container}: {copied.stderr.strip()}") +def local_sandbox_ids() -> list[str]: + """The ids of the local sandboxes with a work folder under the sandbox root. A folder is made before anything its + sandbox runs, so a supervisor that gave a run its own ``AGENT_ENV_LOCAL_SANDBOX_DIR`` cleans up after killing it by + terminating each id here (``LocalSandboxProvider().terminate_sandbox``) and removing its folder.""" + root = _local_sandbox_root_path() + if not root.is_dir(): + return [] + prefix = "agent-env-" + return sorted({entry.name.removeprefix(prefix).rsplit("-", 1)[0] for entry in root.iterdir() + if entry.name.startswith(prefix) and entry.is_dir() and not entry.is_symlink()}) + + def remove_local_work_dir(sandbox_id: str) -> Path | None: """Delete a local sandbox's work folder, and return it, when it lies directly under the sandbox root and isn't a symlink; anything else, a legacy folder under the temp dir included, is left alone. Raises OSError diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index 5d0cf87de..b4fb9dc72 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -1,6 +1,7 @@ """``agent-env run`` on the hello bundle agent-env ships, with no infra: the local stores, a local VM-mode sandbox, and a verifier scoring the files the task loaded into it. Needs `bash` and the usual shell tools on PATH.""" +import asyncio import importlib import json import logging @@ -26,6 +27,11 @@ from agent_env.cli import cli from agent_env.config import configure, reset_config from agent_env.config.paths import state_root +from agent_env.providers.sandbox_providers.local_docker.provider import ( + LocalSandboxProvider, + local_sandbox_ids, + remove_local_work_dir, +) from agent_env.providers.sandbox_providers.sandbox_provider import reset_sandbox_provider from agent_env.store import Filter, LocalSqliteDocumentStore from agent_env.store.routing import namespace_routing @@ -244,6 +250,34 @@ def test_a_signal_mid_run_marks_the_run_cancelled_tears_it_down_and_stops_its_co assert not _alive(int((tmp_path / "nap.pid").read_text())) +def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervisor_to_terminate(bundle, tmp_path): + napping = tmp_path / "napping" + (napping / "tasks").mkdir(parents=True) + (napping / "tasks/nap.json").write_text(json.dumps([ + {"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm", "sandbox_type": "local"}, + {"id": "web", "type": "deploy_sandbox", "sandbox_name": "web", "sandbox_mode": "container", + "sandbox_type": "local", "image": "public.ecr.aws/nginx/nginx:stable-alpine", "port": 80}, + {"id": "nap", "type": "verify_sandbox", "sandbox_name": "box", "base_dir": "/app", "verifier_id": "nap", + "depends_on": ["box", "web"], "criteria": [{"type": "bash_cmd_succeeds", "criterion": "naps", + "bash_cmd": 'sleep 300 & echo $! > "$HOME/nap.pid"; wait'}]}, + ])) + + exit_code, output, held = _signalled([napping], "step 3/3 nap (verify_sandbox)", signal.SIGKILL, group=False) + found = local_sandbox_ids() + for sandbox_id in found: + asyncio.run(LocalSandboxProvider().terminate_sandbox(sandbox_id)) + remove_local_work_dir(sandbox_id) + + assert exit_code == -signal.SIGKILL, output + assert len(held) == len(found) == 2 + assert local_sandbox_ids() == [] + assert not _alive(int((tmp_path / "nap.pid").read_text())) + for sandbox_id in found: + listed = subprocess.run(["docker", "ps", "-aq", "--filter", f"label=agentenv.sandbox={sandbox_id}"], + capture_output=True, text=True, check=True) + assert listed.stdout == "" + + def _alive(pid): for _ in range(100): try: diff --git a/tst/unit/providers/sandbox_providers/local_sandbox_test.py b/tst/unit/providers/sandbox_providers/local_sandbox_test.py index 45d94b92b..a1ee03123 100644 --- a/tst/unit/providers/sandbox_providers/local_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/local_sandbox_test.py @@ -16,7 +16,7 @@ import agent_env.providers.sandbox_providers.local_docker.provider as ls from agent_env.a2a_agent.a2a_agent import A2AAgent from agent_env.config import reset_config, set_object_store -from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox, LocalSandboxProvider +from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox, LocalSandboxProvider, local_sandbox_ids from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_CONTAINER, SANDBOX_MODE_VM from agent_env.store import LocalFilesystemObjectStore @@ -116,6 +116,8 @@ def test_find_work_dir_only_returns_matching_sandbox_id(tmp_path: Path, monkeypa assert LocalSandbox.find_work_dir("local-requested") == requested assert LocalSandbox.find_work_dir("local-missing") is None + (tmp_path / "agent-env-notes.txt").write_text("") + assert local_sandbox_ids() == ["local-other", "local-requested"] def test_find_work_dir_falls_back_to_legacy_tempdir(tmp_path: Path, monkeypatch): From aeb66d538417c8e611510e33aa49e2062e066a90 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 22:42:02 -0800 Subject: [PATCH 2/4] test(run): kill the run once its nap has started Co-Authored-By: Claude Opus 5.5 --- tst/integration/cli/run_bundle_test.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index b4fb9dc72..b41692d14 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -204,10 +204,10 @@ def test_an_installed_bundle_runs_by_name_with_ids_rooted_at_its_package(bundle, assert "demo_installed_bundles" not in sys.modules -def _signalled(args, when, signum=signal.SIGINT, group=True): +def _signalled(args, when, signum=signal.SIGINT, group=True, ready=lambda: True): """Run ``agent-env run ARGS`` in its own process group, as a terminal would, and send ``signum`` to the group - (or to the process alone) once a line containing ``when`` is printed. Returns the exit code, the output, and - what the sandbox root held when the signal went.""" + (or to the process alone) once a line containing ``when`` is printed and ``ready()`` holds. Returns the exit code, + the output, and what the sandbox root held when the signal went.""" process = subprocess.Popen([sys.executable, "-m", "agent_env.cli", "run", *map(str, args)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, start_new_session=True) watchdog = threading.Timer(120, process.kill) @@ -217,6 +217,9 @@ def _signalled(args, when, signum=signal.SIGINT, group=True): for line in process.stdout: lines.append(line) if held is None and when in line: + deadline = time.monotonic() + 30 + while not ready() and time.monotonic() < deadline: + time.sleep(0.05) root = os.environ["AGENT_ENV_LOCAL_SANDBOX_DIR"] held = sorted(os.listdir(root)) if os.path.isdir(root) else [] (os.killpg if group else os.kill)(process.pid, signum) @@ -262,7 +265,9 @@ def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervi "bash_cmd": 'sleep 300 & echo $! > "$HOME/nap.pid"; wait'}]}, ])) - exit_code, output, held = _signalled([napping], "step 3/3 nap (verify_sandbox)", signal.SIGKILL, group=False) + nap = tmp_path / "nap.pid" + exit_code, output, held = _signalled([napping], "step 3/3 nap (verify_sandbox)", signal.SIGKILL, group=False, + ready=nap.exists) found = local_sandbox_ids() for sandbox_id in found: asyncio.run(LocalSandboxProvider().terminate_sandbox(sandbox_id)) @@ -271,7 +276,7 @@ def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervi assert exit_code == -signal.SIGKILL, output assert len(held) == len(found) == 2 assert local_sandbox_ids() == [] - assert not _alive(int((tmp_path / "nap.pid").read_text())) + assert not _alive(int(nap.read_text())) for sandbox_id in found: listed = subprocess.run(["docker", "ps", "-aq", "--filter", f"label=agentenv.sandbox={sandbox_id}"], capture_output=True, text=True, check=True) From fcbc8226d4ee4bdb07472b84e6f13bde6e654d66 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 8 Nov 2026 22:53:59 -0800 Subject: [PATCH 3/4] fix(local): a cut-short port map or a stray folder doesn't stop a supervisor's cleanup get_sandbox reads a port map a killed run left empty as none, and local_sandbox_ids() lists only folders named as sandboxes' are. The test also checks a container-mode sandbox's container, which is named, not labeled, is gone. Co-Authored-By: Claude Opus 5.5 --- .../sandbox_providers/local_docker/provider.py | 15 ++++++++++----- tst/integration/cli/run_bundle_test.py | 7 ++++--- .../sandbox_providers/local_sandbox_test.py | 13 +++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index 81541e4de..fe1eed8e8 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -634,9 +634,12 @@ def local_sandbox_ids() -> list[str]: root = _local_sandbox_root_path() if not root.is_dir(): return [] - prefix = "agent-env-" - return sorted({entry.name.removeprefix(prefix).rsplit("-", 1)[0] for entry in root.iterdir() - if entry.name.startswith(prefix) and entry.is_dir() and not entry.is_symlink()}) + prefix, found = "agent-env-", set() + for entry in root.iterdir(): + sandbox_id, _, suffix = entry.name.removeprefix(prefix).rpartition("-") + if entry.name.startswith(prefix) and sandbox_id and suffix and entry.is_dir() and not entry.is_symlink(): + found.add(sandbox_id) + return sorted(found) def remove_local_work_dir(sandbox_id: str) -> Path | None: @@ -911,8 +914,10 @@ async def get_sandbox(self, sandbox_id: str) -> LocalSandbox: work_dir = LocalSandbox.find_work_dir(sandbox_id) if work_dir is None: raise RuntimeError(f"Local sandbox work directory not found for sandbox_id={sandbox_id!r}") - port_map = work_dir / _PORT_MAP_FILE - ports = {int(c): h for c, h in json.loads(port_map.read_text()).items()} if port_map.exists() else None + try: + ports = {int(c): h for c, h in json.loads((work_dir / _PORT_MAP_FILE).read_text()).items()} + except (OSError, ValueError): # none written, or cut short by a run killed while writing it + ports = None # VM mode even for a container it owns: exec runs on this host, so steps must `docker exec` into it. return LocalSandbox(sandbox_id=sandbox_id, work_dir=work_dir, port_map=ports) diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index b41692d14..4df05a23d 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -278,9 +278,10 @@ def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervi assert local_sandbox_ids() == [] assert not _alive(int(nap.read_text())) for sandbox_id in found: - listed = subprocess.run(["docker", "ps", "-aq", "--filter", f"label=agentenv.sandbox={sandbox_id}"], - capture_output=True, text=True, check=True) - assert listed.stdout == "" + for selector in (f"label=agentenv.sandbox={sandbox_id}", f"name=^agent-{sandbox_id}$"): + listed = subprocess.run(["docker", "ps", "-aq", "--filter", selector], capture_output=True, text=True, + check=True) + assert listed.stdout == "", selector def _alive(pid): diff --git a/tst/unit/providers/sandbox_providers/local_sandbox_test.py b/tst/unit/providers/sandbox_providers/local_sandbox_test.py index a1ee03123..0cf4c5f7b 100644 --- a/tst/unit/providers/sandbox_providers/local_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/local_sandbox_test.py @@ -117,9 +117,22 @@ def test_find_work_dir_only_returns_matching_sandbox_id(tmp_path: Path, monkeypa assert LocalSandbox.find_work_dir("local-requested") == requested assert LocalSandbox.find_work_dir("local-missing") is None (tmp_path / "agent-env-notes.txt").write_text("") + (tmp_path / "agent-env-notes").mkdir() assert local_sandbox_ids() == ["local-other", "local-requested"] +@pytest.mark.asyncio +async def test_a_sandbox_killed_while_writing_its_port_map_is_still_rebuilt_for_teardown(tmp_path: Path, monkeypatch): + monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(tmp_path)) + work_dir = tmp_path / "agent-env-local-cut-abc123" + work_dir.mkdir() + (work_dir / ".port-map.json").write_text("") + + sandbox = await LocalSandboxProvider().get_sandbox("local-cut") + + assert (sandbox.sandbox_id, sandbox.work_dir) == ("local-cut", work_dir) + + def test_find_work_dir_falls_back_to_legacy_tempdir(tmp_path: Path, monkeypatch): """A sandbox created before the work-dir relocation (under the temp dir) is still found.""" new_root = tmp_path / "new" From 5e20adb39cc520fc159d0ecc1af3384f2556edcc Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 9 Nov 2026 08:56:04 -0800 Subject: [PATCH 4/4] fix(computer): a disk launcher stopped before its computer starts exits 143 The launcher runs under set -e, and its TERM trap killed the pid in qemu.pid, which isn't written until QEMU starts. A stop in between failed that kill and exited 1. test_the_launcher_forwards_each_host_port_from_loopback_here_to_the_host_and_seeds_the_trust_files stops the launcher just after it writes the seed, so on a slow runner it hit that window. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/computers/disk/linux/launcher/launch | 2 +- tst/unit/env/envs/test_disk_recipe.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/agent_env/computers/disk/linux/launcher/launch b/src/agent_env/computers/disk/linux/launcher/launch index 9780e3ace..23823708d 100755 --- a/src/agent_env/computers/disk/linux/launcher/launch +++ b/src/agent_env/computers/disk/linux/launcher/launch @@ -130,7 +130,7 @@ for port in ${VM_HOST_PORTS:-}; do fronts+=("$!") done trap 'kill ${fronts[@]+"${fronts[@]}"} ${presser:-} 2>/dev/null || true' EXIT -trap 'kill "$(cat "$control/qemu.pid" 2>/dev/null)" 2>/dev/null; exit 143' TERM INT +trap 'kill "$(cat "$control/qemu.pid" 2>/dev/null)" 2>/dev/null || true; exit 143' TERM INT if [[ "$disk" == *.zst ]]; then zstd -q -d -f "$disk" -o "$state/disk.img" disk=$state/disk.img diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index 744040ffa..cdd448c9e 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -244,6 +244,16 @@ def test_the_launcher_forwards_each_host_port_from_loopback_here_to_the_host_and assert running.wait(10) == 143 +def test_a_launcher_stopped_before_its_computer_starts_exits_as_one_stopped_after_does(launcher): + launch, calls = launcher + + running = launch(VM_DISK="/disk/disk.img.zst", ZSTD_TAKES_S="1") + _logged(calls, "zstd") + running.terminate() + + assert running.wait(10) == 143 + + @pytest.mark.parametrize("host_ports", ["18765", "28765"], ids=["a-guest-ports-proxy", "its-forward"]) def test_the_launcher_refuses_a_host_port_a_guest_ports_proxy_or_forward_takes(launcher, host_ports): launch, calls = launcher