diff --git a/.env.example b/.env.example index 64cf5c726..676974f84 100644 --- a/.env.example +++ b/.env.example @@ -26,6 +26,11 @@ AGENT_ENV_ENVIRONMENT=dev # - a hosted registry (ECR): configure a [stores.image] table in config.toml (no built-in coordinates) # AGENT_ENV_IMAGE_STORE=local +# Where local sandboxes keep their work folders (default ~/.agent-env-sandboxes; on macOS, a path Docker Desktop +# shares). A sandbox's folder is made before anything it runs, so a supervisor that gives a run a folder of its own +# can clean up after killing it: terminate each id local_sandbox_ids() lists, then remove its folder. +# AGENT_ENV_LOCAL_SANDBOX_DIR=~/.agent-env-sandboxes + # Secret store backend (what secret: references in config resolve through): # - "local" (or unset): process env vars, or a flat YAML/JSON file via [stores.secret] in config.toml # - a hosted secret manager (AWS Secrets Manager, Google Cloud Secret Manager): configure a [stores.secret] table (no built-in coordinates) diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 0ad85b9f4..7efbd9008 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -240,7 +240,8 @@ def run_bundle( Each run's sandboxes are torn down as it ends, so the contexts returned record sandboxes that are gone. ``keep`` leaves them up instead, for the caller to use and then remove with ``BundleRun.teardown()``. Ctrl-C or SIGTERM cancels the runs and tears down every one, kept or not, then raises ``RunInterrupted`` - with the result. + with the result. A process killed outright tears down nothing: a supervisor that may have to kill it gives it an + ``AGENT_ENV_LOCAL_SANDBOX_DIR`` of its own, then terminates what ``local_sandbox_ids`` finds there. It runs its own event loop, and turns namespace routing on for the whole process while it runs, so it suits the CLI and scripts, not a service. From async code, call it in a thread; there it can't take over diff --git a/src/agent_env/computers/disk/linux/launcher/launch b/src/agent_env/computers/disk/linux/launcher/launch index 9780e3ace..23823708d 100755 --- a/src/agent_env/computers/disk/linux/launcher/launch +++ b/src/agent_env/computers/disk/linux/launcher/launch @@ -130,7 +130,7 @@ for port in ${VM_HOST_PORTS:-}; do fronts+=("$!") done trap 'kill ${fronts[@]+"${fronts[@]}"} ${presser:-} 2>/dev/null || true' EXIT -trap 'kill "$(cat "$control/qemu.pid" 2>/dev/null)" 2>/dev/null; exit 143' TERM INT +trap 'kill "$(cat "$control/qemu.pid" 2>/dev/null)" 2>/dev/null || true; exit 143' TERM INT if [[ "$disk" == *.zst ]]; then zstd -q -d -f "$disk" -o "$state/disk.img" disk=$state/disk.img diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index 7a3c32c0f..0129aa12c 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -637,6 +637,21 @@ def _copy_into_container(source: Path, container: str, destination: str) -> None raise RuntimeError(f"Could not copy {source} into container {container}: {copied.stderr.strip()}") +def local_sandbox_ids() -> list[str]: + """The ids of the local sandboxes with a work folder under the sandbox root. A folder is made before anything its + sandbox runs, so a supervisor that gave a run its own ``AGENT_ENV_LOCAL_SANDBOX_DIR`` cleans up after killing it by + terminating each id here (``LocalSandboxProvider().terminate_sandbox``) and removing its folder.""" + root = _local_sandbox_root_path() + if not root.is_dir(): + return [] + prefix, found = "agent-env-", set() + for entry in root.iterdir(): + sandbox_id, _, suffix = entry.name.removeprefix(prefix).rpartition("-") + if entry.name.startswith(prefix) and sandbox_id and suffix and entry.is_dir() and not entry.is_symlink(): + found.add(sandbox_id) + return sorted(found) + + def remove_local_work_dir(sandbox_id: str) -> Path | None: """Delete a local sandbox's work folder, and return it, when it lies directly under the sandbox root and isn't a symlink; anything else, a legacy folder under the temp dir included, is left alone. Raises OSError @@ -989,8 +1004,10 @@ def _found(sandbox_id: str) -> LocalSandbox: work_dir = LocalSandbox.find_work_dir(sandbox_id) if work_dir is None: raise RuntimeError(f"Local sandbox work directory not found for sandbox_id={sandbox_id!r}") - port_map = work_dir / _PORT_MAP_FILE - ports = {int(c): h for c, h in json.loads(port_map.read_text()).items()} if port_map.exists() else None + try: + ports = {int(c): h for c, h in json.loads((work_dir / _PORT_MAP_FILE).read_text()).items()} + except (OSError, ValueError): # none written, or cut short by a run killed while writing it + ports = None return LocalSandbox(sandbox_id=sandbox_id, work_dir=work_dir, port_map=ports) diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index 5d0cf87de..4df05a23d 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -1,6 +1,7 @@ """``agent-env run`` on the hello bundle agent-env ships, with no infra: the local stores, a local VM-mode sandbox, and a verifier scoring the files the task loaded into it. Needs `bash` and the usual shell tools on PATH.""" +import asyncio import importlib import json import logging @@ -26,6 +27,11 @@ from agent_env.cli import cli from agent_env.config import configure, reset_config from agent_env.config.paths import state_root +from agent_env.providers.sandbox_providers.local_docker.provider import ( + LocalSandboxProvider, + local_sandbox_ids, + remove_local_work_dir, +) from agent_env.providers.sandbox_providers.sandbox_provider import reset_sandbox_provider from agent_env.store import Filter, LocalSqliteDocumentStore from agent_env.store.routing import namespace_routing @@ -198,10 +204,10 @@ def test_an_installed_bundle_runs_by_name_with_ids_rooted_at_its_package(bundle, assert "demo_installed_bundles" not in sys.modules -def _signalled(args, when, signum=signal.SIGINT, group=True): +def _signalled(args, when, signum=signal.SIGINT, group=True, ready=lambda: True): """Run ``agent-env run ARGS`` in its own process group, as a terminal would, and send ``signum`` to the group - (or to the process alone) once a line containing ``when`` is printed. Returns the exit code, the output, and - what the sandbox root held when the signal went.""" + (or to the process alone) once a line containing ``when`` is printed and ``ready()`` holds. Returns the exit code, + the output, and what the sandbox root held when the signal went.""" process = subprocess.Popen([sys.executable, "-m", "agent_env.cli", "run", *map(str, args)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, start_new_session=True) watchdog = threading.Timer(120, process.kill) @@ -211,6 +217,9 @@ def _signalled(args, when, signum=signal.SIGINT, group=True): for line in process.stdout: lines.append(line) if held is None and when in line: + deadline = time.monotonic() + 30 + while not ready() and time.monotonic() < deadline: + time.sleep(0.05) root = os.environ["AGENT_ENV_LOCAL_SANDBOX_DIR"] held = sorted(os.listdir(root)) if os.path.isdir(root) else [] (os.killpg if group else os.kill)(process.pid, signum) @@ -244,6 +253,37 @@ def test_a_signal_mid_run_marks_the_run_cancelled_tears_it_down_and_stops_its_co assert not _alive(int((tmp_path / "nap.pid").read_text())) +def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervisor_to_terminate(bundle, tmp_path): + napping = tmp_path / "napping" + (napping / "tasks").mkdir(parents=True) + (napping / "tasks/nap.json").write_text(json.dumps([ + {"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm", "sandbox_type": "local"}, + {"id": "web", "type": "deploy_sandbox", "sandbox_name": "web", "sandbox_mode": "container", + "sandbox_type": "local", "image": "public.ecr.aws/nginx/nginx:stable-alpine", "port": 80}, + {"id": "nap", "type": "verify_sandbox", "sandbox_name": "box", "base_dir": "/app", "verifier_id": "nap", + "depends_on": ["box", "web"], "criteria": [{"type": "bash_cmd_succeeds", "criterion": "naps", + "bash_cmd": 'sleep 300 & echo $! > "$HOME/nap.pid"; wait'}]}, + ])) + + nap = tmp_path / "nap.pid" + exit_code, output, held = _signalled([napping], "step 3/3 nap (verify_sandbox)", signal.SIGKILL, group=False, + ready=nap.exists) + found = local_sandbox_ids() + for sandbox_id in found: + asyncio.run(LocalSandboxProvider().terminate_sandbox(sandbox_id)) + remove_local_work_dir(sandbox_id) + + assert exit_code == -signal.SIGKILL, output + assert len(held) == len(found) == 2 + assert local_sandbox_ids() == [] + assert not _alive(int(nap.read_text())) + for sandbox_id in found: + for selector in (f"label=agentenv.sandbox={sandbox_id}", f"name=^agent-{sandbox_id}$"): + listed = subprocess.run(["docker", "ps", "-aq", "--filter", selector], capture_output=True, text=True, + check=True) + assert listed.stdout == "", selector + + def _alive(pid): for _ in range(100): try: diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index 744040ffa..cdd448c9e 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -244,6 +244,16 @@ def test_the_launcher_forwards_each_host_port_from_loopback_here_to_the_host_and assert running.wait(10) == 143 +def test_a_launcher_stopped_before_its_computer_starts_exits_as_one_stopped_after_does(launcher): + launch, calls = launcher + + running = launch(VM_DISK="/disk/disk.img.zst", ZSTD_TAKES_S="1") + _logged(calls, "zstd") + running.terminate() + + assert running.wait(10) == 143 + + @pytest.mark.parametrize("host_ports", ["18765", "28765"], ids=["a-guest-ports-proxy", "its-forward"]) def test_the_launcher_refuses_a_host_port_a_guest_ports_proxy_or_forward_takes(launcher, host_ports): launch, calls = launcher diff --git a/tst/unit/providers/sandbox_providers/local_sandbox_test.py b/tst/unit/providers/sandbox_providers/local_sandbox_test.py index 45d94b92b..0cf4c5f7b 100644 --- a/tst/unit/providers/sandbox_providers/local_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/local_sandbox_test.py @@ -16,7 +16,7 @@ import agent_env.providers.sandbox_providers.local_docker.provider as ls from agent_env.a2a_agent.a2a_agent import A2AAgent from agent_env.config import reset_config, set_object_store -from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox, LocalSandboxProvider +from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox, LocalSandboxProvider, local_sandbox_ids from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_CONTAINER, SANDBOX_MODE_VM from agent_env.store import LocalFilesystemObjectStore @@ -116,6 +116,21 @@ def test_find_work_dir_only_returns_matching_sandbox_id(tmp_path: Path, monkeypa assert LocalSandbox.find_work_dir("local-requested") == requested assert LocalSandbox.find_work_dir("local-missing") is None + (tmp_path / "agent-env-notes.txt").write_text("") + (tmp_path / "agent-env-notes").mkdir() + assert local_sandbox_ids() == ["local-other", "local-requested"] + + +@pytest.mark.asyncio +async def test_a_sandbox_killed_while_writing_its_port_map_is_still_rebuilt_for_teardown(tmp_path: Path, monkeypatch): + monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(tmp_path)) + work_dir = tmp_path / "agent-env-local-cut-abc123" + work_dir.mkdir() + (work_dir / ".port-map.json").write_text("") + + sandbox = await LocalSandboxProvider().get_sandbox("local-cut") + + assert (sandbox.sandbox_id, sandbox.work_dir) == ("local-cut", work_dir) def test_find_work_dir_falls_back_to_legacy_tempdir(tmp_path: Path, monkeypatch):