diff --git a/AGENTS.md b/AGENTS.md index 051f6745c..20fbbdb4a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,6 +89,10 @@ or one of them only where it is used. they keep running; files the install wrote elsewhere (`/opt`, packages) stay. Such an agent fetches the local store's grants only when `grant_advertise_host` names an address this machine reaches, the bridge address say: `host.docker.internal`, which grant URLs name by default, resolves only in containers. +- Local sandbox teardown reports a failure when it cannot list or remove its own Docker containers or + networks, or when marked Linux processes survive. The run keeps the sandbox work folder so teardown + can be retried. With no Docker installed there is nothing to remove; with Docker unreachable, it fails only + for a sandbox whose folder records its own container, compose stack or closed network. - `[agents] default_a2a_agent_id`: the agent a `deploy_agent` step without an id deploys; built-in `@local/agentenv-framework/agents/claude-code`, the `agents` bundle's Claude Code (`src/agent_env/agents/`), overridable by `configure(default_a2a_agent_id=...)`. Any step's diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index 0129aa12c..ee4491c17 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -90,6 +90,8 @@ def _runs_in_container(cmd: list[str]) -> bool: # how much of each stream is kept, the end of it: above the largest output core reads whole, run_code's 50 MiB result. _OUTPUT_GRACE_SECONDS = 1 _OUTPUT_TAIL_BYTES = 64 * 1024 * 1024 +# How much of a failed Docker command's stderr a teardown error carries. +_ERROR_TAIL_CHARS = 300 # Every command a local sandbox runs carries this, set to the sandbox's id, and so does whatever the command starts, so # teardown can find a process left running on this machine, such as the agent a host-mode install started. @@ -335,50 +337,95 @@ async def terminate(self) -> None: this, local runs leak their containers/compose stacks, which squat host ports and block the next deploy. Prod backends override terminate() to tear the whole VM down. Processes its commands left running on this machine are stopped first (on Linux; see ``_marked_pids``), - then, once none is left to run from it, what its steps staged in /tmp is removed. + then, once none is left to run from it, what its steps staged in /tmp is removed. With no Docker installed, + nothing of it can have been started; with Docker unreachable, the teardown fails only for a sandbox whose folder + records Docker objects of its own. """ + failures: list[Exception] = [] try: - if await _stop_marked(self.sandbox_id): - await asyncio.to_thread(_remove_staged, self.sandbox_id) - if self.mode == SANDBOX_MODE_VM and not self.owns_container: - await self._remove_labeled() - finally: # a container that wouldn't go must not keep the compose stack up - await self.exec_script(f"docker rm -f {shlex.quote(self.container_name)} >/dev/null 2>&1 || true") - if self.closed_network is not None: - await self._remove_labeled() - if self.mode == SANDBOX_MODE_VM and (self._work_dir / "docker-compose.yml").exists(): - # No `|| true`: `docker compose down` is idempotent (an already-down stack exits 0), so a - # non-zero exit is a real failure — surface it (exec_script raises, with the compose output) - # instead of silently leaving the stack running and its host ports held. Callers wrap - # terminate() in try/except, so a raised teardown failure is caught, not fatal. + if not await _stop_marked(self.sandbox_id): + raise RuntimeError(f"Sandbox {self.sandbox_id} still has marked processes") + await asyncio.to_thread(_remove_staged, self.sandbox_id) + except Exception as error: + failures.append(error) + try: + code, _, stderr = await self.exec_with_output("docker", "version", "--format", "{{.Server.Version}}") + except FileNotFoundError: + code, stderr = None, "" + if code == 0: + await self._remove_docker_objects(failures) + elif code is not None: + reason = stderr.strip()[-_ERROR_TAIL_CHARS:] + if self._made_docker_objects: + failures.append(RuntimeError(f"Docker can't be reached to remove sandbox {self.sandbox_id}'s container, " + f"compose stack or network: {reason}")) + else: + logger.warning("Could not reach Docker (%s); anything sandbox %s's steps started with it is still there. " + "Remove it with: docker rm -f $(docker ps -aq --filter label=%s=%s)", + reason, self.sandbox_id, SANDBOX_LABEL, self.sandbox_id) + if failures: + raise RuntimeError("Local sandbox teardown incomplete: " + "; ".join(str(error) for error in failures)) from failures[0] + + @property + def _made_docker_objects(self) -> bool: + """Whether its folder records Docker objects of its own: its container, a compose stack or a closed network.""" + return self.owns_container or self.closed_network is not None or (self._work_dir / "docker-compose.yml").exists() + + async def _remove_docker_objects(self, failures: list[Exception]) -> None: + """Remove its container, compose stack and labeled objects, each attempted whatever the others did, adding + what failed to ``failures``.""" + try: + await self._remove_own_container() + except Exception as error: + failures.append(error) + if self.mode == SANDBOX_MODE_VM and (self._work_dir / "docker-compose.yml").exists(): + try: await self.exec_script( f"cd {shlex.quote(str(self._work_dir))} && docker compose down -v --remove-orphans" ) + except Exception as error: + failures.append(error) + if (self.mode == SANDBOX_MODE_VM and not self.owns_container) or self.closed_network is not None: + try: + await self._remove_labeled() + except Exception as error: + failures.append(error) + + async def _remove_own_container(self) -> None: + """Remove this sandbox's named container, or verify that it is absent.""" + name = f"name=^{re.escape(self.container_name)}$" + code, stdout, stderr = await self.exec_with_output("docker", "ps", "-aq", "--filter", name) + if code != 0: + raise RuntimeError(f"Could not list sandbox {self.sandbox_id}'s container: " + f"{stderr.strip()[-_ERROR_TAIL_CHARS:]}") + if containers := stdout.split(): + await self.exec_script(f"docker rm -f {shlex.join(containers)} >/dev/null") async def _remove_labeled(self) -> None: - """Remove what steps started on this host for this sandbox (``run_docker_container``'s containers, their - images and networks), which a real VM would take down with it. Images and networks go best-effort: another - local sandbox's container can still be using one.""" + """Remove labeled containers and networks; images can still be shared and are best-effort.""" label = f"label={SANDBOX_LABEL}={self.sandbox_id}" - try: - exit_code, stdout, stderr = await self.exec_with_output("docker", "ps", "-aq", "--filter", label) - except FileNotFoundError: # no Docker installed: nothing could have been started - return - if exit_code != 0: - # No Docker to reach (a run that never needed it) or Docker is down; either way nothing can be removed now. - logger.warning( - "Could not list %s's containers (%s); anything it started is still there. Remove it with: " - "docker rm -f $(docker ps -aq --filter %s)", self.sandbox_id, stderr.strip()[-300:], label, - ) - return + failures: list[Exception] = [] + for resource, args in ( + ("containers", ("docker", "ps", "-aq", "--filter", label)), + ("networks", ("docker", "network", "ls", "-q", "--filter", label)), + ): + try: + code, stdout, stderr = await self.exec_with_output(*args) + if code != 0: + raise RuntimeError(f"Could not list sandbox {self.sandbox_id}'s {resource}: " + f"{stderr.strip()[-_ERROR_TAIL_CHARS:]}") + if identifiers := stdout.split(): + command = "docker rm -f" if resource == "containers" else "docker network rm" + await self.exec_script(f"{command} {shlex.join(identifiers)} >/dev/null") + except Exception as error: + failures.append(error) q = shlex.quote(label) - if stdout.split(): - await self.exec_script(f"docker rm -f {' '.join(stdout.split())} >/dev/null") await self.exec_script( f"ids=$(docker images -q --filter {q} | sort -u); [ -z \"$ids\" ] || docker rmi -f $ids >/dev/null 2>&1; " - f"ids=$(docker network ls -q --filter {q}); [ -z \"$ids\" ] || docker network rm $ids >/dev/null 2>&1; " f"exit 0" ) + if failures: + raise RuntimeError("; ".join(str(error) for error in failures)) from failures[0] async def exec(self, *command: str) -> Any: """Execute a command locally via subprocess. diff --git a/tst/unit/providers/sandbox_providers/local_sandbox_test.py b/tst/unit/providers/sandbox_providers/local_sandbox_test.py index 0cf4c5f7b..0e6453c11 100644 --- a/tst/unit/providers/sandbox_providers/local_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/local_sandbox_test.py @@ -185,6 +185,12 @@ async def exec_script(self, script, *, max_retries=0): async def exec_with_output(self, *args): self.scripts.append(" ".join(args)) + if args[:2] == ("docker", "version"): + return 0, "29.0.0", "" + if args[:3] == ("docker", "ps", "-aq") and args[-1].startswith("name="): + return 0, self.container_name, "" + if args[:3] == ("docker", "network", "ls"): + return 0, "", "" return self._listed @@ -275,8 +281,7 @@ async def test_terminate_removes_this_sandboxs_own_container(): sandbox.mode = SANDBOX_MODE_CONTAINER await sandbox.terminate() - assert len(sandbox.scripts) == 1 - assert f"docker rm -f {sandbox.container_name}" in sandbox.scripts[0] + assert any(f"docker rm -f {sandbox.container_name}" in script for script in sandbox.scripts) assert sandbox.container_name != "agent-api" # it's the unique local name @@ -314,8 +319,8 @@ async def test_terminate_vm_without_compose_removes_only_an_agent_placed_on_it(t sandbox = _RecordingLocalSandbox(work_dir=tmp_path) await sandbox.terminate() - assert sandbox.scripts[0] == f"docker ps -aq --filter label=agentenv.sandbox={sandbox.sandbox_id}" - assert sandbox.scripts[-1] == f"docker rm -f {sandbox.container_name} >/dev/null 2>&1 || true" + assert f"docker ps -aq --filter label=agentenv.sandbox={sandbox.sandbox_id}" in sandbox.scripts + assert f"docker rm -f {sandbox.container_name} >/dev/null" in sandbox.scripts @pytest.mark.asyncio @@ -325,18 +330,18 @@ async def test_terminate_removes_what_steps_started_for_this_sandbox_only(tmp_pa sandbox = _RecordingLocalSandbox(tmp_path, sandbox_id="local-abc", listed=(0, "c1\nc2\n", "")) await sandbox.terminate() - assert sandbox.scripts[:2] == ["docker ps -aq --filter label=agentenv.sandbox=local-abc", "docker rm -f c1 c2 >/dev/null"] - for listing in ("docker images -q", "docker network ls -q"): - assert f"{listing} --filter label=agentenv.sandbox=local-abc" in sandbox.scripts[2] + assert "docker ps -aq --filter label=agentenv.sandbox=local-abc" in sandbox.scripts + assert "docker rm -f c1 c2 >/dev/null" in sandbox.scripts + assert any("docker images -q --filter label=agentenv.sandbox=local-abc" in script for script in sandbox.scripts) + assert any("docker network ls -q --filter label=agentenv.sandbox=local-abc" in script for script in sandbox.scripts) @pytest.mark.asyncio -async def test_a_failed_lookup_is_reported_with_the_command_that_cleans_up(tmp_path: Path, caplog): +async def test_a_failed_lookup_is_reported_and_containers_are_not_guessed(tmp_path: Path): sandbox = _RecordingLocalSandbox(tmp_path, sandbox_id="local-abc", listed=(1, "", "Cannot connect to the Docker daemon")) - await sandbox.terminate() - - assert "docker rm -f $(docker ps -aq --filter label=agentenv.sandbox=local-abc)" in caplog.text - assert not any(s.startswith("docker rm -f c") for s in sandbox.scripts) + with pytest.raises(RuntimeError, match="Cannot connect to the Docker daemon"): + await sandbox.terminate() + assert not any(script.startswith("docker rm -f c") for script in sandbox.scripts) @pytest.mark.asyncio @@ -348,7 +353,30 @@ async def exec_with_output(self, *args): sandbox = _NoDocker(tmp_path, sandbox_id="local-abc") await sandbox.terminate() - assert sandbox.scripts == [f"docker rm -f {sandbox.container_name} >/dev/null 2>&1 || true"] + assert sandbox.scripts == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("record", [None, ".agent-container-mode", "docker-compose.yml", ".closed-network"], + ids=["none", "its-container", "its-compose-stack", "its-closed-network"]) +async def test_teardown_with_docker_unreachable_fails_only_for_a_sandbox_that_made_docker_objects(tmp_path: Path, + record, caplog): + if record is not None: + (tmp_path / record).write_text("closed-owned" if record == ".closed-network" else "") + + class _DaemonDown(_RecordingLocalSandbox): + async def exec_with_output(self, *args): + self.scripts.append(" ".join(args)) + return 1, "", "Cannot connect to the Docker daemon" + + sandbox = _DaemonDown(tmp_path, sandbox_id="local-abc") + if record is None: + await sandbox.terminate() + assert "docker rm -f $(docker ps -aq --filter label=agentenv.sandbox=local-abc)" in caplog.text + else: + with pytest.raises(RuntimeError, match="Docker can't be reached to remove sandbox local-abc's"): + await sandbox.terminate() + assert sandbox.scripts == ["docker version --format {{.Server.Version}}"] @pytest.mark.asyncio @@ -368,6 +396,22 @@ async def exec_script(self, script, *, max_retries=0): assert any("docker compose down" in s for s in sandbox.scripts) +@pytest.mark.asyncio +async def test_a_network_lookup_failure_still_attempts_compose_teardown(tmp_path: Path): + (tmp_path / "docker-compose.yml").write_text("") + + class _NetworkUnavailable(_RecordingLocalSandbox): + async def exec_with_output(self, *args): + if args[:3] == ("docker", "network", "ls"): + return 1, "", "network lookup failed" + return await super().exec_with_output(*args) + + sandbox = _NetworkUnavailable(tmp_path, sandbox_id="local-abc") + with pytest.raises(RuntimeError, match="network lookup failed"): + await sandbox.terminate() + assert any("docker compose down" in script for script in sandbox.scripts) + + @pytest.mark.asyncio async def test_a_reattached_container_stays_vm_mode_and_owned(tmp_path: Path, monkeypatch): """Its exec runs on this host, so a reattached agent's steps must take the VM path into its container.""" @@ -406,6 +450,9 @@ async def record(self, script, *, max_retries=0): return "" monkeypatch.setattr(LocalSandbox, "exec_script", record) + async def listed(self, *args): + return (0, self.container_name, "") + monkeypatch.setattr(LocalSandbox, "exec_with_output", listed) class _Provider(LocalSandboxProvider): async def create_vm(self, **kwargs): @@ -416,7 +463,7 @@ async def create_vm(self, **kwargs): del scripts[:] await rebuilt.terminate() assert created.owns_container and rebuilt.owns_container and rebuilt.mode == SANDBOX_MODE_VM - assert scripts == [f"docker rm -f {created.container_name} >/dev/null 2>&1 || true"] + assert scripts == [f"docker rm -f {created.container_name} >/dev/null"] @pytest.mark.asyncio @@ -720,9 +767,10 @@ async def test_a_container_with_no_network_takes_its_relay_and_network_with_it(t await sandbox.terminate() - assert sandbox.scripts[:3] == [f"docker rm -f {sandbox.container_name} >/dev/null 2>&1 || true", - "docker ps -aq --filter label=agentenv.sandbox=local-abc", "docker rm -f r1 >/dev/null"] - assert "docker network ls -q --filter label=agentenv.sandbox=local-abc" in sandbox.scripts[3] + assert f"docker rm -f {sandbox.container_name} >/dev/null" in sandbox.scripts + assert "docker rm -f r1 >/dev/null" in sandbox.scripts + assert "docker network ls -q --filter label=agentenv.sandbox=local-abc" in sandbox.scripts + async def _relay_image() -> str: @@ -1098,7 +1146,8 @@ async def test_teardown_keeps_the_staging_a_surviving_process_runs_from(monkeypa sandbox = _RecordingLocalSandbox(work_dir=tmp_path) staged = Path(tempfile.mkdtemp(prefix="install-agent-solver-", suffix=f"-{sandbox.sandbox_id}", dir="/tmp")) try: - await sandbox.terminate() + with pytest.raises(RuntimeError, match="still has marked processes"): + await sandbox.terminate() assert staged.exists() finally: @@ -1248,7 +1297,7 @@ def fail(*args): await provider.create_container(image_name="img:v1", port=8000, env={}) scripts = provider.made[0].scripts assert not any(s.startswith("docker start") for s in scripts) - assert scripts[-1].startswith(f"docker rm -f {provider.made[0].container_name}") + assert any(script.startswith(f"docker rm -f {provider.made[0].container_name}") for script in scripts) def test_the_copy_runs_docker_directly_and_reports_its_error(monkeypatch, tmp_path): diff --git a/tst/unit/task/teardown_test.py b/tst/unit/task/teardown_test.py index ab999b92e..6cc97c1a9 100644 --- a/tst/unit/task/teardown_test.py +++ b/tst/unit/task/teardown_test.py @@ -384,16 +384,55 @@ async def record(self, script, *, max_retries=0): return "" monkeypatch.setattr(LocalSandbox, "exec_script", record) + async def listed(self, *args): + return (0, self.container_name, "") + monkeypatch.setattr(LocalSandbox, "exec_with_output", listed) context = TaskStepContext(deployed_agents=[ DeployedAgent(agent_name="solver", api_url="http://agent", sandbox_id=agent.sandbox_id, sandbox_type="local")]) report = _run(context) assert report.terminated == (RecordedSandbox(agent.sandbox_id, "local", "agent"),) - assert scripts == [f"docker rm -f {agent.container_name} >/dev/null 2>&1 || true"] + assert scripts == [f"docker rm -f {agent.container_name} >/dev/null"] assert list(sandbox_root.iterdir()) == [] +@pytest.mark.parametrize("failure", ["list", "remove", "network"]) +def test_local_docker_refusal_retains_folder_and_retry_removes_it(sandbox_root, tmp_path, monkeypatch, failure): + agent = LocalSandbox() + (agent.work_dir / ".agent-container-mode").write_text(agent.container_name) + if failure == "network": + (agent.work_dir / ".closed-network").write_text("closed-owned") + bindir = tmp_path / "bin" + bindir.mkdir() + docker = bindir / "docker" + permit, alive = tmp_path / "permit", tmp_path / "alive" + alive.touch() + docker.write_text( + f"#!/bin/sh\n" + f"if [ ! -f '{permit}' ] && [ \"$1\" = '{'rm' if failure == 'remove' else 'network' if failure == 'network' else 'ps'}' ]; then " + "echo 'daemon unavailable' >&2; exit 1; fi\n" + f"if [ \"$1\" = ps ] && [ -f '{alive}' ]; then echo owned-container; fi\n" + f"if [ \"$1\" = rm ]; then rm -f '{alive}'; fi\n" + "exit 0\n" + ) + docker.chmod(0o755) + monkeypatch.setenv("PATH", f"{bindir}:{os.environ['PATH']}") + context = TaskStepContext(deployed_agents=[ + DeployedAgent(agent_name="solver", api_url="http://agent", sandbox_id=agent.sandbox_id, sandbox_type="local")]) + + failed = _run(context) + + assert agent.work_dir.is_dir() + assert alive.exists() is (failure != "network") + assert [entry.sandbox_id for entry in failed.still_up] == [agent.sandbox_id] + assert agent.sandbox_id not in context.metadata.get(TORN_DOWN_KEY, []) + permit.touch() + retried = _run(context) + assert retried.terminated == (RecordedSandbox(agent.sandbox_id, "local", "agent"),) + assert not agent.work_dir.exists() and not alive.exists() + + def test_a_slow_folder_removal_does_not_count_against_the_terminate_timeout(sandbox_root, monkeypatch): sandbox = LocalSandbox() context = TaskStepContext(deployed_sandboxes=[