Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@ or one of them only where it is used.
they keep running; files the install wrote elsewhere (`/opt`, packages) stay. Such an agent fetches the
local store's grants only when `grant_advertise_host` names an address this machine reaches, the
bridge address say: `host.docker.internal`, which grant URLs name by default, resolves only in containers.
- Local sandbox teardown reports a failure when it cannot list or remove its own Docker containers or
networks, or when marked Linux processes survive. The run keeps the sandbox work folder so teardown
can be retried. With no Docker installed there is nothing to remove; with Docker unreachable, it fails only
for a sandbox whose folder records its own container, compose stack or closed network.
- `[agents] default_a2a_agent_id`: the agent a `deploy_agent` step without an id deploys; built-in
`@local/agentenv-framework/agents/claude-code`, the `agents` bundle's Claude Code
(`src/agent_env/agents/`), overridable by `configure(default_a2a_agent_id=...)`. Any step's
Expand Down
109 changes: 78 additions & 31 deletions src/agent_env/providers/sandbox_providers/local_docker/provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ def _runs_in_container(cmd: list[str]) -> bool:
# how much of each stream is kept, the end of it: above the largest output core reads whole, run_code's 50 MiB result.
_OUTPUT_GRACE_SECONDS = 1
_OUTPUT_TAIL_BYTES = 64 * 1024 * 1024
# How much of a failed Docker command's stderr a teardown error carries.
_ERROR_TAIL_CHARS = 300

# Every command a local sandbox runs carries this, set to the sandbox's id, and so does whatever the command starts, so
# teardown can find a process left running on this machine, such as the agent a host-mode install started.
Expand Down Expand Up @@ -335,50 +337,95 @@ async def terminate(self) -> None:
this, local runs leak their containers/compose stacks, which squat host ports and block the
next deploy. Prod backends override terminate() to tear the whole VM down.
Processes its commands left running on this machine are stopped first (on Linux; see ``_marked_pids``),
then, once none is left to run from it, what its steps staged in /tmp is removed.
then, once none is left to run from it, what its steps staged in /tmp is removed. With no Docker installed,
nothing of it can have been started; with Docker unreachable, the teardown fails only for a sandbox whose folder
records Docker objects of its own.
"""
failures: list[Exception] = []
try:
if await _stop_marked(self.sandbox_id):
await asyncio.to_thread(_remove_staged, self.sandbox_id)
if self.mode == SANDBOX_MODE_VM and not self.owns_container:
await self._remove_labeled()
finally: # a container that wouldn't go must not keep the compose stack up
await self.exec_script(f"docker rm -f {shlex.quote(self.container_name)} >/dev/null 2>&1 || true")
if self.closed_network is not None:
await self._remove_labeled()
if self.mode == SANDBOX_MODE_VM and (self._work_dir / "docker-compose.yml").exists():
# No `|| true`: `docker compose down` is idempotent (an already-down stack exits 0), so a
# non-zero exit is a real failure — surface it (exec_script raises, with the compose output)
# instead of silently leaving the stack running and its host ports held. Callers wrap
# terminate() in try/except, so a raised teardown failure is caught, not fatal.
if not await _stop_marked(self.sandbox_id):
raise RuntimeError(f"Sandbox {self.sandbox_id} still has marked processes")
await asyncio.to_thread(_remove_staged, self.sandbox_id)
except Exception as error:
failures.append(error)
try:
code, _, stderr = await self.exec_with_output("docker", "version", "--format", "{{.Server.Version}}")
except FileNotFoundError:
code, stderr = None, ""
if code == 0:
await self._remove_docker_objects(failures)
elif code is not None:
reason = stderr.strip()[-_ERROR_TAIL_CHARS:]
if self._made_docker_objects:
failures.append(RuntimeError(f"Docker can't be reached to remove sandbox {self.sandbox_id}'s container, "
f"compose stack or network: {reason}"))
else:
logger.warning("Could not reach Docker (%s); anything sandbox %s's steps started with it is still there. "
"Remove it with: docker rm -f $(docker ps -aq --filter label=%s=%s)",
reason, self.sandbox_id, SANDBOX_LABEL, self.sandbox_id)
if failures:
raise RuntimeError("Local sandbox teardown incomplete: " + "; ".join(str(error) for error in failures)) from failures[0]

@property
def _made_docker_objects(self) -> bool:
"""Whether its folder records Docker objects of its own: its container, a compose stack or a closed network."""
return self.owns_container or self.closed_network is not None or (self._work_dir / "docker-compose.yml").exists()

async def _remove_docker_objects(self, failures: list[Exception]) -> None:
"""Remove its container, compose stack and labeled objects, each attempted whatever the others did, adding
what failed to ``failures``."""
try:
await self._remove_own_container()
except Exception as error:
failures.append(error)
if self.mode == SANDBOX_MODE_VM and (self._work_dir / "docker-compose.yml").exists():
try:
await self.exec_script(
f"cd {shlex.quote(str(self._work_dir))} && docker compose down -v --remove-orphans"
)
except Exception as error:
failures.append(error)
if (self.mode == SANDBOX_MODE_VM and not self.owns_container) or self.closed_network is not None:
try:
await self._remove_labeled()
except Exception as error:
failures.append(error)

async def _remove_own_container(self) -> None:
"""Remove this sandbox's named container, or verify that it is absent."""
name = f"name=^{re.escape(self.container_name)}$"
code, stdout, stderr = await self.exec_with_output("docker", "ps", "-aq", "--filter", name)
if code != 0:
raise RuntimeError(f"Could not list sandbox {self.sandbox_id}'s container: "
f"{stderr.strip()[-_ERROR_TAIL_CHARS:]}")
if containers := stdout.split():
await self.exec_script(f"docker rm -f {shlex.join(containers)} >/dev/null")

async def _remove_labeled(self) -> None:
"""Remove what steps started on this host for this sandbox (``run_docker_container``'s containers, their
images and networks), which a real VM would take down with it. Images and networks go best-effort: another
local sandbox's container can still be using one."""
"""Remove labeled containers and networks; images can still be shared and are best-effort."""
label = f"label={SANDBOX_LABEL}={self.sandbox_id}"
try:
exit_code, stdout, stderr = await self.exec_with_output("docker", "ps", "-aq", "--filter", label)
except FileNotFoundError: # no Docker installed: nothing could have been started
return
if exit_code != 0:
# No Docker to reach (a run that never needed it) or Docker is down; either way nothing can be removed now.
logger.warning(
"Could not list %s's containers (%s); anything it started is still there. Remove it with: "
"docker rm -f $(docker ps -aq --filter %s)", self.sandbox_id, stderr.strip()[-300:], label,
)
return
failures: list[Exception] = []
for resource, args in (
("containers", ("docker", "ps", "-aq", "--filter", label)),
("networks", ("docker", "network", "ls", "-q", "--filter", label)),
):
try:
code, stdout, stderr = await self.exec_with_output(*args)
if code != 0:
raise RuntimeError(f"Could not list sandbox {self.sandbox_id}'s {resource}: "
f"{stderr.strip()[-_ERROR_TAIL_CHARS:]}")
if identifiers := stdout.split():
command = "docker rm -f" if resource == "containers" else "docker network rm"
await self.exec_script(f"{command} {shlex.join(identifiers)} >/dev/null")
except Exception as error:
failures.append(error)
q = shlex.quote(label)
if stdout.split():
await self.exec_script(f"docker rm -f {' '.join(stdout.split())} >/dev/null")
await self.exec_script(
f"ids=$(docker images -q --filter {q} | sort -u); [ -z \"$ids\" ] || docker rmi -f $ids >/dev/null 2>&1; "
f"ids=$(docker network ls -q --filter {q}); [ -z \"$ids\" ] || docker network rm $ids >/dev/null 2>&1; "
f"exit 0"
)
if failures:
raise RuntimeError("; ".join(str(error) for error in failures)) from failures[0]

async def exec(self, *command: str) -> Any:
"""Execute a command locally via subprocess.
Expand Down
87 changes: 68 additions & 19 deletions tst/unit/providers/sandbox_providers/local_sandbox_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,12 @@ async def exec_script(self, script, *, max_retries=0):

async def exec_with_output(self, *args):
self.scripts.append(" ".join(args))
if args[:2] == ("docker", "version"):
return 0, "29.0.0", ""
if args[:3] == ("docker", "ps", "-aq") and args[-1].startswith("name="):
return 0, self.container_name, ""
if args[:3] == ("docker", "network", "ls"):
return 0, "", ""
return self._listed


Expand Down Expand Up @@ -275,8 +281,7 @@ async def test_terminate_removes_this_sandboxs_own_container():
sandbox.mode = SANDBOX_MODE_CONTAINER
await sandbox.terminate()

assert len(sandbox.scripts) == 1
assert f"docker rm -f {sandbox.container_name}" in sandbox.scripts[0]
assert any(f"docker rm -f {sandbox.container_name}" in script for script in sandbox.scripts)
assert sandbox.container_name != "agent-api" # it's the unique local name


Expand Down Expand Up @@ -314,8 +319,8 @@ async def test_terminate_vm_without_compose_removes_only_an_agent_placed_on_it(t
sandbox = _RecordingLocalSandbox(work_dir=tmp_path)
await sandbox.terminate()

assert sandbox.scripts[0] == f"docker ps -aq --filter label=agentenv.sandbox={sandbox.sandbox_id}"
assert sandbox.scripts[-1] == f"docker rm -f {sandbox.container_name} >/dev/null 2>&1 || true"
assert f"docker ps -aq --filter label=agentenv.sandbox={sandbox.sandbox_id}" in sandbox.scripts
assert f"docker rm -f {sandbox.container_name} >/dev/null" in sandbox.scripts


@pytest.mark.asyncio
Expand All @@ -325,18 +330,18 @@ async def test_terminate_removes_what_steps_started_for_this_sandbox_only(tmp_pa
sandbox = _RecordingLocalSandbox(tmp_path, sandbox_id="local-abc", listed=(0, "c1\nc2\n", ""))
await sandbox.terminate()

assert sandbox.scripts[:2] == ["docker ps -aq --filter label=agentenv.sandbox=local-abc", "docker rm -f c1 c2 >/dev/null"]
for listing in ("docker images -q", "docker network ls -q"):
assert f"{listing} --filter label=agentenv.sandbox=local-abc" in sandbox.scripts[2]
assert "docker ps -aq --filter label=agentenv.sandbox=local-abc" in sandbox.scripts
assert "docker rm -f c1 c2 >/dev/null" in sandbox.scripts
assert any("docker images -q --filter label=agentenv.sandbox=local-abc" in script for script in sandbox.scripts)
assert any("docker network ls -q --filter label=agentenv.sandbox=local-abc" in script for script in sandbox.scripts)


@pytest.mark.asyncio
async def test_a_failed_lookup_is_reported_with_the_command_that_cleans_up(tmp_path: Path, caplog):
async def test_a_failed_lookup_is_reported_and_containers_are_not_guessed(tmp_path: Path):
sandbox = _RecordingLocalSandbox(tmp_path, sandbox_id="local-abc", listed=(1, "", "Cannot connect to the Docker daemon"))
await sandbox.terminate()

assert "docker rm -f $(docker ps -aq --filter label=agentenv.sandbox=local-abc)" in caplog.text
assert not any(s.startswith("docker rm -f c") for s in sandbox.scripts)
with pytest.raises(RuntimeError, match="Cannot connect to the Docker daemon"):
await sandbox.terminate()
assert not any(script.startswith("docker rm -f c") for script in sandbox.scripts)


@pytest.mark.asyncio
Expand All @@ -348,7 +353,30 @@ async def exec_with_output(self, *args):
sandbox = _NoDocker(tmp_path, sandbox_id="local-abc")
await sandbox.terminate()

assert sandbox.scripts == [f"docker rm -f {sandbox.container_name} >/dev/null 2>&1 || true"]
assert sandbox.scripts == []


@pytest.mark.asyncio
@pytest.mark.parametrize("record", [None, ".agent-container-mode", "docker-compose.yml", ".closed-network"],
ids=["none", "its-container", "its-compose-stack", "its-closed-network"])
async def test_teardown_with_docker_unreachable_fails_only_for_a_sandbox_that_made_docker_objects(tmp_path: Path,
record, caplog):
if record is not None:
(tmp_path / record).write_text("closed-owned" if record == ".closed-network" else "")

class _DaemonDown(_RecordingLocalSandbox):
async def exec_with_output(self, *args):
self.scripts.append(" ".join(args))
return 1, "", "Cannot connect to the Docker daemon"

sandbox = _DaemonDown(tmp_path, sandbox_id="local-abc")
if record is None:
await sandbox.terminate()
assert "docker rm -f $(docker ps -aq --filter label=agentenv.sandbox=local-abc)" in caplog.text
else:
with pytest.raises(RuntimeError, match="Docker can't be reached to remove sandbox local-abc's"):
await sandbox.terminate()
assert sandbox.scripts == ["docker version --format {{.Server.Version}}"]


@pytest.mark.asyncio
Expand All @@ -368,6 +396,22 @@ async def exec_script(self, script, *, max_retries=0):
assert any("docker compose down" in s for s in sandbox.scripts)


@pytest.mark.asyncio
async def test_a_network_lookup_failure_still_attempts_compose_teardown(tmp_path: Path):
(tmp_path / "docker-compose.yml").write_text("")

class _NetworkUnavailable(_RecordingLocalSandbox):
async def exec_with_output(self, *args):
if args[:3] == ("docker", "network", "ls"):
return 1, "", "network lookup failed"
return await super().exec_with_output(*args)

sandbox = _NetworkUnavailable(tmp_path, sandbox_id="local-abc")
with pytest.raises(RuntimeError, match="network lookup failed"):
await sandbox.terminate()
assert any("docker compose down" in script for script in sandbox.scripts)


@pytest.mark.asyncio
async def test_a_reattached_container_stays_vm_mode_and_owned(tmp_path: Path, monkeypatch):
"""Its exec runs on this host, so a reattached agent's steps must take the VM path into its container."""
Expand Down Expand Up @@ -406,6 +450,9 @@ async def record(self, script, *, max_retries=0):
return ""

monkeypatch.setattr(LocalSandbox, "exec_script", record)
async def listed(self, *args):
return (0, self.container_name, "")
monkeypatch.setattr(LocalSandbox, "exec_with_output", listed)

class _Provider(LocalSandboxProvider):
async def create_vm(self, **kwargs):
Expand All @@ -416,7 +463,7 @@ async def create_vm(self, **kwargs):
del scripts[:]
await rebuilt.terminate()
assert created.owns_container and rebuilt.owns_container and rebuilt.mode == SANDBOX_MODE_VM
assert scripts == [f"docker rm -f {created.container_name} >/dev/null 2>&1 || true"]
assert scripts == [f"docker rm -f {created.container_name} >/dev/null"]


@pytest.mark.asyncio
Expand Down Expand Up @@ -720,9 +767,10 @@ async def test_a_container_with_no_network_takes_its_relay_and_network_with_it(t

await sandbox.terminate()

assert sandbox.scripts[:3] == [f"docker rm -f {sandbox.container_name} >/dev/null 2>&1 || true",
"docker ps -aq --filter label=agentenv.sandbox=local-abc", "docker rm -f r1 >/dev/null"]
assert "docker network ls -q --filter label=agentenv.sandbox=local-abc" in sandbox.scripts[3]
assert f"docker rm -f {sandbox.container_name} >/dev/null" in sandbox.scripts
assert "docker rm -f r1 >/dev/null" in sandbox.scripts
assert "docker network ls -q --filter label=agentenv.sandbox=local-abc" in sandbox.scripts



async def _relay_image() -> str:
Expand Down Expand Up @@ -1098,7 +1146,8 @@ async def test_teardown_keeps_the_staging_a_surviving_process_runs_from(monkeypa
sandbox = _RecordingLocalSandbox(work_dir=tmp_path)
staged = Path(tempfile.mkdtemp(prefix="install-agent-solver-", suffix=f"-{sandbox.sandbox_id}", dir="/tmp"))
try:
await sandbox.terminate()
with pytest.raises(RuntimeError, match="still has marked processes"):
await sandbox.terminate()

assert staged.exists()
finally:
Expand Down Expand Up @@ -1248,7 +1297,7 @@ def fail(*args):
await provider.create_container(image_name="img:v1", port=8000, env={})
scripts = provider.made[0].scripts
assert not any(s.startswith("docker start") for s in scripts)
assert scripts[-1].startswith(f"docker rm -f {provider.made[0].container_name}")
assert any(script.startswith(f"docker rm -f {provider.made[0].container_name}") for script in scripts)


def test_the_copy_runs_docker_directly_and_reports_its_error(monkeypatch, tmp_path):
Expand Down
Loading
Loading