Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ AGENT_ENV_ENVIRONMENT=dev
# - a hosted registry (ECR): configure a [stores.image] table in config.toml (no built-in coordinates)
# AGENT_ENV_IMAGE_STORE=local

# Where local sandboxes keep their work folders (default ~/.agent-env-sandboxes; on macOS, a path Docker Desktop
# shares). A sandbox's folder is made before anything it runs, so a supervisor that gives a run a folder of its own
# can clean up after killing it: terminate each id local_sandbox_ids() lists, then remove its folder.
# AGENT_ENV_LOCAL_SANDBOX_DIR=~/.agent-env-sandboxes

# Secret store backend (what secret: references in config resolve through):
# - "local" (or unset): process env vars, or a flat YAML/JSON file via [stores.secret] in config.toml
# - a hosted secret manager (AWS Secrets Manager, Google Cloud Secret Manager): configure a [stores.secret] table (no built-in coordinates)
Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/bundle/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,8 @@ def run_bundle(
Each run's sandboxes are torn down as it ends, so the contexts returned record sandboxes that are gone.
``keep`` leaves them up instead, for the caller to use and then remove with ``BundleRun.teardown()``.
Ctrl-C or SIGTERM cancels the runs and tears down every one, kept or not, then raises ``RunInterrupted``
with the result.
with the result. A process killed outright tears down nothing: a supervisor that may have to kill it gives it an
``AGENT_ENV_LOCAL_SANDBOX_DIR`` of its own, then terminates what ``local_sandbox_ids`` finds there.

It runs its own event loop, and turns namespace routing on for the whole process while it runs, so it
suits the CLI and scripts, not a service. From async code, call it in a thread; there it can't take over
Expand Down
2 changes: 1 addition & 1 deletion src/agent_env/computers/disk/linux/launcher/launch
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ for port in ${VM_HOST_PORTS:-}; do
fronts+=("$!")
done
trap 'kill ${fronts[@]+"${fronts[@]}"} ${presser:-} 2>/dev/null || true' EXIT
trap 'kill "$(cat "$control/qemu.pid" 2>/dev/null)" 2>/dev/null; exit 143' TERM INT
trap 'kill "$(cat "$control/qemu.pid" 2>/dev/null)" 2>/dev/null || true; exit 143' TERM INT
if [[ "$disk" == *.zst ]]; then
zstd -q -d -f "$disk" -o "$state/disk.img"
disk=$state/disk.img
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -637,6 +637,21 @@ def _copy_into_container(source: Path, container: str, destination: str) -> None
raise RuntimeError(f"Could not copy {source} into container {container}: {copied.stderr.strip()}")


def local_sandbox_ids() -> list[str]:
"""The ids of the local sandboxes with a work folder under the sandbox root. A folder is made before anything its
sandbox runs, so a supervisor that gave a run its own ``AGENT_ENV_LOCAL_SANDBOX_DIR`` cleans up after killing it by
terminating each id here (``LocalSandboxProvider().terminate_sandbox``) and removing its folder."""
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
root = _local_sandbox_root_path()
if not root.is_dir():
return []
prefix, found = "agent-env-", set()
for entry in root.iterdir():
sandbox_id, _, suffix = entry.name.removeprefix(prefix).rpartition("-")
if entry.name.startswith(prefix) and sandbox_id and suffix and entry.is_dir() and not entry.is_symlink():
found.add(sandbox_id)
return sorted(found)


def remove_local_work_dir(sandbox_id: str) -> Path | None:
"""Delete a local sandbox's work folder, and return it, when it lies directly under the sandbox root and
isn't a symlink; anything else, a legacy folder under the temp dir included, is left alone. Raises OSError
Expand Down Expand Up @@ -989,8 +1004,10 @@ def _found(sandbox_id: str) -> LocalSandbox:
work_dir = LocalSandbox.find_work_dir(sandbox_id)
if work_dir is None:
raise RuntimeError(f"Local sandbox work directory not found for sandbox_id={sandbox_id!r}")
port_map = work_dir / _PORT_MAP_FILE
ports = {int(c): h for c, h in json.loads(port_map.read_text()).items()} if port_map.exists() else None
try:
ports = {int(c): h for c, h in json.loads((work_dir / _PORT_MAP_FILE).read_text()).items()}
except (OSError, ValueError): # none written, or cut short by a run killed while writing it
ports = None
return LocalSandbox(sandbox_id=sandbox_id, work_dir=work_dir, port_map=ports)


Expand Down
46 changes: 43 additions & 3 deletions tst/integration/cli/run_bundle_test.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""``agent-env run`` on the hello bundle agent-env ships, with no infra: the local stores, a local VM-mode
sandbox, and a verifier scoring the files the task loaded into it. Needs `bash` and the usual shell tools on PATH."""

import asyncio
import importlib
import json
import logging
Expand All @@ -26,6 +27,11 @@
from agent_env.cli import cli
from agent_env.config import configure, reset_config
from agent_env.config.paths import state_root
from agent_env.providers.sandbox_providers.local_docker.provider import (
LocalSandboxProvider,
local_sandbox_ids,
remove_local_work_dir,
)
from agent_env.providers.sandbox_providers.sandbox_provider import reset_sandbox_provider
from agent_env.store import Filter, LocalSqliteDocumentStore
from agent_env.store.routing import namespace_routing
Expand Down Expand Up @@ -198,10 +204,10 @@ def test_an_installed_bundle_runs_by_name_with_ids_rooted_at_its_package(bundle,
assert "demo_installed_bundles" not in sys.modules


def _signalled(args, when, signum=signal.SIGINT, group=True):
def _signalled(args, when, signum=signal.SIGINT, group=True, ready=lambda: True):
"""Run ``agent-env run ARGS`` in its own process group, as a terminal would, and send ``signum`` to the group
(or to the process alone) once a line containing ``when`` is printed. Returns the exit code, the output, and
what the sandbox root held when the signal went."""
(or to the process alone) once a line containing ``when`` is printed and ``ready()`` holds. Returns the exit code,
the output, and what the sandbox root held when the signal went."""
process = subprocess.Popen([sys.executable, "-m", "agent_env.cli", "run", *map(str, args)], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True, start_new_session=True)
watchdog = threading.Timer(120, process.kill)
Expand All @@ -211,6 +217,9 @@ def _signalled(args, when, signum=signal.SIGINT, group=True):
for line in process.stdout:
lines.append(line)
if held is None and when in line:
deadline = time.monotonic() + 30
while not ready() and time.monotonic() < deadline:
time.sleep(0.05)
Comment on lines +220 to +222

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Readiness timings lack names

The new readiness loop uses inline 30 and 0.05 values. The repository requires descriptive names for magic numbers. Name the readiness timeout and polling interval so their purpose and units are clear. This repository requirement must be satisfied before merging.

Rule Used: Store magic numbers as class or instance variables with descriptive names rather than using them inline in the code. (source)

Learned From
scaleapi/scaleapi#126388

Prompt To Fix With AI
This is a comment left during a code review.
Path: tst/integration/cli/run_bundle_test.py
Line: 220-222

Comment:
**Readiness timings lack names**

The new readiness loop uses inline `30` and `0.05` values. The repository requires descriptive names for magic numbers. Name the readiness timeout and polling interval so their purpose and units are clear. This repository requirement must be satisfied before merging.

**Rule Used:** Store magic numbers as class or instance variables with descriptive names rather than using them inline in the code. ([source](https://app.greptile.com/-/switch-tenant?tenant=scale-ai&next=%2Fcustom-context%3Fmemory%3D002e0051-41ad-46c1-9098-47433c580150))

**Learned From**
[scaleapi/scaleapi#126388](https://github.com/scaleapi/scaleapi/pull/126388)

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Fix in Cursor Fix in Claude Code Fix in Codex

root = os.environ["AGENT_ENV_LOCAL_SANDBOX_DIR"]
held = sorted(os.listdir(root)) if os.path.isdir(root) else []
(os.killpg if group else os.kill)(process.pid, signum)
Expand Down Expand Up @@ -244,6 +253,37 @@ def test_a_signal_mid_run_marks_the_run_cancelled_tears_it_down_and_stops_its_co
assert not _alive(int((tmp_path / "nap.pid").read_text()))


def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervisor_to_terminate(bundle, tmp_path):
napping = tmp_path / "napping"
(napping / "tasks").mkdir(parents=True)
(napping / "tasks/nap.json").write_text(json.dumps([
{"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm", "sandbox_type": "local"},
{"id": "web", "type": "deploy_sandbox", "sandbox_name": "web", "sandbox_mode": "container",
"sandbox_type": "local", "image": "public.ecr.aws/nginx/nginx:stable-alpine", "port": 80},
{"id": "nap", "type": "verify_sandbox", "sandbox_name": "box", "base_dir": "/app", "verifier_id": "nap",
"depends_on": ["box", "web"], "criteria": [{"type": "bash_cmd_succeeds", "criterion": "naps",
"bash_cmd": 'sleep 300 & echo $! > "$HOME/nap.pid"; wait'}]},
]))

nap = tmp_path / "nap.pid"
exit_code, output, held = _signalled([napping], "step 3/3 nap (verify_sandbox)", signal.SIGKILL, group=False,
ready=nap.exists)
found = local_sandbox_ids()
for sandbox_id in found:
asyncio.run(LocalSandboxProvider().terminate_sandbox(sandbox_id))
remove_local_work_dir(sandbox_id)

assert exit_code == -signal.SIGKILL, output
assert len(held) == len(found) == 2
assert local_sandbox_ids() == []
assert not _alive(int(nap.read_text()))
for sandbox_id in found:
for selector in (f"label=agentenv.sandbox={sandbox_id}", f"name=^agent-{sandbox_id}$"):
listed = subprocess.run(["docker", "ps", "-aq", "--filter", selector], capture_output=True, text=True,
check=True)
assert listed.stdout == "", selector


def _alive(pid):
for _ in range(100):
try:
Expand Down
10 changes: 10 additions & 0 deletions tst/unit/env/envs/test_disk_recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,16 @@ def test_the_launcher_forwards_each_host_port_from_loopback_here_to_the_host_and
assert running.wait(10) == 143


def test_a_launcher_stopped_before_its_computer_starts_exits_as_one_stopped_after_does(launcher):
launch, calls = launcher

running = launch(VM_DISK="/disk/disk.img.zst", ZSTD_TAKES_S="1")
_logged(calls, "zstd")
running.terminate()

assert running.wait(10) == 143


@pytest.mark.parametrize("host_ports", ["18765", "28765"], ids=["a-guest-ports-proxy", "its-forward"])
def test_the_launcher_refuses_a_host_port_a_guest_ports_proxy_or_forward_takes(launcher, host_ports):
launch, calls = launcher
Expand Down
17 changes: 16 additions & 1 deletion tst/unit/providers/sandbox_providers/local_sandbox_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import agent_env.providers.sandbox_providers.local_docker.provider as ls
from agent_env.a2a_agent.a2a_agent import A2AAgent
from agent_env.config import reset_config, set_object_store
from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox, LocalSandboxProvider
from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox, LocalSandboxProvider, local_sandbox_ids
from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy
from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_CONTAINER, SANDBOX_MODE_VM
from agent_env.store import LocalFilesystemObjectStore
Expand Down Expand Up @@ -116,6 +116,21 @@ def test_find_work_dir_only_returns_matching_sandbox_id(tmp_path: Path, monkeypa

assert LocalSandbox.find_work_dir("local-requested") == requested
assert LocalSandbox.find_work_dir("local-missing") is None
(tmp_path / "agent-env-notes.txt").write_text("")
(tmp_path / "agent-env-notes").mkdir()
assert local_sandbox_ids() == ["local-other", "local-requested"]


@pytest.mark.asyncio
async def test_a_sandbox_killed_while_writing_its_port_map_is_still_rebuilt_for_teardown(tmp_path: Path, monkeypatch):
monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(tmp_path))
work_dir = tmp_path / "agent-env-local-cut-abc123"
work_dir.mkdir()
(work_dir / ".port-map.json").write_text("")

sandbox = await LocalSandboxProvider().get_sandbox("local-cut")

assert (sandbox.sandbox_id, sandbox.work_dir) == ("local-cut", work_dir)


def test_find_work_dir_falls_back_to_legacy_tempdir(tmp_path: Path, monkeypatch):
Expand Down
Loading