From 8c9db0cdd0c024889af44bf3679484ea2c087d1e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 14 Nov 2026 21:38:32 -0800 Subject: [PATCH 1/5] test(run_bundle): the killed-run test's container is of an image Docker holds, not one pulled from ECR Public test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervisor_to_terminate deployed its container sandbox from public.ecr.aws/nginx/nginx:stable-alpine, which the local sandbox pulled anonymously at test time, and failed with "toomanyrequests: Rate exceeded" whenever ECR Public rate-limited the runner (integration-local, runs 43313173134, 43369378729, 43472071259, 43538837017). It now deploys the registry image the integration tests' throwaway registries run (REGISTRY_IMAGE), which the workflow pulls before the tests, under its digest-pinned RepoDigest: a local sandbox doesn't pull an image its Docker holds under a name pinned to a digest, so the container starts with no registry at all. A machine that doesn't hold the image pulls it once from mirror.gcr.io. Co-Authored-By: Claude Opus 5.5 --- tst/integration/cli/run_bundle_test.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index 9a664d361..6df3a8a36 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -27,6 +27,7 @@ from agent_env.cli import cli from agent_env.config import configure, reset_config from agent_env.config.paths import state_root +from agent_env.providers.sandbox_providers.host_images import present from agent_env.providers.sandbox_providers.local_docker.provider import ( LocalSandboxProvider, local_sandbox_ids, @@ -36,6 +37,7 @@ from agent_env.store import Filter, LocalSqliteDocumentStore from agent_env.store.routing import namespace_routing from agent_env.task import Task +from tst.integration.utils.registry import REGISTRY_IMAGE from tst.util.capabilities import skip_without_gnu_stat pytestmark = pytest.mark.integration @@ -254,13 +256,26 @@ def test_a_signal_mid_run_marks_the_run_cancelled_tears_it_down_and_stops_its_co assert not _alive(int(nap.read_text())) -def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervisor_to_terminate(bundle, tmp_path): +@pytest.fixture(scope="module") +def held_image(): + """The image the tests' throwaway registries run, which the workflow pulls before the tests, under a name pinned to + its digest, so a local sandbox starts a container of it pulling nothing: a pull from a public registry at the test's + time failed whenever ECR Public refused the runner's anonymous pulls.""" + if not present(REGISTRY_IMAGE): + subprocess.run(["docker", "pull", "--quiet", REGISTRY_IMAGE], check=True, capture_output=True) + inspected = subprocess.run(["docker", "image", "inspect", "--format", "{{json .RepoDigests}}", REGISTRY_IMAGE], + check=True, capture_output=True, text=True) + return json.loads(inspected.stdout)[0] + + +def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervisor_to_terminate(bundle, tmp_path, + held_image): napping = tmp_path / "napping" (napping / "tasks").mkdir(parents=True) (napping / "tasks/nap.json").write_text(json.dumps([ {"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm", "sandbox_type": "local"}, {"id": "web", "type": "deploy_sandbox", "sandbox_name": "web", "sandbox_mode": "container", - "sandbox_type": "local", "image": "public.ecr.aws/nginx/nginx:stable-alpine", "port": 80}, + "sandbox_type": "local", "image": held_image, "port": 5000}, {"id": "nap", "type": "verify_sandbox", "sandbox_name": "box", "base_dir": "/app", "verifier_id": "nap", "depends_on": ["box", "web"], "criteria": [{"type": "bash_cmd_succeeds", "criterion": "naps", "bash_cmd": 'sleep 300 & echo $! > "$HOME/nap.pid"; wait'}]}, From c4c5aee56a1e6f3fb4572ec0de26b98f6cfc5578 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 14 Nov 2026 21:38:32 -0800 Subject: [PATCH 2/5] fix(computers): a compose's pulls of its base take turns, so one arch's platform check can't break the other's resolve A compose builds its pairs at once. On this machine's Docker, container/amd64's resolve creates a container of the base (docker create, which pulls it when Docker lacks it) while container/arm64's platform check runs a container of the same base for linux/arm64 (docker run, which pulls too), and neither held the image-pull lock a deploy's pull holds. On Docker's containerd image store the two pulls raced: the amd64 one failed ("could not fetch content descriptor ... from remote: not found") and left the store holding the amd64 manifest without all its layers, so the next compose of the env on the runner failed at once with "failed to get reader from content store: content digest ...: not found" (integration-local-slow 5/12, run 43505816495, browser_gateway_test and clock_desktop_test's chromium compose). The resolver's docker create and the platform check's docker run now hold the image's pull lock (host_images.pull_lock, the one host_images.pull holds), so pulls of one image take turns across a compose's threads and across processes; the integration tier already shares image-pull: locks between its workers. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/builder.py | 19 ++++++---- .../sandbox_providers/host_images.py | 9 ++++- tst/unit/env/envs/test_computer_builder.py | 38 ++++++++++++++++++- 3 files changed, 56 insertions(+), 10 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index f133ca1b1..1a3fc1b0b 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -32,12 +32,13 @@ from agent_env.config import get_config from agent_env.env.envs.computer.utils import layers_in_image from agent_env.env.published_images import COMPUTER_IMAGES -from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag +from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag, pull_lock from agent_env.providers.sandbox_providers.sandbox import upload_vm_file from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM from agent_env.store.base import ObjectAlreadyExistsError from agent_env.store.image_store.oci_registry_credentials import ( is_loopback_host, names_registry, registry_host_from_ref) +from agent_env.store.local_state import holding_locks from agent_env.utils.docker.build import DockerBuildError, build_image if TYPE_CHECKING: @@ -243,10 +244,12 @@ def _docker_here(link: SandboxProvider) -> bool: @functools.cache def _runs_here(platform: str, image: str) -> bool: """Whether this machine's Docker runs ``platform``'s programs, natively or emulated, as a container of ``image`` for - it shows: what its builder lists can leave out a platform it runs, as colima's does Rosetta's linux/amd64.""" + it shows: what its builder lists can leave out a platform it runs, as colima's does Rosetta's linux/amd64. Its pull + of ``image`` takes turns with the others (``pull_lock``), as a compose's other arch resolves on the same base.""" try: - return subprocess.run(["docker", "run", "--rm", "--platform", platform, "--entrypoint", "true", image], - capture_output=True).returncode == 0 + with holding_locks([pull_lock(image)]): + return subprocess.run(["docker", "run", "--rm", "--platform", platform, "--entrypoint", "true", image], + capture_output=True).returncode == 0 except FileNotFoundError: return False @@ -285,15 +288,17 @@ def __init__(self, name: str, work: Path, base: DockerImageArtifact, platform: s def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: """With the build context and what the resolver reads copied into its container, and what it wrote copied out, rather than bind-mounted: a Docker in a VM, as on a Mac, mounts only the folders the VM shares, colima's only the - home, and keeps no owner on them.""" + home, and keeps no owner on them. The container's pull of ``source`` takes turns with the others + (``pull_lock``).""" out = self.work / RESOLVED / "out" out.mkdir(parents=True) (out / "request.json").write_text(json.dumps({**request, "owner": [os.getuid(), os.getgid()]})) shutil.copy(RESOLVER, out / "resolve.py") container, deadline = f"agentenv-resolve-{uuid.uuid4().hex[:12]}", time.monotonic() + timeout try: - _docker("create", "--name", container, "--platform", f"linux/{arch}", "--user", "root", "--entrypoint", - "python3", source, "/out/resolve.py", "/out/request.json", "/out", deadline=deadline) + with holding_locks([pull_lock(source)]): + _docker("create", "--name", container, "--platform", f"linux/{arch}", "--user", "root", "--entrypoint", + "python3", source, "/out/resolve.py", "/out/request.json", "/out", deadline=deadline) _docker("cp", f"{self.work / CONTEXT}/.", f"{container}:/ctx", deadline=deadline) _docker("cp", f"{out}/.", f"{container}:/out", deadline=deadline) ran = subprocess.run(["docker", "start", "--attach", container], capture_output=True, text=True, diff --git a/src/agent_env/providers/sandbox_providers/host_images.py b/src/agent_env/providers/sandbox_providers/host_images.py index 16f50d1f8..55b4019fc 100644 --- a/src/agent_env/providers/sandbox_providers/host_images.py +++ b/src/agent_env/providers/sandbox_providers/host_images.py @@ -137,11 +137,18 @@ async def build_here(image: DockerImageArtifact) -> None: await asyncio.to_thread(build, image) +def pull_lock(image_name: str) -> str: + """The lock a pull of ``image_name`` on this machine holds, ``docker pull``'s or that of a container of an image + Docker lacks: two at once, even of two of its platforms, can leave Docker's containerd image store holding a + platform's manifest without all its layers, and creating a container of it then fails.""" + return f"image-pull:{image_name}" + + def pull(image: DockerImageArtifact) -> None: """Have this machine's Docker hold ``image``, one with an ``image_digest``, under its name, unless it already holds it (``holds_digest``): pulled by its digest, or, when that fails, built from its build context. Raises RuntimeError when it can't be pulled or built.""" - with holding_locks([f"image-pull:{image.pinned_name}"]): + with holding_locks([pull_lock(image.pinned_name)]): if holds_digest(image): return pulled = subprocess.run(["docker", "pull", image.pinned_name], capture_output=True, text=True, errors="replace") diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index 2a4f577a4..b984e3ac0 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -1,7 +1,7 @@ """Where and how a computer env's pairs build: this machine's Docker, natively or emulated, or a VM of the put's sandbox provider, each arch on the first link of a chain that runs it natively; what builds when emulation is off -or no link can build; a VM the provider loses; the VM's shell helper; and what a put says as it goes. Docker, -the registry and the VMs are faked.""" +or no link can build; a VM the provider loses; pulls of a base taking turns; the VM's shell helper; and what a put says +as it goes. Docker, the registry and the VMs are faked.""" import hashlib import io @@ -13,6 +13,7 @@ import sys import tarfile import time +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from types import SimpleNamespace @@ -25,6 +26,7 @@ from agent_env.env import Env from agent_env.env.envs.computer.utils import builder, composer, layers_in_image, packages from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.providers.sandbox_providers import host_images from agent_env.providers.sandbox_providers.chained_sandbox_provider import ChainedSandboxProvider from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider @@ -164,6 +166,38 @@ def docker(command, **kwargs): assert failed.value.log == "Pulling from acme/base" +def test_a_resolves_pull_of_its_base_takes_turns_with_a_platform_checks_and_a_deploys(tmp_path, monkeypatch): + """A compose resolves one arch in a container of the base as it checks the other's with another, each pulled when + Docker lacks the base, and two pulls of one image at once left Docker's containerd image store without layers of + it, which every container of it then failed on.""" + base, digest = "ghcr.io/acme/base:1", "sha256:" + "b" * 64 + pulling, ran, most = [], [], [] + + def docker(command, **kwargs): + if f"{base}@{digest}" in command and command[1] in ("create", "run", "pull"): + pulling.append(command[1]) + most.append(len(pulling)) + time.sleep(0.2) + pulling.remove(command[1]) + ran.append(command[1]) + if command[1] == "cp" and command[3] == str(tmp_path / "work" / builder.RESOLVED / "out"): + (Path(command[3]) / "result.json").write_text(json.dumps({"apt": [], "wheels": {}})) + return subprocess.CompletedProcess(command, 0, "", "") + + _, local = _resolving(tmp_path, monkeypatch, docker) + deployed = DockerImageArtifact(id="base-image", version=1, description="", image_name=base, image_digest=digest) + builder._runs_here.cache_clear() + with ThreadPoolExecutor(max_workers=3) as pool: + calls = [pool.submit(local.resolve, {"apt": [], "python": []}, f"{base}@{digest}", "amd64", 60), + pool.submit(builder._runs_here, "linux/arm64", f"{base}@{digest}"), + pool.submit(host_images.pull, deployed)] + results = [call.result() for call in calls] + builder._runs_here.cache_clear() + + assert results == [{"apt": [], "wheels": {}}, True, None] + assert sorted(ran) == ["create", "pull", "run"] and max(most) == 1 + + def _resolving(tmp_path, monkeypatch, docker): """A builder of this machine's Docker in ``tmp_path``, its build context a driver's, ``docker`` its subprocess.run.""" work = tmp_path / "work" From 17451dc5e2a6cc18715438df8d19c1411de4fb65 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 14 Nov 2026 22:03:38 -0800 Subject: [PATCH 3/5] fix(computers): a resolve's wait for its turn to pull its base counts toward its timeout Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/builder.py | 6 +++-- src/agent_env/store/local_state.py | 22 ++++++++++++++++--- tst/unit/env/envs/test_computer_builder.py | 17 ++++++++++++++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 1a3fc1b0b..9162a6b84 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -289,14 +289,14 @@ def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: """With the build context and what the resolver reads copied into its container, and what it wrote copied out, rather than bind-mounted: a Docker in a VM, as on a Mac, mounts only the folders the VM shares, colima's only the home, and keeps no owner on them. The container's pull of ``source`` takes turns with the others - (``pull_lock``).""" + (``pull_lock``), its wait for its turn within ``timeout`` too.""" out = self.work / RESOLVED / "out" out.mkdir(parents=True) (out / "request.json").write_text(json.dumps({**request, "owner": [os.getuid(), os.getgid()]})) shutil.copy(RESOLVER, out / "resolve.py") container, deadline = f"agentenv-resolve-{uuid.uuid4().hex[:12]}", time.monotonic() + timeout try: - with holding_locks([pull_lock(source)]): + with holding_locks([pull_lock(source)], deadline=deadline): _docker("create", "--name", container, "--platform", f"linux/{arch}", "--user", "root", "--entrypoint", "python3", source, "/out/resolve.py", "/out/request.json", "/out", deadline=deadline) _docker("cp", f"{self.work / CONTEXT}/.", f"{container}:/ctx", deadline=deadline) @@ -308,6 +308,8 @@ def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: _docker("cp", f"{container}:/out/.", str(out), deadline=deadline) except subprocess.TimeoutExpired as e: raise ResolveError(f"resolving took over {timeout} s", str(e.stdout or "")) from None + except TimeoutError: + raise ResolveError(f"resolving took over {timeout} s, waiting for another pull of {source}") from None finally: subprocess.run(["docker", "rm", "--force", container], capture_output=True) return json.loads((out / "result.json").read_text()) diff --git a/src/agent_env/store/local_state.py b/src/agent_env/store/local_state.py index e443083ba..05300b4d0 100644 --- a/src/agent_env/store/local_state.py +++ b/src/agent_env/store/local_state.py @@ -45,10 +45,12 @@ def ensure_state_dir(path: Path) -> None: @contextmanager -def holding_locks(ids: Iterable[str], on_wait: Callable[[], None] | None = None) -> Iterator[None]: +def holding_locks(ids: Iterable[str], on_wait: Callable[[], None] | None = None, + deadline: float | None = None) -> Iterator[None]: """Hold a lock on each of ``ids`` in the per-user state root, so another process writing any of them waits here, calling ``on_wait`` first. The locks are taken in one order, so two holders can't each hold one the - other waits for.""" + other waits for. With a ``deadline`` (``time.monotonic()``), a held lock is tried again every ``_LOCK_POLL_S`` + until then, and TimeoutError raised after.""" if fcntl is None: yield return @@ -65,13 +67,27 @@ def holding_locks(ids: Iterable[str], on_wait: Callable[[], None] | None = None) if on_wait is not None and not waited: on_wait() waited = True - fcntl.flock(fds[-1], fcntl.LOCK_EX) + if deadline is None: + fcntl.flock(fds[-1], fcntl.LOCK_EX) + else: + _lock_by(fds[-1], deadline) yield finally: for fd in fds: os.close(fd) +def _lock_by(fd: int, deadline: float) -> None: + while True: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return + except BlockingIOError: + if time.monotonic() >= deadline: + raise TimeoutError("another holder kept the lock past the deadline") from None + time.sleep(min(_LOCK_POLL_S, max(deadline - time.monotonic(), 0))) + + @asynccontextmanager async def awaiting_lock(id: str, waiting: Callable[[float], Awaitable[None]] | None = None, every_s: float | None = None) -> AsyncIterator[None]: diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index b984e3ac0..ea5f88190 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -27,6 +27,7 @@ from agent_env.env.envs.computer.utils import builder, composer, layers_in_image, packages from agent_env.env.envs.computer.utils.composer import Composing from agent_env.providers.sandbox_providers import host_images +from agent_env.store.local_state import holding_locks from agent_env.providers.sandbox_providers.chained_sandbox_provider import ChainedSandboxProvider from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider @@ -198,6 +199,22 @@ def docker(command, **kwargs): assert sorted(ran) == ["create", "pull", "run"] and max(most) == 1 +def test_a_resolve_waiting_for_another_pull_of_its_base_gives_up_at_its_own_timeout(tmp_path, monkeypatch): + ran = [] + + def docker(command, **kwargs): + ran.append(command[1]) + return subprocess.CompletedProcess(command, 0, "", "") + + _, local = _resolving(tmp_path, monkeypatch, docker) + started = time.monotonic() + with holding_locks([host_images.pull_lock("base")]), \ + pytest.raises(builder.ResolveError, match="resolving took over 1 s, waiting for another pull of base"): + local.resolve({"apt": [], "python": []}, "base", "amd64", 1) + + assert 1 <= time.monotonic() - started < 5 and ran == ["rm"] + + def _resolving(tmp_path, monkeypatch, docker): """A builder of this machine's Docker in ``tmp_path``, its build context a driver's, ``docker`` its subprocess.run.""" work = tmp_path / "work" From aa7eca7167219712ddefe2d3d4c93ac60fb85fd5 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 14 Nov 2026 22:11:12 -0800 Subject: [PATCH 4/5] test(computers): the pull-turns test names its fake pull's duration Co-Authored-By: Claude Opus 5.5 --- tst/unit/env/envs/test_computer_builder.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index ea5f88190..f8f3475c0 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -167,6 +167,10 @@ def docker(command, **kwargs): assert failed.value.log == "Pulling from acme/base" +#: How long a fake pull of the base takes: long enough that pulls not taking turns would overlap. +PULL_S = 0.2 + + def test_a_resolves_pull_of_its_base_takes_turns_with_a_platform_checks_and_a_deploys(tmp_path, monkeypatch): """A compose resolves one arch in a container of the base as it checks the other's with another, each pulled when Docker lacks the base, and two pulls of one image at once left Docker's containerd image store without layers of @@ -178,7 +182,7 @@ def docker(command, **kwargs): if f"{base}@{digest}" in command and command[1] in ("create", "run", "pull"): pulling.append(command[1]) most.append(len(pulling)) - time.sleep(0.2) + time.sleep(PULL_S) pulling.remove(command[1]) ran.append(command[1]) if command[1] == "cp" and command[3] == str(tmp_path / "work" / builder.RESOLVED / "out"): From 12147897a6a4e98ab4f551d48ae65a12d00265e7 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 14 Nov 2026 22:14:30 -0800 Subject: [PATCH 5/5] test(gateway): get_time's test sets the clock still, so a slow runner's second doesn't move it Co-Authored-By: Claude Opus 5.5 --- tst/unit/env/gateway/user_chat_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tst/unit/env/gateway/user_chat_test.py b/tst/unit/env/gateway/user_chat_test.py index c286e9021..115be6609 100644 --- a/tst/unit/env/gateway/user_chat_test.py +++ b/tst/unit/env/gateway/user_chat_test.py @@ -154,7 +154,8 @@ def test_get_time_reaches_a_configured_role_through_a_deny_all_while_the_clock_i _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]}) assert "get_time" not in _names(gateway, "psi_agent") # no clock set, so no get_time at all - assert _req(gateway, "PUT", "/clock/set-time", {"virtual_time": T0}).status_code == 200 + assert _req(gateway, "PUT", "/clock/set-time", {"virtual_time": T0, "virtual_seconds_per_real_second": 0} + ).status_code == 200 assert "get_time" in _names(gateway, "psi_agent") and "get_time" not in _names(gateway, "other") called = _call(gateway, "get_time", {}, "psi_agent") assert called.status_code == 200 and T0 in called.text