diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index f133ca1b1..9162a6b84 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -32,12 +32,13 @@ from agent_env.config import get_config from agent_env.env.envs.computer.utils import layers_in_image from agent_env.env.published_images import COMPUTER_IMAGES -from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag +from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag, pull_lock from agent_env.providers.sandbox_providers.sandbox import upload_vm_file from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM from agent_env.store.base import ObjectAlreadyExistsError from agent_env.store.image_store.oci_registry_credentials import ( is_loopback_host, names_registry, registry_host_from_ref) +from agent_env.store.local_state import holding_locks from agent_env.utils.docker.build import DockerBuildError, build_image if TYPE_CHECKING: @@ -243,10 +244,12 @@ def _docker_here(link: SandboxProvider) -> bool: @functools.cache def _runs_here(platform: str, image: str) -> bool: """Whether this machine's Docker runs ``platform``'s programs, natively or emulated, as a container of ``image`` for - it shows: what its builder lists can leave out a platform it runs, as colima's does Rosetta's linux/amd64.""" + it shows: what its builder lists can leave out a platform it runs, as colima's does Rosetta's linux/amd64. Its pull + of ``image`` takes turns with the others (``pull_lock``), as a compose's other arch resolves on the same base.""" try: - return subprocess.run(["docker", "run", "--rm", "--platform", platform, "--entrypoint", "true", image], - capture_output=True).returncode == 0 + with holding_locks([pull_lock(image)]): + return subprocess.run(["docker", "run", "--rm", "--platform", platform, "--entrypoint", "true", image], + capture_output=True).returncode == 0 except FileNotFoundError: return False @@ -285,15 +288,17 @@ def __init__(self, name: str, work: Path, base: DockerImageArtifact, platform: s def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: """With the build context and what the resolver reads copied into its container, and what it wrote copied out, rather than bind-mounted: a Docker in a VM, as on a Mac, mounts only the folders the VM shares, colima's only the - home, and keeps no owner on them.""" + home, and keeps no owner on them. The container's pull of ``source`` takes turns with the others + (``pull_lock``), its wait for its turn within ``timeout`` too.""" out = self.work / RESOLVED / "out" out.mkdir(parents=True) (out / "request.json").write_text(json.dumps({**request, "owner": [os.getuid(), os.getgid()]})) shutil.copy(RESOLVER, out / "resolve.py") container, deadline = f"agentenv-resolve-{uuid.uuid4().hex[:12]}", time.monotonic() + timeout try: - _docker("create", "--name", container, "--platform", f"linux/{arch}", "--user", "root", "--entrypoint", - "python3", source, "/out/resolve.py", "/out/request.json", "/out", deadline=deadline) + with holding_locks([pull_lock(source)], deadline=deadline): + _docker("create", "--name", container, "--platform", f"linux/{arch}", "--user", "root", "--entrypoint", + "python3", source, "/out/resolve.py", "/out/request.json", "/out", deadline=deadline) _docker("cp", f"{self.work / CONTEXT}/.", f"{container}:/ctx", deadline=deadline) _docker("cp", f"{out}/.", f"{container}:/out", deadline=deadline) ran = subprocess.run(["docker", "start", "--attach", container], capture_output=True, text=True, @@ -303,6 +308,8 @@ def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: _docker("cp", f"{container}:/out/.", str(out), deadline=deadline) except subprocess.TimeoutExpired as e: raise ResolveError(f"resolving took over {timeout} s", str(e.stdout or "")) from None + except TimeoutError: + raise ResolveError(f"resolving took over {timeout} s, waiting for another pull of {source}") from None finally: subprocess.run(["docker", "rm", "--force", container], capture_output=True) return json.loads((out / "result.json").read_text()) diff --git a/src/agent_env/providers/sandbox_providers/host_images.py b/src/agent_env/providers/sandbox_providers/host_images.py index 16f50d1f8..55b4019fc 100644 --- a/src/agent_env/providers/sandbox_providers/host_images.py +++ b/src/agent_env/providers/sandbox_providers/host_images.py @@ -137,11 +137,18 @@ async def build_here(image: DockerImageArtifact) -> None: await asyncio.to_thread(build, image) +def pull_lock(image_name: str) -> str: + """The lock a pull of ``image_name`` on this machine holds, ``docker pull``'s or that of a container of an image + Docker lacks: two at once, even of two of its platforms, can leave Docker's containerd image store holding a + platform's manifest without all its layers, and creating a container of it then fails.""" + return f"image-pull:{image_name}" + + def pull(image: DockerImageArtifact) -> None: """Have this machine's Docker hold ``image``, one with an ``image_digest``, under its name, unless it already holds it (``holds_digest``): pulled by its digest, or, when that fails, built from its build context. Raises RuntimeError when it can't be pulled or built.""" - with holding_locks([f"image-pull:{image.pinned_name}"]): + with holding_locks([pull_lock(image.pinned_name)]): if holds_digest(image): return pulled = subprocess.run(["docker", "pull", image.pinned_name], capture_output=True, text=True, errors="replace") diff --git a/src/agent_env/store/local_state.py b/src/agent_env/store/local_state.py index e443083ba..05300b4d0 100644 --- a/src/agent_env/store/local_state.py +++ b/src/agent_env/store/local_state.py @@ -45,10 +45,12 @@ def ensure_state_dir(path: Path) -> None: @contextmanager -def holding_locks(ids: Iterable[str], on_wait: Callable[[], None] | None = None) -> Iterator[None]: +def holding_locks(ids: Iterable[str], on_wait: Callable[[], None] | None = None, + deadline: float | None = None) -> Iterator[None]: """Hold a lock on each of ``ids`` in the per-user state root, so another process writing any of them waits here, calling ``on_wait`` first. The locks are taken in one order, so two holders can't each hold one the - other waits for.""" + other waits for. With a ``deadline`` (``time.monotonic()``), a held lock is tried again every ``_LOCK_POLL_S`` + until then, and TimeoutError raised after.""" if fcntl is None: yield return @@ -65,13 +67,27 @@ def holding_locks(ids: Iterable[str], on_wait: Callable[[], None] | None = None) if on_wait is not None and not waited: on_wait() waited = True - fcntl.flock(fds[-1], fcntl.LOCK_EX) + if deadline is None: + fcntl.flock(fds[-1], fcntl.LOCK_EX) + else: + _lock_by(fds[-1], deadline) yield finally: for fd in fds: os.close(fd) +def _lock_by(fd: int, deadline: float) -> None: + while True: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return + except BlockingIOError: + if time.monotonic() >= deadline: + raise TimeoutError("another holder kept the lock past the deadline") from None + time.sleep(min(_LOCK_POLL_S, max(deadline - time.monotonic(), 0))) + + @asynccontextmanager async def awaiting_lock(id: str, waiting: Callable[[float], Awaitable[None]] | None = None, every_s: float | None = None) -> AsyncIterator[None]: diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index 9a664d361..6df3a8a36 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -27,6 +27,7 @@ from agent_env.cli import cli from agent_env.config import configure, reset_config from agent_env.config.paths import state_root +from agent_env.providers.sandbox_providers.host_images import present from agent_env.providers.sandbox_providers.local_docker.provider import ( LocalSandboxProvider, local_sandbox_ids, @@ -36,6 +37,7 @@ from agent_env.store import Filter, LocalSqliteDocumentStore from agent_env.store.routing import namespace_routing from agent_env.task import Task +from tst.integration.utils.registry import REGISTRY_IMAGE from tst.util.capabilities import skip_without_gnu_stat pytestmark = pytest.mark.integration @@ -254,13 +256,26 @@ def test_a_signal_mid_run_marks_the_run_cancelled_tears_it_down_and_stops_its_co assert not _alive(int(nap.read_text())) -def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervisor_to_terminate(bundle, tmp_path): +@pytest.fixture(scope="module") +def held_image(): + """The image the tests' throwaway registries run, which the workflow pulls before the tests, under a name pinned to + its digest, so a local sandbox starts a container of it pulling nothing: a pull from a public registry at the test's + time failed whenever ECR Public refused the runner's anonymous pulls.""" + if not present(REGISTRY_IMAGE): + subprocess.run(["docker", "pull", "--quiet", REGISTRY_IMAGE], check=True, capture_output=True) + inspected = subprocess.run(["docker", "image", "inspect", "--format", "{{json .RepoDigests}}", REGISTRY_IMAGE], + check=True, capture_output=True, text=True) + return json.loads(inspected.stdout)[0] + + +def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervisor_to_terminate(bundle, tmp_path, + held_image): napping = tmp_path / "napping" (napping / "tasks").mkdir(parents=True) (napping / "tasks/nap.json").write_text(json.dumps([ {"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm", "sandbox_type": "local"}, {"id": "web", "type": "deploy_sandbox", "sandbox_name": "web", "sandbox_mode": "container", - "sandbox_type": "local", "image": "public.ecr.aws/nginx/nginx:stable-alpine", "port": 80}, + "sandbox_type": "local", "image": held_image, "port": 5000}, {"id": "nap", "type": "verify_sandbox", "sandbox_name": "box", "base_dir": "/app", "verifier_id": "nap", "depends_on": ["box", "web"], "criteria": [{"type": "bash_cmd_succeeds", "criterion": "naps", "bash_cmd": 'sleep 300 & echo $! > "$HOME/nap.pid"; wait'}]}, diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index 2a4f577a4..f8f3475c0 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -1,7 +1,7 @@ """Where and how a computer env's pairs build: this machine's Docker, natively or emulated, or a VM of the put's sandbox provider, each arch on the first link of a chain that runs it natively; what builds when emulation is off -or no link can build; a VM the provider loses; the VM's shell helper; and what a put says as it goes. Docker, -the registry and the VMs are faked.""" +or no link can build; a VM the provider loses; pulls of a base taking turns; the VM's shell helper; and what a put says +as it goes. Docker, the registry and the VMs are faked.""" import hashlib import io @@ -13,6 +13,7 @@ import sys import tarfile import time +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from types import SimpleNamespace @@ -25,6 +26,8 @@ from agent_env.env import Env from agent_env.env.envs.computer.utils import builder, composer, layers_in_image, packages from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.providers.sandbox_providers import host_images +from agent_env.store.local_state import holding_locks from agent_env.providers.sandbox_providers.chained_sandbox_provider import ChainedSandboxProvider from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider @@ -164,6 +167,58 @@ def docker(command, **kwargs): assert failed.value.log == "Pulling from acme/base" +#: How long a fake pull of the base takes: long enough that pulls not taking turns would overlap. +PULL_S = 0.2 + + +def test_a_resolves_pull_of_its_base_takes_turns_with_a_platform_checks_and_a_deploys(tmp_path, monkeypatch): + """A compose resolves one arch in a container of the base as it checks the other's with another, each pulled when + Docker lacks the base, and two pulls of one image at once left Docker's containerd image store without layers of + it, which every container of it then failed on.""" + base, digest = "ghcr.io/acme/base:1", "sha256:" + "b" * 64 + pulling, ran, most = [], [], [] + + def docker(command, **kwargs): + if f"{base}@{digest}" in command and command[1] in ("create", "run", "pull"): + pulling.append(command[1]) + most.append(len(pulling)) + time.sleep(PULL_S) + pulling.remove(command[1]) + ran.append(command[1]) + if command[1] == "cp" and command[3] == str(tmp_path / "work" / builder.RESOLVED / "out"): + (Path(command[3]) / "result.json").write_text(json.dumps({"apt": [], "wheels": {}})) + return subprocess.CompletedProcess(command, 0, "", "") + + _, local = _resolving(tmp_path, monkeypatch, docker) + deployed = DockerImageArtifact(id="base-image", version=1, description="", image_name=base, image_digest=digest) + builder._runs_here.cache_clear() + with ThreadPoolExecutor(max_workers=3) as pool: + calls = [pool.submit(local.resolve, {"apt": [], "python": []}, f"{base}@{digest}", "amd64", 60), + pool.submit(builder._runs_here, "linux/arm64", f"{base}@{digest}"), + pool.submit(host_images.pull, deployed)] + results = [call.result() for call in calls] + builder._runs_here.cache_clear() + + assert results == [{"apt": [], "wheels": {}}, True, None] + assert sorted(ran) == ["create", "pull", "run"] and max(most) == 1 + + +def test_a_resolve_waiting_for_another_pull_of_its_base_gives_up_at_its_own_timeout(tmp_path, monkeypatch): + ran = [] + + def docker(command, **kwargs): + ran.append(command[1]) + return subprocess.CompletedProcess(command, 0, "", "") + + _, local = _resolving(tmp_path, monkeypatch, docker) + started = time.monotonic() + with holding_locks([host_images.pull_lock("base")]), \ + pytest.raises(builder.ResolveError, match="resolving took over 1 s, waiting for another pull of base"): + local.resolve({"apt": [], "python": []}, "base", "amd64", 1) + + assert 1 <= time.monotonic() - started < 5 and ran == ["rm"] + + def _resolving(tmp_path, monkeypatch, docker): """A builder of this machine's Docker in ``tmp_path``, its build context a driver's, ``docker`` its subprocess.run.""" work = tmp_path / "work" diff --git a/tst/unit/env/gateway/user_chat_test.py b/tst/unit/env/gateway/user_chat_test.py index c286e9021..115be6609 100644 --- a/tst/unit/env/gateway/user_chat_test.py +++ b/tst/unit/env/gateway/user_chat_test.py @@ -154,7 +154,8 @@ def test_get_time_reaches_a_configured_role_through_a_deny_all_while_the_clock_i _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]}) assert "get_time" not in _names(gateway, "psi_agent") # no clock set, so no get_time at all - assert _req(gateway, "PUT", "/clock/set-time", {"virtual_time": T0}).status_code == 200 + assert _req(gateway, "PUT", "/clock/set-time", {"virtual_time": T0, "virtual_seconds_per_real_second": 0} + ).status_code == 200 assert "get_time" in _names(gateway, "psi_agent") and "get_time" not in _names(gateway, "other") called = _call(gateway, "get_time", {}, "psi_agent") assert called.status_code == 200 and T0 in called.text