diff --git a/.github/scripts/prune_registry.py b/.github/scripts/prune_registry.py deleted file mode 100644 index 0b2ace7f8..000000000 --- a/.github/scripts/prune_registry.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Keep only the images of a local registry's repository that a run used, before the registry's storage is cached. - -``prune`` drops each tag the registry's log has no manifest request for, then each manifest no remaining tag reaches, -following an image index to the manifests it lists. The registry's ``garbage-collect`` then frees the layers only those -manifests used; with ``--delete-untagged`` it would also delete a kept index's untagged platform manifests. ``links``, -run after it, drops the repository's links to layers it freed. -""" - -from __future__ import annotations - -import argparse -import json -import re -import shutil -import sys -from pathlib import Path - -_V2 = Path("docker") / "registry" / "v2" - - -def main(argv: list[str]) -> int: - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - pruning = commands.add_parser("prune", help="drop the tags and manifests the run didn't use") - linking = commands.add_parser("links", help="drop links to layers garbage-collect freed") - for command in (pruning, linking): - command.add_argument("storage", type=Path, help="the registry's storage root") - command.add_argument("repository") - pruning.add_argument("log", type=Path, help="the registry's log of the run") - args = parser.parse_args(argv) - if args.command == "prune": - kept, dropped = prune(args.storage, args.repository, used_tags(args.log.read_text(errors="replace"), - args.repository)) - print(f"kept {kept} tags of {args.repository}, dropped {dropped}") - else: - print(f"dropped {drop_freed_links(args.storage, args.repository)} links to freed layers") - return 0 - - -def used_tags(log: str, repository: str) -> set[str]: - """The tags the log names a request for a manifest of ``repository`` by.""" - return set(re.findall(rf"/v2/{re.escape(repository)}/manifests/([A-Za-z0-9_][A-Za-z0-9_.-]{{0,127}})(?=[\s\"?])", - log)) - - -def prune(storage: Path, repository: str, used: set[str]) -> tuple[int, int]: - """Drop the tags of ``repository`` not in ``used``, then the manifests no remaining tag reaches; how many tags it - kept and dropped.""" - manifests = storage / _V2 / "repositories" / repository / "_manifests" - tags = _children(manifests / "tags") - for tag in tags: - if tag.name not in used: - shutil.rmtree(tag) - kept = [tag for tag in tags if tag.name in used] - reached = _reached(storage, [(tag / "current" / "link").read_text().strip() for tag in kept]) - for revision in _children(manifests / "revisions" / "sha256"): - if f"sha256:{revision.name}" not in reached: - shutil.rmtree(revision) - return len(kept), len(tags) - len(kept) - - -def drop_freed_links(storage: Path, repository: str) -> int: - """Drop each link of ``repository`` to a layer that's no longer stored; how many.""" - dropped = 0 - for layer in _children(storage / _V2 / "repositories" / repository / "_layers" / "sha256"): - if not _blob(storage, f"sha256:{layer.name}").is_file(): - shutil.rmtree(layer) - dropped += 1 - return dropped - - -def _reached(storage: Path, digests: list[str]) -> set[str]: - """``digests`` and every manifest an image index among them lists, recursively.""" - reached, pending = set(), list(digests) - while pending: - digest = pending.pop() - if digest in reached: - continue - reached.add(digest) - try: - manifest = json.loads(_blob(storage, digest).read_bytes()) - except (FileNotFoundError, ValueError): - continue - pending += [listed["digest"] for listed in manifest.get("manifests", []) if "digest" in listed] - return reached - - -def _children(folder: Path) -> list[Path]: - return [child for child in folder.iterdir() if child.is_dir()] if folder.is_dir() else [] - - -def _blob(storage: Path, digest: str) -> Path: - hex_digest = digest.removeprefix("sha256:") - return storage / _V2 / "blobs" / "sha256" / hex_digest[:2] / hex_digest / "data" - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/.github/scripts/share_computer_images.py b/.github/scripts/share_computer_images.py new file mode 100644 index 000000000..5d3740086 --- /dev/null +++ b/.github/scripts/share_computer_images.py @@ -0,0 +1,50 @@ +"""Copy every composed computer image a slow-tier runner's registry holds to the CI computer image mirror, so later runs +copy them back rather than build them.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import urllib.error +import urllib.request + +from agent_env.env.envs.computer.utils.composed import REPOSITORY, mirror_tag +from agent_env.store.image_store.oci_registry_credentials import RegistryAuth +from agent_env.store.image_store.registry_api import copy_image, pin_digest, pin_if_served + +KEY = re.compile(r"^[0-9a-f]{64}$") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("mirror", help="the mirror repository, as ghcr.io/org/agentenv-ci-computers") + parser.add_argument("--registry", default="localhost:5000", help="the runner's registry") + args = parser.parse_args() + + try: + with urllib.request.urlopen(f"http://{args.registry}/v2/{REPOSITORY}/tags/list") as response: + keys = sorted(tag for tag in json.load(response).get("tags") or [] if KEY.match(tag)) + except urllib.error.HTTPError as e: + if e.code != 404: + raise + keys = [] + auth = RegistryAuth(args.mirror.partition("/")[0], os.environ["GITHUB_ACTOR"], os.environ["GITHUB_TOKEN"]) + for key in keys: + source = pin_digest(f"{args.registry}/{REPOSITORY}:{key}", None) + copy_image(source, f"{args.mirror}:{mirror_tag(key)}", None, auth) + print(f"shared {key}") + print(f"share computer images: {len(keys)} images in {args.mirror}") + if keys: + try: + pin_if_served(f"{args.mirror}:{mirror_tag(keys[0])}", None) + except ValueError as e: + print(f"::warning::the composer reads {args.mirror} anonymously and can't, so every run builds its images; " + f"make the package public: {e}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 3b08d5c52..bdd1c2eca 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -22,6 +22,7 @@ concurrency: env: AGENT_ENV_ENVIRONMENT: dev + COMPUTER_IMAGE_MIRROR: ghcr.io/scaleapi/agentenv-ci-computers PIP_DISABLE_PIP_VERSION_CHECK: "1" jobs: @@ -154,8 +155,14 @@ jobs: shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] runs-on: ubuntu-latest timeout-minutes: 40 + permissions: + contents: read + packages: write steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + # The tests run with packages: write; only the share step, on main, gets the token. + with: + persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -168,29 +175,22 @@ jobs: # starts afterwards. - name: Pull the base images from mirror.gcr.io run: python3 .github/scripts/mirror_base_images.py src tst - # A computer image or disk is tagged with a key of everything it is built from, and the composer reuses one the - # image store already holds for its key. That key includes the lock of the packages the image installs, which the - # composer resolves from Ubuntu's archive, PyPI and download URLs unless the store holds it. The registry keeps - # its storage on the runner, and the computer tests share their locks and files in the packages folder - # (tst/util/package_cache.py); both are restored from what main saved latest this week, so a run neither resolves - # nor builds again what hasn't changed. A shard that runs no computer tests misses. - - name: Week of the computer images' cache + # The computer tests share the package locks a compose resolves, and the files they name, in the packages folder + # (tst/util/package_cache.py). An image's key includes its lock, which the composer otherwise resolves from + # Ubuntu's archive, PyPI and download URLs, so the folder is restored from what main saved latest this week. + - name: Week of the packages' cache id: week run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" - - name: Restore the computer images and packages - id: computers + - name: Restore the packages + id: packages uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: | - ${{ runner.temp }}/registry - ${{ runner.temp }}/packages - key: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }} - restore-keys: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}- + path: ${{ runner.temp }}/packages + key: packages-${{ matrix.shard }}-${{ steps.week.outputs.week }} + restore-keys: packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}- - name: Start the local registry run: | - mkdir -p "$RUNNER_TEMP/registry" - docker run --detach --name registry --publish 5000:5000 --volume "$RUNNER_TEMP/registry:/var/lib/registry" \ - mirror.gcr.io/library/registry:2 + docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2 timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' # x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below. - name: Let the local provider boot disks with KVM @@ -200,49 +200,37 @@ jobs: env: AGENTENV_TEST_SHARD: ${{ matrix.shard }}/12 AGENTENV_TEST_PACKAGES: ${{ runner.temp }}/packages + AGENT_ENV_COMPUTER_IMAGE_MIRROR: ${{ env.COMPUTER_IMAGE_MIRROR }} run: | .venv/bin/python -m pytest -m int_test_slow -n 2 --dist loadfile -rA --timeout=1200 \ --junitxml=junit-integration-slow.xml tst/integration --ignore=tst/integration/env/gateway/gateway_test.py - # Only the computer images this run used are kept, with the packages, and only main saves them, once its tests - # pass, so pull requests reuse main's without filling the repository's cache. An image is used when the registry's - # log has a request for it by its tag (prune_registry.py), so what a week's earlier runs built and this one no - # longer asks for is dropped. The garbage collection frees what only the dropped tags and repositories referenced; - # with --delete-untagged it would also delete the untagged platform manifests of an image index, and their layers. - # The entry is named by the image keys the registry holds and the locks the packages hold, so a run that built a - # new image or resolved a new lock saves a new one. - - name: Keep only the computer images this run used - id: kept + # Only main saves the packages, once its tests pass, so pull requests reuse main's without filling the + # repository's cache. The entry is named by the locks the folder holds, so a run that resolved a new lock saves a + # new one. + - name: Name the packages' cache entry + id: lockkey if: always() run: | - repositories="$RUNNER_TEMP/registry/docker/registry/v2/repositories" - if [ ! -d "$repositories/agentenv-computers" ]; then echo "computers=false" >> "$GITHUB_OUTPUT"; exit 0; fi - docker logs registry > "$RUNNER_TEMP/registry.log" 2>&1 - docker stop registry - sudo find "$repositories" -mindepth 1 -maxdepth 1 ! -name agentenv-computers -exec rm -rf {} + - sudo python3 .github/scripts/prune_registry.py prune "$RUNNER_TEMP/registry" agentenv-computers \ - "$RUNNER_TEMP/registry.log" - docker run --rm --volume "$RUNNER_TEMP/registry:/var/lib/registry" mirror.gcr.io/library/registry:2 \ - garbage-collect /etc/docker/registry/config.yml > /dev/null - sudo python3 .github/scripts/prune_registry.py links "$RUNNER_TEMP/registry" agentenv-computers - mkdir -p "$RUNNER_TEMP/packages" - du -sh "$RUNNER_TEMP/registry" "$RUNNER_TEMP/packages" - if [ -z "$(sudo ls "$repositories/agentenv-computers/_manifests/tags")" ]; then - echo "computers=false" >> "$GITHUB_OUTPUT"; exit 0 - fi - key=$({ sudo ls "$repositories/agentenv-computers/_manifests/tags" - ls "$RUNNER_TEMP/packages/agentenv-packages/locks" 2>/dev/null; } | sort | sha256sum | cut -c1-16) - echo "key=computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}-$key" >> "$GITHUB_OUTPUT" - echo "computers=true" >> "$GITHUB_OUTPUT" - - name: Save the computer images and packages + locks="$RUNNER_TEMP/packages/agentenv-packages/locks" + if [ -z "$(ls "$locks" 2>/dev/null)" ]; then echo "packages=false" >> "$GITHUB_OUTPUT"; exit 0; fi + key=$(ls "$locks" | sort | sha256sum | cut -c1-16) + echo "key=packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}-$key" >> "$GITHUB_OUTPUT" + echo "packages=true" >> "$GITHUB_OUTPUT" + - name: Save the packages if: >- always() && github.event_name == 'push' && steps.tests.outcome == 'success' - && steps.kept.outputs.computers == 'true' && steps.kept.outputs.key != steps.computers.outputs.cache-matched-key + && steps.lockkey.outputs.packages == 'true' && steps.lockkey.outputs.key != steps.packages.outputs.cache-matched-key uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: | - ${{ runner.temp }}/registry - ${{ runner.temp }}/packages - key: ${{ steps.kept.outputs.key }} + path: ${{ runner.temp }}/packages + key: ${{ steps.lockkey.outputs.key }} + - name: Share the computer images + continue-on-error: true + if: >- + always() && steps.tests.outcome == 'success' && github.event_name == 'push' + env: + GITHUB_TOKEN: ${{ github.token }} + run: .venv/bin/python .github/scripts/share_computer_images.py "$COMPUTER_IMAGE_MIRROR" - name: Skips must be declared capability gaps if: always() run: | diff --git a/AGENTS.md b/AGENTS.md index c80ddc3a8..3dacc182d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -392,9 +392,8 @@ orchestrators call it the same way. What callers rely on: `.github/scripts/slow_test_durations.py`. - The computer tests compose in stores of their own, but share the package locks and files a compose resolves (`tst/util/package_cache.py`) through `AGENTENV_TEST_PACKAGES`, a folder for the run unless - it's set. The slow tier caches that folder with the local registry's computer images, so a run - resolves from Ubuntu's archive, PyPI or a download URL only what changed since `main` last saved - them. + it's set. The slow tier caches that folder, so a run resolves from Ubuntu's archive, PyPI or a + download URL only what changed since `main` last saved it. - `tst/manual/` and `tst/benchmarks/` run only by hand, as each module's docstring says; no CI job collects them. - `tst/formal/` holds TLA+ models, run only by hand with each model's `check.sh`; the `AGENTS.md` diff --git a/src/agent_env/config/describe.py b/src/agent_env/config/describe.py index e8e291d7f..73e289cbc 100644 --- a/src/agent_env/config/describe.py +++ b/src/agent_env/config/describe.py @@ -673,6 +673,7 @@ def env_sources() -> tuple[EnvSource, ...]: EnvSource("AGENT_ENV_MODAL_REGION", None, "no file equivalent"), EnvSource("AGENT_ENV_MODAL_APP_NAME", None, "no file equivalent"), EnvSource("AGENT_ENV_FIXTURE_PREFIX", None, "no file equivalent"), + EnvSource("AGENT_ENV_COMPUTER_IMAGE_MIRROR", None, "no file equivalent"), EnvSource("MODAL_TOKEN_ID", None, "credential, no file equivalent"), EnvSource("MODAL_TOKEN_SECRET", None, "credential, no file equivalent"), ] diff --git a/src/agent_env/config/runtime.py b/src/agent_env/config/runtime.py index a98b17005..66326e090 100644 --- a/src/agent_env/config/runtime.py +++ b/src/agent_env/config/runtime.py @@ -24,6 +24,11 @@ objects, image builds, env and agent snapshots, changelogs, default agent and judge trajectories, verifier outputs and the A2A validator's fixtures, so a fresh control-plane backend can write without colliding with existing objects in a shared store. Empty in prod. + +AGENT_ENV_COMPUTER_IMAGE_MIRROR (default ""): a registry repository, read anonymously, holding composed computer images +under their keys and the composer's own digest, as ``ghcr.io/org/computers``. A compose that finds no image of a key in the +env's image store copies the mirror's into it instead of building it. CI sets it so its runners share the images main +built. """ from __future__ import annotations @@ -226,6 +231,9 @@ class Config: fixture_prefix: str = field( default_factory=lambda: os.getenv("AGENT_ENV_FIXTURE_PREFIX", "") ) + computer_image_mirror: str = field( + default_factory=lambda: os.getenv("AGENT_ENV_COMPUTER_IMAGE_MIRROR", "") + ) agent_sandbox_mode: str = field( default_factory=lambda: os.getenv("AGENT_SANDBOX_MODE") or _default_sandbox_mode() ) diff --git a/src/agent_env/env/envs/computer/utils/composed.py b/src/agent_env/env/envs/computer/utils/composed.py index 3ee5281ad..bb13d6b6e 100644 --- a/src/agent_env/env/envs/computer/utils/composed.py +++ b/src/agent_env/env/envs/computer/utils/composed.py @@ -117,6 +117,15 @@ def composition() -> str: return "sha256:" + digest.hexdigest() +MIRROR_PREFIX_LENGTH = 16 + + +def mirror_tag(key: str) -> str: + """The tag of the image of ``key`` in a computer image mirror: the key, under the start of ``composition()``, so a + change to the code that composes an image builds it again rather than reuse one the old code built.""" + return f"{composition().removeprefix('sha256:')[:MIRROR_PREFIX_LENGTH]}-{key.removeprefix('sha256:')}" + + def _rank(result: dict) -> int: """How far a pair got: ready, then built and either unprobed or failed at its probe, then anything short of an image, and below that one another composer recorded. A result takes a stored one's place only when it got at least diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index d9d6c0089..4ab6da95d 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -50,7 +50,7 @@ thin_disk, windows_disk, ) -from agent_env.env.envs.computer.utils.composed import BUILT, REPOSITORY, Record, failed, now, stale +from agent_env.env.envs.computer.utils.composed import BUILT, REPOSITORY, Record, failed, mirror_tag, now, stale from agent_env.env.envs.computer.utils.install import Install, for_os, platform_of from agent_env.env.published_images import DISK, DISK_TOOLS_IMAGE, published_disk, published_pinned from agent_env.providers.sandbox_providers.host_images import platform_tag @@ -58,7 +58,7 @@ from agent_env.store.image_store.image_contents import image_config, labels from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.store.image_store.image_store import ImageStore -from agent_env.store.image_store.registry_api import pin_digest, pin_if_served, write_index +from agent_env.store.image_store.registry_api import copy_image, pin_digest, pin_if_served, write_index from agent_env.store.local_state import holding_locks from agent_env.utils.docker.build import DockerBuildError @@ -771,9 +771,20 @@ def _key(**inputs: Any) -> str: def _stored(env: ComputerEnv, key: str) -> tuple[ImageStore, str, str | None]: """The env's image store, the ref there of the image of ``key``, and that ref pinned when the store serves one - already, for any env that built it.""" + already, for any env that built it, or once it's copied there from the configured computer image mirror.""" store = _image_store(env) - ref = store.image_ref(REPOSITORY, key.removeprefix("sha256:")) + tag = key.removeprefix("sha256:") + ref = store.image_ref(REPOSITORY, tag) + if (pinned := pin_if_served(ref, store.auth(ref))) is not None or not (mirror := get_config().computer_image_mirror): + return store, ref, pinned + try: + if (mirrored := pin_if_served(f"{mirror}:{mirror_tag(tag)}", None)) is None: + return store, ref, None + copy_image(mirrored, ref, None, store.auth(ref)) + except (ValueError, KeyError, OSError) as e: + logger.warning(f"the computer image mirror {mirror} couldn't give {tag}, so it's built: {e}") + return store, ref, None + logger.info(f"copied {tag} from the computer image mirror {mirror}") return store, ref, pin_if_served(ref, store.auth(ref)) diff --git a/src/agent_env/store/image_store/registry_api.py b/src/agent_env/store/image_store/registry_api.py index 4544cfed8..07d92a8ed 100644 --- a/src/agent_env/store/image_store/registry_api.py +++ b/src/agent_env/store/image_store/registry_api.py @@ -1,5 +1,6 @@ """Read an image from its registry over the OCI distribution API: its digest, to pin a tag to the image it names now, -and its manifests and blobs, to inspect what it holds without pulling it; and write an index of images it holds.""" +and its manifests and blobs, to inspect what it holds without pulling it; write an index of images it holds; and copy an +image from one registry to another.""" from __future__ import annotations @@ -7,6 +8,7 @@ import hashlib import json import re +import tempfile from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path @@ -217,6 +219,40 @@ def write_index(ref: str, images: dict[str, str], auth: RegistryAuth | None, *, {"schemaVersion": 2, "mediaType": media, "manifests": manifests}).encode(), media) +def copy_image(source: str, target: str, source_auth: RegistryAuth | None, target_auth: RegistryAuth | None, *, + timeout: float = _TIMEOUT_SECONDS) -> None: + """Copy the image ``source`` names, pinned by its digest, to ``target``'s tag: every blob ``target``'s repository + lacks, then each manifest byte for byte, so the copy keeps ``source``'s digest; an index's platform manifests + first, each by its digest.""" + digest = _parse(source).digest + if digest is None: + raise ValueError(f"{source}: a copy goes by the image's digest, and this names none") + image = _parse(target) + with (_Repository(_parse(source), source_auth, timeout, source) as reading, + _Repository(image, target_auth, timeout, target) as writing, + tempfile.TemporaryDirectory(prefix="agentenv-copy-") as work): + + def manifest(reference: str) -> tuple[bytes, str]: + response = reading.get(f"manifests/{reference}", accept=_MANIFEST_TYPES) + body = response.json() + served = response.headers.get("content-type", "").split(";")[0].strip() or body.get("mediaType", "") + if served in _INDEXES: + for child in body["manifests"]: + manifest(child["digest"]) + else: + for blob in [body["config"], *body["layers"]]: + if writing.has_blob(blob["digest"]): + continue + path = Path(work) / blob["digest"].removeprefix("sha256:") + read_blob(source, blob["digest"], source_auth, path, timeout=timeout) + writing.upload(blob["digest"], path) + path.unlink() + writing.put(f"manifests/{reference}", response.content, served) + return response.content, served + + writing.put(f"manifests/{image.tag or 'latest'}", *manifest(digest)) + + class _Repository: """One image's repository on its registry, read with the Bearer token the registry's first challenge asks for.""" @@ -250,6 +286,29 @@ def put(self, path: str, content: bytes, content_type: str) -> None: if response.status_code not in (200, 201): self._raise_for(response) + def has_blob(self, digest: str) -> bool: + response = self._send("HEAD", f"blobs/{digest}", None, stream=False) + if response.status_code == 404: + return False + self._raise_for(response) + return True + + def upload(self, digest: str, path: Path) -> None: + """Push the file ``path`` as the blob ``digest``, in one request after the registry opens an upload.""" + opened = self._send("POST", "blobs/uploads/", None, stream=False) + if opened.status_code != 202 or "location" not in opened.headers: + self._raise_for(opened) + raise ValueError(f"{self._what}: {self._host} opened no upload for {digest} (HTTP {opened.status_code})") + location = httpx.URL(f"{self._base}/").join(opened.headers["location"]) + base = httpx.URL(self._base) + if (location.scheme, location.netloc) != (base.scheme, base.netloc): + raise ValueError(f"{self._what}: {self._host} sent the upload of {digest} to {location.scheme}://" + f"{location.netloc.decode()}, which its credentials don't go to") + response = self._send("PUT", str(location.copy_merge_params({"digest": digest})), None, stream=False, + content=path, content_type="application/octet-stream") + if response.status_code != 201: + self._raise_for(response) + @contextlib.contextmanager def stream(self, path: str) -> Iterator[httpx.Response]: """``path`` as a streamed response, closed when the block ends.""" @@ -262,19 +321,27 @@ def stream(self, path: str) -> Iterator[httpx.Response]: finally: response.close() - def _send(self, method: str, path: str, accept: str | None, *, stream: bool, content: bytes | None = None, + def _send(self, method: str, path: str, accept: str | None, *, stream: bool, content: bytes | Path | None = None, content_type: str | None = None) -> httpx.Response: + """``path`` is under the repository, or a whole URL the registry gave; a file's ``content`` streams from disk.""" + url = path if path.startswith(("http://", "https://")) else f"{self._base}/{path}" try: - for _ in range(2): + for attempt in range(2): headers = {"Accept": accept} if accept else {} if content_type: headers["Content-Type"] = content_type if self._token: headers["Authorization"] = f"Bearer {self._token}" - request = self._client.build_request(method, f"{self._base}/{path}", headers=headers, content=content) - # Basic credentials go with every request until a Bearer token replaces them. - response = self._client.send(request, auth=None if self._token else self._basic, stream=stream) - if response.status_code != 401 or self._token: + with contextlib.ExitStack() as opened: + body = content + if isinstance(content, Path): + headers["Content-Length"] = str(content.stat().st_size) + body = opened.enter_context(content.open("rb")) + request = self._client.build_request(method, url, headers=headers, content=body) + # Basic credentials go with every request until a Bearer token replaces them. + response = self._client.send(request, auth=None if self._token else self._basic, stream=stream) + # A held token is asked for again once: GHCR scopes it to the first challenge, a pull's before a push. + if response.status_code != 401 or attempt: return response if stream: response.read() diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index e3fdf992b..34b6aeee7 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -46,6 +46,7 @@ probing, resolve_in_base) from agent_env.env.envs.computer.utils import sites as computer_sites from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.store.image_store.registry_api import RegistryRefusedError from agent_env.env.envs.computer.utils.probing import _probe from agent_env.env.envs.computer.env import COMPUTER_TOOLS, ComputerEnv from agent_env.env.envs.computer import app_env @@ -749,6 +750,77 @@ def test_env_build_by_an_installed_alias_composes_under_the_id_of_the_env_it_nam "@local/acme-kit/kit/desktop__computer").image().image_name]) +MIRROR = "ghcr.io/org/computers" + + +def _mirrored(docker, monkeypatch, read=None): + """Move every composed image out of the env's image store into the computer image mirror, as a fresh runner of CI + finds them, and record each copy back: the mirror's refs, read with ``read`` when given.""" + stored = {ref: digest for ref, digest in docker.registry.tags.items() + if f"/{composed.REPOSITORY}:" in ref and ":index-" not in ref} + for ref in stored: + del docker.registry.tags[ref] + mirror = {f"{MIRROR}:{composed.mirror_tag(ref.rpartition(':')[2])}": digest for ref, digest in stored.items()} + copied, real = [], composer.pin_if_served + + def pin(ref, auth): + if ref.startswith(f"{MIRROR}:"): + return read(ref) if read else (f"{ref}@{mirror[ref]}" if ref in mirror else None) + return real(ref, auth) + + def copy(source, target, source_auth, target_auth): + copied.append((source.partition("@")[0], target)) + docker.registry.tags[target] = source.rpartition("@")[2] + + monkeypatch.setattr(composer, "pin_if_served", pin) + monkeypatch.setattr(composer, "copy_image", copy) + monkeypatch.setattr(get_config(), "computer_image_mirror", MIRROR) + return stored, copied + + +def test_an_image_the_store_lacks_is_copied_from_the_computer_image_mirror_and_reused_rather_than_built(base, driver, + docker, + monkeypatch): + first = _put(base, driver) + stored, copied = _mirrored(docker, monkeypatch) + + second = _put(base, driver, id="@local/~/kit/other", environment_name="other") + + reused = Env.get(second.id).metadata["composed"]["container"] + assert len(docker.builds) == 2 and all(r["reused"] for r in reused.values()) + assert sorted(copied) == sorted((f"{MIRROR}:{composed.mirror_tag(ref.rpartition(':')[2])}", ref) for ref in stored) + assert {arch: r["image"] for arch, r in reused.items()} == { + arch: r["image"] for arch, r in Env.get(first.id).metadata["composed"]["container"].items()} + + +def test_a_change_to_the_code_that_composes_an_image_misses_the_mirror_and_builds_it(base, driver, docker, + monkeypatch): + _put(base, driver) + _, copied = _mirrored(docker, monkeypatch) + monkeypatch.setattr(composed, "composition", lambda: "sha256:" + "f" * 64) + + second = _put(base, driver, id="@local/~/kit/other", environment_name="other") + + assert len(docker.builds) == 4 and copied == [] + assert not any(r["reused"] for r in Env.get(second.id).metadata["composed"]["container"].values()) + + +def test_a_computer_image_mirror_that_cant_be_read_leaves_the_image_to_be_built(base, driver, docker, monkeypatch, + caplog): + _put(base, driver) + + def refused(ref): + raise RegistryRefusedError(f"{ref}: ghcr.io refused to serve it (HTTP 403)") + + _, copied = _mirrored(docker, monkeypatch, read=refused) + with caplog.at_level(logging.WARNING, logger=composer.__name__): + second = _put(base, driver, id="@local/~/kit/other", environment_name="other") + + built = Env.get(second.id).metadata["composed"]["container"] + assert len(docker.builds) == 4 and not any(r["reused"] for r in built.values()) and copied == [] + assert f"the computer image mirror {MIRROR} couldn't give" in caplog.text + + def test_builds_of_one_env_at_once_keep_each_others_pairs_and_a_failure_doesnt_replace_a_built_one(base, driver, docker, monkeypatch): diff --git a/tst/unit/store/registry_api_test.py b/tst/unit/store/registry_api_test.py index aec0d54fe..120992dd8 100644 --- a/tst/unit/store/registry_api_test.py +++ b/tst/unit/store/registry_api_test.py @@ -327,3 +327,108 @@ def ghcr(request): assert json.loads(request.content) == {"schemaVersion": 2, "mediaType": written, "manifests": [ amd64_entry, *(children if arm64 == "index" else [{**amd64_entry, "digest": OTHER, "platform": {"architecture": "arm64", "os": "linux"}}])]} + + +def _sha(data: bytes) -> str: + return f"sha256:{hashlib.sha256(data).hexdigest()}" + + +def test_an_image_is_copied_by_its_digest_with_only_the_blobs_the_target_lacks_and_its_manifests_byte_for_byte( + fake_registry): + config, kept, missing = b'{"architecture": "amd64"}', b"layer the target has", b"layer it lacks" + image = json.dumps({"mediaType": OCI_IMAGE, "config": {"digest": _sha(config)}, + "layers": [{"digest": _sha(kept)}, {"digest": _sha(missing)}]}).encode() + index = json.dumps({"mediaType": OCI_INDEX, "manifests": [{"digest": _sha(image)}]}).encode() + source = {_sha(index): (OCI_INDEX, index), _sha(image): (OCI_IMAGE, image)} + blobs = {_sha(blob): blob for blob in (config, kept, missing)} + target_blobs, target_manifests = {_sha(kept): kept}, {} + + def registries(request): + path = request.url.path + if request.url.host == "ghcr.io": + if "/blobs/" in path: + return httpx.Response(200, content=blobs[path.rpartition("/")[2]]) + media, body = source[path.rpartition("/")[2]] + return httpx.Response(200, content=body, headers={"Content-Type": media}) + if request.method == "HEAD": + return httpx.Response(200 if path.rpartition("/")[2] in target_blobs else 404) + if request.method == "POST": + return httpx.Response(202, headers={"Location": "/v2/agentenv-computers/blobs/uploads/u1?_state=s"}) + if "/blobs/uploads/" in path: + assert request.url.params["_state"] == "s" + target_blobs[request.url.params["digest"]] = request.read() + return httpx.Response(201) + target_manifests[path.rpartition("/")[2]] = (request.headers["content-type"], request.content) + return httpx.Response(201) + + seen = fake_registry(registries) + registry_api.copy_image(f"ghcr.io/org/computers:k@{_sha(index)}", "localhost:5000/agentenv-computers:k", None, None) + + assert target_blobs == blobs + assert [r.url.params["digest"] for r in seen if r.method == "PUT" and "/blobs/" in r.url.path] == [ + _sha(config), _sha(missing)] + assert target_manifests == {_sha(image): (OCI_IMAGE, image), _sha(index): (OCI_INDEX, index), "k": (OCI_INDEX, index)} + + +def test_a_copy_needs_the_digest_it_copies(fake_registry): + seen = fake_registry(lambda request: httpx.Response(500)) + + with pytest.raises(ValueError, match="names none"): + registry_api.copy_image("ghcr.io/org/computers:k", "localhost:5000/agentenv-computers:k", None, None) + assert seen == [] + + +def test_a_copy_to_ghcr_asks_again_for_the_push_token_its_first_challenge_didnt_give(fake_registry): + config = b'{"architecture": "amd64"}' + image = json.dumps({"mediaType": OCI_IMAGE, "config": {"digest": _sha(config)}, "layers": []}).encode() + pushed = [] + + def registries(request): + if request.url.host == "localhost": + if "/blobs/" in request.url.path: + return httpx.Response(200, content=config) + return httpx.Response(200, content=image, headers={"Content-Type": OCI_IMAGE}) + if request.url.path == "/token": + assert request.headers["authorization"] == "Basic " + base64.b64encode(b"bot:t").decode() + return httpx.Response(200, json={"token": request.url.params["scope"]}) + writes = request.method in ("POST", "PUT") + scope = "repository:org/computers:pull,push" if writes else "repository:org/computers:pull" + if request.headers.get("authorization") not in (f"Bearer {scope}", "Bearer repository:org/computers:pull,push"): + return httpx.Response(401, headers={"WWW-Authenticate": f'Bearer realm="https://ghcr.io/token",scope="{scope}"'}) + if request.method == "HEAD": + return httpx.Response(404) + if request.method == "POST": + return httpx.Response(202, headers={"Location": "/v2/org/computers/blobs/uploads/u1"}) + pushed.append(request.url.path) + return httpx.Response(201) + + fake_registry(registries) + registry_api.copy_image(f"localhost:5000/agentenv-computers:k@{_sha(image)}", "ghcr.io/org/computers:k", None, + RegistryAuth("ghcr.io", "bot", "t")) + + assert pushed == ["/v2/org/computers/blobs/uploads/u1", f"/v2/org/computers/manifests/{_sha(image)}", + "/v2/org/computers/manifests/k"] + + +@pytest.mark.parametrize("location", ["https://elsewhere.example/v2/org/computers/blobs/uploads/u1", + "http://ghcr.io/v2/org/computers/blobs/uploads/u1"], + ids=["another-host", "plain-http"]) +def test_an_upload_the_registry_sends_elsewhere_is_refused_before_its_credentials_go(fake_registry, location): + config = b'{"architecture": "amd64"}' + image = json.dumps({"mediaType": OCI_IMAGE, "config": {"digest": _sha(config)}, "layers": []}).encode() + + def registries(request): + if request.url.host == "localhost": + if "/blobs/" in request.url.path: + return httpx.Response(200, content=config) + return httpx.Response(200, content=image, headers={"Content-Type": OCI_IMAGE}) + assert request.url.host == "ghcr.io" and request.url.scheme == "https", request.url + if request.method == "HEAD": + return httpx.Response(404) + return httpx.Response(202, headers={"Location": location}) + + fake_registry(registries) + + with pytest.raises(ValueError, match="which its credentials don't go to"): + registry_api.copy_image(f"localhost:5000/agentenv-computers:k@{_sha(image)}", "ghcr.io/org/computers:k", None, + RegistryAuth("ghcr.io", "bot", "t")) diff --git a/tst/unit/test_computer_image_mirror_scripts.py b/tst/unit/test_computer_image_mirror_scripts.py new file mode 100644 index 000000000..622ff2377 --- /dev/null +++ b/tst/unit/test_computer_image_mirror_scripts.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import io +import json +import sys + +import pytest + +from tst.util.scripts import load_script + +KEY, OTHER = "c" * 16 + "-" + "a" * 64, "c" * 16 + "-" + "b" * 64 +RAW, RAW_OTHER = "a" * 64, "b" * 64 + + +def test_a_runner_shares_each_image_under_its_mirror_tag_and_leaves_its_indexes(monkeypatch, capsys): + share = load_script("share_computer_images.py") + tags = {"tags": [RAW, f"index-{RAW_OTHER}", RAW_OTHER]} + copies = [] + monkeypatch.setattr(share.urllib.request, "urlopen", lambda url: io.BytesIO(json.dumps(tags).encode())) + monkeypatch.setattr(share, "pin_digest", lambda ref, auth: f"{ref}@sha256:{ref[-64:]}") + monkeypatch.setattr(share, "mirror_tag", lambda key: f"{'c' * 16}-{key}") + monkeypatch.setattr(share, "copy_image", lambda source, target, source_auth, auth: copies.append( + (source.rpartition("@")[2], target, auth.username))) + monkeypatch.setattr(share, "pin_if_served", lambda ref, auth: f"{ref}@sha256:{RAW}") + monkeypatch.setenv("GITHUB_ACTOR", "bot") + monkeypatch.setenv("GITHUB_TOKEN", "t") + monkeypatch.setattr(sys, "argv", ["share_computer_images.py", "ghcr.io/org/agentenv-ci-computers"]) + + assert share.main() == 0 + assert copies == [(f"sha256:{raw}", f"ghcr.io/org/agentenv-ci-computers:{key}", "bot") + for raw, key in ((RAW, KEY), (RAW_OTHER, OTHER))] + assert "::warning::" not in capsys.readouterr().out + + +def test_a_mirror_the_composer_cant_read_anonymously_warns_to_make_it_public(monkeypatch, capsys): + share = load_script("share_computer_images.py") + monkeypatch.setattr(share.urllib.request, "urlopen", lambda url: io.BytesIO(json.dumps({"tags": [RAW]}).encode())) + monkeypatch.setattr(share, "pin_digest", lambda ref, auth: f"{ref}@sha256:{RAW}") + monkeypatch.setattr(share, "copy_image", lambda *args: None) + + def private(ref, auth): + raise ValueError(f"{ref}: ghcr.io refused to serve it (HTTP 403)") + + monkeypatch.setattr(share, "pin_if_served", private) + monkeypatch.setenv("GITHUB_ACTOR", "bot") + monkeypatch.setenv("GITHUB_TOKEN", "t") + monkeypatch.setattr(sys, "argv", ["share_computer_images.py", "ghcr.io/org/agentenv-ci-computers"]) + + assert share.main() == 0 + assert "make the package public" in capsys.readouterr().out + + +def test_a_runner_that_composed_no_image_shares_nothing_and_passes(monkeypatch, capsys): + share = load_script("share_computer_images.py") + + def no_repository(url): + raise share.urllib.error.HTTPError(url, 404, "Not Found", {}, io.BytesIO(b"")) + + monkeypatch.setattr(share.urllib.request, "urlopen", no_repository) + monkeypatch.setattr(share, "copy_image", lambda *args: pytest.fail("copied with no image to share")) + monkeypatch.setenv("GITHUB_ACTOR", "bot") + monkeypatch.setenv("GITHUB_TOKEN", "t") + monkeypatch.setattr(sys, "argv", ["share_computer_images.py", "ghcr.io/org/agentenv-ci-computers"]) + + assert share.main() == 0 + assert "0 images" in capsys.readouterr().out diff --git a/tst/unit/test_prune_registry_script.py b/tst/unit/test_prune_registry_script.py deleted file mode 100644 index 2b842562c..000000000 --- a/tst/unit/test_prune_registry_script.py +++ /dev/null @@ -1,97 +0,0 @@ -"""``.github/scripts/prune_registry.py``, which keeps only the computer images a slow-tier run used before the local -registry's storage is cached: their tags, the manifests those reach, and the layer links garbage collection leaves.""" - -from __future__ import annotations - -import ast -import hashlib -import json -import sys -from pathlib import Path - -import pytest - -from tst.util.scripts import SCRIPTS, load_script - -SCRIPT = SCRIPTS / "prune_registry.py" -REPOSITORY = "agentenv-computers" -_USED, _STALE = "a" * 64, "b" * 64 - - -@pytest.fixture -def script(): - return load_script(SCRIPT.name) - - -def test_it_imports_only_the_standard_library(): - """It runs under the runner's python3 with sudo, outside the project's environment.""" - imported = set() - for node in ast.walk(ast.parse(SCRIPT.read_text())): - if isinstance(node, ast.Import): - imported |= {alias.name.split(".")[0] for alias in node.names} - elif isinstance(node, ast.ImportFrom) and node.level == 0: - imported.add(node.module.split(".")[0]) - assert imported <= set(sys.stdlib_module_names) | {"__future__"} - - -def test_a_tag_is_used_when_the_log_has_a_request_for_its_manifest_by_it(script): - log = "\n".join([ - f'time="t" level=info msg="response completed" http.request.method=HEAD ' - f'http.request.uri="/v2/{REPOSITORY}/manifests/{_USED}" http.response.status=200', - f'172.17.0.1 - - [10/Nov/2026:05:00:00 +0000] "PUT /v2/{REPOSITORY}/manifests/{"c" * 64} HTTP/1.1" 201 0', - f'172.17.0.1 - - [10/Nov/2026:05:00:01 +0000] "GET /v2/{REPOSITORY}/manifests/sha256:{"d" * 64} HTTP/1.1" 200', - f'172.17.0.1 - - [10/Nov/2026:05:00:02 +0000] "HEAD /v2/other/manifests/{_STALE} HTTP/1.1" 200 0', - ]) - - assert script.used_tags(log, REPOSITORY) == {_USED, "c" * 64} - - -def test_pruning_keeps_the_used_tags_and_every_manifest_their_index_lists(script, tmp_path): - platforms = [_blob(tmp_path, {"layers": [], "n": n}) for n in range(2)] - index = _blob(tmp_path, {"manifests": [{"digest": digest} for digest in platforms]}) - stale = _blob(tmp_path, {"layers": [], "n": "stale"}) - _tag(tmp_path, _USED, index) - _tag(tmp_path, _STALE, stale) - for digest in (index, *platforms, stale): - _revision(tmp_path, digest) - - kept, dropped = script.prune(tmp_path, REPOSITORY, {_USED}) - - manifests = tmp_path / "docker/registry/v2/repositories" / REPOSITORY / "_manifests" - assert (kept, dropped) == (1, 1) - assert sorted(p.name for p in (manifests / "tags").iterdir()) == [_USED] - assert {p.name for p in (manifests / "revisions/sha256").iterdir()} == {d.removeprefix("sha256:") - for d in (index, *platforms)} - - -def test_links_to_layers_garbage_collection_freed_are_dropped(script, tmp_path): - kept = _blob(tmp_path, {"layer": "kept"}) - layers = tmp_path / "docker/registry/v2/repositories" / REPOSITORY / "_layers/sha256" - for digest in (kept, "sha256:" + "e" * 64): - (layers / digest.removeprefix("sha256:")).mkdir(parents=True) - (layers / digest.removeprefix("sha256:") / "link").write_text(digest) - - assert script.drop_freed_links(tmp_path, REPOSITORY) == 1 - assert [p.name for p in layers.iterdir()] == [kept.removeprefix("sha256:")] - - -def _blob(storage: Path, content: dict) -> str: - data = json.dumps(content).encode() - hex_digest = hashlib.sha256(data).hexdigest() - path = storage / "docker/registry/v2/blobs/sha256" / hex_digest[:2] / hex_digest / "data" - path.parent.mkdir(parents=True) - path.write_bytes(data) - return f"sha256:{hex_digest}" - - -def _tag(storage: Path, tag: str, digest: str) -> None: - current = storage / "docker/registry/v2/repositories" / REPOSITORY / "_manifests/tags" / tag / "current" - current.mkdir(parents=True) - (current / "link").write_text(digest) - - -def _revision(storage: Path, digest: str) -> None: - revision = (storage / "docker/registry/v2/repositories" / REPOSITORY / "_manifests/revisions/sha256" - / digest.removeprefix("sha256:")) - revision.mkdir(parents=True) - (revision / "link").write_text(digest)