Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 0 additions & 98 deletions .github/scripts/prune_registry.py

This file was deleted.

50 changes: 50 additions & 0 deletions .github/scripts/share_computer_images.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
"""Copy every composed computer image a slow-tier runner's registry holds to the CI computer image mirror, so later runs
copy them back rather than build them."""

from __future__ import annotations

import argparse
import json
import os
import re
import sys
import urllib.error
import urllib.request

from agent_env.env.envs.computer.utils.composed import REPOSITORY, mirror_tag
from agent_env.store.image_store.oci_registry_credentials import RegistryAuth
from agent_env.store.image_store.registry_api import copy_image, pin_digest, pin_if_served

KEY = re.compile(r"^[0-9a-f]{64}$")


def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("mirror", help="the mirror repository, as ghcr.io/org/agentenv-ci-computers")
parser.add_argument("--registry", default="localhost:5000", help="the runner's registry")
args = parser.parse_args()

try:
with urllib.request.urlopen(f"http://{args.registry}/v2/{REPOSITORY}/tags/list") as response:
keys = sorted(tag for tag in json.load(response).get("tags") or [] if KEY.match(tag))
except urllib.error.HTTPError as e:
if e.code != 404:
raise
keys = []
auth = RegistryAuth(args.mirror.partition("/")[0], os.environ["GITHUB_ACTOR"], os.environ["GITHUB_TOKEN"])
for key in keys:
source = pin_digest(f"{args.registry}/{REPOSITORY}:{key}", None)
copy_image(source, f"{args.mirror}:{mirror_tag(key)}", None, auth)
print(f"shared {key}")
print(f"share computer images: {len(keys)} images in {args.mirror}")
if keys:
try:
pin_if_served(f"{args.mirror}:{mirror_tag(keys[0])}", None)
except ValueError as e:
print(f"::warning::the composer reads {args.mirror} anonymously and can't, so every run builds its images; "
f"make the package public: {e}")
return 0


if __name__ == "__main__":
sys.exit(main())
90 changes: 39 additions & 51 deletions .github/workflows/local-backends.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ concurrency:

env:
AGENT_ENV_ENVIRONMENT: dev
COMPUTER_IMAGE_MIRROR: ghcr.io/scaleapi/agentenv-ci-computers
PIP_DISABLE_PIP_VERSION_CHECK: "1"

jobs:
Expand Down Expand Up @@ -154,8 +155,14 @@ jobs:
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
runs-on: ubuntu-latest
timeout-minutes: 40
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
# The tests run with packages: write; only the share step, on main, gets the token.
with:
persist-credentials: false
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
Expand All @@ -168,29 +175,22 @@ jobs:
# starts afterwards.
- name: Pull the base images from mirror.gcr.io
run: python3 .github/scripts/mirror_base_images.py src tst
# A computer image or disk is tagged with a key of everything it is built from, and the composer reuses one the
# image store already holds for its key. That key includes the lock of the packages the image installs, which the
# composer resolves from Ubuntu's archive, PyPI and download URLs unless the store holds it. The registry keeps
# its storage on the runner, and the computer tests share their locks and files in the packages folder
# (tst/util/package_cache.py); both are restored from what main saved latest this week, so a run neither resolves
# nor builds again what hasn't changed. A shard that runs no computer tests misses.
- name: Week of the computer images' cache
# The computer tests share the package locks a compose resolves, and the files they name, in the packages folder
# (tst/util/package_cache.py). An image's key includes its lock, which the composer otherwise resolves from
# Ubuntu's archive, PyPI and download URLs, so the folder is restored from what main saved latest this week.
- name: Week of the packages' cache
id: week
run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
- name: Restore the computer images and packages
id: computers
- name: Restore the packages
id: packages
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
${{ runner.temp }}/registry
${{ runner.temp }}/packages
key: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}
restore-keys: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}-
path: ${{ runner.temp }}/packages
key: packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}
restore-keys: packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}-
- name: Start the local registry
run: |
mkdir -p "$RUNNER_TEMP/registry"
docker run --detach --name registry --publish 5000:5000 --volume "$RUNNER_TEMP/registry:/var/lib/registry" \
mirror.gcr.io/library/registry:2
docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2
timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done'
# x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below.
- name: Let the local provider boot disks with KVM
Expand All @@ -200,49 +200,37 @@ jobs:
env:
AGENTENV_TEST_SHARD: ${{ matrix.shard }}/12
AGENTENV_TEST_PACKAGES: ${{ runner.temp }}/packages
AGENT_ENV_COMPUTER_IMAGE_MIRROR: ${{ env.COMPUTER_IMAGE_MIRROR }}
run: |
.venv/bin/python -m pytest -m int_test_slow -n 2 --dist loadfile -rA --timeout=1200 \
--junitxml=junit-integration-slow.xml tst/integration --ignore=tst/integration/env/gateway/gateway_test.py
# Only the computer images this run used are kept, with the packages, and only main saves them, once its tests
# pass, so pull requests reuse main's without filling the repository's cache. An image is used when the registry's
# log has a request for it by its tag (prune_registry.py), so what a week's earlier runs built and this one no
# longer asks for is dropped. The garbage collection frees what only the dropped tags and repositories referenced;
# with --delete-untagged it would also delete the untagged platform manifests of an image index, and their layers.
# The entry is named by the image keys the registry holds and the locks the packages hold, so a run that built a
# new image or resolved a new lock saves a new one.
- name: Keep only the computer images this run used
id: kept
# Only main saves the packages, once its tests pass, so pull requests reuse main's without filling the
# repository's cache. The entry is named by the locks the folder holds, so a run that resolved a new lock saves a
# new one.
- name: Name the packages' cache entry
id: lockkey
if: always()
run: |
repositories="$RUNNER_TEMP/registry/docker/registry/v2/repositories"
if [ ! -d "$repositories/agentenv-computers" ]; then echo "computers=false" >> "$GITHUB_OUTPUT"; exit 0; fi
docker logs registry > "$RUNNER_TEMP/registry.log" 2>&1
docker stop registry
sudo find "$repositories" -mindepth 1 -maxdepth 1 ! -name agentenv-computers -exec rm -rf {} +
sudo python3 .github/scripts/prune_registry.py prune "$RUNNER_TEMP/registry" agentenv-computers \
"$RUNNER_TEMP/registry.log"
docker run --rm --volume "$RUNNER_TEMP/registry:/var/lib/registry" mirror.gcr.io/library/registry:2 \
garbage-collect /etc/docker/registry/config.yml > /dev/null
sudo python3 .github/scripts/prune_registry.py links "$RUNNER_TEMP/registry" agentenv-computers
mkdir -p "$RUNNER_TEMP/packages"
du -sh "$RUNNER_TEMP/registry" "$RUNNER_TEMP/packages"
if [ -z "$(sudo ls "$repositories/agentenv-computers/_manifests/tags")" ]; then
echo "computers=false" >> "$GITHUB_OUTPUT"; exit 0
fi
key=$({ sudo ls "$repositories/agentenv-computers/_manifests/tags"
ls "$RUNNER_TEMP/packages/agentenv-packages/locks" 2>/dev/null; } | sort | sha256sum | cut -c1-16)
echo "key=computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}-$key" >> "$GITHUB_OUTPUT"
echo "computers=true" >> "$GITHUB_OUTPUT"
- name: Save the computer images and packages
locks="$RUNNER_TEMP/packages/agentenv-packages/locks"
if [ -z "$(ls "$locks" 2>/dev/null)" ]; then echo "packages=false" >> "$GITHUB_OUTPUT"; exit 0; fi
key=$(ls "$locks" | sort | sha256sum | cut -c1-16)
echo "key=packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}-$key" >> "$GITHUB_OUTPUT"
echo "packages=true" >> "$GITHUB_OUTPUT"
- name: Save the packages
if: >-
always() && github.event_name == 'push' && steps.tests.outcome == 'success'
&& steps.kept.outputs.computers == 'true' && steps.kept.outputs.key != steps.computers.outputs.cache-matched-key
&& steps.lockkey.outputs.packages == 'true' && steps.lockkey.outputs.key != steps.packages.outputs.cache-matched-key
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
${{ runner.temp }}/registry
${{ runner.temp }}/packages
key: ${{ steps.kept.outputs.key }}
path: ${{ runner.temp }}/packages
key: ${{ steps.lockkey.outputs.key }}
- name: Share the computer images
continue-on-error: true
if: >-
always() && steps.tests.outcome == 'success' && github.event_name == 'push'
env:
GITHUB_TOKEN: ${{ github.token }}
run: .venv/bin/python .github/scripts/share_computer_images.py "$COMPUTER_IMAGE_MIRROR"
- name: Skips must be declared capability gaps
if: always()
run: |
Expand Down
5 changes: 2 additions & 3 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -392,9 +392,8 @@ orchestrators call it the same way. What callers rely on:
`.github/scripts/slow_test_durations.py`.
- The computer tests compose in stores of their own, but share the package locks and files a compose
resolves (`tst/util/package_cache.py`) through `AGENTENV_TEST_PACKAGES`, a folder for the run unless
it's set. The slow tier caches that folder with the local registry's computer images, so a run
resolves from Ubuntu's archive, PyPI or a download URL only what changed since `main` last saved
them.
it's set. The slow tier caches that folder, so a run resolves from Ubuntu's archive, PyPI or a
download URL only what changed since `main` last saved it.
- `tst/manual/` and `tst/benchmarks/` run only by hand, as each module's docstring says; no CI job
collects them.
- `tst/formal/` holds TLA+ models, run only by hand with each model's `check.sh`; the `AGENTS.md`
Expand Down
1 change: 1 addition & 0 deletions src/agent_env/config/describe.py
Original file line number Diff line number Diff line change
Expand Up @@ -673,6 +673,7 @@ def env_sources() -> tuple[EnvSource, ...]:
EnvSource("AGENT_ENV_MODAL_REGION", None, "no file equivalent"),
EnvSource("AGENT_ENV_MODAL_APP_NAME", None, "no file equivalent"),
EnvSource("AGENT_ENV_FIXTURE_PREFIX", None, "no file equivalent"),
EnvSource("AGENT_ENV_COMPUTER_IMAGE_MIRROR", None, "no file equivalent"),
EnvSource("MODAL_TOKEN_ID", None, "credential, no file equivalent"),
EnvSource("MODAL_TOKEN_SECRET", None, "credential, no file equivalent"),
]
Expand Down
8 changes: 8 additions & 0 deletions src/agent_env/config/runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@
objects, image builds, env and agent snapshots, changelogs, default agent and judge
trajectories, verifier outputs and the A2A validator's fixtures, so a fresh control-plane
backend can write without colliding with existing objects in a shared store. Empty in prod.

AGENT_ENV_COMPUTER_IMAGE_MIRROR (default ""): a registry repository, read anonymously, holding composed computer images
under their keys and the composer's own digest, as ``ghcr.io/org/computers``. A compose that finds no image of a key in the
env's image store copies the mirror's into it instead of building it. CI sets it so its runners share the images main
built.
"""

from __future__ import annotations
Expand Down Expand Up @@ -226,6 +231,9 @@ class Config:
fixture_prefix: str = field(
default_factory=lambda: os.getenv("AGENT_ENV_FIXTURE_PREFIX", "")
)
computer_image_mirror: str = field(
default_factory=lambda: os.getenv("AGENT_ENV_COMPUTER_IMAGE_MIRROR", "")
)
agent_sandbox_mode: str = field(
default_factory=lambda: os.getenv("AGENT_SANDBOX_MODE") or _default_sandbox_mode()
)
Expand Down
9 changes: 9 additions & 0 deletions src/agent_env/env/envs/computer/utils/composed.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,15 @@ def composition() -> str:
return "sha256:" + digest.hexdigest()


MIRROR_PREFIX_LENGTH = 16


def mirror_tag(key: str) -> str:
"""The tag of the image of ``key`` in a computer image mirror: the key, under the start of ``composition()``, so a
change to the code that composes an image builds it again rather than reuse one the old code built."""
return f"{composition().removeprefix('sha256:')[:MIRROR_PREFIX_LENGTH]}-{key.removeprefix('sha256:')}"


def _rank(result: dict) -> int:
"""How far a pair got: ready, then built and either unprobed or failed at its probe, then anything short of an
image, and below that one another composer recorded. A result takes a stored one's place only when it got at least
Expand Down
19 changes: 15 additions & 4 deletions src/agent_env/env/envs/computer/utils/composer.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,15 @@
thin_disk,
windows_disk,
)
from agent_env.env.envs.computer.utils.composed import BUILT, REPOSITORY, Record, failed, now, stale
from agent_env.env.envs.computer.utils.composed import BUILT, REPOSITORY, Record, failed, mirror_tag, now, stale
from agent_env.env.envs.computer.utils.install import Install, for_os, platform_of
from agent_env.env.published_images import DISK, DISK_TOOLS_IMAGE, published_disk, published_pinned
from agent_env.providers.sandbox_providers.host_images import platform_tag
from agent_env.store.ids import derive_id
from agent_env.store.image_store.image_contents import image_config, labels
from agent_env.store.image_store.oci_registry_credentials import names_registry
from agent_env.store.image_store.image_store import ImageStore
from agent_env.store.image_store.registry_api import pin_digest, pin_if_served, write_index
from agent_env.store.image_store.registry_api import copy_image, pin_digest, pin_if_served, write_index
from agent_env.store.local_state import holding_locks
from agent_env.utils.docker.build import DockerBuildError

Expand Down Expand Up @@ -771,9 +771,20 @@ def _key(**inputs: Any) -> str:

def _stored(env: ComputerEnv, key: str) -> tuple[ImageStore, str, str | None]:
"""The env's image store, the ref there of the image of ``key``, and that ref pinned when the store serves one
already, for any env that built it."""
already, for any env that built it, or once it's copied there from the configured computer image mirror."""
store = _image_store(env)
ref = store.image_ref(REPOSITORY, key.removeprefix("sha256:"))
tag = key.removeprefix("sha256:")
ref = store.image_ref(REPOSITORY, tag)
if (pinned := pin_if_served(ref, store.auth(ref))) is not None or not (mirror := get_config().computer_image_mirror):
return store, ref, pinned
try:
if (mirrored := pin_if_served(f"{mirror}:{mirror_tag(tag)}", None)) is None:
return store, ref, None
copy_image(mirrored, ref, None, store.auth(ref))
except (ValueError, KeyError, OSError) as e:
logger.warning(f"the computer image mirror {mirror} couldn't give {tag}, so it's built: {e}")
return store, ref, None
logger.info(f"copied {tag} from the computer image mirror {mirror}")
return store, ref, pin_if_served(ref, store.auth(ref))


Expand Down
Loading
Loading