From 490ded4e200e941c69d76426b8f017a1728c9b76 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 16 Nov 2026 10:53:51 -0800 Subject: [PATCH 1/3] fix(release): chromium's compose writes its record in a folder of its own, and its first push needn't be answered v0.9.1544 composed and probed chromium, then couldn't write its record: Docker had made the folder the local registry's storage is mounted from root's. The folders are made first. And GHCR may refuse a token a package before its first push, so computer-publish takes an index the registry won't say it has as unpublished, with a warning, and creates it. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/infra_build.py | 8 +++++--- .github/workflows/publish-pypi.yml | 2 ++ tst/unit/test_infra_build_script.py | 20 ++++++++++++++++++-- tst/unit/test_release_infra_images.py | 9 +++++++++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index 47d0b08fc..df8c11312 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -286,9 +286,11 @@ def _publish_computers(folder: Path) -> int: images = {arch: f"{repository}@{digest}" for arch, digest in digests.items()} index = f"{repository}:index-" + hashlib.sha256( ",".join(images[arch] for arch in COMPUTER_ARCHES).encode()).hexdigest() - if (code := _exists(index)) == 2: - return 1 - if code == 1: + served, error = _served(index) + # GHCR may refuse a token a package before its first push, which a release's first of an app is. + if served is None and ": not found" not in error: + print(f"::warning::{index} is taken as unpublished, since the registry won't say: {error}") + if served is None: source = f"https://github.com/{os.environ['GITHUB_REPOSITORY']}" created = _tried_again(lambda: _streamed([ "docker", "buildx", "imagetools", "create", "-t", index, diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index ef4f1be58..9272100b5 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -408,8 +408,10 @@ jobs: python3 .github/scripts/infra_build.py retry "$RUNNER_TEMP/venv/bin/python" -m pip install --quiet \ --index-url https://pypi.org/simple dist/agentenv_framework-*.whl \ "agentenv-framework-protocol[app]==${protocol}" + # The folders first, as Docker makes a missing one it mounts root's, which the compose then can't write in. - name: Start the local registry, its storage in a folder handed on run: | + mkdir -p "$RUNNER_TEMP/computers/registry" "$RUNNER_TEMP/computers/composed" docker run --detach --name registry --publish 5000:5000 \ --volume "$RUNNER_TEMP/computers/registry:/var/lib/registry" mirror.gcr.io/library/registry:2 timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index f012bcfb8..bb58acf81 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -312,7 +312,8 @@ def _composed(folder: Path, changes: dict | None = None, arches=("amd64", "arm64 return folder -def _computer_registry(served: dict[str, str], created: list, children: set[str] | None = None): +def _computer_registry(served: dict[str, str], created: list, children: set[str] | None = None, + before: str = "not found"): """Docker as the registries answer: ``served`` as ``_docker`` serves it, the local registry's composed images, and the agentenv-chromium index, served once created, of ``children``, by default the two images it was made of.""" published = _docker(served) @@ -326,7 +327,7 @@ def run(command, **kw): return published(command, **kw) ref = command[4] if ref == _COMPUTER_INDEX and not created: - return subprocess.CompletedProcess(command, 1, "", f"ERROR: {ref}: not found\n") + return subprocess.CompletedProcess(command, 1, "", f"ERROR: {ref}: {before}\n") if command[5:] == ["--raw"]: raw = raws.get(ref) or (ref == _COMPUTER_INDEX and {"manifests": [ {"digest": digest} for digest in sorted(children or {_AMD64, _ARM64})]}) @@ -357,6 +358,21 @@ def test_publish_computers_joins_each_apps_platform_manifests_into_one_index_and assert released_computers(record) == computers +def test_publish_computers_creates_an_index_ghcr_wont_say_it_has_before_the_packages_first_push(script, monkeypatch, + tmp_path, workflow, + capsys): + created = [] + refused = "denied: requested access to the resource is denied" + monkeypatch.setattr(script.subprocess, "run", _computer_registry({}, created, before=refused)) + monkeypatch.setattr(script.subprocess, "Popen", lambda argv, **kw: (created.append(argv), _Process("", 0))[1]) + + assert _run(script, monkeypatch, "publish-computers", str(_composed(tmp_path))) == 0 + + out = capsys.readouterr().out + assert len(created) == 1 and f"::warning::{_COMPUTER_INDEX} is taken as unpublished" in out + assert "::error::" not in out + + @pytest.mark.parametrize("changes, arches, problem", [ ({"arm64": {"env": "@local/agentenv-framework/browsers/other"}}, ("amd64", "arm64"), "isn't an app a release publishes"), diff --git a/tst/unit/test_release_infra_images.py b/tst/unit/test_release_infra_images.py index 9bb81a882..4ad64d283 100644 --- a/tst/unit/test_release_infra_images.py +++ b/tst/unit/test_release_infra_images.py @@ -237,6 +237,15 @@ def test_what_a_registry_can_rate_limit_is_tried_again(): assert JOBS["infra-build"]["strategy"]["max-parallel"] <= 4 +def test_the_folders_a_computers_compose_writes_are_made_before_docker_mounts_one(): + """Docker makes a missing folder it mounts root's, and the compose then can't write its record beside it, as + v0.9.1544's couldn't.""" + (start,) = [run for run in _runs("computer-build") if "--name registry" in run] + + made = 'mkdir -p "$RUNNER_TEMP/computers/registry" "$RUNNER_TEMP/computers/composed"' + assert made in start and start.index(made) < start.index("docker run") + + def test_the_computers_compose_is_tried_again_in_fresh_stores_while_pypi_doesnt_list_the_protocol(): """Its resolve asks PyPI for the protocol the release published moments earlier, as v0.9.1543's did, too soon.""" (compose,) = [run for run in _runs("computer-build") if "release_computers.py compose" in run] From d771ba46fd399dfa2323dc712a6a2a22bd688c92 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 16 Nov 2026 10:53:52 -0800 Subject: [PATCH 2/3] ci: a temporary dry run of the release's chromium jobs on GitHub's runners, removed before this merges Co-Authored-By: Claude Opus 5.5 --- .github/workflows/computers-dry-run.yml | 110 ++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 .github/workflows/computers-dry-run.yml diff --git a/.github/workflows/computers-dry-run.yml b/.github/workflows/computers-dry-run.yml new file mode 100644 index 000000000..954523afd --- /dev/null +++ b/.github/workflows/computers-dry-run.yml @@ -0,0 +1,110 @@ +# Temporary, for the pull request that adds it, removed before it merges: the release's computer-build steps as publish-pypi.yml +# has them, on GitHub's runners, then computer-publish's join and the record's check with a local registry on +# 5001 standing in for ghcr, then an install of a wheel carrying that record composing chromium by pulling it. +name: Computers dry run +'on': + pull_request: + paths: + - .github/workflows/computers-dry-run.yml +permissions: + contents: read +jobs: + computer-build: + strategy: + fail-fast: true + matrix: + arch: + - amd64 + - arm64 + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + timeout-minutes: 90 + permissions: + contents: read + env: + ARCH: ${{ matrix.arch }} + AGENT_ENV_DOCUMENT_STORE: local + AGENT_ENV_OBJECT_STORE: local + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.12' + - name: Install uv from public PyPI + run: python -m pip install --quiet --index-url https://pypi.org/simple -r .github/ci-requirements.txt + - name: Pin agentenv-framework-protocol in the framework and the shipped bundles, as the release does + run: 'python3 .github/scripts/shipped_bundles.py pin-pyproject + + python3 .github/scripts/shipped_bundles.py pin + + ' + - name: Install agentenv-framework, built as the release builds it, with the protocol it published + run: "uv build --no-sources --wheel --out-dir dist\nprotocol=$(sed -n 's/^version = \"\\(.*\\)\"/\\1/p' packages/agentenv-protocol/pyproject.toml)\npython -m venv \"$RUNNER_TEMP/venv\"\npython3 .github/scripts/infra_build.py\ + \ retry \"$RUNNER_TEMP/venv/bin/python\" -m pip install --quiet \\\n --index-url https://pypi.org/simple dist/agentenv_framework-*.whl \\\n \"agentenv-framework-protocol[app]==${protocol}\"\n" + - name: Start the local registry, its storage in a folder handed on + run: "mkdir -p \"$RUNNER_TEMP/computers/registry\" \"$RUNNER_TEMP/computers/composed\"\ndocker run --detach --name registry --publish 5000:5000 \\\n --volume \"$RUNNER_TEMP/computers/registry:/var/lib/registry\"\ + \ mirror.gcr.io/library/registry:2\ntimeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done'\n" + - name: Compose and probe + env: + XDG_STATE_HOME: ${{ runner.temp }}/state + run: "# shellcheck disable=SC2016 # each try's sh expands them\npython3 .github/scripts/infra_build.py retry sh -c 'rm -rf \"$XDG_STATE_HOME\" &&\n \"$RUNNER_TEMP/venv/bin/python\" .github/scripts/release_computers.py\ + \ compose \"$ARCH\" \\\n \"$RUNNER_TEMP/computers/composed\"'\n" + - name: Stop the local registry + run: docker stop registry + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: computer-${{ matrix.arch }} + path: ${{ runner.temp }}/computers + if-no-files-found: error + retention-days: 1 + computer-publish: + needs: computer-build + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: read + env: + AGENT_ENV_DOCUMENT_STORE: local + AGENT_ENV_OBJECT_STORE: local + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + pattern: computer-* + path: ${{ runner.temp }}/computers + merge-multiple: true + - name: Serve both registries' storage as one local registry, and an empty one on 5001 standing in for ghcr + run: "docker run --detach --name registry --publish 5000:5000 \\\n --volume \"$RUNNER_TEMP/computers/registry:/var/lib/registry\" mirror.gcr.io/library/registry:2\ndocker run --detach --name released\ + \ --publish 5001:5000 mirror.gcr.io/library/registry:2\ntimeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null && curl -sf http://localhost:5001/v2/ >/dev/null; do sleep 1; done'" + - name: Write the stand-in for infra_build.py with ghcr replaced + run: "cat > \"$RUNNER_TEMP/stand_in.py\" <<'PY'\nimport importlib.util, json, os, re, sys\nspec = importlib.util.spec_from_file_location(\"infra_build\", \".github/scripts/infra_build.py\")\nib =\ + \ importlib.util.module_from_spec(spec); spec.loader.exec_module(ib)\nib.REGISTRY = \"localhost:5001/scaleapi\"\nib._sign_in = lambda: None\nif sys.argv[1] == \"verify-record\":\n ib.IMAGES =\ + \ {}\nsys.argv = [\"infra_build.py\", *sys.argv[1:]]\nsys.exit(ib.main())\nPY" + - name: Join each app's images as one index, as computer-publish does, and verify the record + run: 'python3 "$RUNNER_TEMP/stand_in.py" publish-computers "$RUNNER_TEMP/computers/composed" + + python3 -c "import json, os; t = os.environ[''RUNNER_TEMP'']; c = json.load(open(f''{t}/computers/composed/computers.json'')); json.dump({''images'': [], ''computers'': c}, open(f''{t}/record.json'', + ''w''))" + + python3 "$RUNNER_TEMP/stand_in.py" verify-record "$RUNNER_TEMP/record.json"' + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.12' + - name: Install uv from public PyPI + run: python -m pip install --quiet --index-url https://pypi.org/simple -r .github/ci-requirements.txt + - name: An install of a wheel carrying that record + run: "python3 .github/scripts/shipped_bundles.py pin-pyproject\npython3 .github/scripts/shipped_bundles.py pin\ncp \"$RUNNER_TEMP/record.json\" src/agent_env/env/published_images.json\nuv build --no-sources\ + \ --wheel --out-dir dist\nprotocol=$(sed -n 's/^version = \"\\(.*\\)\"/\\1/p' packages/agentenv-protocol/pyproject.toml)\npython -m venv \"$RUNNER_TEMP/user\"\npython3 .github/scripts/infra_build.py\ + \ retry \"$RUNNER_TEMP/user/bin/python\" -m pip install --quiet \\\n --index-url https://pypi.org/simple dist/agentenv_framework-*.whl \\\n \"agentenv-framework-protocol[app]==${protocol}\"" + - name: It composes chromium by pulling the released image, in fresh stores and an empty registry + env: + XDG_STATE_HOME: ${{ runner.temp }}/userstate + run: "set -o pipefail\ndocker rm -f registry\ndocker run --detach --name user --publish 5000:5000 mirror.gcr.io/library/registry:2\ntimeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null;\ + \ do sleep 1; done'\nmkdir -p \"$RUNNER_TEMP/bundle/tasks\"\necho '[{\"id\": \"env\", \"type\": \"deploy_env\", \"env_id\": \"chromium\"}]' > \"$RUNNER_TEMP/bundle/tasks/chromium.json\"\ncd \"$RUNNER_TEMP/bundle\"\ + \ && \"$RUNNER_TEMP/user/bin/agent-env\" run . 2>&1 | tee \"$RUNNER_TEMP/run.log\"\nif grep -E 'resolving packages|building the image' \"$RUNNER_TEMP/run.log\"; then\n echo '::error::the install\ + \ resolved or built chromium, rather than pulling it'; exit 1\nfi\n\"$RUNNER_TEMP/user/bin/python\" -c \"\nfrom agent_env.env import Env\nfrom agent_env.store.routing import namespace_routing\n\ + with namespace_routing():\n pairs = Env.get('@local/agentenv-framework/browsers/chromium').metadata['composed']['container']\nprint({arch: (r['status'], r.get('published'), r['image']) for arch,\ + \ r in pairs.items()})\nassert all(r.get('published') for r in pairs.values()), pairs\n\"" From d16e634a755e9b8bfe9c46db8099336c2e4a05ca Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 16 Nov 2026 11:01:32 -0800 Subject: [PATCH 3/3] ci: remove the temporary dry run of the release's chromium jobs, which passed On GitHub's amd64 and arm64 runners: chromium composed and probed on each, joined as one index in a stand-in for ghcr, the record verified, and an install of a wheel carrying it pulled both arches, resolving and building nothing. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/computers-dry-run.yml | 110 ------------------------ 1 file changed, 110 deletions(-) delete mode 100644 .github/workflows/computers-dry-run.yml diff --git a/.github/workflows/computers-dry-run.yml b/.github/workflows/computers-dry-run.yml deleted file mode 100644 index 954523afd..000000000 --- a/.github/workflows/computers-dry-run.yml +++ /dev/null @@ -1,110 +0,0 @@ -# Temporary, for the pull request that adds it, removed before it merges: the release's computer-build steps as publish-pypi.yml -# has them, on GitHub's runners, then computer-publish's join and the record's check with a local registry on -# 5001 standing in for ghcr, then an install of a wheel carrying that record composing chromium by pulling it. -name: Computers dry run -'on': - pull_request: - paths: - - .github/workflows/computers-dry-run.yml -permissions: - contents: read -jobs: - computer-build: - strategy: - fail-fast: true - matrix: - arch: - - amd64 - - arm64 - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} - timeout-minutes: 90 - permissions: - contents: read - env: - ARCH: ${{ matrix.arch }} - AGENT_ENV_DOCUMENT_STORE: local - AGENT_ENV_OBJECT_STORE: local - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - with: - persist-credentials: false - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 - with: - python-version: '3.12' - - name: Install uv from public PyPI - run: python -m pip install --quiet --index-url https://pypi.org/simple -r .github/ci-requirements.txt - - name: Pin agentenv-framework-protocol in the framework and the shipped bundles, as the release does - run: 'python3 .github/scripts/shipped_bundles.py pin-pyproject - - python3 .github/scripts/shipped_bundles.py pin - - ' - - name: Install agentenv-framework, built as the release builds it, with the protocol it published - run: "uv build --no-sources --wheel --out-dir dist\nprotocol=$(sed -n 's/^version = \"\\(.*\\)\"/\\1/p' packages/agentenv-protocol/pyproject.toml)\npython -m venv \"$RUNNER_TEMP/venv\"\npython3 .github/scripts/infra_build.py\ - \ retry \"$RUNNER_TEMP/venv/bin/python\" -m pip install --quiet \\\n --index-url https://pypi.org/simple dist/agentenv_framework-*.whl \\\n \"agentenv-framework-protocol[app]==${protocol}\"\n" - - name: Start the local registry, its storage in a folder handed on - run: "mkdir -p \"$RUNNER_TEMP/computers/registry\" \"$RUNNER_TEMP/computers/composed\"\ndocker run --detach --name registry --publish 5000:5000 \\\n --volume \"$RUNNER_TEMP/computers/registry:/var/lib/registry\"\ - \ mirror.gcr.io/library/registry:2\ntimeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done'\n" - - name: Compose and probe - env: - XDG_STATE_HOME: ${{ runner.temp }}/state - run: "# shellcheck disable=SC2016 # each try's sh expands them\npython3 .github/scripts/infra_build.py retry sh -c 'rm -rf \"$XDG_STATE_HOME\" &&\n \"$RUNNER_TEMP/venv/bin/python\" .github/scripts/release_computers.py\ - \ compose \"$ARCH\" \\\n \"$RUNNER_TEMP/computers/composed\"'\n" - - name: Stop the local registry - run: docker stop registry - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 - with: - name: computer-${{ matrix.arch }} - path: ${{ runner.temp }}/computers - if-no-files-found: error - retention-days: 1 - computer-publish: - needs: computer-build - runs-on: ubuntu-24.04 - timeout-minutes: 60 - permissions: - contents: read - env: - AGENT_ENV_DOCUMENT_STORE: local - AGENT_ENV_OBJECT_STORE: local - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - with: - persist-credentials: false - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 - with: - pattern: computer-* - path: ${{ runner.temp }}/computers - merge-multiple: true - - name: Serve both registries' storage as one local registry, and an empty one on 5001 standing in for ghcr - run: "docker run --detach --name registry --publish 5000:5000 \\\n --volume \"$RUNNER_TEMP/computers/registry:/var/lib/registry\" mirror.gcr.io/library/registry:2\ndocker run --detach --name released\ - \ --publish 5001:5000 mirror.gcr.io/library/registry:2\ntimeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null && curl -sf http://localhost:5001/v2/ >/dev/null; do sleep 1; done'" - - name: Write the stand-in for infra_build.py with ghcr replaced - run: "cat > \"$RUNNER_TEMP/stand_in.py\" <<'PY'\nimport importlib.util, json, os, re, sys\nspec = importlib.util.spec_from_file_location(\"infra_build\", \".github/scripts/infra_build.py\")\nib =\ - \ importlib.util.module_from_spec(spec); spec.loader.exec_module(ib)\nib.REGISTRY = \"localhost:5001/scaleapi\"\nib._sign_in = lambda: None\nif sys.argv[1] == \"verify-record\":\n ib.IMAGES =\ - \ {}\nsys.argv = [\"infra_build.py\", *sys.argv[1:]]\nsys.exit(ib.main())\nPY" - - name: Join each app's images as one index, as computer-publish does, and verify the record - run: 'python3 "$RUNNER_TEMP/stand_in.py" publish-computers "$RUNNER_TEMP/computers/composed" - - python3 -c "import json, os; t = os.environ[''RUNNER_TEMP'']; c = json.load(open(f''{t}/computers/composed/computers.json'')); json.dump({''images'': [], ''computers'': c}, open(f''{t}/record.json'', - ''w''))" - - python3 "$RUNNER_TEMP/stand_in.py" verify-record "$RUNNER_TEMP/record.json"' - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 - with: - python-version: '3.12' - - name: Install uv from public PyPI - run: python -m pip install --quiet --index-url https://pypi.org/simple -r .github/ci-requirements.txt - - name: An install of a wheel carrying that record - run: "python3 .github/scripts/shipped_bundles.py pin-pyproject\npython3 .github/scripts/shipped_bundles.py pin\ncp \"$RUNNER_TEMP/record.json\" src/agent_env/env/published_images.json\nuv build --no-sources\ - \ --wheel --out-dir dist\nprotocol=$(sed -n 's/^version = \"\\(.*\\)\"/\\1/p' packages/agentenv-protocol/pyproject.toml)\npython -m venv \"$RUNNER_TEMP/user\"\npython3 .github/scripts/infra_build.py\ - \ retry \"$RUNNER_TEMP/user/bin/python\" -m pip install --quiet \\\n --index-url https://pypi.org/simple dist/agentenv_framework-*.whl \\\n \"agentenv-framework-protocol[app]==${protocol}\"" - - name: It composes chromium by pulling the released image, in fresh stores and an empty registry - env: - XDG_STATE_HOME: ${{ runner.temp }}/userstate - run: "set -o pipefail\ndocker rm -f registry\ndocker run --detach --name user --publish 5000:5000 mirror.gcr.io/library/registry:2\ntimeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null;\ - \ do sleep 1; done'\nmkdir -p \"$RUNNER_TEMP/bundle/tasks\"\necho '[{\"id\": \"env\", \"type\": \"deploy_env\", \"env_id\": \"chromium\"}]' > \"$RUNNER_TEMP/bundle/tasks/chromium.json\"\ncd \"$RUNNER_TEMP/bundle\"\ - \ && \"$RUNNER_TEMP/user/bin/agent-env\" run . 2>&1 | tee \"$RUNNER_TEMP/run.log\"\nif grep -E 'resolving packages|building the image' \"$RUNNER_TEMP/run.log\"; then\n echo '::error::the install\ - \ resolved or built chromium, rather than pulling it'; exit 1\nfi\n\"$RUNNER_TEMP/user/bin/python\" -c \"\nfrom agent_env.env import Env\nfrom agent_env.store.routing import namespace_routing\n\ - with namespace_routing():\n pairs = Env.get('@local/agentenv-framework/browsers/chromium').metadata['composed']['container']\nprint({arch: (r['status'], r.get('published'), r['image']) for arch,\ - \ r in pairs.items()})\nassert all(r.get('published') for r in pairs.values()), pairs\n\""