ci(e2e): build the tmachine suite bundles once per run

.github/workflows/branch-e2e.yml

@@ -208,6 +208,62 @@ jobs:
           retention-days: 5
           if-no-files-found: error
 
+  # The test bundles of the tmachine E2E suites rows that don't run
+  # openshell-conformance, built once per run instead of in every row (see
+  # e2e-suites.yml). The rows only run on amd64, so only the x86_64 guest
+  # target is built. Each bundle holds its whole family; a row picks its
+  # package at run time. Rust Cache (rust-cache.yml) saves the cache on main;
+  # keep the env and build commands aligned with its e2e-bundles job.
+  build-e2e-bundles:
+    name: E2E suite bundles (x86_64-unknown-linux-musl)
+    needs: [pr_metadata]
+    if: needs.pr_metadata.outputs.run_integration == 'true'
+    permissions:
+      contents: read
+    runs-on: linux-amd64-cpu8
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+      - uses: ./.github/actions/setup-nix
+        with:
+          prepare-shell: "true"
+          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - uses: ./.github/actions/setup-rust
+        with:
+          cache-key: e2e-bundles-v1-x86_64-unknown-linux-musl
+      - name: Build the bundles
+        env:
+          # One target directory for every family, so later families reuse
+          # testkit-ready and the dependencies the first one built. It is the
+          # root target directory, which setup-rust caches.
+          CARGO_TARGET_DIR: ${{ github.workspace }}/target
+          # Every row downloads the bundles. Line tables keep backtraces
+          # readable at a third of the size of full debuginfo.
+          CARGO_PROFILE_DEV_DEBUG: line-tables-only
+        run: |
+          set -euo pipefail
+          # Retry dependency preparation, then build once.
+          nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner
+          for family in features drivers/podman; do
+            nix run .#openshell-test-runner -- build \
+              --family "$family" --out "artifacts/e2e-bundles/${family//\//-}"
+          done
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: openshell-e2e-bundle-features-x86_64-unknown-linux-musl
+          path: artifacts/e2e-bundles/features
+          retention-days: 5
+          if-no-files-found: error
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: openshell-e2e-bundle-drivers-podman-x86_64-unknown-linux-musl
+          path: artifacts/e2e-bundles/drivers-podman
+          retention-days: 5
+          if-no-files-found: error
+
   build-external-drivers:
     name: ${{ matrix.binary }} (${{ matrix.triple }})
     needs: [pr_metadata, version]
@@ -366,7 +422,7 @@ jobs:
     uses: ./.github/workflows/e2e-suites.yml
     with:
       family: conformance
-      prebuilt-conformance: true
+      prebuilt: true
       source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
       integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
     secrets:
@@ -375,14 +431,15 @@ jobs:
   # Run the testkit feature suites (tests/suites/features) on tmachine. The
   # environments and suites are listed in e2e-suites.yml.
   feature-specific-integration:
-    needs: prepare-integration
+    needs: [prepare-integration, build-e2e-bundles]
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-suites.yml
     with:
       family: features
+      prebuilt: true
       source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
       integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
     secrets:
@@ -391,14 +448,15 @@ jobs:
   # Run the testkit driver suites (tests/suites/drivers) on tmachine. The
   # environments and suites are listed in e2e-suites.yml.
   driver-specific-integration:
-    needs: prepare-integration
+    needs: [prepare-integration, build-e2e-bundles]
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-suites.yml
     with:
       family: drivers
+      prebuilt: true
       source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
       integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
     secrets:

.github/workflows/e2e-suites.yml

@@ -7,8 +7,9 @@
 # optionally, the package in it, the gateway name the installer registers, and
 # the runner label for the environment's architecture. `openshell-test-runner`
 # builds the tests, boots the environment, runs them inside it and checks the
-# results; a conformance row can run the prebuilt `openshell-conformance`
-# instead (`prebuilt-conformance`). The `family` input selects the rows of one
+# results. With `prebuilt`, the rows run what the caller built once for the
+# run instead: conformance rows the static `openshell-conformance`, the other
+# rows their family's test bundle. The `family` input selects the rows of one
 # family, including the workspaces under it (drivers selects drivers/podman);
 # leave it empty to run every row.
 
@@ -30,8 +31,8 @@ on:
         description: Immutable ID of the prepared integration inputs artifact
         required: true
         type: string
-      prebuilt-conformance:
-        description: Run the conformance rows with the openshell-conformance-x86_64-unknown-linux-musl artifact of this run instead of building the suites in every job
+      prebuilt:
+        description: Run the rows with this run's prebuilt tests instead of building them in every job; conformance rows take the openshell-conformance-x86_64-unknown-linux-musl artifact, other rows their family's openshell-e2e-bundle-<family>-x86_64-unknown-linux-musl artifact
         required: false
         type: boolean
         default: false
@@ -105,6 +106,7 @@ jobs:
               | .package //= ""
               | .suite = (if .package == "" then .family else "\(.family)/\(.package)" end)
               | .artifact = ([.suite, .environment, .installer] | join("-") | gsub("/"; "-"))
+              | .bundle = "openshell-e2e-bundle-\(.family | gsub("/"; "-"))-x86_64-unknown-linux-musl"
             )}' <<< "$rows")
           if [ "$(jq '.include | length' <<< "$matrix")" -eq 0 ]; then
             echo "::error::No E2E suites match family '${FAMILY}'"
@@ -167,38 +169,54 @@ jobs:
           artifact-ids: ${{ inputs['integration-inputs-artifact-id'] }}
           path: artifacts
 
-      # The program is built once per run (see build-conformance in
-      # branch-e2e.yml), so the conformance rows need not each build the suites.
+      # The program and the bundles are built once per run (see
+      # build-conformance and build-e2e-bundles in branch-e2e.yml), so the rows
+      # need not each build the suites.
       - name: Download prebuilt openshell-conformance
-        if: inputs.prebuilt-conformance && matrix.family == 'conformance'
+        if: inputs.prebuilt && matrix.family == 'conformance'
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: openshell-conformance-x86_64-unknown-linux-musl
           path: artifacts/prebuilt-conformance
 
+      - name: Download prebuilt test bundle
+        if: inputs.prebuilt && matrix.family != 'conformance'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: ${{ matrix.bundle }}
+          path: artifacts/prebuilt-bundle
+
       - name: Run the suites in this environment
         env:
-          PREBUILT_CONFORMANCE: ${{ inputs.prebuilt-conformance && matrix.family == 'conformance' }}
+          PREBUILT: ${{ inputs.prebuilt }}
           ENVIRONMENT: ${{ matrix.environment }}
           INSTALLER: ${{ matrix.installer }}
           FAMILY: ${{ matrix.family }}
           PACKAGE: ${{ matrix.package }}
           GATEWAY: ${{ matrix.gateway }}
         run: |
-          # Retry dependency preparation, then run the suites once. The runner
-          # builds the suites for the guest, so it needs no prepared archives.
+          # Retry dependency preparation, then run the suites once. Without
+          # prebuilt tests, the runner builds the suites for the guest, so it
+          # needs no prepared archives.
           nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner
           # A prebuilt program replaces the build, and so the package choice.
+          # A prebuilt bundle holds the whole family; the package selects the
+          # tests to run from it.
           selection=()
-          if [ "${PREBUILT_CONFORMANCE}" = "true" ]; then
+          if [ "${PREBUILT}" = "true" ] && [ "${FAMILY}" = "conformance" ]; then
             program="artifacts/prebuilt-conformance/openshell-conformance"
             chmod +x "${program}"
             "${program}" list > /dev/null
             selection=(--program "${program}")
-          elif [ -n "${PACKAGE}" ]; then
-            selection=(--package "${PACKAGE}")
+          else
+            if [ "${PREBUILT}" = "true" ]; then
+              selection=(--bundle artifacts/prebuilt-bundle)
+            fi
+            if [ -n "${PACKAGE}" ]; then
+              selection+=(--package "${PACKAGE}")
+            fi
           fi
-          nix run .#openshell-test-runner -- \
+          nix run .#openshell-test-runner -- run \
             --environment "${ENVIRONMENT}" \
             --installer "${INSTALLER}" \
             --family "${FAMILY}" \

.github/workflows/rust-cache.yml

@@ -3,9 +3,10 @@
 
 name: Rust Cache
 
-# Branch Checks never runs on main, so this saves the rust-checks cache that PR
-# mirrors and merge groups restore. Keep env and build commands aligned with
-# the Branch Checks rust-test job so the cache keys and contents match.
+# Branch Checks and Branch E2E never run on main, so this saves the Rust
+# caches that PR mirrors and merge groups restore. Keep each job's env and
+# build commands aligned with the job that restores its cache, so the cache
+# keys and contents match.
 on:
   push:
     branches:
@@ -65,3 +66,37 @@ jobs:
           cargo nextest run --locked --profile ci --workspace --features openshell-server/test-support --no-run
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml --no-run
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/testkit/Cargo.toml --no-run
+
+  # Restored by build-e2e-bundles in branch-e2e.yml.
+  e2e-bundles:
+    name: E2E suite bundles cache (x86_64-unknown-linux-musl)
+    runs-on: linux-amd64-cpu8
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: ./.github/actions/setup-nix
+        with:
+          prepare-shell: "true"
+          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+      - id: rust
+        uses: ./.github/actions/setup-rust
+        with:
+          cache-key: e2e-bundles-v1-x86_64-unknown-linux-musl
+          save-cache: "true"
+
+      - name: Build the bundles
+        if: steps.rust.outputs.cache-hit != 'true'
+        env:
+          CARGO_TARGET_DIR: ${{ github.workspace }}/target
+          CARGO_PROFILE_DEV_DEBUG: line-tables-only
+        run: |
+          set -euo pipefail
+          nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner
+          for family in features drivers/podman; do
+            nix run .#openshell-test-runner -- build \
+              --family "$family" --out "$RUNNER_TEMP/e2e-bundles/${family//\//-}"
+          done

CI.md

@@ -215,7 +215,7 @@ candidate inputs (see [TESTING.md](TESTING.md)) and narrow the tests with a
 nextest filterset:
 
 ```shell
-nix run .#openshell-test-runner -- \
+nix run .#openshell-test-runner -- run \
   --environment ubuntu-docker-rootful --installer deb \
   --family conformance --package conformance \
   --filter 'test(/^policy_advisor::/)'

CONTRIBUTING.md

@@ -390,7 +390,7 @@ These are the primary `mise` tasks for day-to-day development:
 | `tests/testkit/` | Support library for SDK-driven end-to-end suites |
 | `tests/suites/` | End-to-end suites, grouped by test family |
 | `tests/tmachine/` | QEMU test machines provisioned with Ansible, and the `Session` API that drives them |
-| `tests/runner/` | `openshell-test-runner`: runs a test family in a tmachine environment and checks the results |
+| `tests/runner/` | `openshell-test-runner`: builds a test family for tmachine guests, runs it in a tmachine environment and checks the results |
 | `deploy/`       | Dockerfiles, Helm chart, Kubernetes manifests |
 | `docs/`         | Published Fern docs source, navigation, and content assets |
 | `fern/`         | Fern site config, components, and theme assets |

TESTING.md

@@ -224,7 +224,7 @@ Adapted from the handover notes in NVIDIA/OpenShell#4629 by Evan Lezar.
 
 ### Running suites in a tmachine environment (`tests/runner/`)
 
-`openshell-test-runner` runs a test family inside a tmachine VM, against the
+`openshell-test-runner run` runs a test family inside a tmachine VM, against the
 OpenShell that the VM's installer set up. `--family` is the path of the
 workspace under `tests/suites/`, such as `conformance` or `drivers/podman`, and
 `--package` narrows it to some of its packages. It must name a workspace, so
@@ -233,7 +233,7 @@ selects every row whose family is that name or lies under it, so `drivers`
 selects all the driver suites.
 
 ```shell
-nix run .#openshell-test-runner -- \
+nix run .#openshell-test-runner -- run \
   --environment ubuntu-docker-rootful --installer deb \
   --family conformance --package conformance
 ```
@@ -256,7 +256,13 @@ names, as the program does, such as `--filter policy_advisor::`, rather than a
 nextest filterset. The runner runs the program's `ready`
 and `run` commands in the guest and still writes `junit.xml` and `nextest.log`
 (the program's output) to the results directory. Branch E2E builds
-`openshell-conformance` once and passes it to every conformance row this way. Two runs on one host
+`openshell-conformance` once and passes it to every conformance row this way.
+
+`openshell-test-runner build --family <family> --out <dir>` builds a family's
+test bundle for the guest without booting a VM, and `run --bundle <dir>` runs
+it instead of building one. `--package` then selects the tests to run from the
+bundle, which must hold those packages. Branch E2E builds the `features` and
+`drivers/podman` bundles once and passes them to their rows this way. Two runs on one host
 need different `TMACHINE_SSH_PORT` values (default 2222). The installer
 inputs, such as the Debian package and the images, come from `artifacts/`,
 built by `nix run .#build-artifacts` or downloaded by CI.
@@ -320,7 +326,7 @@ The Keycloak fixture is `tests/support/keycloak`. Run the suite that uses it
 with:
 
 ```shell
-nix run .#openshell-test-runner -- \
+nix run .#openshell-test-runner -- run \
   --environment ubuntu-docker-rootful-keycloak --installer deb \
   --family features --package keycloak-provider-refresh
 ```
@@ -523,7 +529,7 @@ rebuilds only the RPMs. Run conformance against the installed RPMs:
 
 ```shell
 nix run .#build-artifacts
-nix run .#openshell-test-runner -- \
+nix run .#openshell-test-runner -- run \
   --environment fedora-podman-rootful --installer rpm \
   --family conformance --package conformance
 ```
@@ -561,7 +567,7 @@ the gateway with one mode (`default`, `auto`, `keep-id` or `private`); only the
 `binaries` installer applies it:
 
 ```shell
-nix run .#openshell-test-runner -- \
+nix run .#openshell-test-runner -- run \
   --environment fedora-podman-rootless-userns-keep-id --installer binaries \
   --gateway tmachine --family drivers/podman
 ```
@@ -572,7 +578,7 @@ drivers against installed deb packages on Ubuntu and rpm packages on Fedora.
 CI runs it on Docker rootful, Podman rootful, and Podman rootless guests:
 
 ```shell
-nix run .#openshell-test-runner -- \
+nix run .#openshell-test-runner -- run \
   --environment ubuntu-docker-rootful --installer deb \
   --family features --package oci-image
 ```

e2e/policy-advisor/README.md

@@ -66,7 +66,7 @@ candidate, run the `policy-advisor` suite of the `conformance` crate through the
 test runner:
 
 ```bash
-nix run .#openshell-test-runner -- \
+nix run .#openshell-test-runner -- run \
   --environment ubuntu-docker-rootful --installer deb \
   --family conformance --package conformance \
   --filter 'test(/^policy_advisor::/)'