diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 3c501ec79b..fe8dfbed02 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -208,6 +208,62 @@ jobs: retention-days: 5 if-no-files-found: error + # The test bundles of the tmachine E2E suites rows that don't run + # openshell-conformance, built once per run instead of in every row (see + # e2e-suites.yml). The rows only run on amd64, so only the x86_64 guest + # target is built. Each bundle holds its whole family; a row picks its + # package at run time. Rust Cache (rust-cache.yml) saves the cache on main; + # keep the env and build commands aligned with its e2e-bundles job. + build-e2e-bundles: + name: E2E suite bundles (x86_64-unknown-linux-musl) + needs: [pr_metadata] + if: needs.pr_metadata.outputs.run_integration == 'true' + permissions: + contents: read + runs-on: linux-amd64-cpu8 + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.sha }} + - uses: ./.github/actions/setup-nix + with: + prepare-shell: "true" + cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + - uses: ./.github/actions/setup-rust + with: + cache-key: e2e-bundles-v1-x86_64-unknown-linux-musl + - name: Build the bundles + env: + # One target directory for every family, so later families reuse + # testkit-ready and the dependencies the first one built. It is the + # root target directory, which setup-rust caches. + CARGO_TARGET_DIR: ${{ github.workspace }}/target + # Every row downloads the bundles. Line tables keep backtraces + # readable at a third of the size of full debuginfo. + CARGO_PROFILE_DEV_DEBUG: line-tables-only + run: | + set -euo pipefail + # Retry dependency preparation, then build once. + nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner + for family in features drivers/podman; do + nix run .#openshell-test-runner -- build \ + --family "$family" --out "artifacts/e2e-bundles/${family//\//-}" + done + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: openshell-e2e-bundle-features-x86_64-unknown-linux-musl + path: artifacts/e2e-bundles/features + retention-days: 5 + if-no-files-found: error + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: openshell-e2e-bundle-drivers-podman-x86_64-unknown-linux-musl + path: artifacts/e2e-bundles/drivers-podman + retention-days: 5 + if-no-files-found: error + build-external-drivers: name: ${{ matrix.binary }} (${{ matrix.triple }}) needs: [pr_metadata, version] @@ -366,7 +422,7 @@ jobs: uses: ./.github/workflows/e2e-suites.yml with: family: conformance - prebuilt-conformance: true + prebuilt: true source-sha: ${{ needs.prepare-integration.outputs.source_sha }} integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} secrets: @@ -375,7 +431,7 @@ jobs: # Run the testkit feature suites (tests/suites/features) on tmachine. The # environments and suites are listed in e2e-suites.yml. feature-specific-integration: - needs: prepare-integration + needs: [prepare-integration, build-e2e-bundles] permissions: actions: read contents: read @@ -383,6 +439,7 @@ jobs: uses: ./.github/workflows/e2e-suites.yml with: family: features + prebuilt: true source-sha: ${{ needs.prepare-integration.outputs.source_sha }} integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} secrets: @@ -391,7 +448,7 @@ jobs: # Run the testkit driver suites (tests/suites/drivers) on tmachine. The # environments and suites are listed in e2e-suites.yml. driver-specific-integration: - needs: prepare-integration + needs: [prepare-integration, build-e2e-bundles] permissions: actions: read contents: read @@ -399,6 +456,7 @@ jobs: uses: ./.github/workflows/e2e-suites.yml with: family: drivers + prebuilt: true source-sha: ${{ needs.prepare-integration.outputs.source_sha }} integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} secrets: diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 2fef301cae..e91a146d2e 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -7,8 +7,9 @@ # optionally, the package in it, the gateway name the installer registers, and # the runner label for the environment's architecture. `openshell-test-runner` # builds the tests, boots the environment, runs them inside it and checks the -# results; a conformance row can run the prebuilt `openshell-conformance` -# instead (`prebuilt-conformance`). The `family` input selects the rows of one +# results. With `prebuilt`, the rows run what the caller built once for the +# run instead: conformance rows the static `openshell-conformance`, the other +# rows their family's test bundle. The `family` input selects the rows of one # family, including the workspaces under it (drivers selects drivers/podman); # leave it empty to run every row. @@ -30,8 +31,8 @@ on: description: Immutable ID of the prepared integration inputs artifact required: true type: string - prebuilt-conformance: - description: Run the conformance rows with the openshell-conformance-x86_64-unknown-linux-musl artifact of this run instead of building the suites in every job + prebuilt: + description: Run the rows with this run's prebuilt tests instead of building them in every job; conformance rows take the openshell-conformance-x86_64-unknown-linux-musl artifact, other rows their family's openshell-e2e-bundle--x86_64-unknown-linux-musl artifact required: false type: boolean default: false @@ -105,6 +106,7 @@ jobs: | .package //= "" | .suite = (if .package == "" then .family else "\(.family)/\(.package)" end) | .artifact = ([.suite, .environment, .installer] | join("-") | gsub("/"; "-")) + | .bundle = "openshell-e2e-bundle-\(.family | gsub("/"; "-"))-x86_64-unknown-linux-musl" )}' <<< "$rows") if [ "$(jq '.include | length' <<< "$matrix")" -eq 0 ]; then echo "::error::No E2E suites match family '${FAMILY}'" @@ -167,38 +169,54 @@ jobs: artifact-ids: ${{ inputs['integration-inputs-artifact-id'] }} path: artifacts - # The program is built once per run (see build-conformance in - # branch-e2e.yml), so the conformance rows need not each build the suites. + # The program and the bundles are built once per run (see + # build-conformance and build-e2e-bundles in branch-e2e.yml), so the rows + # need not each build the suites. - name: Download prebuilt openshell-conformance - if: inputs.prebuilt-conformance && matrix.family == 'conformance' + if: inputs.prebuilt && matrix.family == 'conformance' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: openshell-conformance-x86_64-unknown-linux-musl path: artifacts/prebuilt-conformance + - name: Download prebuilt test bundle + if: inputs.prebuilt && matrix.family != 'conformance' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ matrix.bundle }} + path: artifacts/prebuilt-bundle + - name: Run the suites in this environment env: - PREBUILT_CONFORMANCE: ${{ inputs.prebuilt-conformance && matrix.family == 'conformance' }} + PREBUILT: ${{ inputs.prebuilt }} ENVIRONMENT: ${{ matrix.environment }} INSTALLER: ${{ matrix.installer }} FAMILY: ${{ matrix.family }} PACKAGE: ${{ matrix.package }} GATEWAY: ${{ matrix.gateway }} run: | - # Retry dependency preparation, then run the suites once. The runner - # builds the suites for the guest, so it needs no prepared archives. + # Retry dependency preparation, then run the suites once. Without + # prebuilt tests, the runner builds the suites for the guest, so it + # needs no prepared archives. nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner # A prebuilt program replaces the build, and so the package choice. + # A prebuilt bundle holds the whole family; the package selects the + # tests to run from it. selection=() - if [ "${PREBUILT_CONFORMANCE}" = "true" ]; then + if [ "${PREBUILT}" = "true" ] && [ "${FAMILY}" = "conformance" ]; then program="artifacts/prebuilt-conformance/openshell-conformance" chmod +x "${program}" "${program}" list > /dev/null selection=(--program "${program}") - elif [ -n "${PACKAGE}" ]; then - selection=(--package "${PACKAGE}") + else + if [ "${PREBUILT}" = "true" ]; then + selection=(--bundle artifacts/prebuilt-bundle) + fi + if [ -n "${PACKAGE}" ]; then + selection+=(--package "${PACKAGE}") + fi fi - nix run .#openshell-test-runner -- \ + nix run .#openshell-test-runner -- run \ --environment "${ENVIRONMENT}" \ --installer "${INSTALLER}" \ --family "${FAMILY}" \ diff --git a/.github/workflows/rust-cache.yml b/.github/workflows/rust-cache.yml index d9ec30c631..dcf7c8027d 100644 --- a/.github/workflows/rust-cache.yml +++ b/.github/workflows/rust-cache.yml @@ -3,9 +3,10 @@ name: Rust Cache -# Branch Checks never runs on main, so this saves the rust-checks cache that PR -# mirrors and merge groups restore. Keep env and build commands aligned with -# the Branch Checks rust-test job so the cache keys and contents match. +# Branch Checks and Branch E2E never run on main, so this saves the Rust +# caches that PR mirrors and merge groups restore. Keep each job's env and +# build commands aligned with the job that restores its cache, so the cache +# keys and contents match. on: push: branches: @@ -65,3 +66,37 @@ jobs: cargo nextest run --locked --profile ci --workspace --features openshell-server/test-support --no-run cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml --no-run cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/testkit/Cargo.toml --no-run + + # Restored by build-e2e-bundles in branch-e2e.yml. + e2e-bundles: + name: E2E suite bundles cache (x86_64-unknown-linux-musl) + runs-on: linux-amd64-cpu8 + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/setup-nix + with: + prepare-shell: "true" + cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + + - id: rust + uses: ./.github/actions/setup-rust + with: + cache-key: e2e-bundles-v1-x86_64-unknown-linux-musl + save-cache: "true" + + - name: Build the bundles + if: steps.rust.outputs.cache-hit != 'true' + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + CARGO_PROFILE_DEV_DEBUG: line-tables-only + run: | + set -euo pipefail + nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner + for family in features drivers/podman; do + nix run .#openshell-test-runner -- build \ + --family "$family" --out "$RUNNER_TEMP/e2e-bundles/${family//\//-}" + done diff --git a/CI.md b/CI.md index 9c9f9cdd16..aaf8a86dda 100644 --- a/CI.md +++ b/CI.md @@ -215,7 +215,7 @@ candidate inputs (see [TESTING.md](TESTING.md)) and narrow the tests with a nextest filterset: ```shell -nix run .#openshell-test-runner -- \ +nix run .#openshell-test-runner -- run \ --environment ubuntu-docker-rootful --installer deb \ --family conformance --package conformance \ --filter 'test(/^policy_advisor::/)' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e800debdf..e9f8fb829d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -390,7 +390,7 @@ These are the primary `mise` tasks for day-to-day development: | `tests/testkit/` | Support library for SDK-driven end-to-end suites | | `tests/suites/` | End-to-end suites, grouped by test family | | `tests/tmachine/` | QEMU test machines provisioned with Ansible, and the `Session` API that drives them | -| `tests/runner/` | `openshell-test-runner`: runs a test family in a tmachine environment and checks the results | +| `tests/runner/` | `openshell-test-runner`: builds a test family for tmachine guests, runs it in a tmachine environment and checks the results | | `deploy/` | Dockerfiles, Helm chart, Kubernetes manifests | | `docs/` | Published Fern docs source, navigation, and content assets | | `fern/` | Fern site config, components, and theme assets | diff --git a/TESTING.md b/TESTING.md index 7443e1380f..d380f81d98 100644 --- a/TESTING.md +++ b/TESTING.md @@ -224,7 +224,7 @@ Adapted from the handover notes in NVIDIA/OpenShell#4629 by Evan Lezar. ### Running suites in a tmachine environment (`tests/runner/`) -`openshell-test-runner` runs a test family inside a tmachine VM, against the +`openshell-test-runner run` runs a test family inside a tmachine VM, against the OpenShell that the VM's installer set up. `--family` is the path of the workspace under `tests/suites/`, such as `conformance` or `drivers/podman`, and `--package` narrows it to some of its packages. It must name a workspace, so @@ -233,7 +233,7 @@ selects every row whose family is that name or lies under it, so `drivers` selects all the driver suites. ```shell -nix run .#openshell-test-runner -- \ +nix run .#openshell-test-runner -- run \ --environment ubuntu-docker-rootful --installer deb \ --family conformance --package conformance ``` @@ -256,7 +256,13 @@ names, as the program does, such as `--filter policy_advisor::`, rather than a nextest filterset. The runner runs the program's `ready` and `run` commands in the guest and still writes `junit.xml` and `nextest.log` (the program's output) to the results directory. Branch E2E builds -`openshell-conformance` once and passes it to every conformance row this way. Two runs on one host +`openshell-conformance` once and passes it to every conformance row this way. + +`openshell-test-runner build --family --out ` builds a family's +test bundle for the guest without booting a VM, and `run --bundle ` runs +it instead of building one. `--package` then selects the tests to run from the +bundle, which must hold those packages. Branch E2E builds the `features` and +`drivers/podman` bundles once and passes them to their rows this way. Two runs on one host need different `TMACHINE_SSH_PORT` values (default 2222). The installer inputs, such as the Debian package and the images, come from `artifacts/`, built by `nix run .#build-artifacts` or downloaded by CI. @@ -320,7 +326,7 @@ The Keycloak fixture is `tests/support/keycloak`. Run the suite that uses it with: ```shell -nix run .#openshell-test-runner -- \ +nix run .#openshell-test-runner -- run \ --environment ubuntu-docker-rootful-keycloak --installer deb \ --family features --package keycloak-provider-refresh ``` @@ -523,7 +529,7 @@ rebuilds only the RPMs. Run conformance against the installed RPMs: ```shell nix run .#build-artifacts -nix run .#openshell-test-runner -- \ +nix run .#openshell-test-runner -- run \ --environment fedora-podman-rootful --installer rpm \ --family conformance --package conformance ``` @@ -561,7 +567,7 @@ the gateway with one mode (`default`, `auto`, `keep-id` or `private`); only the `binaries` installer applies it: ```shell -nix run .#openshell-test-runner -- \ +nix run .#openshell-test-runner -- run \ --environment fedora-podman-rootless-userns-keep-id --installer binaries \ --gateway tmachine --family drivers/podman ``` @@ -572,7 +578,7 @@ drivers against installed deb packages on Ubuntu and rpm packages on Fedora. CI runs it on Docker rootful, Podman rootful, and Podman rootless guests: ```shell -nix run .#openshell-test-runner -- \ +nix run .#openshell-test-runner -- run \ --environment ubuntu-docker-rootful --installer deb \ --family features --package oci-image ``` diff --git a/e2e/policy-advisor/README.md b/e2e/policy-advisor/README.md index adb700306f..a04a8bc996 100644 --- a/e2e/policy-advisor/README.md +++ b/e2e/policy-advisor/README.md @@ -66,7 +66,7 @@ candidate, run the `policy-advisor` suite of the `conformance` crate through the test runner: ```bash -nix run .#openshell-test-runner -- \ +nix run .#openshell-test-runner -- run \ --environment ubuntu-docker-rootful --installer deb \ --family conformance --package conformance \ --filter 'test(/^policy_advisor::/)' diff --git a/tests/runner/src/bundle.rs b/tests/runner/src/bundle.rs index 4bc5da475c..95d0b914ea 100644 --- a/tests/runner/src/bundle.rs +++ b/tests/runner/src/bundle.rs @@ -9,15 +9,20 @@ //! which waits for the gateway before the tests run. The guest needs only the //! `cargo-nextest` binary to run it. //! +//! `build` writes a bundle to a directory, so CI can build it once and every +//! row can run it (`run --bundle`). A `bundle.json` in it names the family +//! and the packages it was built with. +//! //! A family that ships a prebuilt test program, such as `openshell-conformance`, //! can run it instead: the bundle then holds that program alone, built once //! elsewhere for the guest target, and the runner builds nothing. +use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::process::Command; use anyhow::{Context, Result, bail, ensure}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tempfile::TempDir; /// The nextest profile the runner uses in the guest. @@ -41,6 +46,21 @@ path = "junit.xml" /// The prebuilt program's name in a program bundle. pub const PROGRAM_BINARY: &str = "test-program"; +/// The nextest archive in a nextest bundle. +const ARCHIVE: &str = "tests.tar.zst"; +/// The nextest config in a nextest bundle. +const CONFIG: &str = "nextest.toml"; +/// What a nextest bundle was built from. +const MANIFEST: &str = "bundle.json"; + +/// The family and packages a nextest bundle holds. +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +struct Manifest { + family: String, + /// Empty when the bundle holds every package in the family. + packages: Vec, +} + /// What a bundle runs the tests with. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Kind { @@ -80,7 +100,7 @@ pub fn program(program: &Path) -> Result { /// The Rust target for guests. tmachine only boots guests of the host's /// architecture, and static musl binaries don't depend on the guest's libc. -pub fn guest_target() -> Result<&'static str> { +fn guest_target() -> Result<&'static str> { if cfg!(target_arch = "x86_64") { Ok("x86_64-unknown-linux-musl") } else if cfg!(target_arch = "aarch64") { @@ -91,7 +111,7 @@ pub fn guest_target() -> Result<&'static str> { } /// Build `testkit-ready` for the guest and return its path. -pub fn build_ready(root: &Path, target: &str) -> Result { +fn build_ready(root: &Path, target: &str) -> Result { let output = Command::new("cargo") .args(["build", "--locked", "--manifest-path"]) .arg(root.join("tests/testkit/Cargo.toml")) @@ -140,26 +160,41 @@ fn built_executable(messages: &[u8], name: &str) -> Option { .find_map(|message| message.executable) } -/// Build the bundle for a family workspace (`tests/suites/`). With no -/// `packages`, the archive holds every package in the workspace. `ready` is -/// the guest's `testkit-ready`. -pub fn build(workspace: &Path, packages: &[String], target: &str, ready: &Path) -> Result { +/// Build the bundle for `family` into a new temporary directory. +pub fn build_temp(root: &Path, family: &str, packages: &[String]) -> Result { let dir = tempfile::tempdir().context("create bundle directory")?; - let stage = dir.path(); + build(root, family, packages, dir.path())?; + Ok(Bundle(dir, Kind::Nextest)) +} + +/// Build the bundle for `family`, the path of a workspace under +/// `tests/suites/`, into `out`, an empty or missing directory. With no +/// `packages`, the archive holds every package in the workspace. +pub fn build(root: &Path, family: &str, packages: &[String], out: &Path) -> Result<()> { + let workspace = crate::family_workspace(root, family)?; let manifest = workspace.join("Cargo.toml"); ensure!( manifest.is_file(), "{} is not a test family workspace", workspace.display() ); - let archive = stage.join("tests.tar.zst"); + std::fs::create_dir_all(out).with_context(|| format!("create {}", out.display()))?; + ensure!( + std::fs::read_dir(out)?.next().is_none(), + "{} is not empty", + out.display() + ); + let target = guest_target()?; + eprintln!("==> building {READY_BINARY} for {target}"); + let ready = build_ready(root, target)?; + eprintln!("==> building the {family} tests for {target}"); let mut command = Command::new("cargo"); command .args(["nextest", "archive", "--locked", "--manifest-path"]) .arg(&manifest) .args(["--target", target, "--archive-file"]) - .arg(&archive); + .arg(out.join(ARCHIVE)); if packages.is_empty() { command.arg("--workspace"); } @@ -168,11 +203,108 @@ pub fn build(workspace: &Path, packages: &[String], target: &str, ready: &Path) } run(&mut command).with_context(|| format!("build the {} test archive", workspace.display()))?; - copy_manifests(workspace, stage)?; - std::fs::write(stage.join("nextest.toml"), NEXTEST_CONFIG)?; - std::fs::copy(ready, stage.join(READY_BINARY)) + copy_manifests(&workspace, out)?; + std::fs::write(out.join(CONFIG), NEXTEST_CONFIG)?; + std::fs::copy(&ready, out.join(READY_BINARY)) .with_context(|| format!("copy {}", ready.display()))?; - Ok(Bundle(dir, Kind::Nextest)) + let manifest = Manifest { + family: family.to_string(), + packages: packages.to_vec(), + }; + std::fs::write(out.join(MANIFEST), serde_json::to_vec_pretty(&manifest)?)?; + Ok(()) +} + +/// A bundle that `build` wrote to `dir`, for running `packages` of `family` +/// (every package in it when empty). The bundle is copied, so `dir` may have +/// lost the executable bits, as a downloaded CI artifact does. +pub fn prebuilt(dir: &Path, family: &str, packages: &[String]) -> Result { + let read = std::fs::read(dir.join(MANIFEST)) + .with_context(|| format!("{} is not a test bundle: read {MANIFEST}", dir.display()))?; + let manifest: Manifest = serde_json::from_slice(&read) + .with_context(|| format!("parse {}", dir.join(MANIFEST).display()))?; + check_selection(&manifest, family, packages) + .with_context(|| format!("--bundle {}", dir.display()))?; + for name in [ARCHIVE, CONFIG, READY_BINARY] { + ensure!( + dir.join(name).is_file(), + "{} is not a test bundle: it has no {name}", + dir.display() + ); + } + + let copy = tempfile::tempdir().context("create bundle directory")?; + copy_tree(dir, copy.path())?; + let ready = copy.path().join(READY_BINARY); + std::fs::set_permissions(&ready, std::fs::Permissions::from_mode(0o755)) + .with_context(|| format!("make {} executable", ready.display()))?; + Ok(Bundle(copy, Kind::Nextest)) +} + +/// Check that a bundle built from `manifest` holds `packages` of `family`. +fn check_selection(manifest: &Manifest, family: &str, packages: &[String]) -> Result<()> { + ensure!( + manifest.family == family, + "the bundle holds the {} tests, not {family}", + manifest.family + ); + if manifest.packages.is_empty() { + return Ok(()); + } + let missing: Vec<_> = if packages.is_empty() { + vec!["every package"] + } else { + packages + .iter() + .filter(|package| !manifest.packages.contains(package)) + .map(String::as_str) + .collect() + }; + ensure!( + missing.is_empty(), + "the bundle holds only {}, not {}", + manifest.packages.join(", "), + missing.join(", ") + ); + Ok(()) +} + +/// The nextest filtersets that run the tests of `packages` that match any of +/// `filters`. nextest runs the tests that match any `-E`, so the packages and +/// the filters go in one expression. +pub fn narrow(packages: &[String], filters: &[String]) -> Vec { + if packages.is_empty() { + return filters.to_vec(); + } + let packages = packages + .iter() + .map(|package| format!("package(={package})")) + .collect::>() + .join(" | "); + if filters.is_empty() { + return vec![packages]; + } + let filters = filters + .iter() + .map(|filter| format!("({filter})")) + .collect::>() + .join(" | "); + vec![format!("({packages}) & ({filters})")] +} + +/// Copy the files under `src` into `dst`, keeping relative paths. +fn copy_tree(src: &Path, dst: &Path) -> Result<()> { + for entry in std::fs::read_dir(src).with_context(|| format!("read {}", src.display()))? { + let path = entry?.path(); + let dest = dst.join(path.file_name().unwrap_or_default()); + if path.is_dir() { + std::fs::create_dir_all(&dest)?; + copy_tree(&path, &dest)?; + } else { + std::fs::copy(&path, &dest).with_context(|| format!("copy {}", path.display()))?; + } + } + Ok(()) } /// Copy every `Cargo.toml` under `workspace` into `stage`, keeping relative @@ -226,6 +358,76 @@ mod tests { assert!(program(&dir.path().join("missing")).is_err()); } + fn write_bundle(dir: &Path, family: &str, packages: &[&str]) { + std::fs::create_dir_all(dir.join("host-gateway")).unwrap(); + for name in [ARCHIVE, CONFIG, READY_BINARY, "Cargo.toml"] { + std::fs::write(dir.join(name), name).unwrap(); + } + std::fs::write(dir.join("host-gateway/Cargo.toml"), "member").unwrap(); + let manifest = Manifest { + family: family.to_string(), + packages: packages.iter().map(ToString::to_string).collect(), + }; + std::fs::write(dir.join(MANIFEST), serde_json::to_vec(&manifest).unwrap()).unwrap(); + } + + #[test] + fn prebuilt_bundle_is_copied_with_an_executable_ready_check() { + let dir = tempfile::tempdir().unwrap(); + write_bundle(dir.path(), "features", &[]); + let bundle = prebuilt(dir.path(), "features", &["host-gateway".to_string()]).unwrap(); + assert_eq!(bundle.kind(), Kind::Nextest); + assert_ne!(bundle.path(), dir.path()); + assert_eq!( + std::fs::read(bundle.path().join("host-gateway/Cargo.toml")).unwrap(), + b"member" + ); + let mode = std::fs::metadata(bundle.path().join(READY_BINARY)) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o111, 0o111); + } + + #[test] + fn prebuilt_bundle_must_be_complete_and_of_the_family() { + let dir = tempfile::tempdir().unwrap(); + assert!(prebuilt(dir.path(), "features", &[]).is_err()); + write_bundle(dir.path(), "features", &[]); + assert!(prebuilt(dir.path(), "drivers/podman", &[]).is_err()); + std::fs::remove_file(dir.path().join(ARCHIVE)).unwrap(); + assert!(prebuilt(dir.path(), "features", &[]).is_err()); + } + + #[test] + fn selection_must_be_in_the_bundle() { + let some = Manifest { + family: "features".to_string(), + packages: vec!["oci-image".to_string()], + }; + let check = |packages: &[&str]| { + let packages: Vec<_> = packages.iter().map(ToString::to_string).collect(); + check_selection(&some, "features", &packages) + }; + assert!(check(&["oci-image"]).is_ok()); + assert!(check(&["host-gateway"]).is_err()); + assert!(check(&[]).is_err()); + } + + #[test] + fn narrow_combines_packages_and_filters() { + let strings = |items: &[&str]| items.iter().map(ToString::to_string).collect::>(); + assert_eq!(narrow(&[], &strings(&["test(a)"])), strings(&["test(a)"])); + assert_eq!( + narrow(&strings(&["oci-image", "host-gateway"]), &[]), + strings(&["package(=oci-image) | package(=host-gateway)"]) + ); + assert_eq!( + narrow(&strings(&["oci-image"]), &strings(&["test(a)", "test(b)"])), + strings(&["(package(=oci-image)) & ((test(a)) | (test(b)))"]) + ); + } + #[test] fn built_executable_finds_the_named_binary() { let messages = br#"{"reason":"compiler-artifact","target":{"name":"testkit"},"executable":null} diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs index e0a1f8c7cb..bf07210261 100644 --- a/tests/runner/src/main.rs +++ b/tests/runner/src/main.rs @@ -1,9 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! `openshell-test-runner`: run a test family in one tmachine environment. +//! `openshell-test-runner`: build a test family for tmachine guests, and run +//! it in one tmachine environment. //! -//! Run it through `nix run .#openshell-test-runner -- `, which +//! Run it through `nix run .#openshell-test-runner -- `, which //! provides the guest cross-compilation toolchain, QEMU, Ansible, and the //! tmachine configuration. @@ -11,7 +12,7 @@ use std::path::{Path, PathBuf}; use std::process::ExitCode; use anyhow::{Context, Result, ensure}; -use clap::Parser; +use clap::{Args, Parser, Subcommand}; use crate::execute::{Run, Status}; @@ -19,39 +20,68 @@ mod bundle; mod execute; mod test_env; -/// Build a test family for the guest, boot a tmachine environment with -/// OpenShell installed, run the tests in it, and check the results. #[derive(Parser)] #[command(name = "openshell-test-runner")] struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Build a test family's bundle for the guest into a directory, for + /// `run --bundle`. + Build(BuildArgs), + /// Boot a tmachine environment with OpenShell installed, run a test + /// family in it, and check the results. + Run(RunArgs), +} + +/// The tests to build or run. +#[derive(Args)] +struct Selection { + /// The test family: the path of a workspace under `tests/suites/`, e.g. + /// `conformance` or `drivers/podman`. + #[arg(long)] + family: String, + /// Only these packages of the family [default: every package]. + #[arg(long = "package", short = 'p', value_name = "PACKAGE")] + packages: Vec, +} + +#[derive(Args)] +struct BuildArgs { + #[command(flatten)] + selection: Selection, + /// The directory to write the bundle to. It must be empty or missing. + #[arg(long, value_name = "DIR")] + out: PathBuf, +} + +#[derive(Args)] +struct RunArgs { /// The tmachine environment to boot, e.g. `ubuntu-docker-rootful`. #[arg(long)] environment: String, /// The tmachine installer that installs OpenShell, e.g. `deb`. #[arg(long)] installer: String, - /// The test family: the path of a workspace under `tests/suites/`, e.g. - /// `conformance` or `drivers/podman`. - #[arg(long)] - family: String, - /// Only build these packages of the family [default: every package]. - #[arg( - long = "package", - short = 'p', - value_name = "PACKAGE", - conflicts_with = "program" - )] - packages: Vec, + #[command(flatten)] + selection: Selection, /// Narrow the tests that run; repeat to run the tests that match any /// filter. Without `--program` a filter is a nextest filterset, e.g. /// `test(/^smoke::/)`. With it, a filter is a part of a test's name, e.g. /// `smoke::`, as the program takes them. #[arg(long, value_name = "FILTER")] filter: Vec, + /// Run the family's bundle that `build` wrote to this directory instead + /// of building it. `--package` then selects the tests to run from it. + #[arg(long, value_name = "DIR", conflicts_with = "program")] + bundle: Option, /// Run this prebuilt testkit program, such as `openshell-conformance`, /// instead of building the family's tests. It must be a static binary for /// the guest's architecture (a `*-unknown-linux-musl` build). - #[arg(long, value_name = "PATH")] + #[arg(long, value_name = "PATH", conflicts_with = "packages")] program: Option, /// The gateway name the installer registers with the CLI. The packaged /// installers register `openshell`. @@ -70,7 +100,11 @@ struct Cli { async fn main() -> ExitCode { let cli = Cli::parse(); // A SIGTERM from CI cancellation must not leave QEMU holding the SSH port. - match tmachine::until_signal(run(cli)).await { + let result = match cli.command { + Command::Build(args) => tmachine::until_signal(build(args)).await, + Command::Run(args) => tmachine::until_signal(run(args)).await, + }; + match result { // Exit at once: a blocking build or stdin read would otherwise hold // up the runtime's shutdown. Err(status) => std::process::exit(status), @@ -83,28 +117,45 @@ async fn main() -> ExitCode { } } -async fn run(cli: Cli) -> Result { +async fn build(args: BuildArgs) -> Result { + let root = repo_root()?; + let Selection { family, packages } = args.selection; + let out = args.out; + // The build blocks, so it runs off the runtime thread to keep the signal + // handlers responsive. + tokio::task::spawn_blocking(move || { + bundle::build(&root, &family, &packages, &out)?; + eprintln!("==> wrote the {family} bundle to {}", out.display()); + anyhow::Ok(()) + }) + .await + .context("build task panicked")??; + Ok(ExitCode::SUCCESS) +} + +async fn run(args: RunArgs) -> Result { let root = repo_root()?; - let workspace = family_workspace(&root, &cli.family)?; + let Selection { family, packages } = &args.selection; + family_workspace(&root, family)?; // Absolute, because `load_tmachine` changes the working directory. - let results = std::path::absolute(&cli.results) - .with_context(|| format!("resolve {}", cli.results.display()))?; + let results = std::path::absolute(&args.results) + .with_context(|| format!("resolve {}", args.results.display()))?; - let bundle = if let Some(program) = &cli.program { + let mut filters = args.filter.clone(); + let bundle = if let Some(program) = &args.program { eprintln!("==> using the prebuilt {}", program.display()); bundle::program(program)? + } else if let Some(dir) = &args.bundle { + eprintln!("==> using the prebuilt bundle {}", dir.display()); + // A prebuilt bundle may hold more packages than were asked for. + filters = bundle::narrow(packages, &filters); + bundle::prebuilt(dir, family, packages)? } else { - let target = bundle::guest_target()?; // The builds block, so they run off the runtime thread to keep the // signal handlers responsive. tokio::task::spawn_blocking({ - let (root, family, packages) = (root.clone(), cli.family.clone(), cli.packages.clone()); - move || { - eprintln!("==> building {} for {target}", bundle::READY_BINARY); - let ready = bundle::build_ready(&root, target)?; - eprintln!("==> building the {family} tests for {target}"); - bundle::build(&workspace, &packages, target, &ready) - } + let (root, family, packages) = (root.clone(), family.clone(), packages.clone()); + move || bundle::build_temp(&root, &family, &packages) }) .await .context("build task panicked")?? @@ -114,18 +165,18 @@ async fn run(cli: Cli) -> Result { let status = execute::run( &tmachine, &Run { - environment: &cli.environment, - installer: &cli.installer, - gateway: &cli.gateway, + environment: &args.environment, + installer: &args.installer, + gateway: &args.gateway, bundle: &bundle, - filters: &cli.filter, + filters: &filters, results: &results, - keep: cli.keep, + keep: args.keep, }, ) .await; - let label = format!("{} / {} / {}", cli.environment, cli.installer, cli.family); + let label = format!("{} / {} / {family}", args.environment, args.installer); eprintln!("\n{label}: {status}"); eprintln!("Results: {}", results.display()); match &status { diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index 24206097f9..2b715726aa 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -274,7 +274,7 @@ let }; # Runs a test family in a tmachine environment with tests/runner - # (`nix run .#openshell-test-runner -- --environment ubuntu-docker-rootful + # (`nix run .#openshell-test-runner -- run --environment ubuntu-docker-rootful # --installer deb --family conformance`). It cross-builds guest test bundles # and drives tmachine sessions, so it needs the toolchains and the VM tools. testRunner = pkgs.writeShellApplication {