ci.yml

name: ci

# THE CI OF ONE COMMIT, IN STAGES. The design and its measurements are in
# .agents/docs/2026-10-02-pr-ci-acceleration-and-the-toolchain-specification-design.md
# (Part II); the rules it cites are R1-R7 there.
#
#   changes   what the commit changed decides what runs (R2). A change of
#             documentation that nothing reads runs `docs` and nothing else.
#   docs      the checks that need no binary, on every change.
#   build-*   this commit's mcpp, built ONCE per host (R1) by build.yml and
#             uploaded as `mcpp-built-<host>`.
#   the rest  the per-area workflows, called as reusable workflows. Each waits
#             for its own host's build only, takes the binary with
#             .github/actions/use-built-mcpp, and keeps the job names it had as
#             a workflow of its own.
#   e2e-coverage  every e2e test ran somewhere or says why not (R5).
#
# Measured before this shape (2026-10-01, seven commits): 35 to 47 jobs started
# at once against the organisation's 20 slots, 30 to 37 of them built mcpp
# from source, and a commit waited 36 to 50 minutes, 9 to 13 of them in the
# queue.

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
  workflow_dispatch:
    inputs:
      bench:
        description: "measure the engine's own costs (bench/tests)"
        type: boolean
        default: false

# A superseded pull-request run is cancelled. Every push to main runs to the
# end, in a group of its own commit: its build jobs are the only writers of the
# caches (R3), and a cancelled run saves nothing. A group shared by all pushes
# to main would still lose runs, because GitHub keeps one pending run per group
# and cancels the older pending one when a third arrives.
concurrency:
  group: ci-${{ github.event_name == 'push' && github.sha || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
  contents: read
  # `changes` lists the files of the pull request.
  pull-requests: read
  # The e2e shards read the stores of main's last run (.github/actions/e2e-mtest).
  actions: read

jobs:
  changes:
    name: what the change starts
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    outputs:
      code: ${{ steps.classify.outputs.code }}
      set: ${{ steps.set.outputs.set }}
    steps:
      - uses: actions/checkout@v4

      # The set of cases this run runs (design 2026-11-12 §5.11): a pull
      # request's push runs `pr`, the cases its change affects; a pull request
      # labelled `ci:full`, a push to main and a run started by hand run `full`.
      # The label is read when a push starts the run, as `ci:bench` is.
      - name: The set of this run
        id: set
        env:
          EVENT: ${{ github.event_name }}
          LABELS: ${{ join(github.event.pull_request.labels.*.name, ' ') }}
        run: |
          set=full
          if [ "$EVENT" = pull_request ] && [[ " $LABELS " != *" ci:full "* ]]; then set=pr; fi
          echo "set=$set" >> "$GITHUB_OUTPUT"
          echo "the set of this run: $set"

      - name: Classify the changed paths
        id: classify
        env:
          GH_TOKEN: ${{ github.token }}
          EVENT: ${{ github.event_name }}
          REPO: ${{ github.repository }}
          PR: ${{ github.event.pull_request.number }}
          BEFORE: ${{ github.event.before }}
          SHA: ${{ github.sha }}
        run: |
          set -euo pipefail
          full() { echo "code=true" >> "$GITHUB_OUTPUT"; echo "code=true ($1)"; }
          # Anything this step cannot list runs the whole CI: an API error, a
          # force-push whose previous commit is gone, an event without a diff.
          case "$EVENT" in
            pull_request)
              gh api --paginate "repos/$REPO/pulls/$PR/files" \
                --jq '.[] | .filename, (.previous_filename // empty)' > changed.txt \
                || { full "the files of the pull request could not be listed"; exit 0; } ;;
            push)
              if [ -z "$BEFORE" ] || [ "$BEFORE" = 0000000000000000000000000000000000000000 ]; then
                full "a push with no previous commit"; exit 0
              fi
              gh api "repos/$REPO/compare/$BEFORE...$SHA" > compare.json \
                || { full "the push could not be compared with its previous commit"; exit 0; }
              # The compare API lists at most 300 files; a longer list is
              # treated as a change of everything.
              if [ "$(jq '.files | length' compare.json)" -ge 300 ]; then
                full "300 or more changed files"; exit 0
              fi
              jq -r '.files[] | .filename, (.previous_filename // empty)' compare.json > changed.txt ;;
            *)
              full "event $EVENT"; exit 0 ;;
          esac
          python3 .github/tools/classify_changes.py --github-output "$GITHUB_OUTPUT" < changed.txt

  # Each host's e2e shards, by the work its set gives it there (design
  # 2026-11-12 §5.11): `mtest ci plan` divides the selection by the measured
  # history of main's stores (tests/e2e/mtest.toml [ci.plan]), so the shard
  # count follows the work, and a host the set gives nothing starts no shard.
  # A plan that cannot be made gives every host three shards.
  e2e-plan:
    name: the e2e shards of each host
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    outputs:
      linux: ${{ steps.plan.outputs.linux }}
      linux_work: ${{ steps.plan.outputs.linux_work }}
      linux_count: ${{ steps.plan.outputs.linux_count }}
      macos: ${{ steps.plan.outputs.macos }}
      macos_work: ${{ steps.plan.outputs.macos_work }}
      macos_count: ${{ steps.plan.outputs.macos_count }}
      windows: ${{ steps.plan.outputs.windows }}
      windows_work: ${{ steps.plan.outputs.windows_work }}
      windows_count: ${{ steps.plan.outputs.windows_count }}
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/e2e-mtest
        with:
          host: '*'
      - name: The plan
        id: plan
        env:
          SET: ${{ needs.changes.outputs.set }}
        run: |
          set -uo pipefail
          out="$RUNNER_TEMP/plan.out"
          if (cd tests/e2e && GITHUB_OUTPUT="$out" "$E2E_MTEST" ci plan --set "$SET" --format github); then
            # A host given no work keeps a list a matrix accepts; `work` skips it.
            sed 's/=\[\]$/=[1]/' "$out" >> "$GITHUB_OUTPUT"
            for h in linux macos windows; do
              n=$(sed -n "s/^$h=\[\(.*\)\]$/\1/p" "$out" | tr ',' '\n' | grep -c . || true)
              echo "${h}_count=${n:-0}" >> "$GITHUB_OUTPUT"
            done
          else
            echo "::warning::mtest ci plan failed: every host takes three shards"
            for h in linux macos windows; do
              printf '%s=[1,2,3]\n%s_work=true\n%s_count=3\n' "$h" "$h" "$h" >> "$GITHUB_OUTPUT"
            done
          fi

  docs:
    name: documentation and repository checks
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4

      - name: Check version / xlings pin consistency
        run: bash .github/tools/check_version_pins.sh

      - name: Check modules/ wiring
        run: bash .github/tools/check_modules_wiring.sh

      - name: Check that the engine's sources name no more systems than the baseline
        run: python3 .github/tools/check_system_names.py

      - name: Check that the engine's sources name no more toolchain payloads than the baseline
        run: python3 .github/tools/check_payload_names.py

      - name: Check that headers and conditional directives appear only in the system lock files
        run: bash .github/tools/check_include_regions.sh

      - name: Check src/build/prepare* file lengths
        run: bash .github/tools/check_file_lengths.sh

      - name: Check no walk-derived path is narrowed directly
        run: bash .github/tools/check_narrow_conversions.sh

      - name: Where the CI assertions live
        run: bash tools/lint-ci-assertions.sh

      - name: Steps assert what their names say
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          python3 tests/scripts/test_check_workflow_assertions.py
          python3 tests/scripts/test_release_install_verification.py
          python3 .github/tools/check_workflow_assertions.py --check-open

      - name: The release canary runner runs each command under the named bash
        run: python3 tests/scripts/test_release_canaries.py

      - name: The release proposes its index entry from the sealed manifest, GitCode last
        run: python3 tests/scripts/test_index_pr.py

      - name: A build restored from main's run rebuilds what changed
        run: python3 tests/scripts/test_source_mtimes.py

      - name: The protocol table of SPEC-007 names the engine's protocol
        run: python3 tests/scripts/test_protocol_table.py

      - name: The target matrix names every refusal
        run: bash .github/tools/check_matrix_reasons.sh

      - name: Check documentation style and bilingual parity
        run: bash .github/tools/check_docs_style.sh

      - name: Check documentation structure
        run: bash .github/tools/check_docs_structure.sh

      - name: The key registry and the field reference agree
        run: python3 .github/tools/check_manifest_key_classes.py

      - name: The deprecation registry, the code and docs/51 agree
        run: python3 .github/tools/check_compat_registry.py

      - name: Documented target tiers agree with the table
        run: python3 .github/tools/check_target_tiers.py

      - name: Reason tokens agree with the engine, and with the mirror
        run: bash .github/tools/check_reason_tokens.sh

      - name: Only the adapter and the composition root name the provider
        run: python3 .github/tools/check_provider_boundary.py

      - name: The tools of the CI stages pass their fixtures
        run: |
          python3 tests/scripts/test_classify_changes.py
          python3 tests/scripts/test_notify_packages.py
          python3 tests/scripts/test_check_e2e_coverage.py
          python3 tests/scripts/test_test_history.py
          python3 tests/scripts/test_check_default_toolchain_docs.py
          python3 tests/scripts/test_check_provider_boundary.py
          python3 tests/scripts/test_native_candidate_admission.py
          python3 -m unittest discover -s tests/scripts -p test_toolchain_env.py

  build-linux:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    # A failed gate cancels the run (build.yml).
    permissions:
      contents: read
      actions: write
    uses: ./.github/workflows/build.yml
    with:
      host: linux-x86_64
      runs-on: ubuntu-24.04
      # What the Linux consumers install beyond the build's own toolchain: the
      # e2e shards (musl, llvm, mingw-cross), the toolchain legs (musl, llvm).
      prewarm: gcc 16.1.0-musl; llvm 23.1.3; mingw-cross 16.1.0
      # The gate and the tests the Windows cases need are unit-linux-archive's
      # (ci-unit.yml), beside this build: with GCC the gate held this job 18.6
      # minutes after the binary was built, and every Linux job waited for it.
      gate: ''

  build-linux-arm:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    # A failed gate cancels the run (build.yml).
    permissions:
      contents: read
      actions: write
    uses: ./.github/workflows/build.yml
    with:
      host: linux-aarch64
      runs-on: ubuntu-24.04-arm
      prewarm: llvm 23.1.3; gcc 16.1.0-musl

  build-macos:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    # A failed gate cancels the run (build.yml).
    permissions:
      contents: read
      actions: write
    uses: ./.github/workflows/build.yml
    with:
      host: macos-arm64
      runs-on: macos-15

  build-windows:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    # A failed gate cancels the run (build.yml).
    permissions:
      contents: read
      actions: write
    uses: ./.github/workflows/build.yml
    with:
      host: windows-x86_64
      runs-on: windows-latest
      # The MinGW payload every Windows e2e shard installs.
      prewarm: mingw 16.1.0
      # The Windows cases the jobs with Visual Studio masked run (no compiler
      # there), and ci-fresh-install's released mcpp.
      tests: e2e/windows

  # The mcpp .xlings.json names builds the tree as a developer's clone does,
  # and so does the newest release while the pin lags it (ci-bootstrap.yml).
  bootstrap:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    uses: ./.github/workflows/ci-bootstrap.yml

  # THE LINUX UNIT AND C++ E2E TESTS, built once beside the build and run in
  # shards (ci-unit.yml; design 2026-11-14). The archive stage needs no build
  # of this commit: the bootstrap mcpp builds the tests, and not the package's
  # program; the gate and the shards run them against build-linux's mcpp.
  unit-linux-archive:
    needs: changes
    if: needs.changes.outputs.code == 'true'
    # main's measured history is read from its last run.
    permissions:
      contents: read
      actions: read
    uses: ./.github/workflows/ci-unit.yml
    with:
      stage: archive
      host: linux-x86_64
      name: linux
      runs-on: ubuntu-24.04
      # The Windows cases the MinGW cross job runs (the run stage's `cases`).
      tests: e2e/windows

  unit-linux:
    needs: [build-linux, unit-linux-archive, changes]
    # A failed gate cancels the run (ci-unit.yml).
    permissions:
      contents: read
      actions: write
    uses: ./.github/workflows/ci-unit.yml
    with:
      stage: run
      host: linux-x86_64
      name: linux
      runs-on: ubuntu-24.04
      archived: ${{ needs.unit-linux-archive.outputs.archived }}
      # Four shards of four workers: the measured run of 2026-11-13 reached 65
      # of its programs, 3970 s of work, in the 45 minutes of one job.
      shards: '[1,2,3,4]'
      # The Windows cases only a MinGW cross toolchain runs, from Linux:
      # resources (windres -O coff, since GNU ld cannot consume a .res;
      # mcpp#365) on the GNU and clang + MinGW rows, a Windows program packed
      # from Linux (the import table read, not the artifact run), and a
      # library whose legs cross a binary format.
      cases: |
        rows/1058 a gnu abi windows row compiles its resources
        resources/198 windows resources cross
        resources/923 cross dependency images keep their resources
        pack/240 pack pe zip cross
        pack/248 pack library fat pe leg

  linux:
    needs: [build-linux, changes]
    uses: ./.github/workflows/ci-linux.yml
    with:
      # A pull request's label is read when a push starts its run; a run
      # started by hand states it as an input.
      bench: ${{ github.event_name == 'push' || inputs.bench == true || contains(github.event.pull_request.labels.*.name, 'ci:bench') }}
      set: ${{ needs.changes.outputs.set }}

  linux-e2e:
    needs: [build-linux, changes, e2e-plan]
    uses: ./.github/workflows/ci-linux-e2e.yml
    with:
      set: ${{ needs.changes.outputs.set }}
      shards: ${{ needs.e2e-plan.outputs.linux || '[1,2,3]' }}
      count: ${{ needs.e2e-plan.outputs.linux_count || '3' }}
      work: ${{ needs.e2e-plan.outputs.linux_work != 'false' }}

  macos:
    needs: [build-macos, changes]
    uses: ./.github/workflows/ci-macos.yml
    with:
      set: ${{ needs.changes.outputs.set }}

  macos-e2e:
    needs: [build-macos, changes, e2e-plan]
    uses: ./.github/workflows/ci-macos-e2e.yml
    with:
      set: ${{ needs.changes.outputs.set }}
      shards: ${{ needs.e2e-plan.outputs.macos || '[1,2,3]' }}
      count: ${{ needs.e2e-plan.outputs.macos_count || '3' }}
      work: ${{ needs.e2e-plan.outputs.macos_work != 'false' }}

  macos-ios:
    needs: build-macos
    uses: ./.github/workflows/ci-macos-ios.yml

  windows:
    needs: [build-windows, changes]
    uses: ./.github/workflows/ci-windows.yml
    with:
      set: ${{ needs.changes.outputs.set }}

  windows-e2e:
    needs: [build-windows, changes, e2e-plan]
    uses: ./.github/workflows/ci-windows-e2e.yml
    with:
      set: ${{ needs.changes.outputs.set }}
      shards: ${{ needs.e2e-plan.outputs.windows || '[1,2,3]' }}
      count: ${{ needs.e2e-plan.outputs.windows_count || '3' }}
      work: ${{ needs.e2e-plan.outputs.windows_work != 'false' }}

  windows-msvc-xlings:
    needs: build-windows
    uses: ./.github/workflows/ci-windows-msvc-xlings.yml

  cross:
    needs: [build-linux, build-windows]
    uses: ./.github/workflows/cross-build-test.yml

  target-matrix:
    needs: [build-linux, build-linux-arm, build-macos, build-windows]
    uses: ./.github/workflows/ci-target-matrix.yml

  # A home the previous release made, run by this commit's mcpp (D9 of the
  # 2026-10-17 upgrade-continuity design).
  upgrade:
    needs: [build-linux, build-linux-arm, build-macos, build-windows]
    uses: ./.github/workflows/ci-upgrade.yml

  # Pull requests and dispatch only, as when it was a workflow of its own.
  openkal:
    needs: [build-linux, build-linux-arm, build-macos, build-windows]
    if: github.event_name != 'push'
    uses: ./.github/workflows/openkal-cross.yml

  e2e-coverage:
    name: every e2e test runs somewhere
    needs: [changes, linux-e2e, windows-e2e, macos-e2e]
    if: ${{ always() && needs.changes.outputs.code == 'true' && !cancelled() }}
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4

      - name: Fetch the shards' reports
        uses: actions/download-artifact@v4
        with:
          pattern: e2e-report-*
          path: reports
          merge-multiple: true

      - name: Every test ran, is run by a dedicated job, or is excused
        run: python3 .github/tools/check_e2e_coverage.py --reports reports --timings-out timings

      # The measured durations, merged per host, in the format run_all.sh reads
      # from E2E_TIMINGS (a run without mtest).
      - name: Upload the measured durations
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-timings
          path: timings/
          if-no-files-found: ignore
          retention-days: 14

  # THE WORKFLOW RUN'S TESTS AS ONE REPORT (mtest `ci report`): the unit runs
  # each host kept (test-report, mtest-unit-*) and the e2e shards' stores
  # (e2e-store-*), the runs of this workflow run's group together -- the
  # summary, the failures, the slowest cases and the metrics.
  test-report:
    name: the run's tests, reported
    needs: [changes, linux, unit-linux, macos, windows, linux-e2e, macos-e2e, windows-e2e]
    if: ${{ always() && needs.changes.outputs.code == 'true' && !cancelled() }}
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/mtest
      - name: The stores of the run's jobs
        uses: actions/download-artifact@v4
        with:
          pattern: '{mtest-unit-*,e2e-store-*}'
          path: stores
      - name: One report over every job
        shell: bash
        run: |
          set -euo pipefail
          for store in stores/*/; do
            [ -f "$store/index.ndjson" ] || continue
            "$MTEST" store import --from "$store"
          done
          "$MTEST" ci report --group "$GITHUB_RUN_ID" --out test-report.md
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-report
          path: test-report.md
          if-no-files-found: ignore
          retention-days: 14