Repository navigation
ci: workspace mcpp bootstrap pin -> 2026.11.13.1 #1009
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| # THE CI OF ONE COMMIT, IN STAGES. The design and its measurements are in | |
| # .agents/docs/2026-10-02-pr-ci-acceleration-and-the-toolchain-specification-design.md | |
| # (Part II); the rules it cites are R1-R7 there. | |
| # | |
| # changes what the commit changed decides what runs (R2). A change of | |
| # documentation that nothing reads runs `docs` and nothing else. | |
| # docs the checks that need no binary, on every change. | |
| # build-* this commit's mcpp, built ONCE per host (R1) by build.yml and | |
| # uploaded as `mcpp-built-<host>`. | |
| # the rest the per-area workflows, called as reusable workflows. Each waits | |
| # for its own host's build only, takes the binary with | |
| # .github/actions/use-built-mcpp, and keeps the job names it had as | |
| # a workflow of its own. | |
| # e2e-coverage every e2e test ran somewhere or says why not (R5). | |
| # | |
| # Measured before this shape (2026-10-01, seven commits): 35 to 47 jobs started | |
| # at once against the organisation's 20 slots, 30 to 37 of them built mcpp | |
| # from source, and a commit waited 36 to 50 minutes, 9 to 13 of them in the | |
| # queue. | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| workflow_dispatch: | |
| inputs: | |
| bench: | |
| description: "measure the engine's own costs (bench/tests)" | |
| type: boolean | |
| default: false | |
| # A superseded pull-request run is cancelled. Every push to main runs to the | |
| # end, in a group of its own commit: its build jobs are the only writers of the | |
| # caches (R3), and a cancelled run saves nothing. A group shared by all pushes | |
| # to main would still lose runs, because GitHub keeps one pending run per group | |
| # and cancels the older pending one when a third arrives. | |
| concurrency: | |
| group: ci-${{ github.event_name == 'push' && github.sha || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| contents: read | |
| # `changes` lists the files of the pull request. | |
| pull-requests: read | |
| # The e2e shards read the stores of main's last run (.github/actions/e2e-mtest). | |
| actions: read | |
| jobs: | |
| changes: | |
| name: what the change starts | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| outputs: | |
| code: ${{ steps.classify.outputs.code }} | |
| set: ${{ steps.set.outputs.set }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # The set of cases this run runs (design 2026-11-12 §5.11): a pull | |
| # request's push runs `pr`, the cases its change affects; a pull request | |
| # labelled `ci:full`, a push to main and a run started by hand run `full`. | |
| # The label is read when a push starts the run, as `ci:bench` is. | |
| - name: The set of this run | |
| id: set | |
| env: | |
| EVENT: ${{ github.event_name }} | |
| LABELS: ${{ join(github.event.pull_request.labels.*.name, ' ') }} | |
| run: | | |
| set=full | |
| if [ "$EVENT" = pull_request ] && [[ " $LABELS " != *" ci:full "* ]]; then set=pr; fi | |
| echo "set=$set" >> "$GITHUB_OUTPUT" | |
| echo "the set of this run: $set" | |
| - name: Classify the changed paths | |
| id: classify | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| EVENT: ${{ github.event_name }} | |
| REPO: ${{ github.repository }} | |
| PR: ${{ github.event.pull_request.number }} | |
| BEFORE: ${{ github.event.before }} | |
| SHA: ${{ github.sha }} | |
| run: | | |
| set -euo pipefail | |
| full() { echo "code=true" >> "$GITHUB_OUTPUT"; echo "code=true ($1)"; } | |
| # Anything this step cannot list runs the whole CI: an API error, a | |
| # force-push whose previous commit is gone, an event without a diff. | |
| case "$EVENT" in | |
| pull_request) | |
| gh api --paginate "repos/$REPO/pulls/$PR/files" \ | |
| --jq '.[] | .filename, (.previous_filename // empty)' > changed.txt \ | |
| || { full "the files of the pull request could not be listed"; exit 0; } ;; | |
| push) | |
| if [ -z "$BEFORE" ] || [ "$BEFORE" = 0000000000000000000000000000000000000000 ]; then | |
| full "a push with no previous commit"; exit 0 | |
| fi | |
| gh api "repos/$REPO/compare/$BEFORE...$SHA" > compare.json \ | |
| || { full "the push could not be compared with its previous commit"; exit 0; } | |
| # The compare API lists at most 300 files; a longer list is | |
| # treated as a change of everything. | |
| if [ "$(jq '.files | length' compare.json)" -ge 300 ]; then | |
| full "300 or more changed files"; exit 0 | |
| fi | |
| jq -r '.files[] | .filename, (.previous_filename // empty)' compare.json > changed.txt ;; | |
| *) | |
| full "event $EVENT"; exit 0 ;; | |
| esac | |
| python3 .github/tools/classify_changes.py --github-output "$GITHUB_OUTPUT" < changed.txt | |
| # Each host's e2e shards, by the work its set gives it there (design | |
| # 2026-11-12 §5.11): `mtest ci plan` divides the selection by the measured | |
| # history of main's stores (tests/e2e/mtest.toml [ci.plan]), so the shard | |
| # count follows the work, and a host the set gives nothing starts no shard. | |
| # A plan that cannot be made gives every host three shards. | |
| e2e-plan: | |
| name: the e2e shards of each host | |
| needs: changes | |
| if: needs.changes.outputs.code == 'true' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 10 | |
| outputs: | |
| linux: ${{ steps.plan.outputs.linux }} | |
| linux_work: ${{ steps.plan.outputs.linux_work }} | |
| linux_count: ${{ steps.plan.outputs.linux_count }} | |
| macos: ${{ steps.plan.outputs.macos }} | |
| macos_work: ${{ steps.plan.outputs.macos_work }} | |
| macos_count: ${{ steps.plan.outputs.macos_count }} | |
| windows: ${{ steps.plan.outputs.windows }} | |
| windows_work: ${{ steps.plan.outputs.windows_work }} | |
| windows_count: ${{ steps.plan.outputs.windows_count }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/e2e-mtest | |
| with: | |
| host: '*' | |
| - name: The plan | |
| id: plan | |
| env: | |
| SET: ${{ needs.changes.outputs.set }} | |
| run: | | |
| set -uo pipefail | |
| out="$RUNNER_TEMP/plan.out" | |
| if (cd tests/e2e && GITHUB_OUTPUT="$out" "$E2E_MTEST" ci plan --set "$SET" --format github); then | |
| # A host given no work keeps a list a matrix accepts; `work` skips it. | |
| sed 's/=\[\]$/=[1]/' "$out" >> "$GITHUB_OUTPUT" | |
| for h in linux macos windows; do | |
| n=$(sed -n "s/^$h=\[\(.*\)\]$/\1/p" "$out" | tr ',' '\n' | grep -c . || true) | |
| echo "${h}_count=${n:-0}" >> "$GITHUB_OUTPUT" | |
| done | |
| else | |
| echo "::warning::mtest ci plan failed: every host takes three shards" | |
| for h in linux macos windows; do | |
| printf '%s=[1,2,3]\n%s_work=true\n%s_count=3\n' "$h" "$h" "$h" >> "$GITHUB_OUTPUT" | |
| done | |
| fi | |
| docs: | |
| name: documentation and repository checks | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check version / xlings pin consistency | |
| run: bash .github/tools/check_version_pins.sh | |
| - name: Check modules/ wiring | |
| run: bash .github/tools/check_modules_wiring.sh | |
| - name: Check that the engine's sources name no more systems than the baseline | |
| run: python3 .github/tools/check_system_names.py | |
| - name: Check that the engine's sources name no more toolchain payloads than the baseline | |
| run: python3 .github/tools/check_payload_names.py | |
| - name: Check that headers and conditional directives appear only in the system lock files | |
| run: bash .github/tools/check_include_regions.sh | |
| - name: Check src/build/prepare* file lengths | |
| run: bash .github/tools/check_file_lengths.sh | |
| - name: Check no walk-derived path is narrowed directly | |
| run: bash .github/tools/check_narrow_conversions.sh | |
| - name: Where the CI assertions live | |
| run: bash tools/lint-ci-assertions.sh | |
| - name: Steps assert what their names say | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| python3 tests/scripts/test_check_workflow_assertions.py | |
| python3 tests/scripts/test_release_install_verification.py | |
| python3 .github/tools/check_workflow_assertions.py --check-open | |
| - name: The release canary runner runs each command under the named bash | |
| run: python3 tests/scripts/test_release_canaries.py | |
| - name: The protocol table of SPEC-007 names the engine's protocol | |
| run: python3 tests/scripts/test_protocol_table.py | |
| - name: The target matrix names every refusal | |
| run: bash .github/tools/check_matrix_reasons.sh | |
| - name: Check documentation style and bilingual parity | |
| run: bash .github/tools/check_docs_style.sh | |
| - name: Check documentation structure | |
| run: bash .github/tools/check_docs_structure.sh | |
| - name: The key registry and the field reference agree | |
| run: python3 .github/tools/check_manifest_key_classes.py | |
| - name: The deprecation registry, the code and docs/51 agree | |
| run: python3 .github/tools/check_compat_registry.py | |
| - name: Documented target tiers agree with the table | |
| run: python3 .github/tools/check_target_tiers.py | |
| - name: Reason tokens agree with the engine, and with the mirror | |
| run: bash .github/tools/check_reason_tokens.sh | |
| - name: Only the adapter and the composition root name the provider | |
| run: python3 .github/tools/check_provider_boundary.py | |
| - name: The tools of the CI stages pass their fixtures | |
| run: | | |
| python3 tests/scripts/test_classify_changes.py | |
| python3 tests/scripts/test_notify_packages.py | |
| python3 tests/scripts/test_check_e2e_coverage.py | |
| python3 tests/scripts/test_test_history.py | |
| python3 tests/scripts/test_check_default_toolchain_docs.py | |
| python3 tests/scripts/test_check_provider_boundary.py | |
| python3 tests/scripts/test_native_candidate_admission.py | |
| python3 -m unittest discover -s tests/scripts -p test_toolchain_env.py | |
| build-linux: | |
| needs: changes | |
| if: needs.changes.outputs.code == 'true' | |
| # A failed gate cancels the run (build.yml). | |
| permissions: | |
| contents: read | |
| actions: write | |
| uses: ./.github/workflows/build.yml | |
| with: | |
| host: linux-x86_64 | |
| runs-on: ubuntu-24.04 | |
| # What the Linux consumers install beyond the build's own toolchain: the | |
| # e2e shards (musl, llvm, mingw-cross), the toolchain legs (musl, llvm). | |
| prewarm: gcc 16.1.0-musl; llvm 23.1.3; mingw-cross 16.1.0 | |
| # The gate and the tests the Windows cases need are unit-linux-archive's | |
| # (ci-unit.yml), beside this build: with GCC the gate held this job 18.6 | |
| # minutes after the binary was built, and every Linux job waited for it. | |
| gate: '' | |
| build-linux-arm: | |
| needs: changes | |
| if: needs.changes.outputs.code == 'true' | |
| # A failed gate cancels the run (build.yml). | |
| permissions: | |
| contents: read | |
| actions: write | |
| uses: ./.github/workflows/build.yml | |
| with: | |
| host: linux-aarch64 | |
| runs-on: ubuntu-24.04-arm | |
| prewarm: llvm 23.1.3; gcc 16.1.0-musl | |
| build-macos: | |
| needs: changes | |
| if: needs.changes.outputs.code == 'true' | |
| # A failed gate cancels the run (build.yml). | |
| permissions: | |
| contents: read | |
| actions: write | |
| uses: ./.github/workflows/build.yml | |
| with: | |
| host: macos-arm64 | |
| runs-on: macos-15 | |
| build-windows: | |
| needs: changes | |
| if: needs.changes.outputs.code == 'true' | |
| # A failed gate cancels the run (build.yml). | |
| permissions: | |
| contents: read | |
| actions: write | |
| uses: ./.github/workflows/build.yml | |
| with: | |
| host: windows-x86_64 | |
| runs-on: windows-latest | |
| # The MinGW payload every Windows e2e shard installs. | |
| prewarm: mingw 16.1.0 | |
| # The Windows cases the jobs with Visual Studio masked run (no compiler | |
| # there), and ci-fresh-install's released mcpp. | |
| tests: e2e/windows | |
| # The mcpp .xlings.json names builds the tree as a developer's clone does, | |
| # and so does the newest release while the pin lags it (ci-bootstrap.yml). | |
| bootstrap: | |
| needs: changes | |
| if: needs.changes.outputs.code == 'true' | |
| uses: ./.github/workflows/ci-bootstrap.yml | |
| # THE LINUX UNIT AND C++ E2E TESTS, built once beside the build and run in | |
| # shards (ci-unit.yml; design 2026-11-14). The archive stage needs no build | |
| # of this commit: the bootstrap mcpp builds the tests, and not the package's | |
| # program; the gate and the shards run them against build-linux's mcpp. | |
| unit-linux-archive: | |
| needs: changes | |
| if: needs.changes.outputs.code == 'true' | |
| # main's measured history is read from its last run. | |
| permissions: | |
| contents: read | |
| actions: read | |
| uses: ./.github/workflows/ci-unit.yml | |
| with: | |
| stage: archive | |
| host: linux-x86_64 | |
| name: linux | |
| runs-on: ubuntu-24.04 | |
| # The Windows cases the MinGW cross job runs (the run stage's `cases`). | |
| tests: e2e/windows | |
| unit-linux: | |
| needs: [build-linux, unit-linux-archive, changes] | |
| # A failed gate cancels the run (ci-unit.yml). | |
| permissions: | |
| contents: read | |
| actions: write | |
| uses: ./.github/workflows/ci-unit.yml | |
| with: | |
| stage: run | |
| host: linux-x86_64 | |
| name: linux | |
| runs-on: ubuntu-24.04 | |
| archived: ${{ needs.unit-linux-archive.outputs.archived }} | |
| # Four shards of four workers: the measured run of 2026-11-13 reached 65 | |
| # of its programs, 3970 s of work, in the 45 minutes of one job. | |
| shards: '[1,2,3,4]' | |
| # The Windows cases only a MinGW cross toolchain runs, from Linux: | |
| # resources (windres -O coff, since GNU ld cannot consume a .res; | |
| # mcpp#365) on the GNU and clang + MinGW rows, a Windows program packed | |
| # from Linux (the import table read, not the artifact run), and a | |
| # library whose legs cross a binary format. | |
| cases: | | |
| rows/1058 a gnu abi windows row compiles its resources | |
| resources/198 windows resources cross | |
| resources/923 cross dependency images keep their resources | |
| pack/240 pack pe zip cross | |
| pack/248 pack library fat pe leg | |
| linux: | |
| needs: [build-linux, changes] | |
| uses: ./.github/workflows/ci-linux.yml | |
| with: | |
| # A pull request's label is read when a push starts its run; a run | |
| # started by hand states it as an input. | |
| bench: ${{ github.event_name == 'push' || inputs.bench == true || contains(github.event.pull_request.labels.*.name, 'ci:bench') }} | |
| set: ${{ needs.changes.outputs.set }} | |
| linux-e2e: | |
| needs: [build-linux, changes, e2e-plan] | |
| uses: ./.github/workflows/ci-linux-e2e.yml | |
| with: | |
| set: ${{ needs.changes.outputs.set }} | |
| shards: ${{ needs.e2e-plan.outputs.linux || '[1,2,3]' }} | |
| count: ${{ needs.e2e-plan.outputs.linux_count || '3' }} | |
| work: ${{ needs.e2e-plan.outputs.linux_work != 'false' }} | |
| macos: | |
| needs: [build-macos, changes] | |
| uses: ./.github/workflows/ci-macos.yml | |
| with: | |
| set: ${{ needs.changes.outputs.set }} | |
| macos-e2e: | |
| needs: [build-macos, changes, e2e-plan] | |
| uses: ./.github/workflows/ci-macos-e2e.yml | |
| with: | |
| set: ${{ needs.changes.outputs.set }} | |
| shards: ${{ needs.e2e-plan.outputs.macos || '[1,2,3]' }} | |
| count: ${{ needs.e2e-plan.outputs.macos_count || '3' }} | |
| work: ${{ needs.e2e-plan.outputs.macos_work != 'false' }} | |
| macos-ios: | |
| needs: build-macos | |
| uses: ./.github/workflows/ci-macos-ios.yml | |
| windows: | |
| needs: [build-windows, changes] | |
| uses: ./.github/workflows/ci-windows.yml | |
| with: | |
| set: ${{ needs.changes.outputs.set }} | |
| windows-e2e: | |
| needs: [build-windows, changes, e2e-plan] | |
| uses: ./.github/workflows/ci-windows-e2e.yml | |
| with: | |
| set: ${{ needs.changes.outputs.set }} | |
| shards: ${{ needs.e2e-plan.outputs.windows || '[1,2,3]' }} | |
| count: ${{ needs.e2e-plan.outputs.windows_count || '3' }} | |
| work: ${{ needs.e2e-plan.outputs.windows_work != 'false' }} | |
| windows-msvc-xlings: | |
| needs: build-windows | |
| uses: ./.github/workflows/ci-windows-msvc-xlings.yml | |
| cross: | |
| needs: [build-linux, build-windows] | |
| uses: ./.github/workflows/cross-build-test.yml | |
| target-matrix: | |
| needs: [build-linux, build-linux-arm, build-macos, build-windows] | |
| uses: ./.github/workflows/ci-target-matrix.yml | |
| # A home the previous release made, run by this commit's mcpp (D9 of the | |
| # 2026-10-17 upgrade-continuity design). | |
| upgrade: | |
| needs: [build-linux, build-linux-arm, build-macos, build-windows] | |
| uses: ./.github/workflows/ci-upgrade.yml | |
| # Pull requests and dispatch only, as when it was a workflow of its own. | |
| openkal: | |
| needs: [build-linux, build-linux-arm, build-macos, build-windows] | |
| if: github.event_name != 'push' | |
| uses: ./.github/workflows/openkal-cross.yml | |
| e2e-coverage: | |
| name: every e2e test runs somewhere | |
| needs: [changes, linux-e2e, windows-e2e, macos-e2e] | |
| if: ${{ always() && needs.changes.outputs.code == 'true' && !cancelled() }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Fetch the shards' reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: e2e-report-* | |
| path: reports | |
| merge-multiple: true | |
| - name: Every test ran, is run by a dedicated job, or is excused | |
| run: python3 .github/tools/check_e2e_coverage.py --reports reports --timings-out timings | |
| # The measured durations, merged per host, in the format run_all.sh reads | |
| # from E2E_TIMINGS (a run without mtest). | |
| - name: Upload the measured durations | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-timings | |
| path: timings/ | |
| if-no-files-found: ignore | |
| retention-days: 14 | |
| # THE WORKFLOW RUN'S TESTS AS ONE REPORT (mtest `ci report`): the unit runs | |
| # each host kept (test-report, mtest-unit-*) and the e2e shards' stores | |
| # (e2e-store-*), the runs of this workflow run's group together -- the | |
| # summary, the failures, the slowest cases and the metrics. | |
| test-report: | |
| name: the run's tests, reported | |
| needs: [changes, linux, unit-linux, macos, windows, linux-e2e, macos-e2e, windows-e2e] | |
| if: ${{ always() && needs.changes.outputs.code == 'true' && !cancelled() }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ./.github/actions/mtest | |
| - name: The stores of the run's jobs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '{mtest-unit-*,e2e-store-*}' | |
| path: stores | |
| - name: One report over every job | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| for store in stores/*/; do | |
| [ -f "$store/index.ndjson" ] || continue | |
| "$MTEST" store import --from "$store" | |
| done | |
| "$MTEST" ci report --group "$GITHUB_RUN_ID" --out test-report.md | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: test-report | |
| path: test-report.md | |
| if-no-files-found: ignore | |
| retention-days: 14 |