diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index c23ca0ecce..4352b5870d 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -158,6 +158,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Process Coverage + id: process-coverage if: matrix.coverage == 'true' uses: alandefreitas/cpp-actions/process-coverage@v1.9.4 with: @@ -166,3 +167,19 @@ jobs: html-report: true codecov-token: ${{ secrets.CODECOV_TOKEN }} codecov-flags: cpp + + # Save the produced LCOV report so PRs that don't change source files + # can replay it to codecov without re-running a Coverage build. + # `ci-scope-detector.yml` probes this artifact by name and merge-base SHA. + - name: Save coverage for replay + if: | + matrix.coverage == 'true' && + github.event_name == 'push' && + github.ref_name == 'develop' && + steps.process-coverage.outputs.lcov-file != '' + uses: actions/upload-artifact@v4 + with: + name: coverage-develop + path: ${{ steps.process-coverage.outputs.lcov-file }} + retention-days: 90 + if-no-files-found: error diff --git a/.github/workflows/ci-documentation.yml b/.github/workflows/ci-documentation.yml new file mode 100644 index 0000000000..0d2517205c --- /dev/null +++ b/.github/workflows/ci-documentation.yml @@ -0,0 +1,135 @@ +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# +# Per-OS validation that the docs/website/Antora-UI pipelines build +# cleanly with the installed mrdocs binary. Scope-gated: runs when +# source, docs, build, or third-party files changed (i.e. anything +# that can influence rendered documentation). Skipped on tooling / +# ci / toolchain-only PRs. + +name: Documentation + +on: + workflow_call: + inputs: + submatrix: + description: 'JSON-encoded releases submatrix from cpp-matrix' + type: string + required: true + use-develop-binaries: + description: 'true to fetch packages from develop-release instead of this run' + type: boolean + default: false + +jobs: + documentation: + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(inputs.submatrix) }} + + defaults: + run: + shell: bash + + name: ${{ matrix.os }} + runs-on: ${{ matrix.runs-on }} + container: ${{ matrix.container }} + permissions: + contents: read + steps: + - name: Container Bootstrap + uses: alandefreitas/cpp-actions/container-bootstrap@v1.9.4 + + - name: Install packages + uses: alandefreitas/cpp-actions/package-install@v1.9.4 + with: + apt-get: build-essential asciidoctor cmake bzip2 git rsync + + - name: Clone MrDocs + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Setup C++ + uses: alandefreitas/cpp-actions/setup-cpp@v1.9.4 + id: setup-cpp + with: + compiler: ${{ matrix.compiler }} + version: ${{ matrix.version }} + + # Same dual-source pattern as ci-releases.yml: fresh-build artifact + # when we built this run, develop-release fallback otherwise. + - name: Download MrDocs package (fresh build) + if: ${{ !inputs.use-develop-binaries }} + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.mrdocs-release-package-artifact }} + path: packages + + - name: Download MrDocs package (develop-release fallback) + if: ${{ inputs.use-develop-binaries }} + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p packages + case "${{ runner.os }}" in + Linux) pattern='*Linux*' ;; + Windows) pattern='*Windows*' ;; + macOS) pattern='*Darwin*' ;; + *) echo "::error::unknown runner.os: ${{ runner.os }}"; exit 1 ;; + esac + gh release download develop-release --pattern "$pattern" -D packages + + - name: Install MrDocs from Package + run: .github/scripts/install-mrdocs-package.sh + + - name: Generate Landing Page + working-directory: docs/website + run: | + npm ci + node render.js + mkdir -p ../../build/website + cp index.html ../../build/website/index.html + cp robots.txt ../../build/website/robots.txt + cp styles.css ../../build/website/styles.css + cp -r assets ../../build/website/assets + + - name: Generate Antora UI + working-directory: docs/ui + run: | + GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" + export GH_TOKEN + npm ci + npx gulp lint + npx gulp + + - name: Generate Local Documentation + working-directory: docs + run: | + GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" + export GH_TOKEN + set -x + npm ci + npx antora antora-playbook.yml --attribute branchesarray=HEAD --stacktrace --log-level=debug + mkdir -p ../build/docs-local + cp -vr build/site/* ../build/docs-local + + - name: Upload Website as Artifact + uses: actions/upload-artifact@v4 + with: + name: Website ${{ runner.os }} + path: build/website + retention-days: 30 diff --git a/.github/workflows/ci-matrix.yml b/.github/workflows/ci-matrix.yml index e72acb6dd6..1ff1b38d0a 100644 --- a/.github/workflows/ci-matrix.yml +++ b/.github/workflows/ci-matrix.yml @@ -115,6 +115,11 @@ jobs: submatrices: | releases: {{ieq is-release-build 'true'}} + # `coverage` is a raw boolean factor (undefined on non-Coverage rows), + # so `ieq` would crash. Wrap in `#if` like the `is-bottleneck` extra-value + # above to handle undefined cleanly. + coverage: {{#if coverage}}true{{/if}} + releases-and-coverage: {{#if (or (ieq is-release-build 'true') coverage)}}true{{/if}} trace-commands: true github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-publish.yml b/.github/workflows/ci-publish.yml new file mode 100644 index 0000000000..ffcc5ed0e7 --- /dev/null +++ b/.github/workflows/ci-publish.yml @@ -0,0 +1,263 @@ +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# +# Always-run aggregator and (conditionally) the publisher. The first step +# checks every upstream job's result and fails if any failed or were +# cancelled; the remaining publishing steps are gated on `should-publish` +# so the job is cheap on PRs and only renders/publishes on push events to +# develop, master, or tags. This job is the single required status check +# the branch protection rules rely on. + +name: Publish + +on: + workflow_call: + inputs: + submatrix: + description: 'JSON-encoded releases submatrix from cpp-matrix (for platform names)' + type: string + required: true + should-publish: + description: 'true on push to develop/master/tags; gates the actual publishing steps' + type: boolean + default: false + scope-detector-result: + type: string + required: true + cpp-matrix-result: + type: string + required: true + utility-tests-result: + type: string + required: true + build-result: + type: string + required: true + releases-result: + type: string + required: true + documentation-result: + type: string + required: true + +jobs: + publish: + name: Publish + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + # Aggregator: succeed only if every upstream job ended in success or + # skipped. Failure or cancellation fails this job so the branch + # protection gate fails too. + - name: Check upstream status + env: + SCOPE_DETECTOR: ${{ inputs.scope-detector-result }} + CPP_MATRIX: ${{ inputs.cpp-matrix-result }} + UTILITY_TESTS: ${{ inputs.utility-tests-result }} + BUILD: ${{ inputs.build-result }} + RELEASES: ${{ inputs.releases-result }} + DOCUMENTATION: ${{ inputs.documentation-result }} + run: | + set -euo pipefail + fail=0 + for name in SCOPE_DETECTOR CPP_MATRIX UTILITY_TESTS BUILD RELEASES DOCUMENTATION; do + val="${!name}" + case "$val" in + success|skipped) + printf ' %-15s %s\n' "$name" "$val" + ;; + *) + printf ' %-15s %s <-- failure\n' "$name" "$val" + fail=1 + ;; + esac + done + if [[ "$fail" -ne 0 ]]; then + echo "::error::one or more upstream jobs failed or were cancelled" + exit 1 + fi + echo "All upstream jobs OK." + + - name: Install packages + if: inputs.should-publish + uses: alandefreitas/cpp-actions/package-install@v1.9.4 + with: + apt-get: rsync git + + - name: Clone MrDocs + if: inputs.should-publish + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Install Node.js + if: inputs.should-publish + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Download all platform packages + if: inputs.should-publish + uses: actions/download-artifact@v4 + with: + pattern: release-packages-* + path: packages + merge-multiple: true + + - name: Download website (Linux) + if: inputs.should-publish + uses: actions/download-artifact@v4 + with: + name: Website Linux + path: build/website + + - name: Download demos (Linux) + if: inputs.should-publish + uses: actions/download-artifact@v4 + with: + name: demos${{ (contains(fromJSON('["master", "develop"]'), github.ref_name) && format('-{0}', github.ref_name)) || '' }}-Linux + path: . + + - name: Unpack demos + if: inputs.should-publish + run: | + set -euo pipefail + mkdir -p demos + # generate-demos.sh writes a bzip2 archive despite the .gz suffix; + # -xaf auto-detects compression so we don't have to depend on the + # script keeping the same algorithm. + tar -xaf demos.tar.gz -C demos + + - name: Ensure all refs for Antora + if: inputs.should-publish + run: | + set -euo pipefail + # Make sure Antora sees every branch and tag from the upstream repo, + # regardless of who triggered the workflow. + git remote set-url origin https://github.com/cppalliance/mrdocs.git + git fetch --prune --prune-tags origin \ + '+refs/heads/*:refs/remotes/origin/*' \ + '+refs/tags/*:refs/tags/*' + + # Antora's playbook references `./ui/build/ui-bundle.zip` (relative to + # docs/), so we need the gulp-built bundle in this runner. We could pass + # it as an artifact from Documentation/Linux, but rebuilding here is + # cheap and keeps Publish self-contained. + - name: Generate Antora UI + if: inputs.should-publish + working-directory: docs/ui + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + npm ci + npx gulp + + - name: Generate Remote Documentation + if: inputs.should-publish + working-directory: docs + run: | + GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" + export GH_TOKEN + set -x + npm ci + npx antora --clean --fetch antora-playbook.yml --log-level=debug + mkdir -p ../build/website/docs + cp -vr build/site/* ../build/website/docs + + - name: Download previous demos (tags only) + if: inputs.should-publish && startsWith(github.ref, 'refs/tags/') + id: download-prev-demos + uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: demos-develop-Linux + path: demos-previous + + - name: Compare demos (tags only) + if: inputs.should-publish && startsWith(github.ref, 'refs/tags/') && steps.download-prev-demos.outputs.cache-hit == 'true' + id: compare-demos + run: .github/scripts/compare-demos.sh + + - name: Upload demo diff (tags only) + if: inputs.should-publish && startsWith(github.ref, 'refs/tags/') && steps.compare-demos.outputs.diff == 'true' + uses: actions/upload-artifact@v4 + with: + name: demos-diff + path: demos-diff + retention-days: 30 + + - name: Publish website to GitHub Pages + if: inputs.should-publish + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: build/website + force_orphan: true + + - name: Publish website to dev-websites + if: inputs.should-publish + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: | + set -euvx + mkdir -p /home/runner/.ssh + ssh-keyscan dev-websites.cpp.al >> /home/runner/.ssh/known_hosts + chmod 600 /home/runner/.ssh/known_hosts + echo "${{ secrets.DEV_WEBSITES_SSH_KEY }}" > /home/runner/.ssh/github_actions + chmod 600 /home/runner/.ssh/github_actions + ssh-agent -a $SSH_AUTH_SOCK > /dev/null + ssh-add /home/runner/.ssh/github_actions + + rsyncopts=(--recursive --delete --links --times --chmod=D0755,F0755 --compress --compress-choice=zstd --rsh="ssh -o StrictHostKeyChecking=no" --human-readable) + website_dir="ubuntu@dev-websites.cpp.al:/var/www/mrdox.com" + demo_dir="$website_dir/demos/${{ github.ref_name }}" + + time rsync "${rsyncopts[@]}" --exclude=demos/ --exclude=roadmap/ $(pwd)/build/website/ "$website_dir"/ + time rsync "${rsyncopts[@]}" $(pwd)/demos/ "$demo_dir"/ + + - name: Create changelog + if: inputs.should-publish + uses: alandefreitas/cpp-actions/create-changelog@v1.9.4 + with: + output-path: CHANGELOG.md + thank-non-regular: ${{ startsWith(github.ref, 'refs/tags/') }} + github-token: ${{ secrets.GITHUB_TOKEN }} + limit: 150 + update-summary: true + + # For non-tag publishes (develop-release and master-release rolling + # releases), strip the project version out of the package filenames + # and insert the branch name instead. Subsequent pushes overwrite the + # existing GitHub-release assets cleanly. Tag releases keep their + # versioned filenames. + - name: Rebrand branch packages + if: inputs.should-publish && !startsWith(github.ref, 'refs/tags/') + run: | + set -euxo pipefail + cd packages + for f in MrDocs-*.*.*-*.*; do + [ -e "$f" ] || continue + new=$(echo "$f" | sed -E 's|^MrDocs-[0-9]+\.[0-9]+\.[0-9]+-|MrDocs-${{ github.ref_name }}-|') + mv -- "$f" "$new" + done + + - name: Create consolidated GitHub Release + if: inputs.should-publish + uses: softprops/action-gh-release@v2 + with: + files: packages/MrDocs-*-*.* + fail_on_unmatched_files: true + name: ${{ github.ref_name || github.ref }} + tag_name: ${{ github.ref_name || github.ref }}${{ ((!startsWith(github.ref, 'refs/tags/')) && '-release') || '' }} + body_path: CHANGELOG.md + prerelease: false + draft: false + token: ${{ github.token }} diff --git a/.github/workflows/ci-releases.yml b/.github/workflows/ci-releases.yml index 2167bc9634..b9a3a68ca5 100644 --- a/.github/workflows/ci-releases.yml +++ b/.github/workflows/ci-releases.yml @@ -7,28 +7,31 @@ # # Official repository: https://github.com/cppalliance/mrdocs # +# Per-OS smoke validation of the produced mrdocs binary plus demo +# generation against vendored third-party projects. Runs on every PR +# (always required) so the `Releases / Linux/Windows/macOS` status +# checks always reflect an end-to-end install + run of the binary. name: Releases on: workflow_call: inputs: - submatrices: - description: 'JSON-encoded sub-matrices object from cpp-matrix' + submatrix: + description: 'JSON-encoded releases submatrix from cpp-matrix' type: string required: true + use-develop-binaries: + description: 'true to fetch packages from develop-release instead of this run' + type: boolean + default: false jobs: releases: - # `fromJSON(...).releases` deserialises to an array, so comparing it - # to the literal string '[]' never matches. Re-serialise with toJSON - # so the emptiness check actually triggers and the job is skipped - # cleanly on non-release runs (PRs, sanitizer-only matrices, etc.). - if: ${{ toJSON(fromJSON(inputs.submatrices).releases) != '[]' }} strategy: fail-fast: false matrix: - include: ${{ fromJSON(inputs.submatrices).releases }} + include: ${{ fromJSON(inputs.submatrix) }} defaults: run: @@ -38,23 +41,20 @@ jobs: runs-on: ${{ matrix.runs-on }} container: ${{ matrix.container }} permissions: - contents: write - + contents: read steps: - name: Container Bootstrap uses: alandefreitas/cpp-actions/container-bootstrap@v1.9.4 - name: Install packages uses: alandefreitas/cpp-actions/package-install@v1.9.4 - id: package-install with: apt-get: build-essential asciidoctor cmake bzip2 git rsync - name: Clone MrDocs uses: actions/checkout@v4 with: - fetch-depth: 0 - fetch-tags: true + fetch-depth: 1 - name: Install Node.js uses: actions/setup-node@v4 @@ -62,8 +62,8 @@ jobs: node-version: '20' - name: Setup Ninja + if: runner.os == 'Windows' uses: seanmiddleditch/gha-setup-ninja@v5 - if: ${{ runner.os == 'Windows' }} - name: Setup C++ uses: alandefreitas/cpp-actions/setup-cpp@v1.9.4 @@ -72,18 +72,37 @@ jobs: compiler: ${{ matrix.compiler }} version: ${{ matrix.version }} - - name: Download MrDocs package + # When use-develop-binaries=false the package comes from this run's + # ci-build artifact. When true we pull the matching platform asset + # from the rolling develop-release; the scope-detector only sets + # this when it verified the cache is warm. + - name: Download MrDocs package (fresh build) + if: ${{ !inputs.use-develop-binaries }} uses: actions/download-artifact@v4 with: name: ${{ matrix.mrdocs-release-package-artifact }} path: packages + - name: Download MrDocs package (develop-release fallback) + if: ${{ inputs.use-develop-binaries }} + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p packages + case "${{ runner.os }}" in + Linux) pattern='*Linux*' ;; + Windows) pattern='*Windows*' ;; + macOS) pattern='*Darwin*' ;; + *) echo "::error::unknown runner.os: ${{ runner.os }}"; exit 1 ;; + esac + gh release download develop-release --pattern "$pattern" -D packages + - name: Install MrDocs from Package run: .github/scripts/install-mrdocs-package.sh - name: Clone Boost.URL uses: alandefreitas/cpp-actions/boost-clone@v1.9.4 - id: boost-url-clone with: branch: develop modules: url @@ -92,108 +111,6 @@ jobs: modules-exclude-paths: '' trace-commands: true - - name: Generate Landing Page - working-directory: docs/website - run: | - npm ci - node render.js - mkdir -p ../../build/website - cp index.html ../../build/website/index.html - cp robots.txt ../../build/website/robots.txt - cp styles.css ../../build/website/styles.css - cp -r assets ../../build/website/assets - - - name: Generate Antora UI - working-directory: docs/ui - run: | - # This playbook renders the documentation - # content for the website. It includes - # master, develop, and tags. - GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" - export GH_TOKEN - npm ci - npx gulp lint - npx gulp - - # Website publishing gate: - # - Only publish on pushes to master/develop and on tags - # - Only on Linux runners (the publish steps assume GNU tooling) - # - # Use `if: env.PUBLISH_WEBSITE == 'true'` for all steps that - # write into `build/website/` and get deployed to the website. - - name: Set website publish gate - run: | - is_publish_ref='false' - if [[ "${{ github.event_name }}" == 'push' ]]; then - if [[ "${{ github.ref_name }}" == 'master' || "${{ github.ref_name }}" == 'develop' ]]; then - is_publish_ref='true' - fi - if [[ "${{ github.ref }}" == refs/tags/* ]]; then - is_publish_ref='true' - fi - fi - - publish_website="$is_publish_ref" - if [[ "${{ runner.os }}" != 'Linux' ]]; then - publish_website='false' - fi - - { - echo "IS_PUBLISH_REF=$is_publish_ref" - echo "PUBLISH_WEBSITE=$publish_website" - } >> "$GITHUB_ENV" - - - name: Ensure all refs for Antora - if: env.PUBLISH_WEBSITE == 'true' - run: | - set -euo pipefail - # Make sure Antora sees every branch and tag from the upstream repo, - # regardless of who triggered the workflow. - git remote set-url origin https://github.com/cppalliance/mrdocs.git - git fetch --prune --prune-tags origin \ - '+refs/heads/*:refs/remotes/origin/*' \ - '+refs/tags/*:refs/tags/*' - - - name: Generate Remote Documentation - # This step fetches and builds develop, master and all tags. That's - # unrelated to a PR, and is only needed for website publishing. So, skip - # it for a PR. - if: github.event_name != 'pull_request' - working-directory: docs - run: | - # This playbook renders the documentation - # content for the website. It includes - # master, develop, and tags. - GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" - export GH_TOKEN - set -x - npm ci - npx antora --clean --fetch antora-playbook.yml --log-level=debug - mkdir -p ../build/website/docs - cp -vr build/site/* ../build/website/docs - - - name: Upload Website as Artifact - uses: actions/upload-artifact@v4 - with: - name: Website ${{ runner.os }} - path: build/website - retention-days: 30 - - - name: Generate Local Documentation - working-directory: docs - run: | - # This playbook allows us to render the - # documentation content and visualize it - # before a workflow that pushes to the - # website is triggered. - GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" - export GH_TOKEN - set -x - npm ci - npx antora antora-playbook.yml --attribute branchesarray=HEAD --stacktrace --log-level=debug - mkdir -p ../build/docs-local - cp -vr build/site/* ../build/docs-local - - name: Clone Beman.Optional uses: actions/checkout@v4 with: @@ -223,128 +140,23 @@ jobs: path: mp-units - name: Patch Demo Projects - shell: bash run: | set -euo pipefail set -x - for project in beman-optional fmt nlohmann-json mp-units; do src="./examples/third-party/$project" dst="./$project" - [ -d "$src" ] || { echo "Source not found: $src" >&2; exit 1; } mkdir -p "$dst" - - # Mirror contents of $src into $dst, overwriting existing files tar -C "$src" -cf - . | tar -C "$dst" -xpf - done - - name: Generate Demos run: .github/scripts/generate-demos.sh - name: Upload Demos as Artifacts uses: actions/upload-artifact@v4 with: - name: demos${{ (contains(fromJSON('["master", "develop"]'), github.ref_name ) && format('-{0}', github.ref_name)) || '' }}-${{ runner.os }} + name: demos${{ (contains(fromJSON('["master", "develop"]'), github.ref_name) && format('-{0}', github.ref_name)) || '' }}-${{ runner.os }} path: demos.tar.gz - # develop and master are retained for longer so that they can be compared retention-days: ${{ contains(fromJSON('["master", "develop"]'), github.ref_name) && '30' || '1' }} - - - name: Download Previous Demos - if: startsWith(github.ref, 'refs/tags/') && runner.os == 'Linux' - id: download-prev-demos - uses: actions/download-artifact@v4 - continue-on-error: true - with: - name: demos-develop-${{ runner.os }} - path: demos-previous - - - name: Compare demos - if: startsWith(github.ref, 'refs/tags/') && steps.download-prev-demos.outputs.cache-hit == 'true' && runner.os == 'Linux' - id: compare-demos - run: .github/scripts/compare-demos.sh - - - name: Upload Demo Diff as Artifacts - if: startsWith(github.ref, 'refs/tags/') && steps.download-prev-demos.outputs.cache-hit == 'true' && steps.compare-demos.outputs.diff == 'true' && runner.os == 'Linux' - uses: actions/upload-artifact@v4 - with: - name: demos-diff - path: demos-diff - retention-days: 30 - - - name: Publish Website to GitHub Pages - if: env.PUBLISH_WEBSITE == 'true' - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: build/website - force_orphan: true - - - name: Publish website - if: env.PUBLISH_WEBSITE == 'true' - env: - SSH_AUTH_SOCK: /tmp/ssh_agent.sock - run: | - set -euvx - # Add SSH key - mkdir -p /home/runner/.ssh - ssh-keyscan dev-websites.cpp.al >> /home/runner/.ssh/known_hosts - chmod 600 /home/runner/.ssh/known_hosts - echo "${{ secrets.DEV_WEBSITES_SSH_KEY }}" > /home/runner/.ssh/github_actions - chmod 600 /home/runner/.ssh/github_actions - ssh-agent -a $SSH_AUTH_SOCK > /dev/null - ssh-add /home/runner/.ssh/github_actions - - rsyncopts=(--recursive --delete --links --times --chmod=D0755,F0755 --compress --compress-choice=zstd --rsh="ssh -o StrictHostKeyChecking=no" --human-readable) - website_dir="ubuntu@dev-websites.cpp.al:/var/www/mrdox.com" - demo_dir="$website_dir/demos/${{ github.ref_name }}" - - # Copy files: This step will copy the landing page and the documentation to www.mrdocs.com - time rsync "${rsyncopts[@]}" --exclude=demos/ --exclude=roadmap/ $(pwd)/build/website/ "$website_dir"/ - - # Copy demos: This step will copy the demos to www.mrdocs.com/demos - time rsync "${rsyncopts[@]}" $(pwd)/demos/ "$demo_dir"/ - - - name: Create changelog - uses: alandefreitas/cpp-actions/create-changelog@v1.9.4 - with: - output-path: CHANGELOG.md - thank-non-regular: ${{ startsWith(github.ref, 'refs/tags/') }} - github-token: ${{ secrets.GITHUB_TOKEN }} - limit: 150 - update-summary: ${{ runner.os == 'Linux' && 'true' || 'false' }} - - # For non-tag publishes (the develop-release and master-release - # rolling releases), strip the project version out of the package - # filenames and insert the branch name instead. Subsequent pushes - # to the same branch then overwrite the existing GitHub-release - # assets cleanly; without this, a version bump would leave the - # previous version's files behind as stale assets. Tag releases - # keep their versioned filenames. - - name: Rebrand branch packages with the ref name - if: env.IS_PUBLISH_REF == 'true' && !startsWith(github.ref, 'refs/tags/') - run: | - set -euxo pipefail - cd packages - for f in MrDocs-*.*.*-*.*; do - [ -e "$f" ] || continue - new=$(echo "$f" | sed -E 's|^MrDocs-[0-9]+\.[0-9]+\.[0-9]+-|MrDocs-${{ github.ref_name }}-|') - mv -- "$f" "$new" - done - - - name: Create GitHub Package Release - if: env.IS_PUBLISH_REF == 'true' - uses: softprops/action-gh-release@v2 - with: - # After the rebrand step, branch packages have the form - # MrDocs--. (no semver), so the glob - # is broader than the historical MrDocs-*.*.*-*.*. - files: packages/MrDocs-*-*.* - fail_on_unmatched_files: true - name: ${{ github.ref_name || github.ref }} - tag_name: ${{ github.ref_name || github.ref }}${{ ((!startsWith(github.ref, 'refs/tags/')) && '-release') || '' }} - body_path: CHANGELOG.md - prerelease: false - draft: false - token: ${{ github.token }} diff --git a/.github/workflows/ci-scope-detector.yml b/.github/workflows/ci-scope-detector.yml new file mode 100644 index 0000000000..ead59abc57 --- /dev/null +++ b/.github/workflows/ci-scope-detector.yml @@ -0,0 +1,281 @@ +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +# +# Official repository: https://github.com/cppalliance/mrdocs +# +# Classifies the changed files in this run into the Danger scope buckets +# (see `util/danger/logic.ts`) and decides which parts of the build matrix +# still need to execute. +# + +name: Scope Detector + +on: + workflow_call: + outputs: + is-code-change: + description: 'true when files in source/tests/golden-tests/build/third-party changed' + value: ${{ jobs.detect.outputs.is-code-change }} + is-push: + description: 'true when the triggering event is a push' + value: ${{ jobs.detect.outputs.is-push }} + force-full: + description: 'true when [full ci] marker or full-ci label is set' + value: ${{ jobs.detect.outputs.force-full }} + release-cache-warm: + description: 'true when develop-release contains all 3 platform assets' + value: ${{ jobs.detect.outputs.release-cache-warm }} + coverage-cache-warm: + description: 'true when coverage-develop artifact matches the merge-base SHA (strict)' + value: ${{ jobs.detect.outputs.coverage-cache-warm }} + run-release-build: + description: 'build release packages from this run rather than reusing develop-release' + value: ${{ jobs.detect.outputs.run-release-build }} + run-coverage-build: + description: 'build a fresh coverage report rather than replaying develop' + value: ${{ jobs.detect.outputs.run-coverage-build }} + run-full-matrix: + description: 'run the full build matrix including sanitizers' + value: ${{ jobs.detect.outputs.run-full-matrix }} + is-docs-change: + description: 'true when files in the docs scope changed' + value: ${{ jobs.detect.outputs.is-docs-change }} + run-documentation: + description: 'run the per-OS Documentation jobs (Antora + UI + local playbook)' + value: ${{ jobs.detect.outputs.run-documentation }} + matrix-selector: + description: 'which (sub)matrix ci-build should run: full | releases-and-coverage | releases | coverage | none' + value: ${{ jobs.detect.outputs.matrix-selector }} + scopes-json: + description: 'JSON array of scopes touched by this change' + value: ${{ jobs.detect.outputs.scopes-json }} + merge-base-sha: + description: 'merge-base against the base ref, used for cache lookups' + value: ${{ jobs.detect.outputs.merge-base-sha }} + +jobs: + detect: + name: Detect + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: read + outputs: + is-code-change: ${{ steps.classify.outputs.is-code-change }} + is-push: ${{ steps.classify.outputs.is-push }} + force-full: ${{ steps.classify.outputs.force-full }} + release-cache-warm: ${{ steps.probe-release.outputs.warm }} + coverage-cache-warm: ${{ steps.probe-coverage.outputs.warm }} + run-release-build: ${{ steps.derive.outputs.run-release-build }} + run-coverage-build: ${{ steps.derive.outputs.run-coverage-build }} + run-full-matrix: ${{ steps.derive.outputs.run-full-matrix }} + is-docs-change: ${{ steps.classify.outputs.is-docs-change }} + run-documentation: ${{ steps.derive.outputs.run-documentation }} + matrix-selector: ${{ steps.derive.outputs.matrix-selector }} + scopes-json: ${{ steps.classify.outputs.scopes-json }} + merge-base-sha: ${{ steps.classify.outputs.merge-base-sha }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install scope classifier + working-directory: util/danger + run: npm ci --ignore-scripts + + - name: Classify changed files + id: classify + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + BASE_REF: ${{ github.base_ref }} + PR_BODY: ${{ github.event.pull_request.body }} + PR_LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }} + run: | + set -euo pipefail + + # is-push is exposed as its own signal so consumers can react to + # push events without having to re-detect github.event_name. + is_push='false' + if [[ "$GITHUB_EVENT_NAME" == 'push' ]]; then + is_push='true' + fi + + # On PRs the diff is base..HEAD via the merge-base; on push events + # we deliberately leave the file list empty — the derive step forces + # a full matrix on pushes regardless of which files changed. + if [[ "$GITHUB_EVENT_NAME" == 'pull_request' ]]; then + merge_base=$(git merge-base "origin/$BASE_REF" HEAD) + git diff --name-only "$merge_base" HEAD > changed-files.txt + else + merge_base="$(git rev-parse HEAD)" + : > changed-files.txt + fi + + # The classifier is the one in util/danger/logic.ts — same scope + # rules Danger uses, so the two reports stay in sync. + result=$(npm --prefix util/danger run --silent danger:detect-scopes < changed-files.txt) + echo "Detector output: $result" + + # Manual overrides: a `[full ci]` marker in the PR body or a + # `full-ci` label both force the full matrix downstream. + force_full='false' + if [[ -n "${PR_BODY:-}" ]] && grep -qiF '[full ci]' <<< "$PR_BODY"; then + force_full='true' + fi + if [[ -n "${PR_LABELS:-}" ]] && echo "$PR_LABELS" | jq -e 'index("full-ci")' >/dev/null 2>&1; then + force_full='true' + fi + + # docs scope is tracked separately so Documentation jobs can run on + # docs-only PRs (where is-code-change is false but docs still need + # validating across OSes). + is_docs_change='false' + if jq -e '.scopes | index("docs")' <<< "$result" >/dev/null 2>&1; then + is_docs_change='true' + fi + + { + echo "is-push=$is_push" + echo "merge-base-sha=$merge_base" + echo "scopes-json=$(jq -c .scopes <<< "$result")" + echo "is-code-change=$(jq -r .is_code_change <<< "$result")" + echo "is-docs-change=$is_docs_change" + echo "force-full=$force_full" + } >> "$GITHUB_OUTPUT" + + - name: Probe develop-release cache warmth + id: probe-release + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # Probe failure (rate limit, missing release, missing token scope) + # leaves warm=false so we fall back to a fresh build. + warm='false' + + # Fetch the asset filenames of the rolling develop-release. + assets=$(gh release view develop-release --json assets --jq '.assets[].name' 2>/dev/null || true) + + # Cache is warm only if all three platforms are present; one missing + # platform invalidates the whole probe (we'd still need a fresh build + # for that platform to satisfy the required Releases / * checks). + have_linux='false' + have_windows='false' + have_mac='false' + while IFS= read -r asset; do + [[ -z "$asset" ]] && continue + case "$asset" in + *Linux*) have_linux='true' ;; + *Windows*) have_windows='true' ;; + *Darwin*|*macOS*) have_mac='true' ;; + esac + done <<< "$assets" + if [[ "$have_linux" == 'true' && "$have_windows" == 'true' && "$have_mac" == 'true' ]]; then + warm='true' + fi + echo "warm=$warm" >> "$GITHUB_OUTPUT" + + - name: Probe coverage-develop cache warmth (strict) + id: probe-coverage + env: + GH_TOKEN: ${{ github.token }} + MERGE_BASE: ${{ steps.classify.outputs.merge-base-sha }} + run: | + set -euo pipefail + # Strict: the artifact's workflow_run.head_sha must equal the PR's + # merge base. If develop moved forward since the PR forked, the + # match fails and we rebuild coverage — accepting a small false-cold + # rate to keep codecov's diff math honest. + warm='false' + + # Query non-expired coverage-develop artifacts and filter by the + # workflow_run.head_sha (server-side via --jq); take the first hit. + # A probe error (auth, rate limit) silently leaves warm=false. + artifact_id=$(gh api -X GET "/repos/$GITHUB_REPOSITORY/actions/artifacts" \ + -f name=coverage-develop -f per_page=100 \ + --jq '.artifacts[] | select(.expired == false and .workflow_run.head_sha == "'"$MERGE_BASE"'") | .id' \ + 2>/dev/null | head -n1 || true) + if [[ -n "${artifact_id:-}" ]]; then + warm='true' + fi + echo "warm=$warm" >> "$GITHUB_OUTPUT" + + - name: Derive build flags + id: derive + env: + IS_CODE_CHANGE: ${{ steps.classify.outputs.is-code-change }} + IS_DOCS_CHANGE: ${{ steps.classify.outputs.is-docs-change }} + IS_PUSH: ${{ steps.classify.outputs.is-push }} + FORCE_FULL: ${{ steps.classify.outputs.force-full }} + RELEASE_WARM: ${{ steps.probe-release.outputs.warm }} + COVERAGE_WARM: ${{ steps.probe-coverage.outputs.warm }} + run: | + set -euo pipefail + + # Pushes and the [full ci] override force the full matrix; we keep + # is-code-change as a pure file-diff signal so consumers (Danger, + # diagnostics) can read either bit independently. + run_full_matrix='false' + if [[ "$IS_CODE_CHANGE" == 'true' || "$IS_PUSH" == 'true' || "$FORCE_FULL" == 'true' ]]; then + run_full_matrix='true' + fi + + # Need a fresh release build whenever the full matrix runs, or when + # the develop-release cache is cold (e.g. probe failed, missing + # platform asset). Currently always true because the build dispatch + # below uses release-build entries unconditionally as a safety net. + run_release_build='false' + if [[ "$run_full_matrix" == 'true' || "$RELEASE_WARM" != 'true' ]]; then + run_release_build='true' + fi + + # Need a fresh coverage build whenever the full matrix runs, or when + # the coverage-develop cache is cold (no matching merge-base SHA). + # When false, utility-tests replays the cached LCOV to codecov. + run_coverage_build='false' + if [[ "$run_full_matrix" == 'true' || "$COVERAGE_WARM" != 'true' ]]; then + run_coverage_build='true' + fi + + # Documentation jobs run when anything affecting the binary or the + # docs source changed; skipped on tooling/ci/toolchain-only PRs. + run_documentation='false' + if [[ "$run_full_matrix" == 'true' || "$IS_DOCS_CHANGE" == 'true' ]]; then + run_documentation='true' + fi + + # Single string telling ci.yml which (sub)matrix to dispatch into + # ci-build.yml: 'full' uses cpp-matrix.outputs.matrix, the others + # are submatrix keys on cpp-matrix.outputs.submatrices. 'none' skips + # ci-build entirely (both caches warm, no code change). + if [[ "$run_full_matrix" == 'true' ]]; then + matrix_selector='full' + elif [[ "$run_release_build" == 'true' && "$run_coverage_build" == 'true' ]]; then + matrix_selector='releases-and-coverage' + elif [[ "$run_release_build" == 'true' ]]; then + matrix_selector='releases' + elif [[ "$run_coverage_build" == 'true' ]]; then + matrix_selector='coverage' + else + matrix_selector='none' + fi + + { + echo "run-full-matrix=$run_full_matrix" + echo "run-release-build=$run_release_build" + echo "run-coverage-build=$run_coverage_build" + echo "run-documentation=$run_documentation" + echo "matrix-selector=$matrix_selector" + } >> "$GITHUB_OUTPUT" + diff --git a/.github/workflows/ci-utility-tests.yml b/.github/workflows/ci-utility-tests.yml index f1a2069fff..a7dfb96a9e 100644 --- a/.github/workflows/ci-utility-tests.yml +++ b/.github/workflows/ci-utility-tests.yml @@ -12,11 +12,23 @@ name: Utility Tests on: workflow_call: + inputs: + coverage-replay: + description: 'when true, re-upload develop coverage to codecov instead of running a Coverage build' + type: boolean + default: false + merge-base-sha: + description: 'merge-base SHA used to locate the matching coverage-develop artifact' + type: string + default: '' jobs: utility-tests: runs-on: ubuntu-24.04 name: Utility Tests + permissions: + contents: read + actions: read steps: - name: Clone MrDocs @@ -58,3 +70,43 @@ jobs: - name: Verify snippet .cpp files match golden tests run: .github/scripts/verify-snippets.sh + + # When the Coverage matrix entry was skipped because the develop cache is + # warm, fetch develop's LCOV report and re-upload it tagged with the PR's + # commit SHA. Codecov sees zero patch lines and unchanged project totals, + # so codecov/patch and codecov/project post passing statuses without us + # rebuilding the whole project just to remeasure unchanged coverage. + - name: Coverage replay (fetch develop LCOV) + if: inputs.coverage-replay + env: + GH_TOKEN: ${{ github.token }} + MERGE_BASE: ${{ inputs.merge-base-sha }} + run: | + set -euo pipefail + artifact_id=$(gh api -X GET "/repos/$GITHUB_REPOSITORY/actions/artifacts" \ + -f name=coverage-develop -f per_page=100 \ + --jq '.artifacts[] | select(.expired == false and .workflow_run.head_sha == "'"$MERGE_BASE"'") | .id' \ + | head -n1) + if [[ -z "${artifact_id:-}" ]]; then + echo "::error::coverage-develop artifact disappeared between probe and replay" + exit 1 + fi + gh api -H 'Accept: application/vnd.github+json' \ + "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" > coverage.zip + unzip -o coverage.zip -d coverage-develop + lcov_path=$(find coverage-develop -maxdepth 3 -name '*.info' -print -quit) + if [[ -z "${lcov_path:-}" ]]; then + echo "::error::no LCOV file found inside coverage-develop artifact" + exit 1 + fi + echo "LCOV_PATH=$lcov_path" >> "$GITHUB_ENV" + + - name: Coverage replay (upload to codecov) + if: inputs.coverage-replay + uses: codecov/codecov-action@v5 + with: + files: ${{ env.LCOV_PATH }} + flags: cpp + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true + override_commit: ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f31900299b..54a2fb9fa5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,32 +31,111 @@ concurrency: cancel-in-progress: true jobs: + # Classifies changed files into scopes and probes develop-release / coverage + # caches to decide which (sub)matrix the build job should run. + scope-detector: + name: Scope Detector + permissions: + contents: read + actions: read + uses: ./.github/workflows/ci-scope-detector.yml + secrets: inherit + + # Generates the full build matrix and the named submatrices + # (releases, coverage, releases-and-coverage) consumed below. cpp-matrix: name: Generate Test Matrix uses: ./.github/workflows/ci-matrix.yml secrets: inherit + # Bootstrap pytest, Danger.js vitest, YAML schema, snippet verify, plus a + # coverage replay to codecov when the Coverage matrix entry is skipped. + utility-tests: + name: Utility Tests + needs: scope-detector + permissions: + contents: read + actions: read + uses: ./.github/workflows/ci-utility-tests.yml + with: + coverage-replay: ${{ needs.scope-detector.outputs.is-push == 'false' && needs.scope-detector.outputs.run-coverage-build == 'false' && needs.scope-detector.outputs.coverage-cache-warm == 'true' }} + merge-base-sha: ${{ needs.scope-detector.outputs.merge-base-sha }} + secrets: inherit + + # Builds and tests the (sub)matrix selected by scope-detector. Skipped + # entirely when matrix-selector is 'none' (both caches warm, no code). build: name: Build & Test - needs: [cpp-matrix, utility-tests] + needs: [scope-detector, cpp-matrix, utility-tests] + if: needs.scope-detector.outputs.matrix-selector != 'none' permissions: actions: write uses: ./.github/workflows/ci-build.yml with: - matrix: ${{ needs.cpp-matrix.outputs.matrix }} - secrets: inherit - - utility-tests: - name: Utility Tests - uses: ./.github/workflows/ci-utility-tests.yml + # scope-detector picks 'full' | 'releases-and-coverage' | 'releases' + # | 'coverage'. 'full' uses cpp-matrix.matrix, the others index + # into the submatrices. + matrix: >- + ${{ needs.scope-detector.outputs.matrix-selector == 'full' + && needs.cpp-matrix.outputs.matrix + || toJSON(fromJSON(needs.cpp-matrix.outputs.submatrices)[needs.scope-detector.outputs.matrix-selector]) }} secrets: inherit + # Per-OS smoke + demo generation. Always required. Sources packages from + # this run when ci-build ran, from develop-release otherwise. releases: name: Releases - needs: [utility-tests, cpp-matrix, build] + needs: [scope-detector, cpp-matrix, build] + if: | + always() && + needs.scope-detector.result == 'success' && + needs.cpp-matrix.result == 'success' && + (needs.build.result == 'success' || needs.build.result == 'skipped') permissions: - contents: write + contents: read uses: ./.github/workflows/ci-releases.yml with: - submatrices: ${{ needs.cpp-matrix.outputs.submatrices }} + submatrix: ${{ toJSON(fromJSON(needs.cpp-matrix.outputs.submatrices).releases) }} + use-develop-binaries: ${{ needs.scope-detector.outputs.run-release-build == 'false' }} + secrets: inherit + + # Per-OS docs/website/Antora-UI validation. Scope-gated: runs when + # source, docs, build, or third-party changed. + documentation: + name: Documentation + needs: [scope-detector, cpp-matrix, build] + if: | + always() && + needs.scope-detector.outputs.run-documentation == 'true' && + needs.cpp-matrix.result == 'success' && + (needs.build.result == 'success' || needs.build.result == 'skipped') + permissions: + contents: read + uses: ./.github/workflows/ci-documentation.yml + with: + submatrix: ${{ toJSON(fromJSON(needs.cpp-matrix.outputs.submatrices).releases) }} + use-develop-binaries: ${{ needs.scope-detector.outputs.run-release-build == 'false' }} + secrets: inherit + + # Single Linux job that always runs. Aggregates every upstream job's + # result so this status check is the single gate branch protection relies + # on. The actual publishing work (Antora multi-ref render, GH Pages / + # dev-websites rsync, consolidated GitHub Release) is gated on + # `should-publish` and only fires on push to develop/master/tags. + publish: + name: Publish + needs: [scope-detector, cpp-matrix, utility-tests, build, releases, documentation] + if: always() + permissions: + contents: write + uses: ./.github/workflows/ci-publish.yml + with: + submatrix: ${{ toJSON(fromJSON(needs.cpp-matrix.outputs.submatrices).releases) }} + should-publish: ${{ needs.scope-detector.outputs.is-push == 'true' && (github.ref_name == 'develop' || github.ref_name == 'master' || startsWith(github.ref, 'refs/tags/')) }} + scope-detector-result: ${{ needs.scope-detector.result }} + cpp-matrix-result: ${{ needs.cpp-matrix.result }} + utility-tests-result: ${{ needs.utility-tests.result }} + build-result: ${{ needs.build.result }} + releases-result: ${{ needs.releases.result }} + documentation-result: ${{ needs.documentation.result }} secrets: inherit diff --git a/util/danger/README.md b/util/danger/README.md index 17f1d9a2e4..c5d1f420c9 100644 --- a/util/danger/README.md +++ b/util/danger/README.md @@ -36,7 +36,7 @@ npm --prefix util/danger run danger:ci # run Danger in CI mode (requires Git - Aggregate PR source churn triggers a warning above 5000 lines (much more generous than the per-commit limit, since well-sliced commits can still amount to a large overall change). - PR description length is checked against a log-scaled floor (`80 * log2(churn)` characters). Tiny diffs need ~80 chars; ~1k-line changes need ~800; ~30k-line changes need ~1200. HTML comments in the body (e.g. PR template scaffolding) are stripped before measurement. - Feature PRs (`feat:` commits, `feat` PR title, or `feature` label) without any `docs/` change get a light warning. Opt out with the `no-docs-needed` label or a `[skip danger docs]` marker in the PR body. -- The PR template (`.github/pull_request_template.md`) mirrors these rules; following it tends to satisfy all of them automatically. +- The PR template (`.github/pull_request_template.md`) mirrors these rules; following it tends to satisfy all of them automatically. Section headers from the template (any level) that are missing from the PR body are reported as an informational note rather than a warning. ## Updating rules diff --git a/util/danger/detect-scopes.ts b/util/danger/detect-scopes.ts new file mode 100644 index 0000000000..9ab70a212a --- /dev/null +++ b/util/danger/detect-scopes.ts @@ -0,0 +1,56 @@ +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// Copyright (c) 2026 Alan de Freitas (alandefreitas@gmail.com) +// +// Official repository: https://github.com/cppalliance/mrdocs +// +/** + * Classify changed files into Danger scopes and decide whether the change + * can affect the produced mrdocs binary. Reads newline-separated paths from + * stdin and prints a JSON object to stdout that CI consumes via `jq`: + * + * { "scopes": [...], "is_code_change": bool } + * + * Usage: + * git diff --name-only origin/develop...HEAD | \ + * npx --prefix util/danger ts-node util/danger/detect-scopes.ts + */ + +import { affectsBuildPipeline, classifyScope, isCodeChange, scopeDisplayOrder, scopesTouched } from "./logic"; + +async function readStdin(): Promise { + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) { + chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk); + } + return Buffer.concat(chunks).toString("utf8"); +} + +function main(): void { + void readStdin().then((input) => { + const paths = input + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + + const scopes = scopesTouched(paths); + const orderedScopes = scopeDisplayOrder.filter((scope) => scopes.has(scope)); + + const result = { + scopes: orderedScopes, + is_code_change: isCodeChange(scopes) || affectsBuildPipeline(paths), + file_count: paths.length, + }; + + process.stdout.write(JSON.stringify(result) + "\n"); + }); +} + +// Keep classifyScope importable as a side-effect-free re-export for callers +// that want to inspect a single path without spawning a process. +export { classifyScope }; + +main(); diff --git a/util/danger/fixtures/sample-pr.json b/util/danger/fixtures/sample-pr.json index 5231c327d8..ba1277f1a0 100644 --- a/util/danger/fixtures/sample-pr.json +++ b/util/danger/fixtures/sample-pr.json @@ -32,5 +32,6 @@ ], "prBody": "Sample rationale\\n\\nTesting: unit tests locally", "prTitle": "Sample Danger fixture", - "labels": [] + "labels": [], + "prTemplate": "_What this PR does and why._\n\n## Changes\n\n_Replace the lines that apply._\n\n## Testing\n\n_How this change stays tested._\n\n## Documentation\n\n_What was updated._\n" } diff --git a/util/danger/logic.test.ts b/util/danger/logic.test.ts index 0bb09d2247..c6dbde5100 100644 --- a/util/danger/logic.test.ts +++ b/util/danger/logic.test.ts @@ -9,18 +9,41 @@ // import { describe, expect, it } from "vitest"; import { + affectsBuildPipeline, aggregateSizeWarnings, commitSizeInfos, evaluateDanger, expectedBodyLength, + isCodeChange, parseCommitSummary, + parsePrTemplateSections, + prTemplateInfos, basicChecks, + scopesTouched, summarizeScopes, validateCommits, type CommitInfo, type DangerInputs, } from "./logic"; +const sampleTemplate = [ + "", + "", + "_What this PR does and why._", + "", + "## Changes", + "", + "_Replace the lines that apply._", + "", + "## Testing", + "", + "_How this change stays tested._", + "", + "## Documentation", + "", + "_What was updated._", +].join("\n"); + describe("parseCommitSummary", () => { // Ensures we correctly extract type, scope, and subject when format is valid. it("parses valid commit summaries", () => { @@ -139,6 +162,261 @@ describe("commitSizeInfos", () => { }); }); +describe("scopesTouched", () => { + // Classifies a list of file paths into the unique set of scopes touched. + it("returns the unique set of scopes for the given paths", () => { + const scopes = scopesTouched([ + "src/lib/file.cpp", + "include/mrdocs/example.hpp", + "test-files/golden-tests/out.xml", + "docs/index.adoc", + ".github/workflows/ci.yml", + ]); + expect(scopes).toEqual(new Set(["source", "golden-tests", "docs", "ci"])); + }); + + // Empty input → empty set. + it("returns an empty set when no paths are given", () => { + expect(scopesTouched([])).toEqual(new Set()); + }); +}); + +describe("isCodeChange", () => { + // Scopes that can affect the mrdocs binary trigger the full build matrix. + it("is true when source is touched", () => { + expect(isCodeChange(new Set(["source"]))).toBe(true); + }); + + it("is true when tests are touched", () => { + expect(isCodeChange(new Set(["tests"]))).toBe(true); + }); + + it("is true when golden-tests are touched", () => { + expect(isCodeChange(new Set(["golden-tests"]))).toBe(true); + }); + + it("is true when build files are touched", () => { + expect(isCodeChange(new Set(["build"]))).toBe(true); + }); + + it("is true when third-party is touched", () => { + expect(isCodeChange(new Set(["third-party"]))).toBe(true); + }); + + // Meta scopes do not justify rebuilding the binary. + it("is false for docs-only changes", () => { + expect(isCodeChange(new Set(["docs"]))).toBe(false); + }); + + it("is false for ci-only changes", () => { + expect(isCodeChange(new Set(["ci"]))).toBe(false); + }); + + it("is false for tooling-only changes", () => { + expect(isCodeChange(new Set(["tooling"]))).toBe(false); + }); + + it("is false for toolchain or toolchain-tests changes", () => { + expect(isCodeChange(new Set(["toolchain"]))).toBe(false); + expect(isCodeChange(new Set(["toolchain-tests"]))).toBe(false); + }); + + // A mixed change with any code scope still trips the flag. + it("is true when at least one scope is code-relevant", () => { + expect(isCodeChange(new Set(["docs", "ci", "source"]))).toBe(true); + }); + + // Empty set: no change. + it("is false on an empty scope set", () => { + expect(isCodeChange(new Set())).toBe(false); + }); +}); + +describe("affectsBuildPipeline", () => { + // CI workflow files that drive the matrix must trip the full-matrix gate + // even when classified under the `ci` scope (which is not a code change). + it("flags ci-build.yml changes", () => { + expect(affectsBuildPipeline([".github/workflows/ci-build.yml"])).toBe(true); + }); + + it("flags any ci-*.yml workflow", () => { + expect(affectsBuildPipeline([".github/workflows/ci-matrix.yml"])).toBe(true); + expect(affectsBuildPipeline([".github/workflows/ci-release.yml"])).toBe(true); + expect(affectsBuildPipeline([".github/workflows/ci-documentation.yml"])).toBe(true); + expect(affectsBuildPipeline([".github/workflows/ci-publish.yml"])).toBe(true); + expect(affectsBuildPipeline([".github/workflows/ci-scope-detector.yml"])).toBe(true); + expect(affectsBuildPipeline([".github/workflows/ci.yml"])).toBe(true); + }); + + it("flags build / install / demo scripts", () => { + expect(affectsBuildPipeline([".github/scripts/install-mrdocs-package.sh"])).toBe(true); + expect(affectsBuildPipeline([".github/scripts/generate-demos.sh"])).toBe(true); + }); + + it("flags bootstrap entry and sources", () => { + expect(affectsBuildPipeline(["bootstrap.py"])).toBe(true); + expect(affectsBuildPipeline(["util/bootstrap/main.py"])).toBe(true); + }); + + // Workflow files that only drive PR comments / Danger.js posting don't + // affect the build pipeline; they're exercised by their own jobs. + it("does not flag pr-target-checks.yml", () => { + expect(affectsBuildPipeline([".github/workflows/pr-target-checks.yml"])).toBe(false); + }); + + // PR template, gitignore, license: meta files with no build effect. + it("does not flag PR template or meta files", () => { + expect(affectsBuildPipeline([".github/pull_request_template.md"])).toBe(false); + expect(affectsBuildPipeline([".gitignore"])).toBe(false); + expect(affectsBuildPipeline(["LICENSE.txt"])).toBe(false); + }); + + // Bootstrap tests are exercised in utility-tests on every PR, no full + // matrix needed. + it("does not flag bootstrap tests", () => { + expect(affectsBuildPipeline(["util/bootstrap/tests/test_installer.py"])).toBe(false); + }); + + // Danger.js source is exercised by utility-tests' vitest step. + it("does not flag util/danger changes", () => { + expect(affectsBuildPipeline(["util/danger/logic.ts"])).toBe(false); + expect(affectsBuildPipeline(["util/danger/dangerfile.ts"])).toBe(false); + }); + + // Source / docs / build / third-party are already covered by scope-based + // `isCodeChange`; this helper is purely the build-pipeline supplement. + it("does not flag normal source paths", () => { + expect(affectsBuildPipeline(["src/lib/file.cpp"])).toBe(false); + expect(affectsBuildPipeline(["docs/index.adoc"])).toBe(false); + }); + + it("returns false on empty input", () => { + expect(affectsBuildPipeline([])).toBe(false); + }); + + // Normalizes Windows-style separators so backslashes don't bypass the gate. + it("normalizes backslashes", () => { + expect(affectsBuildPipeline([".github\\workflows\\ci-build.yml"])).toBe(true); + }); +}); + +describe("parsePrTemplateSections", () => { + // Pulls H2 headers in document order; ignores intro paragraphs and comments. + it("extracts H2 headers from the template", () => { + expect(parsePrTemplateSections(sampleTemplate)).toEqual(["Changes", "Testing", "Documentation"]); + }); + + // No headers means no sections to enforce. + it("returns an empty array when there are no headers", () => { + expect(parsePrTemplateSections("just a paragraph\n\nno headers here")).toEqual([]); + }); + + // Handles any ATX heading level so future templates can mix H1/H2/H3. + it("extracts headings of any ATX level", () => { + const template = [ + "# Summary", + "## Changes", + "### Subsection", + "#### Detail", + "##### Deeper", + "###### Deepest", + ].join("\n"); + expect(parsePrTemplateSections(template)).toEqual([ + "Summary", + "Changes", + "Subsection", + "Detail", + "Deeper", + "Deepest", + ]); + }); + + // Strips the optional ATX closing run of `#` characters. + it("strips trailing closing hashes from ATX headers", () => { + expect(parsePrTemplateSections("## Changes ##\n### Testing ###")).toEqual(["Changes", "Testing"]); + }); + + // Dedupes by title (case-insensitive) so cross-level repeats don't double-flag. + it("dedupes repeated titles across levels", () => { + const template = ["## Notes", "### notes", "#### NOTES"].join("\n"); + expect(parsePrTemplateSections(template)).toEqual(["Notes"]); + }); +}); + +describe("prTemplateInfos", () => { + // A body that includes every template header produces no info. + it("stays quiet when all template sections are present", () => { + const body = [ + "Rationale for the change.", + "", + "## Changes", + "- Source: tweak.", + "", + "## Testing", + "Ran the suite.", + "", + "## Documentation", + "No user-facing change.", + ].join("\n"); + expect(prTemplateInfos(body, ["Changes", "Testing", "Documentation"])).toEqual([]); + }); + + // Missing sections are listed in the single info message. + it("lists missing sections", () => { + const body = ["Rationale.", "", "## Changes", "- Source: tweak."].join("\n"); + const infos = prTemplateInfos(body, ["Changes", "Testing", "Documentation"]); + expect(infos).toHaveLength(1); + expect(infos[0]).toContain("**Testing**"); + expect(infos[0]).toContain("**Documentation**"); + expect(infos[0]).not.toContain("**Changes**"); + }); + + // Header matching is case-insensitive so contributors don't trip over capitalization. + it("matches headers case-insensitively", () => { + const body = ["## changes", "## testing", "## documentation"].join("\n"); + expect(prTemplateInfos(body, ["Changes", "Testing", "Documentation"])).toEqual([]); + }); + + // The body can use a different heading level than the template — title alone is what matters. + it("matches sections regardless of heading level", () => { + const body = ["# Changes", "### Testing", "###### Documentation"].join("\n"); + expect(prTemplateInfos(body, ["Changes", "Testing", "Documentation"])).toEqual([]); + }); + + // An empty body is already covered by the empty-description warning — skip to avoid noise. + it("skips when the PR body is empty", () => { + expect(prTemplateInfos("", ["Changes", "Testing"])).toEqual([]); + expect(prTemplateInfos(" \n\n ", ["Changes", "Testing"])).toEqual([]); + }); + + // A template with no headers produces no info. + it("skips when the template has no headers", () => { + expect(prTemplateInfos("some body", [])).toEqual([]); + }); + + // Singular wording when exactly one section is missing. + it("uses singular wording for a single missing section", () => { + const body = ["## Changes", "## Testing"].join("\n"); + const infos = prTemplateInfos(body, ["Changes", "Testing", "Documentation"]); + expect(infos[0]).toContain("missing template section:"); + }); +}); + +describe("evaluateDanger pr-template info", () => { + // The new check is wired into the infos channel so it renders as [!NOTE]. + it("surfaces missing template sections via infos", () => { + const result = evaluateDanger({ + files: [{ filename: "src/lib/x.cpp", additions: 10, deletions: 1 }], + commits: [{ sha: "ab", message: "fix: small change" }], + prBody: "A short rationale that explains what is going on, with no template structure at all.", + prTitle: "fix: small change", + labels: [], + prTemplate: sampleTemplate, + }); + expect(result.infos.some((m) => m.includes("missing template sections"))).toBe(true); + }); +}); + describe("starterChecks", () => { // Checks that source changes without accompanying tests produce a warning. it("requests tests when source changes without coverage", () => { diff --git a/util/danger/logic.ts b/util/danger/logic.ts index aecde8d88e..7d78b0ea6f 100644 --- a/util/danger/logic.ts +++ b/util/danger/logic.ts @@ -98,6 +98,7 @@ export interface DangerInputs { prBody: string; prTitle: string; labels: string[]; + prTemplate?: string; } /** @@ -124,6 +125,81 @@ export const scopeDisplayOrder: ScopeKey[] = [ "other", ]; +/** + * Scopes whose changes can affect the produced mrdocs binary or how it is + * tested. CI uses this to decide whether the full build matrix (sanitizers, + * coverage, extras) needs to run, or whether the cheap path (reusing + * develop's release artifacts) is safe. + */ +export const codeChangeScopes: ReadonlySet = new Set([ + "source", + "tests", + "golden-tests", + "build", + "third-party", +]); + +/** + * Paths inside scopes that are otherwise treated as meta (`ci`, `toolchain`) + * but still drive how the binary is built or tested. A change to one of + * these files must trigger a full matrix even if the rest of the diff is + * tooling-only, otherwise the modified workflow / script could be skipped + * via matrix-selector='none' and ship to develop without ever running. + */ +const buildAffectingPathPatterns: RegExp[] = [ + // Workflow files that drive build, matrix, release, docs, or publish. + // pr-target-checks.yml is excluded because it only runs Danger.js. + /^\.github\/workflows\/ci(\.|-)/i, + // Build / install / demo / coverage scripts run inside ci-build, + // ci-release, ci-documentation, ci-publish. + /^\.github\/scripts\//i, + // Bootstrap drives third-party dependency builds inside ci-build. + // The `tests/` subdirectory is exercised by utility-tests on every PR, + // so changes there don't need a full matrix. + /^bootstrap\.py$/i, + /^util\/bootstrap\/(?!tests\/)/i, +]; + +/** + * Classify each path and return the unique set of scopes touched. + */ +export function scopesTouched(paths: Iterable): Set { + const set = new Set(); + for (const path of paths) { + set.add(classifyScope(path)); + } + return set; +} + +/** + * True when any of the supplied scopes can affect the mrdocs binary or its + * tests. + */ +export function isCodeChange(scopes: Iterable): boolean { + for (const scope of scopes) { + if (codeChangeScopes.has(scope)) { + return true; + } + } + return false; +} + +/** + * True when any supplied path drives the build pipeline itself (CI workflow + * files, build scripts, bootstrap). The scope-detector ORs this with + * `isCodeChange` so a CI-only PR that edits ci-build.yml still forces the + * full matrix. + */ +export function affectsBuildPipeline(paths: Iterable): boolean { + for (const path of paths) { + const normalized = path.replace(/\\/g, "/"); + if (buildAffectingPathPatterns.some((pattern) => pattern.test(normalized))) { + return true; + } + } + return false; +} + const allowedTypes = [ "feat", "fix", @@ -505,6 +581,68 @@ export function commitSizeInfos(commits: CommitInfo[]): string[] { return messages; } +// Matches any ATX heading (#, ##, …, ######) with optional trailing # marks per the CommonMark spec. +const atxHeadingPattern = /^[ \t]*#{1,6}[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/; + +/** + * Extract section headers of any level (H1–H6) from a Markdown document. + * + * Used to discover the section structure of the project's PR template so that + * missing sections can be reported back to contributors. Returns titles in + * document order, deduped (case-insensitive) so a template that repeats a + * heading at different levels does not double-flag. + */ +export function parsePrTemplateSections(template: string): string[] { + const sections: string[] = []; + const seen = new Set(); + for (const raw of template.split("\n")) { + const match = raw.match(atxHeadingPattern); + if (!match) continue; + const title = match[1].trim(); + const key = title.toLowerCase(); + if (!seen.has(key)) { + seen.add(key); + sections.push(title); + } + } + return sections; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Report PR-template sections that the body is missing. + * + * Matches headings of any level (so a body can use `### Testing` even if the + * template uses `## Testing`). The template is a convention, not a hard + * requirement, so this is an informational note rather than a warning — the + * empty-description warning already covers the case where the contributor + * wrote nothing at all. + */ +export function prTemplateInfos(prBody: string, templateSections: string[]): string[] { + if (!prBody.trim() || templateSections.length === 0) { + return []; + } + const missing = templateSections.filter((section) => { + const pattern = new RegExp( + `^[ \\t]*#{1,6}[ \\t]+${escapeRegExp(section)}(?:[ \\t]+#+)?[ \\t]*$`, + "im", + ); + return !pattern.test(prBody); + }); + if (missing.length === 0) { + return []; + } + const noun = missing.length === 1 ? "section" : "sections"; + const list = missing.map((name) => `**${name}**`).join(", "); + return [ + `PR description is missing template ${noun}: ${list}. ` + + "Following the [pull request template](.github/pull_request_template.md) helps reviewers find rationale, testing notes, and docs status quickly.", + ]; +} + /** * Warn when the aggregate source-scope churn across the whole PR is large. * @@ -687,7 +825,10 @@ export function evaluateDanger(input: DangerInputs): DangerResult { const summary = summarizeScopes(input.files); const commitValidation = validateCommits(input.commits); - const infos = commitSizeInfos(input.commits); + const infos = [ + ...commitSizeInfos(input.commits), + ...prTemplateInfos(input.prBody || "", parsePrTemplateSections(input.prTemplate || "")), + ]; const warnings = [ ...commitValidation.warnings, diff --git a/util/danger/package.json b/util/danger/package.json index d18896bd17..a055f83189 100644 --- a/util/danger/package.json +++ b/util/danger/package.json @@ -10,7 +10,8 @@ "test": "vitest run", "danger:ci": "danger ci --dangerfile dangerfile.ts", "danger:local": "ts-node --project tsconfig.json run-local.ts", - "danger:scope-map": "ts-node --project tsconfig.json list-scopes.ts" + "danger:scope-map": "ts-node --project tsconfig.json list-scopes.ts", + "danger:detect-scopes": "ts-node --project tsconfig.json detect-scopes.ts" }, "devDependencies": { "@types/node": "^20.14.2", diff --git a/util/danger/runner.ts b/util/danger/runner.ts index 103bbaaedc..4de79be485 100644 --- a/util/danger/runner.ts +++ b/util/danger/runner.ts @@ -7,10 +7,25 @@ // // Official repository: https://github.com/cppalliance/mrdocs // +import { readFileSync } from "node:fs"; +import { join } from "node:path"; import type { DangerDSLType } from "danger"; import { evaluateDanger, type CommitInfo, type FileChange } from "./logic"; import { renderDangerReport } from "./format"; +/** + * Read the project's PR template so rules can check the body against it. + * Missing or unreadable templates degrade gracefully — the template-section + * check just stays quiet rather than failing the run. + */ +function loadPrTemplate(): string { + try { + return readFileSync(join(process.cwd(), ".github/pull_request_template.md"), "utf8"); + } catch { + return ""; + } +} + // Danger provides these as globals at runtime; we declare them for editors/typecheckers. declare const danger: DangerDSLType; declare function markdown(message: string, file?: string, line?: number): void; @@ -113,6 +128,7 @@ export async function runDanger(): Promise { prBody: pr.body || "", prTitle: pr.title || "", labels, + prTemplate: loadPrTemplate(), }); const warnings = [...fetchWarnings, ...evaluation.warnings];