From 5b235452628d94893e042f82aad299c6d2c8ccb8 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 14 Sep 2026 02:24:41 -0700 Subject: [PATCH] ci: gate main on install, invariants, the four engine suites and the parser suites; a skipped suite is a failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Supersedes #418, rebuilt on the consolidated tree and keeping its design: no path filters (a required check that never reports can never merge), one aggregate job for the ruleset, --oracle where ground truth is reachable (javac/javap for Java, the TypeScript compiler for TypeScript and for JavaScript over allowJs/checkJs), Java without the torture families on CI (they need the JVM platform IR), two Python interpreters, JDK 24, the pinned and checksummed Soufflé package, and the compiled engine cached under the rule-set id. Added for the tree as it is now: a fresh-clone job (npm install, typecheck, build, the command runs and the parser writes IR — the failure that reached main in #477), the JavaScript leg, the parser's own suites, the bundle and engine-id preflights among the invariants, Node 22 (node:sqlite), and the workflow is named build so the badge reads it. The parser is in this repository, so the separate-repository fetch, the parser-ref pin and the nightly parser-drift job are gone. main-guard.yml (every commit on main came through a pull request) and lib-coverage.sh are carried over; the Java suite gains --no-torture. --- .github/scripts/run-suite.sh | 25 +++++ .github/workflows/ci.yml | 183 +++++++++++++++++++++++++++++++ .github/workflows/main-guard.yml | 87 +++++++++++++++ graph/test/java/run-tests.sh | 11 +- graph/test/tools/lib-coverage.sh | 69 ++++++++++++ 5 files changed, 372 insertions(+), 3 deletions(-) create mode 100755 .github/scripts/run-suite.sh create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/main-guard.yml create mode 100755 graph/test/tools/lib-coverage.sh diff --git a/.github/scripts/run-suite.sh b/.github/scripts/run-suite.sh new file mode 100755 index 000000000..3eb88d14d --- /dev/null +++ b/.github/scripts/run-suite.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Run one regression suite for CI and turn every way a suite can quietly not test into a +# failure: a suite that skips itself (exit 77), a sub-harness that prints SKIP, a suite that +# passed 0 cases. CI must supply what a suite wants; do not relax this to go green. +# run-suite.sh [suite args...] +set -uo pipefail +lang="${1:?usage: run-suite.sh [args...]}"; shift +root="$(cd "$(dirname "$0")/../.." && pwd)" +[ -f "$root/parser/dist/index.js" ] || { echo "::error::parser/dist/index.js is missing — npm run build did not run, and every suite would skip itself"; exit 1; } +log="$(mktemp)" +echo "── $lang suite" +"$root/bin/axiomcode" test "$lang" "$@" 2>&1 | tee "$log" +rc="${PIPESTATUS[0]}" +if [ "$rc" -eq 77 ]; then + echo "::error::the $lang suite skipped itself (exit 77). A skipped suite is a failed gate."; exit 1 +fi +if grep -qE '(^|[[:space:]])SKIP([: (]|PED)' "$log"; then + echo "::error::a sub-harness in the $lang suite skipped instead of running:" + grep -nE '(^|[[:space:]])SKIP([: (]|PED)' "$log" | sed 's/^/ /' + exit 1 +fi +if [ "$lang" != parser ] && grep -qE '^passed 0([^0-9]|$)|^passed 0,' "$log"; then + echo "::error::the $lang suite passed 0 cases — it asserted nothing."; exit 1 +fi +exit "$rc" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..46f4c984a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,183 @@ +# ───────────────────────────────────────────────────────────────────────────── +# build — the gate every change to main passes through. +# +# Four tiers, cheapest first, all required: +# install a fresh clone installs and builds (parser + engine), and the command runs +# hygiene repository invariants that need no solver and catch silent breakage +# engine the java / typescript / python / javascript regression suites, in parallel +# parser the parser's own suites +# +# NO PATH FILTERS, deliberately. A required check that is skipped by a path filter never +# reports, and a pull request waiting on a check that will never report can never merge. +# ───────────────────────────────────────────────────────────────────────────── +name: build + +on: + push: + branches: [main] + pull_request: + merge_group: + workflow_dispatch: + +# One run per ref. A new push to a pull request cancels the previous run; a run on main is +# always allowed to finish — main's history is the record. +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + NODE_VERSION: '22' # the bundle stage writes graph.sqlite with node:sqlite (≥ 22.5) + +jobs: + install: + name: fresh clone installs, builds, runs + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: '22' } + # No lock file is committed, so this is `npm install`, and `prepare` builds both + # packages. The explicit steps after it mean a prepare-script change cannot silently + # stop compiling this repository. + - run: npm install + - run: npm run typecheck + - run: npm run build + - name: the build produced what the command needs + run: | + test -f parser/dist/index.js || { echo "::error::parser/dist/index.js missing"; exit 1; } + test -f dist/bundle/cli.js || { echo "::error::dist/bundle/cli.js missing"; exit 1; } + test ! -d dist/test || { echo "::error::fixtures were compiled into dist/"; exit 1; } + bin/axiomcode --help | head -1 + bin/axiomcode parser graph/test/java/cases/01-inheritance-override/src /tmp/ir >/dev/null + test -f /tmp/ir/java/all-methods.csv || { echo "::error::the parser wrote no IR"; exit 1; } + + hygiene: + name: repository invariants + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: '22' } + - run: npm install + - name: every fixture input is tracked by git + run: bash graph/test/tools/no-ignored-fixtures.sh + - name: IR staging maps are consistent, per language + run: | + fail=0 + for lang in java python typescript javascript; do + echo "── $lang"; python3 graph/test/tools/check_staging.py --lang "$lang" || fail=1 + done + exit $fail + - name: client->library coverage has not shrunk + run: bash graph/test/tools/lib-coverage.sh + - name: the output bundle is one schema across languages and its doc is current + run: bash graph/test/tools/bundle-test.sh + - name: the engine id is a function of the rules alone + run: bash graph/test/tools/engine-id-test.sh + - name: shell scripts parse + run: | + fail=0 + while IFS= read -r f; do bash -n "$f" || { echo "::error file=$f::does not parse"; fail=1; }; done < <(git ls-files '*.sh') + exit $fail + + engine: + name: engine (${{ matrix.lang }}) + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + # --oracle scores the engine against GROUND TRUTH, not only the goldens. A golden + # says "the same as last time", which a wrong answer satisfies as long as it was + # wrong last time too. The ground truth is built with the toolchain that defines + # the language: javac/javap, the TypeScript compiler (also over JavaScript with + # allowJs/checkJs). No third-party analyzer. + # + # --no-torture for java ONLY: those families need the JVM platform IR staged as a + # library (1.8 GB, built from a JDK source checkout), which cannot live in a + # repository or a cache; without it their receivers are unresolvable BY + # CONSTRUCTION and the red would read as a regression. Six Java cases still cover + # client->library with their own stub libraries. The torture families stay a local + # gate; the suite prints EXCLUDED so they are never mistaken for a pass. + - { lang: java, oracle: '--oracle --no-torture' } + - { lang: typescript, oracle: '--oracle' } + - { lang: javascript, oracle: '--oracle' } + # Python's ground truth is frozen CPython output authored by a separate harness + # CI cannot reach yet, so this leg is goldens-only for now. + - { lang: python, oracle: '' } + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: '22' } + - run: npm install + # TWO interpreters for python: 3.10 for the tier-1 attribution preflight (it reads + # CPython OPCODES, whose shapes are not stable across minor versions, and resolves + # `python3.10` by name); 3.12 for the torture fixtures, whose source uses typing.Self + # (3.11+). The later setup-python wins for plain `python3`, so 3.12 comes second. + - uses: actions/setup-python@v5 + with: { python-version: '3.10' } + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + # JDK 24: the java torture/oracle tooling reads class files with java.lang.classfile, + # which is not final before 24; on an older JDK it exits 77 and silently does not run. + - uses: actions/setup-java@v4 + if: matrix.lang == 'java' + with: { distribution: temurin, java-version: '24' } + # Soufflé is the solver: the engine is compiled from .dl to C++ against its headers, + # so a build of it has to be present the way a compiler has to be present. Pinned to + # the version in graph/pipeline/engine.conf and verified against upstream's checksum. + - name: cache the Soufflé package + uses: actions/cache@v4 + with: + path: ~/souffle-pkg + key: souffle-deb-2.5-ubuntu-2404 + - name: install Soufflé + run: | + set -euo pipefail + . graph/pipeline/engine.conf + deb="x86_64-ubuntu-2404-souffle-${SOUFFLE_VERSION}-Linux.deb" + dir="$HOME/souffle-pkg"; mkdir -p "$dir" + [ -f "$dir/$deb" ] || curl -fsSL --retry 3 -o "$dir/$deb" "https://github.com/souffle-lang/souffle/releases/download/${SOUFFLE_VERSION}/${deb}" + echo "6b86e554f6aa5abf8a8b55d8312ae37c0957c5bd6c9edeea89246db9406f645ec5e600b84fe6636b1c163da556f0da6c3d2dad46c1083413f2fcf4f95b9ac62c $dir/$deb" | sha512sum -c - + sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends "$dir/$deb" + souffle --version | head -2 + # run-souffle.sh caches the compiled engine under the rule-set id; mirroring that key + # here skips a multi-minute C++ build on every run whose rules did not change. + - name: cache the compiled engine + uses: actions/cache@v4 + with: + path: .souffle-cache + key: engine-${{ matrix.lang }}-${{ hashFiles(format('graph/{0}/**/*.dl', matrix.lang), 'graph/pipeline/engine.conf') }} + - name: ${{ matrix.lang }} regression suite + run: bash .github/scripts/run-suite.sh ${{ matrix.lang }} ${{ matrix.oracle }} + + parser: + name: parser suites + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: '22' } + - run: npm install + - run: bash .github/scripts/run-suite.sh parser + + # A single job the ruleset can require. Without it every matrix entry has to be added to + # the protection rules by hand, and a matrix job that fails to start reports nothing — + # which a ruleset reads as "not failing" rather than "did not run". + build: + name: build + runs-on: ubuntu-24.04 + needs: [install, hygiene, engine, parser] + if: always() + steps: + - name: every tier passed + run: | + echo "install=${{ needs.install.result }} hygiene=${{ needs.hygiene.result }} engine=${{ needs.engine.result }} parser=${{ needs.parser.result }}" + [ "${{ needs.install.result }}" = success ] && [ "${{ needs.hygiene.result }}" = success ] && [ "${{ needs.engine.result }}" = success ] && [ "${{ needs.parser.result }}" = success ] diff --git a/.github/workflows/main-guard.yml b/.github/workflows/main-guard.yml new file mode 100644 index 000000000..040023dff --- /dev/null +++ b/.github/workflows/main-guard.yml @@ -0,0 +1,87 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Every commit on `main` should have arrived through a pull request. +# +# This reports when one did not. It is a backstop and not the rule itself: a +# workflow runs after the push has already been accepted, so it can record a +# direct push but never refuse one. The rule that refuses lives in +# .github/scripts/protect-main.sh. +# ───────────────────────────────────────────────────────────────────────────── +name: main-guard + +on: + push: + branches: [main] + +permissions: + contents: read + issues: write + +jobs: + direct-push: + name: every commit on main came from a pull request + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: check the pushed commits + id: check + env: + GH_TOKEN: ${{ github.token }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.event.after }} + run: | + set -uo pipefail + + # A branch created or force-updated from nothing has no usable range. + if [ "$BEFORE" = "0000000000000000000000000000000000000000" ]; then + echo "branch created — nothing to compare"; exit 0 + fi + + orphans=() + while IFS= read -r sha; do + [ -n "$sha" ] || continue + n="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${sha}/pulls" --jq 'length' 2>/dev/null || echo 0)" + if [ "$n" = "0" ]; then + orphans+=("$sha $(git log -1 --format=%s "$sha")") + fi + done < <(git rev-list "${BEFORE}..${AFTER}" 2>/dev/null) + + if [ ${#orphans[@]} -eq 0 ]; then + echo "all pushed commits arrived through a pull request" + exit 0 + fi + + { + echo "### Direct push to \`main\` detected" + echo + echo "These commits are on \`main\` without a pull request:" + echo + printf -- '- %s\n' "${orphans[@]}" + } >> "$GITHUB_STEP_SUMMARY" + + printf '%s\n' "${orphans[@]}" > /tmp/orphans.txt + echo "found=1" >> "$GITHUB_OUTPUT" + echo "::error::${#orphans[@]} commit(s) reached main without a pull request" + exit 1 + + - name: record it as an issue + if: failure() && steps.check.outputs.found == '1' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + title="Direct push to main on $(date -u +%Y-%m-%d)" + # One issue per day, not one per push. + existing="$(gh issue list --state open --search "\"$title\" in:title" --json number --jq '.[0].number' || true)" + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + body="$(printf "Commits reached \`main\` without a pull request:\n\n\`\`\`\n%s\n\`\`\`\n\nRun: %s\n\nReported after the fact: a workflow runs once the push has been accepted, so it can record a direct push but not refuse one. See .github/scripts/protect-main.sh\n" \ + "$(cat /tmp/orphans.txt)" "$run_url")" + if [ -n "${existing:-}" ]; then + gh issue comment "$existing" --body "$body" + else + gh issue create --title "$title" --body "$body" --label platform || \ + gh issue create --title "$title" --body "$body" + fi diff --git a/graph/test/java/run-tests.sh b/graph/test/java/run-tests.sh index 1ef9c26d9..d9e05c2a1 100755 --- a/graph/test/java/run-tests.sh +++ b/graph/test/java/run-tests.sh @@ -21,6 +21,8 @@ # (javac + javap invoke instructions — no third-party analyzer): # every bytecode-declared client->client edge must be present. # ./run-tests.sh --keep keep the per-case work dirs for debugging +# ./run-tests.sh --no-torture skip the torture families (CI: they need the JVM platform IR, +# which cannot be staged there; the suite prints EXCLUDED) # # With --oracle, a case carrying a spring-oracle.conf ALSO boots its sources in a real # AnnotationConfigApplicationContext and scores bean_def / di_edge against what Spring @@ -107,9 +109,9 @@ if ! bash "$ROOT/graph/test/tools/bundle-test.sh"; then fi PARSER="${AXIOM_PARSER:-$ROOT/parser/dist/index.js}" WORK="$HERE/.work" -BLESS=0; KEEP=0; ORACLE=0; FILTERS=() +BLESS=0; KEEP=0; ORACLE=0; NO_TORTURE=0; FILTERS=() for a in "$@"; do case "$a" in - --bless) BLESS=1;; --keep) KEEP=1;; --oracle) ORACLE=1;; + --bless) BLESS=1;; --keep) KEEP=1;; --oracle) ORACLE=1;; --no-torture) NO_TORTURE=1;; -h|--help) sed -n '2,34p' "$0"; exit 0;; *) FILTERS+=("$a");; esac; done # ── PREFLIGHT: every relation the parser emits must actually reach the solver ────────── @@ -370,7 +372,7 @@ done # The cases above each pin ONE rule. This asks what happens when a project uses everything at # once, and reports WHICH construct is the gap rather than one number. It stages the platform IR, # because half the families call java.util types and scoring them without it measures the staging. -if [ -d "$HERE/torture" ] && [ ${#FILTERS[@]} -eq 0 ]; then +if [ -d "$HERE/torture" ] && [ ${#FILTERS[@]} -eq 0 ] && [ "$NO_TORTURE" != 1 ]; then printf '%-34s ' "torture (10 families)" # --bless has to reach the torture harness too, or a run that regenerates every other golden # leaves this one stale and the very next run fails on a diff the operator just approved. @@ -384,6 +386,9 @@ if [ -d "$HERE/torture" ] && [ ${#FILTERS[@]} -eq 0 ]; then fi fi +if [ "$NO_TORTURE" = 1 ] && [ ${#FILTERS[@]} -eq 0 ]; then + echo "torture (10 families) EXCLUDED (--no-torture)" +fi echo "─────────────────────────────────────────────" echo "passed $pass failed $fail" [ $fail -eq 0 ] || { printf 'failing: %s\n' "${failed[*]}"; exit 1; } diff --git a/graph/test/tools/lib-coverage.sh b/graph/test/tools/lib-coverage.sh new file mode 100755 index 000000000..c1b119d18 --- /dev/null +++ b/graph/test/tools/lib-coverage.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# ── The client->library half must not quietly stop being tested ────────────── +# Most of what these suites assert is client->client. The client->library hand-off +# is carried by a smaller set of fixtures, and it is the half that disappears +# silently: delete a golden and the case still runs, still passes, and simply +# stops making the claim. Nothing in the suites notices, because a suite can only +# check the assertions it still has. +# +# So this pins the SHAPE of that coverage, and it needs no parser and no solver: +# +# 1. every TypeScript case carrying a lib/ is solved twice, and both goldens +# exist — the delta between them IS the client->library mapping; +# 2. every Java case carrying a lib-src/ stub library still has its golden; +# 3. the counts never fall. A number here going DOWN is either a deletion that +# wanted review, or coverage lost by accident. +set -uo pipefail +cd "$(dirname "$0")/../.." || exit 2 + +fail=0 + +# Floors, not targets. Raise one when real coverage is added; lowering one is a +# deliberate reduction in what this repository proves, and belongs in a diff. +TS_LIB_FLOOR=23 +JAVA_LIB_FLOOR=6 +PY_LIB_FLOOR=1 + +ts_libs=0; ts_missing=() +for d in test/typescript/cases/*/lib; do + [ -d "$d" ] || continue + c="$(basename "$(dirname "$d")")"; ts_libs=$((ts_libs+1)) + [ -f "test/typescript/expected/$c.edges" ] || ts_missing+=("$c.edges") + [ -f "test/typescript/expected/$c.lib.edges" ] || ts_missing+=("$c.lib.edges") +done + +java_libs=0; java_missing=() +for d in test/java/cases/*/lib-src; do + [ -d "$d" ] || continue + c="$(basename "$(dirname "$d")")"; java_libs=$((java_libs+1)) + [ -f "test/java/expected/$c.edges" ] || java_missing+=("$c.edges") +done + +py_libs=0 +[ -d test/python/torture/lib ] && py_libs=1 + +printf 'client->library coverage: typescript %d (both goldens each), java %d stub libs, python %d torture lib\n' \ + "$ts_libs" "$java_libs" "$py_libs" + +if [ ${#ts_missing[@]} -gt 0 ]; then + echo " a TypeScript case ships a lib/ but is missing a golden, so its client->library half asserts nothing:" + printf ' %s\n' "${ts_missing[@]}"; fail=1 +fi +if [ ${#java_missing[@]} -gt 0 ]; then + echo " a Java case ships a lib-src/ but is missing its golden:" + printf ' %s\n' "${java_missing[@]}"; fail=1 +fi + +check_floor() { # name actual floor + if [ "$2" -lt "$3" ]; then + echo " $1 client->library coverage fell: $2, was $3." + echo " If that removal was intended, lower the floor in this file in the same commit." + fail=1 + fi +} +check_floor typescript "$ts_libs" "$TS_LIB_FLOOR" +check_floor java "$java_libs" "$JAVA_LIB_FLOOR" +check_floor python "$py_libs" "$PY_LIB_FLOOR" + +[ "$fail" = 0 ] && echo " 0 violation(s)" +exit $fail