diff --git a/.github/scripts/run-suite.sh b/.github/scripts/run-suite.sh new file mode 100755 index 00000000..3eb88d14 --- /dev/null +++ b/.github/scripts/run-suite.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Run one regression suite for CI and turn every way a suite can quietly not test into a +# failure: a suite that skips itself (exit 77), a sub-harness that prints SKIP, a suite that +# passed 0 cases. CI must supply what a suite wants; do not relax this to go green. +# run-suite.sh [suite args...] +set -uo pipefail +lang="${1:?usage: run-suite.sh [args...]}"; shift +root="$(cd "$(dirname "$0")/../.." && pwd)" +[ -f "$root/parser/dist/index.js" ] || { echo "::error::parser/dist/index.js is missing — npm run build did not run, and every suite would skip itself"; exit 1; } +log="$(mktemp)" +echo "── $lang suite" +"$root/bin/axiomcode" test "$lang" "$@" 2>&1 | tee "$log" +rc="${PIPESTATUS[0]}" +if [ "$rc" -eq 77 ]; then + echo "::error::the $lang suite skipped itself (exit 77). A skipped suite is a failed gate."; exit 1 +fi +if grep -qE '(^|[[:space:]])SKIP([: (]|PED)' "$log"; then + echo "::error::a sub-harness in the $lang suite skipped instead of running:" + grep -nE '(^|[[:space:]])SKIP([: (]|PED)' "$log" | sed 's/^/ /' + exit 1 +fi +if [ "$lang" != parser ] && grep -qE '^passed 0([^0-9]|$)|^passed 0,' "$log"; then + echo "::error::the $lang suite passed 0 cases — it asserted nothing."; exit 1 +fi +exit "$rc" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..46f4c984 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,183 @@ +# ───────────────────────────────────────────────────────────────────────────── +# build — the gate every change to main passes through. +# +# Four tiers, cheapest first, all required: +# install a fresh clone installs and builds (parser + engine), and the command runs +# hygiene repository invariants that need no solver and catch silent breakage +# engine the java / typescript / python / javascript regression suites, in parallel +# parser the parser's own suites +# +# NO PATH FILTERS, deliberately. A required check that is skipped by a path filter never +# reports, and a pull request waiting on a check that will never report can never merge. +# ───────────────────────────────────────────────────────────────────────────── +name: build + +on: + push: + branches: [main] + pull_request: + merge_group: + workflow_dispatch: + +# One run per ref. A new push to a pull request cancels the previous run; a run on main is +# always allowed to finish — main's history is the record. +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + NODE_VERSION: '22' # the bundle stage writes graph.sqlite with node:sqlite (≥ 22.5) + +jobs: + install: + name: fresh clone installs, builds, runs + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: '22' } + # No lock file is committed, so this is `npm install`, and `prepare` builds both + # packages. The explicit steps after it mean a prepare-script change cannot silently + # stop compiling this repository. + - run: npm install + - run: npm run typecheck + - run: npm run build + - name: the build produced what the command needs + run: | + test -f parser/dist/index.js || { echo "::error::parser/dist/index.js missing"; exit 1; } + test -f dist/bundle/cli.js || { echo "::error::dist/bundle/cli.js missing"; exit 1; } + test ! -d dist/test || { echo "::error::fixtures were compiled into dist/"; exit 1; } + bin/axiomcode --help | head -1 + bin/axiomcode parser graph/test/java/cases/01-inheritance-override/src /tmp/ir >/dev/null + test -f /tmp/ir/java/all-methods.csv || { echo "::error::the parser wrote no IR"; exit 1; } + + hygiene: + name: repository invariants + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: '22' } + - run: npm install + - name: every fixture input is tracked by git + run: bash graph/test/tools/no-ignored-fixtures.sh + - name: IR staging maps are consistent, per language + run: | + fail=0 + for lang in java python typescript javascript; do + echo "── $lang"; python3 graph/test/tools/check_staging.py --lang "$lang" || fail=1 + done + exit $fail + - name: client->library coverage has not shrunk + run: bash graph/test/tools/lib-coverage.sh + - name: the output bundle is one schema across languages and its doc is current + run: bash graph/test/tools/bundle-test.sh + - name: the engine id is a function of the rules alone + run: bash graph/test/tools/engine-id-test.sh + - name: shell scripts parse + run: | + fail=0 + while IFS= read -r f; do bash -n "$f" || { echo "::error file=$f::does not parse"; fail=1; }; done < <(git ls-files '*.sh') + exit $fail + + engine: + name: engine (${{ matrix.lang }}) + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + # --oracle scores the engine against GROUND TRUTH, not only the goldens. A golden + # says "the same as last time", which a wrong answer satisfies as long as it was + # wrong last time too. The ground truth is built with the toolchain that defines + # the language: javac/javap, the TypeScript compiler (also over JavaScript with + # allowJs/checkJs). No third-party analyzer. + # + # --no-torture for java ONLY: those families need the JVM platform IR staged as a + # library (1.8 GB, built from a JDK source checkout), which cannot live in a + # repository or a cache; without it their receivers are unresolvable BY + # CONSTRUCTION and the red would read as a regression. Six Java cases still cover + # client->library with their own stub libraries. The torture families stay a local + # gate; the suite prints EXCLUDED so they are never mistaken for a pass. + - { lang: java, oracle: '--oracle --no-torture' } + - { lang: typescript, oracle: '--oracle' } + - { lang: javascript, oracle: '--oracle' } + # Python's ground truth is frozen CPython output authored by a separate harness + # CI cannot reach yet, so this leg is goldens-only for now. + - { lang: python, oracle: '' } + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: '22' } + - run: npm install + # TWO interpreters for python: 3.10 for the tier-1 attribution preflight (it reads + # CPython OPCODES, whose shapes are not stable across minor versions, and resolves + # `python3.10` by name); 3.12 for the torture fixtures, whose source uses typing.Self + # (3.11+). The later setup-python wins for plain `python3`, so 3.12 comes second. + - uses: actions/setup-python@v5 + with: { python-version: '3.10' } + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + # JDK 24: the java torture/oracle tooling reads class files with java.lang.classfile, + # which is not final before 24; on an older JDK it exits 77 and silently does not run. + - uses: actions/setup-java@v4 + if: matrix.lang == 'java' + with: { distribution: temurin, java-version: '24' } + # Soufflé is the solver: the engine is compiled from .dl to C++ against its headers, + # so a build of it has to be present the way a compiler has to be present. Pinned to + # the version in graph/pipeline/engine.conf and verified against upstream's checksum. + - name: cache the Soufflé package + uses: actions/cache@v4 + with: + path: ~/souffle-pkg + key: souffle-deb-2.5-ubuntu-2404 + - name: install Soufflé + run: | + set -euo pipefail + . graph/pipeline/engine.conf + deb="x86_64-ubuntu-2404-souffle-${SOUFFLE_VERSION}-Linux.deb" + dir="$HOME/souffle-pkg"; mkdir -p "$dir" + [ -f "$dir/$deb" ] || curl -fsSL --retry 3 -o "$dir/$deb" "https://github.com/souffle-lang/souffle/releases/download/${SOUFFLE_VERSION}/${deb}" + echo "6b86e554f6aa5abf8a8b55d8312ae37c0957c5bd6c9edeea89246db9406f645ec5e600b84fe6636b1c163da556f0da6c3d2dad46c1083413f2fcf4f95b9ac62c $dir/$deb" | sha512sum -c - + sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends "$dir/$deb" + souffle --version | head -2 + # run-souffle.sh caches the compiled engine under the rule-set id; mirroring that key + # here skips a multi-minute C++ build on every run whose rules did not change. + - name: cache the compiled engine + uses: actions/cache@v4 + with: + path: .souffle-cache + key: engine-${{ matrix.lang }}-${{ hashFiles(format('graph/{0}/**/*.dl', matrix.lang), 'graph/pipeline/engine.conf') }} + - name: ${{ matrix.lang }} regression suite + run: bash .github/scripts/run-suite.sh ${{ matrix.lang }} ${{ matrix.oracle }} + + parser: + name: parser suites + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: '22' } + - run: npm install + - run: bash .github/scripts/run-suite.sh parser + + # A single job the ruleset can require. Without it every matrix entry has to be added to + # the protection rules by hand, and a matrix job that fails to start reports nothing — + # which a ruleset reads as "not failing" rather than "did not run". + build: + name: build + runs-on: ubuntu-24.04 + needs: [install, hygiene, engine, parser] + if: always() + steps: + - name: every tier passed + run: | + echo "install=${{ needs.install.result }} hygiene=${{ needs.hygiene.result }} engine=${{ needs.engine.result }} parser=${{ needs.parser.result }}" + [ "${{ needs.install.result }}" = success ] && [ "${{ needs.hygiene.result }}" = success ] && [ "${{ needs.engine.result }}" = success ] && [ "${{ needs.parser.result }}" = success ] diff --git a/.github/workflows/main-guard.yml b/.github/workflows/main-guard.yml new file mode 100644 index 00000000..040023df --- /dev/null +++ b/.github/workflows/main-guard.yml @@ -0,0 +1,87 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Every commit on `main` should have arrived through a pull request. +# +# This reports when one did not. It is a backstop and not the rule itself: a +# workflow runs after the push has already been accepted, so it can record a +# direct push but never refuse one. The rule that refuses lives in +# .github/scripts/protect-main.sh. +# ───────────────────────────────────────────────────────────────────────────── +name: main-guard + +on: + push: + branches: [main] + +permissions: + contents: read + issues: write + +jobs: + direct-push: + name: every commit on main came from a pull request + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: check the pushed commits + id: check + env: + GH_TOKEN: ${{ github.token }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.event.after }} + run: | + set -uo pipefail + + # A branch created or force-updated from nothing has no usable range. + if [ "$BEFORE" = "0000000000000000000000000000000000000000" ]; then + echo "branch created — nothing to compare"; exit 0 + fi + + orphans=() + while IFS= read -r sha; do + [ -n "$sha" ] || continue + n="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${sha}/pulls" --jq 'length' 2>/dev/null || echo 0)" + if [ "$n" = "0" ]; then + orphans+=("$sha $(git log -1 --format=%s "$sha")") + fi + done < <(git rev-list "${BEFORE}..${AFTER}" 2>/dev/null) + + if [ ${#orphans[@]} -eq 0 ]; then + echo "all pushed commits arrived through a pull request" + exit 0 + fi + + { + echo "### Direct push to \`main\` detected" + echo + echo "These commits are on \`main\` without a pull request:" + echo + printf -- '- %s\n' "${orphans[@]}" + } >> "$GITHUB_STEP_SUMMARY" + + printf '%s\n' "${orphans[@]}" > /tmp/orphans.txt + echo "found=1" >> "$GITHUB_OUTPUT" + echo "::error::${#orphans[@]} commit(s) reached main without a pull request" + exit 1 + + - name: record it as an issue + if: failure() && steps.check.outputs.found == '1' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + title="Direct push to main on $(date -u +%Y-%m-%d)" + # One issue per day, not one per push. + existing="$(gh issue list --state open --search "\"$title\" in:title" --json number --jq '.[0].number' || true)" + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + body="$(printf "Commits reached \`main\` without a pull request:\n\n\`\`\`\n%s\n\`\`\`\n\nRun: %s\n\nReported after the fact: a workflow runs once the push has been accepted, so it can record a direct push but not refuse one. See .github/scripts/protect-main.sh\n" \ + "$(cat /tmp/orphans.txt)" "$run_url")" + if [ -n "${existing:-}" ]; then + gh issue comment "$existing" --body "$body" + else + gh issue create --title "$title" --body "$body" --label platform || \ + gh issue create --title "$title" --body "$body" + fi diff --git a/graph/test/java/run-tests.sh b/graph/test/java/run-tests.sh index 1ef9c26d..d9e05c2a 100755 --- a/graph/test/java/run-tests.sh +++ b/graph/test/java/run-tests.sh @@ -21,6 +21,8 @@ # (javac + javap invoke instructions — no third-party analyzer): # every bytecode-declared client->client edge must be present. # ./run-tests.sh --keep keep the per-case work dirs for debugging +# ./run-tests.sh --no-torture skip the torture families (CI: they need the JVM platform IR, +# which cannot be staged there; the suite prints EXCLUDED) # # With --oracle, a case carrying a spring-oracle.conf ALSO boots its sources in a real # AnnotationConfigApplicationContext and scores bean_def / di_edge against what Spring @@ -107,9 +109,9 @@ if ! bash "$ROOT/graph/test/tools/bundle-test.sh"; then fi PARSER="${AXIOM_PARSER:-$ROOT/parser/dist/index.js}" WORK="$HERE/.work" -BLESS=0; KEEP=0; ORACLE=0; FILTERS=() +BLESS=0; KEEP=0; ORACLE=0; NO_TORTURE=0; FILTERS=() for a in "$@"; do case "$a" in - --bless) BLESS=1;; --keep) KEEP=1;; --oracle) ORACLE=1;; + --bless) BLESS=1;; --keep) KEEP=1;; --oracle) ORACLE=1;; --no-torture) NO_TORTURE=1;; -h|--help) sed -n '2,34p' "$0"; exit 0;; *) FILTERS+=("$a");; esac; done # ── PREFLIGHT: every relation the parser emits must actually reach the solver ────────── @@ -370,7 +372,7 @@ done # The cases above each pin ONE rule. This asks what happens when a project uses everything at # once, and reports WHICH construct is the gap rather than one number. It stages the platform IR, # because half the families call java.util types and scoring them without it measures the staging. -if [ -d "$HERE/torture" ] && [ ${#FILTERS[@]} -eq 0 ]; then +if [ -d "$HERE/torture" ] && [ ${#FILTERS[@]} -eq 0 ] && [ "$NO_TORTURE" != 1 ]; then printf '%-34s ' "torture (10 families)" # --bless has to reach the torture harness too, or a run that regenerates every other golden # leaves this one stale and the very next run fails on a diff the operator just approved. @@ -384,6 +386,9 @@ if [ -d "$HERE/torture" ] && [ ${#FILTERS[@]} -eq 0 ]; then fi fi +if [ "$NO_TORTURE" = 1 ] && [ ${#FILTERS[@]} -eq 0 ]; then + echo "torture (10 families) EXCLUDED (--no-torture)" +fi echo "─────────────────────────────────────────────" echo "passed $pass failed $fail" [ $fail -eq 0 ] || { printf 'failing: %s\n' "${failed[*]}"; exit 1; } diff --git a/graph/test/tools/lib-coverage.sh b/graph/test/tools/lib-coverage.sh new file mode 100755 index 00000000..c1b119d1 --- /dev/null +++ b/graph/test/tools/lib-coverage.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# ── The client->library half must not quietly stop being tested ────────────── +# Most of what these suites assert is client->client. The client->library hand-off +# is carried by a smaller set of fixtures, and it is the half that disappears +# silently: delete a golden and the case still runs, still passes, and simply +# stops making the claim. Nothing in the suites notices, because a suite can only +# check the assertions it still has. +# +# So this pins the SHAPE of that coverage, and it needs no parser and no solver: +# +# 1. every TypeScript case carrying a lib/ is solved twice, and both goldens +# exist — the delta between them IS the client->library mapping; +# 2. every Java case carrying a lib-src/ stub library still has its golden; +# 3. the counts never fall. A number here going DOWN is either a deletion that +# wanted review, or coverage lost by accident. +set -uo pipefail +cd "$(dirname "$0")/../.." || exit 2 + +fail=0 + +# Floors, not targets. Raise one when real coverage is added; lowering one is a +# deliberate reduction in what this repository proves, and belongs in a diff. +TS_LIB_FLOOR=23 +JAVA_LIB_FLOOR=6 +PY_LIB_FLOOR=1 + +ts_libs=0; ts_missing=() +for d in test/typescript/cases/*/lib; do + [ -d "$d" ] || continue + c="$(basename "$(dirname "$d")")"; ts_libs=$((ts_libs+1)) + [ -f "test/typescript/expected/$c.edges" ] || ts_missing+=("$c.edges") + [ -f "test/typescript/expected/$c.lib.edges" ] || ts_missing+=("$c.lib.edges") +done + +java_libs=0; java_missing=() +for d in test/java/cases/*/lib-src; do + [ -d "$d" ] || continue + c="$(basename "$(dirname "$d")")"; java_libs=$((java_libs+1)) + [ -f "test/java/expected/$c.edges" ] || java_missing+=("$c.edges") +done + +py_libs=0 +[ -d test/python/torture/lib ] && py_libs=1 + +printf 'client->library coverage: typescript %d (both goldens each), java %d stub libs, python %d torture lib\n' \ + "$ts_libs" "$java_libs" "$py_libs" + +if [ ${#ts_missing[@]} -gt 0 ]; then + echo " a TypeScript case ships a lib/ but is missing a golden, so its client->library half asserts nothing:" + printf ' %s\n' "${ts_missing[@]}"; fail=1 +fi +if [ ${#java_missing[@]} -gt 0 ]; then + echo " a Java case ships a lib-src/ but is missing its golden:" + printf ' %s\n' "${java_missing[@]}"; fail=1 +fi + +check_floor() { # name actual floor + if [ "$2" -lt "$3" ]; then + echo " $1 client->library coverage fell: $2, was $3." + echo " If that removal was intended, lower the floor in this file in the same commit." + fail=1 + fi +} +check_floor typescript "$ts_libs" "$TS_LIB_FLOOR" +check_floor java "$java_libs" "$JAVA_LIB_FLOOR" +check_floor python "$py_libs" "$PY_LIB_FLOOR" + +[ "$fail" = 0 ] && echo " 0 violation(s)" +exit $fail