From 6e06be3f96b297d7ef7e61e16586abf98d11b7dc Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Sun, 13 Sep 2026 13:46:37 -0700 Subject: [PATCH 1/9] =?UTF-8?q?build:=20prebuilt=20engine=20binaries=20?= =?UTF-8?q?=E2=80=94=20CI=20builds=20Linux/macOS/Windows=20on=20merge,=20t?= =?UTF-8?q?he=20script=20fetches=20them=20(#454)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running the engine required a Soufflé installation and a C++ toolchain: run-souffle.sh generated C++ from the rules and compiled it on first use, and failed outright without both. Neither is needed at run time. The compiled engine is one self-contained executable (linked against nothing but the C++ runtime) that is already project-independent, and `souffle -g`'s output is portable C++ that needs only Soufflé's headers to compile. The program is now a pure function of the repository: includes are relative to src/ (souffle -I src) and the .input list is derived from the staging maps rather than from a listing of the staged facts, so the same text comes out of every checkout and of CI with no client IR. Its sha256 together with every included file and the PINNED Soufflé version (src/pipeline/engine.conf — a machine without souffle cannot ask for one) is the engine id: path-independent, and different for any change to a rule, a map, the manifest or the pin. Two new modes expose it: --print-engine-id and --emit-program. .github/workflows/engine-binaries.yml, on every merge to main touching the rules: generate the C++ once on Ubuntu with the pinned .deb, compile it on ubuntu-24.04, ubuntu-24.04-arm, macos-14 (universal) and windows-2025 (MSVC) with the same flags the script uses locally and portable targets instead of -march=native, smoke-run each binary on empty inputs, and publish a release tagged engine-- holding the binaries, sha256sum.txt and the generated .cpp. Languages whose id already has a release are skipped. run-souffle.sh keeps its compile branch when souffle is on PATH (warning if the version is not the pinned one). Otherwise it resolves the platform, fetches the asset for its id through gh or curl+GH_TOKEN (the repository is private), verifies the sha256, and caches the binary under its id. No release for the id fails with the two ways out named — never a silently stale binary. A missing basic tool is refused up front, because a missing grep inside a process substitution had silently produced an empty relation list and a wrong id. Guards, both without souffle or network: test/tools/engine-id-test.sh (same id from two paths; changed by a rule, a map, the manifest, the pin; one language's change does not move another's) and test/tools/engine-fetch-test.sh (a fake gh serves a fake engine: fetched, verified, cached, run through to the bundle; tampered checksum refused with nothing cached; absent release explained). Both are preflights of the Java suite. --- .github/workflows/engine-binaries.yml | 214 ++++++++++++++++++++++ README.md | 12 ++ src/pipeline/engine.conf | 14 ++ src/pipeline/portable-stat.sh | 11 ++ src/pipeline/run-souffle.sh | 251 +++++++++++++++++++------- test/java/run-tests.sh | 12 ++ test/tools/engine-fetch-test.sh | 77 ++++++++ test/tools/engine-id-test.sh | 53 ++++++ 8 files changed, 580 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/engine-binaries.yml create mode 100644 src/pipeline/engine.conf create mode 100755 test/tools/engine-fetch-test.sh create mode 100755 test/tools/engine-id-test.sh diff --git a/.github/workflows/engine-binaries.yml b/.github/workflows/engine-binaries.yml new file mode 100644 index 000000000..0eb4b54e3 --- /dev/null +++ b/.github/workflows/engine-binaries.yml @@ -0,0 +1,214 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Prebuilt engine binaries — so running the engine needs no Soufflé and no compiler. +# +# On every merge to main that can change a rule set, for each language whose engine id +# (src/pipeline/run-souffle.sh --print-engine-id) has no release yet: +# generate Ubuntu + the pinned Soufflé .deb: emit the program, `souffle -g` it to portable +# C++ once, and capture the header tree the C++ compiles against. +# build compile that C++ on every platform with the SAME flags the script uses locally +# (-std=c++17 -O3, no OpenMP/zlib/sqlite defines — the binary links against nothing +# but the C++ runtime). Portable target: no -march=native; macOS universal. +# release one GitHub release per (language, id), tagged engine--, holding +# axiom-engine---[.exe], sha256sum.txt, and the generated .cpp. +# run-souffle.sh computes the same id on the user's machine and fetches the matching asset. +# The id is a function of the repository alone (see engine_id in the script), which is what +# makes "no release for this id" mean exactly "these rules have not been merged and built". +# ───────────────────────────────────────────────────────────────────────────── +name: engine-binaries + +on: + push: + branches: [main] + paths: + - 'src/**/*.dl' + - 'src/*/templates/**' + - 'src/pipeline/**' + - '.github/workflows/engine-binaries.yml' + workflow_dispatch: + inputs: + force: + description: rebuild and re-publish even if the release for this id exists + type: boolean + default: false + +permissions: + contents: write + +env: + LANGUAGES: java typescript python + +jobs: + generate: + runs-on: ubuntu-24.04 + outputs: + langs: ${{ steps.plan.outputs.langs }} + steps: + - uses: actions/checkout@v4 + + - name: Install the pinned Soufflé + run: | + . src/pipeline/engine.conf + deb="x86_64-ubuntu-2404-souffle-${SOUFFLE_VERSION}-Linux.deb" + curl -fsSL -o "/tmp/$deb" "https://github.com/souffle-lang/souffle/releases/download/${SOUFFLE_VERSION}/$deb" + sudo apt-get update -qq + sudo apt-get install -y -qq "/tmp/$deb" + souffle --version | head -2 + test -f /usr/include/souffle/CompiledSouffle.h + + - name: Plan — which languages need a build + id: plan + env: + GH_TOKEN: ${{ github.token }} + FORCE: ${{ inputs.force }} + run: | + set -e + mkdir -p out + langs="" + for lang in $LANGUAGES; do + id="$(bash src/pipeline/run-souffle.sh --language "$lang" --print-engine-id)" + echo "$lang: $id" + if [ "$FORCE" != "true" ] && gh release view "engine-$lang-$id" >/dev/null 2>&1; then + echo " release engine-$lang-$id exists — skipping"; continue + fi + printf '%s' "$id" > "out/$lang.id" + langs="$langs \"$lang\"" + done + . src/pipeline/engine.conf; printf '%s' "$SOUFFLE_VERSION" > out/souffle.version + langs="[$(echo $langs | sed 's/ /,/g')]" + echo "langs=$langs" >> "$GITHUB_OUTPUT" + echo "building: $langs" + + - name: Generate portable C++ for each language + if: steps.plan.outputs.langs != '[]' + run: | + set -e + for lang in $LANGUAGES; do + [ -f "out/$lang.id" ] || continue + bash src/pipeline/run-souffle.sh --language "$lang" --emit-program "out/$lang.dl" + # souffle's "No rules/facts defined" notices are expected for the never-staged + # lib-body relations; anything else on stderr is shown. + souffle -I src -g "out/$lang.cpp" "out/$lang.dl" 2> "out/$lang.gen.log" || { cat "out/$lang.gen.log"; exit 1; } + awk '/No rules\/facts defined/{skip=2;next} skip>0{skip--;next} {print}' "out/$lang.gen.log" + ls -la "out/$lang.cpp" + done + cp -r /usr/include/souffle out/souffle-headers + + - uses: actions/upload-artifact@v4 + if: steps.plan.outputs.langs != '[]' + with: + name: generated + path: out + retention-days: 7 + if-no-files-found: error + + build: + needs: generate + if: needs.generate.outputs.langs != '[]' + strategy: + fail-fast: false + matrix: + lang: ${{ fromJson(needs.generate.outputs.langs) }} + target: + - { os: ubuntu-24.04, platform: linux-x86_64 } + - { os: ubuntu-24.04-arm, platform: linux-arm64 } + - { os: macos-14, platform: darwin-universal } + - { os: windows-2025, platform: windows-x86_64 } + runs-on: ${{ matrix.target.os }} + steps: + - uses: actions/download-artifact@v4 + with: + name: generated + path: gen + + - name: Compile (Linux) + if: startsWith(matrix.target.platform, 'linux') + run: | + set -e + # -static-libstdc++/-static-libgcc: the only non-glibc dependency is folded in, so + # the binary runs on any distro with a glibc at least as old as this runner's. + c++ -std=c++17 -O3 -w -static-libstdc++ -static-libgcc -I gen/souffle-headers \ + "gen/${{ matrix.lang }}.cpp" -o "axiom-engine-${{ matrix.lang }}-${{ matrix.target.platform }}" + ls -la axiom-engine-*; ldd axiom-engine-* || true + + - name: Compile (macOS, universal) + if: startsWith(matrix.target.platform, 'darwin') + run: | + set -e + c++ -std=c++17 -O3 -w -arch arm64 -arch x86_64 -mmacosx-version-min=12.0 -I gen/souffle-headers \ + "gen/${{ matrix.lang }}.cpp" -o "axiom-engine-${{ matrix.lang }}-${{ matrix.target.platform }}" + ls -la axiom-engine-*; file axiom-engine-*; otool -L axiom-engine-* + + - uses: ilammy/msvc-dev-cmd@v1 + if: startsWith(matrix.target.platform, 'windows') + with: + arch: x64 + + - name: Compile (Windows, MSVC) + if: startsWith(matrix.target.platform, 'windows') + shell: cmd + run: | + cl /nologo /std:c++17 /O2 /EHsc /bigobj /w /permissive- /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS /DNOMINMAX /I gen\souffle-headers gen\${{ matrix.lang }}.cpp /Fe:axiom-engine-${{ matrix.lang }}-${{ matrix.target.platform }}.exe + if errorlevel 1 exit /b 1 + dir axiom-engine-* + + - name: Smoke — the binary starts and reports its relations + shell: bash + run: | + set -e + bin="$(ls axiom-engine-* | grep -v '\.sha256$' | head -1)" + chmod +x "$bin" 2>/dev/null || true + mkdir -p facts out + # every declared input must exist; an empty file is a valid (empty) relation + sed -n 's/^\.input \([A-Za-z0-9_]*\)(.*/\1/p' "gen/${{ matrix.lang }}.dl" | while read -r r; do : > "facts/$r.facts"; done + "./$bin" -F facts -D out + echo "outputs written: $(ls out | wc -l)" + + - name: Checksum + shell: bash + run: | + bin="$(ls axiom-engine-* | head -1)" + if command -v sha256sum >/dev/null; then sha256sum "$bin" > "$bin.sha256"; else shasum -a 256 "$bin" > "$bin.sha256"; fi + cat "$bin.sha256" + + - uses: actions/upload-artifact@v4 + with: + name: engine-${{ matrix.lang }}-${{ matrix.target.platform }} + path: axiom-engine-* + if-no-files-found: error + + release: + needs: [generate, build] + if: needs.generate.outputs.langs != '[]' + runs-on: ubuntu-24.04 + strategy: + matrix: + lang: ${{ fromJson(needs.generate.outputs.langs) }} + steps: + - uses: actions/download-artifact@v4 + with: + name: generated + path: gen + - uses: actions/download-artifact@v4 + with: + pattern: engine-${{ matrix.lang }}-* + path: bins + merge-multiple: true + + - name: Publish release engine-${{ matrix.lang }}- + env: + GH_TOKEN: ${{ github.token }} + FORCE: ${{ inputs.force }} + run: | + set -e + lang="${{ matrix.lang }}" + id="$(cat "gen/$lang.id")" + tag="engine-$lang-$id" + cat bins/*.sha256 | sed 's# .*/# #' > bins/sha256sum.txt + rm -f bins/*.sha256 + cp "gen/$lang.cpp" "bins/$lang.cpp" + echo "assets:"; ls -la bins + notes="Prebuilt engine for the **$lang** rule set at id \`$id\` (Soufflé $(cat gen/souffle.version), pinned in src/pipeline/engine.conf), built from ${{ github.sha }}. + + \`run-souffle.sh --language $lang\` fetches the asset for its platform when \`souffle\` is not installed. \`$lang.cpp\` is the generated program for anyone who prefers to compile it themselves (needs only the Soufflé headers and a C++17 compiler)." + if [ "$FORCE" = "true" ]; then gh release delete "$tag" -y 2>/dev/null || true; fi + gh release create "$tag" --title "engine · $lang · ${id:0:12}" --notes "$notes" --target "${{ github.sha }}" bins/* diff --git a/README.md b/README.md index 2a2e904c4..56d5a288c 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,12 @@ receiver is a lambda parameter. Restricted to files of 1,000 lines or more, d1 r ## Run +Requirements: Node ≥ 22.5 and a POSIX shell with `awk` (Git Bash on Windows). **No Soufflé and +no C++ compiler**: the rules compile to one self-contained executable, CI builds it for +Linux (x86_64, arm64), macOS (universal) and Windows on every merge to `main`, and the script +fetches the one for your platform on first use — verified by sha256, cached under the rule +set's id. With `souffle` installed the script compiles locally instead, exactly as before. + ```bash npm install && npm run build @@ -140,6 +146,12 @@ bash src/pipeline/run-souffle.sh \ --intermediate --output ``` +The prebuilt binaries live in GitHub releases tagged `engine--`, where `` is +`bash src/pipeline/run-souffle.sh --language --print-engine-id` — a sha256 of the rules +and the pinned Soufflé version (`src/pipeline/engine.conf`), the same from any checkout. The +repository is private, so the fetch authenticates through `gh` or a `GH_TOKEN`. A rule set that +has not been merged has no release: edit rules with Soufflé installed, or merge first. + **Outputs — the same in every language** ([`src/bundle/SCHEMA.md`](src/bundle/SCHEMA.md)) ``` diff --git a/src/pipeline/engine.conf b/src/pipeline/engine.conf new file mode 100644 index 000000000..9f1b1bf3d --- /dev/null +++ b/src/pipeline/engine.conf @@ -0,0 +1,14 @@ +# ───────────────────────────────────────────────────────────────────────────── +# The prebuilt-engine contract. Sourced by run-souffle.sh and by the CI workflow. +# +# SOUFFLE_VERSION is PINNED here, not read from a `souffle --version`, because the machine +# that runs a prebuilt binary has no souffle to ask — and the engine id it computes must be +# the id CI computed. Bumping it changes every language's id, which is what a new code +# generator should do. CI installs exactly this version. +# +# ENGINE_REPO is where the binaries are published: one GitHub release per (language, id), +# tagged engine--, holding axiom-engine--[.exe] and sha256sum.txt. +# Override with AXIOM_ENGINE_REPO to fetch from a fork or a mirror. +# ───────────────────────────────────────────────────────────────────────────── +SOUFFLE_VERSION="2.5" +ENGINE_REPO="${AXIOM_ENGINE_REPO:-AxiomCodeAI/axiom-code-graph}" diff --git a/src/pipeline/portable-stat.sh b/src/pipeline/portable-stat.sh index 363e4df88..77f23a837 100644 --- a/src/pipeline/portable-stat.sh +++ b/src/pipeline/portable-stat.sh @@ -52,3 +52,14 @@ sha1_stdin(){ [ -n "$_SHA1_CMD" ] || { echo "neither shasum nor sha1sum is on PATH" >&2; return 1; } "$_SHA1_CMD" | cut -d' ' -f1 } + +# sha256 of stdin, for the engine id and for verifying a downloaded binary. Same three +# spellings as sha1 above: `shasum -a 256` (macOS, perl shasum in Git Bash) or `sha256sum` +# (coreutils). +if command -v sha256sum >/dev/null 2>&1; then _SHA256_CMD="sha256sum" +elif command -v shasum >/dev/null 2>&1; then _SHA256_CMD="shasum -a 256" +else _SHA256_CMD=""; fi +sha256_stdin(){ + [ -n "$_SHA256_CMD" ] || { echo "neither sha256sum nor shasum is on PATH" >&2; return 1; } + $_SHA256_CMD | cut -d' ' -f1 +} diff --git a/src/pipeline/run-souffle.sh b/src/pipeline/run-souffle.sh index ba6327e4b..9bd6727ae 100755 --- a/src/pipeline/run-souffle.sh +++ b/src/pipeline/run-souffle.sh @@ -3,6 +3,14 @@ # client-ir.map / lib.map (single source of truth). Lib is auto-scoped # to only the signature relations the rules reference (never loads GB-scale bodies). # Usage: run-souffle.sh --client-ir DIR --library DIR --intermediate DIR --output DIR [--language L] [--debug] +# run-souffle.sh --language L --print-engine-id the canonical id of L's compiled engine +# run-souffle.sh --language L --emit-program FILE the Soufflé program CI compiles for L +# +# NO SOUFFLÉ NEEDED TO RUN. The rules compile to one self-contained executable that is +# project-independent; CI builds it for every platform on merge (engine-binaries.yml) and +# publishes it under a release tagged engine--. When `souffle` is not on PATH this +# script fetches that binary for the local platform (once, into the cache) and verifies its +# sha256. With souffle installed it compiles locally as before. See src/pipeline/engine.conf. # # OUTPUT LAYOUT — the same in every language (src/bundle/SCHEMA.md): # $OUT/graph.sqlite the contract: core tables + ext_* tables + the schema catalog @@ -32,6 +40,8 @@ DISPATCH_CAP="${DISPATCH_CAP:-20}" # fan-width cap on virtual dispatch. DEFAUL LANG_ARG="" # which rule set under src// to run. Default java. TAINT="" # --taint on → gate lib→lib GROW on client-seeded data flow (dataflow/taint.dl). Also # settable via env AXIOM_TAINT_GATING=on. Empty = ungated (default behavior). +MODE="run" # run | print-engine-id | emit-program — the last two need no IR and no souffle +EMIT="" while [ $# -gt 0 ]; do case "$1" in --client-ir) CLIENT="$2"; shift 2;; --library) LIB="$2"; shift 2;; --intermediate) INT="$2"; shift 2;; --output) OUT="$2"; shift 2;; @@ -46,6 +56,8 @@ while [ $# -gt 0 ]; do case "$1" in --debug) DEBUG_BUNDLE=1; shift;; # (AXIOM_DEBUG=1 in the environment is the same as --debug — for harnesses that cannot # change the invocation.) + --print-engine-id) MODE="print-engine-id"; shift;; + --emit-program) MODE="emit-program"; EMIT="$2"; shift 2;; *) shift;; esac; done SRC="$(cd "$(dirname "$0")/.." && pwd)" # shellcheck source=portable-stat.sh @@ -56,14 +68,91 @@ ENG="$SRC/$LANG_ARG/engine"; ENG2="$SRC/$LANG_ARG/engine-ii"; DL="$SRC/$LANG_ARG [ -d "$ENG" ] || { echo "no rule set for --language=$LANG_ARG (looked in $ENG)" >&2; exit 1; } # shellcheck source=souffle-include.sh . "$SRC/pipeline/souffle-include.sh" -INNER="$(find_souffle_include)" -# Assert the HEADER, not the directory: `[ -d ]` is the test #216 established cannot tell the two -# install layouts apart, so it would pass a path that then fails at the compiler. -if [ -z "$INNER" ] || [ ! -f "$INNER/souffle/CompiledSouffle.h" ]; then - echo "❌ soufflé headers not found. Install soufflé, or set AXIOM_SOUFFLE_INCLUDE." >&2 - echo " macOS: brew install souffle Debian/Ubuntu: apt-get install souffle" >&2 - exit 1 -fi +# shellcheck source=engine.conf +. "$SRC/pipeline/engine.conf" +# Staging config is PER-LANGUAGE (IR marker + which relations are signatures vs bodies). +# Keeping it here would hardcode Java's entity set into a shared executor. +[ -f "$TPL/staging.conf" ] || { echo "missing $TPL/staging.conf for --language=$LANG_ARG" >&2; exit 1; } +. "$TPL/staging.conf" +ENGINE_II_MODE="${ENGINE_II:-${AXIOM_ENGINE_II:-off}}" + +# The tools every path below relies on. Checked up front because a missing one does not +# always fail loudly: read_map runs inside a process substitution, where a missing grep +# yields an EMPTY relation list — and a different, wrong engine id — under `set -e`. +for t in grep awk sed sort cut tr mktemp uname dirname cat; do + command -v "$t" >/dev/null 2>&1 || { echo "❌ required tool not on PATH: $t" >&2; exit 1; } +done + +# read an import map (relationcsv-basename per line), skipping comments (#) and blanks +read_map(){ grep -vE '^[[:space:]]*(#|$)' "$1"; } + +# ── THE PROGRAM, as a pure function of the repository ──────────────────────────────────── +# Written so that the SAME text comes out of every checkout and of CI: includes are relative +# to src/ (souffle resolves them through -I "$SRC"), and the .input list is derived from the +# maps rather than from a listing of the staged facts dir — so it needs no client IR, and a +# machine that cannot stage (CI) still produces the text the binary was built from. The run +# path asserts below that staging created a facts file for every .input it declares. +# The list of input relations is: every client relation, the lib signature relations, the lib +# body relations (filled per iteration), and the four knob facts. +input_relations(){ + while IFS=$'\t' read -r rel csv; do printf '%s\n' "$rel"; done < <(read_map "$TPL/client-ir.map") + while IFS=$'\t' read -r rel csv; do + case " $LIB_SIG " in *" ${rel#lib_} "*) printf '%s\n' "$rel";; esac + done < <(read_map "$TPL/lib.map") + for r in $LIB_BODY; do printf '%s\n' "$r"; done + printf '%s\n' jdk_max_depth lib_max_depth taint_gating dispatch_cap +} +write_program(){ # $1 = destination file + { + echo "#include \"$LANG_ARG/souffle/decls_base.dl\""; echo "#include \"$LANG_ARG/souffle/decls_all.dl\"" + # rfc4180=true: the IR is CSV, not TSV. The parser quotes any field containing a + # quote, tab or newline and doubles the inner quotes, so reading it as plain TSV hands + # the rules the ESCAPED text. It only bites where a JOINED column contains a quote -- + # which is why it went unnoticed -- but a string forward reference (`-> "Factory"`) + # lands squarely on one, and a field carrying a tab would shift every column after it. + # Souffle parses RFC4180 itself, so this costs one flag rather than a re-encode of + # GB-scale input. + # LC_ALL=C sort: the order is part of the program text, so it must not depend on locale. + input_relations | LC_ALL=C sort -u | while read -r r; do printf '.input %s(IO=file, filename="%s.facts", delimiter="\\t", rfc4180=true)\n' "$r" "$r"; done + for d in projections containment resolution config-resolution expression-resolution call-edge-generation; do + # [ -f ] guard: a phase directory that is empty (or absent for a language that has + # not implemented that layer yet) leaves the glob unexpanded, and souffle's C + # preprocessor then fails on a literal '*.dl' include. + for f in "$ENG/$d/"*.dl; do [ -f "$f" ] && echo "#include \"${f#"$SRC/"}\""; done + done + # engine-ii: the first→third forward-chain engine (mirrors engine/, lib-seeded). Same solve, + # included AFTER engine/ so it reads engine/'s relations (client_calls_lib seed). Glob its + # phase subfolders (both nesting levels; globs are space-safe, the repo path has spaces). + # export/ is doc-only (like engine/export) — skip it. + if [ "$ENGINE_II_MODE" = "on" ]; then + for f in "$ENG2/"*/*.dl "$ENG2/"*/*/*.dl; do + case "$f" in */export/*) continue;; esac + [ -f "$f" ] && echo "#include \"${f#"$SRC/"}\"" + done + fi + # Relative output filenames — the -D at run time supplies the directory. Keeping $OUT out + # of the program makes the compiled binary independent of the output path (better reuse). + while IFS=$'\t' read -r pred file; do [ -n "$pred" ] && printf '.output %s(IO=file, filename="%s", delimiter="\\t")\n' "$pred" "$file"; done < <(LC_ALL=C sort -u "$DL/export_manifest.tsv") + } > "$1" +} +# The engine id: sha256 over the pinned code-generator version, the program text, and every +# file it includes, in include order. A function of the repository alone — the same from any +# path, on any machine, with or without souffle — and different for any rule change. It names +# the local cache entry AND the release CI publishes, which is what lets a machine without +# souffle know which binary is its own. +engine_id(){ + local prog; prog="$(mktemp)"; write_program "$prog" + { printf 'souffle=%s\n' "$SOUFFLE_VERSION"; cat "$prog" + sed -n 's/^#include "\(.*\)"$/\1/p' "$prog" | while read -r inc; do cat "$SRC/$inc"; done + } | sha256_stdin + rm -f "$prog" +} +case "$MODE" in + print-engine-id) engine_id; exit 0;; + emit-program) write_program "$EMIT"; exit 0;; +esac + +[ -n "${CLIENT:-}" ] && [ -n "${INT:-}" ] && [ -n "${OUT:-}" ] || { echo "usage: run-souffle.sh --client-ir DIR --library DIR --intermediate DIR --output DIR [--language L]" >&2; exit 1; } FACTS="$INT/souffle-facts"; rm -rf "$FACTS"; mkdir -p "$FACTS" "$OUT" # raw/ is OWNED: wiped per run so a relation that left the manifest cannot linger from an # earlier run and be mistaken for this one's output. @@ -75,20 +164,12 @@ RAW="$OUT/raw"; rm -rf "$RAW"; mkdir -p "$RAW" CACHE_ROOT="${AXIOM_SOUFFLE_CACHE:-$SRC/../.souffle-cache}"; mkdir -p "$CACHE_ROOT" START_EPOCH=$(date +%s); START_TS=$(date '+%Y-%m-%d %H:%M:%S') -# read an import map (relationcsv-basename per line), skipping comments (#) and blanks -read_map(){ grep -vE '^[[:space:]]*(#|$)' "$1"; } - # Library roots: --library is a comma-separated list of IR roots (each with jdk-style # module sub-folders, or a flat IR dir). The caller (TS) controls which folders/libraries # are loaded; staging concatenates each relation across every module of every root. IFS=',' read -ra LIB_ROOTS <<< "$LIB" # lib_modules ROOT -> the module dirs to stage from (the root itself if it holds the IR, # else its immediate sub-folders — mirrors how the JDK ships sharded modules). -# Staging config is PER-LANGUAGE (IR marker + which relations are signatures vs bodies). -# Keeping it here would hardcode Java's entity set into a shared executor. -[ -f "$TPL/staging.conf" ] || { echo "missing $TPL/staging.conf for --language=$LANG_ARG" >&2; exit 1; } -. "$TPL/staging.conf" - lib_modules(){ if [ -f "$1/$IR_MARKER" ]; then printf '%s\n' "$1"; else for m in "$1"/*/; do [ -d "$m" ] && printf '%s\n' "${m%/}"; done; fi; } # --- CLIENT: stage EVERY mapped relation, empty when the project has no such file --- @@ -233,67 +314,100 @@ echo "▶ dispatch cap = $( [ -s "$FACTS/dispatch_cap.facts" ] && echo "$(cat "$ # (and backed up on ~/Desktop) but excluded from the compiled program; re-enable with # --engine-ii on / AXIOM_ENGINE_II=on. engine/ produces client_calls_lib etc. independently, so # client-only is a complete, valid solve on its own. -ENGINE_II_MODE="${ENGINE_II:-${AXIOM_ENGINE_II:-off}}" echo "▶ engine-ii = $( [ "$ENGINE_II_MODE" = "on" ] && echo 'ON (lib frontier included)' || echo 'OFF (client-only — engine-i)' )" -# --- generate combined program --- +# --- the program, and the binary for it: compiled here, or fetched from CI --- PROG="$INT/souffle-program.dl" -{ - echo "#include \"$DL/decls_base.dl\""; echo "#include \"$DL/decls_all.dl\"" - # rfc4180=true: the IR is CSV, not TSV. The parser quotes any field containing a - # quote, tab or newline and doubles the inner quotes, so reading it as plain TSV hands - # the rules the ESCAPED text. It only bites where a JOINED column contains a quote -- - # which is why it went unnoticed -- but a string forward reference (`-> "Factory"`) - # lands squarely on one, and a field carrying a tab would shift every column after it. - # Souffle parses RFC4180 itself, so this costs one flag rather than a re-encode of - # GB-scale input. - for ff in "$FACTS"/*.facts; do r=$(basename "$ff" .facts); printf '.input %s(IO=file, filename="%s.facts", delimiter="\\t", rfc4180=true)\n' "$r" "$r"; done - for d in projections containment resolution config-resolution expression-resolution call-edge-generation; do - # [ -f ] guard: a phase directory that is empty (or absent for a language that has - # not implemented that layer yet) leaves the glob unexpanded, and souffle's C - # preprocessor then fails on a literal '*.dl' include. - for f in "$ENG/$d/"*.dl; do [ -f "$f" ] && echo "#include \"$f\""; done - done - # engine-ii: the first→third forward-chain engine (mirrors engine/, lib-seeded). Same solve, - # included AFTER engine/ so it reads engine/'s relations (client_calls_lib seed). Glob its - # phase subfolders (both nesting levels; globs are space-safe, the repo path has spaces). - # export/ is doc-only (like engine/export) — skip it. - if [ "$ENGINE_II_MODE" = "on" ]; then - for f in "$ENG2/"*/*.dl "$ENG2/"*/*/*.dl; do - case "$f" in */export/*) continue;; esac - [ -f "$f" ] && echo "#include \"$f\"" - done - fi - # Relative output filenames — the -D at run time supplies the directory. Keeping $OUT out - # of the program makes the compiled binary independent of the output path (better reuse). - while IFS=$'\t' read -r pred file; do [ -n "$pred" ] && printf '.output %s(IO=file, filename="%s", delimiter="\\t")\n' "$pred" "$file"; done < <(sort -u "$DL/export_manifest.tsv") -} > "$PROG" +write_program "$PROG" +# Every declared input must have been staged, or souffle would fail on a missing file after +# the (possibly long) library staging. The program lists inputs from the maps; staging +# created them from the same maps, so a mismatch is a bug in this script, and says so. +for r in $(sed -n 's/^\.input \([A-Za-z0-9_]*\)(.*/\1/p' "$PROG"); do + [ -f "$FACTS/$r.facts" ] || { echo "❌ program declares input $r but staging created no $r.facts" >&2; exit 1; } +done +ENGINE_ID="$(engine_id)" +echo "▶ engine id = $ENGINE_ID (rules + souffle $SOUFFLE_VERSION)" -# --- compile once into a PERSISTENT, content-addressed cache (survives inter/ deletion) --- # What we cache is OUR engine compiled to a native binary (souffle -g turns the .dl rules # into C++, c++ compiles it) — NOT the souffle tool. It depends only on the engine (rules + # decls) and is PROJECT-INDEPENDENT (relative .input/.output), so one binary serves every # project: N concurrent analyses of N different projects all share it. It therefore lives in -# a shared, machine-scoped cache keyed by a content hash — NOT in the per-run intermediate -# (which the pipeline/parser wipes). The hash covers $PROG + every #included decls/engine -# .dl, so any rule/decl change → new hash → new binary; unchanged → instant reuse. Default -# ~/.cache/AxiomCode-Souffle (XDG-aware); delete it to force a clean rebuild, or override -# with AXIOM_SOUFFLE_CACHE. -# Default IN-REPO so a checkout is self-contained and nothing is written outside it -# (.souffle-cache/ is gitignored). Content-addressed, so branches sharing rule text share the -# binary; a fresh clone rebuilds once. Point AXIOM_SOUFFLE_CACHE at a shared machine-scoped -# dir to amortise that across clones. +# a shared, machine-scoped cache keyed by the engine id — NOT in the per-run intermediate +# (which the pipeline/parser wipes). Default IN-REPO so a checkout is self-contained +# (.souffle-cache/ is gitignored); point AXIOM_SOUFFLE_CACHE at a shared dir to amortise it. CACHE_DIR="$CACHE_ROOT" -NEW="$(cat "$PROG" "$DL/decls_base.dl" "$DL/decls_all.dl" "$ENG"/*/*.dl "$ENG"/*/*/*.dl "$ENG2"/*/*.dl "$ENG2"/*/*/*.dl 2>/dev/null | shasum | cut -d' ' -f1)" -BIN="$CACHE_DIR/souffle-engine-$NEW" -if [ ! -x "$BIN" ]; then +EXE=""; case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) EXE=".exe";; esac +BIN="$CACHE_DIR/souffle-engine-$LANG_ARG-$ENGINE_ID$EXE" + +# The platform string CI names its assets by: -. macOS is one universal binary. +engine_platform(){ + local os arch + case "$(uname -s)" in + Linux) os=linux;; Darwin) os=darwin;; MINGW*|MSYS*|CYGWIN*) os=windows;; + *) echo "unsupported platform: $(uname -s)" >&2; return 1;; + esac + case "$(uname -m)" in + x86_64|amd64) arch=x86_64;; arm64|aarch64) arch=arm64;; + *) echo "unsupported architecture: $(uname -m)" >&2; return 1;; + esac + [ "$os" = darwin ] && arch=universal + printf '%s-%s\n' "$os" "$arch" +} +# Fetch the CI-built binary for this platform and id into $BIN, verifying its sha256. Uses +# `gh` when present (the repository is private; gh carries the login), else curl with a +# GH_TOKEN / GITHUB_TOKEN. Downloaded to a temp name and renamed atomically, like the local +# compile, so a concurrent or aborted run never leaves a half-written binary in the cache. +fetch_engine(){ + local platform tag asset tmp sum want + platform="$(engine_platform)" || return 1 + tag="engine-$LANG_ARG-$ENGINE_ID"; asset="axiom-engine-$LANG_ARG-$platform$EXE" + tmp="$(mktemp -d)" + echo "▶ no souffle on PATH — fetching prebuilt engine $asset from $ENGINE_REPO@$tag" + if command -v gh >/dev/null 2>&1; then + gh release download "$tag" -R "$ENGINE_REPO" -p "$asset" -p sha256sum.txt -D "$tmp" 2>"$tmp/err" \ + || { echo "❌ gh could not download $asset from release $tag:" >&2; sed 's/^/ /' "$tmp/err" >&2; rm -rf "$tmp"; return 1; } + else + local token="${GH_TOKEN:-${GITHUB_TOKEN:-}}" + [ -n "$token" ] || { echo "❌ no \`gh\` on PATH and no GH_TOKEN/GITHUB_TOKEN set — cannot fetch from the private release" >&2; rm -rf "$tmp"; return 1; } + local api="https://api.github.com/repos/$ENGINE_REPO/releases/tags/$tag" json + json="$(curl -fsSL -H "Authorization: Bearer $token" -H "Accept: application/vnd.github+json" "$api")" \ + || { echo "❌ release $tag not found in $ENGINE_REPO (is this rule set merged and built?)" >&2; rm -rf "$tmp"; return 1; } + for name in "$asset" sha256sum.txt; do + # the asset's API url is the "url" field of the asset object whose "name" matches + local url; url="$(printf '%s' "$json" | tr -d '\n' | sed 's/{/\n{/g' | grep "\"name\": *\"$name\"" | sed -n 's/.*"url": *"\([^"]*\/assets\/[0-9]*\)".*/\1/p' | head -1)" + [ -n "$url" ] || { echo "❌ release $tag has no asset $name" >&2; rm -rf "$tmp"; return 1; } + curl -fsSL -H "Authorization: Bearer $token" -H "Accept: application/octet-stream" -o "$tmp/$name" "$url" \ + || { echo "❌ download of $name failed" >&2; rm -rf "$tmp"; return 1; } + done + fi + want="$(grep " \*\?$asset\$" "$tmp/sha256sum.txt" | cut -d' ' -f1)" + sum="$(sha256_stdin < "$tmp/$asset")" + if [ -z "$want" ] || [ "$sum" != "$want" ]; then + echo "❌ sha256 mismatch for $asset: got $sum, release says '${want:-}' — refusing to run it" >&2 + rm -rf "$tmp"; return 1 + fi + chmod +x "$tmp/$asset"; mv -f "$tmp/$asset" "$BIN"; rm -rf "$tmp" + echo "▶ verified sha256 $sum → $BIN" +} + +if [ -x "$BIN" ]; then + echo "▶ reusing cached binary" +elif command -v souffle >/dev/null 2>&1; then echo "▶ compiling souffle program (cache miss)..." + INNER="$(find_souffle_include)" + # Assert the HEADER, not the directory: `[ -d ]` is the test #216 established cannot tell + # the two install layouts apart, so it would pass a path that then fails at the compiler. + if [ -z "$INNER" ] || [ ! -f "$INNER/souffle/CompiledSouffle.h" ]; then + echo "❌ soufflé is on PATH but its headers are not. Set AXIOM_SOUFFLE_INCLUDE." >&2; exit 1 + fi + have="$(souffle --version 2>/dev/null | sed -n 's/^Version: *\([0-9][0-9.]*\).*/\1/p' | head -1)" + [ "$have" = "$SOUFFLE_VERSION" ] || echo " ! local souffle is $have, the pinned version is $SOUFFLE_VERSION — a locally compiled engine may differ from CI's" # Generate C++. souffle's "No rules/facts defined" warnings (for the intentionally # unstaged lib-body relations — inert paths) aren't silenced by -w, so filter those 3- # line blocks from stderr; on a real failure, dump the full log and fail. c++ -w # silences the deprecation warnings in souffle's own headers. Compile to a .tmp then # atomically rename, so a concurrent/aborted run never leaves a half-written binary. - if ! souffle -g "$INT/souffle-program.cpp" "$PROG" 2> "$INT/.souffle-gen.log"; then + if ! souffle -I "$SRC" -g "$INT/souffle-program.cpp" "$PROG" 2> "$INT/.souffle-gen.log"; then cat "$INT/.souffle-gen.log" >&2; exit 1 fi awk '/No rules\/facts defined/{skip=2;next} skip>0{skip--;next} {print}' "$INT/.souffle-gen.log" >&2 @@ -305,7 +419,14 @@ if [ ! -x "$BIN" ]; then case "$(uname -s)" in CYGWIN*) CXX_PLATFORM="-Wa,-mbig-obj";; esac c++ -std=c++17 -O3 -march=native -w $CXX_PLATFORM -I "$INNER" "$INT/souffle-program.cpp" -o "$BIN.tmp.$$" mv -f "$BIN.tmp.$$" "$BIN" -else echo "▶ reusing cached binary"; fi +else + fetch_engine || { + echo "❌ no engine for $LANG_ARG@$ENGINE_ID on this machine. Either:" >&2 + echo " • install souffle $SOUFFLE_VERSION to compile locally (macOS: brew install souffle; Ubuntu: the .deb from souffle-lang/souffle releases), or" >&2 + echo " • use a rule set CI has built: merge to main, wait for the engine-binaries workflow, then rerun." >&2 + exit 1 + } +fi # --- STAGE↔SOLVE loop: solve → stage the bodies of methods reached so far → re-solve, until # reachable_method stops growing. Soufflé loads facts up front and can't fetch bodies mid- # solve, so the driver feeds them in reachability order. Each round loads the bodies of ALL @@ -368,7 +489,9 @@ while [ "$iter" -lt 50 ]; do PBIN="$INT/souffle-profile-bin" if [ ! -x "$PBIN" ]; then echo "▶ building profiling binary (once per run dir)..." - souffle -g "$INT/profile-program.cpp" -p "$AXIOM_SOUFFLE_PROFILE" "$PROG" \ + command -v souffle >/dev/null 2>&1 || { echo "❌ profiling needs souffle on PATH (it builds a second binary)" >&2; exit 1; } + INNER="${INNER:-$(find_souffle_include)}" + souffle -I "$SRC" -g "$INT/profile-program.cpp" -p "$AXIOM_SOUFFLE_PROFILE" "$PROG" \ 2> "$INT/.souffle-prof-gen.log" || { cat "$INT/.souffle-prof-gen.log" >&2; exit 1; } c++ -std=c++17 -O3 -march=native -w -I "$INNER" \ "$INT/profile-program.cpp" -o "$PBIN" || exit 1 diff --git a/test/java/run-tests.sh b/test/java/run-tests.sh index 975497f84..fb9e34c38 100755 --- a/test/java/run-tests.sh +++ b/test/java/run-tests.sh @@ -105,6 +105,18 @@ if ! bash "$ROOT/test/tools/bundle-test.sh"; then echo "aborting: the bundle stage does not produce the documented output" exit 1 fi +# ── The engine id and the no-souffle fetch path ────────────────────────────── +# A machine without souffle finds its binary by the id the rules hash to, so the id must be +# the same from any path and different for any rule change; and the fetch must verify the +# sha256 and refuse a mismatch. Both run without souffle or network, in seconds. +if ! bash "$ROOT/test/tools/engine-id-test.sh"; then + echo "aborting: the engine id is not a function of the rules alone" + exit 1 +fi +if ! bash "$ROOT/test/tools/engine-fetch-test.sh"; then + echo "aborting: the prebuilt-engine fetch path does not verify what it runs" + exit 1 +fi PARSER="${AXIOM_PARSER:-$ROOT/../Parser/dist/index.js}" WORK="$HERE/.work" BLESS=0; KEEP=0; ORACLE=0; FILTERS=() diff --git a/test/tools/engine-fetch-test.sh b/test/tools/engine-fetch-test.sh new file mode 100755 index 000000000..1f098f7f7 --- /dev/null +++ b/test/tools/engine-fetch-test.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# The no-souffle path of run-souffle.sh: with `souffle` absent it must fetch the prebuilt +# engine for (language, id, platform), VERIFY its sha256 against the release's +# sha256sum.txt, cache it, and run it through to the bundle — and it must refuse a binary +# whose checksum does not match, leaving nothing in the cache. +# +# No network: a fake `gh` on PATH answers `release download` from a directory this test +# fills, and the "engine" it serves is a shell script that writes the export manifest's +# files into -D (which is all the driver and the bundle stage need from it). +# ───────────────────────────────────────────────────────────────────────────── +set -u +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +RUN="$ROOT/src/pipeline/run-souffle.sh" +fail=0; bad(){ echo " ✗ $*"; fail=$((fail+1)); } +[ -x "$ROOT/node_modules/.bin/tsx" ] || { echo "engine-fetch: SKIP (no node_modules/.bin/tsx — run npm install)"; exit 0; } +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT +mkdir -p "$W/bin" "$W/release" "$W/cache" "$W/ir" "$W/int" "$W/out" +# a PATH with everything the driver and the bundler need, and no souffle +for t in bash sh grep awk sed sort cut tr mktemp uname cat rm cp mv ls dirname basename shasum sha256sum stat date mkdir chmod head tail wc find ln printf tee env node git curl; do + p="$(command -v "$t" 2>/dev/null)" && ln -sf "$p" "$W/bin/$t" +done +ln -sf "$(dirname "$(command -v node)")/npx" "$W/bin/npx" 2>/dev/null || true +. "$ROOT/src/pipeline/engine.conf" + +lang=java +id="$(PATH="$W/bin" bash "$RUN" --language $lang --print-engine-id)" +case "$(uname -s)" in Darwin) platform=darwin-universal;; Linux) platform="linux-$(uname -m | sed 's/aarch64/arm64/;s/amd64/x86_64/')";; *) platform="windows-x86_64";; esac +asset="axiom-engine-$lang-$platform" + +# the fake engine: writes every manifest file, empty, into -D +{ + echo '#!/usr/bin/env bash' + echo 'while [ $# -gt 0 ]; do case "$1" in -D) D="$2"; shift 2;; -F) shift 2;; *) shift;; esac; done' + cut -f2 "$ROOT/src/$lang/souffle/export_manifest.tsv" | sed 's|^|: > "$D/|; s|$|"|' +} > "$W/release/$asset"; chmod +x "$W/release/$asset" +( cd "$W/release" && { command -v sha256sum >/dev/null && sha256sum "$asset" || shasum -a 256 "$asset"; } > sha256sum.txt ) + +# the fake gh: `gh release download TAG -R REPO -p A -p B -D DIR` +cat > "$W/bin/gh" <<'GH' +#!/usr/bin/env bash +[ "$1" = release ] && [ "$2" = download ] || { echo "fake gh: unsupported: $*" >&2; exit 1; } +tag="$3"; shift 3; dest=.; pats=() +while [ $# -gt 0 ]; do case "$1" in -R) shift 2;; -p) pats+=("$2"); shift 2;; -D) dest="$2"; shift 2;; *) shift;; esac; done +[ "$tag" = "$FAKE_TAG" ] || { echo "release not found" >&2; exit 1; } +for p in "${pats[@]}"; do cp "$FAKE_RELEASE/$p" "$dest/" || exit 1; done +GH +chmod +x "$W/bin/gh" +export FAKE_TAG="engine-$lang-$id" FAKE_RELEASE="$W/release" + +run(){ PATH="$W/bin" AXIOM_SOUFFLE_CACHE="$W/cache" bash "$RUN" --language $lang --client-ir "$W/ir" --library "" --intermediate "$W/int" --output "$W/out" > "$W/log" 2>&1; } + +# 1. a good release is fetched, verified, cached and run through to the bundle +if run; then + grep -q "verified sha256" "$W/log" || bad "no 'verified sha256' line in the log" + [ -x "$W/cache/souffle-engine-$lang-$id" ] || bad "fetched binary not cached under its id" + [ -f "$W/out/graph.sqlite" ] || [ -f "$W/out/graph/call_edges.csv" ] || bad "the run did not reach the bundle stage" +else + bad "run with a valid release failed:"; tail -8 "$W/log" | sed 's/^/ /' +fi +# 2. cached: a second run must not download again +: > "$W/log"; run && grep -q "reusing cached binary" "$W/log" || bad "second run did not reuse the cached binary" + +# 3. a tampered binary is refused and nothing is cached +rm -rf "$W/cache"/* "$W/out" +printf '\n# tampered\n' >> "$W/release/$asset" +if run; then bad "a binary with a wrong sha256 was accepted"; else + grep -q "sha256 mismatch" "$W/log" || bad "refusal did not name the sha256 mismatch" + [ ! -e "$W/cache/souffle-engine-$lang-$id" ] || bad "a refused binary was left in the cache" +fi +# 4. no release at all: the two ways out are named +export FAKE_TAG="engine-$lang-nothing"; rm -rf "$W/out" +if run; then bad "a run with no release and no souffle succeeded"; else + grep -q "install souffle" "$W/log" && grep -q "merge to main" "$W/log" || bad "the no-release error does not name both ways out" +fi + +if [ "$fail" -eq 0 ]; then echo "engine-fetch: ok (fetch, verify, cache, refuse tampered, explain absence)"; else echo "engine-fetch: $fail failure(s)"; exit 1; fi diff --git a/test/tools/engine-id-test.sh b/test/tools/engine-id-test.sh new file mode 100755 index 000000000..509cad2ec --- /dev/null +++ b/test/tools/engine-id-test.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# The engine id (run-souffle.sh --print-engine-id) is what lets a machine WITHOUT souffle +# find the binary CI built for its rules, so two properties are load-bearing: +# 1. PATH-INDEPENDENT — the same tree at another path gives the same id (CI's checkout is +# never at the user's path); +# 2. RULE-SENSITIVE — one character changed in one rule file changes it, in every language, +# whether the file is a rule, a map, the manifest, or the pinned souffle version. +# Also: the emitted program contains no absolute path, and neither mode needs souffle. +# ───────────────────────────────────────────────────────────────────────────── +set -u +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +RUN="src/pipeline/run-souffle.sh" +fail=0; bad(){ echo " ✗ $*"; fail=$((fail+1)); } +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT + +# a copy of the tree, at a different path, with souffle hidden from PATH +mkdir -p "$W/copy" "$W/bin" +cp -R "$ROOT/src" "$W/copy/src" +for t in bash grep awk sed sort cut tr mktemp uname cat rm cp mv ls dirname basename shasum sha256sum stat; do + p="$(command -v "$t" 2>/dev/null)" && ln -sf "$p" "$W/bin/$t" +done +id_at(){ ( cd "$1" && PATH="$W/bin" bash "$RUN" --language "$2" --print-engine-id ); } + +for lang in java typescript python; do + a="$(id_at "$ROOT" "$lang")"; b="$(id_at "$W/copy" "$lang")" + case "$a" in [0-9a-f]*) ;; *) bad "$lang: id is not a hex digest: '$a'";; esac + [ "$a" = "$b" ] || bad "$lang: id differs between two paths ($a vs $b)" + ( cd "$W/copy" && PATH="$W/bin" bash "$RUN" --language "$lang" --emit-program "$W/$lang.dl" ) + grep -q '^#include "/' "$W/$lang.dl" && bad "$lang: emitted program embeds an absolute include path" + grep -q "^#include \"$lang/souffle/decls_base.dl\"" "$W/$lang.dl" || bad "$lang: emitted program does not include $lang/souffle/decls_base.dl" + grep -q '^\.input ' "$W/$lang.dl" || bad "$lang: emitted program declares no inputs" +done + +# sensitivity: touch one thing at a time in the copy and expect a new id +before="$(id_at "$W/copy" java)" +mutate(){ # $1 = file, $2 = appended text, $3 = label + cp "$W/copy/$1" "$W/orig"; printf '%s\n' "$2" >> "$W/copy/$1" + after="$(id_at "$W/copy" java)" + [ "$after" != "$before" ] || bad "java: id unchanged after $3" + mv "$W/orig" "$W/copy/$1" +} +f="$(cd "$ROOT" && ls src/java/engine/resolution/*.dl | head -1)" +mutate "$f" "// changed" "editing a rule file ($f)" +mutate "src/java/templates/client-ir.map" "zz_extra_relation all-zz" "adding a staged relation" +mutate "src/java/souffle/export_manifest.tsv" "zz_pred zz.csv" "adding an export" +mutate "src/pipeline/engine.conf" 'SOUFFLE_VERSION="9.9"' "bumping the pinned souffle version" +[ "$(id_at "$W/copy" java)" = "$before" ] || bad "java: id did not return to its original value after the mutations were reverted" +# and a change to one language must not move another's id +tsb="$(id_at "$W/copy" typescript)"; printf '// changed\n' >> "$W/copy/$f" +[ "$(id_at "$W/copy" typescript)" = "$tsb" ] || bad "a java rule change moved the typescript id" + +if [ "$fail" -eq 0 ]; then echo "engine-id: ok (path-independent, rule-sensitive, no souffle needed)"; else echo "engine-id: $fail failure(s)"; exit 1; fi From 36f4995d8e09476121c5f91db020f1615afd43df Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Sun, 13 Sep 2026 13:47:08 -0700 Subject: [PATCH 2/9] build: exercise the engine build matrix on pull requests that touch it (no release) --- .github/workflows/engine-binaries.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/engine-binaries.yml b/.github/workflows/engine-binaries.yml index 0eb4b54e3..05cd37a91 100644 --- a/.github/workflows/engine-binaries.yml +++ b/.github/workflows/engine-binaries.yml @@ -24,6 +24,12 @@ on: - 'src/*/templates/**' - 'src/pipeline/**' - '.github/workflows/engine-binaries.yml' + # A pull request that touches the build itself exercises generate + build (no release), so a + # compiler break on any platform is seen before merge rather than after. + pull_request: + paths: + - 'src/pipeline/**' + - '.github/workflows/engine-binaries.yml' workflow_dispatch: inputs: force: @@ -67,7 +73,7 @@ jobs: for lang in $LANGUAGES; do id="$(bash src/pipeline/run-souffle.sh --language "$lang" --print-engine-id)" echo "$lang: $id" - if [ "$FORCE" != "true" ] && gh release view "engine-$lang-$id" >/dev/null 2>&1; then + if [ "$FORCE" != "true" ] && [ "${{ github.event_name }}" != "pull_request" ] && gh release view "engine-$lang-$id" >/dev/null 2>&1; then echo " release engine-$lang-$id exists — skipping"; continue fi printf '%s' "$id" > "out/$lang.id" @@ -178,7 +184,7 @@ jobs: release: needs: [generate, build] - if: needs.generate.outputs.langs != '[]' + if: needs.generate.outputs.langs != '[]' && github.event_name != 'pull_request' runs-on: ubuntu-24.04 strategy: matrix: From 3423f9a563c9b0af172e3a9d099c8f8689e57d3d Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Sun, 13 Sep 2026 13:48:33 -0700 Subject: [PATCH 3/9] =?UTF-8?q?build:=20the=20generated=20C++=20includes?= =?UTF-8?q?=20souffle/CompiledSouffle.h=20=E2=80=94=20ship=20the=20header?= =?UTF-8?q?=20tree=20under=20that=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/engine-binaries.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/engine-binaries.yml b/.github/workflows/engine-binaries.yml index 05cd37a91..93e055a94 100644 --- a/.github/workflows/engine-binaries.yml +++ b/.github/workflows/engine-binaries.yml @@ -97,7 +97,7 @@ jobs: awk '/No rules\/facts defined/{skip=2;next} skip>0{skip--;next} {print}' "out/$lang.gen.log" ls -la "out/$lang.cpp" done - cp -r /usr/include/souffle out/souffle-headers + cp -r /usr/include/souffle out/souffle - uses: actions/upload-artifact@v4 if: steps.plan.outputs.langs != '[]' @@ -132,7 +132,7 @@ jobs: set -e # -static-libstdc++/-static-libgcc: the only non-glibc dependency is folded in, so # the binary runs on any distro with a glibc at least as old as this runner's. - c++ -std=c++17 -O3 -w -static-libstdc++ -static-libgcc -I gen/souffle-headers \ + c++ -std=c++17 -O3 -w -static-libstdc++ -static-libgcc -I gen \ "gen/${{ matrix.lang }}.cpp" -o "axiom-engine-${{ matrix.lang }}-${{ matrix.target.platform }}" ls -la axiom-engine-*; ldd axiom-engine-* || true @@ -140,7 +140,7 @@ jobs: if: startsWith(matrix.target.platform, 'darwin') run: | set -e - c++ -std=c++17 -O3 -w -arch arm64 -arch x86_64 -mmacosx-version-min=12.0 -I gen/souffle-headers \ + c++ -std=c++17 -O3 -w -arch arm64 -arch x86_64 -mmacosx-version-min=12.0 -I gen \ "gen/${{ matrix.lang }}.cpp" -o "axiom-engine-${{ matrix.lang }}-${{ matrix.target.platform }}" ls -la axiom-engine-*; file axiom-engine-*; otool -L axiom-engine-* @@ -153,7 +153,7 @@ jobs: if: startsWith(matrix.target.platform, 'windows') shell: cmd run: | - cl /nologo /std:c++17 /O2 /EHsc /bigobj /w /permissive- /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS /DNOMINMAX /I gen\souffle-headers gen\${{ matrix.lang }}.cpp /Fe:axiom-engine-${{ matrix.lang }}-${{ matrix.target.platform }}.exe + cl /nologo /std:c++17 /O2 /EHsc /bigobj /w /permissive- /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS /DNOMINMAX /I gen gen\${{ matrix.lang }}.cpp /Fe:axiom-engine-${{ matrix.lang }}-${{ matrix.target.platform }}.exe if errorlevel 1 exit /b 1 dir axiom-engine-* From 35025fc53ae201b676bea19c462d463001c2f0ec Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Sun, 13 Sep 2026 13:59:15 -0700 Subject: [PATCH 4/9] =?UTF-8?q?build:=20Windows=20uses=20Souffl=C3=A9's=20?= =?UTF-8?q?bundled=20getopt=5Flong=20(USE=5FCUSTOM=5FGETOPTLONG)=20instead?= =?UTF-8?q?=20of=20a=20vcpkg=20getopt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/engine-binaries.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/engine-binaries.yml b/.github/workflows/engine-binaries.yml index 93e055a94..449d5502b 100644 --- a/.github/workflows/engine-binaries.yml +++ b/.github/workflows/engine-binaries.yml @@ -153,7 +153,7 @@ jobs: if: startsWith(matrix.target.platform, 'windows') shell: cmd run: | - cl /nologo /std:c++17 /O2 /EHsc /bigobj /w /permissive- /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS /DNOMINMAX /I gen gen\${{ matrix.lang }}.cpp /Fe:axiom-engine-${{ matrix.lang }}-${{ matrix.target.platform }}.exe + cl /nologo /std:c++17 /O2 /EHsc /bigobj /w /permissive- /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS /DNOMINMAX /DUSE_CUSTOM_GETOPTLONG /I gen gen\${{ matrix.lang }}.cpp /Fe:axiom-engine-${{ matrix.lang }}-${{ matrix.target.platform }}.exe if errorlevel 1 exit /b 1 dir axiom-engine-* From fbff769400db87cab205988485a5912e9510d68c Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Sun, 13 Sep 2026 14:22:47 -0700 Subject: [PATCH 5/9] build: run on every merge to main, rebuild only the languages whose id changed, release each language independently --- .github/workflows/engine-binaries.yml | 49 ++++++++++++++++----------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/.github/workflows/engine-binaries.yml b/.github/workflows/engine-binaries.yml index 449d5502b..64f13bc87 100644 --- a/.github/workflows/engine-binaries.yml +++ b/.github/workflows/engine-binaries.yml @@ -1,8 +1,12 @@ # ───────────────────────────────────────────────────────────────────────────── # Prebuilt engine binaries — so running the engine needs no Soufflé and no compiler. # -# On every merge to main that can change a rule set, for each language whose engine id -# (src/pipeline/run-souffle.sh --print-engine-id) has no release yet: +# EVERY merge to main runs this. Each language's engine id (src/pipeline/run-souffle.sh +# --print-engine-id) is a hash of that language's rules alone, so the plan step decides per +# language: an id that already has a release is unchanged and skipped; an id without one is +# built on every platform. A Python-only merge therefore rebuilds Python on all four targets +# and nothing else; a merge touching no rule set builds nothing and finishes in seconds. For +# each language that needs building: # generate Ubuntu + the pinned Soufflé .deb: emit the program, `souffle -g` it to portable # C++ once, and capture the header tree the C++ compiles against. # build compile that C++ on every platform with the SAME flags the script uses locally @@ -19,11 +23,6 @@ name: engine-binaries on: push: branches: [main] - paths: - - 'src/**/*.dl' - - 'src/*/templates/**' - - 'src/pipeline/**' - - '.github/workflows/engine-binaries.yml' # A pull request that touches the build itself exercises generate + build (no release), so a # compiler break on any platform is seen before merge rather than after. pull_request: @@ -51,16 +50,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install the pinned Soufflé - run: | - . src/pipeline/engine.conf - deb="x86_64-ubuntu-2404-souffle-${SOUFFLE_VERSION}-Linux.deb" - curl -fsSL -o "/tmp/$deb" "https://github.com/souffle-lang/souffle/releases/download/${SOUFFLE_VERSION}/$deb" - sudo apt-get update -qq - sudo apt-get install -y -qq "/tmp/$deb" - souffle --version | head -2 - test -f /usr/include/souffle/CompiledSouffle.h - - name: Plan — which languages need a build id: plan env: @@ -74,8 +63,9 @@ jobs: id="$(bash src/pipeline/run-souffle.sh --language "$lang" --print-engine-id)" echo "$lang: $id" if [ "$FORCE" != "true" ] && [ "${{ github.event_name }}" != "pull_request" ] && gh release view "engine-$lang-$id" >/dev/null 2>&1; then - echo " release engine-$lang-$id exists — skipping"; continue + echo " unchanged — release engine-$lang-$id exists"; continue fi + echo " changed — no release for this id; building on every platform" printf '%s' "$id" > "out/$lang.id" langs="$langs \"$lang\"" done @@ -84,6 +74,18 @@ jobs: echo "langs=$langs" >> "$GITHUB_OUTPUT" echo "building: $langs" + + - name: Install the pinned Soufflé + if: steps.plan.outputs.langs != '[]' + run: | + . src/pipeline/engine.conf + deb="x86_64-ubuntu-2404-souffle-${SOUFFLE_VERSION}-Linux.deb" + curl -fsSL -o "/tmp/$deb" "https://github.com/souffle-lang/souffle/releases/download/${SOUFFLE_VERSION}/$deb" + sudo apt-get update -qq + sudo apt-get install -y -qq "/tmp/$deb" + souffle --version | head -2 + test -f /usr/include/souffle/CompiledSouffle.h + - name: Generate portable C++ for each language if: steps.plan.outputs.langs != '[]' run: | @@ -182,9 +184,13 @@ jobs: path: axiom-engine-* if-no-files-found: error + # Per language, and independent of the other languages' builds: `always()` lets this run + # when some build job failed, and the step below publishes a language only if ALL FOUR of + # its binaries exist. A language with a missing platform gets no release — so the next + # merge retries it — while the others publish. release: needs: [generate, build] - if: needs.generate.outputs.langs != '[]' && github.event_name != 'pull_request' + if: always() && needs.generate.result == 'success' && needs.generate.outputs.langs != '[]' && github.event_name != 'pull_request' runs-on: ubuntu-24.04 strategy: matrix: @@ -209,6 +215,11 @@ jobs: lang="${{ matrix.lang }}" id="$(cat "gen/$lang.id")" tag="engine-$lang-$id" + n="$(ls bins/axiom-engine-$lang-* 2>/dev/null | grep -vc '\.sha256$')" + if [ "$n" != 4 ]; then + echo "::warning::$lang: only $n of 4 platform binaries were built — not publishing $tag; the next merge to main retries" + exit 0 + fi cat bins/*.sha256 | sed 's# .*/# #' > bins/sha256sum.txt rm -f bins/*.sha256 cp "gen/$lang.cpp" "bins/$lang.cpp" From 197d594172516b55df732ad78cb14cfdb27ca0ed Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Sun, 13 Sep 2026 14:35:05 -0700 Subject: [PATCH 6/9] =?UTF-8?q?build:=20commit=20the=20rebuilt=20engines?= =?UTF-8?q?=20to=20main=20under=20engine/binaries///=20?= =?UTF-8?q?=E2=80=94=20a=20checkout=20carries=20its=20own=20engines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The publish job places all four binaries of each rebuilt language under engine/binaries///, records the rule-set id in ENGINE_ID, and pushes one commit to main (rebased and retried if main moved during the build); a language missing a platform is left unchanged so the next merge retries it. The plan step now compares each language's id against the committed ENGINE_ID instead of a release. run-souffle.sh resolves the engine in order: the committed binary when its ENGINE_ID equals the checkout's id, a local compile when souffle is on PATH, then the release. The release stays as a secondary copy. engine-fetch-test.sh covers the committed path: used when the id matches, reported and skipped when it does not. --- .gitattributes | 1 + .github/workflows/engine-binaries.yml | 125 +++++++++++++++++--------- README.md | 22 +++-- src/pipeline/run-souffle.sh | 35 ++++++-- test/tools/engine-fetch-test.sh | 33 ++++++- 5 files changed, 158 insertions(+), 58 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..e7321ab22 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +engine/binaries/** binary -text diff --git a/.github/workflows/engine-binaries.yml b/.github/workflows/engine-binaries.yml index 64f13bc87..ccb93c675 100644 --- a/.github/workflows/engine-binaries.yml +++ b/.github/workflows/engine-binaries.yml @@ -3,20 +3,23 @@ # # EVERY merge to main runs this. Each language's engine id (src/pipeline/run-souffle.sh # --print-engine-id) is a hash of that language's rules alone, so the plan step decides per -# language: an id that already has a release is unchanged and skipped; an id without one is -# built on every platform. A Python-only merge therefore rebuilds Python on all four targets -# and nothing else; a merge touching no rule set builds nothing and finishes in seconds. For -# each language that needs building: +# language: an id equal to the one recorded in engine/binaries//ENGINE_ID is unchanged +# and skipped; any other id is built on every platform. A Python-only merge therefore rebuilds +# Python on all four targets and nothing else; a merge touching no rule set builds nothing and +# finishes in seconds. For each language that needs building: # generate Ubuntu + the pinned Soufflé .deb: emit the program, `souffle -g` it to portable # C++ once, and capture the header tree the C++ compiles against. # build compile that C++ on every platform with the SAME flags the script uses locally # (-std=c++17 -O3, no OpenMP/zlib/sqlite defines — the binary links against nothing # but the C++ runtime). Portable target: no -march=native; macOS universal. -# release one GitHub release per (language, id), tagged engine--, holding -# axiom-engine---[.exe], sha256sum.txt, and the generated .cpp. -# run-souffle.sh computes the same id on the user's machine and fetches the matching asset. -# The id is a function of the repository alone (see engine_id in the script), which is what -# makes "no release for this id" mean exactly "these rules have not been merged and built". +# publish COMMIT the binaries to main under engine/binaries/// with the id +# they were built from in engine/binaries//ENGINE_ID — so a checkout carries +# its own engines and nothing is downloaded — and, as a secondary copy, a GitHub +# release tagged engine-- with the same files plus the generated .cpp. +# run-souffle.sh computes the same id on the user's machine and uses the committed binary only +# when the ids agree. The id is a function of the repository alone (see engine_id in the +# script), which is what makes "id differs" mean exactly "these rules were edited after the +# binary was built". # ───────────────────────────────────────────────────────────────────────────── name: engine-binaries @@ -62,10 +65,11 @@ jobs: for lang in $LANGUAGES; do id="$(bash src/pipeline/run-souffle.sh --language "$lang" --print-engine-id)" echo "$lang: $id" - if [ "$FORCE" != "true" ] && [ "${{ github.event_name }}" != "pull_request" ] && gh release view "engine-$lang-$id" >/dev/null 2>&1; then - echo " unchanged — release engine-$lang-$id exists"; continue + have="$(tr -d '[:space:]' < "engine/binaries/$lang/ENGINE_ID" 2>/dev/null || true)" + if [ "$FORCE" != "true" ] && [ "${{ github.event_name }}" != "pull_request" ] && [ "$have" = "$id" ]; then + echo " unchanged — engine/binaries/$lang is at this id"; continue fi - echo " changed — no release for this id; building on every platform" + echo " changed — engine/binaries/$lang is at '${have:-}'; building on every platform" printf '%s' "$id" > "out/$lang.id" langs="$langs \"$lang\"" done @@ -184,48 +188,89 @@ jobs: path: axiom-engine-* if-no-files-found: error - # Per language, and independent of the other languages' builds: `always()` lets this run - # when some build job failed, and the step below publishes a language only if ALL FOUR of - # its binaries exist. A language with a missing platform gets no release — so the next - # merge retries it — while the others publish. - release: + # One job for every language built: `always()` lets it run when some build failed, and a + # language is published only if ALL FOUR of its binaries exist — a language with a missing + # platform is left as it was (the next merge retries it, since its ENGINE_ID still differs) + # while the others go in. One commit on main carries everything that was rebuilt. + publish: needs: [generate, build] if: always() && needs.generate.result == 'success' && needs.generate.outputs.langs != '[]' && github.event_name != 'pull_request' runs-on: ubuntu-24.04 - strategy: - matrix: - lang: ${{ fromJson(needs.generate.outputs.langs) }} steps: + - uses: actions/checkout@v4 + with: + ref: main - uses: actions/download-artifact@v4 with: name: generated path: gen - uses: actions/download-artifact@v4 with: - pattern: engine-${{ matrix.lang }}-* + pattern: engine-* path: bins merge-multiple: true - - name: Publish release engine-${{ matrix.lang }}- + - name: Place the binaries under engine/binaries/// + id: place + run: | + set -e + echo "artifacts:"; ls -la bins + published="" + for lang in $LANGUAGES; do + [ -f "gen/$lang.id" ] || continue + id="$(cat "gen/$lang.id")" + n="$(ls bins/axiom-engine-$lang-* 2>/dev/null | grep -vc '\.sha256$' || true)" + if [ "$n" != 4 ]; then + echo "::warning::$lang: only $n of 4 platform binaries were built — engine/binaries/$lang left unchanged; the next merge to main retries" + continue + fi + rm -rf "engine/binaries/$lang" + for f in bins/axiom-engine-$lang-*; do + case "$f" in *.sha256) continue;; esac + name="$(basename "$f")" + platform="${name#axiom-engine-$lang-}"; platform="${platform%.exe}" + mkdir -p "engine/binaries/$lang/$platform" + cp "$f" "engine/binaries/$lang/$platform/$name" + chmod +x "engine/binaries/$lang/$platform/$name" + done + cat bins/axiom-engine-$lang-*.sha256 | sed 's# .*/# #' > "engine/binaries/$lang/sha256sum.txt" + printf '%s\n' "$id" > "engine/binaries/$lang/ENGINE_ID" + printf 'Built by the engine-binaries workflow from %s.\nRule-set id: %s (src/pipeline/run-souffle.sh --language %s --print-engine-id).\nrun-souffle.sh uses these only while the rules hash to that id.\n' "${{ github.sha }}" "$id" "$lang" > "engine/binaries/$lang/BUILD.txt" + published="$published $lang" + done + echo "published=${published# }" >> "$GITHUB_OUTPUT" + git status --short engine/binaries | head -20 + + - name: Commit to main + if: steps.place.outputs.published != '' + run: | + set -e + git config user.name "engine-binaries" + git config user.email "engine-binaries@users.noreply.github.com" + git add -A engine/binaries + git commit -m "engine: rebuilt binaries for ${{ steps.place.outputs.published }} from ${{ github.sha }}" \ + -m "Built on linux-x86_64, linux-arm64, darwin-universal and windows-x86_64 by the engine-binaries workflow. Each language's ENGINE_ID is the id of the rules it was compiled from; run-souffle.sh uses the binary only while the checkout's rules hash to that id." + # main may have moved during the build; rebase our one commit on top and retry. + for attempt in 1 2 3 4 5; do + git push origin HEAD:main && exit 0 + echo "push rejected (attempt $attempt) — rebasing on the current main" + git fetch origin main && git rebase origin/main + done + echo "::error::could not push the binaries to main after 5 attempts"; exit 1 + + - name: Secondary copy — a release per language + if: steps.place.outputs.published != '' env: GH_TOKEN: ${{ github.token }} - FORCE: ${{ inputs.force }} run: | set -e - lang="${{ matrix.lang }}" - id="$(cat "gen/$lang.id")" - tag="engine-$lang-$id" - n="$(ls bins/axiom-engine-$lang-* 2>/dev/null | grep -vc '\.sha256$')" - if [ "$n" != 4 ]; then - echo "::warning::$lang: only $n of 4 platform binaries were built — not publishing $tag; the next merge to main retries" - exit 0 - fi - cat bins/*.sha256 | sed 's# .*/# #' > bins/sha256sum.txt - rm -f bins/*.sha256 - cp "gen/$lang.cpp" "bins/$lang.cpp" - echo "assets:"; ls -la bins - notes="Prebuilt engine for the **$lang** rule set at id \`$id\` (Soufflé $(cat gen/souffle.version), pinned in src/pipeline/engine.conf), built from ${{ github.sha }}. - - \`run-souffle.sh --language $lang\` fetches the asset for its platform when \`souffle\` is not installed. \`$lang.cpp\` is the generated program for anyone who prefers to compile it themselves (needs only the Soufflé headers and a C++17 compiler)." - if [ "$FORCE" = "true" ]; then gh release delete "$tag" -y 2>/dev/null || true; fi - gh release create "$tag" --title "engine · $lang · ${id:0:12}" --notes "$notes" --target "${{ github.sha }}" bins/* + for lang in ${{ steps.place.outputs.published }}; do + id="$(cat "gen/$lang.id")"; tag="engine-$lang-$id" + mkdir -p "rel/$lang" + cp engine/binaries/$lang/*/axiom-engine-* engine/binaries/$lang/sha256sum.txt "rel/$lang/" + cp "gen/$lang.cpp" "rel/$lang/$lang.cpp" + gh release delete "$tag" -y 2>/dev/null || true + gh release create "$tag" --title "engine · $lang · ${id:0:12}" --target main \ + --notes "Prebuilt engine for the **$lang** rule set at id \`$id\` (Soufflé $(cat gen/souffle.version), pinned in src/pipeline/engine.conf), built from ${{ github.sha }}. The same binaries are committed under engine/binaries/$lang/; \`$lang.cpp\` is the generated program for anyone who prefers to compile it (Soufflé headers + a C++17 compiler)." \ + rel/$lang/* + done diff --git a/README.md b/README.md index 56d5a288c..11b70f14a 100644 --- a/README.md +++ b/README.md @@ -127,10 +127,11 @@ receiver is a lambda parameter. Restricted to files of 1,000 lines or more, d1 r ## Run Requirements: Node ≥ 22.5 and a POSIX shell with `awk` (Git Bash on Windows). **No Soufflé and -no C++ compiler**: the rules compile to one self-contained executable, CI builds it for -Linux (x86_64, arm64), macOS (universal) and Windows on every merge to `main`, and the script -fetches the one for your platform on first use — verified by sha256, cached under the rule -set's id. With `souffle` installed the script compiles locally instead, exactly as before. +no C++ compiler**: the rules compile to one self-contained executable, and CI builds it for +Linux (x86_64, arm64), macOS (universal) and Windows on every merge to `main` and commits it +to `engine/binaries///` — so a checkout already contains the engine for every +platform and nothing is downloaded. With `souffle` installed the script compiles locally +instead, exactly as before. ```bash npm install && npm run build @@ -146,11 +147,14 @@ bash src/pipeline/run-souffle.sh \ --intermediate --output ``` -The prebuilt binaries live in GitHub releases tagged `engine--`, where `` is -`bash src/pipeline/run-souffle.sh --language --print-engine-id` — a sha256 of the rules -and the pinned Soufflé version (`src/pipeline/engine.conf`), the same from any checkout. The -repository is private, so the fetch authenticates through `gh` or a `GH_TOKEN`. A rule set that -has not been merged has no release: edit rules with Soufflé installed, or merge first. +`engine/binaries//ENGINE_ID` records the rule-set id the committed binaries were built +from — `bash src/pipeline/run-souffle.sh --language --print-engine-id`, a sha256 of the +rules and the pinned Soufflé version (`src/pipeline/engine.conf`), the same from any checkout. +The script uses a committed binary only while the checkout's rules hash to that id; rules you +have edited locally need Soufflé installed to compile, or a merge to `main` so CI rebuilds and +commits them. The same binaries are also published as a GitHub release tagged +`engine--`, which the script falls back to fetching (sha256-verified) when neither +a committed nor a locally compiled engine exists. **Outputs — the same in every language** ([`src/bundle/SCHEMA.md`](src/bundle/SCHEMA.md)) diff --git a/src/pipeline/run-souffle.sh b/src/pipeline/run-souffle.sh index 9bd6727ae..627f25727 100755 --- a/src/pipeline/run-souffle.sh +++ b/src/pipeline/run-souffle.sh @@ -8,9 +8,13 @@ # # NO SOUFFLÉ NEEDED TO RUN. The rules compile to one self-contained executable that is # project-independent; CI builds it for every platform on merge (engine-binaries.yml) and -# publishes it under a release tagged engine--. When `souffle` is not on PATH this -# script fetches that binary for the local platform (once, into the cache) and verifies its -# sha256. With souffle installed it compiles locally as before. See src/pipeline/engine.conf. +# COMMITS it under engine/binaries/// next to the id of the rules it was +# built from, so a checkout carries its own engines. The binary is resolved in this order: +# 1. engine/binaries/// — used only if its ENGINE_ID equals the id of the +# rules in this checkout (edited rules never silently run a stale binary); +# 2. a locally compiled engine, when `souffle` is on PATH (the cache under .souffle-cache); +# 3. the GitHub release tagged engine-- (sha256-verified), when it exists. +# See src/pipeline/engine.conf. # # OUTPUT LAYOUT — the same in every language (src/bundle/SCHEMA.md): # $OUT/graph.sqlite the contract: core tables + ext_* tables + the schema catalog @@ -390,7 +394,26 @@ fetch_engine(){ echo "▶ verified sha256 $sum → $BIN" } -if [ -x "$BIN" ]; then +# 1. the binary committed with the repository, if it was built from exactly these rules +COMMITTED_DIR="$SRC/../engine/binaries/$LANG_ARG" +COMMITTED="" +if [ -f "$COMMITTED_DIR/ENGINE_ID" ]; then + if [ "$(tr -d '[:space:]' < "$COMMITTED_DIR/ENGINE_ID")" = "$ENGINE_ID" ]; then + platform="$(engine_platform 2>/dev/null || true)" + cand="$COMMITTED_DIR/$platform/axiom-engine-$LANG_ARG-$platform$EXE" + if [ -n "$platform" ] && [ -f "$cand" ]; then + COMMITTED="$cand"; chmod +x "$COMMITTED" 2>/dev/null || true + else + echo " ! engine/binaries/$LANG_ARG matches these rules but has no binary for ${platform:-this platform}" + fi + else + echo " ! engine/binaries/$LANG_ARG was built from $(cut -c1-12 "$COMMITTED_DIR/ENGINE_ID")…, these rules are ${ENGINE_ID:0:12}… — not using it" + fi +fi + +if [ -n "$COMMITTED" ]; then + BIN="$COMMITTED"; echo "▶ using committed engine ${COMMITTED#"$SRC/../"}" +elif [ -x "$BIN" ]; then echo "▶ reusing cached binary" elif command -v souffle >/dev/null 2>&1; then echo "▶ compiling souffle program (cache miss)..." @@ -421,9 +444,9 @@ elif command -v souffle >/dev/null 2>&1; then mv -f "$BIN.tmp.$$" "$BIN" else fetch_engine || { - echo "❌ no engine for $LANG_ARG@$ENGINE_ID on this machine. Either:" >&2 + echo "❌ no engine for $LANG_ARG@${ENGINE_ID:0:12}… on this machine. Either:" >&2 echo " • install souffle $SOUFFLE_VERSION to compile locally (macOS: brew install souffle; Ubuntu: the .deb from souffle-lang/souffle releases), or" >&2 - echo " • use a rule set CI has built: merge to main, wait for the engine-binaries workflow, then rerun." >&2 + echo " • use a rule set CI has built: merge to main, wait for the engine-binaries workflow to commit engine/binaries/$LANG_ARG, then pull and rerun." >&2 exit 1 } fi diff --git a/test/tools/engine-fetch-test.sh b/test/tools/engine-fetch-test.sh index 1f098f7f7..a4f9c7a20 100755 --- a/test/tools/engine-fetch-test.sh +++ b/test/tools/engine-fetch-test.sh @@ -11,11 +11,15 @@ # ───────────────────────────────────────────────────────────────────────────── set -u ROOT="$(cd "$(dirname "$0")/../.." && pwd)" -RUN="$ROOT/src/pipeline/run-souffle.sh" fail=0; bad(){ echo " ✗ $*"; fail=$((fail+1)); } [ -x "$ROOT/node_modules/.bin/tsx" ] || { echo "engine-fetch: SKIP (no node_modules/.bin/tsx — run npm install)"; exit 0; } W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT -mkdir -p "$W/bin" "$W/release" "$W/cache" "$W/ir" "$W/int" "$W/out" +mkdir -p "$W/bin" "$W/release" "$W/cache" "$W/ir" "$W/int" "$W/out" "$W/tree" +# a copy of the tree, so a committed engine can be planted under engine/binaries/ without +# touching the repository; the bundler's node_modules and tsconfig are shared by link +cp -R "$ROOT/src" "$W/tree/src" +ln -s "$ROOT/node_modules" "$W/tree/node_modules"; ln -s "$ROOT/tsconfig.json" "$W/tree/tsconfig.json"; ln -s "$ROOT/package.json" "$W/tree/package.json" +RUN="$W/tree/src/pipeline/run-souffle.sh" # a PATH with everything the driver and the bundler need, and no souffle for t in bash sh grep awk sed sort cut tr mktemp uname cat rm cp mv ls dirname basename shasum sha256sum stat date mkdir chmod head tail wc find ln printf tee env node git curl; do p="$(command -v "$t" 2>/dev/null)" && ln -sf "$p" "$W/bin/$t" @@ -50,6 +54,29 @@ export FAKE_TAG="engine-$lang-$id" FAKE_RELEASE="$W/release" run(){ PATH="$W/bin" AXIOM_SOUFFLE_CACHE="$W/cache" bash "$RUN" --language $lang --client-ir "$W/ir" --library "" --intermediate "$W/int" --output "$W/out" > "$W/log" 2>&1; } +# 0. a COMMITTED engine whose ENGINE_ID matches the rules is used directly — no download +committed="$W/tree/engine/binaries/$lang" +mkdir -p "$committed/$platform"; cp "$W/release/$asset" "$committed/$platform/$asset"; printf '%s\n' "$id" > "$committed/ENGINE_ID" +export FAKE_TAG="engine-$lang-must-not-be-asked" +if run; then + grep -q "using committed engine" "$W/log" || bad "committed engine with a matching id was not used" + grep -q "fetching prebuilt" "$W/log" && bad "a download was attempted although a matching committed engine exists" + [ -f "$W/out/graph.sqlite" ] || [ -f "$W/out/graph/call_edges.csv" ] || bad "the committed-engine run did not reach the bundle stage" +else + bad "run with a matching committed engine failed:"; tail -8 "$W/log" | sed 's/^/ /' +fi +# 0b. a committed engine built from OTHER rules is ignored, with a message, and the fetch runs +printf 'deadbeef%s\n' "${id:8}" > "$committed/ENGINE_ID"; rm -rf "$W/out" +export FAKE_TAG="engine-$lang-$id" +if run; then + grep -q "not using it" "$W/log" || bad "a stale committed engine was not reported" + grep -q "using committed engine" "$W/log" && bad "a committed engine with a different id was used" + grep -q "verified sha256" "$W/log" || bad "after ignoring the stale committed engine, no fetch happened" +else + bad "run with a stale committed engine and a valid release failed:"; tail -8 "$W/log" | sed 's/^/ /' +fi +rm -rf "$W/tree/engine" "$W/cache"/* "$W/out" + # 1. a good release is fetched, verified, cached and run through to the bundle if run; then grep -q "verified sha256" "$W/log" || bad "no 'verified sha256' line in the log" @@ -74,4 +101,4 @@ if run; then bad "a run with no release and no souffle succeeded"; else grep -q "install souffle" "$W/log" && grep -q "merge to main" "$W/log" || bad "the no-release error does not name both ways out" fi -if [ "$fail" -eq 0 ]; then echo "engine-fetch: ok (fetch, verify, cache, refuse tampered, explain absence)"; else echo "engine-fetch: $fail failure(s)"; exit 1; fi +if [ "$fail" -eq 0 ]; then echo "engine-fetch: ok (committed engine by id, fetch, verify, cache, refuse tampered, explain absence)"; else echo "engine-fetch: $fail failure(s)"; exit 1; fi From 4902b814e8cc87ace176733129fa66a9f9c1e637 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Sun, 13 Sep 2026 18:16:14 -0700 Subject: [PATCH 7/9] build: no pull_request trigger; macOS on a self-hosted arm64 runner; drop the universal binary Hosted macOS minutes bill at 10x and a PR run of the matrix cost ~300 billable minutes; the four-platform compile has been validated, so builds now happen only on merge (and by workflow_dispatch). The macOS job runs on a self-hosted Apple Silicon runner and produces darwin-arm64 only; an Intel row can be added when a user needs it. The platform string on macOS is now darwin- like the others. --- .github/workflows/engine-binaries.yml | 23 ++++++++++++----------- README.md | 2 +- src/pipeline/run-souffle.sh | 3 +-- test/tools/engine-fetch-test.sh | 3 ++- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/engine-binaries.yml b/.github/workflows/engine-binaries.yml index ccb93c675..07fe24a4c 100644 --- a/.github/workflows/engine-binaries.yml +++ b/.github/workflows/engine-binaries.yml @@ -11,7 +11,7 @@ # C++ once, and capture the header tree the C++ compiles against. # build compile that C++ on every platform with the SAME flags the script uses locally # (-std=c++17 -O3, no OpenMP/zlib/sqlite defines — the binary links against nothing -# but the C++ runtime). Portable target: no -march=native; macOS universal. +# but the C++ runtime). Portable target: no -march=native. # publish COMMIT the binaries to main under engine/binaries/// with the id # they were built from in engine/binaries//ENGINE_ID — so a checkout carries # its own engines and nothing is downloaded — and, as a secondary copy, a GitHub @@ -26,12 +26,9 @@ name: engine-binaries on: push: branches: [main] - # A pull request that touches the build itself exercises generate + build (no release), so a - # compiler break on any platform is seen before merge rather than after. - pull_request: - paths: - - 'src/pipeline/**' - - '.github/workflows/engine-binaries.yml' + # No pull_request trigger: macOS minutes bill at 10x and a PR run costs ~300 billable + # minutes. Builds happen on merge; a change to the build itself is validated with + # workflow_dispatch on its branch when that is worth the minutes. workflow_dispatch: inputs: force: @@ -123,7 +120,11 @@ jobs: target: - { os: ubuntu-24.04, platform: linux-x86_64 } - { os: ubuntu-24.04-arm, platform: linux-arm64 } - - { os: macos-14, platform: darwin-universal } + # macOS runs on a SELF-HOSTED Apple Silicon runner (register one under Settings → + # Actions → Runners; it needs the Xcode command-line tools). Hosted macOS minutes + # bill at 10x and the Python build alone took 18 of them. arm64 only: an Intel + # target can be added as a second row when a user needs it. + - { os: [self-hosted, macOS, ARM64], platform: darwin-arm64 } - { os: windows-2025, platform: windows-x86_64 } runs-on: ${{ matrix.target.os }} steps: @@ -142,11 +143,11 @@ jobs: "gen/${{ matrix.lang }}.cpp" -o "axiom-engine-${{ matrix.lang }}-${{ matrix.target.platform }}" ls -la axiom-engine-*; ldd axiom-engine-* || true - - name: Compile (macOS, universal) + - name: Compile (macOS, arm64) if: startsWith(matrix.target.platform, 'darwin') run: | set -e - c++ -std=c++17 -O3 -w -arch arm64 -arch x86_64 -mmacosx-version-min=12.0 -I gen \ + c++ -std=c++17 -O3 -w -arch arm64 -mmacosx-version-min=12.0 -I gen \ "gen/${{ matrix.lang }}.cpp" -o "axiom-engine-${{ matrix.lang }}-${{ matrix.target.platform }}" ls -la axiom-engine-*; file axiom-engine-*; otool -L axiom-engine-* @@ -249,7 +250,7 @@ jobs: git config user.email "engine-binaries@users.noreply.github.com" git add -A engine/binaries git commit -m "engine: rebuilt binaries for ${{ steps.place.outputs.published }} from ${{ github.sha }}" \ - -m "Built on linux-x86_64, linux-arm64, darwin-universal and windows-x86_64 by the engine-binaries workflow. Each language's ENGINE_ID is the id of the rules it was compiled from; run-souffle.sh uses the binary only while the checkout's rules hash to that id." + -m "Built on linux-x86_64, linux-arm64, darwin-arm64 and windows-x86_64 by the engine-binaries workflow. Each language's ENGINE_ID is the id of the rules it was compiled from; run-souffle.sh uses the binary only while the checkout's rules hash to that id." # main may have moved during the build; rebase our one commit on top and retry. for attempt in 1 2 3 4 5; do git push origin HEAD:main && exit 0 diff --git a/README.md b/README.md index 11b70f14a..ea24996ee 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ receiver is a lambda parameter. Restricted to files of 1,000 lines or more, d1 r Requirements: Node ≥ 22.5 and a POSIX shell with `awk` (Git Bash on Windows). **No Soufflé and no C++ compiler**: the rules compile to one self-contained executable, and CI builds it for -Linux (x86_64, arm64), macOS (universal) and Windows on every merge to `main` and commits it +Linux (x86_64, arm64), macOS (arm64) and Windows on every merge to `main` and commits it to `engine/binaries///` — so a checkout already contains the engine for every platform and nothing is downloaded. With `souffle` installed the script compiles locally instead, exactly as before. diff --git a/src/pipeline/run-souffle.sh b/src/pipeline/run-souffle.sh index 627f25727..ce61e686a 100755 --- a/src/pipeline/run-souffle.sh +++ b/src/pipeline/run-souffle.sh @@ -343,7 +343,7 @@ CACHE_DIR="$CACHE_ROOT" EXE=""; case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) EXE=".exe";; esac BIN="$CACHE_DIR/souffle-engine-$LANG_ARG-$ENGINE_ID$EXE" -# The platform string CI names its assets by: -. macOS is one universal binary. +# The platform string CI names its assets by: -. engine_platform(){ local os arch case "$(uname -s)" in @@ -354,7 +354,6 @@ engine_platform(){ x86_64|amd64) arch=x86_64;; arm64|aarch64) arch=arm64;; *) echo "unsupported architecture: $(uname -m)" >&2; return 1;; esac - [ "$os" = darwin ] && arch=universal printf '%s-%s\n' "$os" "$arch" } # Fetch the CI-built binary for this platform and id into $BIN, verifying its sha256. Uses diff --git a/test/tools/engine-fetch-test.sh b/test/tools/engine-fetch-test.sh index a4f9c7a20..f719ec192 100755 --- a/test/tools/engine-fetch-test.sh +++ b/test/tools/engine-fetch-test.sh @@ -29,7 +29,8 @@ ln -sf "$(dirname "$(command -v node)")/npx" "$W/bin/npx" 2>/dev/null || true lang=java id="$(PATH="$W/bin" bash "$RUN" --language $lang --print-engine-id)" -case "$(uname -s)" in Darwin) platform=darwin-universal;; Linux) platform="linux-$(uname -m | sed 's/aarch64/arm64/;s/amd64/x86_64/')";; *) platform="windows-x86_64";; esac +arch="$(uname -m | sed 's/aarch64/arm64/;s/amd64/x86_64/')" +case "$(uname -s)" in Darwin) platform="darwin-$arch";; Linux) platform="linux-$arch";; *) platform="windows-x86_64";; esac asset="axiom-engine-$lang-$platform" # the fake engine: writes every manifest file, empty, into -D From d4af09fba404a6dc1b0f6becd9cdc96341cb6ab9 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Sun, 13 Sep 2026 21:35:18 -0700 Subject: [PATCH 8/9] build: engines ship on npm as @axiomcode/engine--; publish is a dispatch-only workflow (dry run by default) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running the engine needed Soufflé and a C++ toolchain. The compiled engine is one self-contained executable per language and platform, so CI builds them (build-engines.yml: generate the portable C++ once on Ubuntu with the pinned Soufflé, compile on linux-x64, linux-arm64, win32-x64 and a self-hosted darwin-arm64, smoke-run each) and publish-npm.yml assembles one package per platform — @axiomcode/engine--, every language's engine under / with its ENGINE_ID — and publishes them. The workflow runs only on demand (version, dry_run=true by default, macos) or on a v* tag; a dry run packs and prints without uploading. This package lists the four as optionalDependencies, so `npm install` fetches exactly the one npm's os/cpu filter matches; nothing is committed to git. run-souffle.sh resolves the engine in order: the installed package when its ENGINE_ID equals the checkout's rule hash (edited rules never run a stale binary), else a local compile when souffle is present, else an error naming both ways out. The program the id hashes is a pure function of the repository (relative includes, inputs derived from the maps), exposed as --print-engine-id and --emit-program; the pinned Soufflé version lives in graph/pipeline/engine.conf. Guards, without souffle or network, as Java-suite preflights: engine-id-test.sh (same id from two paths; changed by a rule, a map, the manifest, the pin) and engine-package-test.sh (a hand-made engine package: used when its id matches, refused with both ways out when not, absence explained). --- .github/workflows/build-engines.yml | 143 ++++++++++++++ .github/workflows/publish-npm.yml | 76 ++++++++ .gitignore | 2 + README.md | 15 +- bin/axiomcode | 3 +- graph/pipeline/engine.conf | 15 ++ graph/pipeline/portable-stat.sh | 10 + graph/pipeline/run-souffle.sh | 243 +++++++++++++++++------- graph/test/java/run-tests.sh | 13 ++ graph/test/tools/engine-id-test.sh | 53 ++++++ graph/test/tools/engine-package-test.sh | 63 ++++++ package.json | 8 +- packaging/assemble-engine-package.sh | 25 +++ packaging/engine-package.json | 11 ++ 14 files changed, 602 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/build-engines.yml create mode 100644 .github/workflows/publish-npm.yml create mode 100644 graph/pipeline/engine.conf create mode 100755 graph/test/tools/engine-id-test.sh create mode 100755 graph/test/tools/engine-package-test.sh create mode 100755 packaging/assemble-engine-package.sh create mode 100644 packaging/engine-package.json diff --git a/.github/workflows/build-engines.yml b/.github/workflows/build-engines.yml new file mode 100644 index 000000000..7806840ea --- /dev/null +++ b/.github/workflows/build-engines.yml @@ -0,0 +1,143 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Build the engine binaries — every language, every platform — as artifacts. +# +# Reusable (workflow_call) so publish-npm.yml can build then publish in one run, and +# dispatchable on its own to check that the rules still compile everywhere without +# publishing anything. Soufflé is a BUILD-time dependency only: `souffle -g` turns each +# language's rules into portable C++ once on Ubuntu (the pinned .deb), and every platform +# compiles that C++ with its own C++17 compiler against Soufflé's headers. The result is +# one self-contained executable per language per platform, linked against nothing but the +# C++ runtime. Same flags as the local compile (no OpenMP/zlib/sqlite), portable targets. +# +# Artifacts: engines-/ holding /axiomcode-engine-[.exe] + /ENGINE_ID +# Platforms are named the npm way (process.platform-process.arch): darwin-arm64, linux-x64, +# linux-arm64, win32-x64. macOS builds on a SELF-HOSTED Apple Silicon runner (hosted macOS +# minutes bill at 10x); pass macos=false to skip it while no runner is registered. +# ───────────────────────────────────────────────────────────────────────────── +name: build-engines + +on: + workflow_call: + inputs: + macos: + description: build the darwin-arm64 engines on the self-hosted macOS runner + type: boolean + default: true + workflow_dispatch: + inputs: + macos: + description: build the darwin-arm64 engines on the self-hosted macOS runner + type: boolean + default: false + +env: + LANGUAGES: java typescript python + +jobs: + generate: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Install the pinned Soufflé + run: | + . graph/pipeline/engine.conf + deb="x86_64-ubuntu-2404-souffle-${SOUFFLE_VERSION}-Linux.deb" + curl -fsSL -o "/tmp/$deb" "https://github.com/souffle-lang/souffle/releases/download/${SOUFFLE_VERSION}/$deb" + sudo apt-get update -qq && sudo apt-get install -y -qq "/tmp/$deb" + souffle --version | head -2 + test -f /usr/include/souffle/CompiledSouffle.h + - name: Generate portable C++ per language + run: | + set -e + mkdir -p gen + for lang in $LANGUAGES; do + id="$(bash graph/pipeline/run-souffle.sh --language "$lang" --print-engine-id)" + echo "$lang: $id"; printf '%s' "$id" > "gen/$lang.id" + bash graph/pipeline/run-souffle.sh --language "$lang" --emit-program "gen/$lang.dl" + souffle -I graph -g "gen/$lang.cpp" "gen/$lang.dl" 2> "gen/$lang.gen.log" || { cat "gen/$lang.gen.log"; exit 1; } + awk '/No rules\/facts defined/{skip=2;next} skip>0{skip--;next} {print}' "gen/$lang.gen.log" + done + cp -r /usr/include/souffle gen/souffle + - uses: actions/upload-artifact@v4 + with: { name: generated, path: gen, retention-days: 3, if-no-files-found: error } + + build: + needs: generate + strategy: + fail-fast: false + matrix: + target: + - { os: ubuntu-24.04, platform: linux-x64 } + - { os: ubuntu-24.04-arm, platform: linux-arm64 } + - { os: windows-2025, platform: win32-x64 } + runs-on: ${{ matrix.target.os }} + steps: + - uses: actions/download-artifact@v4 + with: { name: generated, path: gen } + - name: Compile every language (Linux) + if: startsWith(matrix.target.platform, 'linux') + run: | + set -e + for lang in $LANGUAGES; do + mkdir -p "engines/$lang" + c++ -std=c++17 -O3 -w -static-libstdc++ -static-libgcc -I gen "gen/$lang.cpp" -o "engines/$lang/axiomcode-engine-$lang" + cp "gen/$lang.id" "engines/$lang/ENGINE_ID" + done + ls -la engines/*; ldd engines/java/axiomcode-engine-java || true + - uses: ilammy/msvc-dev-cmd@v1 + if: startsWith(matrix.target.platform, 'win32') + with: { arch: x64 } + - name: Compile every language (Windows, MSVC) + if: startsWith(matrix.target.platform, 'win32') + shell: cmd + run: | + for %%L in (java typescript python) do ( + mkdir engines\%%L + cl /nologo /std:c++17 /O2 /EHsc /bigobj /w /permissive- /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS /DNOMINMAX /DUSE_CUSTOM_GETOPTLONG /I gen gen\%%L.cpp /Fe:engines\%%L\axiomcode-engine-%%L.exe + if errorlevel 1 exit /b 1 + copy gen\%%L.id engines\%%L\ENGINE_ID + ) + dir /s engines + - name: Smoke — every binary starts on empty inputs + shell: bash + run: | + set -e + for lang in $LANGUAGES; do + bin="$(ls engines/$lang/axiomcode-engine-$lang* )"; chmod +x "$bin" 2>/dev/null || true + mkdir -p "facts-$lang" "out-$lang" + sed -n 's/^\.input \([A-Za-z0-9_]*\)(.*/\1/p' "gen/$lang.dl" | while read -r r; do : > "facts-$lang/$r.facts"; done + "./$bin" -F "facts-$lang" -D "out-$lang" + echo "$lang: ok ($(ls out-$lang | wc -l) relations written)" + done + - uses: actions/upload-artifact@v4 + with: + name: engines-${{ matrix.target.platform }} + path: engines + if-no-files-found: error + + build-macos: + needs: generate + if: inputs.macos + runs-on: [self-hosted, macOS, ARM64] + steps: + - uses: actions/download-artifact@v4 + with: { name: generated, path: gen } + - name: Compile every language (macOS arm64) + run: | + set -e + for lang in $LANGUAGES; do + mkdir -p "engines/$lang" + c++ -std=c++17 -O3 -w -arch arm64 -mmacosx-version-min=12.0 -I gen "gen/$lang.cpp" -o "engines/$lang/axiomcode-engine-$lang" + cp "gen/$lang.id" "engines/$lang/ENGINE_ID" + done + otool -L engines/java/axiomcode-engine-java + - name: Smoke — every binary starts on empty inputs + run: | + set -e + for lang in $LANGUAGES; do + mkdir -p "facts-$lang" "out-$lang" + sed -n 's/^\.input \([A-Za-z0-9_]*\)(.*/\1/p' "gen/$lang.dl" | while read -r r; do : > "facts-$lang/$r.facts"; done + "./engines/$lang/axiomcode-engine-$lang" -F "facts-$lang" -D "out-$lang" + done + - uses: actions/upload-artifact@v4 + with: { name: engines-darwin-arm64, path: engines, if-no-files-found: error } diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml new file mode 100644 index 000000000..a6255048e --- /dev/null +++ b/.github/workflows/publish-npm.yml @@ -0,0 +1,76 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Publish the engine packages to npm: @axiomcode/engine--, one per platform, each +# holding every language's engine for that platform. This package lists them as optional +# dependencies, so `npm install` fetches exactly the one for the machine — no Soufflé, no +# compiler, no download of ours. +# +# Runs ONLY when asked: +# • manually (Actions → publish-npm → Run workflow): version, dry_run (default TRUE — packs +# and prints exactly what would be published, uploads nothing), macos. +# • on a version tag `v1.2.3`: a real publish of that version. +# Needs the NPM_TOKEN secret (an npmjs automation token with publish rights on @axiomcode) +# for a real publish; a dry run needs nothing. +# ───────────────────────────────────────────────────────────────────────────── +name: publish-npm + +on: + workflow_dispatch: + inputs: + version: + description: version to publish (e.g. 0.1.0) + required: true + type: string + dry_run: + description: pack and print, publish nothing + type: boolean + default: true + macos: + description: include darwin-arm64 (needs the self-hosted macOS runner) + type: boolean + default: true + push: + tags: ['v*'] + +permissions: + contents: read + id-token: write + +jobs: + engines: + uses: ./.github/workflows/build-engines.yml + with: + macos: ${{ github.event_name == 'push' || inputs.macos }} + + publish: + needs: engines + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + registry-url: 'https://registry.npmjs.org' + - uses: actions/download-artifact@v4 + with: { pattern: engines-*, path: artifacts } + - name: Assemble one package per platform + run: | + set -e + if [ "${{ github.event_name }}" = push ]; then version="${GITHUB_REF_NAME#v}"; else version="${{ inputs.version }}"; fi + echo "version=$version" >> "$GITHUB_ENV" + for d in artifacts/engines-*; do + platform="${d#artifacts/engines-}" + bash packaging/assemble-engine-package.sh "$platform" "$version" "$d" "packages/engine-$platform" + cat "packages/engine-$platform/package.json" + done + - name: Publish (or dry-run) + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + DRY: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} + run: | + set -e + flag=""; [ "$DRY" = true ] && flag="--dry-run" + for p in packages/engine-*; do + echo "══ $p $flag" + ( cd "$p" && npm publish --access public $flag ) + done + [ "$DRY" = true ] && echo "DRY RUN — nothing was uploaded. Re-run with dry_run=false, or push tag v$version, to publish." || echo "published version $version" diff --git a/.gitignore b/.gitignore index 2f9db48a7..478ff7342 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ # ── Dependencies ───────────────────────────────────────────────────────────── node_modules/ +# and the bare name: `node_modules/` matches a directory only, so a symlink named node_modules slips past it +node_modules jspm_packages/ web_modules/ .pnp diff --git a/README.md b/README.md index a9c1b476b..285e4cf65 100644 --- a/README.md +++ b/README.md @@ -144,11 +144,14 @@ bin/axiomcode test [java|typescript|python|parser|all] `bin/axiomcode` is the whole pipeline as subcommands: the parser (`parser/`) extracts a relational IR from the source — every language it finds, in one pass — the engine (`graph/`) solves each language separately, and `//graph.sqlite` is the result (graphs are per language; a Java→TypeScript -call is not an edge in either). **No -Soufflé and no C++ compiler**: the rules compile to one self-contained executable, CI builds it for -Linux (x86_64, arm64), macOS (arm64) and Windows on every merge to `main` and commits it under -`binaries///`, so a checkout carries the engine for every platform. With `souffle` -installed the engine compiles locally instead. +call is not an edge in either). **No Soufflé and no C++ compiler:** the rules compile to one +self-contained executable per language and platform, CI publishes those to npm as +`@axiomcode/engine--`, and this package lists them as optional dependencies — so `npm install` +fetches exactly the one for your machine (Linux x64/arm64, macOS arm64, Windows x64). With `souffle` +installed the engine compiles locally instead; rules edited after the last published engines fall back +to that automatically (the script compares the package's `ENGINE_ID` to the checkout's rules and never +runs a stale binary). Publishing: `Actions → publish-npm → Run workflow` (dry run by default), or push a +`v*` tag. **Outputs — the same in every language** ([`graph/bundle/SCHEMA.md`](graph/bundle/SCHEMA.md)) @@ -214,7 +217,7 @@ graph/ the engine pipeline/run-souffle.sh fact staging, engine resolution (committed / compiled / fetched), stage↔solve loop, then the bundle stage bundle/ the output contract: schema as data (SCHEMA.md), per-language adapters, writers test// the engine's regression suites, torture harnesses, oracles (graph/test/tools: platform preflights) -binaries/// CI-built engines, committed on merge (ENGINE_ID = the rules they were built from) +packaging/ template + assembler for the @axiomcode/engine-- packages (publish-npm.yml) ``` ## Tests diff --git a/bin/axiomcode b/bin/axiomcode index 622f28eb7..3ea3c5dde 100755 --- a/bin/axiomcode +++ b/bin/axiomcode @@ -26,7 +26,8 @@ # bin/axiomcode engine --language L --client-ir / --out [options] # bin/axiomcode test [java|typescript|python|parser|all] [suite options] # -# Requires Node ≥ 22.5; no Soufflé or compiler (binaries/, or a local souffle if present). +# Requires Node ≥ 22.5; no Soufflé or compiler (the engine comes from npm as @axiomcode/engine--, +# or is compiled locally when souffle is present). # ───────────────────────────────────────────────────────────────────────────── set -eu ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" diff --git a/graph/pipeline/engine.conf b/graph/pipeline/engine.conf new file mode 100644 index 000000000..0f0158c62 --- /dev/null +++ b/graph/pipeline/engine.conf @@ -0,0 +1,15 @@ +# ───────────────────────────────────────────────────────────────────────────── +# The prebuilt-engine contract. Sourced by run-souffle.sh and by the CI workflows. +# +# SOUFFLE_VERSION is PINNED here, not read from a `souffle --version`, because the machine +# that runs a prebuilt binary has no souffle to ask — and the engine id it computes must be +# the id CI computed. Bumping it changes every language's id, which is what a new code +# generator should do. CI installs exactly this version. +# +# ENGINE_PACKAGE_SCOPE is the npm scope the engine packages are published under: +# /engine-- (darwin-arm64, linux-x64, linux-arm64, win32-x64), each holding +# every language's engine for that platform under / with its ENGINE_ID. This package +# lists them as optionalDependencies, so `npm install` fetches the one for the machine. +# ───────────────────────────────────────────────────────────────────────────── +SOUFFLE_VERSION="2.5" +ENGINE_PACKAGE_SCOPE="@axiomcode" diff --git a/graph/pipeline/portable-stat.sh b/graph/pipeline/portable-stat.sh index 3be93fdf0..2e45972d8 100644 --- a/graph/pipeline/portable-stat.sh +++ b/graph/pipeline/portable-stat.sh @@ -52,3 +52,13 @@ sha1_stdin(){ [ -n "$_SHA1_CMD" ] || { echo "neither shasum nor sha1sum is on PATH" >&2; return 1; } "$_SHA1_CMD" | cut -d' ' -f1 } + +# sha256 of stdin, for the engine id. Same three spellings as sha1 above: `shasum -a 256` +# (macOS, perl shasum in Git Bash) or `sha256sum` (coreutils). +if command -v sha256sum >/dev/null 2>&1; then _SHA256_CMD="sha256sum" +elif command -v shasum >/dev/null 2>&1; then _SHA256_CMD="shasum -a 256" +else _SHA256_CMD=""; fi +sha256_stdin(){ + [ -n "$_SHA256_CMD" ] || { echo "neither sha256sum nor shasum is on PATH" >&2; return 1; } + $_SHA256_CMD | cut -d' ' -f1 +} diff --git a/graph/pipeline/run-souffle.sh b/graph/pipeline/run-souffle.sh index cc75f7c4a..97f0fbf14 100755 --- a/graph/pipeline/run-souffle.sh +++ b/graph/pipeline/run-souffle.sh @@ -3,6 +3,18 @@ # client-ir.map / lib.map (single source of truth). Lib is auto-scoped # to only the signature relations the rules reference (never loads GB-scale bodies). # Usage: run-souffle.sh --client-ir DIR --library DIR --intermediate DIR --output DIR [--language L] [--debug] +# run-souffle.sh --language L --print-engine-id the canonical id of L's compiled engine +# run-souffle.sh --language L --emit-program FILE the Soufflé program CI compiles for L +# +# NO SOUFFLÉ NEEDED TO RUN. The rules compile to one self-contained executable that is +# project-independent; CI builds it for every platform and publishes it on npm as +# @axiomcode/engine-- (publish-npm.yml), which this package lists as an optional +# dependency so `npm install` fetches exactly the one for the machine. The binary is +# resolved in this order: +# 1. node_modules/@axiomcode/engine-// — used only if its ENGINE_ID equals +# the id of the rules in this checkout (edited rules never silently run a stale binary); +# 2. a locally compiled engine, when `souffle` is on PATH (cached under .souffle-cache). +# See graph/pipeline/engine.conf. # # OUTPUT LAYOUT — the same in every language (graph/bundle/SCHEMA.md): # $OUT/graph.sqlite the contract: core tables + ext_* tables + the schema catalog @@ -33,6 +45,8 @@ DISPATCH_CAP="${DISPATCH_CAP:-20}" # fan-width cap on virtual dispatch. DEFAUL LANG_ARG="" # which rule set under graph// to run. Default java. TAINT="" # --taint on → gate lib→lib GROW on client-seeded data flow (dataflow/taint.dl). Also # settable via env AXIOM_TAINT_GATING=on. Empty = ungated (default behavior). +MODE="run" # run | print-engine-id | emit-program — the last two need no IR and no souffle +EMIT="" while [ $# -gt 0 ]; do case "$1" in --client-ir) CLIENT="$2"; shift 2;; --library) LIB="$2"; shift 2;; --intermediate) INT="$2"; shift 2;; --output) OUT="$2"; shift 2;; @@ -41,6 +55,8 @@ while [ $# -gt 0 ]; do case "$1" in --lib-depth) LIB_DEPTH="$2"; shift 2;; --taint) TAINT="$2"; shift 2;; --language) LANG_ARG="$2"; shift 2;; + --print-engine-id) MODE="print-engine-id"; shift;; + --emit-program) MODE="emit-program"; EMIT="$2"; shift 2;; # graph.sqlite is the deliverable; csv/*.csv is a debugging view of the same core # tables. --debug asks for both. (An older Node with no node:sqlite writes the CSVs # regardless, because otherwise the run would produce no consumer-facing output.) @@ -50,6 +66,7 @@ while [ $# -gt 0 ]; do case "$1" in # change the invocation.) *) shift;; esac; done SRC="$(cd "$(dirname "$0")/.." && pwd)" +PKG="$(cd "$SRC/.." && pwd)" # the package root: package.json, node_modules, parser/, graph/ # shellcheck source=portable-stat.sh . "$SRC/pipeline/portable-stat.sh" # Rules are PER-LANGUAGE and live under graph//; the executor itself is shared. @@ -58,14 +75,88 @@ ENG="$SRC/$LANG_ARG/engine"; ENG2="$SRC/$LANG_ARG/engine-ii"; DL="$SRC/$LANG_ARG [ -d "$ENG" ] || { echo "no rule set for --language=$LANG_ARG (looked in $ENG)" >&2; exit 1; } # shellcheck source=souffle-include.sh . "$SRC/pipeline/souffle-include.sh" -INNER="$(find_souffle_include)" -# Assert the HEADER, not the directory: `[ -d ]` is the test #216 established cannot tell the two -# install layouts apart, so it would pass a path that then fails at the compiler. -if [ -z "$INNER" ] || [ ! -f "$INNER/souffle/CompiledSouffle.h" ]; then - echo "❌ soufflé headers not found. Install soufflé, or set AXIOM_SOUFFLE_INCLUDE." >&2 - echo " macOS: brew install souffle Debian/Ubuntu: apt-get install souffle" >&2 - exit 1 -fi +# shellcheck source=engine.conf +. "$SRC/pipeline/engine.conf" +# Staging config is PER-LANGUAGE (IR marker + which relations are signatures vs bodies). +# Keeping it here would hardcode Java's entity set into a shared executor. +[ -f "$TPL/staging.conf" ] || { echo "missing $TPL/staging.conf for --language=$LANG_ARG" >&2; exit 1; } +. "$TPL/staging.conf" +ENGINE_II_MODE="${ENGINE_II:-${AXIOM_ENGINE_II:-off}}" + +# The tools every path below relies on. Checked up front because a missing one does not +# always fail loudly: read_map runs inside a process substitution, where a missing grep +# yields an EMPTY relation list — and a different, wrong engine id — under `set -e`. +for t in grep awk sed sort cut tr mktemp uname dirname cat; do + command -v "$t" >/dev/null 2>&1 || { echo "❌ required tool not on PATH: $t" >&2; exit 1; } +done + +# read an import map (relationcsv-basename per line), skipping comments (#) and blanks +read_map(){ grep -vE '^[[:space:]]*(#|$)' "$1"; } + +# ── THE PROGRAM, as a pure function of the repository ──────────────────────────────────── +# Written so that the SAME text comes out of every checkout and of CI: includes are relative +# to graph/ (souffle resolves them through -I "$SRC"), and the .input list is derived from the +# maps rather than from a listing of the staged facts dir — so it needs no client IR, and a +# machine that cannot stage (CI) still produces the text the binary was built from. The run +# path asserts below that staging created a facts file for every .input it declares. +# The list of input relations is: every client relation, the lib signature relations, the lib +# body relations (filled per iteration), and the four knob facts. +input_relations(){ + while IFS=$'\t' read -r rel csv; do printf '%s\n' "$rel"; done < <(read_map "$TPL/client-ir.map") + while IFS=$'\t' read -r rel csv; do + case " $LIB_SIG " in *" ${rel#lib_} "*) printf '%s\n' "$rel";; esac + done < <(read_map "$TPL/lib.map") + for r in $LIB_BODY; do printf '%s\n' "$r"; done + printf '%s\n' jdk_max_depth lib_max_depth taint_gating dispatch_cap +} +write_program(){ # $1 = destination file + { + echo "#include \"$LANG_ARG/souffle/decls_base.dl\""; echo "#include \"$LANG_ARG/souffle/decls_all.dl\"" + # rfc4180=true: the IR is CSV, not TSV. The parser quotes any field containing a + # quote, tab or newline and doubles the inner quotes, so reading it as plain TSV hands + # the rules the ESCAPED text. Souffle parses RFC4180 itself, so this costs one flag + # rather than a re-encode of GB-scale input. + # LC_ALL=C sort: the order is part of the program text, so it must not depend on locale. + input_relations | LC_ALL=C sort -u | while read -r r; do printf '.input %s(IO=file, filename="%s.facts", delimiter="\\t", rfc4180=true)\n' "$r" "$r"; done + for d in projections containment resolution config-resolution expression-resolution call-edge-generation; do + # [ -f ] guard: a phase directory that is empty (or absent for a language that has + # not implemented that layer yet) leaves the glob unexpanded, and souffle's C + # preprocessor then fails on a literal '*.dl' include. + for f in "$ENG/$d/"*.dl; do [ -f "$f" ] && echo "#include \"${f#"$SRC/"}\""; done + done + # engine-ii: the first→third forward-chain engine (mirrors engine/, lib-seeded). Same solve, + # included AFTER engine/ so it reads engine/'s relations (client_calls_lib seed). Glob its + # phase subfolders (both nesting levels; globs are space-safe, the repo path has spaces). + # export/ is doc-only (like engine/export) — skip it. + if [ "$ENGINE_II_MODE" = "on" ]; then + for f in "$ENG2/"*/*.dl "$ENG2/"*/*/*.dl; do + case "$f" in */export/*) continue;; esac + [ -f "$f" ] && echo "#include \"${f#"$SRC/"}\"" + done + fi + # Relative output filenames — the -D at run time supplies the directory. Keeping $OUT out + # of the program makes the compiled binary independent of the output path (better reuse). + while IFS=$'\t' read -r pred file; do [ -n "$pred" ] && printf '.output %s(IO=file, filename="%s", delimiter="\\t")\n' "$pred" "$file"; done < <(LC_ALL=C sort -u "$DL/export_manifest.tsv") + } > "$1" +} +# The engine id: sha256 over the pinned code-generator version, the program text, and every +# file it includes, in include order. A function of the repository alone — the same from any +# path, on any machine, with or without souffle — and different for any rule change. It names +# the local cache entry AND the engine package CI publishes, which is what lets a machine +# without souffle know which binary is its own. +engine_id(){ + local prog; prog="$(mktemp)"; write_program "$prog" + { printf 'souffle=%s\n' "$SOUFFLE_VERSION"; cat "$prog" + sed -n 's/^#include "\(.*\)"$/\1/p' "$prog" | while read -r inc; do cat "$SRC/$inc"; done + } | sha256_stdin + rm -f "$prog" +} +case "$MODE" in + print-engine-id) engine_id; exit 0;; + emit-program) write_program "$EMIT"; exit 0;; +esac + +[ -n "${CLIENT:-}" ] && [ -n "${INT:-}" ] && [ -n "${OUT:-}" ] || { echo "usage: run-souffle.sh --client-ir DIR --library DIR --intermediate DIR --output DIR [--language L]" >&2; exit 1; } FACTS="$INT/souffle-facts"; rm -rf "$FACTS"; mkdir -p "$FACTS" "$OUT" # raw/ is OWNED: wiped per run so a relation that left the manifest cannot linger from an # earlier run and be mistaken for this one's output. @@ -77,20 +168,12 @@ RAW="$OUT/raw"; rm -rf "$RAW"; mkdir -p "$RAW" CACHE_ROOT="${AXIOM_SOUFFLE_CACHE:-$SRC/../.souffle-cache}"; mkdir -p "$CACHE_ROOT" START_EPOCH=$(date +%s); START_TS=$(date '+%Y-%m-%d %H:%M:%S') -# read an import map (relationcsv-basename per line), skipping comments (#) and blanks -read_map(){ grep -vE '^[[:space:]]*(#|$)' "$1"; } - # Library roots: --library is a comma-separated list of IR roots (each with jdk-style # module sub-folders, or a flat IR dir). The caller (TS) controls which folders/libraries # are loaded; staging concatenates each relation across every module of every root. IFS=',' read -ra LIB_ROOTS <<< "$LIB" # lib_modules ROOT -> the module dirs to stage from (the root itself if it holds the IR, # else its immediate sub-folders — mirrors how the JDK ships sharded modules). -# Staging config is PER-LANGUAGE (IR marker + which relations are signatures vs bodies). -# Keeping it here would hardcode Java's entity set into a shared executor. -[ -f "$TPL/staging.conf" ] || { echo "missing $TPL/staging.conf for --language=$LANG_ARG" >&2; exit 1; } -. "$TPL/staging.conf" - lib_modules(){ if [ -f "$1/$IR_MARKER" ]; then printf '%s\n' "$1"; else for m in "$1"/*/; do [ -d "$m" ] && printf '%s\n' "${m%/}"; done; fi; } # --- CLIENT: stage EVERY mapped relation, empty when the project has no such file --- @@ -235,79 +318,98 @@ echo "▶ dispatch cap = $( [ -s "$FACTS/dispatch_cap.facts" ] && echo "$(cat "$ # (and backed up on ~/Desktop) but excluded from the compiled program; re-enable with # --engine-ii on / AXIOM_ENGINE_II=on. engine/ produces client_calls_lib etc. independently, so # client-only is a complete, valid solve on its own. -ENGINE_II_MODE="${ENGINE_II:-${AXIOM_ENGINE_II:-off}}" echo "▶ engine-ii = $( [ "$ENGINE_II_MODE" = "on" ] && echo 'ON (lib frontier included)' || echo 'OFF (client-only — engine-i)' )" -# --- generate combined program --- +# --- the program, and the binary for it: from npm, or compiled here --- PROG="$INT/souffle-program.dl" -{ - echo "#include \"$DL/decls_base.dl\""; echo "#include \"$DL/decls_all.dl\"" - # rfc4180=true: the IR is CSV, not TSV. The parser quotes any field containing a - # quote, tab or newline and doubles the inner quotes, so reading it as plain TSV hands - # the rules the ESCAPED text. It only bites where a JOINED column contains a quote -- - # which is why it went unnoticed -- but a string forward reference (`-> "Factory"`) - # lands squarely on one, and a field carrying a tab would shift every column after it. - # Souffle parses RFC4180 itself, so this costs one flag rather than a re-encode of - # GB-scale input. - for ff in "$FACTS"/*.facts; do r=$(basename "$ff" .facts); printf '.input %s(IO=file, filename="%s.facts", delimiter="\\t", rfc4180=true)\n' "$r" "$r"; done - for d in projections containment resolution config-resolution expression-resolution call-edge-generation; do - # [ -f ] guard: a phase directory that is empty (or absent for a language that has - # not implemented that layer yet) leaves the glob unexpanded, and souffle's C - # preprocessor then fails on a literal '*.dl' include. - for f in "$ENG/$d/"*.dl; do [ -f "$f" ] && echo "#include \"$f\""; done - done - # engine-ii: the first→third forward-chain engine (mirrors engine/, lib-seeded). Same solve, - # included AFTER engine/ so it reads engine/'s relations (client_calls_lib seed). Glob its - # phase subfolders (both nesting levels; globs are space-safe, the repo path has spaces). - # export/ is doc-only (like engine/export) — skip it. - if [ "$ENGINE_II_MODE" = "on" ]; then - for f in "$ENG2/"*/*.dl "$ENG2/"*/*/*.dl; do - case "$f" in */export/*) continue;; esac - [ -f "$f" ] && echo "#include \"$f\"" - done - fi - # Relative output filenames — the -D at run time supplies the directory. Keeping $OUT out - # of the program makes the compiled binary independent of the output path (better reuse). - while IFS=$'\t' read -r pred file; do [ -n "$pred" ] && printf '.output %s(IO=file, filename="%s", delimiter="\\t")\n' "$pred" "$file"; done < <(sort -u "$DL/export_manifest.tsv") -} > "$PROG" +write_program "$PROG" +# Every declared input must have been staged, or souffle would fail on a missing file after +# the (possibly long) library staging. The program lists inputs from the maps; staging +# created them from the same maps, so a mismatch is a bug in this script, and says so. +for r in $(sed -n 's/^\.input \([A-Za-z0-9_]*\)(.*/\1/p' "$PROG"); do + [ -f "$FACTS/$r.facts" ] || { echo "❌ program declares input $r but staging created no $r.facts" >&2; exit 1; } +done +ENGINE_ID="$(engine_id)" +echo "▶ engine id = $ENGINE_ID (rules + souffle $SOUFFLE_VERSION)" -# --- compile once into a PERSISTENT, content-addressed cache (survives inter/ deletion) --- # What we cache is OUR engine compiled to a native binary (souffle -g turns the .dl rules # into C++, c++ compiles it) — NOT the souffle tool. It depends only on the engine (rules + # decls) and is PROJECT-INDEPENDENT (relative .input/.output), so one binary serves every -# project: N concurrent analyses of N different projects all share it. It therefore lives in -# a shared, machine-scoped cache keyed by a content hash — NOT in the per-run intermediate -# (which the pipeline/parser wipes). The hash covers $PROG + every #included decls/engine -# .dl, so any rule/decl change → new hash → new binary; unchanged → instant reuse. Default -# ~/.cache/AxiomCode-Souffle (XDG-aware); delete it to force a clean rebuild, or override -# with AXIOM_SOUFFLE_CACHE. -# Default IN-REPO so a checkout is self-contained and nothing is written outside it -# (.souffle-cache/ is gitignored). Content-addressed, so branches sharing rule text share the -# binary; a fresh clone rebuilds once. Point AXIOM_SOUFFLE_CACHE at a shared machine-scoped -# dir to amortise that across clones. +# project. It lives in a shared, machine-scoped cache keyed by the engine id — NOT in the +# per-run intermediate. Default IN-REPO so a checkout is self-contained (.souffle-cache/ is +# gitignored); point AXIOM_SOUFFLE_CACHE at a shared dir to amortise it. CACHE_DIR="$CACHE_ROOT" -NEW="$(cat "$PROG" "$DL/decls_base.dl" "$DL/decls_all.dl" "$ENG"/*/*.dl "$ENG"/*/*/*.dl "$ENG2"/*/*.dl "$ENG2"/*/*/*.dl 2>/dev/null | shasum | cut -d' ' -f1)" -BIN="$CACHE_DIR/souffle-engine-$NEW" -if [ ! -x "$BIN" ]; then +EXE=""; case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) EXE=".exe";; esac +BIN="$CACHE_DIR/souffle-engine-$LANG_ARG-$ENGINE_ID$EXE" + +# The platform string, in npm's spelling (process.platform-process.arch), because that is +# how the engine packages are named: darwin-arm64, linux-x64, linux-arm64, win32-x64. +engine_platform(){ + local os arch + case "$(uname -s)" in + Linux) os=linux;; Darwin) os=darwin;; MINGW*|MSYS*|CYGWIN*) os=win32;; + *) echo "unsupported platform: $(uname -s)" >&2; return 1;; + esac + case "$(uname -m)" in + x86_64|amd64) arch=x64;; arm64|aarch64) arch=arm64;; + *) echo "unsupported architecture: $(uname -m)" >&2; return 1;; + esac + printf '%s-%s\n' "$os" "$arch" +} +# 1. the engine package npm installed for this machine, if it was built from exactly these +# rules. Found by walking up from the package root the way node would, so a checkout's own +# node_modules and a global install both work. +PACKAGED="" +platform="$(engine_platform 2>/dev/null || true)" +if [ -n "$platform" ]; then + d="$PKG" + while [ "$d" != / ]; do + pkgdir="$d/node_modules/$ENGINE_PACKAGE_SCOPE/engine-$platform" + if [ -d "$pkgdir" ]; then + have="$(tr -d '[:space:]' < "$pkgdir/$LANG_ARG/ENGINE_ID" 2>/dev/null || true)" + cand="$pkgdir/$LANG_ARG/axiomcode-engine-$LANG_ARG$EXE" + if [ "$have" = "$ENGINE_ID" ] && [ -f "$cand" ]; then PACKAGED="$cand"; chmod +x "$cand" 2>/dev/null || true + elif [ -n "$have" ]; then echo " ! $ENGINE_PACKAGE_SCOPE/engine-$platform holds $LANG_ARG at ${have:0:12}…, these rules are ${ENGINE_ID:0:12}… — not using it (publish a new engine version for these rules)" + else echo " ! $ENGINE_PACKAGE_SCOPE/engine-$platform has no $LANG_ARG engine"; fi + break + fi + d="$(dirname "$d")" + done +fi + +if [ -n "$PACKAGED" ]; then + BIN="$PACKAGED"; echo "▶ using packaged engine $ENGINE_PACKAGE_SCOPE/engine-$platform ($LANG_ARG)" +elif [ -x "$BIN" ]; then + echo "▶ reusing cached binary" +elif command -v souffle >/dev/null 2>&1; then echo "▶ compiling souffle program (cache miss)..." + INNER="$(find_souffle_include)" + # Assert the HEADER, not the directory: `[ -d ]` is the test #216 established cannot tell + # the two install layouts apart, so it would pass a path that then fails at the compiler. + if [ -z "$INNER" ] || [ ! -f "$INNER/souffle/CompiledSouffle.h" ]; then + echo "❌ soufflé is on PATH but its headers are not. Set AXIOM_SOUFFLE_INCLUDE." >&2; exit 1 + fi + have="$(souffle --version 2>/dev/null | sed -n 's/^Version: *\([0-9][0-9.]*\).*/\1/p' | head -1)" + [ "$have" = "$SOUFFLE_VERSION" ] || echo " ! local souffle is $have, the pinned version is $SOUFFLE_VERSION — a locally compiled engine may differ from CI's" # Generate C++. souffle's "No rules/facts defined" warnings (for the intentionally # unstaged lib-body relations — inert paths) aren't silenced by -w, so filter those 3- # line blocks from stderr; on a real failure, dump the full log and fail. c++ -w # silences the deprecation warnings in souffle's own headers. Compile to a .tmp then # atomically rename, so a concurrent/aborted run never leaves a half-written binary. - if ! souffle -g "$INT/souffle-program.cpp" "$PROG" 2> "$INT/.souffle-gen.log"; then + if ! souffle -I "$SRC" -g "$INT/souffle-program.cpp" "$PROG" 2> "$INT/.souffle-gen.log"; then cat "$INT/.souffle-gen.log" >&2; exit 1 fi awk '/No rules\/facts defined/{skip=2;next} skip>0{skip--;next} {print}' "$INT/.souffle-gen.log" >&2 - # Platform-conditional compile flags. On Cygwin the COFF object format caps a single - # object at 32768 sections, and soufflé's generated translation unit for this rule set - # (338 relations, 42 .dl files → a 5.6 MB binary) blows past it. -Wa,-mbig-obj lifts the - # cap. Reported working upstream, though on soufflé 1.5.1 — untested here on 2.5. CXX_PLATFORM="" case "$(uname -s)" in CYGWIN*) CXX_PLATFORM="-Wa,-mbig-obj";; esac c++ -std=c++17 -O3 -march=native -w $CXX_PLATFORM -I "$INNER" "$INT/souffle-program.cpp" -o "$BIN.tmp.$$" mv -f "$BIN.tmp.$$" "$BIN" -else echo "▶ reusing cached binary"; fi +else + echo "❌ no engine for $LANG_ARG@${ENGINE_ID:0:12}… on this machine. Either:" >&2 + echo " • run \`npm install\` here — it fetches $ENGINE_PACKAGE_SCOPE/engine- for this machine (if these rules have been published), or" >&2 + echo " • install souffle $SOUFFLE_VERSION to compile locally (macOS: brew install souffle; Ubuntu: the .deb from souffle-lang/souffle releases)." >&2 + exit 1 +fi # --- STAGE↔SOLVE loop: solve → stage the bodies of methods reached so far → re-solve, until # reachable_method stops growing. Soufflé loads facts up front and can't fetch bodies mid- # solve, so the driver feeds them in reachability order. Each round loads the bodies of ALL @@ -370,7 +472,9 @@ while [ "$iter" -lt 50 ]; do PBIN="$INT/souffle-profile-bin" if [ ! -x "$PBIN" ]; then echo "▶ building profiling binary (once per run dir)..." - souffle -g "$INT/profile-program.cpp" -p "$AXIOM_SOUFFLE_PROFILE" "$PROG" \ + command -v souffle >/dev/null 2>&1 || { echo "❌ profiling needs souffle on PATH (it builds a second binary)" >&2; exit 1; } + INNER="${INNER:-$(find_souffle_include)}" + souffle -I "$SRC" -g "$INT/profile-program.cpp" -p "$AXIOM_SOUFFLE_PROFILE" "$PROG" \ 2> "$INT/.souffle-prof-gen.log" || { cat "$INT/.souffle-prof-gen.log" >&2; exit 1; } c++ -std=c++17 -O3 -march=native -w -I "$INNER" \ "$INT/profile-program.cpp" -o "$PBIN" || exit 1 @@ -413,7 +517,6 @@ echo "Elapsed (solve): $((SOLVE_EPOCH-START_EPOCH))s" # bundle can never be built from a stale dist/ (the failure mode a compiled step invites); # an installed package has no devDependencies and runs the compiled dist/bundle/cli.js that # `npm run build` produced. Neither present is a setup error, and says so. -PKG="$SRC/.." if [ -x "$PKG/node_modules/.bin/tsx" ]; then # --tsconfig, explicitly: cli.ts imports its neighbours through the @/ alias, and tsx # resolves that from the tsconfig it finds relative to the CALLER's working directory — diff --git a/graph/test/java/run-tests.sh b/graph/test/java/run-tests.sh index 1ef9c26d9..11a690a70 100755 --- a/graph/test/java/run-tests.sh +++ b/graph/test/java/run-tests.sh @@ -105,6 +105,19 @@ if ! bash "$ROOT/graph/test/tools/bundle-test.sh"; then echo "aborting: the bundle stage does not produce the documented output" exit 1 fi +# ── The engine id and the packaged-engine path ─────────────────────────────── +# A machine without souffle finds its binary by the id the rules hash to, so the id must be +# the same from any path and different for any rule change; and the engine package npm +# installed must be used only when its ENGINE_ID matches. Both run without souffle or +# network, in seconds. +if ! bash "$ROOT/graph/test/tools/engine-id-test.sh"; then + echo "aborting: the engine id is not a function of the rules alone" + exit 1 +fi +if ! bash "$ROOT/graph/test/tools/engine-package-test.sh"; then + echo "aborting: the packaged-engine path does not check what it runs" + exit 1 +fi PARSER="${AXIOM_PARSER:-$ROOT/parser/dist/index.js}" WORK="$HERE/.work" BLESS=0; KEEP=0; ORACLE=0; FILTERS=() diff --git a/graph/test/tools/engine-id-test.sh b/graph/test/tools/engine-id-test.sh new file mode 100755 index 000000000..e4f48c06e --- /dev/null +++ b/graph/test/tools/engine-id-test.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# The engine id (run-souffle.sh --print-engine-id) is what lets a machine WITHOUT souffle +# find the binary CI built for its rules, so two properties are load-bearing: +# 1. PATH-INDEPENDENT — the same tree at another path gives the same id (CI's checkout is +# never at the user's path); +# 2. RULE-SENSITIVE — one character changed in one rule file changes it, in every language, +# whether the file is a rule, a map, the manifest, or the pinned souffle version. +# Also: the emitted program contains no absolute path, and neither mode needs souffle. +# ───────────────────────────────────────────────────────────────────────────── +set -u +ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" # the repository root, found by its marker +RUN="graph/pipeline/run-souffle.sh" +fail=0; bad(){ echo " ✗ $*"; fail=$((fail+1)); } +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT + +# a copy of the tree, at a different path, with souffle hidden from PATH +mkdir -p "$W/copy" "$W/bin" +cp -R "$ROOT/graph" "$W/copy/graph"; cp "$ROOT/package.json" "$W/copy/package.json" +for t in bash grep awk sed sort cut tr mktemp uname cat rm cp mv ls dirname basename shasum sha256sum stat; do + p="$(command -v "$t" 2>/dev/null)" && ln -sf "$p" "$W/bin/$t" +done +id_at(){ ( cd "$1" && PATH="$W/bin" bash "$RUN" --language "$2" --print-engine-id ); } + +for lang in java typescript python; do + a="$(id_at "$ROOT" "$lang")"; b="$(id_at "$W/copy" "$lang")" + case "$a" in [0-9a-f]*) ;; *) bad "$lang: id is not a hex digest: '$a'";; esac + [ "$a" = "$b" ] || bad "$lang: id differs between two paths ($a vs $b)" + ( cd "$W/copy" && PATH="$W/bin" bash "$RUN" --language "$lang" --emit-program "$W/$lang.dl" ) + grep -q '^#include "/' "$W/$lang.dl" && bad "$lang: emitted program embeds an absolute include path" + grep -q "^#include \"$lang/souffle/decls_base.dl\"" "$W/$lang.dl" || bad "$lang: emitted program does not include $lang/souffle/decls_base.dl" + grep -q '^\.input ' "$W/$lang.dl" || bad "$lang: emitted program declares no inputs" +done + +# sensitivity: touch one thing at a time in the copy and expect a new id +before="$(id_at "$W/copy" java)" +mutate(){ # $1 = file, $2 = appended text, $3 = label + cp "$W/copy/$1" "$W/orig"; printf '%s\n' "$2" >> "$W/copy/$1" + after="$(id_at "$W/copy" java)" + [ "$after" != "$before" ] || bad "java: id unchanged after $3" + mv "$W/orig" "$W/copy/$1" +} +f="$(cd "$ROOT" && ls graph/java/engine/resolution/*.dl | head -1)" +mutate "$f" "// changed" "editing a rule file ($f)" +mutate "graph/java/templates/client-ir.map" "zz_extra_relation all-zz" "adding a staged relation" +mutate "graph/java/souffle/export_manifest.tsv" "zz_pred zz.csv" "adding an export" +mutate "graph/pipeline/engine.conf" 'SOUFFLE_VERSION="9.9"' "bumping the pinned souffle version" +[ "$(id_at "$W/copy" java)" = "$before" ] || bad "java: id did not return to its original value after the mutations were reverted" +# and a change to one language must not move another's id +tsb="$(id_at "$W/copy" typescript)"; printf '// changed\n' >> "$W/copy/$f" +[ "$(id_at "$W/copy" typescript)" = "$tsb" ] || bad "a java rule change moved the typescript id" + +if [ "$fail" -eq 0 ]; then echo "engine-id: ok (path-independent, rule-sensitive, no souffle needed)"; else echo "engine-id: $fail failure(s)"; exit 1; fi diff --git a/graph/test/tools/engine-package-test.sh b/graph/test/tools/engine-package-test.sh new file mode 100755 index 000000000..5e56548ab --- /dev/null +++ b/graph/test/tools/engine-package-test.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# The no-souffle path of run-souffle.sh: with `souffle` absent it must find the engine that +# npm installed for this machine — node_modules/@axiomcode/engine--// — use it +# ONLY when that package's ENGINE_ID equals the id of the rules in the checkout, run it +# through to the bundle, and otherwise refuse with the two ways out named. +# +# No network, no npm: a copy of the tree gets a hand-made engine package, and the "engine" +# in it is a shell script that writes the export manifest's files into -D (which is all the +# driver and the bundle stage need from it). +# ───────────────────────────────────────────────────────────────────────────── +set -u +ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" # the repository root, found by its marker +fail=0; bad(){ echo " ✗ $*"; fail=$((fail+1)); } +[ -x "$ROOT/node_modules/.bin/tsx" ] || { echo "engine-package: SKIP (no node_modules/.bin/tsx — run npm install)"; exit 0; } +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT +mkdir -p "$W/bin" "$W/ir" "$W/int" "$W/out" "$W/tree" +# a copy of the tree with its own node_modules dir (the real one linked in for the bundler) +cp -R "$ROOT/graph" "$W/tree/graph"; cp "$ROOT/package.json" "$ROOT/tsconfig.json" "$W/tree/" +mkdir -p "$W/tree/node_modules"; ln -s "$ROOT/node_modules/.bin" "$W/tree/node_modules/.bin" +for d in "$ROOT"/node_modules/*/; do n="$(basename "$d")"; [ "$n" = "@axiomcode" ] && continue; ln -s "${d%/}" "$W/tree/node_modules/$n"; done +RUN="$W/tree/graph/pipeline/run-souffle.sh" +# a PATH with everything the driver and the bundler need, and no souffle +for t in bash sh grep awk sed sort cut tr mktemp uname cat rm cp mv ls dirname basename shasum sha256sum stat date mkdir chmod head tail wc find ln printf tee env node git; do + p="$(command -v "$t" 2>/dev/null)" && ln -sf "$p" "$W/bin/$t" +done +. "$ROOT/graph/pipeline/engine.conf" + +lang=java +id="$(PATH="$W/bin" bash "$RUN" --language $lang --print-engine-id)" +arch="$(uname -m | sed 's/aarch64/arm64/;s/amd64|x86_64/x64/;s/x86_64/x64/')" +case "$(uname -s)" in Darwin) platform="darwin-$arch";; Linux) platform="linux-$arch";; *) platform="win32-x64";; esac +pkg="$W/tree/node_modules/$ENGINE_PACKAGE_SCOPE/engine-$platform"; mkdir -p "$pkg/$lang" +# the fake engine: writes every manifest file, empty, into -D +{ + echo '#!/usr/bin/env bash' + echo 'while [ $# -gt 0 ]; do case "$1" in -D) D="$2"; shift 2;; -F) shift 2;; *) shift;; esac; done' + cut -f2 "$ROOT/graph/$lang/souffle/export_manifest.tsv" | sed 's|^|: > "$D/|; s|$|"|' +} > "$pkg/$lang/axiomcode-engine-$lang"; chmod +x "$pkg/$lang/axiomcode-engine-$lang" +printf '%s\n' "$id" > "$pkg/$lang/ENGINE_ID" + +run(){ PATH="$W/bin" AXIOM_SOUFFLE_CACHE="$W/cache" bash "$RUN" --language $lang --client-ir "$W/ir" --library "" --intermediate "$W/int" --output "$W/out" > "$W/log" 2>&1; } + +# 1. the packaged engine with a matching id is used, and the run reaches the bundle +if run; then + grep -q "using packaged engine" "$W/log" || bad "packaged engine with a matching id was not used" + [ -f "$W/out/graph.sqlite" ] || [ -f "$W/out/csv/call_edges.csv" ] || bad "the run did not reach the bundle stage" +else + bad "run with a matching packaged engine failed:"; tail -8 "$W/log" | sed 's/^/ /' +fi +# 2. a package built from OTHER rules is reported and refused; no souffle → the two ways out +printf 'deadbeef%s\n' "${id:8}" > "$pkg/$lang/ENGINE_ID"; rm -rf "$W/out" +if run; then bad "a packaged engine with a different id was used"; else + grep -q "not using it" "$W/log" || bad "a stale packaged engine was not reported" + grep -q "npm install" "$W/log" && grep -q "install souffle" "$W/log" || bad "the refusal does not name both ways out" +fi +# 3. no package at all → the same explanation +rm -rf "$W/tree/node_modules/$ENGINE_PACKAGE_SCOPE" "$W/out" +if run; then bad "a run with no engine package and no souffle succeeded"; else + grep -q "npm install" "$W/log" || bad "the no-package error does not point at npm install" +fi + +if [ "$fail" -eq 0 ]; then echo "engine-package: ok (packaged engine by id, stale package refused, absence explained)"; else echo "engine-package: $fail failure(s)"; exit 1; fi diff --git a/package.json b/package.json index 9c8ef5d70..7e0e279df 100644 --- a/package.json +++ b/package.json @@ -25,5 +25,11 @@ "workspaces": [ "parser" ], - "license": "FSL-1.1-Apache-2.0" + "license": "FSL-1.1-Apache-2.0", + "optionalDependencies": { + "@axiomcode/engine-darwin-arm64": "0.1.0", + "@axiomcode/engine-linux-x64": "0.1.0", + "@axiomcode/engine-linux-arm64": "0.1.0", + "@axiomcode/engine-win32-x64": "0.1.0" + } } diff --git a/packaging/assemble-engine-package.sh b/packaging/assemble-engine-package.sh new file mode 100755 index 000000000..4dab479ae --- /dev/null +++ b/packaging/assemble-engine-package.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Assemble one @axiomcode/engine-- npm package from the compiled engines. +# assemble-engine-package.sh +# holds /axiomcode-engine-[.exe] and /ENGINE_ID for every +# language CI built for that platform. The package carries them verbatim plus a package.json +# whose os/cpu fields let npm install it only on a matching machine. +set -eu +platform="$1"; version="$2"; src="$3"; out="$4" +os="${platform%%-*}"; cpu="${platform#*-}" +HERE="$(cd "$(dirname "$0")" && pwd)" +. "$HERE/../graph/pipeline/engine.conf" +rm -rf "$out"; mkdir -p "$out" +cp -R "$src"/. "$out/" +langs="$(ls -d "$out"/*/ | xargs -n1 basename | tr '\n' ' ')" +sed -e "s|@@SCOPE@@|$ENGINE_PACKAGE_SCOPE|g" -e "s|@@PLATFORM@@|$platform|g" -e "s|@@VERSION@@|$version|g" \ + -e "s|@@OS@@|$os|g" -e "s|@@CPU@@|$cpu|g" -e "s|@@LANGS@@|${langs% }|g" -e "s|@@SOUFFLE@@|$SOUFFLE_VERSION|g" \ + "$HERE/engine-package.json" > "$out/package.json" +cp "$HERE/../LICENSE.md" "$out/LICENSE.md" +{ echo "# $ENGINE_PACKAGE_SCOPE/engine-$platform"; echo + echo "Prebuilt AxiomCode code-graph engines for $os/$cpu: ${langs% }. Installed automatically as an" + echo "optional dependency of the code-graph package on a matching machine; not meant to be used directly." + echo; for l in $langs; do echo "- $l: rules id \`$(cat "$out/$l/ENGINE_ID")\`"; done +} > "$out/README.md" +chmod +x "$out"/*/axiomcode-engine-* 2>/dev/null || true +echo "assembled $out: $(ls "$out" | tr '\n' ' ')" diff --git a/packaging/engine-package.json b/packaging/engine-package.json new file mode 100644 index 000000000..e63533dfc --- /dev/null +++ b/packaging/engine-package.json @@ -0,0 +1,11 @@ +{ + "name": "@@SCOPE@@/engine-@@PLATFORM@@", + "version": "@@VERSION@@", + "description": "AxiomCode code-graph engines (@@LANGS@@) compiled for @@OS@@/@@CPU@@ with Soufflé @@SOUFFLE@@. Installed as an optional dependency of the code-graph package; npm selects this package by os/cpu.", + "license": "FSL-1.1-Apache-2.0", + "os": ["@@OS@@"], + "cpu": ["@@CPU@@"], + "files": ["*/axiomcode-engine-*", "*/ENGINE_ID", "README.md", "LICENSE.md"], + "repository": { "type": "git", "url": "git+https://github.com/AxiomCodeAI/axiom-code-graph.git" }, + "publishConfig": { "access": "public" } +} From 36d0725d77927af3be03df491d5b4368176200b0 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 14 Sep 2026 02:04:50 -0700 Subject: [PATCH 9/9] =?UTF-8?q?build:=20the=20JavaScript=20engine=20is=20b?= =?UTF-8?q?uilt=20and=20shipped=20like=20the=20other=20three=20=E2=80=94?= =?UTF-8?q?=20LANGUAGES=20gains=20javascript=20in=20the=20generate,=20Linu?= =?UTF-8?q?x,=20Windows=20and=20smoke=20steps,=20and=20in=20the=20engine-i?= =?UTF-8?q?d=20preflight?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-engines.yml | 4 ++-- graph/test/tools/engine-id-test.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-engines.yml b/.github/workflows/build-engines.yml index 7806840ea..326e1cd28 100644 --- a/.github/workflows/build-engines.yml +++ b/.github/workflows/build-engines.yml @@ -31,7 +31,7 @@ on: default: false env: - LANGUAGES: java typescript python + LANGUAGES: java typescript python javascript jobs: generate: @@ -91,7 +91,7 @@ jobs: if: startsWith(matrix.target.platform, 'win32') shell: cmd run: | - for %%L in (java typescript python) do ( + for %%L in (java typescript python javascript) do ( mkdir engines\%%L cl /nologo /std:c++17 /O2 /EHsc /bigobj /w /permissive- /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS /DNOMINMAX /DUSE_CUSTOM_GETOPTLONG /I gen gen\%%L.cpp /Fe:engines\%%L\axiomcode-engine-%%L.exe if errorlevel 1 exit /b 1 diff --git a/graph/test/tools/engine-id-test.sh b/graph/test/tools/engine-id-test.sh index e4f48c06e..3602ea3c9 100755 --- a/graph/test/tools/engine-id-test.sh +++ b/graph/test/tools/engine-id-test.sh @@ -22,7 +22,7 @@ for t in bash grep awk sed sort cut tr mktemp uname cat rm cp mv ls dirname base done id_at(){ ( cd "$1" && PATH="$W/bin" bash "$RUN" --language "$2" --print-engine-id ); } -for lang in java typescript python; do +for lang in java typescript python javascript; do a="$(id_at "$ROOT" "$lang")"; b="$(id_at "$W/copy" "$lang")" case "$a" in [0-9a-f]*) ;; *) bad "$lang: id is not a hex digest: '$a'";; esac [ "$a" = "$b" ] || bad "$lang: id differs between two paths ($a vs $b)"