From 2e7b26f6ceee8c69fd1b748b4231386cb9939590 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Thu, 17 Sep 2026 22:49:25 -0700 Subject: [PATCH] build: engines ship on npm as @axiomcode/engine-- Restores the work reverted in #903 so it can be reviewed before it ships. Unchanged from what was on main, with the fixes that landed on top of it kept: - the collation pin from #895 stays inside write_program, so the engine id is a function of the rules and not of the user's locale. Without it a published engine is refused for java and python on any machine with a UTF-8 collation, which is the common configuration. - bin, files and dependencies from #886 stay, so an install can actually run. optionalDependencies returns with the feature that reads them. Known gaps, both measured and neither fixed here: - linux-arm64 cannot install at all (#901). No dependency in the 0.21.x tree-sitter generation ships an arm64 prebuild, and a nested duplicate cannot build even with a toolchain. Unrelated to the engine: the install fails before any engine is resolved. - nothing builds the binaries while the CI workflows are out of the tree, so a release means building by hand on each platform. --- .gitignore | 2 + bin/axiomcode | 3 +- graph/pipeline/engine.conf | 15 ++ graph/pipeline/portable-stat.sh | 10 + graph/pipeline/run-souffle.sh | 269 +++++++++++++++------- graph/test/java/run-tests.sh | 13 ++ graph/test/tools/engine-id-locale-test.sh | 4 - graph/test/tools/engine-id-test.sh | 74 ++++++ graph/test/tools/engine-package-test.sh | 82 +++++++ package.json | 6 + packaging/assemble-engine-package.sh | 25 ++ packaging/engine-package.json | 11 + 12 files changed, 428 insertions(+), 86 deletions(-) create mode 100644 graph/pipeline/engine.conf create mode 100755 graph/test/tools/engine-id-test.sh create mode 100755 graph/test/tools/engine-package-test.sh create mode 100755 packaging/assemble-engine-package.sh create mode 100644 packaging/engine-package.json diff --git a/.gitignore b/.gitignore index 10ae6b97..c639db15 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ # ── Dependencies ───────────────────────────────────────────────────────────── node_modules node_modules/ +# and the bare name: `node_modules/` matches a directory only, so a symlink named node_modules slips past it +node_modules jspm_packages/ web_modules/ .pnp diff --git a/bin/axiomcode b/bin/axiomcode index 912eb94c..529c92e7 100755 --- a/bin/axiomcode +++ b/bin/axiomcode @@ -26,7 +26,8 @@ # bin/axiomcode engine --language L --client-ir / --out [options] # bin/axiomcode test [java|typescript|python|javascript|parser|all] [suite options] # -# Requires Node ≥ 22.5; no Soufflé or compiler (binaries/, or a local souffle if present). +# Requires Node ≥ 22.5; no Soufflé or compiler (the engine comes from npm as @axiomcode/engine--, +# or is compiled locally when souffle is present). # ───────────────────────────────────────────────────────────────────────────── set -eu # RESOLVE $0 THROUGH SYMLINKS BEFORE THE WALK. npm installs a `bin` entry as a link in diff --git a/graph/pipeline/engine.conf b/graph/pipeline/engine.conf new file mode 100644 index 00000000..0f0158c6 --- /dev/null +++ b/graph/pipeline/engine.conf @@ -0,0 +1,15 @@ +# ───────────────────────────────────────────────────────────────────────────── +# The prebuilt-engine contract. Sourced by run-souffle.sh and by the CI workflows. +# +# SOUFFLE_VERSION is PINNED here, not read from a `souffle --version`, because the machine +# that runs a prebuilt binary has no souffle to ask — and the engine id it computes must be +# the id CI computed. Bumping it changes every language's id, which is what a new code +# generator should do. CI installs exactly this version. +# +# ENGINE_PACKAGE_SCOPE is the npm scope the engine packages are published under: +# /engine-- (darwin-arm64, linux-x64, linux-arm64, win32-x64), each holding +# every language's engine for that platform under / with its ENGINE_ID. This package +# lists them as optionalDependencies, so `npm install` fetches the one for the machine. +# ───────────────────────────────────────────────────────────────────────────── +SOUFFLE_VERSION="2.5" +ENGINE_PACKAGE_SCOPE="@axiomcode" diff --git a/graph/pipeline/portable-stat.sh b/graph/pipeline/portable-stat.sh index 3be93fdf..2e45972d 100644 --- a/graph/pipeline/portable-stat.sh +++ b/graph/pipeline/portable-stat.sh @@ -52,3 +52,13 @@ sha1_stdin(){ [ -n "$_SHA1_CMD" ] || { echo "neither shasum nor sha1sum is on PATH" >&2; return 1; } "$_SHA1_CMD" | cut -d' ' -f1 } + +# sha256 of stdin, for the engine id. Same three spellings as sha1 above: `shasum -a 256` +# (macOS, perl shasum in Git Bash) or `sha256sum` (coreutils). +if command -v sha256sum >/dev/null 2>&1; then _SHA256_CMD="sha256sum" +elif command -v shasum >/dev/null 2>&1; then _SHA256_CMD="shasum -a 256" +else _SHA256_CMD=""; fi +sha256_stdin(){ + [ -n "$_SHA256_CMD" ] || { echo "neither sha256sum nor shasum is on PATH" >&2; return 1; } + $_SHA256_CMD | cut -d' ' -f1 +} diff --git a/graph/pipeline/run-souffle.sh b/graph/pipeline/run-souffle.sh index 20d4da5e..a04763c7 100755 --- a/graph/pipeline/run-souffle.sh +++ b/graph/pipeline/run-souffle.sh @@ -3,6 +3,18 @@ # client-ir.map / lib.map (single source of truth). Lib is auto-scoped # to only the signature relations the rules reference (never loads GB-scale bodies). # Usage: run-souffle.sh --client-ir DIR --library DIR --intermediate DIR --output DIR [--language L] [--debug] +# run-souffle.sh --language L --print-engine-id the canonical id of L's compiled engine +# run-souffle.sh --language L --emit-program FILE the Soufflé program CI compiles for L +# +# NO SOUFFLÉ NEEDED TO RUN. The rules compile to one self-contained executable that is +# project-independent; CI builds it for every platform and publishes it on npm as +# @axiomcode/engine--, which this package lists as an optional +# dependency so `npm install` fetches exactly the one for the machine. The binary is +# resolved in this order: +# 1. node_modules/@axiomcode/engine-// — used only if its ENGINE_ID equals +# the id of the rules in this checkout (edited rules never silently run a stale binary); +# 2. a locally compiled engine, when `souffle` is on PATH (cached under .souffle-cache). +# See graph/pipeline/engine.conf. # # OUTPUT LAYOUT — the same in every language (graph/bundle/SCHEMA.md): # $OUT/graph.sqlite the contract: core tables + ext_* tables + the schema catalog @@ -33,6 +45,8 @@ DISPATCH_CAP="${DISPATCH_CAP:-20}" # fan-width cap on virtual dispatch. DEFAUL LANG_ARG="" # which rule set under graph// to run. Default java. TAINT="" # --taint on → gate lib→lib GROW on client-seeded data flow (dataflow/taint.dl). Also # settable via env AXIOM_TAINT_GATING=on. Empty = ungated (default behavior). +MODE="run" # run | print-engine-id | emit-program — the last two need no IR and no souffle +EMIT="" while [ $# -gt 0 ]; do case "$1" in --client-ir) CLIENT="$2"; shift 2;; --library) LIB="$2"; shift 2;; --intermediate) INT="$2"; shift 2;; --output) OUT="$2"; shift 2;; @@ -41,6 +55,8 @@ while [ $# -gt 0 ]; do case "$1" in --lib-depth) LIB_DEPTH="$2"; shift 2;; --taint) TAINT="$2"; shift 2;; --language) LANG_ARG="$2"; shift 2;; + --print-engine-id) MODE="print-engine-id"; shift;; + --emit-program) MODE="emit-program"; EMIT="$2"; shift 2;; # graph.sqlite is the deliverable; csv/*.csv is a debugging view of the same core # tables. --debug asks for both. (An older Node with no node:sqlite writes the CSVs # regardless, because otherwise the run would produce no consumer-facing output.) @@ -50,6 +66,7 @@ while [ $# -gt 0 ]; do case "$1" in # change the invocation.) *) shift;; esac; done SRC="$(cd "$(dirname "$0")/.." && pwd)" +PKG="$(cd "$SRC/.." && pwd)" # the package root: package.json, node_modules, parser/, graph/ # shellcheck source=portable-stat.sh . "$SRC/pipeline/portable-stat.sh" # shellcheck source=lib-cache-key.sh @@ -60,14 +77,103 @@ ENG="$SRC/$LANG_ARG/engine"; ENG2="$SRC/$LANG_ARG/engine-ii"; DL="$SRC/$LANG_ARG [ -d "$ENG" ] || { echo "no rule set for --language=$LANG_ARG (looked in $ENG)" >&2; exit 1; } # shellcheck source=souffle-include.sh . "$SRC/pipeline/souffle-include.sh" -INNER="$(find_souffle_include)" -# Assert the HEADER, not the directory: `[ -d ]` is the test #216 established cannot tell the two -# install layouts apart, so it would pass a path that then fails at the compiler. -if [ -z "$INNER" ] || [ ! -f "$INNER/souffle/CompiledSouffle.h" ]; then - echo "❌ soufflé headers not found. Install soufflé, or set AXIOM_SOUFFLE_INCLUDE." >&2 - echo " macOS: brew install souffle Debian/Ubuntu: apt-get install souffle" >&2 - exit 1 -fi +# shellcheck source=engine.conf +. "$SRC/pipeline/engine.conf" +# Staging config is PER-LANGUAGE (IR marker + which relations are signatures vs bodies). +# Keeping it here would hardcode Java's entity set into a shared executor. +[ -f "$TPL/staging.conf" ] || { echo "missing $TPL/staging.conf for --language=$LANG_ARG" >&2; exit 1; } +. "$TPL/staging.conf" +ENGINE_II_MODE="${ENGINE_II:-${AXIOM_ENGINE_II:-off}}" + +# The tools every path below relies on. Checked up front because a missing one does not +# always fail loudly: read_map runs inside a process substitution, where a missing grep +# yields an EMPTY relation list — and a different, wrong engine id — under `set -e`. +for t in grep awk sed sort cut tr mktemp uname dirname cat; do + command -v "$t" >/dev/null 2>&1 || { echo "❌ required tool not on PATH: $t" >&2; exit 1; } +done + +# read an import map (relationcsv-basename per line), skipping comments (#) and blanks +read_map(){ grep -vE '^[[:space:]]*(#|$)' "$1"; } + +# ── THE PROGRAM, as a pure function of the repository ──────────────────────────────────── +# Written so that the SAME text comes out of every checkout and of CI: includes are relative +# to graph/ (souffle resolves them through -I "$SRC"), and the .input list is derived from the +# maps rather than from a listing of the staged facts dir — so it needs no client IR, and a +# machine that cannot stage (CI) still produces the text the binary was built from. The run +# path asserts below that staging created a facts file for every .input it declares. +# The list of input relations is: every client relation, the lib signature relations, the lib +# body relations (filled per iteration), and the four knob facts. +input_relations(){ + while IFS=$'\t' read -r rel csv; do printf '%s\n' "$rel"; done < <(read_map "$TPL/client-ir.map") + while IFS=$'\t' read -r rel csv; do + case " $LIB_SIG " in *" ${rel#lib_} "*) printf '%s\n' "$rel";; esac + done < <(read_map "$TPL/lib.map") + for r in $LIB_BODY; do printf '%s\n' "$r"; done + printf '%s\n' jdk_max_depth lib_max_depth taint_gating dispatch_cap +} +write_program(){ # $1 = destination file + # COLLATION IS PART OF THE PROGRAM TEXT, so it is pinned here rather than inherited. The + # #include lines below come from shell globs, and bash orders a glob by LC_COLLATE, not by + # byte value. A UTF-8 collation ignores punctuation when comparing, so call-site.dl and + # callee-resolution.dl swap places against their byte order. The include order is part of + # the program text, the program text is hashed, and that hash is the engine id -- so + # whether a published binary is accepted becomes a function of the user's locale rather + # than of the rules, and the refusal names the rules. CI runs under a C-ish locale while a + # UTF-8 locale is the default on most Linux desktops, in macOS terminals and in Git Bash, + # so the mismatch is the common case. Measured: java and python ids differ between macOS + # and MSYS2, and between LC_ALL=C and en_US.UTF-8 on glibc; typescript and javascript + # agree only because no pair of their filenames collides. Invisible on macOS, whose + # collation matches C either way, which is why it survived. The sorts below were already + # forced to C for this reason; the globs were not. + # See issue #895 and graph/test/tools/engine-id-locale-test.sh. + local LC_ALL=C LC_COLLATE=C + { + echo "#include \"$LANG_ARG/souffle/decls_base.dl\""; echo "#include \"$LANG_ARG/souffle/decls_all.dl\"" + # rfc4180=true: the IR is CSV, not TSV. The parser quotes any field containing a + # quote, tab or newline and doubles the inner quotes, so reading it as plain TSV hands + # the rules the ESCAPED text. Souffle parses RFC4180 itself, so this costs one flag + # rather than a re-encode of GB-scale input. + # LC_ALL=C sort: the order is part of the program text, so it must not depend on locale. + input_relations | LC_ALL=C sort -u | while read -r r; do printf '.input %s(IO=file, filename="%s.facts", delimiter="\\t", rfc4180=true)\n' "$r" "$r"; done + for d in projections containment resolution config-resolution expression-resolution call-edge-generation framework-behavior; do + # [ -f ] guard: a phase directory that is empty (or absent for a language that has + # not implemented that layer yet) leaves the glob unexpanded, and souffle's C + # preprocessor then fails on a literal '*.dl' include. + for f in "$ENG/$d/"*.dl; do [ -f "$f" ] && echo "#include \"${f#"$SRC/"}\""; done + done + # engine-ii: the first→third forward-chain engine (mirrors engine/, lib-seeded). Same solve, + # included AFTER engine/ so it reads engine/'s relations (client_calls_lib seed). Glob its + # phase subfolders (both nesting levels; globs are space-safe, the repo path has spaces). + # export/ is doc-only (like engine/export) — skip it. + if [ "$ENGINE_II_MODE" = "on" ]; then + for f in "$ENG2/"*/*.dl "$ENG2/"*/*/*.dl; do + case "$f" in */export/*) continue;; esac + [ -f "$f" ] && echo "#include \"${f#"$SRC/"}\"" + done + fi + # Relative output filenames — the -D at run time supplies the directory. Keeping $OUT out + # of the program makes the compiled binary independent of the output path (better reuse). + while IFS=$'\t' read -r pred file; do [ -n "$pred" ] && printf '.output %s(IO=file, filename="%s", delimiter="\\t")\n' "$pred" "$file"; done < <(LC_ALL=C sort -u "$DL/export_manifest.tsv") + } > "$1" +} +# The engine id: sha256 over the pinned code-generator version, the program text, and every +# file it includes, in include order. A function of the repository alone — the same from any +# path, on any machine, with or without souffle — and different for any rule change. It names +# the local cache entry AND the engine package CI publishes, which is what lets a machine +# without souffle know which binary is its own. +engine_id(){ + local prog; prog="$(mktemp)"; write_program "$prog" + { printf 'souffle=%s\n' "$SOUFFLE_VERSION"; cat "$prog" + sed -n 's/^#include "\(.*\)"$/\1/p' "$prog" | while read -r inc; do cat "$SRC/$inc"; done + } | sha256_stdin + rm -f "$prog" +} +case "$MODE" in + print-engine-id) engine_id; exit 0;; + emit-program) write_program "$EMIT"; exit 0;; +esac + +[ -n "${CLIENT:-}" ] && [ -n "${INT:-}" ] && [ -n "${OUT:-}" ] || { echo "usage: run-souffle.sh --client-ir DIR --library DIR --intermediate DIR --output DIR [--language L]" >&2; exit 1; } FACTS="$INT/souffle-facts"; rm -rf "$FACTS"; mkdir -p "$FACTS" "$OUT" # raw/ is OWNED: wiped per run so a relation that left the manifest cannot linger from an # earlier run and be mistaken for this one's output. @@ -79,20 +185,12 @@ RAW="$OUT/raw"; rm -rf "$RAW"; mkdir -p "$RAW" CACHE_ROOT="${AXIOM_SOUFFLE_CACHE:-$SRC/../.souffle-cache}"; mkdir -p "$CACHE_ROOT" START_EPOCH=$(date +%s); START_TS=$(date '+%Y-%m-%d %H:%M:%S') -# read an import map (relationcsv-basename per line), skipping comments (#) and blanks -read_map(){ grep -vE '^[[:space:]]*(#|$)' "$1"; } - # Library roots: --library is a comma-separated list of IR roots (each with jdk-style # module sub-folders, or a flat IR dir). The caller (TS) controls which folders/libraries # are loaded; staging concatenates each relation across every module of every root. IFS=',' read -ra LIB_ROOTS <<< "$LIB" # lib_modules ROOT -> the module dirs to stage from (the root itself if it holds the IR, # else its immediate sub-folders — mirrors how the JDK ships sharded modules). -# Staging config is PER-LANGUAGE (IR marker + which relations are signatures vs bodies). -# Keeping it here would hardcode Java's entity set into a shared executor. -[ -f "$TPL/staging.conf" ] || { echo "missing $TPL/staging.conf for --language=$LANG_ARG" >&2; exit 1; } -. "$TPL/staging.conf" - lib_modules(){ if [ -f "$1/$IR_MARKER" ]; then printf '%s\n' "$1"; else for m in "$1"/*/; do [ -d "$m" ] && printf '%s\n' "${m%/}"; done; fi; } # --- CLIENT: stage EVERY mapped relation, empty when the project has no such file --- @@ -211,90 +309,98 @@ echo "▶ dispatch cap = $( [ -s "$FACTS/dispatch_cap.facts" ] && echo "$(cat "$ # (and backed up on ~/Desktop) but excluded from the compiled program; re-enable with # --engine-ii on / AXIOM_ENGINE_II=on. engine/ produces client_calls_lib etc. independently, so # client-only is a complete, valid solve on its own. -ENGINE_II_MODE="${ENGINE_II:-${AXIOM_ENGINE_II:-off}}" echo "▶ engine-ii = $( [ "$ENGINE_II_MODE" = "on" ] && echo 'ON (lib frontier included)' || echo 'OFF (client-only — engine-i)' )" -# --- generate combined program --- -# COLLATION IS PART OF THE PROGRAM TEXT, so it is pinned here rather than inherited. The -# #include lines below come from shell globs, and bash orders a glob by LC_COLLATE, not by -# byte value. A UTF-8 collation ignores punctuation when comparing, so call-site.dl and -# callee-resolution.dl swap places against their byte order. The include order is part of the -# program text and the program text is hashed, so the compiled-binary cache key becomes a -# function of the user's locale rather than of the rules: a locale change forces a full -# recompile, and two agents on one machine with different locales never share a binary. -# Reproduced on glibc and on MSYS2; invisible on macOS, whose collation matches C either way. -# The sorts below were already forced to C for this reason; the globs were not. See #895. -export LC_ALL=C LC_COLLATE=C - +# --- the program, and the binary for it: from npm, or compiled here --- PROG="$INT/souffle-program.dl" -{ - echo "#include \"$DL/decls_base.dl\""; echo "#include \"$DL/decls_all.dl\"" - # rfc4180=true: the IR is CSV, not TSV. The parser quotes any field containing a - # quote, tab or newline and doubles the inner quotes, so reading it as plain TSV hands - # the rules the ESCAPED text. It only bites where a JOINED column contains a quote -- - # which is why it went unnoticed -- but a string forward reference (`-> "Factory"`) - # lands squarely on one, and a field carrying a tab would shift every column after it. - # Souffle parses RFC4180 itself, so this costs one flag rather than a re-encode of - # GB-scale input. - for ff in "$FACTS"/*.facts; do r=$(basename "$ff" .facts); printf '.input %s(IO=file, filename="%s.facts", delimiter="\\t", rfc4180=true)\n' "$r" "$r"; done - for d in projections containment resolution config-resolution expression-resolution call-edge-generation framework-behavior; do - # [ -f ] guard: a phase directory that is empty (or absent for a language that has - # not implemented that layer yet) leaves the glob unexpanded, and souffle's C - # preprocessor then fails on a literal '*.dl' include. - for f in "$ENG/$d/"*.dl; do [ -f "$f" ] && echo "#include \"$f\""; done - done - # engine-ii: the first→third forward-chain engine (mirrors engine/, lib-seeded). Same solve, - # included AFTER engine/ so it reads engine/'s relations (client_calls_lib seed). Glob its - # phase subfolders (both nesting levels; globs are space-safe, the repo path has spaces). - # export/ is doc-only (like engine/export) — skip it. - if [ "$ENGINE_II_MODE" = "on" ]; then - for f in "$ENG2/"*/*.dl "$ENG2/"*/*/*.dl; do - case "$f" in */export/*) continue;; esac - [ -f "$f" ] && echo "#include \"$f\"" - done - fi - # Relative output filenames — the -D at run time supplies the directory. Keeping $OUT out - # of the program makes the compiled binary independent of the output path (better reuse). - while IFS=$'\t' read -r pred file; do [ -n "$pred" ] && printf '.output %s(IO=file, filename="%s", delimiter="\\t")\n' "$pred" "$file"; done < <(sort -u "$DL/export_manifest.tsv") -} > "$PROG" +write_program "$PROG" +# Every declared input must have been staged, or souffle would fail on a missing file after +# the (possibly long) library staging. The program lists inputs from the maps; staging +# created them from the same maps, so a mismatch is a bug in this script, and says so. +for r in $(sed -n 's/^\.input \([A-Za-z0-9_]*\)(.*/\1/p' "$PROG"); do + [ -f "$FACTS/$r.facts" ] || { echo "❌ program declares input $r but staging created no $r.facts" >&2; exit 1; } +done +ENGINE_ID="$(engine_id)" +echo "▶ engine id = $ENGINE_ID (rules + souffle $SOUFFLE_VERSION)" -# --- compile once into a PERSISTENT, content-addressed cache (survives inter/ deletion) --- # What we cache is OUR engine compiled to a native binary (souffle -g turns the .dl rules # into C++, c++ compiles it) — NOT the souffle tool. It depends only on the engine (rules + # decls) and is PROJECT-INDEPENDENT (relative .input/.output), so one binary serves every -# project: N concurrent analyses of N different projects all share it. It therefore lives in -# a shared, machine-scoped cache keyed by a content hash — NOT in the per-run intermediate -# (which the pipeline/parser wipes). The hash covers $PROG + every #included decls/engine -# .dl, so any rule/decl change → new hash → new binary; unchanged → instant reuse. Default -# ~/.cache/AxiomCode-Souffle (XDG-aware); delete it to force a clean rebuild, or override -# with AXIOM_SOUFFLE_CACHE. -# Default IN-REPO so a checkout is self-contained and nothing is written outside it -# (.souffle-cache/ is gitignored). Content-addressed, so branches sharing rule text share the -# binary; a fresh clone rebuilds once. Point AXIOM_SOUFFLE_CACHE at a shared machine-scoped -# dir to amortise that across clones. +# project. It lives in a shared, machine-scoped cache keyed by the engine id — NOT in the +# per-run intermediate. Default IN-REPO so a checkout is self-contained (.souffle-cache/ is +# gitignored); point AXIOM_SOUFFLE_CACHE at a shared dir to amortise it. CACHE_DIR="$CACHE_ROOT" -NEW="$(cat "$PROG" "$DL/decls_base.dl" "$DL/decls_all.dl" "$ENG"/*/*.dl "$ENG"/*/*/*.dl "$ENG2"/*/*.dl "$ENG2"/*/*/*.dl 2>/dev/null | shasum | cut -d' ' -f1)" -BIN="$CACHE_DIR/souffle-engine-$NEW" -if [ ! -x "$BIN" ]; then +EXE=""; case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) EXE=".exe";; esac +BIN="$CACHE_DIR/souffle-engine-$LANG_ARG-$ENGINE_ID$EXE" + +# The platform string, in npm's spelling (process.platform-process.arch), because that is +# how the engine packages are named: darwin-arm64, linux-x64, linux-arm64, win32-x64. +engine_platform(){ + local os arch + case "$(uname -s)" in + Linux) os=linux;; Darwin) os=darwin;; MINGW*|MSYS*|CYGWIN*) os=win32;; + *) echo "unsupported platform: $(uname -s)" >&2; return 1;; + esac + case "$(uname -m)" in + x86_64|amd64) arch=x64;; arm64|aarch64) arch=arm64;; + *) echo "unsupported architecture: $(uname -m)" >&2; return 1;; + esac + printf '%s-%s\n' "$os" "$arch" +} +# 1. the engine package npm installed for this machine, if it was built from exactly these +# rules. Found by walking up from the package root the way node would, so a checkout's own +# node_modules and a global install both work. +PACKAGED="" +platform="$(engine_platform 2>/dev/null || true)" +if [ -n "$platform" ]; then + d="$PKG" + while [ "$d" != / ]; do + pkgdir="$d/node_modules/$ENGINE_PACKAGE_SCOPE/engine-$platform" + if [ -d "$pkgdir" ]; then + have="$(tr -d '[:space:]' < "$pkgdir/$LANG_ARG/ENGINE_ID" 2>/dev/null || true)" + cand="$pkgdir/$LANG_ARG/axiomcode-engine-$LANG_ARG$EXE" + if [ "$have" = "$ENGINE_ID" ] && [ -f "$cand" ]; then PACKAGED="$cand"; chmod +x "$cand" 2>/dev/null || true + elif [ -n "$have" ]; then echo " ! $ENGINE_PACKAGE_SCOPE/engine-$platform holds $LANG_ARG at ${have:0:12}…, these rules are ${ENGINE_ID:0:12}… — not using it (publish a new engine version for these rules)" + else echo " ! $ENGINE_PACKAGE_SCOPE/engine-$platform has no $LANG_ARG engine"; fi + break + fi + d="$(dirname "$d")" + done +fi + +if [ -n "$PACKAGED" ]; then + BIN="$PACKAGED"; echo "▶ using packaged engine $ENGINE_PACKAGE_SCOPE/engine-$platform ($LANG_ARG)" +elif [ -x "$BIN" ]; then + echo "▶ reusing cached binary" +elif command -v souffle >/dev/null 2>&1; then echo "▶ compiling souffle program (cache miss)..." + INNER="$(find_souffle_include)" + # Assert the HEADER, not the directory: `[ -d ]` is the test #216 established cannot tell + # the two install layouts apart, so it would pass a path that then fails at the compiler. + if [ -z "$INNER" ] || [ ! -f "$INNER/souffle/CompiledSouffle.h" ]; then + echo "❌ soufflé is on PATH but its headers are not. Set AXIOM_SOUFFLE_INCLUDE." >&2; exit 1 + fi + have="$(souffle --version 2>/dev/null | sed -n 's/^Version: *\([0-9][0-9.]*\).*/\1/p' | head -1)" + [ "$have" = "$SOUFFLE_VERSION" ] || echo " ! local souffle is $have, the pinned version is $SOUFFLE_VERSION — a locally compiled engine may differ from CI's" # Generate C++. souffle's "No rules/facts defined" warnings (for the intentionally # unstaged lib-body relations — inert paths) aren't silenced by -w, so filter those 3- # line blocks from stderr; on a real failure, dump the full log and fail. c++ -w # silences the deprecation warnings in souffle's own headers. Compile to a .tmp then # atomically rename, so a concurrent/aborted run never leaves a half-written binary. - if ! souffle -g "$INT/souffle-program.cpp" "$PROG" 2> "$INT/.souffle-gen.log"; then + if ! souffle -I "$SRC" -g "$INT/souffle-program.cpp" "$PROG" 2> "$INT/.souffle-gen.log"; then cat "$INT/.souffle-gen.log" >&2; exit 1 fi awk '/No rules\/facts defined/{skip=2;next} skip>0{skip--;next} {print}' "$INT/.souffle-gen.log" >&2 - # Platform-conditional compile flags. On Cygwin the COFF object format caps a single - # object at 32768 sections, and soufflé's generated translation unit for this rule set - # (338 relations, 42 .dl files → a 5.6 MB binary) blows past it. -Wa,-mbig-obj lifts the - # cap. Reported working upstream, though on soufflé 1.5.1 — untested here on 2.5. CXX_PLATFORM="" case "$(uname -s)" in CYGWIN*) CXX_PLATFORM="-Wa,-mbig-obj";; esac c++ -std=c++17 -O3 -march=native -w $CXX_PLATFORM -I "$INNER" "$INT/souffle-program.cpp" -o "$BIN.tmp.$$" mv -f "$BIN.tmp.$$" "$BIN" -else echo "▶ reusing cached binary"; fi +else + echo "❌ no engine for $LANG_ARG@${ENGINE_ID:0:12}… on this machine. Either:" >&2 + echo " • run \`npm install\` here — it fetches $ENGINE_PACKAGE_SCOPE/engine- for this machine (if these rules have been published), or" >&2 + echo " • install souffle $SOUFFLE_VERSION to compile locally (macOS: brew install souffle; Ubuntu: the .deb from souffle-lang/souffle releases)." >&2 + exit 1 +fi # --- STAGE↔SOLVE loop: solve → stage the bodies of methods reached so far → re-solve, until # reachable_method stops growing. Soufflé loads facts up front and can't fetch bodies mid- # solve, so the driver feeds them in reachability order. Each round loads the bodies of ALL @@ -357,7 +463,9 @@ while [ "$iter" -lt 50 ]; do PBIN="$INT/souffle-profile-bin" if [ ! -x "$PBIN" ]; then echo "▶ building profiling binary (once per run dir)..." - souffle -g "$INT/profile-program.cpp" -p "$AXIOM_SOUFFLE_PROFILE" "$PROG" \ + command -v souffle >/dev/null 2>&1 || { echo "❌ profiling needs souffle on PATH (it builds a second binary)" >&2; exit 1; } + INNER="${INNER:-$(find_souffle_include)}" + souffle -I "$SRC" -g "$INT/profile-program.cpp" -p "$AXIOM_SOUFFLE_PROFILE" "$PROG" \ 2> "$INT/.souffle-prof-gen.log" || { cat "$INT/.souffle-prof-gen.log" >&2; exit 1; } c++ -std=c++17 -O3 -march=native -w -I "$INNER" \ "$INT/profile-program.cpp" -o "$PBIN" || exit 1 @@ -400,7 +508,6 @@ echo "Elapsed (solve): $((SOLVE_EPOCH-START_EPOCH))s" # bundle can never be built from a stale dist/ (the failure mode a compiled step invites); # an installed package has no devDependencies and runs the compiled dist/bundle/cli.js that # `npm run build` produced. Neither present is a setup error, and says so. -PKG="$SRC/.." if [ -x "$PKG/node_modules/.bin/tsx" ]; then # --tsconfig, explicitly: cli.ts imports its neighbours through the @/ alias, and tsx # resolves that from the tsconfig it finds relative to the CALLER's working directory — diff --git a/graph/test/java/run-tests.sh b/graph/test/java/run-tests.sh index 413a7600..a5a14149 100755 --- a/graph/test/java/run-tests.sh +++ b/graph/test/java/run-tests.sh @@ -124,6 +124,19 @@ if ! bash "$ROOT/graph/test/tools/bundle-test.sh"; then echo "aborting: the bundle stage does not produce the documented output" exit 1 fi +# ── The engine id and the packaged-engine path ─────────────────────────────── +# A machine without souffle finds its binary by the id the rules hash to, so the id must be +# the same from any path and different for any rule change; and the engine package npm +# installed must be used only when its ENGINE_ID matches. Both run without souffle or +# network, in seconds. +if ! bash "$ROOT/graph/test/tools/engine-id-test.sh"; then + echo "aborting: the engine id is not a function of the rules alone" + exit 1 +fi +if ! bash "$ROOT/graph/test/tools/engine-package-test.sh"; then + echo "aborting: the packaged-engine path does not check what it runs" + exit 1 +fi PARSER="${AXIOM_PARSER:-$ROOT/parser/dist/index.js}" WORK="$HERE/.work" BLESS=0; KEEP=0; ORACLE=0; FILTERS=() diff --git a/graph/test/tools/engine-id-locale-test.sh b/graph/test/tools/engine-id-locale-test.sh index 732e1ff1..7d86aacf 100755 --- a/graph/test/tools/engine-id-locale-test.sh +++ b/graph/test/tools/engine-id-locale-test.sh @@ -68,11 +68,7 @@ RS="$ROOT/graph/pipeline/run-souffle.sh" # a different guard for a different line, so it reported the pin present after the pin had # been deleted. That is the same shape of defect this whole test exists for: a check that # cannot fail is not a check. -# The program is assembled inside write_program() where that exists, and inline under the -# "generate combined program" marker where it does not. Accept either, so this keeps -# measuring the guard across both shapes of the executor. wp="$(grep -n 'write_program()' "$RS" | head -1 | cut -d: -f1)" -[ -n "$wp" ] || wp="$(grep -n 'generate combined program' "$RS" | head -1 | cut -d: -f1)" gl="$(awk -v s="${wp:-1}" 'NR>s && /for f in "\$ENG/ {print NR; exit}' "$RS")" if [ -z "$wp" ] || [ -z "$gl" ]; then bad "cannot locate write_program and its first rule glob in run-souffle.sh; this test no longer measures anything" diff --git a/graph/test/tools/engine-id-test.sh b/graph/test/tools/engine-id-test.sh new file mode 100755 index 00000000..700058bd --- /dev/null +++ b/graph/test/tools/engine-id-test.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# The engine id (run-souffle.sh --print-engine-id) is what lets a machine WITHOUT souffle +# find the binary CI built for its rules, so two properties are load-bearing: +# 1. PATH-INDEPENDENT — the same tree at another path gives the same id (CI's checkout is +# never at the user's path); +# 2. RULE-SENSITIVE — one character changed in one rule file changes it, in every language, +# whether the file is a rule, a map, the manifest, or the pinned souffle version. +# Also: the emitted program contains no absolute path, and neither mode needs souffle. +# ───────────────────────────────────────────────────────────────────────────── +set -u +ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" # the repository root, found by its marker +RUN="graph/pipeline/run-souffle.sh" +fail=0; bad(){ echo " ✗ $*"; fail=$((fail+1)); } +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT + +# a copy of the tree, at a different path, with souffle hidden from PATH +mkdir -p "$W/copy" +cp -R "$ROOT/graph" "$W/copy/graph"; cp "$ROOT/package.json" "$W/copy/package.json" + +# SOUFFLE IS HIDDEN BY REMOVING ITS DIRECTORY, not by rebuilding a minimal PATH from a +# whitelist of symlinks. The whitelist cannot work on macOS: /usr/bin/shasum is a perl +# script and the system perl dispatches on the script's CANONICAL path, so a symlink to +# it, or even a copy of it, is refused with "perl version 5.30.3 can't run ". +# The sandbox was then left with no digest tool at all -- and on a machine without +# coreutils there is no sha256sum to fall back to -- so every check failed for a reason +# that had nothing to do with the engine id. Dropping one directory hides souffle and +# leaves every other tool where the system expects to find it. +SOUFFLE_BIN="$(command -v souffle 2>/dev/null || true)" +if [ -n "$SOUFFLE_BIN" ]; then + SOUFFLE_DIR="$(cd "$(dirname "$SOUFFLE_BIN")" && pwd)" + SANDBOX_PATH="$(printf '%s' "$PATH" | tr ':' '\n' | while IFS= read -r d; do + [ -n "$d" ] || continue + rd="$(cd "$d" 2>/dev/null && pwd)" || continue + [ "$rd" = "$SOUFFLE_DIR" ] || printf '%s:' "$d" + done)" + SANDBOX_PATH="${SANDBOX_PATH%:}" +else + SANDBOX_PATH="$PATH" +fi +command -v souffle >/dev/null 2>&1 && PATH="$SANDBOX_PATH" command -v souffle >/dev/null 2>&1 \ + && { echo " ✗ sandbox PATH still finds souffle"; fail=$((fail+1)); } + +id_at(){ ( cd "$1" && PATH="$SANDBOX_PATH" bash "$RUN" --language "$2" --print-engine-id ); } + +for lang in java typescript python javascript; do + a="$(id_at "$ROOT" "$lang")"; b="$(id_at "$W/copy" "$lang")" + case "$a" in [0-9a-f]*) ;; *) bad "$lang: id is not a hex digest: '$a'";; esac + [ "$a" = "$b" ] || bad "$lang: id differs between two paths ($a vs $b)" + ( cd "$W/copy" && PATH="$SANDBOX_PATH" bash "$RUN" --language "$lang" --emit-program "$W/$lang.dl" ) + grep -q '^#include "/' "$W/$lang.dl" && bad "$lang: emitted program embeds an absolute include path" + grep -q "^#include \"$lang/souffle/decls_base.dl\"" "$W/$lang.dl" || bad "$lang: emitted program does not include $lang/souffle/decls_base.dl" + grep -q '^\.input ' "$W/$lang.dl" || bad "$lang: emitted program declares no inputs" +done + +# sensitivity: touch one thing at a time in the copy and expect a new id +before="$(id_at "$W/copy" java)" +mutate(){ # $1 = file, $2 = appended text, $3 = label + cp "$W/copy/$1" "$W/orig"; printf '%s\n' "$2" >> "$W/copy/$1" + after="$(id_at "$W/copy" java)" + [ "$after" != "$before" ] || bad "java: id unchanged after $3" + mv "$W/orig" "$W/copy/$1" +} +f="$(cd "$ROOT" && ls graph/java/engine/resolution/*.dl | head -1)" +mutate "$f" "// changed" "editing a rule file ($f)" +mutate "graph/java/templates/client-ir.map" "zz_extra_relation all-zz" "adding a staged relation" +mutate "graph/java/souffle/export_manifest.tsv" "zz_pred zz.csv" "adding an export" +mutate "graph/pipeline/engine.conf" 'SOUFFLE_VERSION="9.9"' "bumping the pinned souffle version" +[ "$(id_at "$W/copy" java)" = "$before" ] || bad "java: id did not return to its original value after the mutations were reverted" +# and a change to one language must not move another's id +tsb="$(id_at "$W/copy" typescript)"; printf '// changed\n' >> "$W/copy/$f" +[ "$(id_at "$W/copy" typescript)" = "$tsb" ] || bad "a java rule change moved the typescript id" + +if [ "$fail" -eq 0 ]; then echo "engine-id: ok (path-independent, rule-sensitive, no souffle needed)"; else echo "engine-id: $fail failure(s)"; exit 1; fi diff --git a/graph/test/tools/engine-package-test.sh b/graph/test/tools/engine-package-test.sh new file mode 100755 index 00000000..c2a25d48 --- /dev/null +++ b/graph/test/tools/engine-package-test.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# The no-souffle path of run-souffle.sh: with `souffle` absent it must find the engine that +# npm installed for this machine — node_modules/@axiomcode/engine--// — use it +# ONLY when that package's ENGINE_ID equals the id of the rules in the checkout, run it +# through to the bundle, and otherwise refuse with the two ways out named. +# +# No network, no npm: a copy of the tree gets a hand-made engine package, and the "engine" +# in it is a shell script that writes the export manifest's files into -D (which is all the +# driver and the bundle stage need from it). +# ───────────────────────────────────────────────────────────────────────────── +set -u +ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")" # the repository root, found by its marker +fail=0; bad(){ echo " ✗ $*"; fail=$((fail+1)); } +[ -x "$ROOT/node_modules/.bin/tsx" ] || { echo "engine-package: SKIP (no node_modules/.bin/tsx — run npm install)"; exit 0; } +W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT +mkdir -p "$W/bin" "$W/ir" "$W/int" "$W/out" "$W/tree" +# a copy of the tree with its own node_modules dir (the real one linked in for the bundler) +cp -R "$ROOT/graph" "$W/tree/graph"; cp "$ROOT/package.json" "$ROOT/tsconfig.json" "$W/tree/" +mkdir -p "$W/tree/node_modules"; ln -s "$ROOT/node_modules/.bin" "$W/tree/node_modules/.bin" +for d in "$ROOT"/node_modules/*/; do n="$(basename "$d")"; [ "$n" = "@axiomcode" ] && continue; ln -s "${d%/}" "$W/tree/node_modules/$n"; done +RUN="$W/tree/graph/pipeline/run-souffle.sh" +# a PATH with everything the driver and the bundler need, and no souffle +# SOUFFLE IS HIDDEN BY REMOVING ITS DIRECTORY from PATH, not by rebuilding a minimal +# PATH from a whitelist of symlinks. The whitelist cannot work on macOS: /usr/bin/shasum +# is a perl script and the system perl dispatches on the script's CANONICAL path, so a +# symlink to it, or a copy of it, is refused with "perl version 5.30.3 can't run ". +# The sandbox was then left with no digest tool, the library cache key came back empty, +# and the run refused to proceed -- a failure with nothing to do with packaging. +# Same reasoning as engine-id-test.sh; keep the two in step. +SOUFFLE_BIN="$(command -v souffle 2>/dev/null || true)" +if [ -n "$SOUFFLE_BIN" ]; then + SOUFFLE_DIR="$(cd "$(dirname "$SOUFFLE_BIN")" && pwd)" + SANDBOX_PATH="$(printf '%s' "$PATH" | tr ':' '\n' | while IFS= read -r d; do + [ -n "$d" ] || continue + rd="$(cd "$d" 2>/dev/null && pwd)" || continue + [ "$rd" = "$SOUFFLE_DIR" ] || printf '%s:' "$d" + done)" + SANDBOX_PATH="${SANDBOX_PATH%:}" +else + SANDBOX_PATH="$PATH" +fi +for t in ; do + p="$(command -v "$t" 2>/dev/null)" && ln -sf "$p" "$W/bin/$t" +done +. "$ROOT/graph/pipeline/engine.conf" + +lang=java +id="$(PATH="$SANDBOX_PATH" bash "$RUN" --language $lang --print-engine-id)" +arch="$(uname -m | sed 's/aarch64/arm64/;s/amd64|x86_64/x64/;s/x86_64/x64/')" +case "$(uname -s)" in Darwin) platform="darwin-$arch";; Linux) platform="linux-$arch";; *) platform="win32-x64";; esac +pkg="$W/tree/node_modules/$ENGINE_PACKAGE_SCOPE/engine-$platform"; mkdir -p "$pkg/$lang" +# the fake engine: writes every manifest file, empty, into -D +{ + echo '#!/usr/bin/env bash' + echo 'while [ $# -gt 0 ]; do case "$1" in -D) D="$2"; shift 2;; -F) shift 2;; *) shift;; esac; done' + cut -f2 "$ROOT/graph/$lang/souffle/export_manifest.tsv" | sed 's|^|: > "$D/|; s|$|"|' +} > "$pkg/$lang/axiomcode-engine-$lang"; chmod +x "$pkg/$lang/axiomcode-engine-$lang" +printf '%s\n' "$id" > "$pkg/$lang/ENGINE_ID" + +run(){ PATH="$SANDBOX_PATH" AXIOM_SOUFFLE_CACHE="$W/cache" bash "$RUN" --language $lang --client-ir "$W/ir" --library "" --intermediate "$W/int" --output "$W/out" > "$W/log" 2>&1; } + +# 1. the packaged engine with a matching id is used, and the run reaches the bundle +if run; then + grep -q "using packaged engine" "$W/log" || bad "packaged engine with a matching id was not used" + [ -f "$W/out/graph.sqlite" ] || [ -f "$W/out/csv/call_edges.csv" ] || bad "the run did not reach the bundle stage" +else + bad "run with a matching packaged engine failed:"; tail -8 "$W/log" | sed 's/^/ /' +fi +# 2. a package built from OTHER rules is reported and refused; no souffle → the two ways out +printf 'deadbeef%s\n' "${id:8}" > "$pkg/$lang/ENGINE_ID"; rm -rf "$W/out" +if run; then bad "a packaged engine with a different id was used"; else + grep -q "not using it" "$W/log" || bad "a stale packaged engine was not reported" + grep -q "npm install" "$W/log" && grep -q "install souffle" "$W/log" || bad "the refusal does not name both ways out" +fi +# 3. no package at all → the same explanation +rm -rf "$W/tree/node_modules/$ENGINE_PACKAGE_SCOPE" "$W/out" +if run; then bad "a run with no engine package and no souffle succeeded"; else + grep -q "npm install" "$W/log" || bad "the no-package error does not point at npm install" +fi + +if [ "$fail" -eq 0 ]; then echo "engine-package: ok (packaged engine by id, stale package refused, absence explained)"; else echo "engine-package: $fail failure(s)"; exit 1; fi diff --git a/package.json b/package.json index 65c99857..500d357b 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,12 @@ "parser" ], "license": "FSL-1.1-Apache-2.0", + "optionalDependencies": { + "@axiomcode/engine-darwin-arm64": "0.1.0", + "@axiomcode/engine-linux-x64": "0.1.0", + "@axiomcode/engine-linux-arm64": "0.1.0", + "@axiomcode/engine-win32-x64": "0.1.0" + }, "bin": { "axiomcode": "bin/axiomcode" }, diff --git a/packaging/assemble-engine-package.sh b/packaging/assemble-engine-package.sh new file mode 100755 index 00000000..4dab479a --- /dev/null +++ b/packaging/assemble-engine-package.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Assemble one @axiomcode/engine-- npm package from the compiled engines. +# assemble-engine-package.sh +# holds /axiomcode-engine-[.exe] and /ENGINE_ID for every +# language CI built for that platform. The package carries them verbatim plus a package.json +# whose os/cpu fields let npm install it only on a matching machine. +set -eu +platform="$1"; version="$2"; src="$3"; out="$4" +os="${platform%%-*}"; cpu="${platform#*-}" +HERE="$(cd "$(dirname "$0")" && pwd)" +. "$HERE/../graph/pipeline/engine.conf" +rm -rf "$out"; mkdir -p "$out" +cp -R "$src"/. "$out/" +langs="$(ls -d "$out"/*/ | xargs -n1 basename | tr '\n' ' ')" +sed -e "s|@@SCOPE@@|$ENGINE_PACKAGE_SCOPE|g" -e "s|@@PLATFORM@@|$platform|g" -e "s|@@VERSION@@|$version|g" \ + -e "s|@@OS@@|$os|g" -e "s|@@CPU@@|$cpu|g" -e "s|@@LANGS@@|${langs% }|g" -e "s|@@SOUFFLE@@|$SOUFFLE_VERSION|g" \ + "$HERE/engine-package.json" > "$out/package.json" +cp "$HERE/../LICENSE.md" "$out/LICENSE.md" +{ echo "# $ENGINE_PACKAGE_SCOPE/engine-$platform"; echo + echo "Prebuilt AxiomCode code-graph engines for $os/$cpu: ${langs% }. Installed automatically as an" + echo "optional dependency of the code-graph package on a matching machine; not meant to be used directly." + echo; for l in $langs; do echo "- $l: rules id \`$(cat "$out/$l/ENGINE_ID")\`"; done +} > "$out/README.md" +chmod +x "$out"/*/axiomcode-engine-* 2>/dev/null || true +echo "assembled $out: $(ls "$out" | tr '\n' ' ')" diff --git a/packaging/engine-package.json b/packaging/engine-package.json new file mode 100644 index 00000000..e63533df --- /dev/null +++ b/packaging/engine-package.json @@ -0,0 +1,11 @@ +{ + "name": "@@SCOPE@@/engine-@@PLATFORM@@", + "version": "@@VERSION@@", + "description": "AxiomCode code-graph engines (@@LANGS@@) compiled for @@OS@@/@@CPU@@ with Soufflé @@SOUFFLE@@. Installed as an optional dependency of the code-graph package; npm selects this package by os/cpu.", + "license": "FSL-1.1-Apache-2.0", + "os": ["@@OS@@"], + "cpu": ["@@CPU@@"], + "files": ["*/axiomcode-engine-*", "*/ENGINE_ID", "README.md", "LICENSE.md"], + "repository": { "type": "git", "url": "git+https://github.com/AxiomCodeAI/axiom-code-graph.git" }, + "publishConfig": { "access": "public" } +}