build: engines ship on npm as @axiomcode/engine-<os>-<cpu>

.gitignore

@@ -10,6 +10,8 @@
 # ── Dependencies ─────────────────────────────────────────────────────────────
 node_modules
 node_modules/
+# and the bare name: `node_modules/` matches a directory only, so a symlink named node_modules slips past it
+node_modules
 jspm_packages/
 web_modules/
 .pnp

bin/axiomcode

@@ -26,7 +26,8 @@
 #   bin/axiomcode engine --language L --client-ir <ir-dir>/<lang> --out <dir> [options]
 #   bin/axiomcode test   [java|typescript|python|javascript|parser|all] [suite options]
 #
-# Requires Node ≥ 22.5; no Soufflé or compiler (binaries/, or a local souffle if present).
+# Requires Node ≥ 22.5; no Soufflé or compiler (the engine comes from npm as @axiomcode/engine-<os>-<cpu>,
+# or is compiled locally when souffle is present).
 # ─────────────────────────────────────────────────────────────────────────────
 set -eu
 # RESOLVE $0 THROUGH SYMLINKS BEFORE THE WALK. npm installs a `bin` entry as a link in

graph/pipeline/engine.conf

@@ -0,0 +1,15 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# The prebuilt-engine contract. Sourced by run-souffle.sh and by the CI workflows.
+#
+# SOUFFLE_VERSION is PINNED here, not read from a `souffle --version`, because the machine
+# that runs a prebuilt binary has no souffle to ask — and the engine id it computes must be
+# the id CI computed. Bumping it changes every language's id, which is what a new code
+# generator should do. CI installs exactly this version.
+#
+# ENGINE_PACKAGE_SCOPE is the npm scope the engine packages are published under:
+# <scope>/engine-<os>-<cpu> (darwin-arm64, linux-x64, linux-arm64, win32-x64), each holding
+# every language's engine for that platform under <lang>/ with its ENGINE_ID. This package
+# lists them as optionalDependencies, so `npm install` fetches the one for the machine.
+# ─────────────────────────────────────────────────────────────────────────────
+SOUFFLE_VERSION="2.5"
+ENGINE_PACKAGE_SCOPE="@axiomcode"

graph/pipeline/portable-stat.sh

@@ -52,3 +52,13 @@ sha1_stdin(){
   [ -n "$_SHA1_CMD" ] || { echo "neither shasum nor sha1sum is on PATH" >&2; return 1; }
   "$_SHA1_CMD" | cut -d' ' -f1
 }
+
+# sha256 of stdin, for the engine id. Same three spellings as sha1 above: `shasum -a 256`
+# (macOS, perl shasum in Git Bash) or `sha256sum` (coreutils).
+if command -v sha256sum >/dev/null 2>&1;  then _SHA256_CMD="sha256sum"
+elif command -v shasum >/dev/null 2>&1;   then _SHA256_CMD="shasum -a 256"
+else _SHA256_CMD=""; fi
+sha256_stdin(){
+  [ -n "$_SHA256_CMD" ] || { echo "neither sha256sum nor shasum is on PATH" >&2; return 1; }
+  $_SHA256_CMD | cut -d' ' -f1
+}

graph/test/java/run-tests.sh

@@ -124,6 +124,19 @@ if ! bash "$ROOT/graph/test/tools/bundle-test.sh"; then
   echo "aborting: the bundle stage does not produce the documented output"
   exit 1
 fi
+# ── The engine id and the packaged-engine path ───────────────────────────────
+# A machine without souffle finds its binary by the id the rules hash to, so the id must be
+# the same from any path and different for any rule change; and the engine package npm
+# installed must be used only when its ENGINE_ID matches. Both run without souffle or
+# network, in seconds.
+if ! bash "$ROOT/graph/test/tools/engine-id-test.sh"; then
+  echo "aborting: the engine id is not a function of the rules alone"
+  exit 1
+fi
+if ! bash "$ROOT/graph/test/tools/engine-package-test.sh"; then
+  echo "aborting: the packaged-engine path does not check what it runs"
+  exit 1
+fi
 PARSER="${AXIOM_PARSER:-$ROOT/parser/dist/index.js}"
 WORK="$HERE/.work"
 BLESS=0; KEEP=0; ORACLE=0; FILTERS=()

graph/test/tools/engine-id-locale-test.sh

@@ -68,11 +68,7 @@ RS="$ROOT/graph/pipeline/run-souffle.sh"
 # a different guard for a different line, so it reported the pin present after the pin had
 # been deleted. That is the same shape of defect this whole test exists for: a check that
 # cannot fail is not a check.
-# The program is assembled inside write_program() where that exists, and inline under the
-# "generate combined program" marker where it does not. Accept either, so this keeps
-# measuring the guard across both shapes of the executor.
 wp="$(grep -n 'write_program()' "$RS" | head -1 | cut -d: -f1)"
-[ -n "$wp" ] || wp="$(grep -n 'generate combined program' "$RS" | head -1 | cut -d: -f1)"
 gl="$(awk -v s="${wp:-1}" 'NR>s && /for f in "\$ENG/ {print NR; exit}' "$RS")"
 if [ -z "$wp" ] || [ -z "$gl" ]; then
   bad "cannot locate write_program and its first rule glob in run-souffle.sh; this test no longer measures anything"

graph/test/tools/engine-id-test.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────────────────────
+# The engine id (run-souffle.sh --print-engine-id) is what lets a machine WITHOUT souffle
+# find the binary CI built for its rules, so two properties are load-bearing:
+#   1. PATH-INDEPENDENT — the same tree at another path gives the same id (CI's checkout is
+#      never at the user's path);
+#   2. RULE-SENSITIVE — one character changed in one rule file changes it, in every language,
+#      whether the file is a rule, a map, the manifest, or the pinned souffle version.
+# Also: the emitted program contains no absolute path, and neither mode needs souffle.
+# ─────────────────────────────────────────────────────────────────────────────
+set -u
+ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")"  # the repository root, found by its marker
+RUN="graph/pipeline/run-souffle.sh"
+fail=0; bad(){ echo "  ✗ $*"; fail=$((fail+1)); }
+W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT
+
+# a copy of the tree, at a different path, with souffle hidden from PATH
+mkdir -p "$W/copy"
+cp -R "$ROOT/graph" "$W/copy/graph"; cp "$ROOT/package.json" "$W/copy/package.json"
+
+# SOUFFLE IS HIDDEN BY REMOVING ITS DIRECTORY, not by rebuilding a minimal PATH from a
+# whitelist of symlinks. The whitelist cannot work on macOS: /usr/bin/shasum is a perl
+# script and the system perl dispatches on the script's CANONICAL path, so a symlink to
+# it, or even a copy of it, is refused with "perl version 5.30.3 can't run <path>".
+# The sandbox was then left with no digest tool at all -- and on a machine without
+# coreutils there is no sha256sum to fall back to -- so every check failed for a reason
+# that had nothing to do with the engine id. Dropping one directory hides souffle and
+# leaves every other tool where the system expects to find it.
+SOUFFLE_BIN="$(command -v souffle 2>/dev/null || true)"
+if [ -n "$SOUFFLE_BIN" ]; then
+  SOUFFLE_DIR="$(cd "$(dirname "$SOUFFLE_BIN")" && pwd)"
+  SANDBOX_PATH="$(printf '%s' "$PATH" | tr ':' '\n' | while IFS= read -r d; do
+    [ -n "$d" ] || continue
+    rd="$(cd "$d" 2>/dev/null && pwd)" || continue
+    [ "$rd" = "$SOUFFLE_DIR" ] || printf '%s:' "$d"
+  done)"
+  SANDBOX_PATH="${SANDBOX_PATH%:}"
+else
+  SANDBOX_PATH="$PATH"
+fi
+command -v souffle >/dev/null 2>&1 && PATH="$SANDBOX_PATH" command -v souffle >/dev/null 2>&1 \
+  && { echo "  ✗ sandbox PATH still finds souffle"; fail=$((fail+1)); }
+
+id_at(){ ( cd "$1" && PATH="$SANDBOX_PATH" bash "$RUN" --language "$2" --print-engine-id ); }
+
+for lang in java typescript python javascript; do
+  a="$(id_at "$ROOT" "$lang")"; b="$(id_at "$W/copy" "$lang")"
+  case "$a" in [0-9a-f]*) ;; *) bad "$lang: id is not a hex digest: '$a'";; esac
+  [ "$a" = "$b" ] || bad "$lang: id differs between two paths ($a vs $b)"
+  ( cd "$W/copy" && PATH="$SANDBOX_PATH" bash "$RUN" --language "$lang" --emit-program "$W/$lang.dl" )
+  grep -q '^#include "/' "$W/$lang.dl" && bad "$lang: emitted program embeds an absolute include path"
+  grep -q "^#include \"$lang/souffle/decls_base.dl\"" "$W/$lang.dl" || bad "$lang: emitted program does not include $lang/souffle/decls_base.dl"
+  grep -q '^\.input ' "$W/$lang.dl" || bad "$lang: emitted program declares no inputs"
+done
+
+# sensitivity: touch one thing at a time in the copy and expect a new id
+before="$(id_at "$W/copy" java)"
+mutate(){ # $1 = file, $2 = appended text, $3 = label
+  cp "$W/copy/$1" "$W/orig"; printf '%s\n' "$2" >> "$W/copy/$1"
+  after="$(id_at "$W/copy" java)"
+  [ "$after" != "$before" ] || bad "java: id unchanged after $3"
+  mv "$W/orig" "$W/copy/$1"
+}
+f="$(cd "$ROOT" && ls graph/java/engine/resolution/*.dl | head -1)"
+mutate "$f" "// changed" "editing a rule file ($f)"
+mutate "graph/java/templates/client-ir.map" "zz_extra_relation	all-zz" "adding a staged relation"
+mutate "graph/java/souffle/export_manifest.tsv" "zz_pred	zz.csv" "adding an export"
+mutate "graph/pipeline/engine.conf" 'SOUFFLE_VERSION="9.9"' "bumping the pinned souffle version"
+[ "$(id_at "$W/copy" java)" = "$before" ] || bad "java: id did not return to its original value after the mutations were reverted"
+# and a change to one language must not move another's id
+tsb="$(id_at "$W/copy" typescript)"; printf '// changed\n' >> "$W/copy/$f"
+[ "$(id_at "$W/copy" typescript)" = "$tsb" ] || bad "a java rule change moved the typescript id"
+
+if [ "$fail" -eq 0 ]; then echo "engine-id: ok (path-independent, rule-sensitive, no souffle needed)"; else echo "engine-id: $fail failure(s)"; exit 1; fi

graph/test/tools/engine-package-test.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────────────────────
+# The no-souffle path of run-souffle.sh: with `souffle` absent it must find the engine that
+# npm installed for this machine — node_modules/@axiomcode/engine-<os>-<cpu>/<lang>/ — use it
+# ONLY when that package's ENGINE_ID equals the id of the rules in the checkout, run it
+# through to the bundle, and otherwise refuse with the two ways out named.
+#
+# No network, no npm: a copy of the tree gets a hand-made engine package, and the "engine"
+# in it is a shell script that writes the export manifest's files into -D (which is all the
+# driver and the bundle stage need from it).
+# ─────────────────────────────────────────────────────────────────────────────
+set -u
+ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")"  # the repository root, found by its marker
+fail=0; bad(){ echo "  ✗ $*"; fail=$((fail+1)); }
+[ -x "$ROOT/node_modules/.bin/tsx" ] || { echo "engine-package: SKIP (no node_modules/.bin/tsx — run npm install)"; exit 0; }
+W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT
+mkdir -p "$W/bin" "$W/ir" "$W/int" "$W/out" "$W/tree"
+# a copy of the tree with its own node_modules dir (the real one linked in for the bundler)
+cp -R "$ROOT/graph" "$W/tree/graph"; cp "$ROOT/package.json" "$ROOT/tsconfig.json" "$W/tree/"
+mkdir -p "$W/tree/node_modules"; ln -s "$ROOT/node_modules/.bin" "$W/tree/node_modules/.bin"
+for d in "$ROOT"/node_modules/*/; do n="$(basename "$d")"; [ "$n" = "@axiomcode" ] && continue; ln -s "${d%/}" "$W/tree/node_modules/$n"; done
+RUN="$W/tree/graph/pipeline/run-souffle.sh"
+# a PATH with everything the driver and the bundler need, and no souffle
+# SOUFFLE IS HIDDEN BY REMOVING ITS DIRECTORY from PATH, not by rebuilding a minimal
+# PATH from a whitelist of symlinks. The whitelist cannot work on macOS: /usr/bin/shasum
+# is a perl script and the system perl dispatches on the script's CANONICAL path, so a
+# symlink to it, or a copy of it, is refused with "perl version 5.30.3 can't run <path>".
+# The sandbox was then left with no digest tool, the library cache key came back empty,
+# and the run refused to proceed -- a failure with nothing to do with packaging.
+# Same reasoning as engine-id-test.sh; keep the two in step.
+SOUFFLE_BIN="$(command -v souffle 2>/dev/null || true)"
+if [ -n "$SOUFFLE_BIN" ]; then
+  SOUFFLE_DIR="$(cd "$(dirname "$SOUFFLE_BIN")" && pwd)"
+  SANDBOX_PATH="$(printf '%s' "$PATH" | tr ':' '\n' | while IFS= read -r d; do
+    [ -n "$d" ] || continue
+    rd="$(cd "$d" 2>/dev/null && pwd)" || continue
+    [ "$rd" = "$SOUFFLE_DIR" ] || printf '%s:' "$d"
+  done)"
+  SANDBOX_PATH="${SANDBOX_PATH%:}"
+else
+  SANDBOX_PATH="$PATH"
+fi
+for t in ; do
+  p="$(command -v "$t" 2>/dev/null)" && ln -sf "$p" "$W/bin/$t"
+done
+. "$ROOT/graph/pipeline/engine.conf"
+
+lang=java
+id="$(PATH="$SANDBOX_PATH" bash "$RUN" --language $lang --print-engine-id)"
+arch="$(uname -m | sed 's/aarch64/arm64/;s/amd64|x86_64/x64/;s/x86_64/x64/')"
+case "$(uname -s)" in Darwin) platform="darwin-$arch";; Linux) platform="linux-$arch";; *) platform="win32-x64";; esac
+pkg="$W/tree/node_modules/$ENGINE_PACKAGE_SCOPE/engine-$platform"; mkdir -p "$pkg/$lang"
+# the fake engine: writes every manifest file, empty, into -D
+{
+  echo '#!/usr/bin/env bash'
+  echo 'while [ $# -gt 0 ]; do case "$1" in -D) D="$2"; shift 2;; -F) shift 2;; *) shift;; esac; done'
+  cut -f2 "$ROOT/graph/$lang/souffle/export_manifest.tsv" | sed 's|^|: > "$D/|; s|$|"|'
+} > "$pkg/$lang/axiomcode-engine-$lang"; chmod +x "$pkg/$lang/axiomcode-engine-$lang"
+printf '%s\n' "$id" > "$pkg/$lang/ENGINE_ID"
+
+run(){ PATH="$SANDBOX_PATH" AXIOM_SOUFFLE_CACHE="$W/cache" bash "$RUN" --language $lang --client-ir "$W/ir" --library "" --intermediate "$W/int" --output "$W/out" > "$W/log" 2>&1; }
+
+# 1. the packaged engine with a matching id is used, and the run reaches the bundle
+if run; then
+  grep -q "using packaged engine" "$W/log" || bad "packaged engine with a matching id was not used"
+  [ -f "$W/out/graph.sqlite" ] || [ -f "$W/out/csv/call_edges.csv" ] || bad "the run did not reach the bundle stage"
+else
+  bad "run with a matching packaged engine failed:"; tail -8 "$W/log" | sed 's/^/      /'
+fi
+# 2. a package built from OTHER rules is reported and refused; no souffle → the two ways out
+printf 'deadbeef%s\n' "${id:8}" > "$pkg/$lang/ENGINE_ID"; rm -rf "$W/out"
+if run; then bad "a packaged engine with a different id was used"; else
+  grep -q "not using it" "$W/log" || bad "a stale packaged engine was not reported"
+  grep -q "npm install" "$W/log" && grep -q "install souffle" "$W/log" || bad "the refusal does not name both ways out"
+fi
+# 3. no package at all → the same explanation
+rm -rf "$W/tree/node_modules/$ENGINE_PACKAGE_SCOPE" "$W/out"
+if run; then bad "a run with no engine package and no souffle succeeded"; else
+  grep -q "npm install" "$W/log" || bad "the no-package error does not point at npm install"
+fi
+
+if [ "$fail" -eq 0 ]; then echo "engine-package: ok (packaged engine by id, stale package refused, absence explained)"; else echo "engine-package: $fail failure(s)"; exit 1; fi

package.json

@@ -26,6 +26,12 @@
     "parser"
   ],
   "license": "FSL-1.1-Apache-2.0",
+  "optionalDependencies": {
+    "@axiomcode/engine-darwin-arm64": "0.1.0",
+    "@axiomcode/engine-linux-x64": "0.1.0",
+    "@axiomcode/engine-linux-arm64": "0.1.0",
+    "@axiomcode/engine-win32-x64": "0.1.0"
+  },
   "bin": {
     "axiomcode": "bin/axiomcode"
   },

packaging/assemble-engine-package.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Assemble one @axiomcode/engine-<os>-<cpu> npm package from the compiled engines.
+#   assemble-engine-package.sh <platform> <version> <engines-dir> <out-dir>
+# <engines-dir> holds <lang>/axiomcode-engine-<lang>[.exe] and <lang>/ENGINE_ID for every
+# language CI built for that platform. The package carries them verbatim plus a package.json
+# whose os/cpu fields let npm install it only on a matching machine.
+set -eu
+platform="$1"; version="$2"; src="$3"; out="$4"
+os="${platform%%-*}"; cpu="${platform#*-}"
+HERE="$(cd "$(dirname "$0")" && pwd)"
+. "$HERE/../graph/pipeline/engine.conf"
+rm -rf "$out"; mkdir -p "$out"
+cp -R "$src"/. "$out/"
+langs="$(ls -d "$out"/*/ | xargs -n1 basename | tr '\n' ' ')"
+sed -e "s|@@SCOPE@@|$ENGINE_PACKAGE_SCOPE|g" -e "s|@@PLATFORM@@|$platform|g" -e "s|@@VERSION@@|$version|g" \
+    -e "s|@@OS@@|$os|g" -e "s|@@CPU@@|$cpu|g" -e "s|@@LANGS@@|${langs% }|g" -e "s|@@SOUFFLE@@|$SOUFFLE_VERSION|g" \
+    "$HERE/engine-package.json" > "$out/package.json"
+cp "$HERE/../LICENSE.md" "$out/LICENSE.md"
+{ echo "# $ENGINE_PACKAGE_SCOPE/engine-$platform"; echo
+  echo "Prebuilt AxiomCode code-graph engines for $os/$cpu: ${langs% }. Installed automatically as an"
+  echo "optional dependency of the code-graph package on a matching machine; not meant to be used directly."
+  echo; for l in $langs; do echo "- $l: rules id \`$(cat "$out/$l/ENGINE_ID")\`"; done
+} > "$out/README.md"
+chmod +x "$out"/*/axiomcode-engine-* 2>/dev/null || true
+echo "assembled $out: $(ls "$out" | tr '\n' ' ')"

packaging/engine-package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@@SCOPE@@/engine-@@PLATFORM@@",
+  "version": "@@VERSION@@",
+  "description": "AxiomCode code-graph engines (@@LANGS@@) compiled for @@OS@@/@@CPU@@ with Soufflé @@SOUFFLE@@. Installed as an optional dependency of the code-graph package; npm selects this package by os/cpu.",
+  "license": "FSL-1.1-Apache-2.0",
+  "os": ["@@OS@@"],
+  "cpu": ["@@CPU@@"],
+  "files": ["*/axiomcode-engine-*", "*/ENGINE_ID", "README.md", "LICENSE.md"],
+  "repository": { "type": "git", "url": "git+https://github.com/AxiomCodeAI/axiom-code-graph.git" },
+  "publishConfig": { "access": "public" }
+}