From c6501adf329282d1820650bce362517ef2ac1374 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 25 Sep 2026 11:27:43 -0700 Subject: [PATCH] =?UTF-8?q?engines:=20ship=20the=20query=20programs=20comp?= =?UTF-8?q?iled,=20so=20impact=20and=20path=20need=20no=20Souffl=C3=A9=20(?= =?UTF-8?q?#1330)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build-engines compiles impact.dl, path.dl, path-opt.dl and path-every.dl on all five platforms next to the language engines, keyed by dl_program.py's own rules id, and checks each platform's binaries against the interpreter on a small fixture. The engine package ships them under queries/, dl_program.py uses the one built from these rules before it tries a local compile, and publish refuses a platform that lacks them. The nightly e2e now indexes, runs impact and path from the installed tarballs with no souffle on PATH. A local dl/.cache no longer reaches the npm package. --- .github/scripts/e2e-install.sh | 35 ++++++++- .github/scripts/query-smoke.sh | 60 +++++++++++++++ .github/workflows/build-engines.yml | 48 +++++++++++- .github/workflows/ci.yml | 2 +- .github/workflows/publish-npm.yml | 9 +++ package.json | 1 + packaging/assemble-engine-package.sh | 12 ++- packaging/engine-package.json | 2 +- .../skills/axiomcode/scripts/axiomcode-impact | 5 +- .../skills/axiomcode/scripts/axiomcode-path | 3 +- .../skills/axiomcode/scripts/dl_program.py | 74 +++++++++++++++++-- 11 files changed, 233 insertions(+), 18 deletions(-) create mode 100755 .github/scripts/query-smoke.sh diff --git a/.github/scripts/e2e-install.sh b/.github/scripts/e2e-install.sh index 5205c0ab..c98b5179 100755 --- a/.github/scripts/e2e-install.sh +++ b/.github/scripts/e2e-install.sh @@ -8,7 +8,9 @@ # + /ENGINE_ID). This packs @axiomcode/code-graph and that platform's engine # package exactly as publish-npm.yml would, installs both into an empty project, and # runs `axiomcode` on with souffle NOT on PATH. It passes only if the -# installed package found its engine package, used it, and wrote a graph with edges. +# installed package found its engine package, used it, and wrote a graph with edges — +# and then indexed that source and answered `impact` and `path` from the query programs +# the engine package ships (#1330), still with no souffle to compile or interpret them. # # The test suites cannot see any of this: they run from the checkout, where a missing # `files` entry, a broken bin, or an engine package the driver does not find all go @@ -58,4 +60,33 @@ edges="$(node -e ' console.log(db.prepare("SELECT COUNT(*) AS n FROM call_edges").get().n); ' "$db" 2>/dev/null)" [ -n "$edges" ] && [ "$edges" -gt 0 ] || fail "the graph has no call edges (${edges:-unreadable})" -echo "e2e: ok — installed from the tarballs, used the packaged $platform engine, $edges call edges" + +echo "── axiomcode index, impact and path on a copy of $(basename "$src"), no souffle" +cp -R "$src" "$W/repo" +PATH="$SANDBOX_PATH" "$bin" index "$W/repo" > "$W/index.log" 2>&1 || { tail -15 "$W/index.log"; fail "axiomcode index exited non-zero"; } +ready="$(grep 'datalog rules ready' "$W/index.log" || true)"; echo " $ready" +# every program from the engine package: not compiled here (no souffle), not left to the interpreter +echo "$ready" | grep -Eq 'ready: ([0-9]+)/\1 compiled' && ! echo "$ready" | grep -Eq '=(cached|interpreter)' \ + || fail "the query programs did not all come from the engine package: $ready" +# one resolved call in the graph: its caller and its callee, by simple name +pair="$(node -e ' + const { DatabaseSync } = require("node:sqlite"); + const db = new DatabaseSync(process.argv[1], { readOnly: true }); + const r = db.prepare(`SELECT c.name AS a, m.name AS b FROM call_edges e JOIN methods m ON m.id = e.callee_method_id + JOIN methods c ON c.id = e.caller_id WHERE e.tier = ? AND m.name NOT LIKE ? AND c.name NOT LIKE ? AND c.name <> m.name + ORDER BY c.name, m.name LIMIT 1`).get("known_edge", "%<%", "%<%"); + if (r) console.log(r.a + " " + r.b); +' "$W/repo/.axiomcode/out/graph.sqlite" 2>/dev/null)" +[ -n "$pair" ] || fail "the indexed graph has no resolved call to ask about" +caller="${pair% *}"; callee="${pair#* }" +( cd "$W/repo" && PATH="$SANDBOX_PATH" "$bin" impact "$callee" ) > "$W/impact.log" 2>&1; rc=$? +sed 's/^/ /' "$W/impact.log" | head -8 +[ "$rc" -eq 0 ] || fail "axiomcode impact $callee exited $rc" +# path answers from SQL by default; AXIOMCODE_DATALOG=1 runs the shipped path programs instead +for every in "" --every; do + ( cd "$W/repo" && PATH="$SANDBOX_PATH" AXIOMCODE_DATALOG=1 "$bin" path "$caller" "$callee" $every ) > "$W/path.log" 2>&1; rc=$? + sed 's/^/ /' "$W/path.log" | head -4 + [ "$rc" -eq 0 ] || fail "axiomcode path $caller $callee $every exited $rc" + grep -q 'reached' "$W/path.log" || fail "axiomcode path $caller $callee $every found no route over a resolved call" +done +echo "e2e: ok — installed from the tarballs, used the packaged $platform engine, $edges call edges; impact and path answered from the packaged query programs" diff --git a/.github/scripts/query-smoke.sh b/.github/scripts/query-smoke.sh new file mode 100755 index 00000000..76a358c1 --- /dev/null +++ b/.github/scripts/query-smoke.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# ───────────────────────────────────────────────────────────────────────────── +# The query programs answer the same compiled as interpreted, on every platform. +# +# query-smoke.sh expect (generate job, with Soufflé) +# query-smoke.sh check (each platform, no Soufflé) +# +# `expect` writes a small call graph as facts (fixture/) and what the Soufflé +# interpreter derives from it for every .dl (expected//). `check` runs +# each platform's axiomcode-query- on the same facts and compares, relation +# by relation. Starting on empty inputs would pass a binary that answers nothing; +# this fails one whose answers differ, and fails if the fixture reaches nothing. +# ───────────────────────────────────────────────────────────────────────────── +set -euo pipefail +mode="${1:?usage: query-smoke.sh expect|check [bin-dir]}"; G="${2:?gen-queries-dir}" +EXE=""; case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) EXE=".exe";; esac +inputs() { sed -n 's/.*\.input \([A-Za-z0-9_]*\).*/\1/p' "$1"; } +norm() { for f in "$1"/*.csv; do [ -e "$f" ] || continue; printf '== %s\n' "$(basename "$f")"; tr -d '\r' < "$f" | LC_ALL=C sort; done; } + +case "$mode" in + expect) + F="$G/fixture"; rm -rf "$F"; mkdir -p "$F" + for dl in "$G"/*.dl; do inputs "$dl" | while read -r r; do : > "$F/$r.facts"; done; done + # t -> a -> b -> c, and x -> c by name only; one path query from t to c + printf 'm:t\tm:a\tcall\nm:a\tm:b\tcall\nm:b\tm:c\tcall\n' > "$F/edge.facts" + printf 't\tm:t\na\tm:a\nb\tm:b\nc\tm:c\nx\tm:x\n' > "$F/named.facts" + printf 'm:x\tc\n' > "$F/byname.facts" + printf 'q\tm:t\n' > "$F/src.facts" + printf 'q\tm:c\n' > "$F/dst.facts" + # impact: what changing c reaches, through the same chain's resolved call sites + printf 'q\tmethod\tm:c\tc\n' > "$F/target.facts" + printf 'm:t\tm:a\tclient\tT.java\t3\nm:a\tm:b\tclient\tA.java\t4\nm:b\tm:c\tclient\tB.java\t5\n' > "$F/calls.facts" + printf 'm:a\tmethod\nm:b\tmethod\nm:c\tmethod\nm:t\tmethod\n' > "$F/kind.facts" + for dl in "$G"/*.dl; do + q="$(basename "$dl" .dl)"; E="$G/expected/$q"; rm -rf "$E"; mkdir -p "$E" + souffle -F "$F" -D "$E" "$dl" + n="$(cat "$E"/*.csv 2>/dev/null | wc -l | tr -d ' ')" + echo "query $q: interpreter derived $n rows" + done + for q in impact path; do + [ "$(cat "$G"/expected/$q/*.csv | wc -l)" -gt 0 ] || { echo "::error::the fixture reaches nothing in $q.dl"; exit 1; } + done + ;; + check) + B="${3:?bin-dir}"; W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT; bad=0 + for dl in "$G"/*.dl; do + q="$(basename "$dl" .dl)"; bin="$B/axiomcode-query-$q$EXE" + [ -f "$bin" ] || { echo "::error::no binary for $q"; bad=1; continue; } + chmod +x "$bin" 2>/dev/null || true + mkdir -p "$W/$q"; "$bin" -F "$G/fixture" -D "$W/$q" + if diff <(norm "$G/expected/$q") <(norm "$W/$q") > "$W/$q.diff"; then + echo "query $q: ok ($(norm "$W/$q" | grep -vc '^==') rows, same as the interpreter)" + else + echo "::error::query $q: the compiled program answers differently"; head -40 "$W/$q.diff"; bad=1 + fi + done + exit "$bad" + ;; + *) echo "unknown mode $mode" >&2; exit 2;; +esac diff --git a/.github/workflows/build-engines.yml b/.github/workflows/build-engines.yml index 167fbf1f..fcb18e83 100644 --- a/.github/workflows/build-engines.yml +++ b/.github/workflows/build-engines.yml @@ -9,7 +9,12 @@ # one self-contained executable per language per platform, linked against nothing but the # C++ runtime. Same flags as the local compile (no OpenMP/zlib/sqlite), portable targets. # -# Artifacts: engines-/ holding /axiomcode-engine-[.exe] + /ENGINE_ID +# Artifacts: engines-/ holding /axiomcode-engine-[.exe] + /ENGINE_ID, +# and queries/axiomcode-query-[.exe] + queries/.id for every query program the CLI +# runs (plugins/axiomcode/skills/axiomcode/scripts/dl/.dl: impact, path, path-opt, +# path-every). Without those, `axiomcode impact` needs Soufflé and a C++ compiler on the user's +# machine (#1330). A query binary's id is dl_program.py's own key, so the CLI finds it by the +# same hash it would cache a local compile under. # Platforms are named the npm way (process.platform-process.arch): darwin-arm64, darwin-x64, # linux-x64, linux-arm64, win32-x64, each on a standard GitHub-hosted runner (free for a # public repository; macOS builds on macos-15 for Apple Silicon and macos-15-intel for Intel). @@ -71,9 +76,21 @@ jobs: souffle -I graph -g "gen/$lang.cpp" "gen/$lang.dl" 2> "gen/$lang.gen.log" || { cat "gen/$lang.gen.log"; exit 1; } awk '/No rules\/facts defined/{skip=2;next} skip>0{skip--;next} {print}' "gen/$lang.gen.log" done + # the query programs: self-contained (no #include), keyed by dl_program.py itself + mkdir -p gen/queries + for dl in plugins/axiomcode/skills/axiomcode/scripts/dl/*.dl; do + q="$(basename "$dl" .dl)" + python3 plugins/axiomcode/skills/axiomcode/scripts/dl_program.py --print-id "$dl" | tr -d '\n' > "gen/queries/$q.id" + echo "query $q: $(cat "gen/queries/$q.id")" + cp "$dl" "gen/queries/$q.dl" + souffle -g "gen/queries/$q.cpp" "$dl" 2> "gen/queries/$q.gen.log" || { cat "gen/queries/$q.gen.log"; exit 1; } + done + # the fixture and what the interpreter derives from it; each platform's binaries must match + bash .github/scripts/query-smoke.sh expect gen/queries + cp .github/scripts/query-smoke.sh gen/queries/smoke.sh cp -r /usr/include/souffle gen/souffle # one key for the whole set; a partial match restores the previous set - echo "key=$(cat gen/*.id | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT" + echo "key=$(cat gen/*.id gen/queries/*.id | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT" # The compile flags live in THIS file and ENGINE_ID does not cover them, so its # hash prefixes the key AND the restore prefix: a flag change restores nothing. # Computed here because the build jobs never check the repository out. @@ -112,6 +129,13 @@ jobs: c++ -std=c++17 -O3 -w -static-libstdc++ -static-libgcc -I gen "gen/$lang.cpp" -o "engines/$lang/axiomcode-engine-$lang" cp "gen/$lang.id" "engines/$lang/ENGINE_ID" done + mkdir -p engines/queries + for cpp in gen/queries/*.cpp; do + q="$(basename "$cpp" .cpp)" + if cmp -s "gen/queries/$q.id" "engines/queries/$q.id"; then echo "query $q: cached, rules unchanged"; continue; fi + c++ -std=c++17 -O3 -w -static-libstdc++ -static-libgcc -I gen "$cpp" -o "engines/queries/axiomcode-query-$q" + cp "gen/queries/$q.id" "engines/queries/$q.id" + done ls -la engines/*; ldd engines/java/axiomcode-engine-java || true - uses: ilammy/msvc-dev-cmd@v1 if: startsWith(matrix.target.platform, 'win32') @@ -131,6 +155,17 @@ jobs: echo %%L: cached, rules unchanged ) ) + if not exist engines\queries mkdir engines\queries + for %%Q in (gen\queries\*.cpp) do ( + fc /b gen\queries\%%~nQ.id engines\queries\%%~nQ.id >nul 2>&1 + if errorlevel 1 ( + cl /nologo /std:c++17 /O2 /EHsc /bigobj /w /permissive- /Zc:__cplusplus /D_CRT_SECURE_NO_WARNINGS /DNOMINMAX /DUSE_CUSTOM_GETOPTLONG /I gen %%Q /Fe:engines\queries\axiomcode-query-%%~nQ.exe + if errorlevel 1 exit /b 1 + copy /y gen\queries\%%~nQ.id engines\queries\%%~nQ.id + ) else ( + echo query %%~nQ: cached, rules unchanged + ) + ) dir /s engines - name: Smoke — every binary starts on empty inputs shell: bash @@ -143,6 +178,7 @@ jobs: "./$bin" -F "facts-$lang" -D "out-$lang" echo "$lang: ok ($(ls out-$lang | wc -l) relations written)" done + bash gen/queries/smoke.sh check gen/queries engines/queries - name: save the engines for the next run if: ${{ !inputs.fresh && steps.restore.outputs.cache-hit != 'true' }} uses: actions/cache/save@v4 @@ -187,6 +223,13 @@ jobs: c++ -std=c++17 -O3 -w -arch "$ARCH" -mmacosx-version-min=12.0 -I gen "gen/$lang.cpp" -o "engines/$lang/axiomcode-engine-$lang" cp "gen/$lang.id" "engines/$lang/ENGINE_ID" done + mkdir -p engines/queries + for cpp in gen/queries/*.cpp; do + q="$(basename "$cpp" .cpp)" + if cmp -s "gen/queries/$q.id" "engines/queries/$q.id"; then echo "query $q: cached, rules unchanged"; continue; fi + c++ -std=c++17 -O3 -w -arch "$ARCH" -mmacosx-version-min=12.0 -I gen "$cpp" -o "engines/queries/axiomcode-query-$q" + cp "gen/queries/$q.id" "engines/queries/$q.id" + done file engines/java/axiomcode-engine-java otool -L engines/java/axiomcode-engine-java - name: Smoke — every binary starts on empty inputs @@ -197,6 +240,7 @@ jobs: sed -n 's/^\.input \([A-Za-z0-9_]*\)(.*/\1/p' "gen/$lang.dl" | while read -r r; do : > "facts-$lang/$r.facts"; done "./engines/$lang/axiomcode-engine-$lang" -F "facts-$lang" -D "out-$lang" done + bash gen/queries/smoke.sh check gen/queries engines/queries - name: save the engines for the next run if: ${{ !inputs.fresh && steps.restore.outputs.cache-hit != 'true' }} uses: actions/cache/save@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5623fa16..4ea7827e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,7 @@ jobs: code="$code$(printf '%s\n' "$files" | grep -E '^(graph|parser)/' || true)" # ENGINES: what the platform build compiles or packages. engines="$(printf '%s\n' "$files" | grep -E \ - -e '\.dl$' -e '^graph/pipeline/' -e '^packaging/' \ + -e '\.dl$' -e '^graph/pipeline/' -e '^packaging/' -e 'scripts/dl_program\.py$' -e '^\.github/scripts/query-smoke\.sh$' \ -e '^\.github/workflows/build-engines\.yml$' -e '^package\.json$' || true)" [ -n "$code" ] && echo "code=true" >> "$GITHUB_OUTPUT" || echo "code=false" >> "$GITHUB_OUTPUT" # Only a pull request INTO main builds the platform engines; into dev they wait. diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml index 8651517a..99625a60 100644 --- a/.github/workflows/publish-npm.yml +++ b/.github/workflows/publish-npm.yml @@ -103,6 +103,15 @@ jobs: echo "::warning::no engines were built for $platform" missing=1 fi + # and every query program the CLI runs, or impact/path need soufflé there (#1330) + for dl in plugins/axiomcode/skills/axiomcode/scripts/dl/*.dl; do + q="$(basename "$dl" .dl)" + if ! cmp -s <(python3 plugins/axiomcode/skills/axiomcode/scripts/dl_program.py --print-id "$dl") \ + <(cat "artifacts/engines-$platform/queries/$q.id" 2>/dev/null; echo); then + echo "::warning::$platform has no query program for these $q.dl rules" + missing=1 + fi + done done if [ "$missing" = 1 ] && [ "$DRY" != true ]; then echo "::error::a real publish needs every platform package.json pins"; exit 1 diff --git a/package.json b/package.json index 0248c3ab..7315e065 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "plugins/axiomcode/", "!plugins/axiomcode/validate/", "!plugins/**/__pycache__/", + "!plugins/**/dl/.cache/", "parser/dist/", "parser/package.json", "README.md", diff --git a/packaging/assemble-engine-package.sh b/packaging/assemble-engine-package.sh index df1c2078..baa88736 100755 --- a/packaging/assemble-engine-package.sh +++ b/packaging/assemble-engine-package.sh @@ -2,7 +2,8 @@ # Assemble one @axiomcode/engine-- npm package from the compiled engines. # assemble-engine-package.sh # holds /axiomcode-engine-[.exe] and /ENGINE_ID for every -# language CI built for that platform. The package carries them verbatim plus a package.json +# language CI built for that platform, and queries/axiomcode-query-[.exe] + queries/.id +# for every query program the CLI runs (impact, path …). The package carries them verbatim plus a package.json # whose os/cpu fields let npm install it only on a matching machine. set -eu platform="$1"; version="$2"; src="$3"; out="$4" @@ -11,7 +12,8 @@ HERE="$(cd "$(dirname "$0")" && pwd)" . "$HERE/../graph/pipeline/engine.conf" rm -rf "$out"; mkdir -p "$out" cp -R "$src"/. "$out/" -langs="$(ls -d "$out"/*/ | xargs -n1 basename | tr '\n' ' ')" +langs="$(ls -d "$out"/*/ | xargs -n1 basename | grep -vx queries | tr '\n' ' ')" +queries="$(ls "$out"/queries/*.id 2>/dev/null | xargs -n1 basename 2>/dev/null | sed 's/\.id$//' | tr '\n' ' ')" case "$platform" in darwin-arm64) label="macOS (Apple Silicon)";; darwin-x64) label="macOS (Intel)";; linux-x64) label="Linux x64";; linux-arm64) label="Linux arm64";; win32-x64) label="Windows x64";; @@ -28,6 +30,10 @@ cp "$HERE/../LICENSE.md" "$out/LICENSE.md" echo; echo "Languages: ${langs% }." echo; echo "Rules each engine was built from:"; echo for l in $langs; do echo "- $l: \`$(cat "$out/$l/ENGINE_ID")\`"; done + if [ -n "$queries" ]; then + echo; echo "Query programs (\`axiomcode impact\`, \`axiomcode path\` …), by the rules each was built from:"; echo + for q in $queries; do echo "- $q: \`$(cat "$out/queries/$q.id")\`"; done + fi } > "$out/README.md" -chmod +x "$out"/*/axiomcode-engine-* 2>/dev/null || true +chmod +x "$out"/*/axiomcode-engine-* "$out"/queries/axiomcode-query-* 2>/dev/null || true echo "assembled $out: $(ls "$out" | tr '\n' ' ')" diff --git a/packaging/engine-package.json b/packaging/engine-package.json index b8fc3b53..976d8ec3 100644 --- a/packaging/engine-package.json +++ b/packaging/engine-package.json @@ -5,7 +5,7 @@ "license": "FSL-1.1-Apache-2.0", "os": ["@@OS@@"], "cpu": ["@@CPU@@"], - "files": ["*/axiomcode-engine-*", "*/ENGINE_ID", "README.md", "LICENSE.md"], + "files": ["*/axiomcode-engine-*", "*/ENGINE_ID", "queries/axiomcode-query-*", "queries/*.id", "README.md", "LICENSE.md"], "repository": { "type": "git", "url": "git+https://github.com/AxiomCodeAI/axiomcodegraph.git" }, "publishConfig": { "access": "public" } } diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact index c98a9ba8..1417b60f 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact @@ -1237,8 +1237,9 @@ class Impact: if os.environ.get('AXIOMCODE_BACKEND'): print('backend=datalog', file=sys.stderr) prog = self.program() # the compiled rules, or the interpreter if prog[0] == 'souffle' and not shutil.which('souffle'): - die("this query needs soufflé and none is installed (brew install souffle-lang/souffle/souffle) — or a compiled\n" - f" rule binary in {os.path.relpath(os.path.join(HERE, 'dl', '.cache'))}/, which runs without it") + die(f"no compiled impact.dl for these rules on this machine: the {dl_program.SCOPE}/engine-{dl_program.npm_platform()} package\n" + " ships one for each release (reinstall @axiomcode/code-graph so it is fetched), or install soufflé\n" + " (brew install souffle-lang/souffle/souffle) to compile or interpret the rules here") prof('query facts written'); r = subprocess.run(prog + ['-F', F, '-D', O], capture_output=True, text=True); prof('souffle returned') if r.returncode: die("souffle failed:\n" + r.stderr[-1200:]) out = {} diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path index 976d9ed6..d6d843a5 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path @@ -554,7 +554,8 @@ def run_dl(g, queries, programs=('path.dl',)): # is only needed for the ones that fall back to the interpreter. progs = {p: dl_program.program(p) for p in programs} if any(v[0] == 'souffle' for v in progs.values()) and not shutil.which('souffle'): - die("souffle is not installed (brew install souffle-lang/souffle/souffle)") + die(f"no compiled path rules for this version on this machine: the {dl_program.SCOPE}/engine-{dl_program.npm_platform()} package\n" + " ships them for each release (reinstall @axiomcode/code-graph), or install soufflé (brew install souffle-lang/souffle/souffle)") F = tempfile.mkdtemp(prefix='axpath-'); O = tempfile.mkdtemp(prefix='axpath-out-') for n in ('byname', 'named'): os.symlink(os.path.join(g.facts, n + '.facts'), os.path.join(F, n + '.facts')) if g.EXTRA: diff --git a/plugins/axiomcode/skills/axiomcode/scripts/dl_program.py b/plugins/axiomcode/skills/axiomcode/scripts/dl_program.py index 584945e0..d19b61e1 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/dl_program.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/dl_program.py @@ -15,12 +15,14 @@ which was the whole of that benchmark's wall-clock regression. A killed compile leaves no `.nocompile` marker either (it did not fail, it was killed), so it cannot even record its own defeat. """ -import hashlib, os, shutil, subprocess, sys +import hashlib, os, platform, shutil, subprocess, sys HERE = os.path.dirname(os.path.abspath(__file__)) DL_DIR = os.path.join(HERE, 'dl') CACHE = os.path.join(DL_DIR, '.cache') HOOKS_DIR = os.path.abspath(os.path.join(HERE, '..', '..', '..', 'hooks')) +SCOPE = '@axiomcode' # graph/pipeline/engine.conf's ENGINE_PACKAGE_SCOPE +EXE = '.exe' if os.name == 'nt' else '' def souffle_include(): @@ -59,15 +61,71 @@ def resolve(dl): return os.path.join(DL_DIR, dl) +def rules_id(dl): + """what a compiled program is keyed by: its rules, and nothing else. CI names the binaries it ships with this same + function (`dl_program.py --print-id`), so a shipped binary and a local compile agree on which rules they hold.""" + return hashlib.sha1(open(dl, 'rb').read()).hexdigest()[:16] + + +def npm_platform(): + """this machine in npm's spelling (process.platform-process.arch), which is how the engine packages are named""" + o = {'darwin': 'darwin', 'win32': 'win32', 'cygwin': 'win32', 'msys': 'win32'}.get(sys.platform, 'linux' if sys.platform.startswith('linux') else None) + a = {'x86_64': 'x64', 'amd64': 'x64', 'arm64': 'arm64', 'aarch64': 'arm64'}.get(platform.machine().lower()) + return f'{o}-{a}' if o and a else None + + +def engine_roots(): + """where the installed engine package can be found from, in order: this plugin (inside the npm package, or a + checkout with its own node_modules), the engine named by AXIOMCODE_ENGINE, the `axiomcode` on PATH. A plugin a host + copied under its own directory has no node_modules above it, which is why the last two exist.""" + roots = [HERE] + if os.environ.get('AXIOMCODE_ENGINE'): roots.append(os.environ['AXIOMCODE_ENGINE']) + b = shutil.which('axiomcode') + if b: roots.append(os.path.dirname(os.path.dirname(os.path.realpath(b)))) + return roots + + +def packaged(stem, key): + """the query binary the engine package for this machine ships, when it was built from exactly these rules. Walks + up from each root the way node resolves a package, so a local node_modules and a global install both work. A + package holding other rules is reported once and not used — running it would answer from rules this plugin is not.""" + plat = npm_platform() + if not plat: return None + seen = set() + for root in engine_roots(): + d = os.path.abspath(root) + while True: + q = os.path.join(d, 'node_modules', SCOPE, f'engine-{plat}', 'queries') + if q not in seen and os.path.isdir(q): + seen.add(q) + binp = os.path.join(q, f'axiomcode-query-{stem}{EXE}') + try: have = open(os.path.join(q, f'{stem}.id')).read().strip() + except OSError: have = '' + if have == key and os.path.isfile(binp): + if EXE == '' and not os.access(binp, os.X_OK): + try: os.chmod(binp, 0o755) + except OSError: pass + return binp + if have: print(f" ! {SCOPE}/engine-{plat} holds {stem}.dl at {have}, these rules are {key} — not using it", file=sys.stderr) + up = os.path.dirname(d) + if up == d: break + d = up + return None + + def program(dl, verbose=True): - """the argv prefix to run this program: [], or ['souffle',
] when there is no compiler, when a + """the argv prefix to run this program: [] when it was built from these rules (no + soufflé, no compiler — what an npm install gets), else [], or ['souffle',
] when there is no compiler, when a compile has already failed here (the failure is recorded, so a broken toolchain costs one attempt rather than one per query — the visible half of #816), or under AXIOMCODE_INTERPRET.""" dl = resolve(dl) # keyed by the RULES alone. The soufflé runtime is compiled INTO the binary, so the version that generated it does # not matter at run time; and keying on it meant a cached (or shipped) binary was unusable without soufflé present. - key = hashlib.sha1(open(dl, 'rb').read()).hexdigest()[:16] + key = rules_id(dl) stem = os.path.splitext(os.path.basename(dl))[0] + if not os.environ.get('AXIOMCODE_INTERPRET'): + shipped = packaged(stem, key) + if shipped: return [shipped] binp = os.path.join(CACHE, f'{stem}-{key}'); nope = binp + '.nocompile' if os.path.exists(binp): return [binp] if os.path.exists(nope) or not shutil.which('c++') or not shutil.which('souffle') or os.environ.get('AXIOMCODE_INTERPRET'): @@ -109,12 +167,16 @@ def warm(verbose=True): done = [] for dl in all_programs(): pre = program(dl, verbose=False) - done.append((os.path.basename(dl), 'cached' if pre and not pre[0].endswith('souffle') else 'interpreter')) + how = 'interpreter' if not pre or pre[0] == 'souffle' else 'cached' if pre[0].startswith(CACHE) else 'packaged' + done.append((os.path.basename(dl), how)) if verbose: - ok = sum(1 for _, s in done if s == 'cached') + ok = sum(1 for _, s in done if s != 'interpreter') print(f"datalog rules ready: {ok}/{len(done)} compiled (" + ', '.join(f'{n}={s}' for n, s in done) + ")") return done if __name__ == '__main__': - warm() + if sys.argv[1:2] == ['--print-id']: + for a in sys.argv[2:]: print(rules_id(a if os.path.exists(a) else resolve(a))) + else: + warm()