fix: pin the collation so the program text depends on the rules, not the locale

graph/pipeline/run-souffle.sh

@@ -112,6 +112,21 @@ input_relations(){
   printf '%s\n' jdk_max_depth lib_max_depth taint_gating dispatch_cap
 }
 write_program(){ # $1 = destination file
+  # COLLATION IS PART OF THE PROGRAM TEXT, so it is pinned here rather than inherited. The
+  # #include lines below come from shell globs, and bash orders a glob by LC_COLLATE, not by
+  # byte value. A UTF-8 collation ignores punctuation when comparing, so call-site.dl and
+  # callee-resolution.dl swap places against their byte order. The include order is part of
+  # the program text, the program text is hashed, and that hash is the engine id -- so
+  # whether a published binary is accepted becomes a function of the user's locale rather
+  # than of the rules, and the refusal names the rules. CI runs under a C-ish locale while a
+  # UTF-8 locale is the default on most Linux desktops, in macOS terminals and in Git Bash,
+  # so the mismatch is the common case. Measured: java and python ids differ between macOS
+  # and MSYS2, and between LC_ALL=C and en_US.UTF-8 on glibc; typescript and javascript
+  # agree only because no pair of their filenames collides. Invisible on macOS, whose
+  # collation matches C either way, which is why it survived. The sorts below were already
+  # forced to C for this reason; the globs were not.
+  # See issue #895 and graph/test/tools/engine-id-locale-test.sh.
+  local LC_ALL=C LC_COLLATE=C
   {
     echo "#include \"$LANG_ARG/souffle/decls_base.dl\""; echo "#include \"$LANG_ARG/souffle/decls_all.dl\""
     # rfc4180=true: the IR is CSV, not TSV. The parser quotes any field containing a

graph/test/java/run-tests.sh

@@ -99,6 +99,14 @@ if ! bash "$ROOT/graph/test/tools/souffle-include-test.sh"; then
   echo "aborting: the soufflé include path does not resolve to a compilable -I"
   exit 1
 fi
+# ── The program text must not follow the user's locale ─────────────────────
+# Bash orders a glob by LC_COLLATE, so an unpinned collation made the cache key, and with it
+# whether a published engine is accepted, a function of the environment rather than the rules.
+# Invisible on macOS, whose collation matches C either way. See #895.
+if ! bash "$ROOT/graph/test/tools/engine-id-locale-test.sh"; then
+  echo "aborting: the program text depends on the shell locale"
+  exit 1
+fi
 # ── The bundle stage must build the language-neutral output ─────────────────
 # Every solve below ends by joining the raw relations to the IR and writing graph.sqlite
 # (graph/bundle/SCHEMA.md); graph/*.csv is the same core tables and is written only under

graph/test/tools/engine-id-locale-test.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# ─────────────────────────────────────────────────────────────────────────────
+# The program text, and therefore the cache key and the engine id, must be a function of the
+# RULES ALONE and not of the environment the shell happens to run in.
+#
+# It was a function of the locale. The #include lines are assembled from shell globs, and
+# bash orders a glob by LC_COLLATE rather than by byte value: a UTF-8 collation ignores
+# punctuation, so call-site.dl and callee-resolution.dl swap places against their byte order.
+# Same rules, different locale, different hash. CI runs under a C-ish locale and a user
+# typically does not, so published engines were refused on the common configuration, with an
+# error naming the rules rather than the locale. See issue #895.
+#
+# THIS CANNOT BE ASSERTED ON macOS ALONE. Its collation behaves like C under both locales, so
+# the first attempt to reproduce the defect there found no difference and read as a clean bill
+# of health. The test therefore compares the ORDER A GLOB PRODUCES for a fixture built to
+# contain a colliding pair, which is checkable everywhere, and only then checks the program
+# text itself. A platform that cannot distinguish the two collations reports SKIP for the
+# first half rather than passing it silently.
+# ─────────────────────────────────────────────────────────────────────────────
+set -uo pipefail
+ROOT="$(d="$(cd "$(dirname "$0")" && pwd)"; while [ "$d" != / ] && { [ ! -f "$d/package.json" ] || [ ! -d "$d/graph" ]; }; do d="$(dirname "$d")"; done; echo "$d")"
+fail=0; checks=0
+ok(){  checks=$((checks+1)); [ -n "${ENGINE_ID_LOCALE_VERBOSE:-}" ] && printf '  ok    %s\n' "$1"; return 0; }
+bad(){ checks=$((checks+1)); printf '  FAIL  %s\n' "$1"; fail=1; }
+
+# --- does this machine's libc distinguish the two collations at all? ----------------------
+W="$(mktemp -d)"; trap 'rm -rf "$W"' EXIT
+( cd "$W" && touch call-site.dl callee-resolution.dl )
+order(){ ( cd "$W" && LC_ALL="$1" bash -c 'for f in *.dl; do printf "%s " "$f"; done' ); }
+c_order="$(order C)"; u_order="$(order en_US.UTF-8)"
+if [ "$c_order" = "$u_order" ]; then
+  echo "  skip  this libc collates the fixture identically under C and en_US.UTF-8 (macOS does);"
+  echo "        the ordering half of this test cannot run here -- assert it on glibc or MSYS2"
+else
+  ok "the two collations do differ here, so the guard is meaningful ($c_order/ $u_order)"
+fi
+
+# --- the real assertion: the program text does not move with the locale -------------------
+# Compared as TEXT rather than as a hash, so a failure names the line that moved instead of
+# only reporting two different hashes.
+have_lang(){ ls -d "$ROOT/graph/$1/engine" >/dev/null 2>&1; }
+for lang in java typescript python javascript; do
+  have_lang "$lang" || continue
+  a="$W/$lang.C.dl"; b="$W/$lang.U.dl"
+  LC_ALL=C            bash "$ROOT/graph/pipeline/run-souffle.sh" --language "$lang" --emit-program "$a" >/dev/null 2>&1
+  LC_ALL=en_US.UTF-8  bash "$ROOT/graph/pipeline/run-souffle.sh" --language "$lang" --emit-program "$b" >/dev/null 2>&1
+  if [ ! -s "$a" ] || [ ! -s "$b" ]; then
+    # --emit-program lands with #478; until then the executor has no way to print the program
+    # without solving, and this half cannot run. Skip loudly rather than pass on nothing.
+    echo "  skip  $lang: --emit-program produced nothing (needs the emit flag)"
+    continue
+  fi
+  if cmp -s "$a" "$b"; then ok "$lang program text is identical under C and en_US.UTF-8"
+  else bad "$lang program text MOVES with the locale: $(diff "$a" "$b" | grep '^[<>]' | head -2 | tr '\n' ' ')"; fi
+done
+
+# --- the guard itself, asserted on EVERY platform ------------------------------------------
+# The two halves above both skip on a machine that cannot tell the collations apart and on a
+# checkout without --emit-program, which is how this test came to report PASS having asserted
+# nothing at all. This half runs everywhere: the executor must pin the collation before it
+# assembles the program, and it must do so BEFORE the first #include glob, or the pin is
+# decoration. Grepping for the mechanism is weaker than measuring the output, but a guard that
+# is silently deleted is exactly how the defect returns, and a test that cannot fail anywhere
+# is worth less than one that can fail somewhere.
+RS="$ROOT/graph/pipeline/run-souffle.sh"
+# Scoped to the REGION that assembles the program: from write_program to its first rule glob.
+# A bare search for LC_ALL matches the `LC_ALL=C sort` calls elsewhere in the file, which are
+# a different guard for a different line, so it reported the pin present after the pin had
+# been deleted. That is the same shape of defect this whole test exists for: a check that
+# cannot fail is not a check.
+wp="$(grep -n 'write_program()' "$RS" | head -1 | cut -d: -f1)"
+gl="$(awk -v s="${wp:-1}" 'NR>s && /for f in "\$ENG/ {print NR; exit}' "$RS")"
+if [ -z "$wp" ] || [ -z "$gl" ]; then
+  bad "cannot locate write_program and its first rule glob in run-souffle.sh; this test no longer measures anything"
+else
+  # COMMENTS DO NOT COUNT. The guard is described in a comment right above itself, and that
+  # comment names LC_ALL, so a naive match reported the pin present after the pin was deleted.
+  # Skip comment lines and the `LC_ALL=C sort` calls, which guard a different line.
+  pin="$(awk -v a="$wp" -v b="$gl" 'NR>a && NR<b && !/^[[:space:]]*#/ && /LC_(ALL|COLLATE)=C/ && !/sort/ {print NR; exit}' "$RS")"
+  if [ -n "$pin" ]; then ok "collation pinned at line $pin, inside write_program (lines $wp-$gl)"
+  else bad "no LC_ALL/LC_COLLATE pin between write_program (line $wp) and its first rule glob (line $gl): the program order follows the user's locale (#895)"; fi
+fi
+
+[ "$checks" -gt 0 ] || { echo "  FAIL  this test asserted nothing on this machine"; fail=1; }
+printf 'engine-id-locale: %d checks, %s\n' "$checks" "$([ $fail -eq 0 ] && echo PASS || echo FAIL)"
+exit $fail