impact/path: one facts export per graph, however many queries arrive at once

plugins/axiomcode/skills/axiomcode/scripts/ax_text.py

@@ -644,6 +644,13 @@ def block(repo, asked, scope=None, why='unresolved', rows=ROWS):
         if rest > 0:
             lines.append(f"    … +{rest} more line(s): git grep -n{'w' if w else ''} -F -e '{n}'" if not pat else
                          f"    … +{rest} more line(s): git grep -nP -e '{pat.pattern}'")
+        # a name no graph declares that ARRIVES THROUGH AN IMPORT is the signature of an unstaged dependency, the
+        # commonest setup gap there is: without the hint this answer is indistinguishable from an engine gap, and the
+        # fix (stage the dependency's source) is one flag away. Only for undeclared names: a string or a resolved
+        # target wants no staging advice.
+        if why == 'undeclared' and any(re.match(r'\s*(import\b|using\b|from\s)', t.strip()) and n in t for _f, _l, t in hits):
+            lines.append(f"    this name arrives through an import, so it is likely declared in a dependency: calls through "
+                         f"it resolve once that dependency's source is staged — `axiomcode index --library <its source dir>`")
     if not lines: return ''
     if approxed: lines.append("next: [approx] rows are text placed in the declaration that holds it, not resolved edges: read the "
                               "evidence line, then `impact <that declaration>` for what a change reaches")

plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact

@@ -1275,8 +1275,17 @@ class Impact:
         # a cache written by an older program, or one an interrupted run left half written, breaks every query with
         # "Cannot open fact file X.facts": accept it only when it holds every relation the program reads
         need = set(re.findall(r'\.decl (\w+)\([^)]*\)\s+\.input', open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dl', 'impact.dl')).read())) - PER_QUERY
-        if os.path.exists(stamp) and open(stamp).read() == want and all(os.path.exists(os.path.join(D, n + '.facts')) for n in need): return D
-        os.makedirs(D, exist_ok=True); W = lambda n, rows: g.write(n, rows, D)
+        def fresh():
+            try: return open(stamp).read() == want and all(os.path.exists(os.path.join(D, n + '.facts')) for n in need)
+            except OSError: return False
+        if fresh(): return D
+        os.makedirs(D, exist_ok=True)
+        if not P.export_lock(D, fresh): return D           # another process just exported these same facts
+        try: return self._export(D, stamp, want)
+        finally: P.export_unlock(D)
+
+    def _export(self, D, stamp, want):
+        g = self.g; W = lambda n, rows: g.write(n, rows, D)
         # owner display -> type id. A display name is NOT unique: two files of one test suite commonly
         # declare a class of the same name, and keyed on the name alone the second one's members were
         # filed under the first one's type. Everything scope/2 reaches then crossed between them --

plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path

@@ -128,6 +128,43 @@ def replace_file(tmp, dst):
     shutil.copyfile(tmp, dst); os.remove(tmp)
 
 
+def export_lock(d, fresh):
+    """ONE exporter per facts directory. A query that found the stamp stale while another process (the build's
+    background warm-up, a concurrent query, several hooks at once) was already writing these facts ran the whole
+    export AGAIN beside it: on an 8,619-file repository the first query after `index` cost 193 s against 8.8 s
+    once warm, every concurrent query the same again, and the copies thrashed each other. Returns True with the
+    lock held — the caller exports, then export_unlock in a finally — or False once another process finished the
+    same export (`fresh` says so) and the caller just reads it. A lock whose writer is gone is taken over."""
+    lock = os.path.join(d, '.exporting')
+    while True:
+        try:
+            fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+            os.write(fd, str(os.getpid()).encode()); os.close(fd); return True
+        except FileExistsError:
+            pass
+        gone = False
+        try:
+            os.kill(int(open(lock).read().strip() or '0'), 0)
+        except (ProcessLookupError, ValueError):
+            gone = True
+        except PermissionError:
+            pass                                 # alive, another user's process: wait for it
+        except OSError:                          # Windows cannot probe another session's pid: fall back to the lock's age
+            try: gone = time.time() - os.path.getmtime(lock) > 900
+            except OSError: continue             # the lock vanished under us: contend for it again
+        if gone:
+            try: os.remove(lock)
+            except OSError: pass
+            continue
+        time.sleep(0.5)
+        if fresh(): return False
+
+
+def export_unlock(d):
+    try: os.remove(os.path.join(d, '.exporting'))
+    except OSError: pass
+
+
 class G:
     def __init__(self, repo):
         self.repo = os.path.realpath(repo or '.')
@@ -948,8 +985,16 @@ class G:
     EXPORT_VERSION = '4'
     def export(self):
         stamp = os.path.join(self.facts, 'stamp'); want = f"{self.db_mtime}:{self.EXPORT_VERSION}"
-        if os.path.exists(stamp) and open(stamp).read() == want: return
+        def fresh():
+            try: return open(stamp).read() == want
+            except OSError: return False
+        if fresh(): return
         os.makedirs(self.facts, exist_ok=True)
+        if not export_lock(self.facts, fresh): return      # another process just exported these same facts
+        try: self._export(stamp, want)
+        finally: export_unlock(self.facts)
+
+    def _export(self, stamp, want):
         # `generated` is a client declaration too: a Lombok accessor, a record member, a C# auto-property. The engine
         # synthesises it and RESOLVES the call sites that name it, so filtering the edge out lost every call into a
         # generated member — which, on a project whose models are all @Data, is most of what touches the models.

tests/export_singleflight.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""tests/export_singleflight.py — one facts export per graph, however many queries arrive at once.
+
+The facts a query solves over (out/dl, out/dl/impact) are exported once per graph.sqlite and reused. The export
+had no lock: a query that found the stamp stale while another process was already writing the same facts — the
+build's background warm-up, a second query, several hooks at once — ran the WHOLE export again beside it. On an
+8,619-file repository the first query after `index` cost 193 s against 8.8 s warm, and every concurrent query
+paid the same again. Now the first comer takes out/dl/impact/.exporting and the rest wait for its stamp; a lock
+whose writer died is taken over.
+
+Checks, on one indexed fixture:
+  1. wait:     a lock held by a LIVE process makes a stale --warm wait, and when the holder restores the fresh
+               stamp and releases, the waiter returns WITHOUT re-exporting (the stamp file is untouched).
+  2. takeover: a lock left by a DEAD process does not block — the next --warm removes it and exports.
+  3. unlock:   a finished export leaves no .exporting behind.
+
+Indexes one case, so it needs the engine (AXIOMCODE_ENGINE, as tests/run.py).
+
+    python3 tests/export_singleflight.py
+"""
+import os, shutil, subprocess, sys, tempfile, threading, time
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+AX = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode')
+IMPACT = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode-impact')
+CASE = os.path.join(ROOT, 'tests', 'cases', 'java', 'containment-not-from-a-shared-span', 'src')
+
+
+def run(*a, timeout=600):
+    p = subprocess.run(['bash', AX] + list(a), capture_output=True, text=True, timeout=timeout)
+    return p.returncode, p.stdout + p.stderr
+
+
+def main():
+    fails = []
+    def check(ok, why, detail=''):
+        print(('ok   ' if ok else 'FAIL ') + why + ('' if ok else '\n     ' + detail.strip().replace('\n', '\n     ')))
+        if not ok: fails.append(why)
+
+    work = tempfile.mkdtemp(prefix='axiomcode-singleflight-')
+    try:
+        repo = os.path.join(work, 'repo'); shutil.copytree(CASE, repo)
+        rc, out = run('index', repo, '--lang', 'java')
+        if rc: print(f"FAIL index: {out[-300:]}"); return 1
+        rc, out = run('impact', '--warm', repo)
+        check(rc == 0 and 'impact facts ready' in out, 'a first --warm exports the facts', out[-300:])
+        D = os.path.join(repo, '.axiomcode', 'out', 'dl', 'impact')
+        stamp = os.path.join(D, 'stamp'); lock = os.path.join(D, '.exporting')
+        check(not os.path.exists(lock), 'a finished export leaves no .exporting behind', 'lock file still there')
+
+        # 1. WAIT, DON'T RE-EXPORT: hold the lock as a live process, hide the stamp so the waiter sees stale facts,
+        # then put the fresh stamp back and release. The waiter must return 0 having written nothing: the stamp's
+        # mtime is the proof, set well in the past so any rewrite moves it.
+        held = open(lock, 'w'); held.write(str(os.getpid())); held.flush()
+        past = time.time() - 3600; os.utime(stamp, (past, past)); before = os.path.getmtime(stamp)
+        saved = stamp + '.aside'; os.rename(stamp, saved)
+
+        def release():
+            time.sleep(2); os.rename(saved, stamp); held.close(); os.remove(lock)
+        t = threading.Thread(target=release); t.start()
+        t0 = time.time(); rc, out = run('impact', '--warm', repo, timeout=120); waited = time.time() - t0
+        t.join()
+        check(rc == 0, 'a --warm against a held lock returns 0 once the holder finishes', out[-300:])
+        check(waited >= 2, f'it waited for the holder ({waited:.1f}s)', 'returned before the lock was released')
+        check(os.path.getmtime(stamp) == before, 'and re-exported nothing: the stamp is the one the holder left',
+              'stamp mtime moved — the waiter exported over the holder')
+
+        # 2. TAKEOVER: a dead writer's lock does not block. Plant a lock naming a pid that is gone, stale the
+        # stamp for real, and the next --warm must remove the lock and export.
+        os.remove(stamp)
+        open(lock, 'w').write('999999999')
+        t0 = time.time(); rc, out = run('impact', '--warm', repo, timeout=120)
+        check(rc == 0 and os.path.exists(stamp), 'a dead writer\'s lock is taken over and the export runs', out[-300:])
+        check(not os.path.exists(lock), 'and the taken-over lock is released', 'lock file still there')
+    finally:
+        shutil.rmtree(work, ignore_errors=True)
+
+    print(('FAIL: ' + '; '.join(fails)) if fails else 'all export_singleflight checks passed')
+    return 1 if fails else 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())