From 82a0ab22fa181b98f75d6a4d41357d3f95870087 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:57:03 -0700 Subject: [PATCH 1/2] impact/path: one facts export per graph, however many queries arrive at once The export had no lock: a query that found the stamp stale while another process was already writing the same facts (the background warm-up after index, a second query, several hooks at once) ran the whole export again beside it. Measured on an 8,619-file Java repository: the first query after index cost 193 s against 8.8 s warm, and every concurrent query paid the same again while thrashing the writer. The first comer now takes /.exporting and the others wait for its stamp, then answer from the facts it wrote; a lock whose writer is gone is taken over. tests/export_singleflight.py pins the wait, the takeover and the unlock. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../skills/axiomcode/scripts/axiomcode-impact | 13 ++- .../skills/axiomcode/scripts/axiomcode-path | 47 ++++++++++- tests/export_singleflight.py | 83 +++++++++++++++++++ 3 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 tests/export_singleflight.py diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact index e8186697..191cb9f4 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact @@ -1275,8 +1275,17 @@ class Impact: # a cache written by an older program, or one an interrupted run left half written, breaks every query with # "Cannot open fact file X.facts": accept it only when it holds every relation the program reads need = set(re.findall(r'\.decl (\w+)\([^)]*\)\s+\.input', open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dl', 'impact.dl')).read())) - PER_QUERY - if os.path.exists(stamp) and open(stamp).read() == want and all(os.path.exists(os.path.join(D, n + '.facts')) for n in need): return D - os.makedirs(D, exist_ok=True); W = lambda n, rows: g.write(n, rows, D) + def fresh(): + try: return open(stamp).read() == want and all(os.path.exists(os.path.join(D, n + '.facts')) for n in need) + except OSError: return False + if fresh(): return D + os.makedirs(D, exist_ok=True) + if not P.export_lock(D, fresh): return D # another process just exported these same facts + try: return self._export(D, stamp, want) + finally: P.export_unlock(D) + + def _export(self, D, stamp, want): + g = self.g; W = lambda n, rows: g.write(n, rows, D) # owner display -> type id. A display name is NOT unique: two files of one test suite commonly # declare a class of the same name, and keyed on the name alone the second one's members were # filed under the first one's type. Everything scope/2 reaches then crossed between them -- diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path index e719552e..997e29fe 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path @@ -128,6 +128,43 @@ def replace_file(tmp, dst): shutil.copyfile(tmp, dst); os.remove(tmp) +def export_lock(d, fresh): + """ONE exporter per facts directory. A query that found the stamp stale while another process (the build's + background warm-up, a concurrent query, several hooks at once) was already writing these facts ran the whole + export AGAIN beside it: on an 8,619-file repository the first query after `index` cost 193 s against 8.8 s + once warm, every concurrent query the same again, and the copies thrashed each other. Returns True with the + lock held — the caller exports, then export_unlock in a finally — or False once another process finished the + same export (`fresh` says so) and the caller just reads it. A lock whose writer is gone is taken over.""" + lock = os.path.join(d, '.exporting') + while True: + try: + fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + os.write(fd, str(os.getpid()).encode()); os.close(fd); return True + except FileExistsError: + pass + gone = False + try: + os.kill(int(open(lock).read().strip() or '0'), 0) + except (ProcessLookupError, ValueError): + gone = True + except PermissionError: + pass # alive, another user's process: wait for it + except OSError: # Windows cannot probe another session's pid: fall back to the lock's age + try: gone = time.time() - os.path.getmtime(lock) > 900 + except OSError: continue # the lock vanished under us: contend for it again + if gone: + try: os.remove(lock) + except OSError: pass + continue + time.sleep(0.5) + if fresh(): return False + + +def export_unlock(d): + try: os.remove(os.path.join(d, '.exporting')) + except OSError: pass + + class G: def __init__(self, repo): self.repo = os.path.realpath(repo or '.') @@ -948,8 +985,16 @@ class G: EXPORT_VERSION = '4' def export(self): stamp = os.path.join(self.facts, 'stamp'); want = f"{self.db_mtime}:{self.EXPORT_VERSION}" - if os.path.exists(stamp) and open(stamp).read() == want: return + def fresh(): + try: return open(stamp).read() == want + except OSError: return False + if fresh(): return os.makedirs(self.facts, exist_ok=True) + if not export_lock(self.facts, fresh): return # another process just exported these same facts + try: self._export(stamp, want) + finally: export_unlock(self.facts) + + def _export(self, stamp, want): # `generated` is a client declaration too: a Lombok accessor, a record member, a C# auto-property. The engine # synthesises it and RESOLVES the call sites that name it, so filtering the edge out lost every call into a # generated member — which, on a project whose models are all @Data, is most of what touches the models. diff --git a/tests/export_singleflight.py b/tests/export_singleflight.py new file mode 100644 index 00000000..61ad051e --- /dev/null +++ b/tests/export_singleflight.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""tests/export_singleflight.py — one facts export per graph, however many queries arrive at once. + +The facts a query solves over (out/dl, out/dl/impact) are exported once per graph.sqlite and reused. The export +had no lock: a query that found the stamp stale while another process was already writing the same facts — the +build's background warm-up, a second query, several hooks at once — ran the WHOLE export again beside it. On an +8,619-file repository the first query after `index` cost 193 s against 8.8 s warm, and every concurrent query +paid the same again. Now the first comer takes out/dl/impact/.exporting and the rest wait for its stamp; a lock +whose writer died is taken over. + +Checks, on one indexed fixture: + 1. wait: a lock held by a LIVE process makes a stale --warm wait, and when the holder restores the fresh + stamp and releases, the waiter returns WITHOUT re-exporting (the stamp file is untouched). + 2. takeover: a lock left by a DEAD process does not block — the next --warm removes it and exports. + 3. unlock: a finished export leaves no .exporting behind. + +Indexes one case, so it needs the engine (AXIOMCODE_ENGINE, as tests/run.py). + + python3 tests/export_singleflight.py +""" +import os, shutil, subprocess, sys, tempfile, threading, time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +AX = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode') +IMPACT = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode-impact') +CASE = os.path.join(ROOT, 'tests', 'cases', 'java', 'containment-not-from-a-shared-span', 'src') + + +def run(*a, timeout=600): + p = subprocess.run(['bash', AX] + list(a), capture_output=True, text=True, timeout=timeout) + return p.returncode, p.stdout + p.stderr + + +def main(): + fails = [] + def check(ok, why, detail=''): + print(('ok ' if ok else 'FAIL ') + why + ('' if ok else '\n ' + detail.strip().replace('\n', '\n '))) + if not ok: fails.append(why) + + work = tempfile.mkdtemp(prefix='axiomcode-singleflight-') + try: + repo = os.path.join(work, 'repo'); shutil.copytree(CASE, repo) + rc, out = run('index', repo, '--lang', 'java') + if rc: print(f"FAIL index: {out[-300:]}"); return 1 + rc, out = run('impact', '--warm', repo) + check(rc == 0 and 'impact facts ready' in out, 'a first --warm exports the facts', out[-300:]) + D = os.path.join(repo, '.axiomcode', 'out', 'dl', 'impact') + stamp = os.path.join(D, 'stamp'); lock = os.path.join(D, '.exporting') + check(not os.path.exists(lock), 'a finished export leaves no .exporting behind', 'lock file still there') + + # 1. WAIT, DON'T RE-EXPORT: hold the lock as a live process, hide the stamp so the waiter sees stale facts, + # then put the fresh stamp back and release. The waiter must return 0 having written nothing: the stamp's + # mtime is the proof, set well in the past so any rewrite moves it. + held = open(lock, 'w'); held.write(str(os.getpid())); held.flush() + past = time.time() - 3600; os.utime(stamp, (past, past)); before = os.path.getmtime(stamp) + saved = stamp + '.aside'; os.rename(stamp, saved) + + def release(): + time.sleep(2); os.rename(saved, stamp); held.close(); os.remove(lock) + t = threading.Thread(target=release); t.start() + t0 = time.time(); rc, out = run('impact', '--warm', repo, timeout=120); waited = time.time() - t0 + t.join() + check(rc == 0, 'a --warm against a held lock returns 0 once the holder finishes', out[-300:]) + check(waited >= 2, f'it waited for the holder ({waited:.1f}s)', 'returned before the lock was released') + check(os.path.getmtime(stamp) == before, 'and re-exported nothing: the stamp is the one the holder left', + 'stamp mtime moved — the waiter exported over the holder') + + # 2. TAKEOVER: a dead writer's lock does not block. Plant a lock naming a pid that is gone, stale the + # stamp for real, and the next --warm must remove the lock and export. + os.remove(stamp) + open(lock, 'w').write('999999999') + t0 = time.time(); rc, out = run('impact', '--warm', repo, timeout=120) + check(rc == 0 and os.path.exists(stamp), 'a dead writer\'s lock is taken over and the export runs', out[-300:]) + check(not os.path.exists(lock), 'and the taken-over lock is released', 'lock file still there') + finally: + shutil.rmtree(work, ignore_errors=True) + + print(('FAIL: ' + '; '.join(fails)) if fails else 'all export_singleflight checks passed') + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main()) From 1205b15a157fd6e4bb75e43b43d3db3e7365e730 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Fri, 2 Oct 2026 17:02:33 -0700 Subject: [PATCH 2/2] impact: an undeclared name that arrives through an import hints at --library staging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A name no graph declares, reached only by text, whose lines include an import of it is the signature of an unstaged dependency — until now the answer was indistinguishable from an engine gap, and the fix is one flag away. One hint line under the [text] rows, only for undeclared names. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- plugins/axiomcode/skills/axiomcode/scripts/ax_text.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_text.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_text.py index ada6242b..3ef55ac3 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_text.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_text.py @@ -644,6 +644,13 @@ def block(repo, asked, scope=None, why='unresolved', rows=ROWS): if rest > 0: lines.append(f" … +{rest} more line(s): git grep -n{'w' if w else ''} -F -e '{n}'" if not pat else f" … +{rest} more line(s): git grep -nP -e '{pat.pattern}'") + # a name no graph declares that ARRIVES THROUGH AN IMPORT is the signature of an unstaged dependency, the + # commonest setup gap there is: without the hint this answer is indistinguishable from an engine gap, and the + # fix (stage the dependency's source) is one flag away. Only for undeclared names: a string or a resolved + # target wants no staging advice. + if why == 'undeclared' and any(re.match(r'\s*(import\b|using\b|from\s)', t.strip()) and n in t for _f, _l, t in hits): + lines.append(f" this name arrives through an import, so it is likely declared in a dependency: calls through " + f"it resolve once that dependency's source is staged — `axiomcode index --library `") if not lines: return '' if approxed: lines.append("next: [approx] rows are text placed in the declaration that holds it, not resolved edges: read the " "evidence line, then `impact ` for what a change reaches")