plugin: a corrupt graph.sqlite is said, moved aside and rebuilt, never a traceback

plugins/axiomcode/skills/axiomcode/scripts/ax_contract.py

@@ -479,7 +479,8 @@ def ensure_graph(repo, db):
         # relink above took every pointer that leaves this .axiomcode/out: one still here is this repository's own
         try: gone = f" (.axiomcode/out/graph.sqlite points at {os.readlink(db)}, which is not there)"
         except OSError: gone = ''
-        print(f"the graph of {repo} is missing (a build was interrupted){gone} — rebuilding it; the baseline edits are measured against is kept …", file=sys.stderr)
+        why = "it was corrupt and was moved aside" if os.path.exists(os.path.join(ax_fresh.out_dir(rr), 'corrupt')) else "a build was interrupted"
+        print(f"the graph of {repo} is missing ({why}){gone} — rebuilding it; the baseline edits are measured against is kept …", file=sys.stderr)
         env = dict(os.environ, AXIOMCODE_KEEP_BASE='1')
     else:
         print(f"no graph for {repo} yet — building one (this is the only slow call; later ones read it) …", file=sys.stderr)
@@ -534,6 +535,94 @@ def _start_build(argv, log, env):
     return subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=log, stderr=log, close_fds=True, env=env)
 
 
+def usable_graph(repo, gdir):
+    """The graph directory a verb reads: `gdir` itself, unless its graph.sqlite is CORRUPT (a disk that filled, a copy cut
+    short). Then it is never read — every verb died in a Python traceback on it — but moved aside and rebuilt: the main
+    graph through ensure_graph (a shell waits for it; the MCP server starts it and answers at once), another language's
+    in the background. Until the rebuild is in, the last good graph answers (the baseline a refresh kept), and says so
+    on a `graph refresh:` line, which the MCP server carries into the answer. With none, the verb stops in words."""
+    db = os.path.join(gdir, 'out', 'graph.sqlite')
+    if not os.path.exists(db): return gdir
+    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); import ax_fresh
+    why = ax_fresh.graph_corrupt(db)
+    if not why: return gdir
+    return _corrupt(repo, gdir, db, why)
+
+
+def _corrupt(repo, gdir, db, why):
+    import ax_fresh
+    rr = os.path.realpath(repo)
+    lang = ax_fresh.graph_lang()
+    # a graph someone placed with AXIOMCODE_GRAPH is theirs: said, never moved or rebuilt. One of this repository's own
+    # language graphs (the dispatcher sets AXIOMCODE_GRAPH to it, with AXIOMCODE_GRAPH_LANG) is ours to repair
+    placed = os.environ.get('AXIOMCODE_GRAPH')
+    own = not placed or (lang and os.path.realpath(placed) == os.path.realpath(os.path.join(rr, '.axiomcode', 'lang', lang)))
+    what = f"the {lang} graph" if lang else "the graph"
+    if not own:
+        _stop(f"{what} at {db} is corrupt ({why}); AXIOMCODE_GRAPH placed it, so it is not rebuilt here — rebuild it where it was built")
+    moved = ax_fresh.quarantine(rr, db, why)
+    print(f"graph refresh: {what} at {db} is corrupt ({why}) — " + (f"moved aside to {moved} and rebuilding it" if moved else "rebuilding it"),
+          file=sys.stderr, flush=True)
+    if not lang:
+        if ensure_graph(repo, db) and os.path.exists(db) and not ax_fresh.graph_corrupt(db): return gdir
+    elif not ax_fresh.building(rr):
+        _rebuild_in_background(rr)
+    good = ax_fresh.last_good_graph(rr)
+    if good:
+        print(f"graph refresh: until the rebuild is in, this answer comes from the last good graph ({good}), which predates "
+              f"the edits since it was built", file=sys.stderr, flush=True)
+        return good
+    _stop(_BUILD_NOTE[-1] if _BUILD_NOTE else
+        f"{what} at {db} was corrupt and there is no earlier graph to answer from; a rebuild is "
+        f"{'running' if ax_fresh.building(rr) else 'needed'} — ask again when it is done, or run `axiomcode index {repo}`")
+
+
+def _stop(msg):
+    """the verb's refusal, as the verbs' own die() gives it: the words on stdout, exit 2"""
+    print(msg); sys.exit(2)
+
+
+def _rebuild_in_background(repo):
+    """start a detached build of the repository, keeping the baseline, as a repair does (another language's graph was
+    corrupt; the build rebuilds every language, and the corrupt marker keeps it from calling the tree up to date)"""
+    build = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'axiomcode-build')
+    if not os.path.exists(build): return
+    env = dict(os.environ, AXIOMCODE_KEEP_BASE='1', AXIOMCODE_BACKGROUND='1', AXIOMCODE_REFRESH_REASON='a corrupt graph')
+    for k in ('AXIOMCODE_GRAPH', 'AXIOMCODE_GRAPH_LANG', 'AXIOMCODE_FANOUT', 'AXIOMCODE_LANG'): env.pop(k, None)
+    try:
+        log = open(os.path.join(repo, '.axiomcode', 'refresh.log'), 'a')
+        kw = dict(start_new_session=True) if os.name != 'nt' else dict(creationflags=0x00000008 | 0x00000200)
+        subprocess.Popen([os.environ.get('AXIOMCODE_BASH') or 'bash', build, repo], stdin=subprocess.DEVNULL, stdout=log, stderr=log,
+                         close_fds=True, env=env, **kw)
+    except OSError: pass
+
+
+def on_corrupt(repo, db):
+    """A graph that reads cleanly when opened can still hold a malformed page that only a query reaches. Installed by a
+    verb once its graph is open: an sqlite error that escapes the verb is checked with quick_check, and when the graph is
+    corrupt the verb says so and moves it aside for a rebuild instead of printing a traceback. Any other error is
+    reported as it was."""
+    prev = sys.excepthook
+    def hook(t, e, tb):
+        import sqlite3
+        if isinstance(e, sqlite3.DatabaseError) and os.path.exists(db):
+            sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); import ax_fresh
+            why = ax_fresh.graph_corrupt(db, thorough=True)
+            if why:
+                rr = os.path.realpath(repo); lang = ax_fresh.graph_lang()
+                placed = os.environ.get('AXIOMCODE_GRAPH')
+                own = not placed or (lang and os.path.realpath(placed) == os.path.realpath(os.path.join(rr, '.axiomcode', 'lang', lang)))
+                moved = ax_fresh.quarantine(rr, db, why) if own else ''
+                if own and not ax_fresh.building(rr): _rebuild_in_background(rr)
+                print(f"{'the ' + lang + ' graph' if lang else 'the graph'} at {db} is corrupt ({why}) — "
+                      + (f"moved aside to {moved}; it is being rebuilt, ask again in a minute" if moved else
+                         "rebuild it where it was built" if not own else f"run `axiomcode index {repo}` to rebuild it"),
+                      file=sys.stderr, flush=True)
+                sys.stdout.flush(); os._exit(1)                # SystemExit raised in an excepthook is itself reported
+        prev(t, e, tb)
+    sys.excepthook = hook
+
+
 def _published(repo):
     """the main graph is out while the build goes on solving the repository's other languages: the answer is given now,
     and names on stderr (a `graph refresh:` line, which the MCP server carries into the answer) what it cannot see yet"""

plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py

@@ -438,6 +438,57 @@ def relink(repo, locked=False, say=True):
     if say: print(msg, file=sys.stderr)
     return msg
 
+def graph_corrupt(db, thorough=False):
+    """why the graph at db cannot be read ('' when it can). A disk that filled mid-write, a copy cut short or a crash can
+    leave graph.sqlite malformed, and every verb then died in a Python traceback (`file is not a database`, `database disk
+    image is malformed`) instead of saying what was wrong. The probe is cheap (the schema is read, as every verb reads it
+    first); only when it fails, or with `thorough` (a verb that hit an error mid-answer), is quick_check asked, since on a
+    large graph it reads every page. An empty file is not corrupt: it is a graph that was never indexed."""
+    import sqlite3
+    try:
+        con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
+    except sqlite3.Error as e: return str(e)
+    try:
+        try:
+            con.execute("SELECT count(*) FROM sqlite_master").fetchone()
+            if not thorough: return ''
+        except sqlite3.OperationalError as e:
+            if 'locked' in str(e) or 'unable to open' in str(e): return ''   # busy or gone, not malformed
+            first = str(e)
+        except sqlite3.DatabaseError as e: first = str(e)
+        else: first = ''
+        try:
+            bad = [r[0] for r in con.execute("PRAGMA quick_check(3)").fetchall() if r[0] != 'ok']
+        except sqlite3.DatabaseError as e: return first or str(e)
+        return first or '; '.join(bad)
+    finally: con.close()
+
+def quarantine(repo, db, why):
+    """move a corrupt graph out of the way so the next build replaces it rather than reads it, and mark the repository so
+    that build cannot take the unchanged files for an up-to-date graph (axiomcode-build reads .axiomcode/out/corrupt). A
+    pointer's TARGET is moved and the pointer left dangling, which every verb already treats as a graph to repair (a
+    rebuild that keeps the baseline). Returns where the corrupt file went, or '' when it could not be moved."""
+    real = os.path.realpath(db); dst = real + '.corrupt'
+    try:
+        os.replace(real, dst)
+        for ext in ('-wal', '-shm', '-journal'):
+            if os.path.exists(real + ext): os.replace(real + ext, dst + ext)
+    except OSError: dst = ''
+    try:
+        os.makedirs(out_dir(repo), exist_ok=True)
+        with open(os.path.join(out_dir(repo), 'corrupt'), 'w') as f: f.write(f"{db}: {why}\n")
+    except OSError: pass
+    return dst
+
+def last_good_graph(repo):
+    """a graph directory that can still answer when the current graph is corrupt: the baseline graph a refresh kept
+    (.axiomcode/base, or base/lang/<lang> for another language), when it reads cleanly. It describes an earlier tree, so
+    an answer from it must say so. None when there is none."""
+    root = os.path.join(repo, '.axiomcode', 'base')
+    d = os.path.join(root, 'lang', graph_lang()) if graph_lang() else root
+    db = os.path.join(d, 'out', 'graph.sqlite')
+    return d if os.path.isfile(db) and os.path.getsize(db) and not graph_corrupt(db) else None
+
 def graph_lang():
     """the language whose graph a verb reads when it is not the main one (AXIOMCODE_GRAPH_LANG, set by the dispatcher
     as it asks each language's graph in turn), else ''"""

plugins/axiomcode/skills/axiomcode/scripts/ax_langs.py

@@ -92,6 +92,9 @@ def main(argv):
     named, outside = by_scope(named)
     named, others = by_landing(named)
     answered = [n for n in named if n[2] == 0]
+    # a graph that could not answer at all keeps its `graph refresh:` lines (a corrupt graph moved aside and being rebuilt,
+    # ax_contract.usable_graph): dropped with its refusal, the answer read as whole while a language was missing from it
+    notes = ''.join(l + '\n' for n in named if n[2] != 0 and answered for l in n[4].splitlines() if l.startswith('graph refresh:'))
     if not answered:
         # a graph that does not hold the --in scope says only that; the graph that holds it has the refusal that
         # matters (nothing under it matches, or the name is not there), so a scope in one language is judged by it
@@ -110,7 +113,7 @@ def main(argv):
             print(json.dumps(base, indent=1))
         else:
             print(json.dumps(base, indent=1) if not isinstance(base, str) else base, end='' if isinstance(base, str) else '\n')
-        sys.stderr.write(''.join(n[4] for n in answered))
+        sys.stderr.write(''.join(n[4] for n in answered) + notes)
         return 0
 
     show = answered or named
@@ -121,6 +124,7 @@ def main(argv):
         if len(show) > 1 or not is_main:
             print(('' if i == 0 else '\n') + f"══ {lang} graph ══" + ('' if is_main else f"   (.axiomcode/lang/{lang})"), flush=True)
         sys.stdout.write(out); sys.stdout.flush(); sys.stderr.write(err); sys.stderr.flush()
+    if notes: sys.stderr.write(notes); sys.stderr.flush()
     if answered and others:
         print(f"\nthe --from name is also declared in the {', '.join(others)} graph(s), where its flow reaches "
               "half as many of the task's words or fewer; --lang <language> asks one of them")

plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build

@@ -244,13 +244,16 @@ LANGDIR="$REPO/.axiomcode/lang"      # one graph directory per language other th
 # file the parser reads with its hash; when none differs the graph is current whatever git says: a dirty tree whose
 # edits are already indexed is not rebuilt, and a tree edited and restored, which git calls clean against a graph built
 # from the edit, is. A commit that changed no file moves the stamp, not the graph.
-if [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then
+# A GRAPH A QUERY FOUND CORRUPT (ax_fresh.quarantine leaves .axiomcode/out/corrupt) is rebuilt even though no file moved:
+# the files still match the table, so without this the build said "up to date" over a graph that could not be read.
+if [ -f "$OUT/corrupt" ]; then echo "rebuilding: $(head -1 "$OUT/corrupt")"; fi
+if [ ! -f "$OUT/corrupt" ] && [ -f "$OUT/graph.sqlite" ] && [ -f "$OUT/stamp" ] && [ ! -f "$OUT/partial" ] && AXIOMCODE_ENGINE="$ENGINE" python3 "$H/ax_fresh.py" uptodate "$REPO" "$LANGS" "${AXIOMCODE_SRC:-}" "${STAMP#"$PREFIX"}"; then
   if [ "$(cat "$OUT/stamp")" != "$STAMP" ]; then echo "$STAMP" > "$OUT/stamp"; INDEXED_TREE="$OLD_INDEXED"; set_base "$OLD_INDEXED"; PREV=""; keep_base_graph; commit_base; fi
   has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO"
   [ -f "$REPO/.axiomcode/engine" ] || printf '%s\n' "$ENGINE" > "$REPO/.axiomcode/engine" || true
   echo "graph up to date (${LANGS//,/, }) at $OUT/graph.sqlite"; exit 0
 fi
-if [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then
+if [ ! -f "$OUT/corrupt" ] && [ ! -f "$OUT/files.json" ] && [ -f "$OUT/stamp" ] && [ "$(cat "$OUT/stamp")" = "$STAMP" ] && [ -f "$OUT/graph.sqlite" ] && [ -z "$(cd "$REPO" && git status --porcelain 2>/dev/null | head -1)" ]; then
   has_symbols "$OUT/graph.sqlite" || python3 "$H/axiomcode-index" "$REPO"
   python3 "$H/axiomcode-impact" --warm "$REPO" || true   # export the graph's facts now (~9 s on 1,373 files): the first
 # query pays it otherwise, and for the plugin that first query is inside hooks/changes.py's timeout=14, several at once.
@@ -418,6 +421,7 @@ publish_main(){
   done
   commit_base
   rm -rf "$PREV"
+  rm -f "$OUT/corrupt" "$OUT/$LANG_ARG/graph.sqlite.corrupt"* "$LANGDIR"/*/out/graph.sqlite.corrupt*   # a sound graph replaced what was quarantined
   MAIN_DONE=1; PUBLISHING=""
   # the languages still unsolved are named in $OUT/partial: the graph is not up to date while it is there, so the next
   # build (a refresh the next query starts, or an index) builds them rather than calling the file table current

plugins/axiomcode/skills/axiomcode/scripts/axiomcode-graph

@@ -31,7 +31,7 @@ the caller, never a drawn edge.
 import os, re, sys, json, time, sqlite3, collections, subprocess
 
 HERE = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, HERE); import ax_contract
+sys.path.insert(0, HERE); import ax_contract, ax_fresh
 TESTRE = re.compile(r'(^|/)(tests?|testing|__tests__|test-data|fixtures?)(/|$)|(^|/)test_|conftest\.py$|/src/test/|Tests?\.java$|\.(spec|test)\.[cm]?[tj]sx?$')
 METHOD_KINDS = ('function', 'method', 'constructor', 'module')
 TYPE_KINDS = ('class', 'interface', 'enum', 'type', 'namespace')
@@ -51,13 +51,17 @@ def graph_dbs(repo, gdir):
         ld = os.path.join(repo, '.axiomcode', 'lang')
         for l in sorted(os.listdir(ld)) if os.path.isdir(ld) else ():
             d = os.path.join(ld, l, 'out', 'graph.sqlite')
-            if os.path.exists(d): dbs.append(d)
+            if not os.path.exists(d): continue
+            why = ax_fresh.graph_corrupt(d)
+            # another language's corrupt graph is left out of the page, said, rather than ending the page in a traceback
+            if why: print(f"graph refresh: the {l} graph at {d} is corrupt ({why}) and is left out of this page; `axiomcode index` rebuilds it", file=sys.stderr)
+            else: dbs.append(d)
     return dbs
 
 
 def export(repo, page=None, how=''):
     repo = os.path.realpath(repo or '.')
-    gdir = os.environ.get('AXIOMCODE_GRAPH') or os.path.join(repo, '.axiomcode')
+    gdir = ax_contract.usable_graph(repo, os.environ.get('AXIOMCODE_GRAPH') or os.path.join(repo, '.axiomcode'))
     db = os.path.join(gdir, 'out', 'graph.sqlite')
     # the same door every other verb goes through: built on demand under the CLI, refused in words
     # that name the one command that helps under anything else.
@@ -76,7 +80,7 @@ def export(repo, page=None, how=''):
         if r['line'] and len(sites[c]) < 400: sites[c].append([r['line'], r['written'] or '', target, r['tier']])
     skipped, tiers = None, collections.Counter()
     for db in graph_dbs(repo, gdir):
-        con = sqlite3.connect(db); con.row_factory = sqlite3.Row
+        con = sqlite3.connect(db); con.row_factory = sqlite3.Row; ax_contract.on_corrupt(repo, db)
         q = lambda sql: con.execute(sql).fetchall()
         has = lambda t: bool(con.execute("SELECT 1 FROM sqlite_master WHERE name=?", (t,)).fetchone())
         run = {r['key']: r['value'] for r in q("SELECT key, value FROM run")}; runs.append(run)

plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path

@@ -116,6 +116,8 @@ class G:
         self.repo = os.path.realpath(repo or '.')
         os.environ.setdefault('AXIOMCODE_REPO', self.repo)       # which repository's recorded engine dl_program looks in
         gdir = os.environ.get('AXIOMCODE_GRAPH') or os.path.join(self.repo, '.axiomcode')
+        # a CORRUPT graph is moved aside and rebuilt, and the last good graph answers meanwhile (ax_contract.usable_graph)
+        gdir = ax_contract.usable_graph(self.repo, gdir)
         self.db = os.path.join(gdir, 'out', 'graph.sqlite')
         # ONE door for every verb. impact, changed, context and test-impact all load this file as a
         # module and construct this class, so a repository with no graph is handled here, once, for
@@ -125,6 +127,7 @@ class G:
         if not os.path.exists(self.db) and not ax_contract.ensure_graph(self.repo, self.db):
             die(ax_contract.no_graph(self.repo, self.db))
         self.connect()
+        ax_contract.on_corrupt(self.repo, self.db)
         if not self.has('symbols'):
             # a graph that was built but never indexed: the index is this tool's own, and adding it
             # is seconds, so asking the caller to run one more command buys nothing.