From b88d7326b8f1b63f8276b02d960739a43d293815 Mon Sep 17 00:00:00 2001 From: Swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Mon, 28 Sep 2026 03:27:29 -0700 Subject: [PATCH] graph: draw the existing graph, and every rebuild keeps the languages the index chose `axiomcode graph` ran axiomcode-build with no flags, which detects every language in the tree. On a repository indexed with --lang it rebuilt the whole graph on every call, started JavaScript/TypeScript compiles for stray files, left a graph for each, and recorded the detected languages as the index's choice, so every later refresh solved them too. - graph draws from graph.sqlite when the file table says it is current, and otherwise rebuilds with the recorded --lang/--src/--library (ax_fresh.rebuild_env, shared with the background refresh) - axiomcode-build: a build that names no language keeps an explicit --lang (and --src) recorded in the file table (ax_fresh.py chosen), so the refresh, a broken-graph repair, a bare index and graph never solve a language the index left out; detected languages are still detected again - the page draws every language graph the repository has - prose counts, the page's absolute path, and `axiomcode index --library` as the advice; help, SKILL.md, the MCP description and README agree - tests/graph_verb.py: 21 checks with a detected-languages control --- README.md | 2 +- plugins/axiomcode/mcp/server.py | 2 +- plugins/axiomcode/skills/axiomcode/SKILL.md | 5 +- .../skills/axiomcode/scripts/ax_fresh.py | 29 +- .../skills/axiomcode/scripts/axiomcode | 16 +- .../skills/axiomcode/scripts/axiomcode-build | 22 +- .../skills/axiomcode/scripts/axiomcode-graph | 298 ++++++++++++------ skills/axiomcode/SKILL.md | 5 +- tests/README.md | 3 + tests/graph_verb.py | 165 ++++++++++ 10 files changed, 421 insertions(+), 126 deletions(-) create mode 100644 tests/graph_verb.py diff --git a/README.md b/README.md index 93413fa5b..3bd2a0187 100644 --- a/README.md +++ b/README.md @@ -308,7 +308,7 @@ about one change had to read 95 of 43,793 methods, and every true direct caller | `axiomcode test-impact` | which tests have to run for the current edit, with the chain that reaches each | | `axiomcode changed` | which declarations an edit changed, and how (signature, type, body, added, removed). `--impact` adds what that reaches | | `axiomcode context ""` | where a task's words land in the code, when you have a problem statement and not yet a name | -| `axiomcode graph` | the whole graph as one self-contained HTML page, at `.axiomcode/graph/graph.html` | +| `axiomcode graph` | the whole graph as one self-contained HTML page, at `.axiomcode/graph/graph.html`, drawn from the existing graph (rebuilt first only when stale, with the flags it was indexed with) | | `axiomcode index` | build or rebuild the graph explicitly; `--lang`, `--src` and `--library` narrow it | | `axiomcode mcp` | serve the graph to an agent as MCP tools over stdio | diff --git a/plugins/axiomcode/mcp/server.py b/plugins/axiomcode/mcp/server.py index 86783fa4a..806610123 100755 --- a/plugins/axiomcode/mcp/server.py +++ b/plugins/axiomcode/mcp/server.py @@ -170,7 +170,7 @@ def axiomcode_test_impact(repo: str = ".", range: str = '', staged: bool = False @srv.tool() def axiomcode_graph(repo: str = ".", out: str = '') -> str: - """Draw the graph as one interactive HTML page (/.axiomcode/graph/graph.html, or out=); runs index first if there is no graph yet.""" + """Draw the graph as one interactive HTML page, for a person: every language the repository was indexed in, at /.axiomcode/graph/graph.html or out=. Drawn from the existing graph when it is up to date (seconds, no engine run); a graph that is out of date is rebuilt first with the --lang, --src and --library it was indexed with, never for a language the index left out; with no graph yet the repository is indexed first. Answers with what it drew, in prose, and the page's absolute path.""" return run(['graph', repo] + (['--out', out] if out else [])) if __name__ == '__main__': diff --git a/plugins/axiomcode/skills/axiomcode/SKILL.md b/plugins/axiomcode/skills/axiomcode/SKILL.md index eaba7b67f..90fc4af3e 100644 --- a/plugins/axiomcode/skills/axiomcode/SKILL.md +++ b/plugins/axiomcode/skills/axiomcode/SKILL.md @@ -27,7 +27,7 @@ Read / Grep results as `graph: …` lines. | "how does A reach B" · "everything that reaches X" | `axiomcode path A B` · `axiomcode path '*' X` | | "what did my edit touch" · "which tests do I run" | `axiomcode changed --impact` · `axiomcode test-impact` | | "is it safe to delete X" | `axiomcode impact X --delete` | -| the graph as a page for a human · this repo should prefer the graph, once | `axiomcode graph` · `axiomcode install` | +| the graph as a page for a human · this repo should prefer the graph, once | `axiomcode graph` (drawn from the existing graph in seconds; a stale one is rebuilt first with the flags it was indexed with; prints the page's absolute path) · `axiomcode install` | Rules that decide whether an answer means anything: @@ -38,7 +38,8 @@ Rules that decide whether an answer means anything: briefly on its own only when the answer touches an edited file and the rebuild is nearly done. - **Before a delete or a rename, ask with `--fresh`** (MCP `fresh=true`): it waits for the rebuild, printing its progress, and answers from a graph that includes every edit. - A manual `index` with different flags (e.g. dropping `--library`) rebuilds a worse graph over the good one. + A manual `index` with different flags rebuilds a worse graph over the good one. A bare `index`, the background + refresh and `graph` keep the `--lang` (and `--src`, `--library`) the graph was indexed with; pass `--lang` to change it. - A repo in several languages is indexed in all of them, one graph each, and every query asks each graph; calls are not followed from one language to another. `--lang` restricts it, `--src src` narrows it; `--library ` so calls into dependencies resolve (without it they are `ambiguous_unknown` — do not quote that resolution rate). diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py index 82e9850d7..2c09172de 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_fresh.py @@ -27,6 +27,7 @@ ax_fresh.py worker the worker itself (what kick detaches) ax_fresh.py lock take the build lock on an fd the calling shell holds open ax_fresh.py count the source files of each language, walked as the parser walks + ax_fresh.py chosen the --lang and --src an explicit index chose, which a rebuild keeps Environment: AXIOMCODE_NO_REFRESH=1 turns every trigger off; AXIOMCODE_REFRESH_DEBOUNCE (seconds, default 2) is the quiet window; AXIOMCODE_FRESH_WAIT (seconds, default 30) is the most a query whose answer touches an edited file @@ -309,6 +310,18 @@ def legacy_params(repo): src = os.path.relpath(os.path.realpath(run.get('source_dir') or repo), os.path.realpath(repo)) return dict(lang=run.get('language', ''), src_arg='' if src == '.' or src.startswith('..') else src, library=run.get('library_roots') or '') +def rebuild_env(t, **extra): + """the environment that rebuilds the graph a file table `t` describes, with the language, --src and --library it + was built with: what the background refresh runs, and what `axiomcode graph` runs when the graph is stale. Languages + DETECTED are detected again (a language the repository gained since gets its graph); only a --lang the user gave is + kept. A table from before this was recorded counts as detected: it was built with one language even where the + repository had several, and keeping that would keep the others out for good.""" + env = dict(os.environ, AXIOMCODE_LANG='' if t.get('lang_auto', True) else t.get('lang', ''), AXIOMCODE_SRC=t.get('src_arg', ''), **extra) + env.pop('AXIOMCODE_LIBRARY', None) + env.pop('AXIOMCODE_LANG_AUTO', None); env.pop('AXIOMCODE_GRAPH_LANG', None) + if t.get('library'): env['AXIOMCODE_LIBRARY'] = t['library'] + return env + def _flock(fd, block): """an advisory lock on fd; the OS releases it when the last holder of the file description exits, so a killed build never leaves a lock behind (no stale-lock timeout to guess)""" @@ -447,14 +460,7 @@ def worker(repo): if st.get('failed_table') == fp: return 0 n = sum(len(x) for x in c) why = (f"{n} file(s) changed" if n else f"HEAD moved to {(head(repo) or '')[:10]}") + f", found by {os.environ.get('AXIOMCODE_REFRESH_TRIGGER') or 'an edit'}" - # languages DETECTED are detected again (a language the repository gained since gets its graph); only a - # --lang the user gave is kept. A table from before this was recorded counts as detected: it was built with - # one language even where the repository had several, and keeping that would keep the others out for good - env = dict(os.environ, AXIOMCODE_LANG='' if t.get('lang_auto', True) else t.get('lang', ''), AXIOMCODE_SRC=t.get('src_arg', ''), - AXIOMCODE_BACKGROUND='1', AXIOMCODE_REFRESH_REASON=why) - env.pop('AXIOMCODE_LIBRARY', None) - env.pop('AXIOMCODE_LANG_AUTO', None); env.pop('AXIOMCODE_GRAPH_LANG', None) - if t.get('library'): env['AXIOMCODE_LIBRARY'] = t['library'] + env = rebuild_env(t, AXIOMCODE_BACKGROUND='1', AXIOMCODE_REFRESH_REASON=why) t0 = time.time(); write_state(repo, state='building', started=t0, files=sum(len(x) for x in c)) if any(c): print(f"{time.strftime('%H:%M:%S')} refresh: {sum(len(x) for x in c)} file(s) changed ({', '.join((c[0] + c[1] + c[2])[:5])}) — rebuilding", flush=True) else: print(f"{time.strftime('%H:%M:%S')} refresh: HEAD moved — moving the baseline to it", flush=True) @@ -826,6 +832,13 @@ def main(argv): subdirs[:] = [x for x in subdirs if not prunes('python', x, os.path.basename(d))] n += sum(1 for f in files if python_script(os.path.join(d, f), f)) print(n); return 0 + if cmd == 'chosen': + # the --lang (and --src) the graph here was indexed with, as "\t", when they were CHOSEN (an explicit + # --lang); nothing when its languages were detected, or there is no graph. axiomcode-build keeps them for a rebuild + # that names no language, so no rebuild path solves a language the index left out + t = load_table(repo) + if has_graph(repo) and t and t.get('lang_auto') is False and t.get('lang'): print(f"{t['lang']}\t{t.get('src_arg', '')}") + return 0 if cmd == 'uptodate': # exit 0 when the recorded table was built with this language / --src / --library and no file differs lang, src_arg, lib = argv[3], argv[4] if len(argv) > 4 else '', argv[5] if len(argv) > 5 else '' diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode index f4e6b18d9..b5e608584 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode @@ -10,10 +10,12 @@ # --in is repeatable and takes a list, and several are COMBINED, not intersected, so a change that # spans two roots is answerable in one call. # Ends by saying what it could not see. Every other verb needs a name you already have; this one does not. -# axiomcode graph [] [--out ] [same flags] -# the graph as one page. Runs the pipeline only when the repo has no up-to-date graph (~1 s otherwise). -# defaults to the current directory; the page goes to /.axiomcode/graph/graph.html, or to --out: a folder -# gets /.html, a .html path is used as given. The page embeds the sources, so a node opens its code. +# axiomcode graph [] [--out ] +# the graph as one page, every language the repository was indexed in. Drawn from the existing graph when it is up +# to date (seconds, no engine run); rebuilt first only when a source file changed, and then with the --lang, --src +# and --library it was indexed with, never for a language the index left out; indexed first when there is none. +# Prints what it drew and the page's absolute path: /.axiomcode/graph/graph.html, or --out (a folder gets +# /.html, a .html path is used as given). The page embeds the sources, so a node opens its code. # axiomcode path [] [--every | --paths N] [--all] [--limit N] [--in ] [--json] [--fresh] # the shortest chain of calls from A to B per reached target, and with --every all the routes; through what? # '*' as one endpoint: path '*' X = everything that can reach X, with the entry points among them; path X '*' = everything X reaches. Endpoints exactly as written in the code: Owner.method, @@ -115,8 +117,8 @@ case "${ARGS[0]:-}" in -h|--help) if [ -n "$cmd" ] && [ "${#ARGS[@]}" = 1 ]; the # check and an exec. The repository is the last argument that is a directory, as the verbs themselves take it, else the # working directory. No graph: nothing here, the verb builds its first one. # `changed` and `test-impact` ask about an edit, against the graph of the baseline (kept aside by a refresh), so they do -# not wait for the graph; they wait only when HEAD moved and the baseline has to follow it. `graph` builds (axiomcode-build) -# before it draws, so it is current by construction. +# not wait for the graph; they wait only when HEAD moved and the baseline has to follow it. `graph` checks the graph +# itself: it draws from it when it is current, and rebuilds it with its own recorded flags only when it is not. # A flag's VALUE is never the repository: `--in plugins/core` names a directory under it, and taking that for the # repository found no .axiomcode/lang there, so a repository in several languages was asked in its main graph alone and # a scope that only another language's graph holds was refused as absent. @@ -143,7 +145,7 @@ case "$cmd" in impact) exec python3 ${Q:+"$Q" "$FR"} "$H/axiomcode-impact" ${ARGS[@]+"${ARGS[@]}"} ;; changed) exec python3 ${Q:+"$Q" "$FR"} "$H/axiomcode-changed" ${ARGS[@]+"${ARGS[@]}"} ;; test-impact|tests) exec python3 ${Q:+"$Q" "$FR"} "$H/axiomcode-test-impact" ${ARGS[@]+"${ARGS[@]}"} ;; - graph) exec python3 "$H/axiomcode-graph" build ${ARGS[@]+"${ARGS[@]}"} ;; + graph) exec python3 "$H/axiomcode-graph" ${ARGS[@]+"${ARGS[@]}"} ;; install) exec python3 "$H/axiomcode-install" ${ARGS[@]+"${ARGS[@]}"} ;; --verbs) verbs ;; # what this dispatcher answers to; bin/axiomcode asks, so the two cannot drift ""|-h|--help|help) if [ ${#ARGS[@]} -gt 0 ]; then verbhelp "${ARGS[0]}"; else helptext; fi ;; diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build index a2c2b97d2..decd239ea 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-build @@ -5,7 +5,9 @@ # AXIOMCODE_ENGINE checkout of axiomcodegraph (default: the one this skill lives in) # AXIOMCODE_SRC subtree to analyse (e.g. one module of a monorepo); the graph still lives at /.axiomcode # AXIOMCODE_LANG restrict to java | typescript | python | javascript | csharp, or a comma list of them, the first being -# the main one (default: every language with files; the main one is the language with the most) +# the main one (default: every language with files; the main one is the language with the most). +# A rebuild that names no language keeps the ones an earlier --lang chose (and its --src); pass +# --lang again, or AXIOMCODE_REINDEX=1, to choose again # AXIOMCODE_LIBRARY comma-separated library roots (source trees or IR) passed as --library; without them, calls into # dependencies are declared unknown rather than guessed # A repository in several languages gets one graph per language. The main language's is the one it always had, @@ -15,7 +17,23 @@ # language's compile; until the last one is in, .axiomcode/out/building names the languages still to come. set -euo pipefail H="$(cd "$(dirname "$0")" && pwd)" -REPO="$(cd "${1:-.}" && pwd)"; SRC="$REPO/${AXIOMCODE_SRC:-}"; SRC="${SRC%/}" +REPO="$(cd "${1:-.}" && pwd)" +# A REBUILD THAT NAMES NO LANGUAGE KEEPS THE LANGUAGES THE INDEX CHOSE. Every rebuild comes here: an explicit `index`, +# the background refresh after an edit or a commit, a query that repairs a broken graph, `graph`. One that named no +# language detected every language in the tree, so on a repository indexed with --lang csharp it solved a few stray +# .ts and .js files too (a JavaScript Souffle compile of up to an hour), left a graph for each that later queries +# answered from, and recorded the detected list as the index's choice, so every refresh after it did the same. When +# the graph here was built with an explicit --lang (lang_auto false in its file table), a request with no language +# keeps that --lang, and the --src with it when none is given; a graph whose languages were detected is detected again. +if [ -z "${AXIOMCODE_LANG:-}" ] && [ -z "${AXIOMCODE_REINDEX:-}" ]; then + KEPT="$(python3 "$H/ax_fresh.py" chosen "$REPO" 2>/dev/null || true)" + if [ -n "$KEPT" ]; then + AXIOMCODE_LANG="${KEPT%%$'\t'*}"; export AXIOMCODE_LANG + [ -n "${AXIOMCODE_SRC:-}" ] || { AXIOMCODE_SRC="${KEPT#*$'\t'}"; export AXIOMCODE_SRC; } + [ -n "${AXIOMCODE_BACKGROUND:-}" ] || echo "keeping the languages this graph was indexed with (--lang $AXIOMCODE_LANG${AXIOMCODE_SRC:+ --src $AXIOMCODE_SRC}); pass --lang, or AXIOMCODE_REINDEX=1, to choose again" + fi +fi +SRC="$REPO/${AXIOMCODE_SRC:-}"; SRC="${SRC%/}" # THE ENGINE IS CHOSEN THE SAME WAY EVERY TIME, AND A BUILD THAT FINDS NONE SAYS WHERE IT LOOKED. The background # refresh runs this script from a hook, with the hook's PATH, the hook's Node and, installed from a marketplace, from a # plugin COPIED outside any checkout; an explicit `axiomcode index` runs it from inside the npm package. Each found an diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-graph b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-graph index bd414596c..38a3b38a9 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-graph +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-graph @@ -1,13 +1,25 @@ #!/usr/bin/env python3 -"""axiomcode-graph — the graph of a codebase, from the engine, as one page. +"""axiomcode graph — the graph of a codebase as one interactive page. - axiomcode-graph build [] [--library [,…]] [--out ] run the engine (axiomcode-build) then write the page - axiomcode-graph export [] [--out ] write the page from an existing graph.sqlite (no engine run) + axiomcode graph [] [--out ] + axiomcode graph [] --lang [,…] [--src ] [--library [,…]] [--out …] ---library (or AXIOMCODE_LIBRARY) names the dependency roots — for Java the JDK and the third-party sources/IR. With them a call -into a dependency resolves to a BLACK BOX (one per library type, drawn outside the disc in its own colour); without them it -is unresolved, and the resolution rate is understated. The rate the page reports is over calls into client code; external -calls are counted separately. +With no flags the page is drawn from the graph `axiomcode index` built, and the engine does not run: + - the graph is up to date (no source file differs from the ones it was built from): the page is written from it + in seconds. + - it is out of date: it is rebuilt first exactly as it was indexed, with the same --lang, --src and --library, so + only the languages the index chose are solved and no graph appears for a language it left out. + - there is no graph yet: the repository is indexed first (every language present, or the flags given). +Flags given here are a request for that graph, as `axiomcode index` takes them. + +A repository indexed in several languages is drawn in all of them, on one page. The command prints what it drew as +prose, and the page's absolute path last. `axiomcode graph export []` writes the page from the graph as it is, +with no freshness check. + +--library names the dependency roots (for Java the JDK and the third-party sources/IR), and is given to +`axiomcode index`. With them a call into a dependency resolves to a BLACK BOX (one per library type, drawn outside the +disc in its own colour); without them it is unresolved, and the resolution rate is understated. The rate the page +reports is over calls into client code; external calls are counted separately. The page (default /.axiomcode/graph/graph.html; one file, no network) draws the codebase as a disc: directories in the centre, their files on the next ring, the types in those files, and every method on the @@ -31,57 +43,29 @@ def _b36(n): return s -def export(repo, page=None): +def graph_dbs(repo, gdir): + """every graph to draw: the main language's, then each other language's (.axiomcode/lang/), as the query verbs + ask them. A graph named in AXIOMCODE_GRAPH was chosen by the caller and is drawn alone.""" + dbs = [os.path.join(gdir, 'out', 'graph.sqlite')] + if not os.environ.get('AXIOMCODE_GRAPH'): + ld = os.path.join(repo, '.axiomcode', 'lang') + for l in sorted(os.listdir(ld)) if os.path.isdir(ld) else (): + d = os.path.join(ld, l, 'out', 'graph.sqlite') + if os.path.exists(d): dbs.append(d) + return dbs + + +def export(repo, page=None, how=''): repo = os.path.realpath(repo or '.') gdir = os.environ.get('AXIOMCODE_GRAPH') or os.path.join(repo, '.axiomcode') db = os.path.join(gdir, 'out', 'graph.sqlite') # the same door every other verb goes through: built on demand under the CLI, refused in words # that name the one command that helps under anything else. if not os.path.exists(db) and not ax_contract.ensure_graph(repo, db): sys.exit(ax_contract.no_graph(repo, db)) - con = sqlite3.connect(db); con.row_factory = sqlite3.Row - q = lambda sql: con.execute(sql).fetchall() - has = lambda t: bool(con.execute("SELECT 1 FROM sqlite_master WHERE name=?", (t,)).fetchone()) - run = {r['key']: r['value'] for r in q("SELECT key, value FROM run")} - src = run.get('source_dir') or repo - def rel(p): - a = p if os.path.isabs(p) else os.path.join(src, p) - a = os.path.realpath(a) if os.path.exists(a) else os.path.normpath(a) - r = os.path.relpath(a, repo).replace(os.sep, '/'); return p.lstrip('/') if r.startswith('..') else r - - # ── declarations: types and methods, with the display names the query skill uses when the index is there ── - nodes = {} - owner = {r['id']: r['owner_type_id'] for r in q("SELECT id, owner_type_id FROM methods WHERE provenance='client'")} - if has('symbols'): - for r in q("SELECT * FROM symbols WHERE (method_id IS NOT NULL OR type_id IS NOT NULL)"): - if r['method_id'] and r['kind'] in METHOD_KINDS: - nodes[r['method_id']] = dict(id=r['method_id'], k='method', sk=r['kind'], l=r['display'], f=r['file'], a=r['line'], b=r['end_line'], t=int(bool(r['is_test'])), sig=r['signature'] or '') - elif r['type_id'] and r['kind'] in TYPE_KINDS: - nodes[r['type_id']] = dict(id=r['type_id'], k='type', sk=r['kind'], l=r['display'], f=r['file'], a=r['line'], b=r['end_line'], t=int(bool(r['is_test']))) - else: - cat = lambda c: 'interface' if 'INTERFACE' in c else 'enum' if 'ENUM' in c else 'namespace' if 'NAMESPACE' in c else 'type' if 'ALIAS' in c or 'TYPEDEF' in c else 'class' - for r in q("SELECT * FROM types WHERE provenance='client'"): - f = rel(r['file_path']); nodes[r['id']] = dict(id=r['id'], k='type', sk=cat(r['category']), l=r['name'].replace('$', '.'), f=f, a=r['start_line'], b=r['end_line'], t=int(bool(TESTRE.search(f)))) - for r in q("SELECT * FROM methods WHERE provenance='client'"): - f = rel(r['file_path']); o = (r['owner_qualified_name'] or '').replace('#', '.').split('.')[-1].replace('$', '.') - sk = 'constructor' if 'CONSTRUCTOR' in r['kind'] else 'module' if 'MODULE' in r['kind'] else 'method' if r['owner_type_id'] else 'function' - nodes[r['id']] = dict(id=r['id'], k='method', sk=sk, l=f"{o}.{r['name']}" if o else r['name'], f=f, a=r['start_line'], b=r['end_line'], t=int(bool(TESTRE.search(f))), sig=r['signature'] or '') - lib = {r['id']: (r['owner_qualified_name'] or r['qualified_name'].rsplit('.', 1)[0], r['name']) for r in q("SELECT id, qualified_name, name, owner_qualified_name FROM methods WHERE provenance<>'client'")} - - # ── containment: dir → file → type → method (nested types under their outer; free functions under the file) ── - nesting = {r['type_id']: r['outer_type_id'] for r in q("SELECT type_id, outer_type_id FROM nesting")} if has('nesting') else {} - for n in list(nodes.values()): - fid = 'F:' + n['f'] - if fid not in nodes: - d = os.path.dirname(n['f']) or '.' - nodes[fid] = dict(id=fid, k='file', l=n['f'], f=n['f'], t=int(bool(TESTRE.search(n['f']))), p='D:' + d) - if 'D:' + d not in nodes: nodes['D:' + d] = dict(id='D:' + d, k='dir', l=d, f='', t=int(bool(TESTRE.search(d + '/')))) - if n['k'] == 'type': n['p'] = nesting[n['id']] if nesting.get(n['id']) in nodes else fid - elif n['k'] == 'method': o = owner.get(n['id']); n['p'] = o if o in nodes else fid - - # ── calls: one edge per (caller, callee) with the best tier and the site count; unresolved → a count on the caller ── - # only an ambiguous_* tier is unresolved. boundary_lib is a call into a library; the *_terminal, implicit_constructor and - # fan_capped tiers are correct ends (the platform, a synthesised constructor, a computed target) — the same rule as the - # query skill, so the two never disagree about what is a blind spot + # A REPOSITORY IN SEVERAL LANGUAGES is drawn in all of them: one page, each language's graph merged into it (their ids + # carry the language, so none collide; a directory holding two languages' files is one node). No edge joins two + # languages, as no query follows one. + nodes, owners, lib, nesting, anc, langs, runs = {}, {}, {}, {}, collections.defaultdict(set), [], [] E, unresolved, terminal, LE = {}, collections.Counter(), collections.Counter(), {} boxes = collections.defaultdict(lambda: dict(sites=0, callers=set(), methods=collections.Counter())) # one black box per library type def box(c, owner, name): @@ -90,39 +74,92 @@ def export(repo, page=None): sites = collections.defaultdict(list) # per caller: [line, written name, target id ('' = none), tier] — what makes a call in the source a link def site(c, r, target): if r['line'] and len(sites[c]) < 400: sites[c].append([r['line'], r['written'] or '', target, r['tier']]) - for r in q("SELECT e.caller_id, e.callee_method_id, e.tier, e.callee_label, s.start_line AS line, s.callee_name AS written FROM call_edges e LEFT JOIN call_sites s ON s.id = e.call_site_id"): - c, t = r['caller_id'], r['callee_method_id'] - if c not in nodes: continue - if not t or t not in nodes: - if t in lib: box(c, *lib[t]); site(c, r, 'L:' + lib[t][0]) - elif r['tier'] == 'boundary_lib' and r['callee_label']: - # Python / JavaScript: the library target is a label — builtin:len, external:asyncio.get_running_loop — not a method row - prov, _, name = r['callee_label'].partition(':'); owner, _, meth = name.rpartition('.') - owner = owner or (('builtins' if prov == 'builtin' else name)); box(c, owner, meth or name); site(c, r, 'L:' + owner) - elif (r['tier'] or '').startswith('ambiguous'): unresolved[c] += 1; site(c, r, '') - else: terminal[c] += 1 - continue - site(c, r, t) - e = E.get((c, t)) - if e is None: E[(c, t)] = dict(s=c, t=t, tier=r['tier'], w=1) + skipped, tiers = None, collections.Counter() + for db in graph_dbs(repo, gdir): + con = sqlite3.connect(db); con.row_factory = sqlite3.Row + q = lambda sql: con.execute(sql).fetchall() + has = lambda t: bool(con.execute("SELECT 1 FROM sqlite_master WHERE name=?", (t,)).fetchone()) + run = {r['key']: r['value'] for r in q("SELECT key, value FROM run")}; runs.append(run) + if run.get('language') and run['language'] not in langs: langs.append(run['language']) + src = run.get('source_dir') or repo + def rel(p, src=src): + a = p if os.path.isabs(p) else os.path.join(src, p) + a = os.path.realpath(a) if os.path.exists(a) else os.path.normpath(a) + r = os.path.relpath(a, repo).replace(os.sep, '/'); return p.lstrip('/') if r.startswith('..') else r + + # ── declarations: types and methods, with the display names the query skill uses when the index is there ── + owners.update({r['id']: r['owner_type_id'] for r in q("SELECT id, owner_type_id FROM methods WHERE provenance='client'")}) + if has('symbols'): + for r in q("SELECT * FROM symbols WHERE (method_id IS NOT NULL OR type_id IS NOT NULL)"): + if r['method_id'] and r['kind'] in METHOD_KINDS: + nodes[r['method_id']] = dict(id=r['method_id'], k='method', sk=r['kind'], l=r['display'], f=r['file'], a=r['line'], b=r['end_line'], t=int(bool(r['is_test'])), sig=r['signature'] or '') + elif r['type_id'] and r['kind'] in TYPE_KINDS: + nodes[r['type_id']] = dict(id=r['type_id'], k='type', sk=r['kind'], l=r['display'], f=r['file'], a=r['line'], b=r['end_line'], t=int(bool(r['is_test']))) else: - e['w'] += 1 - if r['tier'] == 'known_edge': e['tier'] = 'known_edge' - # a call through a holder the engine could name (#1206): the caller reaches every function the holder may hold. The call - # site itself resolves to the holder's bodiless base (Function.apply), so without these the stored function has no caller - if has('ext_fn_value_call'): - for r in q("SELECT DISTINCT c0, c1 FROM ext_fn_value_call"): - c, t = r[0], r[1] - if c in nodes and t in nodes and (c, t) not in E: E[(c, t)] = dict(s=c, t=t, tier='fn_value', w=1) + cat = lambda c: 'interface' if 'INTERFACE' in c else 'enum' if 'ENUM' in c else 'namespace' if 'NAMESPACE' in c else 'type' if 'ALIAS' in c or 'TYPEDEF' in c else 'class' + for r in q("SELECT * FROM types WHERE provenance='client'"): + f = rel(r['file_path']); nodes[r['id']] = dict(id=r['id'], k='type', sk=cat(r['category']), l=r['name'].replace('$', '.'), f=f, a=r['start_line'], b=r['end_line'], t=int(bool(TESTRE.search(f)))) + for r in q("SELECT * FROM methods WHERE provenance='client'"): + f = rel(r['file_path']); o = (r['owner_qualified_name'] or '').replace('#', '.').split('.')[-1].replace('$', '.') + sk = 'constructor' if 'CONSTRUCTOR' in r['kind'] else 'module' if 'MODULE' in r['kind'] else 'method' if r['owner_type_id'] else 'function' + nodes[r['id']] = dict(id=r['id'], k='method', sk=sk, l=f"{o}.{r['name']}" if o else r['name'], f=f, a=r['start_line'], b=r['end_line'], t=int(bool(TESTRE.search(f))), sig=r['signature'] or '') + lib.update({r['id']: (r['owner_qualified_name'] or r['qualified_name'].rsplit('.', 1)[0], r['name']) for r in q("SELECT id, qualified_name, name, owner_qualified_name FROM methods WHERE provenance<>'client'")}) + if has('nesting'): nesting.update({r['type_id']: r['outer_type_id'] for r in q("SELECT type_id, outer_type_id FROM nesting")}) + + # ── calls: one edge per (caller, callee) with the best tier and the site count; unresolved → a count on the caller ── + # only an ambiguous_* tier is unresolved. boundary_lib is a call into a library; the *_terminal, implicit_constructor and + # fan_capped tiers are correct ends (the platform, a synthesised constructor, a computed target) — the same rule as the + # query skill, so the two never disagree about what is a blind spot + for r in q("SELECT e.caller_id, e.callee_method_id, e.tier, e.callee_label, s.start_line AS line, s.callee_name AS written FROM call_edges e LEFT JOIN call_sites s ON s.id = e.call_site_id"): + c, t = r['caller_id'], r['callee_method_id'] + if c not in nodes: continue + if not t or t not in nodes: + if t in lib: box(c, *lib[t]); site(c, r, 'L:' + lib[t][0]) + elif r['tier'] == 'boundary_lib' and r['callee_label']: + # Python / JavaScript: the library target is a label — builtin:len, external:asyncio.get_running_loop — not a method row + prov, _, name = r['callee_label'].partition(':'); owner, _, meth = name.rpartition('.') + owner = owner or (('builtins' if prov == 'builtin' else name)); box(c, owner, meth or name); site(c, r, 'L:' + owner) + elif (r['tier'] or '').startswith('ambiguous'): unresolved[c] += 1; site(c, r, '') + else: terminal[c] += 1 + continue + site(c, r, t) + e = E.get((c, t)) + if e is None: E[(c, t)] = dict(s=c, t=t, tier=r['tier'], w=1) + else: + e['w'] += 1 + if r['tier'] == 'known_edge': e['tier'] = 'known_edge' + # a call through a holder the engine could name (#1206): the caller reaches every function the holder may hold. The call + # site itself resolves to the holder's bodiless base (Function.apply), so without these the stored function has no caller + if has('ext_fn_value_call'): + for r in q("SELECT DISTINCT c0, c1 FROM ext_fn_value_call"): + c, t = r[0], r[1] + if c in nodes and t in nodes and (c, t) not in E: E[(c, t)] = dict(s=c, t=t, tier='fn_value', w=1) + for r in q("SELECT method_id, reason FROM entry_points"): + if r['method_id'] in nodes: nodes[r['method_id']].setdefault('e', []).append(r['reason']) + for r in q("SELECT type_id, ancestor_type_id FROM type_ancestors"): anc[r['type_id']].add(r['ancestor_type_id']) + # two shapes share the name: the bundle's (file_path, reason, construct, start_line, …) and the index's (file, reason), which + # the index writes over the bundle's — read whichever is there + if has('skipped'): + cols = {r[1] for r in q("PRAGMA table_info(skipped)")}; col = lambda c: c if c in cols else 'NULL' + fc = 'file_path' if 'file_path' in cols else 'file' + skipped = (skipped or []) + [dict(f=rel(r[0]), r=r[1] or '', a=r[2], d=r[3] or '') for r in q(f"SELECT {fc}, reason, {col('start_line')}, {col('detail')} FROM skipped WHERE {fc} <> '' ORDER BY reason, {fc}")] + tiers.update(r[0] for r in q("SELECT tier FROM call_edges")) + con.close() + + # ── containment: dir → file → type → method (nested types under their outer; free functions under the file) ── + for n in list(nodes.values()): + fid = 'F:' + n['f'] + if fid not in nodes: + d = os.path.dirname(n['f']) or '.' + nodes[fid] = dict(id=fid, k='file', l=n['f'], f=n['f'], t=int(bool(TESTRE.search(n['f']))), p='D:' + d) + if 'D:' + d not in nodes: nodes['D:' + d] = dict(id='D:' + d, k='dir', l=d, f='', t=int(bool(TESTRE.search(d + '/')))) + if n['k'] == 'type': n['p'] = nesting[n['id']] if nesting.get(n['id']) in nodes else fid + elif n['k'] == 'method': o = owners.get(n['id']); n['p'] = o if o in nodes else fid for m, n in unresolved.items(): nodes[m]['u'] = n for m, ss in sites.items(): nodes[m]['cs'] = ss for owner, b in boxes.items(): nodes['L:' + owner] = dict(id='L:' + owner, k='lib', l=owner, f='', sites=b['sites'], callers=len(b['callers']), methods=[f"{k}×{v}" if v > 1 else k for k, v in b['methods'].most_common(8)]) - for r in q("SELECT method_id, reason FROM entry_points"): - if r['method_id'] in nodes: nodes[r['method_id']].setdefault('e', []).append(r['reason']) X = [] # extends / implements: direct ancestors only (type_ancestors is a closure) - anc = collections.defaultdict(set) - for r in q("SELECT type_id, ancestor_type_id FROM type_ancestors"): anc[r['type_id']].add(r['ancestor_type_id']) for t, A in anc.items(): if t in nodes: for a in A: @@ -133,21 +170,13 @@ def export(repo, page=None): for n in nodes.values(): if deg[n['id']]: n['d'] = deg[n['id']] - # two shapes share the name: the bundle's (file_path, reason, construct, start_line, …) and the index's (file, reason), which - # the index writes over the bundle's — read whichever is there - skipped = None - if has('skipped'): - cols = {r[1] for r in q("PRAGMA table_info(skipped)")}; col = lambda c: c if c in cols else 'NULL' - fc = 'file_path' if 'file_path' in cols else 'file' - skipped = [dict(f=rel(r[0]), r=r[1] or '', a=r[2], d=r[3] or '') for r in q(f"SELECT {fc}, reason, {col('start_line')}, {col('detail')} FROM skipped WHERE {fc} <> '' ORDER BY reason, {fc}")] - tiers = collections.Counter(r[0] for r in q("SELECT tier FROM call_edges")) sites = sum(tiers.values()); unres = sum(v for t, v in tiers.items() if t.startswith('ambiguous')) client = tiers.get('known_edge', 0) + tiers.get('multi_inferred', 0); external = sum(b['sites'] for b in boxes.values()) terminal_n = sites - unres - client - external - data = dict(meta=dict(repo=os.path.basename(repo), language=run.get('language', '?'), built=time.strftime('%Y-%m-%d %H:%M'), sites=sites, + data = dict(meta=dict(repo=os.path.basename(repo), language=', '.join(langs) or '?', built=time.strftime('%Y-%m-%d %H:%M'), sites=sites, client_sites=client, external_sites=external, terminal_sites=terminal_n, unresolved=unres, boxes=len(boxes), client_rate=round(100 * client / max(1, sites - external - terminal_n)), overall_rate=round(100 * (sites - unres) / max(1, sites)), - tiers=dict(tiers), library_roots=run.get('library_roots', '') or (run.get('language') != 'java' and 'n/a') or '', counts=dict(collections.Counter(n['k'] for n in nodes.values())), edges=len(E)), + tiers=dict(tiers), library_roots=runs[0].get('library_roots', '') or ('java' not in langs and 'n/a') or '', counts=dict(collections.Counter(n['k'] for n in nodes.values())), edges=len(E)), nodes=list(nodes.values()), calls=list(E.values()), libcalls=list(LE.values()), extends=X, skipped=skipped) data['meta']['fn_value_edges'] = sum(1 for e in E.values() if e['tier'] == 'fn_value') short = {} @@ -174,29 +203,92 @@ def export(repo, page=None): data['files'] = files; data['meta']['repo_path'] = repo; data['meta']['sources'] = bool(files) tpl = open(os.path.join(HERE, 'viewer.html'), encoding='utf-8').read() open(page, 'w', encoding='utf-8').write(tpl.replace('/*__DATA__*/null', json.dumps(data, separators=(',', ':')).replace(',`")) - print(f"page: {os.path.relpath(page)}") + m, n = data['meta'], data['meta']['counts'] + # PROSE, NOT A DICT: an agent relays this line to a person, and a raw {'dir': …} or a ../../../ path it had to resolve + # itself was what it relayed + if how: print(how) + k = lambda v, one, many: f"{v:,} {one if v == 1 else many}" + print(f"graph of {repo} ({m['language']}): {k(n.get('method', 0), 'function or method', 'functions and methods')}, " + f"{k(n.get('type', 0), 'type', 'types')}, {k(n.get('file', 0), 'file', 'files')} in {k(n.get('dir', 0), 'directory', 'directories')}; " + f"{k(m['edges'], 'call edge', 'call edges')}, {len(X):,} extends/implements.") + print(f"calls: {k(m['sites'], 'call site', 'call sites')}. Of the calls into this repository's own code {m['client_rate']}% resolved " + f"({m['client_sites']:,} resolved, {m['unresolved']:,} unresolved); {m['external_sites']:,} go into " + f"{k(m['boxes'], 'library type', 'library types')}, drawn as black boxes; {m['terminal_sites']:,} end at the platform.") + if not m['library_roots']: + print(f"indexed without library roots: calls into the JDK and other dependencies cannot resolve and count as unresolved. " + f"To resolve them, index again with them: axiomcode index {repo} --library ,") + print(f"page: {os.path.abspath(page)}") -def build(repo, page, library=None): - repo = os.path.realpath(repo or '.') - if library: os.environ['AXIOMCODE_LIBRARY'] = ','.join(os.path.abspath(os.path.expanduser(x)) for x in library.split(',')) +def run_build(repo, env=None): + """axiomcode-build on repo, its progress on this process's stdout; exits with its status when it fails""" cands = [os.path.join(HERE, 'axiomcode-build'), os.path.join(os.environ.get('AXIOMCODE_ENGINE', ''), 'skills', 'axiomcode', 'scripts', 'axiomcode-build')] b = next((os.path.realpath(c) for c in cands if os.path.isfile(c)), None) if not b: sys.exit("axiomcode-build not found beside this script (set AXIOMCODE_ENGINE to the engine checkout)") - r = subprocess.run([os.environ.get('AXIOMCODE_BASH') or 'bash', b, repo]) # a bash script: Windows cannot exec it itself + sys.stdout.flush() + r = subprocess.run([os.environ.get('AXIOMCODE_BASH') or 'bash', b, repo], env=env) # a bash script: Windows cannot exec it itself if r.returncode: sys.exit(r.returncode) - export(repo, page) + + +def recorded(repo): + """the file table of the graph that is there: the languages, --src and --library it was indexed with. A graph from + before the table has its main language, --src and --library in its run table; its other languages are the graphs + beside it. None when there is nothing to read them from.""" + import ax_fresh + t = ax_fresh.load_table(repo) + if t: return t + t = ax_fresh.legacy_params(repo) + if not t: return None + ld = os.path.join(repo, '.axiomcode', 'lang') + others = sorted(l for l in (os.listdir(ld) if os.path.isdir(ld) else ()) if os.path.exists(os.path.join(ld, l, 'out', 'graph.sqlite'))) + return dict(t, lang=','.join([t['lang']] + [l for l in others if l != t['lang']]), lang_auto=False, legacy=True) + + +def build(repo, page, library=None): + """the page, from the graph that is there when it is current, else from a rebuild of it with the flags it was indexed + with. THE ENGINE RUNS ONLY WHEN IT HAS TO, AND ONLY FOR THE LANGUAGES THE INDEX CHOSE. This called axiomcode-build with + no flags, which detects every language in the tree: on a repository indexed with --lang python its answer never matched + the recorded graph, so every call rebuilt the graph from scratch in every language present, a JavaScript or TypeScript + compile for a few stray files included, and left a graph for each of them that later queries answered from.""" + import ax_fresh + repo = os.path.realpath(repo or '.') + if library: os.environ['AXIOMCODE_LIBRARY'] = ','.join(os.path.abspath(os.path.expanduser(x)) for x in library.split(',')) + db = os.path.join(repo, '.axiomcode', 'out', 'graph.sqlite') + asked = [f"{k}={os.environ[v]}" for k, v in (('--lang', 'AXIOMCODE_LANG'), ('--src', 'AXIOMCODE_SRC'), ('--library', 'AXIOMCODE_LIBRARY')) if os.environ.get(v)] + if os.environ.get('AXIOMCODE_GRAPH'): return export(repo, page) # a graph someone else built and placed: drawn as it is + if not os.path.lexists(db): + # no graph yet: the first index, with what was asked for (every language present, when nothing was) + t0 = time.time(); run_build(repo) + return export(repo, page, how=f"no graph yet: indexed {repo}{' with ' + ' '.join(asked) if asked else ''} in {time.time() - t0:.0f} s.") + if asked: + # flags given here are a request for that graph: axiomcode-build keeps the one there when it already is that graph + t0 = time.time(); run_build(repo) + return export(repo, page, how=f"graph built with {' '.join(asked)} ({time.time() - t0:.0f} s).") + t = recorded(repo) + if t is None: + return export(repo, page, how="drawn from the graph as it is: it records no index flags to rebuild it with.") + c = None if t.get('legacy') else ax_fresh.changes(repo, t) + langs = (t.get('lang') or '?').replace(',', ', ') + if c is not None and not any(c) and not ax_fresh.graph_broken(repo): + return export(repo, page, how=f"graph up to date ({langs}): drawn from it, no rebuild.") + n = sum(len(x) for x in c) if c else 0 + flags = ' '.join(f for f in (('--lang ' + t['lang']) if not t.get('lang_auto', True) else 'every language present', + ('--src ' + t['src_arg']) if t.get('src_arg') else '', ('--library ' + t['library']) if t.get('library') else '') if f) + print(f"the graph is out of date ({n} file(s) changed since it was built): rebuilding it as it was indexed ({flags}) …" if n else + f"rebuilding the graph as it was indexed ({flags}) …", flush=True) + t0 = time.time(); run_build(repo, ax_fresh.rebuild_env(t, AXIOMCODE_REFRESH_REASON='axiomcode graph')) + return export(repo, page, how=f"graph rebuilt ({flags}) in {time.time() - t0:.0f} s.") if __name__ == '__main__': a = sys.argv[1:]; page = None; library = None if '--out' in a: i = a.index('--out'); page = os.path.abspath(os.path.expanduser(a[i + 1])); del a[i:i + 2] if '--library' in a: i = a.index('--library'); library = a[i + 1]; del a[i:i + 2] - verb, pos = (a[0] if a else ''), a[1:] - if verb == 'build': build(pos[0] if pos else '.', page, library) - elif verb == 'export': export(pos[0] if pos else '.', page) - else: print(__doc__.strip()); sys.exit(0 if verb in ('', '-h', '--help') else 2) + if a and a[0] in ('-h', '--help'): print(__doc__.strip()); sys.exit(0) + # `axiomcode graph []` is the verb; `build` and `export` are the older spellings, still taken. A directory + # named `build` or `export` is a repository, not a subcommand + verb = a[0] if a and a[0] in ('build', 'export', 'draw') and not os.path.isdir(a[0]) else 'draw' + pos = a[1:] if a and verb == a[0] else a + if len(pos) > 1 or any(x.startswith('-') for x in pos): print(__doc__.strip(), file=sys.stderr); sys.exit(2) + if verb == 'export': export(pos[0] if pos else '.', page, how="drawn from the graph as it is (export: no freshness check, no rebuild).") + else: build(pos[0] if pos else '.', page, library) diff --git a/skills/axiomcode/SKILL.md b/skills/axiomcode/SKILL.md index b2923a602..80ea06323 100644 --- a/skills/axiomcode/SKILL.md +++ b/skills/axiomcode/SKILL.md @@ -27,7 +27,7 @@ Read / Grep results as `graph: …` lines. | "how does A reach B" · "everything that reaches X" | `axiomcode path A B` · `axiomcode path '*' X` | | "what did my edit touch" · "which tests do I run" | `axiomcode changed --impact` · `axiomcode test-impact` | | "is it safe to delete X" | `axiomcode impact X --delete` | -| the graph as a page for a human · this repo should prefer the graph, once | `axiomcode graph` · `axiomcode install` | +| the graph as a page for a human · this repo should prefer the graph, once | `axiomcode graph` (drawn from the existing graph in seconds; a stale one is rebuilt first with the flags it was indexed with; prints the page's absolute path) · `axiomcode install` | Rules that decide whether an answer means anything: @@ -38,7 +38,8 @@ Rules that decide whether an answer means anything: briefly on its own only when the answer touches an edited file and the rebuild is nearly done. - **Before a delete or a rename, ask with `--fresh`** (MCP `fresh=true`): it waits for the rebuild, printing its progress, and answers from a graph that includes every edit. - A manual `index` with different flags (e.g. dropping `--library`) rebuilds a worse graph over the good one. + A manual `index` with different flags rebuilds a worse graph over the good one. A bare `index`, the background + refresh and `graph` keep the `--lang` (and `--src`, `--library`) the graph was indexed with; pass `--lang` to change it. - A repo in several languages is indexed in all of them, one graph each, and every query asks each graph; calls are not followed from one language to another. `--lang` restricts it, `--src src` narrows it; `--library ` so calls into dependencies resolve (without it they are `ambiguous_unknown` — do not quote that resolution rate). diff --git a/tests/README.md b/tests/README.md index c75741715..f0d520e6c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -18,6 +18,9 @@ One check needs no graph and is its own script: python3 tests/refresh.py the graph refreshes itself after an edit in every language: a query sees the edit, `changed` answers the same before and after, a burst costs one rebuild and queries during it answer (#1305; builds real graphs, needs the engine) + python3 tests/graph_verb.py `axiomcode graph` draws the existing graph and rebuilds a stale one with the flags it + was indexed with; no rebuild path (refresh, repair, bare index) solves a language an + explicit --lang left out (builds real graphs, needs the engine) python3 tests/indexed_tree.py changed compares against the tree the graph was indexed from, so an index taken with uncommitted edits reports only later edits (#1222) python3 tests/mcp.py `axiomcode mcp` answers initialize, lists every tool and runs one, diff --git a/tests/graph_verb.py b/tests/graph_verb.py new file mode 100644 index 000000000..5c9fa025f --- /dev/null +++ b/tests/graph_verb.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""tests/graph_verb.py — `axiomcode graph` draws the graph the index built, and rebuilds it only as it was indexed. + +It used to run axiomcode-build with no flags. That detects every language in the tree, so on a repository indexed with +--lang python the build never matched the recorded graph: every call rebuilt it from scratch in every language +present, a JavaScript or TypeScript compile for a few stray files included, and left a graph for each of them that +later queries answered from. Its output was a raw Python dict and a relative ../../ page path. + +Two throwaway repositories: + + indexed Python under app/, with a stray JavaScript file beside it, indexed with --lang python --src app + current the page is drawn from the graph: no engine run (the graph file is untouched), no graph for another + language, prose counts and the page's absolute path + stale after an edit the graph is rebuilt with --lang python --src app, the page shows the edit, and still no + JavaScript graph appears + control Python and Java, indexed with no --lang (every language): the page draws both, and both graphs stay; + detected languages are not pinned, so a rebuild still detects them + rebuilds C# with a stray TypeScript front end, indexed with --lang csharp: an edit plus a commit (the background + refresh), a bare `index` and a query that repairs a broken graph each solve C# alone + surfaces `axiomcode help graph` and the MCP tool's description describe this, not `axiomcode-graph build` + + python3 tests/graph_verb.py [-v] +""" +import os, shutil, subprocess, sys, tempfile, time + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +AX = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode') +MCP = os.path.join(ROOT, 'plugins', 'axiomcode', 'mcp', 'server.py') +FRESH = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'ax_fresh.py') + +INDEXED = { + 'app/shop/__init__.py': '', + 'app/shop/orders.py': 'def price(n):\n return n * 3\n\n\ndef order_total(n):\n return price(n) + 1\n', + 'app/shop/cli.py': 'from shop.orders import order_total\n\n\ndef main():\n return order_total(2)\n', + 'app/static/widget.js': 'function strayWidget(a) {\n return a + 1\n}\n\nmodule.exports = { strayWidget }\n', + 'tools/other.py': 'def outside_src():\n return 1\n', +} +CSHARP = { + 'App/App.csproj': '\n net8.0\n\n', + 'App/Orders.cs': 'namespace App;\n\npublic class OrderService\n{\n public decimal Total(int widgets) => Price(widgets) * 2;\n\n private decimal Price(int widgets) => widgets * 3m;\n}\n', + 'App/ClientApp/tsconfig.json': '{ "compilerOptions": { "strict": true } }\n', + 'App/ClientApp/main.ts': 'export function strayFrontEnd(): number {\n return 1\n}\n', +} +MIXED = { + 'pom.xml': '\n 4.0.0\n example\n mixed\n 1.0\n\n', + 'src/main/java/app/Billing.java': 'package app;\n\npublic class Billing {\n public int chargeAccount(int n) {\n return fee(n) + n;\n }\n\n int fee(int n) {\n return 2;\n }\n}\n', + 'src/main/java/app/Invoice.java': 'package app;\n\npublic class Invoice {\n public int issueInvoice() {\n return new Billing().chargeAccount(3);\n }\n}\n', + 'scripts/report/__init__.py': '', + 'scripts/report/make.py': 'def render_report(x):\n return str(x)\n\n\ndef main():\n return render_report(1)\n', +} + + +def sh(cwd, *cmd, env=None): + return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + + +def make(root, files): + for rel, text in files.items(): + os.makedirs(os.path.dirname(os.path.join(root, rel)), exist_ok=True) + open(os.path.join(root, rel), 'w').write(text) + for cmd in (('git', 'init', '-q'), ('git', 'add', '-A'), ('git', '-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qm', 'base')): + sh(root, *cmd) + + +def main(argv): + verbose = '-v' in argv + fails = [] + + def check(ok, why, detail=''): + print(('ok ' if ok else 'FAIL ') + why + ('' if ok and not verbose or not detail else '\n ' + detail.strip()[-1500:].replace('\n', '\n '))) + if not ok: fails.append(why) + + work = os.path.realpath(tempfile.mkdtemp(prefix='axiomcode-graphverb-')) # macOS: /var is /private/var, and the page is named by its real path + # the engine is this checkout's, unless the caller names a built one (a worktree whose parser is not built) + env = dict(os.environ, AXIOMCODE_ENGINE=os.environ.get('AXIOMCODE_ENGINE') or ROOT, AXIOMCODE_NO_REFRESH='1') + for k in ('AXIOMCODE_LANG', 'AXIOMCODE_SRC', 'AXIOMCODE_LIBRARY', 'AXIOMCODE_GRAPH'): env.pop(k, None) + try: + # ── indexed with --lang python --src app ────────────────────────────────────────────────────────────── + repo = os.path.join(work, 'indexed'); make(repo, INDEXED) + ax = os.path.join(repo, '.axiomcode'); db = os.path.join(ax, 'out', 'graph.sqlite') + b = sh(repo, 'bash', AX, 'index', repo, '--lang', 'python', '--src', 'app', env=env) + check(b.returncode == 0 and os.path.exists(db), 'setup: the repository indexes with --lang python --src app', b.stdout + b.stderr) + if b.returncode: return 1 + before = (os.path.realpath(db), os.stat(os.path.realpath(db)).st_mtime_ns) + + t0 = time.time(); g = sh(repo, 'bash', AX, 'graph', repo, env=env); took = time.time() - t0 + page = os.path.join(ax, 'graph', 'graph.html') + check(g.returncode == 0 and os.path.exists(page), 'current: the page is written', g.stdout + g.stderr) + check((os.path.realpath(db), os.stat(os.path.realpath(db)).st_mtime_ns) == before and 'no rebuild' in g.stdout and 'building' not in g.stdout, + f'current: the page is drawn from the graph there, the engine does not run ({took:.1f} s)', g.stdout + g.stderr) + check(not os.path.exists(os.path.join(ax, 'lang')) and not os.path.exists(os.path.join(ax, 'out', 'javascript')), + 'current: no graph appears for the stray JavaScript file the index left out', g.stdout) + check(f'page: {page}' in g.stdout and '../' not in g.stdout and "{'" not in g.stdout and ' functions and methods' in g.stdout, + 'current: the output is prose, and names the page by its absolute path', g.stdout) + check('axiomcode-graph build' not in g.stdout + g.stderr, 'current: it advises no command an agent does not have', g.stdout + g.stderr) + html = open(page).read() if os.path.exists(page) else '' + check('order_total' in html and 'strayWidget' not in html and 'outside_src' not in html, + 'current: the page holds the indexed Python and nothing outside --lang / --src', '') + + # an edit: the graph is stale, and is rebuilt as it was indexed + with open(os.path.join(repo, 'app', 'shop', 'orders.py'), 'a') as f: f.write('\n\ndef refund_order(n):\n return -order_total(n)\n') + g = sh(repo, 'bash', AX, 'graph', repo, env=env) + html = open(page).read() if os.path.exists(page) else '' + check(g.returncode == 0 and 'refund_order' in html and 'rebuilt (--lang python --src app)' in g.stdout, + 'stale: the graph is rebuilt with the --lang and --src it was indexed with, and the page shows the edit', g.stdout + g.stderr) + check(not os.path.exists(os.path.join(ax, 'lang')) and 'building python graph' in g.stdout and 'javascript graph' not in g.stdout and '+ javascript' not in g.stdout, + 'stale: the rebuild solves Python alone; no JavaScript graph appears', g.stdout) + check('outside_src' not in html, 'stale: the rebuild keeps --src (a file outside it is not drawn)', '') + + # ── control: indexed with no --lang, every language present ────────────────────────────────────────── + mixed = os.path.join(work, 'mixed'); make(mixed, MIXED) + b = sh(mixed, 'bash', AX, 'index', mixed, env=env) + mx = os.path.join(mixed, '.axiomcode') + both = [os.path.exists(os.path.join(mx, 'out', 'graph.sqlite')), os.path.exists(os.path.join(mx, 'lang', 'python', 'out', 'graph.sqlite'))] + check(b.returncode == 0 and all(both), 'control: a Java and Python repository indexed with no --lang has both graphs', b.stdout + b.stderr) + g = sh(mixed, 'bash', AX, 'graph', mixed, env=env) + mpage = os.path.join(mx, 'graph', 'graph.html') + html = open(mpage).read() if os.path.exists(mpage) else '' + check(g.returncode == 0 and 'chargeAccount' in html and 'render_report' in html and any(l.startswith('graph of ') and 'java' in l and 'python' in l for l in g.stdout.splitlines()), + 'control: the page draws every language the repository was indexed in', g.stdout + g.stderr) + check(os.path.exists(os.path.join(mx, 'lang', 'python', 'out', 'graph.sqlite')) and 'no rebuild' in g.stdout, + 'control: drawing it keeps both graphs and rebuilds neither', g.stdout) + c = sh(mixed, sys.executable, FRESH, 'chosen', mixed, env=env) + check(c.returncode == 0 and c.stdout.strip() == '', 'control: languages that were detected are not pinned; a rebuild detects them again', c.stdout) + + # ── every rebuild path: C# indexed with --lang csharp, a stray TypeScript front end beside it ───────────── + cs = os.path.join(work, 'csharp'); make(cs, CSHARP); cx = os.path.join(cs, '.axiomcode') + b = sh(cs, 'bash', AX, 'index', cs, '--lang', 'csharp', env=env) + check(b.returncode == 0 and os.path.exists(os.path.join(cx, 'out', 'graph.sqlite')) and not os.path.exists(os.path.join(cx, 'lang')), + 'rebuilds: a C# repository indexes with --lang csharp and no TypeScript graph', b.stdout + b.stderr) + c = sh(cs, sys.executable, FRESH, 'chosen', cs, env=env) + check(c.stdout.strip() == 'csharp', 'rebuilds: the file table records csharp as chosen', c.stdout) + # an edit, committed, then the background refresh: the path a hook or the MCP server's timer starts + with open(os.path.join(cs, 'App', 'Orders.cs'), 'a') as f: f.write('\n// committed edit\n') + sh(cs, 'git', '-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qam', 'edit') + w = sh(cs, sys.executable, FRESH, 'worker', cs, env=dict(env, AXIOMCODE_REFRESH_DEBOUNCE='0.3')) + stamp = lambda: open(os.path.join(cx, 'out', 'stamp')).read().strip() if os.path.exists(os.path.join(cx, 'out', 'stamp')) else '' + no_ts = lambda: not os.path.exists(os.path.join(cx, 'lang')) and not os.path.exists(os.path.join(cx, 'out', 'typescript')) + check(w.returncode == 0 and 'rebuilding' in w.stdout and '-csharp-' in stamp() and no_ts(), + 'rebuilds: an edit plus a commit refreshes C# alone; no TypeScript graph, no TypeScript solve', w.stdout + stamp()) + # a bare index, as an agent re-runs one "to make sure" + b = sh(cs, 'bash', AX, 'index', cs, env=env) + check(b.returncode == 0 and 'keeping the languages this graph was indexed with (--lang csharp)' in b.stdout and '-csharp-' in stamp() and no_ts(), + 'rebuilds: a bare index keeps --lang csharp and says so', b.stdout + b.stderr) + # a query that finds the graph pointer broken repairs it, with the same languages + ptr = os.path.join(cx, 'out', 'graph.sqlite'); tgt = os.path.realpath(ptr) + os.rename(tgt, tgt + '.gone') + q = sh(cs, 'bash', AX, 'impact', 'OrderService.Total', cs, env=env) + check(q.returncode == 0 and os.path.exists(ptr) and '-csharp-' in stamp() and no_ts(), + 'rebuilds: a query repairing a broken graph builds C# alone', q.stdout[-600:] + q.stderr[-600:]) + + # ── surfaces ────────────────────────────────────────────────────────────────────────────────────────── + h = sh(ROOT, 'bash', AX, 'help', 'graph', env=env) + check(h.returncode == 0 and 'axiomcode graph []' in h.stdout and 'axiomcode-graph build' not in h.stdout and 'as it was indexed' in h.stdout, + '`axiomcode help graph` names the verb agents call and says a stale graph is rebuilt as it was indexed', h.stdout) + doc = open(MCP).read(); i = doc.index('def axiomcode_graph('); doc = doc[i:i + 1200] + check('it was indexed with' in doc and 'absolute path' in doc, 'the MCP axiomcode_graph description says the same', doc) + finally: + shutil.rmtree(work, ignore_errors=True) + print(f"\n{'FAIL' if fails else 'ok'}: {len(fails)} of the checks above failed" if fails else '\nok: every check passed') + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv))