From b79147e4aed46db11dae6b01fa2c03480be4688d Mon Sep 17 00:00:00 2001 From: swapnil Date: Wed, 30 Sep 2026 02:39:29 -0700 Subject: [PATCH 1/2] tests/run.py --jobs N runs cases side by side; CI runs the query cases four at a time The query-case step made engine (typescript) a 28-minute job (19 of them the cases, one after another). Each case indexes its own directory and shares nothing but the compiled rules, so the first case runs alone (it compiles and caches them) and the rest run N at a time; each case's output is printed as one block, in case order. TypeScript locally: 175 s -> 41 s with --jobs 6, the same 204/204 checks and the same case order and outcomes. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- tests/run.py | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5987254c..cb6710f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -456,7 +456,7 @@ jobs: env: AXIOM_PARSER: ${{ github.workspace }}/parser/dist/index.js AXIOM_SOUFFLE_CACHE: ${{ github.workspace }}/.souffle-cache - run: python3 tests/run.py --lang ${{ matrix.lang }} + run: python3 tests/run.py --lang ${{ matrix.lang }} --jobs 4 # the small surface as users and agents get it: find / impact / path / tests through the installed command and # the MCP server, answered as places with their code, and the direct calls and flags that keep the old answers diff --git a/tests/run.py b/tests/run.py index a35f564a..863a7c06 100755 --- a/tests/run.py +++ b/tests/run.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""tests/run.py [ …] [--lang java|python|typescript|javascript] [--keep] [-v] +"""tests/run.py [ …] [--lang java|python|typescript|javascript] [--keep] [-v] [--jobs N] What the plugin CLAIMS to find, checked on code that is small enough to read. Each case is a directory under tests/cases/// holding a tiny synthetic project and a case.json: @@ -32,26 +32,29 @@ and neither announces itself as a want/avoid mismatch. If anyone ever "simplifies" `pending` into a skip, that is the property they will have removed. """ -import json, os, re, shutil, subprocess, sys +import builtins, concurrent.futures, io, json, os, re, shutil, subprocess, sys HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE) AX = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode') args = sys.argv[1:]; keep = '--keep' in args; verbose = '-v' in args lang = args[args.index('--lang') + 1] if '--lang' in args else None -only = [a for a in args if not a.startswith('-') and a not in (lang,)] +jobs = int(args[args.index('--jobs') + 1]) if '--jobs' in args else 1 +only = [a for a in args if not a.startswith('-') and a not in (lang, str(jobs))] cases = [] for l in sorted(os.listdir(os.path.join(HERE, 'cases'))): if lang and l != lang: continue d = os.path.join(HERE, 'cases', l) for c in sorted(os.listdir(d)): if os.path.isfile(os.path.join(d, c, 'case.json')) and (not only or c in only or l in only): cases.append((l, c, os.path.join(d, c))) -fail = tot = pend = 0 -for l, name, path in cases: +def run_case(case): + """one case: index it, run its checks; its output as one block and its counts, so cases can run side by side""" + l, name, path = case; buf = io.StringIO(); fail = tot = pend = 0 + def print(*a, flush=False, **k): builtins.print(*a, file=buf, **k) print(f"… {l}/{name}", flush=True) spec = json.load(open(os.path.join(path, 'case.json'))) build = ['bash', AX, 'index', path, '--lang', spec.get('lang', l)] + (['--src', spec['src']] if spec.get('src') else []) \ + (['--library', os.path.join(path, spec['library'])] if spec.get('library') else []) # a staged dependency root, relative to the case r = subprocess.run(build, capture_output=True, text=True) - if r.returncode: print(f"FAIL {l}/{name}: index failed: {(r.stderr or r.stdout)[-300:]}"); fail += 1; continue + if r.returncode: print(f"FAIL {l}/{name}: index failed: {(r.stderr or r.stdout)[-300:]}"); return buf.getvalue(), 0, 1, 0 for stmt in spec.get('sql', []): # facts a framework extension would have written subprocess.run(['sqlite3', os.path.join(path, '.axiomcode', 'out', 'graph.sqlite'), stmt], capture_output=True, text=True) for ch in spec['checks']: @@ -97,5 +100,17 @@ print(' ' + '\n '.join(text.strip().split('\n')[:14])) elif verbose: print(f"ok {l}/{name}: {ch['why']}") if not keep: shutil.rmtree(os.path.join(path, '.axiomcode'), ignore_errors=True) + return buf.getvalue(), tot, fail, pend + + +# CASES RUN SIDE BY SIDE with --jobs N: each indexes its own directory and shares nothing but the compiled rules, so the +# first case runs alone (it compiles and caches them) and the rest run N at a time. Output is printed in case order. +fail = tot = pend = 0 +def report(res): + global fail, tot, pend + text, t, f, p = res; sys.stdout.write(text); sys.stdout.flush(); tot += t; fail += f; pend += p +if cases: report(run_case(cases[0])) +with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, jobs)) as ex: + for res in ex.map(run_case, cases[1:]): report(res) print(f"\n{tot - fail - pend} of {tot} check(s) passed in {len(cases)} case(s)" + (f" - {pend} PENDING" if pend else '') + ('' if not fail else f" - {fail} FAILED")) sys.exit(1 if fail else 0) From 1d50368e31470ddd27144b5667748ce340b5eb35 Mon Sep 17 00:00:00 2001 From: swapnil Date: Wed, 30 Sep 2026 02:55:39 -0700 Subject: [PATCH 2/2] tests: the uncertain-row evidence case runs in each language's own leg typescript/evidence-for-uncertain-rows indexed five languages, so the typescript leg compiled the java, csharp, python and javascript engines it has no cache for: 964 s of the leg's 19-minute query-case step, where the median case takes 4 s. The case is now one per language (ts 9 checks, js 1, python 1, java 2, csharp 1), each indexed in the leg whose engine is already built; the 14 checks are unchanged. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../evidence-for-uncertain-rows/case.json | 21 +++ .../evidence-for-uncertain-rows/cs/Stock.cs | 0 .../evidence-for-uncertain-rows/case.json | 31 ++++ .../java/app/Cache.java | 0 .../java/app/Pipeline.java | 0 .../java/app/Store.java | 0 .../evidence-for-uncertain-rows/case.json | 22 +++ .../evidence-for-uncertain-rows/js/client.js | 0 .../evidence-for-uncertain-rows/js/store.js | 0 .../evidence-for-uncertain-rows/case.json | 22 +++ .../evidence-for-uncertain-rows/py/billing.py | 0 .../evidence-for-uncertain-rows/py/gateway.py | 0 .../evidence-for-uncertain-rows/case.json | 171 ++++++++++++------ 13 files changed, 213 insertions(+), 54 deletions(-) create mode 100644 tests/cases/csharp/evidence-for-uncertain-rows/case.json rename tests/cases/{typescript => csharp}/evidence-for-uncertain-rows/cs/Stock.cs (100%) create mode 100644 tests/cases/java/evidence-for-uncertain-rows/case.json rename tests/cases/{typescript => java}/evidence-for-uncertain-rows/java/app/Cache.java (100%) rename tests/cases/{typescript => java}/evidence-for-uncertain-rows/java/app/Pipeline.java (100%) rename tests/cases/{typescript => java}/evidence-for-uncertain-rows/java/app/Store.java (100%) create mode 100644 tests/cases/javascript/evidence-for-uncertain-rows/case.json rename tests/cases/{typescript => javascript}/evidence-for-uncertain-rows/js/client.js (100%) rename tests/cases/{typescript => javascript}/evidence-for-uncertain-rows/js/store.js (100%) create mode 100644 tests/cases/python/evidence-for-uncertain-rows/case.json rename tests/cases/{typescript => python}/evidence-for-uncertain-rows/py/billing.py (100%) rename tests/cases/{typescript => python}/evidence-for-uncertain-rows/py/gateway.py (100%) diff --git a/tests/cases/csharp/evidence-for-uncertain-rows/case.json b/tests/cases/csharp/evidence-for-uncertain-rows/case.json new file mode 100644 index 00000000..f745109f --- /dev/null +++ b/tests/cases/csharp/evidence-for-uncertain-rows/case.json @@ -0,0 +1,21 @@ +{ + "lang": "csharp", + "src": ".", + "checks": [ + { + "why": "C#: a dynamic field is why the call is a name-match; the resolved call on the typed field beside it gets nothing", + "run": [ + "impact", + "cs/Stock.cs:5", + "--evidence", + "--grep" + ], + "want": [ + "decided L11: private readonly dynamic _meter; [field \u00b7 type dynamic]" + ], + "avoid": [ + "decided L10" + ] + } + ] +} diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/cs/Stock.cs b/tests/cases/csharp/evidence-for-uncertain-rows/cs/Stock.cs similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/cs/Stock.cs rename to tests/cases/csharp/evidence-for-uncertain-rows/cs/Stock.cs diff --git a/tests/cases/java/evidence-for-uncertain-rows/case.json b/tests/cases/java/evidence-for-uncertain-rows/case.json new file mode 100644 index 00000000..4ba282c0 --- /dev/null +++ b/tests/cases/java/evidence-for-uncertain-rows/case.json @@ -0,0 +1,31 @@ +{ + "lang": "java", + "src": ".", + "checks": [ + { + "why": "Java: a one-of-a-set call through a JDK functional interface decides on the field that holds it", + "run": [ + "impact", + "java/app/Pipeline.java:7", + "--evidence", + "--grep" + ], + "want": [ + "decided L6: private final Function loader; [field \u00b7 type Function]" + ] + }, + { + "why": "Java: path's hop through the same call carries the same decider", + "run": [ + "path", + "Cache.load", + "Pipeline.apply", + "--evidence", + "--grep" + ], + "want": [ + "private final Function loader;" + ] + } + ] +} diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/java/app/Cache.java b/tests/cases/java/evidence-for-uncertain-rows/java/app/Cache.java similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/java/app/Cache.java rename to tests/cases/java/evidence-for-uncertain-rows/java/app/Cache.java diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/java/app/Pipeline.java b/tests/cases/java/evidence-for-uncertain-rows/java/app/Pipeline.java similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/java/app/Pipeline.java rename to tests/cases/java/evidence-for-uncertain-rows/java/app/Pipeline.java diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/java/app/Store.java b/tests/cases/java/evidence-for-uncertain-rows/java/app/Store.java similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/java/app/Store.java rename to tests/cases/java/evidence-for-uncertain-rows/java/app/Store.java diff --git a/tests/cases/javascript/evidence-for-uncertain-rows/case.json b/tests/cases/javascript/evidence-for-uncertain-rows/case.json new file mode 100644 index 00000000..e5351252 --- /dev/null +++ b/tests/cases/javascript/evidence-for-uncertain-rows/case.json @@ -0,0 +1,22 @@ +{ + "lang": "javascript", + "src": ".", + "checks": [ + { + "why": "JavaScript: an untyped parameter is the decider of a name-match; a field assigned in the constructor decides nothing that is resolved", + "run": [ + "impact", + "js/store.js:2", + "--evidence", + "--grep" + ], + "want": [ + "decided L11: cached(res) { [param]", + "decided L16: export function remote(api) { [param]" + ], + "avoid": [ + "this.shelf = new Shelf()" + ] + } + ] +} diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/js/client.js b/tests/cases/javascript/evidence-for-uncertain-rows/js/client.js similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/js/client.js rename to tests/cases/javascript/evidence-for-uncertain-rows/js/client.js diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/js/store.js b/tests/cases/javascript/evidence-for-uncertain-rows/js/store.js similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/js/store.js rename to tests/cases/javascript/evidence-for-uncertain-rows/js/store.js diff --git a/tests/cases/python/evidence-for-uncertain-rows/case.json b/tests/cases/python/evidence-for-uncertain-rows/case.json new file mode 100644 index 00000000..d449c094 --- /dev/null +++ b/tests/cases/python/evidence-for-uncertain-rows/case.json @@ -0,0 +1,22 @@ +{ + "lang": "python", + "src": ".", + "checks": [ + { + "why": "Python: the receiver's type is set from an __init__ parameter of its OWN class, not the same-named attribute another class in the file annotates", + "run": [ + "impact", + "py/gateway.py:2", + "--evidence", + "--grep" + ], + "want": [ + "py/billing.py:33: return self.gw.charge(-amount)", + "decided L30: self.gw = gw [assigned]" + ], + "avoid": [ + "def __init__(self, gw: Gateway): [constructor parameter" + ] + } + ] +} diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/py/billing.py b/tests/cases/python/evidence-for-uncertain-rows/py/billing.py similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/py/billing.py rename to tests/cases/python/evidence-for-uncertain-rows/py/billing.py diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/py/gateway.py b/tests/cases/python/evidence-for-uncertain-rows/py/gateway.py similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/py/gateway.py rename to tests/cases/python/evidence-for-uncertain-rows/py/gateway.py diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/case.json b/tests/cases/typescript/evidence-for-uncertain-rows/case.json index c301fdb6..bbad3d4d 100644 --- a/tests/cases/typescript/evidence-for-uncertain-rows/case.json +++ b/tests/cases/typescript/evidence-for-uncertain-rows/case.json @@ -1,88 +1,151 @@ { - "lang": "typescript,javascript,python,java,csharp", + "lang": "typescript", "src": ".", "checks": [ { "why": "a [by name] row carries the line that decides its receiver: a Map field and an `any` parameter show the name-match is another type; the resolved row gets no evidence", - "run": ["impact", "ts/orders.ts:2", "--evidence", "--grep"], - "want": ["ts/callers.ts:16: return this.seen.get(key);", "decided L14: private readonly seen = new Map(); [field · type new Map(…)]", - "decided L20: export function viaAny(box: any, id: string): string { [param · type any]", "only through it:"], - "avoid": ["decided L7"] + "run": [ + "impact", + "ts/orders.ts:2", + "--evidence", + "--grep" + ], + "want": [ + "ts/callers.ts:16: return this.seen.get(key);", + "decided L14: private readonly seen = new Map(); [field \u00b7 type new Map(\u2026)]", + "decided L20: export function viaAny(box: any, id: string): string { [param \u00b7 type any]", + "only through it:" + ], + "avoid": [ + "decided L7" + ] }, { "why": "a dispatch-key row decides on where the key comes from, and a registration row on the constant it registers under, followed into the module that defines it", - "run": ["impact", "ts/bus.ts:16", "--evidence", "--grep"], - "want": ["decided L10: emit(topic: string, id: string): void { [dispatch key topic]", - "decided ts/callers.ts:4: export const ORDER_PLACED = 'order.placed'; [registration key ORDER_PLACED]"] + "run": [ + "impact", + "ts/bus.ts:16", + "--evidence", + "--grep" + ], + "want": [ + "decided L10: emit(topic: string, id: string): void { [dispatch key topic]", + "decided ts/callers.ts:4: export const ORDER_PLACED = 'order.placed'; [registration key ORDER_PLACED]" + ] }, { "why": "--json rows gain evidence {call, decider} and only_through {callables, tests}; alongside rows become a count", - "run": ["impact", "ts/orders.ts:2", "--evidence", "--json"], + "run": [ + "impact", + "ts/orders.ts:2", + "--evidence", + "--json" + ], "stdout_json": true, - "want": ["\"evidence\"", "\"decider\"", "\"only_through\"", "\"callables\"", "\"alongside_count\""] + "want": [ + "\"evidence\"", + "\"decider\"", + "\"only_through\"", + "\"callables\"", + "\"alongside_count\"" + ] }, { "why": "the prose answer says the same, in one section after the rows it qualifies", - "run": ["impact", "ts/orders.ts:2", "--evidence"], - "want": ["evidence for the 2 strongest non-exact rows", "call L16: return this.seen.get(key);", "only through it: 1 callable(s), 0 test(s)", "--drop asks again without that row"] + "run": [ + "impact", + "ts/orders.ts:2", + "--evidence" + ], + "want": [ + "evidence for the 2 strongest non-exact rows", + "call L16: return this.seen.get(key);", + "only through it: 1 callable(s), 0 test(s)", + "--drop asks again without that row" + ] }, { "why": "--exact asks again with exact edges only: the name-matches and what stands on them go, the resolved caller stays", - "run": ["impact", "ts/orders.ts:2", "--exact", "--grep"], - "want": ["return this.store.get(id);", "asked again with exact edges only"], - "avoid": ["box.get(id)", "this.seen.get(key)"] + "run": [ + "impact", + "ts/orders.ts:2", + "--exact", + "--grep" + ], + "want": [ + "return this.store.get(id);", + "asked again with exact edges only" + ], + "avoid": [ + "box.get(id)", + "this.seen.get(key)" + ] }, { "why": "--drop asks again without that one row", - "run": ["impact", "ts/orders.ts:2", "--drop", "ts/callers.ts:21", "--grep"], - "want": ["this.seen.get(key)", "without ts/callers.ts:21"], - "avoid": ["box.get(id)"] + "run": [ + "impact", + "ts/orders.ts:2", + "--drop", + "ts/callers.ts:21", + "--grep" + ], + "want": [ + "this.seen.get(key)", + "without ts/callers.ts:21" + ], + "avoid": [ + "box.get(id)" + ] }, { "why": "control: an answer with only [resolved] rows is byte-identical with evidence on and off", - "run": ["impact", "ts/ledger.ts:2", "--evidence"], - "same_as": ["impact", "ts/ledger.ts:2", "--no-evidence"], - "want": ["[resolved] sum"] + "run": [ + "impact", + "ts/ledger.ts:2", + "--evidence" + ], + "same_as": [ + "impact", + "ts/ledger.ts:2", + "--no-evidence" + ], + "want": [ + "[resolved] sum" + ] }, { "why": "control: the same resolved-only answer as --json and as grep rows is byte-identical too", - "run": ["impact", "ts/ledger.ts:2", "--evidence", "--json"], - "same_as": ["impact", "ts/ledger.ts:2", "--json"], + "run": [ + "impact", + "ts/ledger.ts:2", + "--evidence", + "--json" + ], + "same_as": [ + "impact", + "ts/ledger.ts:2", + "--json" + ], "stdout_json": true }, { "why": "control: evidence is off by default, so an answer with uncertain rows is the answer it was", - "run": ["impact", "ts/orders.ts:2", "--grep"], - "same_as": ["impact", "ts/orders.ts:2", "--no-evidence", "--grep"], - "avoid": ["decided ", "only through it"] - }, - { - "why": "JavaScript: an untyped parameter is the decider of a name-match; a field assigned in the constructor decides nothing that is resolved", - "run": ["impact", "js/store.js:2", "--evidence", "--grep"], - "want": ["decided L11: cached(res) { [param]", "decided L16: export function remote(api) { [param]"], - "avoid": ["this.shelf = new Shelf()"] - }, - { - "why": "Python: the receiver's type is set from an __init__ parameter of its OWN class, not the same-named attribute another class in the file annotates", - "run": ["impact", "py/gateway.py:2", "--evidence", "--grep"], - "want": ["py/billing.py:33: return self.gw.charge(-amount)", "decided L30: self.gw = gw [assigned]"], - "avoid": ["def __init__(self, gw: Gateway): [constructor parameter"] - }, - { - "why": "Java: a one-of-a-set call through a JDK functional interface decides on the field that holds it", - "run": ["impact", "java/app/Pipeline.java:7", "--evidence", "--grep"], - "want": ["decided L6: private final Function loader; [field · type Function]"] - }, - { - "why": "Java: path's hop through the same call carries the same decider", - "run": ["path", "Cache.load", "Pipeline.apply", "--evidence", "--grep"], - "want": ["private final Function loader;"] - }, - { - "why": "C#: a dynamic field is why the call is a name-match; the resolved call on the typed field beside it gets nothing", - "run": ["impact", "cs/Stock.cs:5", "--evidence", "--grep"], - "want": ["decided L11: private readonly dynamic _meter; [field · type dynamic]"], - "avoid": ["decided L10"] + "run": [ + "impact", + "ts/orders.ts:2", + "--grep" + ], + "same_as": [ + "impact", + "ts/orders.ts:2", + "--no-evidence", + "--grep" + ], + "avoid": [ + "decided ", + "only through it" + ] } ] }