diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5987254c..cb6710f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -456,7 +456,7 @@ jobs: env: AXIOM_PARSER: ${{ github.workspace }}/parser/dist/index.js AXIOM_SOUFFLE_CACHE: ${{ github.workspace }}/.souffle-cache - run: python3 tests/run.py --lang ${{ matrix.lang }} + run: python3 tests/run.py --lang ${{ matrix.lang }} --jobs 4 # the small surface as users and agents get it: find / impact / path / tests through the installed command and # the MCP server, answered as places with their code, and the direct calls and flags that keep the old answers diff --git a/tests/cases/csharp/evidence-for-uncertain-rows/case.json b/tests/cases/csharp/evidence-for-uncertain-rows/case.json new file mode 100644 index 00000000..f745109f --- /dev/null +++ b/tests/cases/csharp/evidence-for-uncertain-rows/case.json @@ -0,0 +1,21 @@ +{ + "lang": "csharp", + "src": ".", + "checks": [ + { + "why": "C#: a dynamic field is why the call is a name-match; the resolved call on the typed field beside it gets nothing", + "run": [ + "impact", + "cs/Stock.cs:5", + "--evidence", + "--grep" + ], + "want": [ + "decided L11: private readonly dynamic _meter; [field \u00b7 type dynamic]" + ], + "avoid": [ + "decided L10" + ] + } + ] +} diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/cs/Stock.cs b/tests/cases/csharp/evidence-for-uncertain-rows/cs/Stock.cs similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/cs/Stock.cs rename to tests/cases/csharp/evidence-for-uncertain-rows/cs/Stock.cs diff --git a/tests/cases/java/evidence-for-uncertain-rows/case.json b/tests/cases/java/evidence-for-uncertain-rows/case.json new file mode 100644 index 00000000..4ba282c0 --- /dev/null +++ b/tests/cases/java/evidence-for-uncertain-rows/case.json @@ -0,0 +1,31 @@ +{ + "lang": "java", + "src": ".", + "checks": [ + { + "why": "Java: a one-of-a-set call through a JDK functional interface decides on the field that holds it", + "run": [ + "impact", + "java/app/Pipeline.java:7", + "--evidence", + "--grep" + ], + "want": [ + "decided L6: private final Function loader; [field \u00b7 type Function]" + ] + }, + { + "why": "Java: path's hop through the same call carries the same decider", + "run": [ + "path", + "Cache.load", + "Pipeline.apply", + "--evidence", + "--grep" + ], + "want": [ + "private final Function loader;" + ] + } + ] +} diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/java/app/Cache.java b/tests/cases/java/evidence-for-uncertain-rows/java/app/Cache.java similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/java/app/Cache.java rename to tests/cases/java/evidence-for-uncertain-rows/java/app/Cache.java diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/java/app/Pipeline.java b/tests/cases/java/evidence-for-uncertain-rows/java/app/Pipeline.java similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/java/app/Pipeline.java rename to tests/cases/java/evidence-for-uncertain-rows/java/app/Pipeline.java diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/java/app/Store.java b/tests/cases/java/evidence-for-uncertain-rows/java/app/Store.java similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/java/app/Store.java rename to tests/cases/java/evidence-for-uncertain-rows/java/app/Store.java diff --git a/tests/cases/javascript/evidence-for-uncertain-rows/case.json b/tests/cases/javascript/evidence-for-uncertain-rows/case.json new file mode 100644 index 00000000..e5351252 --- /dev/null +++ b/tests/cases/javascript/evidence-for-uncertain-rows/case.json @@ -0,0 +1,22 @@ +{ + "lang": "javascript", + "src": ".", + "checks": [ + { + "why": "JavaScript: an untyped parameter is the decider of a name-match; a field assigned in the constructor decides nothing that is resolved", + "run": [ + "impact", + "js/store.js:2", + "--evidence", + "--grep" + ], + "want": [ + "decided L11: cached(res) { [param]", + "decided L16: export function remote(api) { [param]" + ], + "avoid": [ + "this.shelf = new Shelf()" + ] + } + ] +} diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/js/client.js b/tests/cases/javascript/evidence-for-uncertain-rows/js/client.js similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/js/client.js rename to tests/cases/javascript/evidence-for-uncertain-rows/js/client.js diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/js/store.js b/tests/cases/javascript/evidence-for-uncertain-rows/js/store.js similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/js/store.js rename to tests/cases/javascript/evidence-for-uncertain-rows/js/store.js diff --git a/tests/cases/python/evidence-for-uncertain-rows/case.json b/tests/cases/python/evidence-for-uncertain-rows/case.json new file mode 100644 index 00000000..d449c094 --- /dev/null +++ b/tests/cases/python/evidence-for-uncertain-rows/case.json @@ -0,0 +1,22 @@ +{ + "lang": "python", + "src": ".", + "checks": [ + { + "why": "Python: the receiver's type is set from an __init__ parameter of its OWN class, not the same-named attribute another class in the file annotates", + "run": [ + "impact", + "py/gateway.py:2", + "--evidence", + "--grep" + ], + "want": [ + "py/billing.py:33: return self.gw.charge(-amount)", + "decided L30: self.gw = gw [assigned]" + ], + "avoid": [ + "def __init__(self, gw: Gateway): [constructor parameter" + ] + } + ] +} diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/py/billing.py b/tests/cases/python/evidence-for-uncertain-rows/py/billing.py similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/py/billing.py rename to tests/cases/python/evidence-for-uncertain-rows/py/billing.py diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/py/gateway.py b/tests/cases/python/evidence-for-uncertain-rows/py/gateway.py similarity index 100% rename from tests/cases/typescript/evidence-for-uncertain-rows/py/gateway.py rename to tests/cases/python/evidence-for-uncertain-rows/py/gateway.py diff --git a/tests/cases/typescript/evidence-for-uncertain-rows/case.json b/tests/cases/typescript/evidence-for-uncertain-rows/case.json index c301fdb6..bbad3d4d 100644 --- a/tests/cases/typescript/evidence-for-uncertain-rows/case.json +++ b/tests/cases/typescript/evidence-for-uncertain-rows/case.json @@ -1,88 +1,151 @@ { - "lang": "typescript,javascript,python,java,csharp", + "lang": "typescript", "src": ".", "checks": [ { "why": "a [by name] row carries the line that decides its receiver: a Map field and an `any` parameter show the name-match is another type; the resolved row gets no evidence", - "run": ["impact", "ts/orders.ts:2", "--evidence", "--grep"], - "want": ["ts/callers.ts:16: return this.seen.get(key);", "decided L14: private readonly seen = new Map(); [field · type new Map(…)]", - "decided L20: export function viaAny(box: any, id: string): string { [param · type any]", "only through it:"], - "avoid": ["decided L7"] + "run": [ + "impact", + "ts/orders.ts:2", + "--evidence", + "--grep" + ], + "want": [ + "ts/callers.ts:16: return this.seen.get(key);", + "decided L14: private readonly seen = new Map(); [field \u00b7 type new Map(\u2026)]", + "decided L20: export function viaAny(box: any, id: string): string { [param \u00b7 type any]", + "only through it:" + ], + "avoid": [ + "decided L7" + ] }, { "why": "a dispatch-key row decides on where the key comes from, and a registration row on the constant it registers under, followed into the module that defines it", - "run": ["impact", "ts/bus.ts:16", "--evidence", "--grep"], - "want": ["decided L10: emit(topic: string, id: string): void { [dispatch key topic]", - "decided ts/callers.ts:4: export const ORDER_PLACED = 'order.placed'; [registration key ORDER_PLACED]"] + "run": [ + "impact", + "ts/bus.ts:16", + "--evidence", + "--grep" + ], + "want": [ + "decided L10: emit(topic: string, id: string): void { [dispatch key topic]", + "decided ts/callers.ts:4: export const ORDER_PLACED = 'order.placed'; [registration key ORDER_PLACED]" + ] }, { "why": "--json rows gain evidence {call, decider} and only_through {callables, tests}; alongside rows become a count", - "run": ["impact", "ts/orders.ts:2", "--evidence", "--json"], + "run": [ + "impact", + "ts/orders.ts:2", + "--evidence", + "--json" + ], "stdout_json": true, - "want": ["\"evidence\"", "\"decider\"", "\"only_through\"", "\"callables\"", "\"alongside_count\""] + "want": [ + "\"evidence\"", + "\"decider\"", + "\"only_through\"", + "\"callables\"", + "\"alongside_count\"" + ] }, { "why": "the prose answer says the same, in one section after the rows it qualifies", - "run": ["impact", "ts/orders.ts:2", "--evidence"], - "want": ["evidence for the 2 strongest non-exact rows", "call L16: return this.seen.get(key);", "only through it: 1 callable(s), 0 test(s)", "--drop asks again without that row"] + "run": [ + "impact", + "ts/orders.ts:2", + "--evidence" + ], + "want": [ + "evidence for the 2 strongest non-exact rows", + "call L16: return this.seen.get(key);", + "only through it: 1 callable(s), 0 test(s)", + "--drop asks again without that row" + ] }, { "why": "--exact asks again with exact edges only: the name-matches and what stands on them go, the resolved caller stays", - "run": ["impact", "ts/orders.ts:2", "--exact", "--grep"], - "want": ["return this.store.get(id);", "asked again with exact edges only"], - "avoid": ["box.get(id)", "this.seen.get(key)"] + "run": [ + "impact", + "ts/orders.ts:2", + "--exact", + "--grep" + ], + "want": [ + "return this.store.get(id);", + "asked again with exact edges only" + ], + "avoid": [ + "box.get(id)", + "this.seen.get(key)" + ] }, { "why": "--drop asks again without that one row", - "run": ["impact", "ts/orders.ts:2", "--drop", "ts/callers.ts:21", "--grep"], - "want": ["this.seen.get(key)", "without ts/callers.ts:21"], - "avoid": ["box.get(id)"] + "run": [ + "impact", + "ts/orders.ts:2", + "--drop", + "ts/callers.ts:21", + "--grep" + ], + "want": [ + "this.seen.get(key)", + "without ts/callers.ts:21" + ], + "avoid": [ + "box.get(id)" + ] }, { "why": "control: an answer with only [resolved] rows is byte-identical with evidence on and off", - "run": ["impact", "ts/ledger.ts:2", "--evidence"], - "same_as": ["impact", "ts/ledger.ts:2", "--no-evidence"], - "want": ["[resolved] sum"] + "run": [ + "impact", + "ts/ledger.ts:2", + "--evidence" + ], + "same_as": [ + "impact", + "ts/ledger.ts:2", + "--no-evidence" + ], + "want": [ + "[resolved] sum" + ] }, { "why": "control: the same resolved-only answer as --json and as grep rows is byte-identical too", - "run": ["impact", "ts/ledger.ts:2", "--evidence", "--json"], - "same_as": ["impact", "ts/ledger.ts:2", "--json"], + "run": [ + "impact", + "ts/ledger.ts:2", + "--evidence", + "--json" + ], + "same_as": [ + "impact", + "ts/ledger.ts:2", + "--json" + ], "stdout_json": true }, { "why": "control: evidence is off by default, so an answer with uncertain rows is the answer it was", - "run": ["impact", "ts/orders.ts:2", "--grep"], - "same_as": ["impact", "ts/orders.ts:2", "--no-evidence", "--grep"], - "avoid": ["decided ", "only through it"] - }, - { - "why": "JavaScript: an untyped parameter is the decider of a name-match; a field assigned in the constructor decides nothing that is resolved", - "run": ["impact", "js/store.js:2", "--evidence", "--grep"], - "want": ["decided L11: cached(res) { [param]", "decided L16: export function remote(api) { [param]"], - "avoid": ["this.shelf = new Shelf()"] - }, - { - "why": "Python: the receiver's type is set from an __init__ parameter of its OWN class, not the same-named attribute another class in the file annotates", - "run": ["impact", "py/gateway.py:2", "--evidence", "--grep"], - "want": ["py/billing.py:33: return self.gw.charge(-amount)", "decided L30: self.gw = gw [assigned]"], - "avoid": ["def __init__(self, gw: Gateway): [constructor parameter"] - }, - { - "why": "Java: a one-of-a-set call through a JDK functional interface decides on the field that holds it", - "run": ["impact", "java/app/Pipeline.java:7", "--evidence", "--grep"], - "want": ["decided L6: private final Function loader; [field · type Function]"] - }, - { - "why": "Java: path's hop through the same call carries the same decider", - "run": ["path", "Cache.load", "Pipeline.apply", "--evidence", "--grep"], - "want": ["private final Function loader;"] - }, - { - "why": "C#: a dynamic field is why the call is a name-match; the resolved call on the typed field beside it gets nothing", - "run": ["impact", "cs/Stock.cs:5", "--evidence", "--grep"], - "want": ["decided L11: private readonly dynamic _meter; [field · type dynamic]"], - "avoid": ["decided L10"] + "run": [ + "impact", + "ts/orders.ts:2", + "--grep" + ], + "same_as": [ + "impact", + "ts/orders.ts:2", + "--no-evidence", + "--grep" + ], + "avoid": [ + "decided ", + "only through it" + ] } ] } diff --git a/tests/run.py b/tests/run.py index a35f564a..863a7c06 100755 --- a/tests/run.py +++ b/tests/run.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""tests/run.py [ …] [--lang java|python|typescript|javascript] [--keep] [-v] +"""tests/run.py [ …] [--lang java|python|typescript|javascript] [--keep] [-v] [--jobs N] What the plugin CLAIMS to find, checked on code that is small enough to read. Each case is a directory under tests/cases/// holding a tiny synthetic project and a case.json: @@ -32,26 +32,29 @@ and neither announces itself as a want/avoid mismatch. If anyone ever "simplifies" `pending` into a skip, that is the property they will have removed. """ -import json, os, re, shutil, subprocess, sys +import builtins, concurrent.futures, io, json, os, re, shutil, subprocess, sys HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE) AX = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode') args = sys.argv[1:]; keep = '--keep' in args; verbose = '-v' in args lang = args[args.index('--lang') + 1] if '--lang' in args else None -only = [a for a in args if not a.startswith('-') and a not in (lang,)] +jobs = int(args[args.index('--jobs') + 1]) if '--jobs' in args else 1 +only = [a for a in args if not a.startswith('-') and a not in (lang, str(jobs))] cases = [] for l in sorted(os.listdir(os.path.join(HERE, 'cases'))): if lang and l != lang: continue d = os.path.join(HERE, 'cases', l) for c in sorted(os.listdir(d)): if os.path.isfile(os.path.join(d, c, 'case.json')) and (not only or c in only or l in only): cases.append((l, c, os.path.join(d, c))) -fail = tot = pend = 0 -for l, name, path in cases: +def run_case(case): + """one case: index it, run its checks; its output as one block and its counts, so cases can run side by side""" + l, name, path = case; buf = io.StringIO(); fail = tot = pend = 0 + def print(*a, flush=False, **k): builtins.print(*a, file=buf, **k) print(f"… {l}/{name}", flush=True) spec = json.load(open(os.path.join(path, 'case.json'))) build = ['bash', AX, 'index', path, '--lang', spec.get('lang', l)] + (['--src', spec['src']] if spec.get('src') else []) \ + (['--library', os.path.join(path, spec['library'])] if spec.get('library') else []) # a staged dependency root, relative to the case r = subprocess.run(build, capture_output=True, text=True) - if r.returncode: print(f"FAIL {l}/{name}: index failed: {(r.stderr or r.stdout)[-300:]}"); fail += 1; continue + if r.returncode: print(f"FAIL {l}/{name}: index failed: {(r.stderr or r.stdout)[-300:]}"); return buf.getvalue(), 0, 1, 0 for stmt in spec.get('sql', []): # facts a framework extension would have written subprocess.run(['sqlite3', os.path.join(path, '.axiomcode', 'out', 'graph.sqlite'), stmt], capture_output=True, text=True) for ch in spec['checks']: @@ -97,5 +100,17 @@ print(' ' + '\n '.join(text.strip().split('\n')[:14])) elif verbose: print(f"ok {l}/{name}: {ch['why']}") if not keep: shutil.rmtree(os.path.join(path, '.axiomcode'), ignore_errors=True) + return buf.getvalue(), tot, fail, pend + + +# CASES RUN SIDE BY SIDE with --jobs N: each indexes its own directory and shares nothing but the compiled rules, so the +# first case runs alone (it compiles and caches them) and the rest run N at a time. Output is printed in case order. +fail = tot = pend = 0 +def report(res): + global fail, tot, pend + text, t, f, p = res; sys.stdout.write(text); sys.stdout.flush(); tot += t; fail += f; pend += p +if cases: report(run_case(cases[0])) +with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, jobs)) as ex: + for res in ex.map(run_case, cases[1:]): report(res) print(f"\n{tot - fail - pend} of {tot} check(s) passed in {len(cases)} case(s)" + (f" - {pend} PENDING" if pend else '') + ('' if not fail else f" - {fail} FAILED")) sys.exit(1 if fail else 0)