tests/run.py --jobs N; CI runs the query cases four at a time

.github/workflows/ci.yml

@@ -456,7 +456,7 @@ jobs:
         env:
           AXIOM_PARSER: ${{ github.workspace }}/parser/dist/index.js
           AXIOM_SOUFFLE_CACHE: ${{ github.workspace }}/.souffle-cache
-        run: python3 tests/run.py --lang ${{ matrix.lang }}
+        run: python3 tests/run.py --lang ${{ matrix.lang }} --jobs 4
 
       # the small surface as users and agents get it: find / impact / path / tests through the installed command and
       # the MCP server, answered as places with their code, and the direct calls and flags that keep the old answers

tests/cases/csharp/evidence-for-uncertain-rows/case.json

@@ -0,0 +1,21 @@
+{
+ "lang": "csharp",
+ "src": ".",
+ "checks": [
+  {
+   "why": "C#: a dynamic field is why the call is a name-match; the resolved call on the typed field beside it gets nothing",
+   "run": [
+    "impact",
+    "cs/Stock.cs:5",
+    "--evidence",
+    "--grep"
+   ],
+   "want": [
+    "decided  L11: private readonly dynamic _meter;  [field \u00b7 type dynamic]"
+   ],
+   "avoid": [
+    "decided  L10"
+   ]
+  }
+ ]
+}

tests/cases/java/evidence-for-uncertain-rows/case.json

@@ -0,0 +1,31 @@
+{
+ "lang": "java",
+ "src": ".",
+ "checks": [
+  {
+   "why": "Java: a one-of-a-set call through a JDK functional interface decides on the field that holds it",
+   "run": [
+    "impact",
+    "java/app/Pipeline.java:7",
+    "--evidence",
+    "--grep"
+   ],
+   "want": [
+    "decided  L6: private final Function<String, String> loader;  [field \u00b7 type Function]"
+   ]
+  },
+  {
+   "why": "Java: path's hop through the same call carries the same decider",
+   "run": [
+    "path",
+    "Cache.load",
+    "Pipeline.apply",
+    "--evidence",
+    "--grep"
+   ],
+   "want": [
+    "private final Function<String, String> loader;"
+   ]
+  }
+ ]
+}

tests/cases/javascript/evidence-for-uncertain-rows/case.json

@@ -0,0 +1,22 @@
+{
+ "lang": "javascript",
+ "src": ".",
+ "checks": [
+  {
+   "why": "JavaScript: an untyped parameter is the decider of a name-match; a field assigned in the constructor decides nothing that is resolved",
+   "run": [
+    "impact",
+    "js/store.js:2",
+    "--evidence",
+    "--grep"
+   ],
+   "want": [
+    "decided  L11: cached(res) {  [param]",
+    "decided  L16: export function remote(api) {  [param]"
+   ],
+   "avoid": [
+    "this.shelf = new Shelf()"
+   ]
+  }
+ ]
+}

tests/cases/python/evidence-for-uncertain-rows/case.json

@@ -0,0 +1,22 @@
+{
+ "lang": "python",
+ "src": ".",
+ "checks": [
+  {
+   "why": "Python: the receiver's type is set from an __init__ parameter of its OWN class, not the same-named attribute another class in the file annotates",
+   "run": [
+    "impact",
+    "py/gateway.py:2",
+    "--evidence",
+    "--grep"
+   ],
+   "want": [
+    "py/billing.py:33: return self.gw.charge(-amount)",
+    "decided  L30: self.gw = gw  [assigned]"
+   ],
+   "avoid": [
+    "def __init__(self, gw: Gateway):  [constructor parameter"
+   ]
+  }
+ ]
+}

tests/cases/typescript/evidence-for-uncertain-rows/case.json

@@ -1,88 +1,151 @@
 {
- "lang": "typescript,javascript,python,java,csharp",
+ "lang": "typescript",
  "src": ".",
  "checks": [
   {
    "why": "a [by name] row carries the line that decides its receiver: a Map field and an `any` parameter show the name-match is another type; the resolved row gets no evidence",
-   "run": ["impact", "ts/orders.ts:2", "--evidence", "--grep"],
-   "want": ["ts/callers.ts:16: return this.seen.get(key);", "decided  L14: private readonly seen = new Map<string, string>();  [field · type new Map(…)]",
-            "decided  L20: export function viaAny(box: any, id: string): string {  [param · type any]", "only through it:"],
-   "avoid": ["decided  L7"]
+   "run": [
+    "impact",
+    "ts/orders.ts:2",
+    "--evidence",
+    "--grep"
+   ],
+   "want": [
+    "ts/callers.ts:16: return this.seen.get(key);",
+    "decided  L14: private readonly seen = new Map<string, string>();  [field \u00b7 type new Map(\u2026)]",
+    "decided  L20: export function viaAny(box: any, id: string): string {  [param \u00b7 type any]",
+    "only through it:"
+   ],
+   "avoid": [
+    "decided  L7"
+   ]
   },
   {
    "why": "a dispatch-key row decides on where the key comes from, and a registration row on the constant it registers under, followed into the module that defines it",
-   "run": ["impact", "ts/bus.ts:16", "--evidence", "--grep"],
-   "want": ["decided  L10: emit(topic: string, id: string): void {  [dispatch key topic]",
-            "decided  ts/callers.ts:4: export const ORDER_PLACED = 'order.placed';  [registration key ORDER_PLACED]"]
+   "run": [
+    "impact",
+    "ts/bus.ts:16",
+    "--evidence",
+    "--grep"
+   ],
+   "want": [
+    "decided  L10: emit(topic: string, id: string): void {  [dispatch key topic]",
+    "decided  ts/callers.ts:4: export const ORDER_PLACED = 'order.placed';  [registration key ORDER_PLACED]"
+   ]
   },
   {
    "why": "--json rows gain evidence {call, decider} and only_through {callables, tests}; alongside rows become a count",
-   "run": ["impact", "ts/orders.ts:2", "--evidence", "--json"],
+   "run": [
+    "impact",
+    "ts/orders.ts:2",
+    "--evidence",
+    "--json"
+   ],
    "stdout_json": true,
-   "want": ["\"evidence\"", "\"decider\"", "\"only_through\"", "\"callables\"", "\"alongside_count\""]
+   "want": [
+    "\"evidence\"",
+    "\"decider\"",
+    "\"only_through\"",
+    "\"callables\"",
+    "\"alongside_count\""
+   ]
   },
   {
    "why": "the prose answer says the same, in one section after the rows it qualifies",
-   "run": ["impact", "ts/orders.ts:2", "--evidence"],
-   "want": ["evidence for the 2 strongest non-exact rows", "call     L16: return this.seen.get(key);", "only through it: 1 callable(s), 0 test(s)", "--drop <a row's file:line> asks again without that row"]
+   "run": [
+    "impact",
+    "ts/orders.ts:2",
+    "--evidence"
+   ],
+   "want": [
+    "evidence for the 2 strongest non-exact rows",
+    "call     L16: return this.seen.get(key);",
+    "only through it: 1 callable(s), 0 test(s)",
+    "--drop <a row's file:line> asks again without that row"
+   ]
   },
   {
    "why": "--exact asks again with exact edges only: the name-matches and what stands on them go, the resolved caller stays",
-   "run": ["impact", "ts/orders.ts:2", "--exact", "--grep"],
-   "want": ["return this.store.get(id);", "asked again with exact edges only"],
-   "avoid": ["box.get(id)", "this.seen.get(key)"]
+   "run": [
+    "impact",
+    "ts/orders.ts:2",
+    "--exact",
+    "--grep"
+   ],
+   "want": [
+    "return this.store.get(id);",
+    "asked again with exact edges only"
+   ],
+   "avoid": [
+    "box.get(id)",
+    "this.seen.get(key)"
+   ]
   },
   {
    "why": "--drop <file:line> asks again without that one row",
-   "run": ["impact", "ts/orders.ts:2", "--drop", "ts/callers.ts:21", "--grep"],
-   "want": ["this.seen.get(key)", "without ts/callers.ts:21"],
-   "avoid": ["box.get(id)"]
+   "run": [
+    "impact",
+    "ts/orders.ts:2",
+    "--drop",
+    "ts/callers.ts:21",
+    "--grep"
+   ],
+   "want": [
+    "this.seen.get(key)",
+    "without ts/callers.ts:21"
+   ],
+   "avoid": [
+    "box.get(id)"
+   ]
   },
   {
    "why": "control: an answer with only [resolved] rows is byte-identical with evidence on and off",
-   "run": ["impact", "ts/ledger.ts:2", "--evidence"],
-   "same_as": ["impact", "ts/ledger.ts:2", "--no-evidence"],
-   "want": ["[resolved] sum"]
+   "run": [
+    "impact",
+    "ts/ledger.ts:2",
+    "--evidence"
+   ],
+   "same_as": [
+    "impact",
+    "ts/ledger.ts:2",
+    "--no-evidence"
+   ],
+   "want": [
+    "[resolved] sum"
+   ]
   },
   {
    "why": "control: the same resolved-only answer as --json and as grep rows is byte-identical too",
-   "run": ["impact", "ts/ledger.ts:2", "--evidence", "--json"],
-   "same_as": ["impact", "ts/ledger.ts:2", "--json"],
+   "run": [
+    "impact",
+    "ts/ledger.ts:2",
+    "--evidence",
+    "--json"
+   ],
+   "same_as": [
+    "impact",
+    "ts/ledger.ts:2",
+    "--json"
+   ],
    "stdout_json": true
   },
   {
    "why": "control: evidence is off by default, so an answer with uncertain rows is the answer it was",
-   "run": ["impact", "ts/orders.ts:2", "--grep"],
-   "same_as": ["impact", "ts/orders.ts:2", "--no-evidence", "--grep"],
-   "avoid": ["decided ", "only through it"]
-  },
-  {
-   "why": "JavaScript: an untyped parameter is the decider of a name-match; a field assigned in the constructor decides nothing that is resolved",
-   "run": ["impact", "js/store.js:2", "--evidence", "--grep"],
-   "want": ["decided  L11: cached(res) {  [param]", "decided  L16: export function remote(api) {  [param]"],
-   "avoid": ["this.shelf = new Shelf()"]
-  },
-  {
-   "why": "Python: the receiver's type is set from an __init__ parameter of its OWN class, not the same-named attribute another class in the file annotates",
-   "run": ["impact", "py/gateway.py:2", "--evidence", "--grep"],
-   "want": ["py/billing.py:33: return self.gw.charge(-amount)", "decided  L30: self.gw = gw  [assigned]"],
-   "avoid": ["def __init__(self, gw: Gateway):  [constructor parameter"]
-  },
-  {
-   "why": "Java: a one-of-a-set call through a JDK functional interface decides on the field that holds it",
-   "run": ["impact", "java/app/Pipeline.java:7", "--evidence", "--grep"],
-   "want": ["decided  L6: private final Function<String, String> loader;  [field · type Function]"]
-  },
-  {
-   "why": "Java: path's hop through the same call carries the same decider",
-   "run": ["path", "Cache.load", "Pipeline.apply", "--evidence", "--grep"],
-   "want": ["private final Function<String, String> loader;"]
-  },
-  {
-   "why": "C#: a dynamic field is why the call is a name-match; the resolved call on the typed field beside it gets nothing",
-   "run": ["impact", "cs/Stock.cs:5", "--evidence", "--grep"],
-   "want": ["decided  L11: private readonly dynamic _meter;  [field · type dynamic]"],
-   "avoid": ["decided  L10"]
+   "run": [
+    "impact",
+    "ts/orders.ts:2",
+    "--grep"
+   ],
+   "same_as": [
+    "impact",
+    "ts/orders.ts:2",
+    "--no-evidence",
+    "--grep"
+   ],
+   "avoid": [
+    "decided ",
+    "only through it"
+   ]
   }
  ]
 }

tests/run.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""tests/run.py [<case> …] [--lang java|python|typescript|javascript] [--keep] [-v]
+"""tests/run.py [<case> …] [--lang java|python|typescript|javascript] [--keep] [-v] [--jobs N]
 
 What the plugin CLAIMS to find, checked on code that is small enough to read. Each case is a directory under
 tests/cases/<language>/<name>/ holding a tiny synthetic project and a case.json:
@@ -32,26 +32,29 @@
   and neither announces itself as a want/avoid mismatch. If anyone ever "simplifies" `pending` into a skip, that is
   the property they will have removed.
 """
-import json, os, re, shutil, subprocess, sys
+import builtins, concurrent.futures, io, json, os, re, shutil, subprocess, sys
 HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE)
 AX = os.path.join(ROOT, 'plugins', 'axiomcode', 'skills', 'axiomcode', 'scripts', 'axiomcode')
 args = sys.argv[1:]; keep = '--keep' in args; verbose = '-v' in args
 lang = args[args.index('--lang') + 1] if '--lang' in args else None
-only = [a for a in args if not a.startswith('-') and a not in (lang,)]
+jobs = int(args[args.index('--jobs') + 1]) if '--jobs' in args else 1
+only = [a for a in args if not a.startswith('-') and a not in (lang, str(jobs))]
 cases = []
 for l in sorted(os.listdir(os.path.join(HERE, 'cases'))):
     if lang and l != lang: continue
     d = os.path.join(HERE, 'cases', l)
     for c in sorted(os.listdir(d)):
         if os.path.isfile(os.path.join(d, c, 'case.json')) and (not only or c in only or l in only): cases.append((l, c, os.path.join(d, c)))
-fail = tot = pend = 0
-for l, name, path in cases:
+def run_case(case):
+    """one case: index it, run its checks; its output as one block and its counts, so cases can run side by side"""
+    l, name, path = case; buf = io.StringIO(); fail = tot = pend = 0
+    def print(*a, flush=False, **k): builtins.print(*a, file=buf, **k)
     print(f"… {l}/{name}", flush=True)
     spec = json.load(open(os.path.join(path, 'case.json')))
     build = ['bash', AX, 'index', path, '--lang', spec.get('lang', l)] + (['--src', spec['src']] if spec.get('src') else []) \
         + (['--library', os.path.join(path, spec['library'])] if spec.get('library') else [])   # a staged dependency root, relative to the case
     r = subprocess.run(build, capture_output=True, text=True)
-    if r.returncode: print(f"FAIL {l}/{name}: index failed: {(r.stderr or r.stdout)[-300:]}"); fail += 1; continue
+    if r.returncode: print(f"FAIL {l}/{name}: index failed: {(r.stderr or r.stdout)[-300:]}"); return buf.getvalue(), 0, 1, 0
     for stmt in spec.get('sql', []):                                  # facts a framework extension would have written
         subprocess.run(['sqlite3', os.path.join(path, '.axiomcode', 'out', 'graph.sqlite'), stmt], capture_output=True, text=True)
     for ch in spec['checks']:
@@ -97,5 +100,17 @@
             print('     ' + '\n     '.join(text.strip().split('\n')[:14]))
         elif verbose: print(f"ok   {l}/{name}: {ch['why']}")
     if not keep: shutil.rmtree(os.path.join(path, '.axiomcode'), ignore_errors=True)
+    return buf.getvalue(), tot, fail, pend
+
+
+# CASES RUN SIDE BY SIDE with --jobs N: each indexes its own directory and shares nothing but the compiled rules, so the
+# first case runs alone (it compiles and caches them) and the rest run N at a time. Output is printed in case order.
+fail = tot = pend = 0
+def report(res):
+    global fail, tot, pend
+    text, t, f, p = res; sys.stdout.write(text); sys.stdout.flush(); tot += t; fail += f; pend += p
+if cases: report(run_case(cases[0]))
+with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, jobs)) as ex:
+    for res in ex.map(run_case, cases[1:]): report(res)
 print(f"\n{tot - fail - pend} of {tot} check(s) passed in {len(cases)} case(s)" + (f" - {pend} PENDING" if pend else '') + ('' if not fail else f" - {fail} FAILED"))
 sys.exit(1 if fail else 0)