test-impact: a test helper is the route, not a test; far tests listed apart

plugins/axiomcode/skills/axiomcode/scripts/ax_grep.py

@@ -142,8 +142,13 @@ def more(k, n=1): rest[k] = rest.get(k, 0) + n
         row = ('module' if mod else 'reached', site(code, r['at'], f"hop {r['hops']}" + (' · module scope' if mod else '') +
                                                      (' · test' if r.get('test') else '') + stale(r), r['display']))
         (modules if mod else rows).append(row)
-    for r, k in per_test_file(d.get('tests', [])):
-        rows.append(('test', site(code, r['at'], f"test · {TAG.get(r.get('certainty'), r.get('certainty'))} · hop {r['hops']}" + more_in_file(k) + stale(r), r['display'])))
+    # a test file that loads the changed file before one that never imports it (impact's loads_change)
+    far_files = {f for f in {(t.get('at') or '').rpartition(':')[0] for t in d.get('tests', [])}
+                 if all(t.get('loads_change') is False for t in d.get('tests', []) if (t.get('at') or '').rpartition(':')[0] == f)}
+    near_some = any(t.get('loads_change') is not False for t in d.get('tests', []))
+    for r, k in sorted(per_test_file(d.get('tests', [])), key=lambda rk: near_some and rk[0]['at'].rpartition(':')[0] in far_files):
+        far = ' · further out: never imports the change' if near_some and r['at'].rpartition(':')[0] in far_files else ''
+        rows.append(('test', site(code, r['at'], f"test · {TAG.get(r.get('certainty'), r.get('certainty'))} · hop {r['hops']}" + far + more_in_file(k) + stale(r), r['display'])))
     for r in d.get('stub_tests', []):
         rows.append(('stub', site(code, r['at'], f"test · stubs it{stale(r)}", r['display'])))
     # a test file's top level is already its test row above
@@ -238,8 +243,13 @@ def test_impact(d, code):
     rows = []
     for f in d.get('edited_test_files', []):
         rows.append(('test', f"{f}:1: (edited test file)  [test · edited]"))
+    # the tests that load the changed file first; a file further out (it never imports the change, or reaches it only
+    # many hops away) after them, said so
+    further = set(d.get('further_test_files') or [])
     for t, k in per_test_file(d.get('tests', [])):
-        rows.append(('test', ev(site(code, t['at'], f"test · {TAG.get(t.get('certainty'), t.get('certainty'))} · hop {t['hops']}{more_in_file(k)}{stale(t)}", t['display']), t)))
+        far = (' · further out: never imports the change' if t.get('loads_change') is False else ' · further out') \
+            if (t.get('at') or '').rpartition(':')[0] in further else ''
+        rows.append(('test', ev(site(code, t['at'], f"test · {TAG.get(t.get('certainty'), t.get('certainty'))} · hop {t['hops']}{far}{more_in_file(k)}{stale(t)}", t['display']), t)))
     # a changed file no graph follows (a script, a fixture) is run by the test files that name it in their text
     names = {}
     for f, v in (d.get('named_in_test_text') or {}).items():
@@ -249,6 +259,7 @@ def test_impact(d, code):
         rows.append(('text test', f"{t}:1: (names {', '.join(dict.fromkeys(ns))})  [test · text]"))
     foot = ev_foot(d)
     if d.get('command'): foot.append(f"run: {d['command']}")
+    if d.get('command_further'): foot.append(f"run: (then, further out) {d['command_further']}")
     if d.get('bound'): foot.append(f"bound: {d['bound']}")
     return rows, {}, foot
 

plugins/axiomcode/skills/axiomcode/scripts/axiomcode-test-impact

@@ -62,6 +62,8 @@ is not the same as a change that does nothing, and the rung on each row is where
 import importlib.machinery, importlib.util, json, os, re, subprocess, sys, collections, functools, posixpath, shlex
 
 TESTY = re.compile(r'(^|[/_.-])(test|tests|spec|specs|__tests__|it)([/_.-]|$)', re.I)
+# a test that loads the change and reaches it within this many hops is listed first; beyond it, further out
+NEAR_HOPS = 3
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, HERE)
@@ -2034,6 +2036,19 @@ def main(argv):
     case_data_tests = {i for i, r in tests.items() if r.get('at') and ax_caserun.in_case_dir(real, r['at'].split(':')[0])}
     for i in case_data_tests: tests.pop(i, None)
     files = sorted({r.get('at', '').split(':')[0] for r in tests.values() if r.get('at')})
+    # NEAREST FIRST, AND FURTHER OUT APART: a test whose file (or the fixture file its runner loads) never imports the
+    # changed file reaches it only through a key join, a dispatch resolved across services, or a registration the
+    # process it runs in never makes (impact's loads_change). Still listed, after the tests that load the change.
+    # One that loads it but reaches it only beyond NEAR_HOPS (a whole service started for an unrelated flow, whose
+    # route crosses a bus and a handler table) is further out too: impact's own measurement puts a sound route within
+    # 3 hops at about 9 in 10 right, and a hub answer at 3 in 10. Where the import graph is unknown (no loads_change),
+    # nothing moves.
+    near = lambda r: r.get('loads_change') is None or (r['loads_change'] and (r.get('hops') or 0) <= NEAR_HOPS)
+    further = sorted({f for f in files} - {(r.get('at') or '').split(':')[0] for r in tests.values() if near(r)})
+    if further and len(further) < len(files):
+        files = [f for f in files if f not in set(further)]
+    else:
+        further = []                                    # nothing loads it, or everything does: one list, as before
     # A selected test carrying no location drops out of `files`, and `files` is what the command PRINTS and
     # what the run command is built from — so the answer names it nowhere and the suite runs without it.
     # This is the third instance of one bug: a row discarded for being malformed in one field. The other two
@@ -2060,13 +2075,18 @@ def main(argv):
         if classes and lang in ('java', 'csharp') else classes
     replaced = concrete_test_classes(db, run_classes)[1] if lang in ('java', 'csharp') else {}
     cmd_all = command_for(lang, run_files, run_classes + ([os.path.splitext(os.path.basename(f))[0] for f in edited] if classes else []), db, repo)
+    cmd_further = command_for(lang, further, [], db, repo) if further else None
+    in_file = lambda f: sum(1 for r in tests.values() if r.get('at', '').startswith(f + ':'))
 
     import ax_evidence
     if as_json:
         print(json.dumps(ax_evidence.annotate('test-impact', {'changed': [{'symbol': by_target[t]['symbol'], 'kind': by_target[t]['kind'],
                                        'target': t} for t in targets],
                           'test_files': files, 'test_classes': classes,
-                          'tests': [dict(r, pulled_in_by=sorted(pulled[i])) for i, r in tests.items()],
+                          'further_test_files': further, 'command_further': cmd_further,
+                          # the tests that load the change first, then the ones further out, each nearest first
+                          'tests': [dict(r, pulled_in_by=sorted(pulled[i])) for i, r in
+                                    sorted(tests.items(), key=lambda kv: (kv[1].get('at', '').split(':')[0] in further,))],
                           'edited_test_files': edited,
                           'command': cmd_all,
                           'abstract_replaced': replaced,
@@ -2141,12 +2161,20 @@ def main(argv):
             print("\n  " + cmd_all.replace("\n", "\n  "))
             print_unrun(lang, repo, run_files)
     elif tests:
-        print(f"\ntests to run: {len(tests)} test(s) in {len(files)} file(s)")
+        print(f"\ntests to run: {sum(in_file(f) for f in files)} test(s) in {len(files)} file(s)"
+              + (" that load the changed file" if further else ""))
         for f in files[:limit]:
-            n = sum(1 for r in tests.values() if r.get('at', '').startswith(f + ':'))
-            print(f"    {f}   ({n} test(s))")
+            print(f"    {f}   ({in_file(f)} test(s))")
         if len(files) > limit:
             print(f"    … +{len(files)-limit} more file(s) (--limit N)")
+        if further:
+            print(f"\nfurther out: {sum(in_file(f) for f in further)} test(s) in {len(further)} file(s) whose file never imports the"
+                  f" changed one (a key, a dispatch or a registration joins them to it) or reaches it only beyond {NEAR_HOPS} hops;"
+                  " run after the ones above:")
+            for f in further[:limit]:
+                print(f"    {f}   ({in_file(f)} test(s))")
+            if len(further) > limit:
+                print(f"    … +{len(further)-limit} more file(s) (--limit N)")
         if why:
             print("\n  why each one is here:")
             for i, r in sorted(tests.items(), key=lambda kv: (kv[1].get('hops') or 99, kv[1].get('display', '')))[:limit]:
@@ -2167,6 +2195,8 @@ def main(argv):
         if cmd_all:
             print("\n  " + cmd_all.replace("\n", "\n  "))
         print_unrun(lang, repo, run_files)
+        if cmd_further:
+            print("\n  then, further out: " + cmd_further.replace("\n", "\n  "))
     elif edited and cmd_all:
         print("\n  " + cmd_all.replace("\n", "\n  "))
         print_unrun(lang, repo, run_files)

plugins/axiomcode/skills/axiomcode/scripts/dl/impact.dl

@@ -943,9 +943,14 @@ up_running(q, a, d+1) :- up_running(q, b, d), edge(a, b, "known_edge"), !state_g
 up_running_g(q, a, t, f, d+1) :- up_running(q, f, d), state_gate(a, f, t), edge(a, f, "known_edge"), d < 40.
 up_running_g(q, c, t, f, d+1) :- up_running_g(q, a, t, f, d), edge(c, a, "known_edge"), state_world(c, t), d < 40.
 up_running(q, c, d+1) :- up_running_g(q, a, t, f, d), edge(c, a, "known_edge"), state_exit(c, a, t, f), d < 40.
+// A module-level CONST asked about is reached by its module body only because that body declares it: its value is
+// read by the callables that import it (resolved reads, listed as such), and importing the module runs nothing new
+// for them. Hopping on from that module named every importer's module as broken at import, the readers again.
+.decl declares_seed_const(q:symbol, mod:symbol)
+declares_seed_const(q, mod) :- target(q, "field", fl, _), field(fl, mod, _, _, _), kind(mod, "module").
 .decl import_hop(q:symbol, a:symbol, b:symbol)
-import_hop(q, a, mod) :- up_running(q, mod, _), kind(mod, "module"), decl_file(mod, g), imports_file(f, g),
-                         decl_file(a, f), kind(a, "module"), a != mod.
+import_hop(q, a, mod) :- up_running(q, mod, _), kind(mod, "module"), !declares_seed_const(q, mod), decl_file(mod, g),
+                         imports_file(f, g), decl_file(a, f), kind(a, "module"), a != mod.
 up(q, a, d+1) :- up_running(q, b, d), import_hop(q, a, b), d < 40.
 parent_up(q, a, b, "at import") :- reach(q, a, d), d > 0, reach(q, b, d1), d1 = d - 1, import_hop(q, a, b).
 .output reach

plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py

@@ -685,6 +685,21 @@ def parent_up(edges, depth):
 # a file the runner imports for its fixtures and hooks and never collects tests from
 NON_TEST_FILE = _re.compile(r'(^|/)conftest\.py$')
 _RET_TYPE = _re.compile(r'\)\s*:\s*(.+)$')
+# NO RUNNER OF THESE LANGUAGES COLLECTS A FUNCTION BY ITS NAME: node:test, jest, vitest and mocha run the callback handed
+# to test(…) / it(…), which the registrar rule finds on the declaration's own line. Read by name, a helper beside the
+# tests (`export function testApp()` in test/harness.js) was a test, counted as "1 test" in a file no runner collects.
+_JS_FILE = _re.compile(r'\.(?:[cm]?[jt]sx?)$')
+# a module-level Python function is collected only from a file the runner collects (pytest's python_files, unittest's
+# test*.py): a `def test_client()` in tests/helpers.py is a helper the tests import
+_PY_COLLECTED = _re.compile(r'(^|/)(test[^/]*|[^/]*_tests?)\.py$')
+
+
+def _module_level_helper(name, owner, file):
+    """a callable named like a test that no runner of its language collects by that name"""
+    f = file or ''
+    if _JS_FILE.search(f): return True
+    if f.endswith('.py') and not owner: return not _PY_COLLECTED.search(f) or not (name or '').startswith('test')
+    return False
 
 
 def _short_decoration(d):
@@ -708,6 +723,7 @@ def named_test(name, decs, owner, file, signature):
     if not (name or '').startswith(('test', 'it')): return False
     if any(NON_TEST_DECOR.match(_short_decoration(d)) for d in decs or ()): return False
     if NON_TEST_FILE.search(file or ''): return False
+    if _module_level_helper(name, owner, file): return False
     own = (owner or '').split('.')[-1]
     if own and not TEST_OWNER.search(own): return False
     # JUnit 3 reads the convention on `public void testX()`. A method that DECLARES a return

tests/cases/javascript/a-helper-is-the-route-not-the-test/case.json

@@ -0,0 +1,18 @@
+{"lang": "javascript", "src": ".",
+ "checks": [
+  {"why": "a helper beside the tests (test/helpers.js, a function named like a test) is the route to the change, not a test: no runner collects it, so the test that calls it is listed and the helper is not",
+   "run": ["test-impact", "{repo}", "services/search/src/indexer.js"],
+   "want": ["services/search/test/search.test.js   (1 test(s))", "services/search/test/indexer.test.js   (1 test(s))"],
+   "avoid": ["helpers.js   (", "helpers.js)"]},
+  {"why": "a test whose file never imports the changed one (joined to it only by a topic key and a handler table) is listed apart, after the tests that load it, with its own command",
+   "run": ["test-impact", "{repo}", "services/search/src/indexer.js"],
+   "want": ["tests to run: 3 test(s) in 3 file(s) that load the changed file", "further out: 2 test(s) in 2 file(s)", "then, further out: (cd services/producer && node --test test/service.test.js)"]},
+  {"why": "a test that loads the change but reaches it only beyond 3 hops (a pipeline of four calls) is further out too; CONTROL: a test that imports the change through a workspace package's export subpath (@demo/search/indexer) loads it, and stays with the near tests",
+   "run": ["test-impact", "{repo}", "services/search/src/indexer.js", "--json"],
+   "want": ["\"further_test_files\": [\n  \"services/producer/test/service.test.js\",\n  \"services/search/test/pipeline.test.js\"\n ]"],
+   "avoid": ["\"services/api/test/api.test.js\"\n ],\n \"command_further\""]},
+  {"why": "CONTROL: impact still counts every test that reaches the change, the key-joined one included",
+   "run": ["impact", "index", "--tests-only"],
+   "want": ["services/producer/test/service.test.js (1)", "services/api/test/api.test.js (1)", "services/search/test/search.test.js (1)"],
+   "avoid": ["treat them as tests to run"]}
+ ]}

tests/cases/javascript/a-helper-is-the-route-not-the-test/package.json

@@ -0,0 +1 @@
+{"name": "demo", "private": true, "workspaces": ["packages/*", "services/*"]}

tests/cases/javascript/a-helper-is-the-route-not-the-test/packages/topics/index.js

@@ -0,0 +1 @@
+export const TOPICS = Object.freeze({ CREATED: 'doc.created' });

tests/cases/javascript/a-helper-is-the-route-not-the-test/packages/topics/package.json

@@ -0,0 +1 @@
+{"name": "@demo/topics", "type": "module", "main": "index.js"}

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/package.json

@@ -0,0 +1 @@
+{"name": "@demo/api", "type": "module", "main": "src/api.js", "scripts": {"test": "node --test test/*.test.js"}}

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/src/api.js

@@ -0,0 +1,3 @@
+import { index } from '@demo/search/indexer';
+
+export function reindex(doc) { return index({ payload: doc }); }

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/test/api.test.js

@@ -0,0 +1,6 @@
+import { test } from 'node:test';
+import { reindex } from '@demo/api';
+
+test('reindex goes through the search package', () => {
+  reindex({ id: 'd1' });
+});

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/package.json

@@ -0,0 +1 @@
+{"name": "@demo/producer", "type": "module", "main": "src/service.js", "scripts": {"test": "node --test test/*.test.js"}}

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/src/service.js

@@ -0,0 +1,10 @@
+import { TOPICS } from '@demo/topics';
+
+export class DocService {
+  constructor(bus) { this.bus = bus; }
+
+  create(doc) {
+    this.bus.publish(TOPICS.CREATED, doc);
+    return doc;
+  }
+}

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/test/service.test.js

@@ -0,0 +1,7 @@
+import { test } from 'node:test';
+import { DocService } from '../src/service.js';
+
+test('create publishes the created event', () => {
+  const sent = [];
+  new DocService({ publish: (type, doc) => sent.push([type, doc]) }).create({ id: 'd1' });
+});

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/package.json

@@ -0,0 +1,2 @@
+{"name": "@demo/search", "type": "module", "exports": {".": "./src/index.js", "./indexer": "./src/indexer.js"},
+ "scripts": {"test": "node --test test/*.test.js"}}

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/handlers.js

@@ -0,0 +1,6 @@
+import { TOPICS } from '@demo/topics';
+import { index } from './indexer.js';
+
+export const handlers = {
+  [TOPICS.CREATED]: async (env) => index(env),
+};

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/index.js

@@ -0,0 +1,10 @@
+import { handlers } from './handlers.js';
+
+export async function consume(table, msg) {
+  const handler = table[msg.type];
+  if (handler) return handler(msg);
+}
+
+export function start() {
+  return (msg) => consume(handlers, msg);
+}

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/indexer.js

@@ -0,0 +1 @@
+export function index(env) { return env.payload; }

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/pipeline.js

@@ -0,0 +1,7 @@
+import { index } from './indexer.js';
+
+function store(env) { return index(env); }
+function enrich(env) { return store(env); }
+function validate(env) { return enrich(env); }
+
+export function ingest(env) { return validate(env); }

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/helpers.js

@@ -0,0 +1,7 @@
+import { start } from '../src/index.js';
+
+// a helper the tests call, named like a test: no runner collects it
+export function testApp() {
+  const app = start();
+  return app({ type: 'doc.created', payload: 1 });
+}

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/indexer.test.js

@@ -0,0 +1,6 @@
+import { test } from 'node:test';
+import { index } from '../src/indexer.js';
+
+test('index returns the payload', () => {
+  index({ payload: 1 });
+});

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/pipeline.test.js

@@ -0,0 +1,6 @@
+import { test } from 'node:test';
+import { ingest } from '../src/pipeline.js';
+
+test('ingest walks the whole pipeline', () => {
+  ingest({ payload: 1 });
+});

tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/search.test.js

@@ -0,0 +1,6 @@
+import { test } from 'node:test';
+import { testApp } from './helpers.js';
+
+test('a created document is indexed through the app', async () => {
+  await testApp();
+});

tests/cases/python/conftest-fixture/case.json

@@ -6,4 +6,11 @@
    "avoid": ["test_other.py"]},
   {"why": "an autouse fixture runs before every test in its scope, named by nothing at all",
    "run": ["impact", "reset", "--tests"],
-   "want": ["test_widget.py", "test_other.py"]}]}
+   "want": ["test_widget.py", "test_other.py"]},
+  {"why": "a module-level function named like a test in a module pytest does not collect (tests/helpers.py) is the route to the change, not a test: the test that calls it is selected, the helper is not",
+   "run": ["impact", "total", "--tests"],
+   "want": ["tests: 1 of 3 test method(s)", "test_total.py"],
+   "avoid": ["treat them as tests to run"]},
+  {"why": "CONTROL: a module-level test_ function in a collected test module is still a test",
+   "run": ["impact", "reset", "--tests"],
+   "want": ["test_other_is_true"]}]}

tests/cases/python/conftest-fixture/lib.py

@@ -4,3 +4,7 @@ def make_widget():
 
 def reset():
     return 0
+
+
+def total():
+    return 2

tests/cases/python/conftest-fixture/tests/helpers.py

@@ -0,0 +1,6 @@
+from lib import total
+
+
+# a helper the tests import, named like a test: pytest collects nothing from helpers.py
+def testing_app():
+    return total()

tests/cases/python/conftest-fixture/tests/test_total.py

@@ -0,0 +1,5 @@
+from helpers import testing_app
+
+
+def test_total_is_two():
+    assert testing_app() == 2