From 9f127c5378e1c6d961a1ebc717559478a52f1592 Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:44:01 -0700 Subject: [PATCH] test-impact: a helper beside the tests is the route, not a test; far tests listed apart - A JavaScript / TypeScript function is never a test by its name (runners collect the callback handed to test()/it()), and a module-level Python test_* function counts only in a module pytest collects: a helper in test/helpers.js or tests/helpers.py was "1 test" in a file no runner collects. - impact marks each test with loads_change: whether its file (or the fixture / conftest files its runner loads) imports the changed file, through the import closure. - test-impact lists first the files that load the change within 3 hops, then "further out" the rest (a key or registration joins them, or the route is long), with its own run command. - imports_fact resolves an ES module import written with its extension and a workspace package's specifier (package.json exports / main); IMPACT_VERSION 63. - The import-time hop no longer fires from the module that merely declares a const asked about: its readers resolve, and every importer's module was listed again (exposed once .js imports resolved). Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../skills/axiomcode/scripts/ax_grep.py | 17 +++- .../skills/axiomcode/scripts/axiomcode-impact | 95 ++++++++++++++++++- .../axiomcode/scripts/axiomcode-test-impact | 38 +++++++- .../skills/axiomcode/scripts/dl/impact.dl | 9 +- .../skills/axiomcode/scripts/graph_sql.py | 16 ++++ .../case.json | 18 ++++ .../package.json | 1 + .../packages/topics/index.js | 1 + .../packages/topics/package.json | 1 + .../services/api/package.json | 1 + .../services/api/src/api.js | 3 + .../services/api/test/api.test.js | 6 ++ .../services/producer/package.json | 1 + .../services/producer/src/service.js | 10 ++ .../services/producer/test/service.test.js | 7 ++ .../services/search/package.json | 2 + .../services/search/src/handlers.js | 6 ++ .../services/search/src/index.js | 10 ++ .../services/search/src/indexer.js | 1 + .../services/search/src/pipeline.js | 7 ++ .../services/search/test/helpers.js | 7 ++ .../services/search/test/indexer.test.js | 6 ++ .../services/search/test/pipeline.test.js | 6 ++ .../services/search/test/search.test.js | 6 ++ tests/cases/python/conftest-fixture/case.json | 9 +- tests/cases/python/conftest-fixture/lib.py | 4 + .../python/conftest-fixture/tests/helpers.py | 6 ++ .../conftest-fixture/tests/test_total.py | 5 + 28 files changed, 284 insertions(+), 15 deletions(-) create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/case.json create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/package.json create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/packages/topics/index.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/packages/topics/package.json create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/package.json create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/src/api.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/test/api.test.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/package.json create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/src/service.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/test/service.test.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/package.json create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/handlers.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/index.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/indexer.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/pipeline.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/helpers.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/indexer.test.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/pipeline.test.js create mode 100644 tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/search.test.js create mode 100644 tests/cases/python/conftest-fixture/tests/helpers.py create mode 100644 tests/cases/python/conftest-fixture/tests/test_total.py diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_grep.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_grep.py index 33c4e80b5..5e4d56383 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_grep.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_grep.py @@ -142,8 +142,13 @@ def more(k, n=1): rest[k] = rest.get(k, 0) + n row = ('module' if mod else 'reached', site(code, r['at'], f"hop {r['hops']}" + (' · module scope' if mod else '') + (' · test' if r.get('test') else '') + stale(r), r['display'])) (modules if mod else rows).append(row) - for r, k in per_test_file(d.get('tests', [])): - rows.append(('test', site(code, r['at'], f"test · {TAG.get(r.get('certainty'), r.get('certainty'))} · hop {r['hops']}" + more_in_file(k) + stale(r), r['display']))) + # a test file that loads the changed file before one that never imports it (impact's loads_change) + far_files = {f for f in {(t.get('at') or '').rpartition(':')[0] for t in d.get('tests', [])} + if all(t.get('loads_change') is False for t in d.get('tests', []) if (t.get('at') or '').rpartition(':')[0] == f)} + near_some = any(t.get('loads_change') is not False for t in d.get('tests', [])) + for r, k in sorted(per_test_file(d.get('tests', [])), key=lambda rk: near_some and rk[0]['at'].rpartition(':')[0] in far_files): + far = ' · further out: never imports the change' if near_some and r['at'].rpartition(':')[0] in far_files else '' + rows.append(('test', site(code, r['at'], f"test · {TAG.get(r.get('certainty'), r.get('certainty'))} · hop {r['hops']}" + far + more_in_file(k) + stale(r), r['display']))) for r in d.get('stub_tests', []): rows.append(('stub', site(code, r['at'], f"test · stubs it{stale(r)}", r['display']))) # a test file's top level is already its test row above @@ -238,8 +243,13 @@ def test_impact(d, code): rows = [] for f in d.get('edited_test_files', []): rows.append(('test', f"{f}:1: (edited test file) [test · edited]")) + # the tests that load the changed file first; a file further out (it never imports the change, or reaches it only + # many hops away) after them, said so + further = set(d.get('further_test_files') or []) for t, k in per_test_file(d.get('tests', [])): - rows.append(('test', ev(site(code, t['at'], f"test · {TAG.get(t.get('certainty'), t.get('certainty'))} · hop {t['hops']}{more_in_file(k)}{stale(t)}", t['display']), t))) + far = (' · further out: never imports the change' if t.get('loads_change') is False else ' · further out') \ + if (t.get('at') or '').rpartition(':')[0] in further else '' + rows.append(('test', ev(site(code, t['at'], f"test · {TAG.get(t.get('certainty'), t.get('certainty'))} · hop {t['hops']}{far}{more_in_file(k)}{stale(t)}", t['display']), t))) # a changed file no graph follows (a script, a fixture) is run by the test files that name it in their text names = {} for f, v in (d.get('named_in_test_text') or {}).items(): @@ -249,6 +259,7 @@ def test_impact(d, code): rows.append(('text test', f"{t}:1: (names {', '.join(dict.fromkeys(ns))}) [test · text]")) foot = ev_foot(d) if d.get('command'): foot.append(f"run: {d['command']}") + if d.get('command_further'): foot.append(f"run: (then, further out) {d['command_further']}") if d.get('bound'): foot.append(f"bound: {d['bound']}") return rows, {}, foot diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact index c46974568..a7fd4f7ae 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact @@ -827,14 +827,56 @@ class Impact: # member of it, so every prefix is tried, longest first. parts = cand.split('.') tgt = next((mod[p] for p in ('.'.join(parts[:k]) for k in range(len(parts), 0, -1)) if p in mod), None) + elif raw.startswith('.'): + tgt = self._js_file(os.path.normpath(os.path.join(os.path.dirname(f), raw)), byfile) else: - if not raw.startswith('.'): continue # a package, not a file in this repository - base = os.path.normpath(os.path.join(os.path.dirname(f), raw)) - tgt = next((c for c in (base + e for e in ('.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', - '/index.ts', '/index.tsx', '/index.js')) if c in byfile), None) + tgt = self._workspace_import(raw, byfile) # a package of this repository (a workspace), else none if tgt and tgt != f: out.add((f, tgt)) return sorted(out) + JS_RESOLVE = ('', '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '/index.ts', '/index.tsx', '/index.js', '/index.mjs') + + def _js_file(self, base, byfile): + # an ES module names the file with its extension (`./a.js`), and TypeScript writes `./a.js` for `./a.ts` + stems = [base] + ([base[:-len(x)] for x in ('.js', '.mjs', '.cjs', '.jsx') if base.endswith(x)][:1]) + return next((c for s in stems for c in (s + e for e in self.JS_RESOLVE) if c in byfile), None) + + def _workspace_import(self, spec, byfile): + """the file a bare specifier names when it is a PACKAGE OF THIS REPOSITORY (a monorepo workspace: `@acme/bus`, + `@acme/bus/kafka`), read the way node reads it: the package.json `exports` entry for the subpath, else `main` / + `module`, else its index. A package installed from a registry has no package.json here and names no file.""" + if not hasattr(self, '_pkgs'): + self._pkgs = {} + for d in sorted({os.path.dirname(f) for f in byfile}): + while d and d not in self._pkgs: + try: + with open(os.path.join(self.g.repo, d, 'package.json'), encoding='utf-8', errors='replace') as h: pj = json.load(h) + except (OSError, ValueError): + pj = None + self._pkgs[d] = pj if isinstance(pj, dict) and isinstance(pj.get('name'), str) else None + d = os.path.dirname(d) + self._pkg_by_name = {pj['name']: (d, pj) for d, pj in sorted(self._pkgs.items()) if pj} + name = '/'.join(spec.split('/')[:2 if spec.startswith('@') else 1]) + if name not in self._pkg_by_name: return None + d, pj = self._pkg_by_name[name] + sub = '.' + spec[len(name):] + def pick(v): # a conditional export: the first condition that names a path + if isinstance(v, str): return v + if isinstance(v, dict): + for k in ('import', 'node', 'require', 'default', *v.keys()): + p = pick(v.get(k)) if k in v else None + if p: return p + return None + ex = pj.get('exports') + if isinstance(ex, dict) and any(k.startswith('.') for k in ex): + entry = pick(ex.get(sub)) + elif sub == '.': + entry = pick(ex) or pj.get('module') or pj.get('main') or 'index' + else: + entry = sub + if not entry: return None + return self._js_file(os.path.normpath(os.path.join(d, entry)), byfile) + def _nonsource(self): """the non-source scan, cached per graph (ax_nonsource.py): a query opens only the files that changed since the last one read them, and searches only those whose words include a name asked for. AXIOMCODE_NO_SCAN_CACHE=1 @@ -1269,7 +1311,7 @@ class Impact: W('cs_fixture_type', sorted(x for x in fixt if not x[0].startswith('collection:'))) # ── facts: the graph, exported once (reused while graph.sqlite is unchanged) ──────────────────────────────── - IMPACT_VERSION = '62' # 62: state_gate, state_gate_alloc, state_call_alloc, state_call_open, state_world, the callbacks one instance was given and the allocation each caller's receiver may be (JavaScript instance-state.dl); 61: filter_run, a servlet filter a test's context holds (added to HttpSecurity in a configuration the test loads, or a component) and the test methods that send a request through it; 57: a TypeScript object literal key is a ref of entity kind OBJECT_PROPERTY_KEY, kept past a bound access on its line; 56: reg_key_fact carries a handler table's entries (kind table), literal a table key written as a dotted string or through a constant, and test_code; 55: cs_data_source, cs_data_type, cs_fixture_type, the C# test links a runner makes from a data attribute or a class/collection fixture (#1498, #1499); 53: implicit_new, the type a C# `new T()` constructs where T writes no constructor (#1473); 52: test_method holds a method under a composed or derived test marker declared in the repository (a Java annotation meta-annotated @Test, a C# attribute derived from FactAttribute: #1418, #1497; 51 was the C# test-links branch's number, landed as 55); 48: sigtype, a parameter / return position type_use resolves to a type, read before the textuse grep (#1422), and persist_field, the properties a persistence query reads (#1461); 47: test_method / fixture from graph_sql's one classification (a tear-down such as @AfterEach or [TestCleanup] is a fixture, [TestInitialize] is no test, an @Override is no named test: #1417 #1419 #1502), and reg_key_fact drops a string a decoration does not register under (#1413); 46: accessor carries the wither and an isX boolean's setX / withX, and a generated builder or fluent setter the engine resolved is a writer (#1404, #1409); 45: runs_before, a C# set-up an NUnit [SetUpFixture] or an MSTest assembly initializer runs for tests outside its type (#1501), stub rows for a member a Moq Protected() setup names by string (#1540), cs_config_literal for a Section:Key configuration key (#1443), and lex_parent puts a lambda under the declaration on its own line (#1556); 44: a C# MEMBER_ACCESS ref is qualified, so its qualifier decides (#1445); 41: spawns_fact, a test that runs a script by its path (ax_spawn.py); 40: test_method holds a script test's module (a test-tree file run as a program, no framework: graph_sql.script_tests); 39: a chained route link's `calls` row and `registration` label sit on the link's own line, with its own verb and path; 38: reg_key_fact drops a decoration string with a space in it (a description, not a key); 37: via_base / via_site, a caller that reaches a declaration through a base it is override-equivalent to (#1542), and injected_bean, the bean an injection point was wired to (#1384); 36: handoff_at, route_arg, callable_const, init_wrapper, init_alias, returns_fn — a const holding a wrapped handler registered at a route; 35: 0.1.5's 33 (#1598, the defines edges the path export links) joined 0.1.6's 33, two different fact sets under one number; 33 (0.1.6): calls carries the tier "stub" for a call inside a mock's stub or verification, reg_verb / lit_verb join a route by its HTTP method, and a handler's route joins its type's prefix; 32: cert_tier's why is worded per tier (an event_dispatch row says it sends the request or event), and the route facts #1633 changed (#1510), which merged without a bump; 31: event_dispatch edges (a published event reaches its listeners, #1391) and the pytest fixture_injection reading (#1631) change impact's facts; 30: registers, a bean another class's annotation registers (#1396, #1414); 29: the edges it links from the path export changed (#1402), and a cache written before it must not survive; 28: reexport_from, the file an `export *` line re-exports; 27: framework, the engine's framework_edge (#1509); 24: the test* naming convention requires a test class as owner (#1181); 23: owner/member disambiguated by file, two classes of one name no longer merging (#1188); 22: lex_parent, the innermost declaration enclosing each one by span (#1183); 21: cert_tier, the tier -> certainty table the call rules join on (#1131); 20: faccess, the engine's resolved field accesses (#1071); 3: decl_file facts (the import-time test-file rule); 14: the registration-key + IMPACT_VERSION = '63' # 63: imports_fact resolves an import written with its extension and a workspace package's specifier, and test_method takes no JavaScript / TypeScript helper by its name; 62: state_gate, state_gate_alloc, state_call_alloc, state_call_open, state_world, the callbacks one instance was given and the allocation each caller's receiver may be (JavaScript instance-state.dl); 61: filter_run, a servlet filter a test's context holds (added to HttpSecurity in a configuration the test loads, or a component) and the test methods that send a request through it; 57: a TypeScript object literal key is a ref of entity kind OBJECT_PROPERTY_KEY, kept past a bound access on its line; 56: reg_key_fact carries a handler table's entries (kind table), literal a table key written as a dotted string or through a constant, and test_code; 55: cs_data_source, cs_data_type, cs_fixture_type, the C# test links a runner makes from a data attribute or a class/collection fixture (#1498, #1499); 53: implicit_new, the type a C# `new T()` constructs where T writes no constructor (#1473); 52: test_method holds a method under a composed or derived test marker declared in the repository (a Java annotation meta-annotated @Test, a C# attribute derived from FactAttribute: #1418, #1497; 51 was the C# test-links branch's number, landed as 55); 48: sigtype, a parameter / return position type_use resolves to a type, read before the textuse grep (#1422), and persist_field, the properties a persistence query reads (#1461); 47: test_method / fixture from graph_sql's one classification (a tear-down such as @AfterEach or [TestCleanup] is a fixture, [TestInitialize] is no test, an @Override is no named test: #1417 #1419 #1502), and reg_key_fact drops a string a decoration does not register under (#1413); 46: accessor carries the wither and an isX boolean's setX / withX, and a generated builder or fluent setter the engine resolved is a writer (#1404, #1409); 45: runs_before, a C# set-up an NUnit [SetUpFixture] or an MSTest assembly initializer runs for tests outside its type (#1501), stub rows for a member a Moq Protected() setup names by string (#1540), cs_config_literal for a Section:Key configuration key (#1443), and lex_parent puts a lambda under the declaration on its own line (#1556); 44: a C# MEMBER_ACCESS ref is qualified, so its qualifier decides (#1445); 41: spawns_fact, a test that runs a script by its path (ax_spawn.py); 40: test_method holds a script test's module (a test-tree file run as a program, no framework: graph_sql.script_tests); 39: a chained route link's `calls` row and `registration` label sit on the link's own line, with its own verb and path; 38: reg_key_fact drops a decoration string with a space in it (a description, not a key); 37: via_base / via_site, a caller that reaches a declaration through a base it is override-equivalent to (#1542), and injected_bean, the bean an injection point was wired to (#1384); 36: handoff_at, route_arg, callable_const, init_wrapper, init_alias, returns_fn — a const holding a wrapped handler registered at a route; 35: 0.1.5's 33 (#1598, the defines edges the path export links) joined 0.1.6's 33, two different fact sets under one number; 33 (0.1.6): calls carries the tier "stub" for a call inside a mock's stub or verification, reg_verb / lit_verb join a route by its HTTP method, and a handler's route joins its type's prefix; 32: cert_tier's why is worded per tier (an event_dispatch row says it sends the request or event), and the route facts #1633 changed (#1510), which merged without a bump; 31: event_dispatch edges (a published event reaches its listeners, #1391) and the pytest fixture_injection reading (#1631) change impact's facts; 30: registers, a bean another class's annotation registers (#1396, #1414); 29: the edges it links from the path export changed (#1402), and a cache written before it must not survive; 28: reexport_from, the file an `export *` line re-exports; 27: framework, the engine's framework_edge (#1509); 24: the test* naming convention requires a test class as owner (#1181); 23: owner/member disambiguated by file, two classes of one name no longer merging (#1188); 22: lex_parent, the innermost declaration enclosing each one by span (#1183); 21: cert_tier, the tier -> certainty table the call rules join on (#1131); 20: faccess, the engine's resolved field accesses (#1071); 3: decl_file facts (the import-time test-file rule); 14: the registration-key # layer; 15: the registration facts (two 14s landed independently, which is exactly the collision this # guards); 16: regsite folded into ax_registration's reg_key_fact; 20: implements_pair (#1011); 17/18: the tagged-template test registrar # (it.each`…`) and its table span @@ -2736,6 +2778,43 @@ def main(argv): # framework's, not an edge, which is why `chains` (verified hop by hop below) keeps the test's own chain. def test_route(m, fx): return [m] + I.chain(fx, parent) if fx else chains.get(m, []) + # DOES THE TEST LOAD THE CHANGE? A test runs code in its own process only once its file, or the fixture's file the + # runner loads for it (and in Python the conftest.py files above it), imports the changed file, directly or through + # other files. A route that never does -- a key join from a test that only writes the topic string, a dispatch + # resolved to a table in a service the test never starts -- is a lead further out, not a test of the change: on + # one monorepo 19 test files were selected for a handler table's module and 2 of them import it. None where the + # language has no import graph here (imports_fact covers Python, JavaScript and TypeScript). + _rev = None; _importers = {} + def loads_change(m, fx): + nonlocal _rev + tf = g.sym[m].get('file') or '' + route = test_route(m, fx) + seed_file = (g.sym.get(route[-1]) or {}).get('file') if route else None + if not tf or not seed_file or os.path.splitext(tf)[1].lower() not in I.IMPORT_RE: return None + if tf == seed_file or test_cert.get(m) in ('spawns', 'at import'): return True + if _rev is None: + _rev = collections.defaultdict(set) + try: + for ln in open(os.path.join(g.facts, 'impact', 'imports_fact.facts'), encoding='utf-8', errors='replace'): + a, _, b = ln.rstrip('\n').partition('\t') + if a and b: _rev[b].add(a) + except OSError: + _rev = {} # no import facts: nothing can be said either way + if not _rev: return None + if seed_file not in _importers: + seen, todo = {seed_file}, [seed_file] + while todo: + for a in _rev.get(todo.pop(), ()): + if a not in seen: seen.add(a); todo.append(a) + _importers[seed_file] = seen + loaders = {tf} | ({g.sym[fx].get('file')} if fx and fx in g.sym else set()) + if tf.endswith('.py'): + d_ = os.path.dirname(tf) + while True: + loaders.add(f"{d_}/conftest.py" if d_ else 'conftest.py') + if not d_: break + d_ = os.path.dirname(d_) + return bool(loaders & _importers[seed_file]) # a MODULE counted as a test is a script test (graph_sql.script_tests): it is labelled with the command it runs by script_ids = {m for m in tests if g.sym[m]['kind'] == 'module'} _runs = {} @@ -2959,6 +3038,7 @@ def main(argv): 'alongside': [{'id': c, 'display': g.disp(c), 'role': 'co-located', 'why': why, 'reasons': direct_reasons.get((c, _grp(role)), []), 'certainty': cert, 'at': loc, 'for': sorted(direct_for[(c, _grp(role))])} for c, role, why, cert, loc, n in sorted(D, key=lambda x: (g.disp(x[0]), x[2], x[4], x[0])) if cert == 'alongside'], 'reached': [{'id': m, 'display': g.disp(m), 'hops': d, 'for': sorted(reach_from[m]), 'at': g.loc(m), 'test': bool(g.sym[m]['is_test'])} for m, d in sorted(reached.items(), key=lambda x: (x[1], g.disp(x[0]), g.loc(x[0]), x[0]))], 'tests': [{'id': m, 'display': g.disp(m), 'owner': g.sym[m]['owner'], 'name': g.sym[m]['name'], 'hops': d, 'via': g.disp(fx) if fx else None, 'at': g.loc(m), 'certainty': test_cert.get(m), 'chain': [g.disp(x) for x in test_route(m, fx)], + **({'loads_change': lc} if (lc := loads_change(m, fx)) is not None else {}), **({'script': True, 'run': script_cmd(m)} if m in script_ids else {})} for m, (d, fx) in sorted(tests.items(), key=lambda kv: (kv[1][0], g.disp(kv[0]), g.loc(kv[0]), kv[0]))], 'framework_entries': [{'id': m, 'display': g.disp(m), 'signal': sig, 'at': g.loc(m)} for m, sig in fw_ent], 'framework_grep': fw_grep, @@ -3334,6 +3414,11 @@ def main(argv): # a callable in a file whose test is counted runs when that file does (`main` under a script's main guard, the # describe block around a counted `it`): the count already credits it test_rows = [m for m in test_rows if g.sym[m]['file'] not in tfiles] + # a helper beside the tests (test/helpers.js, tests/helpers.py) that a counted test calls on its way to the change is + # that test's route, not an uncredited test: the count already runs it + if test_rows: + on_route = {x for t, v in tests.items() for x in test_route(t, v[1] if isinstance(v, tuple) and len(v) > 1 else None)} + test_rows = [m for m in test_rows if m not in on_route] if test_rows: print(f" {'and ' if tests else 'but '}{len(test_rows)} callable(s) in test code listed above reach it by a route this count does not credit " "(a runner entry it does not collect, a remote or framework hop) — treat them as tests to run: " diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-test-impact b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-test-impact index d086a90de..29e9f97e9 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-test-impact +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-test-impact @@ -62,6 +62,8 @@ is not the same as a change that does nothing, and the rung on each row is where import importlib.machinery, importlib.util, json, os, re, subprocess, sys, collections, functools, posixpath, shlex TESTY = re.compile(r'(^|[/_.-])(test|tests|spec|specs|__tests__|it)([/_.-]|$)', re.I) +# a test that loads the change and reaches it within this many hops is listed first; beyond it, further out +NEAR_HOPS = 3 HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) @@ -2034,6 +2036,19 @@ def main(argv): case_data_tests = {i for i, r in tests.items() if r.get('at') and ax_caserun.in_case_dir(real, r['at'].split(':')[0])} for i in case_data_tests: tests.pop(i, None) files = sorted({r.get('at', '').split(':')[0] for r in tests.values() if r.get('at')}) + # NEAREST FIRST, AND FURTHER OUT APART: a test whose file (or the fixture file its runner loads) never imports the + # changed file reaches it only through a key join, a dispatch resolved across services, or a registration the + # process it runs in never makes (impact's loads_change). Still listed, after the tests that load the change. + # One that loads it but reaches it only beyond NEAR_HOPS (a whole service started for an unrelated flow, whose + # route crosses a bus and a handler table) is further out too: impact's own measurement puts a sound route within + # 3 hops at about 9 in 10 right, and a hub answer at 3 in 10. Where the import graph is unknown (no loads_change), + # nothing moves. + near = lambda r: r.get('loads_change') is None or (r['loads_change'] and (r.get('hops') or 0) <= NEAR_HOPS) + further = sorted({f for f in files} - {(r.get('at') or '').split(':')[0] for r in tests.values() if near(r)}) + if further and len(further) < len(files): + files = [f for f in files if f not in set(further)] + else: + further = [] # nothing loads it, or everything does: one list, as before # A selected test carrying no location drops out of `files`, and `files` is what the command PRINTS and # what the run command is built from — so the answer names it nowhere and the suite runs without it. # This is the third instance of one bug: a row discarded for being malformed in one field. The other two @@ -2060,13 +2075,18 @@ def main(argv): if classes and lang in ('java', 'csharp') else classes replaced = concrete_test_classes(db, run_classes)[1] if lang in ('java', 'csharp') else {} cmd_all = command_for(lang, run_files, run_classes + ([os.path.splitext(os.path.basename(f))[0] for f in edited] if classes else []), db, repo) + cmd_further = command_for(lang, further, [], db, repo) if further else None + in_file = lambda f: sum(1 for r in tests.values() if r.get('at', '').startswith(f + ':')) import ax_evidence if as_json: print(json.dumps(ax_evidence.annotate('test-impact', {'changed': [{'symbol': by_target[t]['symbol'], 'kind': by_target[t]['kind'], 'target': t} for t in targets], 'test_files': files, 'test_classes': classes, - 'tests': [dict(r, pulled_in_by=sorted(pulled[i])) for i, r in tests.items()], + 'further_test_files': further, 'command_further': cmd_further, + # the tests that load the change first, then the ones further out, each nearest first + 'tests': [dict(r, pulled_in_by=sorted(pulled[i])) for i, r in + sorted(tests.items(), key=lambda kv: (kv[1].get('at', '').split(':')[0] in further,))], 'edited_test_files': edited, 'command': cmd_all, 'abstract_replaced': replaced, @@ -2141,12 +2161,20 @@ def main(argv): print("\n " + cmd_all.replace("\n", "\n ")) print_unrun(lang, repo, run_files) elif tests: - print(f"\ntests to run: {len(tests)} test(s) in {len(files)} file(s)") + print(f"\ntests to run: {sum(in_file(f) for f in files)} test(s) in {len(files)} file(s)" + + (" that load the changed file" if further else "")) for f in files[:limit]: - n = sum(1 for r in tests.values() if r.get('at', '').startswith(f + ':')) - print(f" {f} ({n} test(s))") + print(f" {f} ({in_file(f)} test(s))") if len(files) > limit: print(f" … +{len(files)-limit} more file(s) (--limit N)") + if further: + print(f"\nfurther out: {sum(in_file(f) for f in further)} test(s) in {len(further)} file(s) whose file never imports the" + f" changed one (a key, a dispatch or a registration joins them to it) or reaches it only beyond {NEAR_HOPS} hops;" + " run after the ones above:") + for f in further[:limit]: + print(f" {f} ({in_file(f)} test(s))") + if len(further) > limit: + print(f" … +{len(further)-limit} more file(s) (--limit N)") if why: print("\n why each one is here:") for i, r in sorted(tests.items(), key=lambda kv: (kv[1].get('hops') or 99, kv[1].get('display', '')))[:limit]: @@ -2167,6 +2195,8 @@ def main(argv): if cmd_all: print("\n " + cmd_all.replace("\n", "\n ")) print_unrun(lang, repo, run_files) + if cmd_further: + print("\n then, further out: " + cmd_further.replace("\n", "\n ")) elif edited and cmd_all: print("\n " + cmd_all.replace("\n", "\n ")) print_unrun(lang, repo, run_files) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/dl/impact.dl b/plugins/axiomcode/skills/axiomcode/scripts/dl/impact.dl index 83d418e32..f8d70e364 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/dl/impact.dl +++ b/plugins/axiomcode/skills/axiomcode/scripts/dl/impact.dl @@ -943,9 +943,14 @@ up_running(q, a, d+1) :- up_running(q, b, d), edge(a, b, "known_edge"), !state_g up_running_g(q, a, t, f, d+1) :- up_running(q, f, d), state_gate(a, f, t), edge(a, f, "known_edge"), d < 40. up_running_g(q, c, t, f, d+1) :- up_running_g(q, a, t, f, d), edge(c, a, "known_edge"), state_world(c, t), d < 40. up_running(q, c, d+1) :- up_running_g(q, a, t, f, d), edge(c, a, "known_edge"), state_exit(c, a, t, f), d < 40. +// A module-level CONST asked about is reached by its module body only because that body declares it: its value is +// read by the callables that import it (resolved reads, listed as such), and importing the module runs nothing new +// for them. Hopping on from that module named every importer's module as broken at import, the readers again. +.decl declares_seed_const(q:symbol, mod:symbol) +declares_seed_const(q, mod) :- target(q, "field", fl, _), field(fl, mod, _, _, _), kind(mod, "module"). .decl import_hop(q:symbol, a:symbol, b:symbol) -import_hop(q, a, mod) :- up_running(q, mod, _), kind(mod, "module"), decl_file(mod, g), imports_file(f, g), - decl_file(a, f), kind(a, "module"), a != mod. +import_hop(q, a, mod) :- up_running(q, mod, _), kind(mod, "module"), !declares_seed_const(q, mod), decl_file(mod, g), + imports_file(f, g), decl_file(a, f), kind(a, "module"), a != mod. up(q, a, d+1) :- up_running(q, b, d), import_hop(q, a, b), d < 40. parent_up(q, a, b, "at import") :- reach(q, a, d), d > 0, reach(q, b, d1), d1 = d - 1, import_hop(q, a, b). .output reach diff --git a/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py b/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py index 538ca7e09..db0458a3e 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py @@ -685,6 +685,21 @@ def parent_up(edges, depth): # a file the runner imports for its fixtures and hooks and never collects tests from NON_TEST_FILE = _re.compile(r'(^|/)conftest\.py$') _RET_TYPE = _re.compile(r'\)\s*:\s*(.+)$') +# NO RUNNER OF THESE LANGUAGES COLLECTS A FUNCTION BY ITS NAME: node:test, jest, vitest and mocha run the callback handed +# to test(…) / it(…), which the registrar rule finds on the declaration's own line. Read by name, a helper beside the +# tests (`export function testApp()` in test/harness.js) was a test, counted as "1 test" in a file no runner collects. +_JS_FILE = _re.compile(r'\.(?:[cm]?[jt]sx?)$') +# a module-level Python function is collected only from a file the runner collects (pytest's python_files, unittest's +# test*.py): a `def test_client()` in tests/helpers.py is a helper the tests import +_PY_COLLECTED = _re.compile(r'(^|/)(test[^/]*|[^/]*_tests?)\.py$') + + +def _module_level_helper(name, owner, file): + """a callable named like a test that no runner of its language collects by that name""" + f = file or '' + if _JS_FILE.search(f): return True + if f.endswith('.py') and not owner: return not _PY_COLLECTED.search(f) or not (name or '').startswith('test') + return False def _short_decoration(d): @@ -708,6 +723,7 @@ def named_test(name, decs, owner, file, signature): if not (name or '').startswith(('test', 'it')): return False if any(NON_TEST_DECOR.match(_short_decoration(d)) for d in decs or ()): return False if NON_TEST_FILE.search(file or ''): return False + if _module_level_helper(name, owner, file): return False own = (owner or '').split('.')[-1] if own and not TEST_OWNER.search(own): return False # JUnit 3 reads the convention on `public void testX()`. A method that DECLARES a return diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/case.json b/tests/cases/javascript/a-helper-is-the-route-not-the-test/case.json new file mode 100644 index 000000000..d429fc0a7 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/case.json @@ -0,0 +1,18 @@ +{"lang": "javascript", "src": ".", + "checks": [ + {"why": "a helper beside the tests (test/helpers.js, a function named like a test) is the route to the change, not a test: no runner collects it, so the test that calls it is listed and the helper is not", + "run": ["test-impact", "{repo}", "services/search/src/indexer.js"], + "want": ["services/search/test/search.test.js (1 test(s))", "services/search/test/indexer.test.js (1 test(s))"], + "avoid": ["helpers.js (", "helpers.js)"]}, + {"why": "a test whose file never imports the changed one (joined to it only by a topic key and a handler table) is listed apart, after the tests that load it, with its own command", + "run": ["test-impact", "{repo}", "services/search/src/indexer.js"], + "want": ["tests to run: 3 test(s) in 3 file(s) that load the changed file", "further out: 2 test(s) in 2 file(s)", "then, further out: (cd services/producer && node --test test/service.test.js)"]}, + {"why": "a test that loads the change but reaches it only beyond 3 hops (a pipeline of four calls) is further out too; CONTROL: a test that imports the change through a workspace package's export subpath (@demo/search/indexer) loads it, and stays with the near tests", + "run": ["test-impact", "{repo}", "services/search/src/indexer.js", "--json"], + "want": ["\"further_test_files\": [\n \"services/producer/test/service.test.js\",\n \"services/search/test/pipeline.test.js\"\n ]"], + "avoid": ["\"services/api/test/api.test.js\"\n ],\n \"command_further\""]}, + {"why": "CONTROL: impact still counts every test that reaches the change, the key-joined one included", + "run": ["impact", "index", "--tests-only"], + "want": ["services/producer/test/service.test.js (1)", "services/api/test/api.test.js (1)", "services/search/test/search.test.js (1)"], + "avoid": ["treat them as tests to run"]} + ]} diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/package.json b/tests/cases/javascript/a-helper-is-the-route-not-the-test/package.json new file mode 100644 index 000000000..d0426ba55 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/package.json @@ -0,0 +1 @@ +{"name": "demo", "private": true, "workspaces": ["packages/*", "services/*"]} diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/packages/topics/index.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/packages/topics/index.js new file mode 100644 index 000000000..0ea6e0714 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/packages/topics/index.js @@ -0,0 +1 @@ +export const TOPICS = Object.freeze({ CREATED: 'doc.created' }); diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/packages/topics/package.json b/tests/cases/javascript/a-helper-is-the-route-not-the-test/packages/topics/package.json new file mode 100644 index 000000000..968e6c388 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/packages/topics/package.json @@ -0,0 +1 @@ +{"name": "@demo/topics", "type": "module", "main": "index.js"} diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/package.json b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/package.json new file mode 100644 index 000000000..acdbe729e --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/package.json @@ -0,0 +1 @@ +{"name": "@demo/api", "type": "module", "main": "src/api.js", "scripts": {"test": "node --test test/*.test.js"}} diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/src/api.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/src/api.js new file mode 100644 index 000000000..2e1d55843 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/src/api.js @@ -0,0 +1,3 @@ +import { index } from '@demo/search/indexer'; + +export function reindex(doc) { return index({ payload: doc }); } diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/test/api.test.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/test/api.test.js new file mode 100644 index 000000000..dffd5d019 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/api/test/api.test.js @@ -0,0 +1,6 @@ +import { test } from 'node:test'; +import { reindex } from '@demo/api'; + +test('reindex goes through the search package', () => { + reindex({ id: 'd1' }); +}); diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/package.json b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/package.json new file mode 100644 index 000000000..f540c1225 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/package.json @@ -0,0 +1 @@ +{"name": "@demo/producer", "type": "module", "main": "src/service.js", "scripts": {"test": "node --test test/*.test.js"}} diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/src/service.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/src/service.js new file mode 100644 index 000000000..98543c0c8 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/src/service.js @@ -0,0 +1,10 @@ +import { TOPICS } from '@demo/topics'; + +export class DocService { + constructor(bus) { this.bus = bus; } + + create(doc) { + this.bus.publish(TOPICS.CREATED, doc); + return doc; + } +} diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/test/service.test.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/test/service.test.js new file mode 100644 index 000000000..6ef201c13 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/producer/test/service.test.js @@ -0,0 +1,7 @@ +import { test } from 'node:test'; +import { DocService } from '../src/service.js'; + +test('create publishes the created event', () => { + const sent = []; + new DocService({ publish: (type, doc) => sent.push([type, doc]) }).create({ id: 'd1' }); +}); diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/package.json b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/package.json new file mode 100644 index 000000000..791f28c13 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/package.json @@ -0,0 +1,2 @@ +{"name": "@demo/search", "type": "module", "exports": {".": "./src/index.js", "./indexer": "./src/indexer.js"}, + "scripts": {"test": "node --test test/*.test.js"}} diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/handlers.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/handlers.js new file mode 100644 index 000000000..69a88e465 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/handlers.js @@ -0,0 +1,6 @@ +import { TOPICS } from '@demo/topics'; +import { index } from './indexer.js'; + +export const handlers = { + [TOPICS.CREATED]: async (env) => index(env), +}; diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/index.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/index.js new file mode 100644 index 000000000..111c5894a --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/index.js @@ -0,0 +1,10 @@ +import { handlers } from './handlers.js'; + +export async function consume(table, msg) { + const handler = table[msg.type]; + if (handler) return handler(msg); +} + +export function start() { + return (msg) => consume(handlers, msg); +} diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/indexer.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/indexer.js new file mode 100644 index 000000000..917635b5e --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/indexer.js @@ -0,0 +1 @@ +export function index(env) { return env.payload; } diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/pipeline.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/pipeline.js new file mode 100644 index 000000000..e75f8af54 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/src/pipeline.js @@ -0,0 +1,7 @@ +import { index } from './indexer.js'; + +function store(env) { return index(env); } +function enrich(env) { return store(env); } +function validate(env) { return enrich(env); } + +export function ingest(env) { return validate(env); } diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/helpers.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/helpers.js new file mode 100644 index 000000000..e59844016 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/helpers.js @@ -0,0 +1,7 @@ +import { start } from '../src/index.js'; + +// a helper the tests call, named like a test: no runner collects it +export function testApp() { + const app = start(); + return app({ type: 'doc.created', payload: 1 }); +} diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/indexer.test.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/indexer.test.js new file mode 100644 index 000000000..4d47fbc1e --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/indexer.test.js @@ -0,0 +1,6 @@ +import { test } from 'node:test'; +import { index } from '../src/indexer.js'; + +test('index returns the payload', () => { + index({ payload: 1 }); +}); diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/pipeline.test.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/pipeline.test.js new file mode 100644 index 000000000..708c9413f --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/pipeline.test.js @@ -0,0 +1,6 @@ +import { test } from 'node:test'; +import { ingest } from '../src/pipeline.js'; + +test('ingest walks the whole pipeline', () => { + ingest({ payload: 1 }); +}); diff --git a/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/search.test.js b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/search.test.js new file mode 100644 index 000000000..8021bfa38 --- /dev/null +++ b/tests/cases/javascript/a-helper-is-the-route-not-the-test/services/search/test/search.test.js @@ -0,0 +1,6 @@ +import { test } from 'node:test'; +import { testApp } from './helpers.js'; + +test('a created document is indexed through the app', async () => { + await testApp(); +}); diff --git a/tests/cases/python/conftest-fixture/case.json b/tests/cases/python/conftest-fixture/case.json index e334c16a3..6a32efee0 100644 --- a/tests/cases/python/conftest-fixture/case.json +++ b/tests/cases/python/conftest-fixture/case.json @@ -6,4 +6,11 @@ "avoid": ["test_other.py"]}, {"why": "an autouse fixture runs before every test in its scope, named by nothing at all", "run": ["impact", "reset", "--tests"], - "want": ["test_widget.py", "test_other.py"]}]} + "want": ["test_widget.py", "test_other.py"]}, + {"why": "a module-level function named like a test in a module pytest does not collect (tests/helpers.py) is the route to the change, not a test: the test that calls it is selected, the helper is not", + "run": ["impact", "total", "--tests"], + "want": ["tests: 1 of 3 test method(s)", "test_total.py"], + "avoid": ["treat them as tests to run"]}, + {"why": "CONTROL: a module-level test_ function in a collected test module is still a test", + "run": ["impact", "reset", "--tests"], + "want": ["test_other_is_true"]}]} diff --git a/tests/cases/python/conftest-fixture/lib.py b/tests/cases/python/conftest-fixture/lib.py index 1318ec7fe..ce7b91657 100644 --- a/tests/cases/python/conftest-fixture/lib.py +++ b/tests/cases/python/conftest-fixture/lib.py @@ -4,3 +4,7 @@ def make_widget(): def reset(): return 0 + + +def total(): + return 2 diff --git a/tests/cases/python/conftest-fixture/tests/helpers.py b/tests/cases/python/conftest-fixture/tests/helpers.py new file mode 100644 index 000000000..cfaf3034d --- /dev/null +++ b/tests/cases/python/conftest-fixture/tests/helpers.py @@ -0,0 +1,6 @@ +from lib import total + + +# a helper the tests import, named like a test: pytest collects nothing from helpers.py +def testing_app(): + return total() diff --git a/tests/cases/python/conftest-fixture/tests/test_total.py b/tests/cases/python/conftest-fixture/tests/test_total.py new file mode 100644 index 000000000..e87039f3c --- /dev/null +++ b/tests/cases/python/conftest-fixture/tests/test_total.py @@ -0,0 +1,5 @@ +from helpers import testing_app + + +def test_total_is_two(): + assert testing_app() == 2