diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact index a55fec968..6c7d459f9 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact @@ -63,6 +63,16 @@ _spec = importlib.util.spec_from_loader('axpath', importlib.machinery.SourceFile P = importlib.util.module_from_spec(_spec); _spec.loader.exec_module(P) G, run_dl, die, BODILESS_KINDS = P.G, P.run_dl, P.die, P.BODILESS_KINDS + +def script_run(repo, rel): + """the command a script test (graph_sql.script_test_file) is run by: the one test-impact gives for the file — its + package's script, its own usage line, the interpreter — so impact and test-impact name the same command""" + global _TI + if '_TI' not in globals(): + spec = importlib.util.spec_from_loader('axtestimpact', importlib.machinery.SourceFileLoader('axtestimpact', os.path.join(HERE, 'axiomcode-test-impact'))) + _TI = importlib.util.module_from_spec(spec); spec.loader.exec_module(_TI) + return _TI.script_command(repo, rel) + # written per query (or supplied by the path tool's own export), so the cached export is NOT expected to hold them. Leaving # `nonsource` / `qual_name` / `edge` / `named` out of this set made the completeness check below unsatisfiable, so the 15 MB # export re-ran on EVERY query — the cache never hit once. @@ -736,7 +746,7 @@ class Impact: return [ln for ln in range(s['line'], min(s['end_line'], len(L)) + 1) if pat.search(L[ln - 1])] # ── facts: the graph, exported once (reused while graph.sqlite is unchanged) ──────────────────────────────── - IMPACT_VERSION = '39' # 39: a chained route link's `calls` row and `registration` label sit on the link's own line, with its own verb and path; 38: reg_key_fact drops a decoration string with a space in it (a description, not a key); 37: via_base / via_site, a caller that reaches a declaration through a base it is override-equivalent to (#1542), and injected_bean, the bean an injection point was wired to (#1384); 36: handoff_at, route_arg, callable_const, init_wrapper, init_alias, returns_fn — a const holding a wrapped handler registered at a route; 35: 0.1.5's 33 (#1598, the defines edges the path export links) joined 0.1.6's 33, two different fact sets under one number; 33 (0.1.6): calls carries the tier "stub" for a call inside a mock's stub or verification, reg_verb / lit_verb join a route by its HTTP method, and a handler's route joins its type's prefix; 32: cert_tier's why is worded per tier (an event_dispatch row says it sends the request or event), and the route facts #1633 changed (#1510), which merged without a bump; 31: event_dispatch edges (a published event reaches its listeners, #1391) and the pytest fixture_injection reading (#1631) change impact's facts; 30: registers, a bean another class's annotation registers (#1396, #1414); 29: the edges it links from the path export changed (#1402), and a cache written before it must not survive; 28: reexport_from, the file an `export *` line re-exports; 27: framework, the engine's framework_edge (#1509); 24: the test* naming convention requires a test class as owner (#1181); 23: owner/member disambiguated by file, two classes of one name no longer merging (#1188); 22: lex_parent, the innermost declaration enclosing each one by span (#1183); 21: cert_tier, the tier -> certainty table the call rules join on (#1131); 20: faccess, the engine's resolved field accesses (#1071); 3: decl_file facts (the import-time test-file rule); 14: the registration-key + IMPACT_VERSION = '40' # 40: test_method holds a script test's module (a test-tree file run as a program, no framework: graph_sql.script_tests); 39: a chained route link's `calls` row and `registration` label sit on the link's own line, with its own verb and path; 38: reg_key_fact drops a decoration string with a space in it (a description, not a key); 37: via_base / via_site, a caller that reaches a declaration through a base it is override-equivalent to (#1542), and injected_bean, the bean an injection point was wired to (#1384); 36: handoff_at, route_arg, callable_const, init_wrapper, init_alias, returns_fn — a const holding a wrapped handler registered at a route; 35: 0.1.5's 33 (#1598, the defines edges the path export links) joined 0.1.6's 33, two different fact sets under one number; 33 (0.1.6): calls carries the tier "stub" for a call inside a mock's stub or verification, reg_verb / lit_verb join a route by its HTTP method, and a handler's route joins its type's prefix; 32: cert_tier's why is worded per tier (an event_dispatch row says it sends the request or event), and the route facts #1633 changed (#1510), which merged without a bump; 31: event_dispatch edges (a published event reaches its listeners, #1391) and the pytest fixture_injection reading (#1631) change impact's facts; 30: registers, a bean another class's annotation registers (#1396, #1414); 29: the edges it links from the path export changed (#1402), and a cache written before it must not survive; 28: reexport_from, the file an `export *` line re-exports; 27: framework, the engine's framework_edge (#1509); 24: the test* naming convention requires a test class as owner (#1181); 23: owner/member disambiguated by file, two classes of one name no longer merging (#1188); 22: lex_parent, the innermost declaration enclosing each one by span (#1183); 21: cert_tier, the tier -> certainty table the call rules join on (#1131); 20: faccess, the engine's resolved field accesses (#1071); 3: decl_file facts (the import-time test-file rule); 14: the registration-key # layer; 15: the registration facts (two 14s landed independently, which is exactly the collision this # guards); 16: regsite folded into ax_registration's reg_key_fact; 20: implements_pair (#1011); 17/18: the tagged-template test registrar # (it.each`…`) and its table span @@ -1106,7 +1116,13 @@ class Impact: L = self.lines(sy['file']) if sy['line'] - 1 < len(L) and REGISTRAR.search(L[sy['line'] - 1]): tm.append(i) elif any(f == sy['file'] and a <= sy['line'] <= b for f, a, b in tables): tm.append(i) - fx = [i for i, s in g.sym.items() if s['is_test'] and ((s.get('type_id') and not s.get('method_id')) or s['kind'] in ('constructor', 'module') or s['name'] in FIXTURE_NAMES or any(FIXTURE_DECOR.match(d) for d in decs.get(i, ())))] + # …and a SCRIPT TEST, a test-tree file run as a program with no framework: its module is the test (graph_sql, + # the one rule the hooks' fast path reads too) + import graph_sql + scripts = graph_sql.script_tests([(i, s['kind'], s.get('file')) for i, s in g.sym.items() if s['is_test'] and s.get('method_id') and s.get('file')], + lambda f: '\n'.join(self.lines(f)), set(tm)) + tm += sorted(scripts) + fx = [i for i, s in g.sym.items() if s['is_test'] and i not in scripts and ((s.get('type_id') and not s.get('method_id')) or s['kind'] in ('constructor', 'module') or s['name'] in FIXTURE_NAMES or any(FIXTURE_DECOR.match(d) for d in decs.get(i, ())))] W('test_method', [(m,) for m in tm]); W('fixture', [(m,) for m in fx]) # ── which fixture runs before which test, when nothing calls it ──────────────────────────────────────── # pytest hands a fixture to a test BY THE NAME OF A PARAMETER, and the fixture that serves a whole directory @@ -1957,6 +1973,12 @@ def main(argv): at_d = [x[2] for x in res['test_hit'] if x[0] == m and int(x[1]) == int(d)] _vias = sorted(v for v in at_d if v) tests[m] = (int(d), None if any(not v for v in at_d) else (_vias[0] if _vias else None)) + # A SCRIPT TEST (a module counted as a test) runs its own functions: `runCase` / `main` beside it is its body, not a + # fixture run before it, so its own route is the one to report even when it is a hop longer than the carrier's + for m, (d, via) in list(tests.items()): + if via and g.sym[m]['kind'] == 'module' and (g.sym.get(via) or {}).get('file') == g.sym[m]['file']: + own = [int(x[1]) for x in res['test_hit'] if x[0] == m and not x[2] and int(x[1]) <= depth] + if own: tests[m] = (min(own), None) # the same test, inherited: the subclass runs the base's test method, in its own file inh = {} for s_, m, d in {(x[0], x[1], int(x[2])) for x in res['inherited_test']}: @@ -1990,6 +2012,13 @@ def main(argv): u = g.q(f"SELECT count(*) n FROM unresolved_sites WHERE caller_id IN ({','.join('?' * len(inside))})", *inside)[0]['n'] if inside else 0 # the check that means something: every printed chain hop, and every resolved direct entry, is looked up again in graph.sqlite chains = {m: I.chain(m, parent) for m in tests} + # a MODULE counted as a test is a script test (graph_sql.script_tests): it is labelled with the command it runs by + script_ids = {m for m in tests if g.sym[m]['kind'] == 'module'} + _runs = {} + def script_cmd(m): + f = g.sym[m]['file'] + if f not in _runs: _runs[f] = script_run(g.repo, f) + return _runs[f] # A TEST THAT HOLDS A MOCK OF A TYPE ON ITS ROUTE never runs that type's methods (ax_edges.mocked_types): a web test # with `@MockBean OrderService` reaches the controller over HTTP and stops at the mock, so a route through # OrderService is not a route, whatever its rung. Read off the route the answer prints (the nearest one); the test @@ -2187,7 +2216,8 @@ def main(argv): # whatever the solver happened to emit — which is not the same between backends. 'direct': [{'id': c, 'display': g.disp(c), 'role': role, 'why': why, 'also': direct_also.get((c, _grp(role)), []), 'reasons': direct_reasons.get((c, _grp(role)), []), 'certainty': cert, 'at': loc, 'sites': n, 'for': sorted(direct_for[(c, _grp(role))])} for c, role, why, cert, loc, n in sorted(D, key=lambda x: (CERT[x[3]], x[1], g.disp(x[0]), x[2], x[4], x[0]))], 'reached': [{'id': m, 'display': g.disp(m), 'hops': d, 'for': sorted(reach_from[m]), 'at': g.loc(m), 'test': bool(g.sym[m]['is_test'])} for m, d in sorted(reached.items(), key=lambda x: (x[1], g.disp(x[0]), g.loc(x[0]), x[0]))], - 'tests': [{'id': m, 'display': g.disp(m), 'owner': g.sym[m]['owner'], 'name': g.sym[m]['name'], 'hops': d, 'via': g.disp(fx) if fx else None, 'at': g.loc(m), 'certainty': test_cert.get(m), 'chain': [g.disp(x) for x in chains.get(m, [])]} for m, (d, fx) in sorted(tests.items(), key=lambda kv: (kv[1][0], g.disp(kv[0]), g.loc(kv[0]), kv[0]))], + 'tests': [{'id': m, 'display': g.disp(m), 'owner': g.sym[m]['owner'], 'name': g.sym[m]['name'], 'hops': d, 'via': g.disp(fx) if fx else None, 'at': g.loc(m), 'certainty': test_cert.get(m), 'chain': [g.disp(x) for x in chains.get(m, [])], + **({'script': True, 'run': script_cmd(m)} if m in script_ids else {})} for m, (d, fx) in sorted(tests.items(), key=lambda kv: (kv[1][0], g.disp(kv[0]), g.loc(kv[0]), kv[0]))], 'framework_entries': [{'id': m, 'display': g.disp(m), 'signal': sig, 'at': g.loc(m)} for m, sig in fw_ent], 'framework_grep': fw_grep, 'stub_tests': [{'id': m, 'display': g.disp(m), 'owner': g.sym[m]['owner'], 'name': g.sym[m]['name'], 'at': g.loc(m), 'certainty': 'stubs it', @@ -2525,6 +2555,9 @@ def main(argv): test_rows = sorted(({m for m, _ in ent if _is_t(m)} | {c for c, role, why, cert, loc, n in D if _is_t(c) and cert not in ('alongside', 'stubs it') and not why.startswith('stubs a method of this name')}) - set(tests) - set(stubbed), key=lambda m: (g.disp(m), g.loc(m))) + # a callable in a file whose test is counted runs when that file does (`main` under a script's main guard, the + # describe block around a counted `it`): the count already credits it + test_rows = [m for m in test_rows if g.sym[m]['file'] not in tfiles] if test_rows: print(f" {'and ' if tests else 'but '}{len(test_rows)} callable(s) in test code listed above reach it by a route this count does not credit " "(a runner entry it does not collect, a remote or framework hop) — treat them as tests to run: " @@ -2594,13 +2627,16 @@ def main(argv): # when the file's tests share it, a count when they do not (each one's own is under --why) fxs = {g.disp(tests[m][1]) for m in ms if tests[m][1]} via = (f" via {next(iter(fxs))}" if len(fxs) == 1 else f" via {len(fxs)} fixtures" if fxs else '') + # a script test is a whole file run as a program: the command it runs by, never a framework's + sc = next((m for m in ms if m in script_ids), None) + head = f"{f} (script: {script_cmd(sc) or 'no command found: run the file as its project does'})" if sc else f"{f} ({len(ms)}){via}" if want_why: - print(f" {f} ({len(ms)}){via}:") + print(f" {head}:") for m in shown: r = route(m); print(f" {tname(m)} {tests[m][0]} hop(s)" + (f" {r}" if r else '')) if len(ms) > limit: print(f" … +{len(ms) - limit} more (--limit N)") else: - print(f" {f} ({len(ms)}){via}: " + ', '.join(tname(m) for m in shown) + (f" … +{len(ms) - limit} more (--limit N)" if len(ms) > limit else '')) + print(f" {head}: " + ', '.join(tname(m) for m in shown) + (f" … +{len(ms) - limit} more (--limit N)" if len(ms) > limit else '')) if not want_why: print(" --why adds each test's route (with --tests-in to keep it short)") if want_tests and stubbed: sfiles = collections.defaultdict(list) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-test-impact b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-test-impact index 07b2bc1c2..176289684 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-test-impact +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-test-impact @@ -60,6 +60,7 @@ TESTY = re.compile(r'(^|[/_.-])(test|tests|spec|specs|__tests__|it)([/_.-]|$)', HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) +import graph_sql def run(script, args): @@ -1429,6 +1430,10 @@ def js_plan(repo, files): cmd = _script_for(repo, pkg, f[len(pkg) + 1:] if pkg else f) if cmd: singles.append((pkg, cmd)) + elif graph_sql.script_test_file(f, t := _read(os.path.join(repo, f), 400_000)) and graph_sql.script_main_guard(f, t): + # a script test no package script or usage line names runs under its interpreter only when it guards its + # own entry; an unguarded one is named as run by nothing (#1570), never a guessed framework or interpreter + singles.append(('', f"{'npx tsx' if f.endswith(('.ts', '.mts', '.cts', '.tsx')) else 'node'} {shlex.quote(f)}")) else: unrun.append(f) def cd(pkg, cmd): @@ -1441,6 +1446,12 @@ def js_plan(repo, files): return cmds, unrun +def script_command(repo, rel): + """the command a script test is run by, the one this command prints for it (impact labels the test with it)""" + plan = py_plan if rel.endswith('.py') else js_plan + return "\n".join(plan(repo, [rel])[0]) or None + + def print_unrun(lang, repo, files): """Say which selected files no runner collects, so a reader does not take the command as the whole run.""" if lang == 'python': @@ -1826,7 +1837,7 @@ def main(argv): # A row the answer named that carries no owner still has to reach the reader. Dropping it for being # malformed is how a selection gets quietly shorter, which is the one failure this command exists to # prevent — the same shape as a dropped edge or an answer that could not be parsed. - ownerless = [r for r in tests.values() if not r.get('owner') and r.get('at')] + ownerless = [r for r in tests.values() if not r.get('owner') and r.get('at') and not r.get('script')] # a script is its file run_files = sorted(set(files) | set(edited)) lang = (changed.get('lang') or '').lower() or _guess_lang(run_files or changed_files) # the graph `impact` answered from, asked again for which selected classes are abstract (command_for) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py b/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py index d0c59fde7..3f51c1f07 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py @@ -549,6 +549,54 @@ def parent_up(edges, depth): EACH_TABLE = re.compile(r'\b(it|test|bench|describe)\s*\.\s*each\b') +# A SCRIPT TEST: a file under the test tree that calls no test framework and is run as a program — `test/run.js` +# calling `runCase(...)` at top level, a `tests/check.py` with its own main guard. Its module IS the test: impact walked +# to it and labelled it [test], then counted "0 of 0", and test-impact said no test reaches the change. A file that +# declares a framework test is its runner's; one that exports (a helper), sits in a fixture or support directory, or +# is a runner's setup or config file is not run on its own. Nor is test DATA: the inputs a test reads (a case +# directory, a fixture project with its own src/, a corpus) and the tooling beside the tests (tools/, bin/, a harness) +# lie under a test tree too, and on one repository they were two thirds of what the rule matched before it said so. +PY_SCRIPT_MAIN = re.compile(r'^if\s+__name__\s*==\s*[\'"]__main__[\'"]\s*:', re.M) +JS_EXPORTS = re.compile(r'^\s*(?:export\s|module\.exports\b|exports\.[\w$]+\s*=)', re.M) +JS_FRAMEWORK = re.compile(r'(? { + it('adds a bang', () => { + expect(shout(sample())).toBe('HI!'); + }); +}); diff --git a/tests/cases/python/script-runner-is-a-test/app/__init__.py b/tests/cases/python/script-runner-is-a-test/app/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/cases/python/script-runner-is-a-test/app/pricing.py b/tests/cases/python/script-runner-is-a-test/app/pricing.py new file mode 100644 index 000000000..984987ebb --- /dev/null +++ b/tests/cases/python/script-runner-is-a-test/app/pricing.py @@ -0,0 +1,6 @@ +def price(amount): + return round(amount * 1.2, 2) + + +def discount(amount): + return round(price(amount) * 0.9, 2) diff --git a/tests/cases/python/script-runner-is-a-test/case.json b/tests/cases/python/script-runner-is-a-test/case.json new file mode 100644 index 000000000..9902e9c59 --- /dev/null +++ b/tests/cases/python/script-runner-is-a-test/case.json @@ -0,0 +1,17 @@ +{"lang": "python", "src": ".", + "checks": [ + {"why": "a runner under the test tree with its own main guard and no test framework is a test: impact counts it and names the interpreter that runs it", + "run": ["impact", "price", "--tests"], + "want": ["check_prices.py (script: python tests/check_prices.py)", "test_discount.py"], + "avoid": ["server.py (script", "0 of"]}, + {"why": "the same runner counts in the summary line, not only in the listing", + "run": ["impact", "price"], + "want": ["tests: 2 of 2 test method(s)"]}, + {"why": "CONTROL: a pytest test keeps its selection, and a runner that does not reach the change is not listed", + "run": ["impact", "discount", "--tests"], + "want": ["test_discount.py"], + "avoid": ["check_prices"]}, + {"why": "the script test carries its run command in --json, which test-impact builds its command from", + "run": ["impact", "price", "--tests", "--json"], + "stdout_json": true, + "want": ["\"script\": true", "\"run\": \"python tests/check_prices.py\""]}]} diff --git a/tests/cases/python/script-runner-is-a-test/tests/check_prices.py b/tests/cases/python/script-runner-is-a-test/tests/check_prices.py new file mode 100644 index 000000000..57a32d9e7 --- /dev/null +++ b/tests/cases/python/script-runner-is-a-test/tests/check_prices.py @@ -0,0 +1,10 @@ +from app.pricing import price + + +def main(): + assert price(10) == 12.0 + print("ok") + + +if __name__ == '__main__': + main() diff --git a/tests/cases/python/script-runner-is-a-test/tests/fixtures/server.py b/tests/cases/python/script-runner-is-a-test/tests/fixtures/server.py new file mode 100644 index 000000000..3476bbe25 --- /dev/null +++ b/tests/cases/python/script-runner-is-a-test/tests/fixtures/server.py @@ -0,0 +1,3 @@ +from app.pricing import price + +PRICE = price(1) diff --git a/tests/cases/python/script-runner-is-a-test/tests/test_discount.py b/tests/cases/python/script-runner-is-a-test/tests/test_discount.py new file mode 100644 index 000000000..9f5f042e9 --- /dev/null +++ b/tests/cases/python/script-runner-is-a-test/tests/test_discount.py @@ -0,0 +1,5 @@ +from app.pricing import discount + + +def test_discount(): + assert discount(10) == 10.8 diff --git a/tests/script_tests.py b/tests/script_tests.py new file mode 100644 index 000000000..eb60a43c1 --- /dev/null +++ b/tests/script_tests.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""tests/script_tests.py — test-impact selects a script-style runner, with the command the project runs it by (T2-1). + +A runner under the test tree that calls no test framework — `test/run.js` calling `runCase(...)` at top level, a +`tests/check.py` with its own main guard — was walked to and labelled [test] by impact, then counted "0 of 0", and +test-impact said no test reaches the change. Each run copies the golden fixture into a throwaway git repository, +indexes it, edits the changed function's body and asks test-impact what to run. The script is selected with its own +command (the package script that names it, or the interpreter) and a framework test beside it keeps its command. + + python3 tests/script_tests.py +""" +import os, shutil, subprocess, sys, tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +AX = os.path.join(ROOT, 'bin', 'axiomcode') +CASES = os.path.join(ROOT, 'tests', 'cases') + + +def sh(cwd, *cmd, env=None): + return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + + +def main(): + fails = [] + def check(ok, why, detail=''): + print(('ok ' if ok else 'FAIL ') + why + ('' if ok else '\n ' + detail.strip().replace('\n', '\n '))) + if not ok: fails.append(why) + + env = dict(os.environ, AXIOMCODE_ENGINE=ROOT, AXIOMCODE_NO_REFRESH='1') + # (language, fixture, --src, file to edit, old text, new text, wanted lines, unwanted lines) + runs = [ + ('javascript', 'script-runner-is-a-test', 'src', 'src/lib/text.js', 'trim().toLowerCase()', 'trim().toLowerCase().normalize()', + ['src/test/run.js', '(cd src && npm run test)', 'node src/test/smoke.js', '(cd src && npx vitest run test/text.test.js)'], + ['no test in the graph reaches', 'src/test/helpers.js']), + ('python', 'script-runner-is-a-test', '.', 'app/pricing.py', 'amount * 1.2', 'amount * 1.25', + ['tests/check_prices.py', 'python tests/check_prices.py', 'pytest tests/test_discount.py'], + ['no test in the graph reaches', 'tests/fixtures/server.py', 'pytest tests/check_prices.py']), + ] + for lang, case, src, rel, old, new, want, avoid in runs: + work = tempfile.mkdtemp(prefix='axiomcode-script-tests-') + try: + repo = os.path.join(work, 'repo') + shutil.copytree(os.path.join(CASES, lang, case), repo, ignore=shutil.ignore_patterns('.axiomcode', 'case.json')) + for cmd in (('git', 'init', '-q'), ('git', 'add', '-A'), + ('git', '-c', 'user.email=t@t', '-c', 'user.name=t', 'commit', '-qm', 'base')): + sh(repo, *cmd) + built = sh(repo, AX, 'index', '.', '--lang', lang, '--src', src, env=env) + check(built.returncode == 0, f'{lang}: the fixture indexes', built.stdout + built.stderr) + f = os.path.join(repo, rel); text = open(f).read() + open(f, 'w').write(text.replace(old, new)) + r = sh(repo, AX, 'test-impact', '.', env=env) + out = r.stdout + r.stderr + check(r.returncode == 0 and 'Traceback' not in out, f'{lang}: test-impact answers', out) + for w in want: check(w in out, f'{lang}: test-impact names {w!r}', out) + for a in avoid: check(a not in out, f'{lang}: test-impact does not name {a!r}', out) + finally: + shutil.rmtree(work, ignore_errors=True) + print(f"\n{'ok' if not fails else f'{len(fails)} FAILED'}") + return 1 if fails else 0 + + +if __name__ == '__main__': + sys.exit(main())