From 0548acdfc1de49f747b0546ccf7e0d194e419d7e Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 30 Sep 2026 05:06:07 -0700 Subject: [PATCH 1/2] context/impact: lockfiles and manifest package lists are never text leads; --in narrows text files A context question with a common word (debug, optional, string) listed package-lock.json and package.json lines as "text files that name these declarations" and made one of them the next step, even when --in named a directory holding neither file. - ax_nonsource: lockfiles of every package manager (npm, pnpm, yarn, NuGet, Gradle, Poetry, uv, Composer, Cargo, Go) are out of scope; a manifest's metadata and dependency lists (package.json, pyproject.toml, PackageReference) and package.json keys are skipped by line. Script commands and tool configuration values are still searched. - context: the caller's --in narrows text bindings as it narrows declarations; a mapper XML whose namespace names the declaring type stays (bound by the declaration, not the word). - ax_pages: a --source answer with no flow and no text row now ends with a next step. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../case.json | 18 ++++++++++++++++++ .../config/app.json | 3 +++ .../package.json | 15 +++++++++++++++ .../packages.lock.json | 8 ++++++++ .../src/a/app.js | 14 ++++++++++++++ .../src/a/routes.json | 3 +++ .../src/b/worker.js | 3 +++ 7 files changed, 64 insertions(+) create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/case.json create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/config/app.json create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/package.json create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/packages.lock.json create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/src/a/app.js create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/src/a/routes.json create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/src/b/worker.js diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/case.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/case.json new file mode 100644 index 00000000..3f88d68a --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/case.json @@ -0,0 +1,18 @@ +{"lang": "javascript", "src": "src", + "checks": [ + {"why": "--in narrows the text files a context answer lists as it narrows the declarations: a JSON under the scope that names a found function is listed, a lockfile, a manifest and a config file outside it are not, and the next step stays inside the scope", + "run": ["context", "app wiring: createApp, config, debug output", "--in", "src/a"], + "want": ["text files that name these declarations", "src/a/routes.json:2", "next: read src/a/"], + "avoid": ["package-lock.json", "packages.lock.json", "package.json:", "config/app.json"]}, + {"why": "--source prints each file's declarations as code instead of a name list; the answer still ends with a next step, on the first file shown", + "run": ["context", "app wiring: createApp and its output", "--in", "src/a", "--source"], + "want": ["next: answer from the code of src/a/app.js shown first above"], + "avoid": ["package-lock.json", "package.json:"]}, + {"why": "without --in, a lockfile is never a text lead and a manifest's description, dependency lists and script names are not either; a manifest script command and a config file that name the function are (control: the manifest is filtered by line, not dropped)", + "run": ["context", "app wiring: createApp, config, debug output"], + "want": ["src/a/routes.json:2", "config/app.json:2", "package.json:6"], + "avoid": ["package-lock.json", "packages.lock.json", "package.json:4", "package.json:7", "package.json:10", "package.json:13"]}, + {"why": "impact lists the same text files as bound from outside the source: the lockfiles and the manifest's dependency lists are not among them", + "run": ["impact", "debug"], + "want": ["src/a/routes.json", "config/app.json"], + "avoid": ["package-lock.json", "packages.lock.json", "package.json:7", "package.json:10"]}]} diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/config/app.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/config/app.json new file mode 100644 index 00000000..6741e398 --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/config/app.json @@ -0,0 +1,3 @@ +{ + "startup": "debug" +} diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/package.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/package.json new file mode 100644 index 00000000..62b94574 --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/package.json @@ -0,0 +1,15 @@ +{ + "name": "wiring-sample", + "version": "1.0.0", + "description": "Sample app: turns debug (verbose) output on from config", + "scripts": { + "trace": "debug", + "debug": "node src/a/app.js" + }, + "dependencies": { + "debug": "^4.1.0" + }, + "optionalDependencies": { + "optional": "^0.1.4" + } +} diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/packages.lock.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/packages.lock.json new file mode 100644 index 00000000..c7a71061 --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/packages.lock.json @@ -0,0 +1,8 @@ +{ + "version": 1, + "dependencies": { + "net8.0": { + "optional": { "type": "Direct", "requested": "[1.0.0, )" } + } + } +} diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/src/a/app.js b/tests/cases/javascript/context-text-leads-skip-lockfiles/src/a/app.js new file mode 100644 index 00000000..6662599a --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/src/a/app.js @@ -0,0 +1,14 @@ +import { readFileSync } from 'node:fs'; + +export function debug(config) { + return config.verbose === true; +} + +export function optional(value, fallback) { + return value === undefined ? fallback : value; +} + +export function createApp(config) { + const routes = JSON.parse(readFileSync(new URL('./routes.json', import.meta.url), 'utf8')); + return { routes, verbose: debug(config), port: optional(config.port, 8080) }; +} diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/src/a/routes.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/src/a/routes.json new file mode 100644 index 00000000..67dbe729 --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/src/a/routes.json @@ -0,0 +1,3 @@ +{ + "/health": { "handler": "debug" } +} diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/src/b/worker.js b/tests/cases/javascript/context-text-leads-skip-lockfiles/src/b/worker.js new file mode 100644 index 00000000..856ee0d5 --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/src/b/worker.js @@ -0,0 +1,3 @@ +export function startWorker(config) { + return { queue: config.queue }; +} From 5e2290d800a6aa5ac9e8438c0b94482161bd758c Mon Sep 17 00:00:00 2001 From: swapnil <78632212+swapnilpaliwal-sd@users.noreply.github.com> Date: Wed, 30 Sep 2026 11:59:05 -0700 Subject: [PATCH 2/2] context/impact: manifest package lists are never text leads, in any manifest; --in text dirs stay in scope The previous commit on this branch added the case but not the code it checks, and its npm lockfile fixture was gitignored, so CI ran the checks against the old scripts. - ax_nonsource: one manifest filter decided by where a word sits (entry, table, block, element), not by line: package.json, bower.json, deno.json, composer.json by JSON entry (a one-line manifest keeps its scripts); pnpm-workspace.yaml and conda environment.yml by block; pyproject.toml and Pipfile by table; requirements files whole; pom.xml, MSBuild, packages.config and .nuspec by package element. More lockfiles recognised. - context: the word-matched rows of a text-only --in use the same filter; a directory the main graph holds is no longer text to another language's graph; with several graphs, a text-only --in is answered once and never by a package outside it. Co-authored-by: axiomcode-bot[bot] <334110751+axiomcode-bot[bot]@users.noreply.github.com> --- .../skills/axiomcode/scripts/ax_nonsource.py | 143 +++++++++++++++++- .../skills/axiomcode/scripts/ax_pages.py | 13 +- .../axiomcode/scripts/axiomcode-context | 63 +++++--- .../context-answers-what-was-asked/case.json | 4 + .../bower.json | 1 + .../case.json | 12 +- .../deno.json | 4 + .../package-lock.json | 19 +++ .../packages/min/package.json | 1 + .../pnpm-workspace.yaml | 2 + .../requirements.txt | 1 + .../tools/lint/package.json | 6 + .../tools/lint/rules.json | 3 + tests/multi_language.py | 13 +- 14 files changed, 257 insertions(+), 28 deletions(-) create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/bower.json create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/deno.json create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/package-lock.json create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/packages/min/package.json create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/pnpm-workspace.yaml create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/requirements.txt create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/tools/lint/package.json create mode 100644 tests/cases/javascript/context-text-leads-skip-lockfiles/tools/lint/rules.json diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_nonsource.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_nonsource.py index 10b1e929..2e127ec7 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_nonsource.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_nonsource.py @@ -65,16 +65,151 @@ def _classify(fp): # callable: called (`note(`), quoted as a value (`"note"`, `'note'`), or qualified with `#`, `::` or `->` # (`Owner#note`; a CSS `#note` selector is not one). A dotted `Owner.note` is matched as the qualified name itself and never reaches this rule. The # rest are PROSE: counted and grep-able, not listed as places a rename breaks. -OUT_OF_SCOPE_EXT = {'.sh', '.bash', '.zsh', '.ksh', '.dl'} +# +# LOCKFILES AND MANIFEST LISTS. A lockfile is written by a package manager, never by hand, and every word in it is a +# package name, a version or a flag: `"optional": true`, `"debug": "^4.1.0"`. A manifest's metadata (name, +# description, keywords) and its dependency lists are the same: they name packages, not callables. A method named +# `debug`, `optional` or `string` matched there is a package or a word, never a binding, and in a JavaScript +# repository it outnumbered every real one (a context question's next step became a line of package-lock.json). +# The rest of a manifest (scripts, tasks, tool configuration) is kept: that is where a name can be referred to. A +# JSON manifest's KEYS are not: `"start":` names an npm script and `"testEnvironment":` an option, so only the values +# are searched. What is skipped is decided by where the word sits in the file's structure (the entry, table, block or +# element that holds it), not by the line alone: a one-line manifest holds its scripts and its dependencies together. +OUT_OF_SCOPE_EXT = {'.sh', '.bash', '.zsh', '.ksh', '.dl', '.lock', '.lockfile'} SHELL_SHEBANG = re.compile(r'#!\s*\S*(?:/|\s)(?:env\s+)?(?:ba|z|k|da)?sh\b') +LOCKFILES = {'package-lock.json', 'npm-shrinkwrap.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lock', 'deno.lock', + 'packages.lock.json', 'project.assets.json', 'paket.lock', 'gradle.lockfile', 'poetry.lock', 'uv.lock', + 'pipfile.lock', 'pdm.lock', 'conda-lock.yml', 'composer.lock', 'gemfile.lock', 'cargo.lock', 'go.sum'} +# JSON manifests: the top-level keys that describe the package or list other packages +_JSON_META = r'name|version|description|keywords|authors?|contributors|maintainers|license|homepage|repository|bugs|funding|private' +JSON_LIST_KEY = { + 'package.json': re.compile(r'(?i)^(' + _JSON_META + r'|type|engines|os|cpu|publishConfig|workspaces|packageManager|' + r'overrides|resolutions|pnpm|\w*dependencies(Meta)?)$'), + 'bower.json': re.compile(r'(?i)^(' + _JSON_META + r'|ignore|resolutions|\w*dependencies)$'), + 'deno.json': re.compile(r'^(name|version|imports|scopes|importMap|lock|nodeModulesDir|vendor|workspace|patch|links)$'), + 'composer.json': re.compile(r'(?i)^(' + _JSON_META + r'|type|support|require(-dev)?|conflict|replace|provide|suggest|' + r'repositories|minimum-stability|prefer-stable)$')} +JSON_LIST_KEY['deno.jsonc'] = JSON_LIST_KEY['deno.json'] +# YAML manifests: the top-level blocks that list packages (a conda environment, a pnpm workspace and its catalog) +YAML_LIST_KEY = { + 'environment.yml': re.compile(r'^(name|channels|dependencies|prefix)$'), + 'pnpm-workspace.yaml': re.compile(r'^(packages|catalogs?|overrides|patchedDependencies|\w*BuiltDependencies|' + r'peerDependencyRules|allowedDeprecatedVersions|packageExtensions)$')} +YAML_LIST_KEY['environment.yaml'] = YAML_LIST_KEY['environment.yml'] +# TOML manifests: the tables and keys that list packages +PY_DEP_TABLE = re.compile(r'^\[\s*(dependency-groups|project\.optional-dependencies|tool\.poetry(\.group\.[^\]]+)?\.(dev-)?dependencies|' + r'tool\.pdm\.dev-dependencies|tool\.uv)\s*\]') +PIPFILE_TABLE = re.compile(r'^\[\s*(packages|dev-packages|requires|source|[\w-]+-packages)\s*\]') +PY_DEP_KEY = re.compile(r'^\s*(dependencies|requires|dev-dependencies|optional-dependencies)\s*=') +# a file that is nothing but a list of packages +REQUIREMENTS = re.compile(r'^(requirements|constraints)[\w.-]*\.(txt|in)$') +# XML manifests: the elements that name a package (MSBuild, packages.config, .nuspec) and a POM's dependency blocks +XML_PKG_LINE = re.compile(r'<\s*(PackageReference|PackageVersion|package|dependency)\b[^>]*\b(Include|Update|id)\s*=') +POM_BLOCK = re.compile(r'<(/?)(dependencies|dependencyManagement|parent|exclusions)>') +POM_COORD = re.compile(r'^\s*<(groupId|artifactId|version|packaging|name|description|url|scope|type|classifier|optional|' + r'modelVersion|id|tags|authors|owners)>[^<]*\s*$') + + +def is_lockfile(rel): + """a file a package manager writes: every word in it is a package, a version or a flag""" + return os.path.basename(rel).lower() in LOCKFILES + + +def _json_spans(text, list_key): + """{line: [(start col, end col)]} of the strings a word is not matched in: each string under a top-level key that + `list_key` matches, and each key at any depth. A string-aware scan, so a minified manifest is split by entry too.""" + out, depth, want_key, listed, line, bol, i, n = {}, 0, False, False, 1, 0, 0, len(text) + while i < n: + c = text[i] + if c == '"': + j = i + 1 + while j < n and text[j] != '"': j += 2 if text[j] == '\\' else 1 + k = j + 1 + while k < n and text[k] in ' \t\r\n': k += 1 + if depth == 1 and want_key: want_key, listed = False, bool(list_key.match(text[i + 1:j])) + if (k < n and text[k] == ":") or (depth >= 1 and listed): + out.setdefault(line, []).append((i - bol, j + 1 - bol)) + nl = text.count('\n', i, j) + if nl: line += nl; bol = text.rindex('\n', i, j) + 1 + i = j + 1 + continue + if text.startswith('//', i): # a JSONC comment (deno.jsonc) + j = text.find('\n', i); i = n if j < 0 else j + continue + if c in '{[': + depth += 1 + if depth == 1: want_key = c == '{' + elif c in '}]': depth -= 1 + elif c == ',' and depth == 1: want_key = True + elif c == '\n': line += 1; bol = i + 1 + i += 1 + return out + + +def _unquoted(s): + """a TOML line without its strings and its comment: what is left are the brackets that open and close a list""" + return re.sub(r'"(?:\\.|[^"\\])*"|\'[^\']*\'', '""', s).split('#', 1)[0] + + +def _toml_lines(text, table, key=None): + """the lines inside a table `table` matches, and those of a `key = [ ... ]` list outside one""" + out, in_table, open_brackets = set(), False, 0 + for i, ln in enumerate(text.split('\n'), 1): + if open_brackets > 0: # inside a `dependencies = [ ... ]` spread over lines + out.add(i); s = _unquoted(ln); open_brackets += s.count('[') - s.count(']'); continue + if ln.lstrip().startswith('['): in_table = bool(table.match(ln.strip())); continue + if in_table: out.add(i); continue + if key and key.match(ln): + out.add(i); s = _unquoted(ln); open_brackets = s.count('[') - s.count(']') + return out + + +def _yaml_lines(text, block): + """the lines of the top-level YAML blocks `block` matches: the key's line and every indented or list line under it""" + out, inside = set(), False + for i, ln in enumerate(text.split('\n'), 1): + m = re.match(r'([\w.-]+)\s*:', ln) + if m: inside = bool(block.match(m.group(1))) + elif ln[:1] not in ('', ' ', '\t', '-', '#'): inside = False + if inside: out.add(i) + return out + + +def _xml_lines(base, text): + """the lines of an XML manifest that name a package: a POM's dependency blocks and coordinates, a NuGet element""" + lines, out, depth = text.split('\n'), set(), 0 + for i, ln in enumerate(lines, 1): + if base == 'pom.xml': + opened = depth > 0 + for m in POM_BLOCK.finditer(ln): depth += -1 if m.group(1) else 1 + if opened or depth > 0 or POM_COORD.match(ln): out.add(i) + elif XML_PKG_LINE.search(ln) or (base.endswith('.nuspec') and POM_COORD.match(ln)): out.add(i) + return out + + +def manifest_skip(rel, text): + """-> skip(line, col): True where a word written there sits in a lockfile, a manifest's metadata or one of its + dependency lists (see LOCKFILES AND MANIFEST LISTS); None for any other file""" + base = os.path.basename(rel).lower() + if is_lockfile(rel) or REQUIREMENTS.match(base): return lambda _l, _c: True + if base in JSON_LIST_KEY: + spans = _json_spans(text, JSON_LIST_KEY[base]) + return lambda l, c: any(a <= c < b for a, b in spans.get(l, ())) + if base in YAML_LIST_KEY: lines = _yaml_lines(text, YAML_LIST_KEY[base]) + elif base == 'pyproject.toml': lines = _toml_lines(text, PY_DEP_TABLE, PY_DEP_KEY) + elif base == 'pipfile': lines = _toml_lines(text, PIPFILE_TABLE) + elif base in ('pom.xml', 'packages.config') or base.endswith(('.csproj', '.fsproj', '.vbproj', '.props', '.targets', '.nuspec')): + lines = _xml_lines(base, text) + else: return None + return lambda l, _c: l in lines COMMON = re.compile(r'[a-z]+') _QUOTES = '"\'`' QUALIFIER = re.compile(r'[\w$)\]>](?:#|::|->)$') # `Owner#note`, `Owner::note`, `$obj->note`; not a CSS `#note` selector def out_of_scope(rel, text): - """a shell script or a Datalog file: a name matched there is never a binding (owner's rule)""" - if os.path.splitext(rel)[1].lower() in OUT_OF_SCOPE_EXT: return True + """a shell script, a Datalog file or a lockfile: a name matched there is never a binding (owner's rule)""" + if os.path.splitext(rel)[1].lower() in OUT_OF_SCOPE_EXT or is_lockfile(rel): return True return not os.path.splitext(rel)[1] and bool(SHELL_SHEBANG.match(text[:120])) @@ -211,9 +346,11 @@ def hits(self, names): try: text = open(os.path.join(self.repo, rel), errors='replace').read() except OSError: continue if not any(n in text for n in names) or out_of_scope(rel, text): continue + skip = manifest_skip(rel, text) for i, line in enumerate(text.split('\n'), 1): shaped = {} for m in pat.finditer(line): + if skip and skip(i, m.start(1)): continue n = m.group(1); hits.append((n, rel, i)) shaped[n] = shaped.get(n, False) or not is_common(n) or code_shaped(line, m.start(1), m.end(1)) self.prose.update((n, rel, i) for n, ok in shaped.items() if not ok) diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_pages.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_pages.py index 94915f7f..28aabf2e 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_pages.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_pages.py @@ -300,10 +300,17 @@ def next_context(text): "step's BODY, not only the line shown, since the body is the explanation and the flow is only its spine " "(`--source` prints it)." + gap) m = re.search(r'^\s+(?:hop \d+|name only, no call path)\s+(\S+)\s+\(\d+ symbol\(s\)\)[^\n]*\n\s+-> ([^\n]+)', text, re.M) + if m: + f = m.group(1); syms = [x.strip() for x in m.group(2).split(',') if x.strip()][:2] + return (f"next: read {f} first — it holds {' and '.join(syms)}; then `impact ` for what a change " + "to it reaches. The other files are ranked context, not a reading list") + # --source prints each file's declarations as code (`name (file:line)` and its lines) instead of the `->` list + m = re.search(r'^\s+(?:hop \d+|name only, no call path)\s+(\S+)\s+\(\d+ symbol\(s\)\)[^\n]*\n((?:\s+\S+ \(\S+:\d+\)\n(?:\s+(?:\d+|) \| [^\n]*\n)*)+)', + text, re.M) if not m: return '' - f = m.group(1); syms = [x.strip() for x in m.group(2).split(',') if x.strip()][:2] - return (f"next: read {f} first — it holds {' and '.join(syms)}; then `impact ` for what a change " - "to it reaches. The other files are ranked context, not a reading list") + syms = re.findall(r'^\s+(\S+) \(\S+:\d+\)$', m.group(2), re.M)[:2] + return (f"next: answer from the code of {m.group(1)} shown first above — {' and '.join(syms)}; then `impact ` for what a change to it reaches. The other files are ranked context, not a reading list") def next_changed(text): if re.search(r'^(no change|no git base)', text, re.M): return '' diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-context b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-context index d7077e55..9bd4561d 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-context +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-context @@ -482,11 +482,12 @@ def graph_langs(g): def indexed_files(g, langs): - """every file some graph of this repository holds""" + """every file some graph of this repository holds. The main graph's dir is '' in `langs` (the default the verbs + read); asked from another language's graph, it is read from its place, or a directory only it holds reads as text""" import sqlite3 held = {(sy.get('file') or '') for sy in g.all_sym.values()} for l, d in langs.items(): - if not d: continue + if not d: d = os.path.join(g.repo, '.axiomcode') try: con = sqlite3.connect(f"file:{os.path.join(d, 'out', 'graph.sqlite')}?mode=ro", uri=True) held |= {r[0] for r in con.execute("SELECT DISTINCT file FROM symbols")} @@ -578,23 +579,39 @@ def nonsource(g): def text_term_hits(g, dirs, terms, cap=200): - """[(file, first line, [terms])] -- files under `dirs` whose text uses the task's words, most words first""" + """[(file, first line, [terms])] -- files under `dirs` whose text uses the task's words, most words first. A word + in a lockfile or in a manifest's package lists is never one (ax_nonsource.py: it names a package, not the task).""" + import ax_nonsource pats = {t: re.compile(r'(?i)(?= 3} out = [] for d in dirs: for root, _ds, fs in os.walk(os.path.join(g.repo, d)): for fn in fs: rel = os.path.relpath(os.path.join(root, fn), g.repo).replace(os.sep, '/') + if ax_nonsource.is_lockfile(rel): continue ln = source_lines(g.repo, rel) + skip =ax_nonsource.manifest_skip(rel, '\n'.join(ln)) or (lambda _l, _c: False) got, first = [], 0 for t, pt in pats.items(): - k = next((i for i, x in enumerate(ln, 1) if pt.search(x)), 0) + k = next((i for i, x in enumerate(ln, 1) for m in pt.finditer(x) if not skip(i, m.start())), 0) if k: got.append(t); first = min(first or k, k) if got: out.append((rel, first, got)) if len(out) >= cap: break return sorted(out, key=lambda r: (-len(r[2]), r[0])) +def print_text_hits(g, dirs, terms): + """print the text files under `dirs` that use the task's words; -> how many there are""" + hits = text_term_hits(g, dirs, terms) + if hits: + print(f"\ntext files under {', '.join(d + '/' for d in dirs)} that use the task's words (not source; " + "matched by word, not by any declaration):") + for f, l, ts in hits[:BINDINGS_SHOWN]: + print(f" {f + ':' + str(l):56} {', '.join(ts)}") + RESULT['text_bindings'] = [{'file': f, 'line': l, 'terms': ts} for f, l, ts in hits] + return len(hits) + + ASK_EXT = [(re.compile(r'\b(sql|quer(y|ies)|statements?|mapper|mappings?|migrations?|schema)\b', re.I), ('.xml', '.sql')), (re.compile(r'\b(config(uration|ured)?|propert(y|ies)|settings?|ya?ml|toml|ini|env)\b', re.I), ('.properties', '.yml', '.yaml', '.toml', '.ini', '.conf', '.cfg', '.env')), @@ -607,10 +624,12 @@ def asked_exts(text): return tuple(e for rx, es in ASK_EXT if rx.search(text or '') for e in es) -def text_bindings(g, sids, under=(), want_ext=()): +def text_bindings(g, sids, under=(), want_ext=(), scope=None): """[(file, line, declaration, name written there)] -- a non-source file that writes the name of one of `sids` as a whole word. A mapper XML with a namespace binds only the type that namespace names (a statement id repeated - in another mapper's XML is not this method's SQL).""" + in another mapper's XML is not this method's SQL). `scope` (the caller's --in) keeps only the files under it, as + it keeps only the declarations under it, except a mapper whose namespace names the type: that binding is proved + by the declaration, not by the word, wherever the file sits.""" want = collections.defaultdict(set) # name -> {sid} decl_count = collections.Counter(sy.get('name') for sy in g.sym.values() if not (sy.get('file') or '').startswith('<')) for sid in sids: @@ -642,6 +661,7 @@ def text_bindings(g, sids, under=(), want_ext=()): if nm in common and not ns: continue elif nm in common: continue else: cands = sorted(want[nm]) + if scope and not scope(f) and not (f.endswith('.xml') and ns_of.get(f)): continue key = (f, nm) if key in seen: continue seen.add(key) @@ -653,7 +673,7 @@ def text_bindings(g, sids, under=(), want_ext=()): return [r for _k, r in sorted(out)] -def asked_bound(g, scored, text, under=(), cap=300): +def asked_bound(g, scored, text, under=(), cap=300, scope=None): """{symbol id} among the multi-term matches that a text file of the kind the question asks about binds by name. Empty unless the question asks about such a file (SQL, configuration, a template): only then is the tree walked, and only then does it break a tie between equally scored entry points (pick_seeds).""" @@ -662,7 +682,7 @@ def asked_bound(g, scored, text, under=(), cap=300): cands = sorted((sid for sid, (_sc, m) in scored.items() if len(m) >= 2), key=lambda s: -scored[s][0])[:cap] by_disp = collections.defaultdict(set) for sid in cands: by_disp[g.disp(sid)].add(sid) - return frozenset(sid for f, _l, d, _n in text_bindings(g, cands, under=under, want_ext=kinds) + return frozenset(sid for f, _l, d, _n in text_bindings(g, cands, under=under, want_ext=kinds, scope=scope) if f.endswith(kinds) for sid in by_disp.get(d, ())) @@ -1199,8 +1219,14 @@ def main(argv): RESULT['not_indexed'] = list(notice) for ln in notice: print(ln) if notice: print() + text_only_in = False if text_dirs: + asked_in = bool(scopes) scopes = [x for x in scopes if x.strip('/').lstrip('./') not in text_dirs] + text_only_in = asked_in and not scopes and not scope_offered + # every --in names text no graph holds: one graph lists it (the one that says so, above), and another has + # nothing inside the caller's scope to add — its only package is outside it and would become the next step + if fan and text_only_in and not notice: return 3 # AN --in NO GRAPH HOLDS AND NO DIRECTORY IS was refused with a menu, once per language graph, and no answer: the # agent then searched by hand. A typo is not a question about nothing; the task is answered at the root (or under # the scopes that are real) and the line says what was dropped and what is close. A scope another language's graph @@ -1288,12 +1314,21 @@ def main(argv): # is naming a change that spans them, and intersecting those two paths answers nothing (#1029) specs = [scope_spec(sc, g.repo) for sc in scopes] in_scope = lambda f: any(under_scope(f, sp) for sp in specs) + # the text files are narrowed by the CALLER's --in only: the sole package this graph indexes is not a statement + # about where its SQL or configuration lives, and a guessed scope narrows nothing + fixed_specs = [scope_spec(sc, g.repo) for sc in fixed] + text_scope = (lambda f: any(under_scope(f, sp) for sp in fixed_specs)) if fixed_specs else None if scopes and not scope_offered: scored = {sid: v for sid, v in scored.items() if in_scope(g.sym.get(sid, {}).get('file'))} # what the question NAMES comes before what its words match: a declaration spelled out, a route quoted named = named_declarations(g, body) + route_seeds(g, body) if scopes and not scope_offered: named = [(sid, why) for sid, why in named if in_scope(g.sym.get(sid, {}).get('file'))] + if not scored and not named and not have_from and text_only_in: + # the caller's --in holds only text: its files that use the task's words are the answer, and the package this + # graph put in its place holding none of them is no reason to refuse it + if not print_text_hits(g, text_dirs, terms): print(f"no file under {', '.join(d + '/' for d in text_dirs)} uses the task's words") + return 0 if not scored and not named and not have_from: # the scope is real (require_scope proved it) but holds nothing matching — offer what is under it, # ranked the same way the scope-less menu is: by where the words land, never by how big a directory is @@ -1307,7 +1342,7 @@ def main(argv): # themselves, and on Python every class has one, so they can be a third of this list while # naming nowhere to go. scored = {sid: v for sid, v in scored.items() if not is_synthetic(g.sym.get(sid, {}).get('name'))} - seeds = pick_seeds(g, scored, terms, seeds_wanted, named, prefer=asked_bound(g, scored, body, text_dirs)) + seeds = pick_seeds(g, scored, terms, seeds_wanted, named, prefer=asked_bound(g, scored, body, text_dirs, scope=text_scope)) RESULT['terms'] = list(terms) RESULT['scope'], RESULT['scope_offered'] = (scopes[0] if scopes else None), bool(scope_offered) @@ -1525,7 +1560,7 @@ def main(argv): depth[x] = d + 1 near = list(seed_ids) + sorted(ring, key=lambda x: (not bodiless(g, x), depth[x], g.disp(x) or '', x))[:300] kinds = asked_exts(body) - binds = text_bindings(g, near, under=text_dirs, want_ext=kinds) + binds = text_bindings(g, near, under=text_dirs, want_ext=kinds, scope=text_scope) # the question named a kind of file (SQL: mapper XML or .sql): when some binding is of that kind, the others # (a Postman collection, a README table that spells the method) are not what was asked if kinds and any(f.endswith(kinds) for f, _l, _d, _n in binds): @@ -1538,13 +1573,7 @@ def main(argv): if len(binds) > BINDINGS_SHOWN: print(f" … +{len(binds) - BINDINGS_SHOWN} more") RESULT['text_bindings'] = [{'file': f, 'line': l, 'declaration': d, 'name': nm} for f, l, d, nm in binds] elif text_dirs: - hits = text_term_hits(g, text_dirs, terms) - if hits: - print(f"\ntext files under {', '.join(d + '/' for d in text_dirs)} that use the task's words (not source; " - "matched by word, not by any declaration):") - for f, l, ts in hits[:BINDINGS_SHOWN]: - print(f" {f + ':' + str(l):56} {', '.join(ts)}") - RESULT['text_bindings'] = [{'file': f, 'line': l, 'terms': ts} for f, l, ts in hits] + print_text_hits(g, text_dirs, terms) print("\n" + BOUND) print(" narrow with `impact --in ` or `path '*' --in `.") diff --git a/tests/cases/java/context-answers-what-was-asked/case.json b/tests/cases/java/context-answers-what-was-asked/case.json index 333ae910..777f3f3e 100644 --- a/tests/cases/java/context-answers-what-was-asked/case.json +++ b/tests/cases/java/context-answers-what-was-asked/case.json @@ -5,6 +5,10 @@ "want": ["text files that name these declarations", "src/main/resources/mapper/OrderMapper.xml:3", "names findByNumber (OrderMapper.findByNumber)", "next: read src/main/resources/mapper/OrderMapper.xml:3"], "avoid": ["InvoiceMapper.xml"]}, + {"why": "control for --in narrowing text files: a mapper XML whose namespace names the declaring type is bound by that declaration, so it is listed even when --in names only the source tree", + "run": ["context", "which SQL runs when an order is loaded by number", "--in", "src/main/java"], + "want": ["src/main/resources/mapper/OrderMapper.xml:3", "names findByNumber (OrderMapper.findByNumber)"], + "avoid": ["InvoiceMapper.xml"]}, {"why": "--in on a directory that holds no source is accepted: it says the graph cannot see it and lists the text files under it bound to what the question names", "run": ["context", "which SQL runs for findByNumber", "--in", "src/main/resources"], "want": ["not indexed: src/main/resources/", "named in the task (findByNumber)", "src/main/resources/mapper/OrderMapper.xml:3"], diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/bower.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/bower.json new file mode 100644 index 00000000..5721bd26 --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/bower.json @@ -0,0 +1 @@ +{ "name": "wiring-sample", "dependencies": { "debug": "1.0.0" } } diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/case.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/case.json index 3f88d68a..720b5f37 100644 --- a/tests/cases/javascript/context-text-leads-skip-lockfiles/case.json +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/case.json @@ -8,11 +8,15 @@ "run": ["context", "app wiring: createApp and its output", "--in", "src/a", "--source"], "want": ["next: answer from the code of src/a/app.js shown first above"], "avoid": ["package-lock.json", "package.json:"]}, + {"why": "--in a directory no graph holds lists its text files by word, and a manifest's dependency line there is not one; its rule file is (control)", + "run": ["context", "lint tooling: which debug rule the linter loads", "--in", "tools/lint"], + "want": ["not indexed: tools/lint/", "tools/lint/rules.json:2", "next: read tools/lint/"], + "avoid": ["tools/lint/package.json", "package-lock.json"]}, {"why": "without --in, a lockfile is never a text lead and a manifest's description, dependency lists and script names are not either; a manifest script command and a config file that name the function are (control: the manifest is filtered by line, not dropped)", "run": ["context", "app wiring: createApp, config, debug output"], - "want": ["src/a/routes.json:2", "config/app.json:2", "package.json:6"], + "want": ["config/app.json:2", "package.json:6"], "avoid": ["package-lock.json", "packages.lock.json", "package.json:4", "package.json:7", "package.json:10", "package.json:13"]}, - {"why": "impact lists the same text files as bound from outside the source: the lockfiles and the manifest's dependency lists are not among them", + {"why": "impact lists the same text files as bound from outside the source: lockfiles, the manifest's dependency lists and every other manifest's package list (a pnpm catalog, bower and deno imports, a requirements file) are not among them; a script or task value that names the function is, in a one-line manifest too (control)", "run": ["impact", "debug"], - "want": ["src/a/routes.json", "config/app.json"], - "avoid": ["package-lock.json", "packages.lock.json", "package.json:7", "package.json:10"]}]} + "want": ["src/a/routes.json", "config/app.json", "deno.json:3", "packages/min/package.json:1"], + "avoid": ["package-lock.json", "packages.lock.json", "package.json:7", "package.json:10", "pnpm-workspace.yaml", "bower.json", "deno.json:2", "requirements.txt"]}]} diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/deno.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/deno.json new file mode 100644 index 00000000..817764be --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/deno.json @@ -0,0 +1,4 @@ +{ + "imports": { "debug": "npm:debug@4.1.0" }, + "tasks": { "trace": "debug" } +} diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/package-lock.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/package-lock.json new file mode 100644 index 00000000..c25c2e2c --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/package-lock.json @@ -0,0 +1,19 @@ +{ + "name": "wiring-sample", + "lockfileVersion": 3, + "packages": { + "node_modules/debug": { + "version": "4.3.4", + "dependencies": { + "ms": "2.1.2" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "optional": true + } + }, + "dependencies": { + "debug": "^4.1.0" + } +} diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/packages/min/package.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/packages/min/package.json new file mode 100644 index 00000000..16825bfa --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/packages/min/package.json @@ -0,0 +1 @@ +{"name":"min","dependencies":{"debug":"^4"},"scripts":{"trace":"debug"}} diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/pnpm-workspace.yaml b/tests/cases/javascript/context-text-leads-skip-lockfiles/pnpm-workspace.yaml new file mode 100644 index 00000000..b91fa7b9 --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +catalog: + debug: ^4.1.0 diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/requirements.txt b/tests/cases/javascript/context-text-leads-skip-lockfiles/requirements.txt new file mode 100644 index 00000000..8058cbc2 --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/requirements.txt @@ -0,0 +1 @@ +debug==0.1 diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/tools/lint/package.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/tools/lint/package.json new file mode 100644 index 00000000..36702d1f --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/tools/lint/package.json @@ -0,0 +1,6 @@ +{ + "name": "lint-tools", + "devDependencies": { + "debug": "^4.1.0" + } +} diff --git a/tests/cases/javascript/context-text-leads-skip-lockfiles/tools/lint/rules.json b/tests/cases/javascript/context-text-leads-skip-lockfiles/tools/lint/rules.json new file mode 100644 index 00000000..64d7d3b4 --- /dev/null +++ b/tests/cases/javascript/context-text-leads-skip-lockfiles/tools/lint/rules.json @@ -0,0 +1,3 @@ +{ + "rules": { "no-debug": "warn" } +} diff --git a/tests/multi_language.py b/tests/multi_language.py index 7f0f644d..39e2d5b2 100644 --- a/tests/multi_language.py +++ b/tests/multi_language.py @@ -50,7 +50,8 @@ 'lib/pytools/probe.py': 'def emit(x):\n return [x]\n', 'src/cli.ts': 'import { run } from \'./main\'\n\nexport function main(): number {\n return run()\n}\n', 'jslib/package.json': '{ "name": "jslib", "version": "1.0.0", "main": "index.js" }\n', - 'jslib/index.js': 'function helper(a) {\n return a + 1\n}\n\nfunction api(a) {\n return helper(a) * 2\n}\n\nmodule.exports = { api }\n', + 'notes/steps.json': '{ "steps": ["total"] }\n', + 'jslib/index.js':'function helper(a) {\n return a + 1\n}\n\nfunction api(a) {\n return helper(a) * 2\n}\n\nmodule.exports = { api }\n', } # a Maven project whose build wrote javadoc: target/ beside the pom.xml, and a copy committed for a docs site @@ -212,6 +213,16 @@ def check(ok, why, detail=''): s = sh(repo, AX, 'context', 'emit a value', '.', '--in', 'tools', env=quiet) check(s.returncode == 0 and 'tools/gen/make.py' in s.stdout and 'lib/pytools' not in s.stdout, 'scope: context --in tools keeps lib/pytools/ out too', s.stdout + s.stderr) + # a directory only the MAIN graph holds, asked from every graph: another language's graph once read it as text + # no graph holds, listed its source files as text rows and ended on a next step outside the scope + s = sh(repo, AX, 'context', 'compute the area of a shape and add up a total', '.', '--in', 'src', env=quiet) + check(s.returncode == 0 and 'not indexed: src/' not in s.stdout and 'tools/' not in s.stdout and 'src/shape.ts' in s.stdout, + 'scope: a directory the main graph holds is not text to the other graphs', s.stdout + s.stderr) + # a directory no graph holds, named by --in: its text files are listed once, and every next step stays in it + s = sh(repo, AX, 'context', 'which steps add up a total', '.', '--in', 'notes', env=quiet) + nexts = [l for l in s.stdout.splitlines() if l.startswith('next:')] + check(s.returncode == 0 and s.stdout.count('notes/steps.json') == 1 and nexts and all('notes/' in l for l in nexts), + 'scope: --in a text-only directory is listed by one graph, and no next step leaves it', s.stdout + s.stderr) # ── --from ──────────────────────────────────────────────────────────────────────────────────────────── # `main` is declared in the typescript graph and twice in the python one: the flow starts where the task's