diff --git a/bin/axiomcode b/bin/axiomcode index c3948dfa..99f3f668 100755 --- a/bin/axiomcode +++ b/bin/axiomcode @@ -295,6 +295,13 @@ case "$cmd" in echo "▶ parsing $src → IR" "$0" parser "$src" "$ir" ${popts[@]+"${popts[@]}"} > "$int/parse.log" 2>&1 || { echo "❌ parser failed — see $int/parse.log" >&2; tail -5 "$int/parse.log" >&2; exit 1; } head -1 "$int/parse.log" + # WHAT AN INSTALLED PYTHON LIBRARY HANDS BACK to the objects the client gives it (graph/python/libsum): read from + # the project's environment, cached by package and version, staged beside the client IR. Never fatal -- without + # it the engine sees no library hand-backs, exactly as before. AXIOMCODE_LIB_CALLBACKS=0 turns it off. + if [ -d "$ir/python" ] && [ "${AXIOMCODE_LIB_CALLBACKS:-1}" != 0 ]; then + python3 "$ROOT/graph/python/libsum/libsum.py" --src "$src" --out "$ir/python" > "$int/libsum.log" 2>&1 \ + && tail -1 "$int/libsum.log" || echo "⚠️ library callback summaries failed (see $int/libsum.log); continuing without them" >&2 + fi # which languages the IR holds — one folder each — and which of them have an engine found=(); for d in "$ir"/*/; do l="$(basename "$d")"; [ -d "$ROOT/graph/$l/engine" ] && found+=("$l"); done [ ${#found[@]} -gt 0 ] || { echo "❌ no language with an engine in the IR (folders: $(ls "$ir" | tr '\n' ' '))" >&2; exit 1; } diff --git a/graph/bundle/SCHEMA.md b/graph/bundle/SCHEMA.md index 2c17b391..a78cdc69 100644 --- a/graph/bundle/SCHEMA.md +++ b/graph/bundle/SCHEMA.md @@ -442,7 +442,7 @@ One row per place a call is written (or, for a synthesised edge, the construct t - **typescript** — end_line / end_column come from the expression row; the call-site row itself records only the start. - **javascript** — caller_id is the parser's enclosing method, or the module initializer for top-level code. end_line / end_column come from the expression row. `require()` is a module edge, not a call site. - **typescript** — PROPERTY_READ and PROPERTY_WRITE rows are accessor invocations with no written call: the site is the property-access expression that runs the getter or setter, positioned from the expressions table, and callee_name is NULL because nothing was written; the accessor's name is on the callee's methods row. Filter them out with kind NOT IN (…) when counting calls. -- **python** — PROPERTY_READ, CONTEXT_MANAGER, ITERATION_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls. +- **python** — PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls. - **python** — The id is an EXPRESSION hash for a written call; a DECORATOR hash (PY_DECORATOR_…) for DECORATOR_APPLICATION and DECORATOR_* sites, positioned at the decorator line; and the class's TYPE hash for METACLASS_CREATION, positioned at the class declaration. ### `call_edges` @@ -531,6 +531,7 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h | `DECORATOR_*` | python | Applying an unparenthesised decorator; the suffix is the parser's decorator kind: BARE, ATTRIBUTE, SUBSCRIPT, EXPRESSION (and CALL/ATTRIBUTE_CALL when the factory expression is not itself a call site). The site is the decorator hash. | | `METACLASS_CREATION` | python | A class statement invokes its metaclass's `__new__` / `__init__` at import time, whether the metaclass is written on the statement (`class X(metaclass=M)`) or inherited from a base, and the nearest base's `__init_subclass__`. No written call; the site is the class's type hash. | | `PROPERTY_READ` | python | Reading `obj.attr` where `attr` is a `@property` runs the getter; reading `Cls.attr` where the METACLASS defines `attr` as a property runs that getter. No written call; the site is the attribute-access expression. | +| `PROPERTY_WRITE` | python | Assigning `obj.attr = v` where `attr` is a `@property` with a setter runs the setter; `del obj.attr` runs its deleter. No written call; the site is the attribute-access expression. | | `CONTEXT_MANAGER` | python | `with expr:` runs `__enter__` / `__exit__` (or the async pair). No written call; the site is the context-manager expression. | | `ITERATION_PROTOCOL` | python | `for x in expr:` (and comprehensions) runs `__iter__` / `__next__` (or the async pair). No written call; the site is the iterated expression. | | `SUBSCRIPT_PROTOCOL` | python | `x[k]` runs `__getitem__` (and `x[k] = v` / `del x[k]` the setter and deleter) of the receiver's class. No written call; the site is the subscript expression. Its own kind so it is never counted as a written call. | diff --git a/graph/bundle/schema.ts b/graph/bundle/schema.ts index 2e9af322..ac5e5a3c 100644 --- a/graph/bundle/schema.ts +++ b/graph/bundle/schema.ts @@ -667,6 +667,7 @@ export const VOCAB: readonly VocabSpec[] = [ { table: 'call_edges', column: 'kind', value: 'DECORATOR_*', languages: P, meaning: 'Applying an unparenthesised decorator; the suffix is the parser\'s decorator kind: BARE, ATTRIBUTE, SUBSCRIPT, EXPRESSION (and CALL/ATTRIBUTE_CALL when the factory expression is not itself a call site). The site is the decorator hash.' }, { table: 'call_edges', column: 'kind', value: 'METACLASS_CREATION', languages: P, meaning: 'A class statement invokes its metaclass\'s `__new__` / `__init__` at import time, whether the metaclass is written on the statement (`class X(metaclass=M)`) or inherited from a base, and the nearest base\'s `__init_subclass__`. No written call; the site is the class\'s type hash.' }, { table: 'call_edges', column: 'kind', value: 'PROPERTY_READ', languages: P, meaning: 'Reading `obj.attr` where `attr` is a `@property` runs the getter; reading `Cls.attr` where the METACLASS defines `attr` as a property runs that getter. No written call; the site is the attribute-access expression.' }, + { table: 'call_edges', column: 'kind', value: 'PROPERTY_WRITE', languages: P, meaning: 'Assigning `obj.attr = v` where `attr` is a `@property` with a setter runs the setter; `del obj.attr` runs its deleter. No written call; the site is the attribute-access expression.' }, { table: 'call_edges', column: 'kind', value: 'CONTEXT_MANAGER', languages: P, meaning: '`with expr:` runs `__enter__` / `__exit__` (or the async pair). No written call; the site is the context-manager expression.' }, { table: 'call_edges', column: 'kind', value: 'ITERATION_PROTOCOL', languages: P, meaning: '`for x in expr:` (and comprehensions) runs `__iter__` / `__next__` (or the async pair). No written call; the site is the iterated expression.' }, { table: 'call_edges', column: 'kind', value: 'SUBSCRIPT_PROTOCOL', languages: P, meaning: '`x[k]` runs `__getitem__` (and `x[k] = v` / `del x[k]` the setter and deleter) of the receiver\'s class. No written call; the site is the subscript expression. Its own kind so it is never counted as a written call.' }, @@ -748,7 +749,7 @@ export const NOTES: readonly NoteSpec[] = [ { language: 'typescript', table: 'overrides', note: 'EMPTY — this table is Java-shaped. The TypeScript dispatch envelope is in dispatch_candidates, with basis `nominal` or `structural`.' }, { language: 'typescript', table: 'type_instantiated', note: 'Every row has how = `new`. Not restricted to client provenance: a type the library constructs is still a type that exists at run time, and dropping it would narrow the envelope unsoundly.' }, { language: 'typescript', table: 'call_sites', note: 'PROPERTY_READ and PROPERTY_WRITE rows are accessor invocations with no written call: the site is the property-access expression that runs the getter or setter, positioned from the expressions table, and callee_name is NULL because nothing was written; the accessor\'s name is on the callee\'s methods row. Filter them out with kind NOT IN (…) when counting calls.' }, - { language: 'python', table: 'call_sites', note: 'PROPERTY_READ, CONTEXT_MANAGER, ITERATION_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls.' }, + { language: 'python', table: 'call_sites', note: 'PROPERTY_READ, PROPERTY_WRITE, CONTEXT_MANAGER, ITERATION_PROTOCOL, METACLASS_CREATION and DYNAMIC_CALL rows are protocol or indirect edges with no written call: their site is the expression that triggers them, and callee_name is always NULL because nothing was written. SUBSCRIPT_CALL is NULL only when the subscript is not a written name (measured 206 of 337 rows on a Python subject). Filter them out with kind NOT IN (…) when counting calls.' }, { language: 'python', table: 'call_sites', note: 'The id is an EXPRESSION hash for a written call; a DECORATOR hash (PY_DECORATOR_…) for DECORATOR_APPLICATION and DECORATOR_* sites, positioned at the decorator line; and the class\'s TYPE hash for METACLASS_CREATION, positioned at the class declaration.' }, { language: 'python', table: 'call_edges', note: 'A `boundary_lib` edge may point at a builtin (callee_provenance builtin, callee_label `builtin:NAME`) or at an unstaged import path (callee_provenance external) — neither has a methods row.' }, { language: 'java', table: 'call_edges', note: 'A `boundary_lib` edge with callee_provenance external names a method of an ancestor type no staged IR declares (callee_label `external:.`, no methods row). A site whose receiver is declared as such a type is multi_inferred even with one client override: the platform method itself, and the platform\'s own subclasses, are the other possible targets. Stage the library to replace the label with the real method.' }, diff --git a/graph/pipeline/run-souffle.sh b/graph/pipeline/run-souffle.sh index b0e9fd5b..6b907570 100755 --- a/graph/pipeline/run-souffle.sh +++ b/graph/pipeline/run-souffle.sh @@ -201,6 +201,7 @@ write_program(){ _SU=() t+="#include \"$LANG_ARG/souffle/decls_base.dl\"$nl#include \"$LANG_ARG/souffle/decls_all.dl\"$nl" map_rels_su "$TPL/client-ir.map" || return 1 + if [ -f "$TPL/client-extra.map" ]; then map_rels_su "$TPL/client-extra.map" || return 1; fi map_rels_su "$TPL/lib.map" sig || return 1 for r in $LIB_BODY; do _SU+=("$r"); done _SU+=(jdk_max_depth lib_max_depth taint_gating dispatch_cap dispatch_closed_world) @@ -254,7 +255,8 @@ write_program(){ # lost on either side shows up as a difference. Every .input and .output line must be exactly # the expected one, each exactly once, with nothing extra. Exit status only: no pipe to lose. verify_program(){ - awk -v cmap="$TPL/client-ir.map" -v lmap="$TPL/lib.map" -v man="$DL/export_manifest.tsv" \ + local xmap=""; [ -f "$TPL/client-extra.map" ] && xmap="$TPL/client-extra.map" + awk -v cmap="$TPL/client-ir.map" -v xmap="$xmap" -v lmap="$TPL/lib.map" -v man="$DL/export_manifest.tsv" \ -v libsig=" $LIB_SIG " -v extra="$LIB_BODY jdk_max_depth lib_max_depth taint_gating dispatch_cap dispatch_closed_world" ' function want(line) { if (!(line in need)) { need[line] = 1; n++ } } function inp(r) { want(".input " r "(IO=file, filename=\"" r ".facts\", delimiter=\"\\t\", rfc4180=true)") } @@ -269,7 +271,7 @@ verify_program(){ close(path) } BEGIN { - maprels(cmap, 0); maprels(lmap, 1) + maprels(cmap, 0); maprels(lmap, 1); if (xmap != "") maprels(xmap, 0) m = split(extra, e, " "); for (i = 1; i <= m; i++) if (e[i] != "") inp(e[i]) while ((rc = (getline l < man)) > 0) { sub(/^\t+/, "", l); k = index(l, "\t"); p = l; f = "" @@ -813,7 +815,7 @@ while IFS=$'\t' read -r rel csv; do if [ -f "$CLIENT/$csv.csv" ]; then awk -F'\t' 'NR==1{n=NF; next} NF==n{print; next} {bad++} END{if(bad>0) printf " ! dropped %d malformed row(s) from %s\n", bad, FILENAME > "/dev/stderr"}' "$CLIENT/$csv.csv" > "$FACTS/$rel.facts" else : > "$FACTS/$rel.facts"; fi CLIENT_INPUTS="$CLIENT_INPUTS$rel"$'\n' -done < <(read_map "$TPL/client-ir.map") +done < <(read_map "$TPL/client-ir.map"; if [ -f "$TPL/client-extra.map" ]; then read_map "$TPL/client-extra.map"; fi) # --- LIB: signature relations, concatenated across every module of every root --- # CACHED. This concatenation reads the ENTIRE library IR (the JDK alone is 2.0 GB in, diff --git a/graph/python/engine/call-edge-generation/call_chain.dl b/graph/python/engine/call-edge-generation/call_chain.dl index b1c334c5..6c35b6cf 100644 --- a/graph/python/engine/call-edge-generation/call_chain.dl +++ b/graph/python/engine/call-edge-generation/call_chain.dl @@ -156,6 +156,31 @@ method_returns_method("client", m, r) :- expr_call_candidate(site, g), method_returns_method("client", g, r). +// ── a call that hands back one of its arguments (py_returns_arg, resolution/builtins.dl) ── +// Matched through the import that binds the callee, never by the bare name: +// `functools.update_wrapper(...)` / `t.cast(...)` where the receiver is the name an `import` +// bound, or `update_wrapper(...)` bound by `from functools import update_wrapper`. A local +// function that happens to share the name does neither. +py_passthrough_arg(site, pos) :- + call_name(site, fn), call_receiver_object(site, obj), + expr_binding("client", rb, ctx, obj), ctx != "STORE", binding_lookup("client", rb, rb2), + import_binding("client", rb2, i), import_decl("client", k, mod, _, i), + (k = "MODULE_IMPORT" ; k = "MODULE_IMPORT_ALIAS"), // `import typing as t` is the alias kind + py_returns_arg(path, pos), cat(mod, cat(".", fn)) = path. +py_passthrough_arg(site, pos) :- + call_callee_is_value(site), call_callee_expr(site, callee), + expr_binding("client", rb, ctx, callee), ctx != "STORE", binding_lookup("client", rb, rb2), + import_binding("client", rb2, i), import_decl("client", _, path, _, i), + py_returns_arg(path, pos). +// the expression a value really is, through any number of such calls +py_passthrough_root(e, e) :- method_return_value_expr("client", _, e). +py_passthrough_root(e, x) :- + py_passthrough_root(e, c), call_of_expr(c, site), py_passthrough_arg(site, pos), + call_arg(site, pos, x). +method_returns_method("client", m, r) :- + method_return_value_expr("client", m, e), py_passthrough_root(e, x), x != e, + expr_names_method("client", x, r). + // ── decorator_hits_lib(SiteKey, LibMethodHash) ─────────────────────────────── // `@abstractmethod` names `abc.abstractmethod`, which HAS Python source and is in the // staged stdlib IR — it is a library boundary, not a blind spot. A BARE decorator has no @@ -320,6 +345,25 @@ iter_protocol_edge(src, caller, m) :- expr_type("client", src, t), iter_protocol_target(t, m), expr_ultimate_method("client", src, caller). +// ── property_write_edge(WriteExprHash, CallerMethodHash, AccessorMethodHash) ── +// The store and delete halves of the property protocol: `obj.x = v` calls x's setter, +// `del obj.x` its deleter (type_property_accessor, resolution/attribute-lookup.dl). Same +// shape as a read, keyed on the access's own name context. +property_write_edge(e, caller, m) :- + expr_node("client", "ATTRIBUTE_ACCESS", _, n, e), + expr_name_context("client", "STORE", e), + expr_parent("client", e, "ATTRIBUTE_OBJECT", _, obj), + expr_type("client", obj, t), + type_property_accessor("client", t, n, "PROPERTY_SETTER", m), + expr_ultimate_method("client", e, caller). +property_write_edge(e, caller, m) :- + expr_node("client", "ATTRIBUTE_ACCESS", _, n, e), + expr_name_context("client", "DEL", e), + expr_parent("client", e, "ATTRIBUTE_OBJECT", _, obj), + expr_type("client", obj, t), + type_property_accessor("client", t, n, "PROPERTY_DELETER", m), + expr_ultimate_method("client", e, caller). + // ── property_read_edge(ReadExprHash, CallerMethodHash, GetterMethodHash) ───── property_read_edge(e, caller, getter) :- expr_node("client", "ATTRIBUTE_ACCESS", _, n, e), @@ -668,6 +712,7 @@ subscript_protocol_edge(sub, caller, m) :- // from the graph rather than being re-tiered. A dropped edge is worse than a mislabelled // one, and the golden caught it. protocol_edge(e, d, caller, m) :- property_read_edge(e, caller, m), method_decl(_, d, _, _, _, m). +protocol_edge(e, d, caller, m) :- property_write_edge(e, caller, m), method_decl(_, d, _, _, _, m). protocol_edge(e, d, caller, m) :- with_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). protocol_edge(e, d, caller, m) :- iter_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). protocol_edge(e, d, caller, m) :- subscript_protocol_edge(e, caller, m), method_decl(_, d, _, _, _, m). @@ -731,6 +776,12 @@ call_chain_edge(e, caller, "-", getter, "client", cls, "PROPERTY_READ") :- property_read_edge(e, caller, getter), method_decl(_, d, _, _, _, getter), protocol_edge_class(e, d, cls). +// PROPERTY WRITE — `obj.x = v` runs x's setter and `del obj.x` its deleter. Likewise no +// call site, and its own kind so it is never counted as a written call. +call_chain_edge(e, caller, "-", m, "client", cls, "PROPERTY_WRITE") :- + property_write_edge(e, caller, m), method_decl(_, d, _, _, _, m), + protocol_edge_class(e, d, cls). + // CONTEXT MANAGER — the same shape: an edge with no call site, its own kind so it can // never be mistaken for a written call. call_chain_edge(cm, caller, "-", m, "client", cls, "CONTEXT_MANAGER") :- diff --git a/graph/python/engine/expression-resolution/expr-type.dl b/graph/python/engine/expression-resolution/expr-type.dl index d46b6eca..ed530619 100644 --- a/graph/python/engine/expression-resolution/expr-type.dl +++ b/graph/python/engine/expression-resolution/expr-type.dl @@ -585,6 +585,15 @@ binding_declared_nominal(p, bind, d) :- binding_declared_ref(p, bind, r), type_ref_resolved(p, d, r), !type_is_structural(p, d). +// `x: Optional[Order] = ...` / `x: Order | None = ...` IS an Order (or None) — the rule the +// parameter, return and field annotations already have (resolution/annotations.dl, +// "Optional[X] IS X"), which the local annotation alone was missing: a local written +// with the commonest modern spelling stayed untyped while the same annotation on a +// parameter resolved. +binding_declared_nominal(p, bind, t) :- + type_ref_owner(p, bind, "BINDING", r), + type_ref(p, k, "VARIABLE_ANNOTATION", _, _, r), annotation_optional_kind(k), + type_ref_element(p, r, t), !type_is_structural(p, t). // The FK is resolved on only 14 of 479 variable annotations, so the name-based lookup // beside it is what carries this -- the same two clauses the parameter path uses, with // the same kind restriction. A SUBSCRIPT or a UNION names no single type and is served by diff --git a/graph/python/engine/framework-behavior/dispatch.dl b/graph/python/engine/framework-behavior/dispatch.dl index 0ede5dbf..c2c8dc38 100644 --- a/graph/python/engine/framework-behavior/dispatch.dl +++ b/graph/python/engine/framework-behavior/dispatch.dl @@ -412,6 +412,53 @@ py_fixture_requested(m) :- py_fixture_injection(_, m, _). framework_edge(from, to, "fixture_injection", name, "by_name") :- py_fixture_injection(from, to, name). +// ── 2a. THE SAME LOOKUP, FROM SYNTAX ALONE ────────────────────────────────────── +// resolution/value-flow.dl types a parameter by the fixture serving it, so it needs the +// runner's lookup INSIDE the resolution fixpoint. py_fixture_injection cannot be read +// there: a fixture a conftest star-imports is found through module_member_method, which +// is resolution, and the nearest-conftest MAX over it would then be a cyclic aggregate +// souffle refuses to stratify. This chain keeps the runner's order (class, own module, +// nearest conftest by its declaring file) and leaves out the two clauses that need +// resolution — a star-imported fixture and a pytest_plugins one. Those still reach their +// tests through the injection edge; they only go untyped. +.decl py_fixs_in_file(fix:symbol, file:symbol) +py_fixs_in_file(fix, p) :- py_fixture_decl(fix, _), method_file("client", p, fix). +.decl py_fixs_same_module(req:symbol, name:symbol, fix:symbol) +py_fixs_same_module(req, name, fix) :- py_fixture_request(req, name), + py_fixture_decl(fix, name), fix != req, !py_fixture_class(fix, _), + !py_fixture_class_shadowed(req, name), + method_file("client", p, req), py_fixs_in_file(fix, p). +.decl py_fixs_conftest_cand(req:symbol, name:symbol, fix:symbol, depth:number) +py_fixs_conftest_cand(req, name, fix, dl) :- py_fixture_request(req, name), + py_fixture_decl(fix, name), fix != req, !py_fixture_class(fix, _), + py_fixs_in_file(fix, cp), py_fixture_scope_file(c), + strlen(cp) >= strlen(c), substr(cp, strlen(cp) - strlen(c), strlen(c)) = c, + d = substr(cp, 0, strlen(cp) - strlen(c)), + method_file("client", p, req), + strlen(p) > strlen(d), substr(p, 0, strlen(d)) = d, dl = strlen(d). +.decl py_fixs_nearest(req:symbol, name:symbol, depth:number) +py_fixs_nearest(req, name, m) :- py_fixs_conftest_cand(req, name, _, _), + m = max dl : { py_fixs_conftest_cand(req, name, _, dl) }. +.decl py_fixs_injection(from:symbol, to:symbol, name:symbol) +py_fixs_injection(req, fix, name) :- py_fixture_in_class(req, name, fix). +py_fixs_injection(req, fix, name) :- py_fixs_same_module(req, name, fix). +py_fixs_injection(req, fix, name) :- py_fixs_conftest_cand(req, name, fix, dl), + !py_fixture_class_shadowed(req, name), !py_fixs_same_module(req, name, _), + py_fixs_nearest(req, name, dl). + +// ── 2b. FIXTURE VALUE (what the runner hands the parameter) ──────────────────── +// py_fixture_value_type(Fixture, Type) — the value a fixture hands over: its return or +// yield value's type, or its declared return. Consumed by resolution/value-flow.dl, where +// the runner's call `test(fixture_value)` is one more argument reaching a parameter. +.decl py_fixture_value_type(fix:symbol, t:symbol) +py_fixture_value_type(fix, t) :- py_fixture_decl(fix, _), + method_return_value_expr("client", fix, e), expr_type("client", e, t). +py_fixture_value_type(fix, t) :- py_fixture_decl(fix, _), + method_declared_return_type("client", fix, d), declared_dispatch("client", d, t). +py_fixture_value_type(fix, t) :- py_fixture_decl(fix, _), + expr_node("client", "YIELD", _, _, y), expr_ultimate_method("client", y, fix), + expr_parent("client", y, "YIELD_VALUE", _, v), expr_type("client", v, t). + // ───────────────────────────────────────────────────────────────────────────── // 3. URL DISPATCH (route table -> view) // ───────────────────────────────────────────────────────────────────────────── diff --git a/graph/python/engine/framework-behavior/library-callbacks.dl b/graph/python/engine/framework-behavior/library-callbacks.dl new file mode 100644 index 00000000..e17d9043 --- /dev/null +++ b/graph/python/engine/framework-behavior/library-callbacks.dl @@ -0,0 +1,368 @@ +// ============================================================================ +// FRAMEWORK-BEHAVIOR · LIBRARY CALLBACKS (the hop a dependency makes back into the client) +// +// ── THE GAP ───────────────────────────────────────────────────────────────── +// The engine reads a dependency's signatures, never its bodies. So when a library calls +// BACK into client code, the call graph stops at the library frame: a test client's +// `client.get("/")` runs the `open()` a client subclass overrides and then the +// application object it was built around (`app(environ, start_response)`); a CLI test +// runner's `runner.invoke(cli)` runs `cli.main()`, which runs the command's callback; an +// `ExitStack.enter_context(cm)` runs `cm.__enter__`. Measured on a 16-repository Python +// oracle, a library frame is the first broken hop behind most test-impact misses on the +// web-framework subject: the test calls the library, the library calls the code under +// change, and the graph has only the first half. +// +// ── WHERE THE KNOWLEDGE COMES FROM ────────────────────────────────────────── +// graph/python/libsum/libsum.py reads the installed library's SOURCE once (cached by the +// installed distributions and the client's imports, never solved here) and stages these tables +// beside the client IR, for the library classes and functions the client can name: +// py_lib_callback(Callee, Root, Pos, Path, Member, Via, Type) +// calling the library callable Callee hands back `Member` on the object that arrived +// as its parameter Root (written by the client at position Pos, -1 = the receiver +// `self`), after following the attribute Path ("" = the object itself); Via is the +// library function where that invocation is written; Type the class the library declares that +// object to be, "" when it declares none +// py_lib_ancestor(Class, Ancestor) a library class's library ancestors +// py_lib_field_init(Class, Field, Param, Pos) +// constructing Class stores the argument Param on the new object as Field +// py_lib_alias(Alias, Canonical) +// a name a package re-exports +// py_lib_decorates(Decorator, Form, Class, Field) +// `@Decorator` (DIRECT) or `@Decorator(...)` (FACTORY) turns a def into a Class whose Field +// holds it; Class "" when the def itself comes back +// No row for a library written in C: a declared blind spot, never a guess. +// +// ── WHAT IT EMITS, AND WHY NOT AS A CALL ──────────────────────────────────── +// lib_callback_edge(Caller, Target, Callee, Member, Via, How) +// Caller the client method containing the call into the library +// Target the client method the library is summarised to call back +// Callee the library callable the client called (canonical name) +// How receiver Target is a member of the object the call was made on +// (a client subclass overriding what the library's own code calls) +// receiver_field ... of an object the library holds in a field of it +// argument ... of an object (or the callable itself) passed as an argument +// argument_field ... of an object held in a field of an argument +// The contract of this phase holds: it only ADDS a relation. The hop is a library's +// behaviour summarised from its source, joined to the client by a member NAME on a type +// the client handed over; a resolved call is stronger knowledge, so call_chain_edge is +// untouched and a consumer opts in (impact walks it as its own hop, `library callback`). +// No rule here names a library or a framework: every name comes from the staged tables. +// ============================================================================ + +// ── the dotted path a client expression spells through an unlinked import ─── +// `Client` after `from webkit.testing import Client`, `clikit.command` after `import clikit`: the import's path, +// then each attribute read off it. Only an import that linked to nothing in the client starts one. +.decl lcb_path_of(e:symbol, p:symbol) +lcb_path_of(e, p) :- + expr_binding("client", b, ctx, e), ctx != "STORE", + binding_lookup("client", b, b2), import_unlinked(b2, p). +lcb_path_of(e, cat(p, cat(".", n))) :- + expr_node("client", "ATTRIBUTE_ACCESS", _, n, e), n != "", + expr_parent("client", e, "ATTRIBUTE_OBJECT", _, obj), lcb_path_of(obj, p), strlen(p) < 200. + +// ── its canonical name, following the package's re-exports ────────────────── +.decl lcb_named(p:symbol) +lcb_named(p) :- lcb_path_of(_, p). +.decl lcb_canon(p:symbol, c:symbol) +lcb_canon(p, c) :- lcb_named(p), py_lib_alias(p, c). +lcb_canon(p, p) :- lcb_named(p), !py_lib_alias(p, _). +.decl lcb_expr_names(e:symbol, c:symbol) +lcb_expr_names(e, c) :- lcb_path_of(e, p), lcb_canon(p, c). + +// ── a client class whose base is a library class ───────────────────────────── +// Written `class AppClient(Client)` with `Client` imported from a dependency that was not +// staged: the base resolves to nothing and type_base_unresolved records it by text. Here it +// is named by the import the text binds to, and inherited through client subclasses. +.decl lcb_direct_ext_base(t:symbol, c:symbol) +lcb_direct_ext_base(t, c) :- + type_base_slot("client", t, _, "NAME", _, _, bh), base_expr("client", bh, e), lcb_expr_names(e, c). +lcb_direct_ext_base(t, c) :- + type_base_slot("client", t, _, "SUBSCRIPT", _, _, bh), base_expr("client", bh, e), + expr_parent("client", e, "SUBSCRIPT_OBJECT", _, obj), lcb_expr_names(obj, c). +.decl lcb_ext_class(t:symbol, c:symbol) +lcb_ext_class(t, c) :- lcb_direct_ext_base(t, c). +lcb_ext_class(t, c) :- type_ancestor("client", t, a), lcb_direct_ext_base(a, c). + +// a client type and every client subclass of it: the object a library calls back may be a +// subclass of the type the call site was written against +.decl lcb_self_type(t:symbol, s:symbol) +lcb_self_type(t, t) :- lcb_ext_class(t, _). +lcb_self_type(t, s) :- lcb_ext_class(t, _), type_ancestor("client", s, t). + +.decl lcb_has_rows(c:symbol) +lcb_has_rows(c) :- py_lib_callback(c, _, _, _, _, _, _). + +// ── lcb_entry(Site, Callee, SelfType) — a client call into a summarised library callable ── +// SelfType is the client type the library's `self` is, or "" when the receiver is not a +// client object. +.decl lcb_entry(site:symbol, callee:symbol, selfT:symbol) +// (a) a member the client type inherits from its library base and does not define itself: +// `client.get("/")` on an AppClient-like subclass, `self.get(...)` inside one +lcb_entry(site, q, t) :- + call_receiver_object(site, obj), expr_type("client", obj, t), + call_name(site, n), n != "", + !expr_call_candidate(site, _), + lcb_ext_class(t, c), q = cat(c, cat(".", n)), lcb_has_rows(q). +// (b) super().n(...) whose MRO leaves the client for a library base +lcb_entry(site, q, a) :- + call_recv_kind(site, "SUPER"), call_super_anchor(site, _, a), + call_name(site, n), n != "", + !expr_call_candidate(site, _), + lcb_ext_class(a, c), q = cat(c, cat(".", n)), lcb_has_rows(q). +// (c) a library function or class called by a name the client imported +lcb_entry(site, c, "") :- + call_callee_is_value(site), call_callee_expr(site, callee), + lcb_expr_names(callee, c), lcb_has_rows(c). +// (d) a member of a library object the client built: `runner = CliRunner(); runner.invoke(cli)`, or one a +// fixture or a helper returned. Its `self` is a library object, so it has no client type; what the client +// gave its constructor is reached through the allocation (lcb_lobj_entry below). +lcb_entry(site, q, "") :- lcb_lobj_entry(site, q, _). +// (e) constructing a client class whose constructor is the library base's: its __init__ may +// call members the client overrides +lcb_entry(site, c, t) :- + call_constructs_type(site, t), lcb_ext_class(t, c), + !type_constructor("client", t, _), lcb_has_rows(c). + +// (f) a hand-back to a member the client object does not define but INHERITS from a library class: the +// library calls `cli.main()` on the command it was given, `main` is the library base's own, and that +// method's hand-backs apply to the same object (its make_context override). Same site, same caller. +lcb_entry(site, q2, t) :- + lcb_arg(site, q, r, a), py_lib_callback(q, r, _, "", n, _, _), + expr_type("client", a, t), !mro_lookup("client", t, n, _), + lcb_ext_class(t, c), q2 = cat(c, cat(".", n)), lcb_has_rows(q2). +lcb_entry(site, q2, t) :- + lcb_entry(site, q, t), t != "", py_lib_callback(q, "self", _, "", n, _, _), + !mro_lookup("client", t, n, _), + lcb_ext_class(t, c), q2 = cat(c, cat(".", n)), lcb_has_rows(q2). + +// ── lcb_ext_instance(Expr, Class, Alloc) — an expression holding an object of a library class ── +// Alloc is the call site that constructed it. Followed through a single-binding local, a client function's +// return value, and the fixture a test parameter is filled from. +.decl lcb_ext_instance(e:symbol, c:symbol, alloc:symbol) +lcb_ext_instance(e, c, site) :- + call_of_expr(e, site), call_callee_is_value(site), call_callee_expr(site, callee), + lcb_expr_names(callee, c), lcb_is_class(c). +lcb_ext_instance(e, c, al) :- + expr_binding("client", bref, ctx, e), ctx != "STORE", + binding_lookup("client", bref, bind), + binding_rebinding("client", "1", _, _, bind), + expr_binding("client", bind, "STORE", tgt), + assign_pair("client", tgt, val), + lcb_ext_instance(val, c, al). +lcb_ext_instance(e, c, al) :- + call_of_expr(e, site), expr_call_candidate(site, m), + method_return_value_expr("client", m, r), lcb_ext_instance(r, c, al). +lcb_ext_instance(e, c, al) :- + expr_names_param("client", e, ph), param_decl("client", pn, _, _, req, ph), + py_fixs_injection(req, fix, pn), method_return_value_expr("client", fix, r), lcb_ext_instance(r, c, al). +lcb_ext_instance(e, c, al) :- + expr_names_param("client", e, ph), param_decl("client", pn, _, _, req, ph), + py_fixs_injection(req, fix, pn), + expr_node("client", "YIELD", _, _, y), expr_ultimate_method("client", y, fix), + expr_parent("client", y, "YIELD_VALUE", _, v), lcb_ext_instance(v, c, al). +.decl lcb_is_class(c:symbol) +lcb_is_class(c) :- py_lib_field_init(c, _, _, _). +lcb_is_class(c) :- py_lib_callback(c, "self", _, _, _, _, _). + +// ── LIBRARY OBJECTS THE CLIENT HOLDS ──────────────────────────────────────── +// lcb_lobj(Expr, Class, Obj): Expr holds a library object of Class, identified by Obj -- the call site that +// constructed it, or the client function a library decorator wrapped into it (`@command() def greet()` makes +// `greet` a Command whose `callback` is the def). Its fields are what the client gave it. +.decl lcb_lobj(e:symbol, c:symbol, obj:symbol) +lcb_lobj(e, c, al) :- lcb_ext_instance(e, c, al). +lcb_lobj(e, c, m) :- lcb_names_def(e, m), lcb_deco_obj(m, c, _). + +// the def a name denotes, decorated or not (the decorated name's VALUE is what the decorator returned) +.decl lcb_names_def(e:symbol, m:symbol) +lcb_names_def(e, m) :- expr_binding("client", bref, ctx, e), ctx != "STORE", binding_lookup("client", bref, b), + method_declaring_binding("client", b, m). +lcb_names_def(e, m) :- expr_binding("client", bref, ctx, e), ctx != "STORE", binding_lookup("client", bref, b), + import_binds_method("client", b, m). +lcb_names_def(e, m) :- expr_attr_names_method("client", e, m). + +// the library callable a decorator expression applies: `@q` (direct) or `@q(...)` (factory) +.decl lcb_deco_callable(d:symbol, q:symbol, form:symbol) +lcb_deco_callable(d, q, "DIRECT") :- decorator_expr("client", e, d), !call_of_expr(e, _), lcb_expr_names(e, q). +lcb_deco_callable(d, q, "FACTORY") :- decorator_expr("client", e, d), call_of_expr(e, site), + call_callee_expr(site, callee), lcb_expr_names(callee, q). +lcb_deco_callable(d, q, "FACTORY") :- decorator_expr("client", e, d), call_of_expr(e, site), + call_receiver_object(site, obj), expr_type("client", obj, t), call_name(site, n), !mro_lookup("client", t, n, _), + lcb_ext_class(t, c), q = cat(c, cat(".", n)). +// a decorator that hands the function straight back (registers it, records an option on it) +.decl lcb_deco_identity(d:symbol) +lcb_deco_identity(d) :- lcb_deco_callable(d, q, form), py_lib_decorates(q, form, "", ""). +lcb_deco_identity(d) :- decorator_builtin("client", bk, d), decorator_transparent(bk). +// the def m enters application stage k unchanged: every decorator applied before it handed it straight back +.decl lcb_deco_blocked(m:symbol, k:number) +lcb_deco_blocked(m, k) :- deco_at(m, k, _), deco_at(m, k2, d2), k2 < k, !lcb_deco_identity(d2), !deco_stage(m, k2 + 1, m). +.decl lcb_deco_obj(m:symbol, c:symbol, f:symbol) +lcb_deco_obj(m, c, f) :- deco_at(m, k, d), !lcb_deco_blocked(m, k), lcb_deco_callable(d, q, form), + py_lib_decorates(q, form, c, f), c != "". + +// ── entries on a library object, and what its fields hand back ────────────── +.decl lcb_lobj_entry(site:symbol, callee:symbol, obj:symbol) +lcb_lobj_entry(site, q, o) :- + call_receiver_object(site, recv), lcb_lobj(recv, c, o), + call_name(site, n), n != "", + !expr_call_candidate(site, _), + q = cat(c, cat(".", n)), lcb_has_rows(q). +// a library object handed to a library callable that calls one of the object's own (library) members +lcb_lobj_entry(site, q2, o) :- + lcb_arg(site, q, r, a), py_lib_callback(q, r, _, "", n, _, _), lcb_lobj(a, c, o), + q2 = cat(c, cat(".", n)), lcb_has_rows(q2). + +// what a library object's field holds that offers member n: the argument its construction was given, or the +// function a decorator wrapped (called as `__call__`) +.decl lcb_lobj_field_target(o:symbol, f:symbol, n:symbol, m:symbol) +lcb_lobj_field_target(al, f, n, m) :- lcb_alloc_field(al, f, a), lcb_member_of_expr(a, n, m). +lcb_lobj_field_target(m, f, "__call__", m) :- lcb_deco_obj(m, _, f), lcb_field_wanted(m, f). +.decl lcb_field_wanted(o:symbol, f:symbol) +lcb_field_wanted(o, f) :- lcb_lobj_entry(_, q, o), py_lib_callback(q, "self", _, f, _, _, _), f != "". +lcb_field_wanted(o, f) :- lcb_arg(site, q, r, a), py_lib_callback(q, r, _, f, _, _, _), f != "", lcb_lobj(a, _, o), + lcb_entry(site, q, _). + +// what the constructor of a library object stored in a field, as the client wrote it at the allocation +.decl lcb_alloc_field(al:symbol, f:symbol, a:symbol) +lcb_alloc_field(al, f, a) :- + lcb_field_wanted(al, f), call_callee_expr(al, callee), lcb_expr_names(callee, c), + py_lib_field_init(c, f, _, pos), call_arg(al, ps, a), to_number(ps) = to_number(pos). +lcb_alloc_field(al, f, a) :- + lcb_field_wanted(al, f), call_callee_expr(al, callee), lcb_expr_names(callee, c), + py_lib_field_init(c, f, pn, _), call_kwarg(al, pn, a). + +// ── the argument the client wrote for a summarised parameter ───────────────── +.decl lcb_arg(site:symbol, callee:symbol, root:symbol, a:symbol) +lcb_arg(site, q, r, a) :- + lcb_entry(site, q, _), py_lib_callback(q, r, pos, _, _, _, _), r != "self", + to_number(pos) >= 0, call_arg(site, ps, a), to_number(ps) = to_number(pos). +lcb_arg(site, q, r, a) :- + lcb_entry(site, q, _), py_lib_callback(q, r, _, _, _, _, _), r != "self", + call_kwarg(site, r, a). + +// ── what a library-held field of a client object holds ─────────────────────── +// lcb_field_holds(T, Field, Expr): an object of client type T was built with Expr as the +// argument the library constructor stores in Field (the construction of T or of a client +// subclass, or T's own super().__init__(...) call) +.decl lcb_ctor_store(t:symbol, f:symbol, param:symbol, pos:symbol) +lcb_ctor_store(t, f, pn, pos) :- lcb_ext_class(t, c), py_lib_field_init(c, f, pn, pos). +.decl lcb_field_holds(t:symbol, f:symbol, e:symbol) +lcb_field_holds(t, f, a) :- + lcb_ctor_store(t, f, _, pos), lcb_self_type(t, s), call_constructs_type(site, s), + call_arg(site, ps, a), to_number(ps) = to_number(pos). +lcb_field_holds(t, f, a) :- + lcb_ctor_store(t, f, pn, _), lcb_self_type(t, s), call_constructs_type(site, s), + call_kwarg(site, pn, a). +lcb_field_holds(t, f, a) :- + lcb_ctor_store(t, f, _, pos), call_recv_kind(site, "SUPER"), call_super_anchor(site, _, t), + call_name(site, "__init__"), call_arg(site, ps, a), to_number(ps) = to_number(pos). +lcb_field_holds(t, f, a) :- + lcb_ctor_store(t, f, pn, _), call_recv_kind(site, "SUPER"), call_super_anchor(site, _, t), + call_name(site, "__init__"), call_kwarg(site, pn, a). + +// ── the client member an object expression offers under a name ─────────────── +// the type the expression has, or a client subclass of it; `__call__` on an expression that +// IS a function or a bound method is that function +.decl lcb_member_of_expr(e:symbol, n:symbol, m:symbol) +lcb_member_of_expr(e, n, m) :- + lcb_wanted(e, n), expr_type("client", e, t), lcb_sub(t, s), mro_lookup("client", s, n, m), + method_owner("client", _, m). +lcb_member_of_expr(e, "__call__", m) :- lcb_wanted(e, "__call__"), expr_denotes_method("client", e, m). +lcb_member_of_expr(e, "__call__", m) :- lcb_wanted(e, "__call__"), expr_names_method("client", e, m). +.decl lcb_sub(t:symbol, s:symbol) +lcb_sub(t, t) :- lcb_wanted(e, _), expr_type("client", e, t). +lcb_sub(t, s) :- lcb_wanted(e, _), expr_type("client", e, t), type_ancestor("client", s, t). + +// the member of a client object's own field, as the client wrote it (`self.app = app`) +.decl lcb_member_of_field(t:symbol, f:symbol, n:symbol, m:symbol) +lcb_member_of_field(t, f, n, m) :- + lcb_wanted_field(t, f, n), field_decl("client", f, owner, _, fh), + (owner = t ; type_ancestor("client", t, owner)), + field_type("client", fh, x), mro_lookup("client", x, n, m), method_owner("client", _, m). + +// demand: only the (expression, member) pairs a summary asks about are looked up +.decl lcb_wanted(e:symbol, n:symbol) +lcb_wanted(a, n) :- lcb_arg(site, q, r, a), py_lib_callback(q, r, _, "", n, _, _), lcb_entry(site, q, _). +lcb_wanted(e, n) :- lcb_entry(_, q, t), t != "", py_lib_callback(q, "self", _, f, n, _, _), f != "", + lcb_field_holds(t, f, e). +lcb_wanted(a, n) :- lcb_alloc_field(al, f, a), lcb_lobj_entry(_, q, al), py_lib_callback(q, "self", _, f, n, _, _). +lcb_wanted(e, n) :- lcb_alloc_field(al, f, e), lcb_lobj(a, _, al), lcb_arg(_, q, r, a), py_lib_callback(q, r, _, f, n, _, _). +lcb_wanted(e, n) :- lcb_arg(site, q, r, a), py_lib_callback(q, r, _, f, n, _, _), f != "", lcb_entry(site, q, _), + expr_type("client", a, t), lcb_field_holds(t, f, e). +.decl lcb_wanted_field(t:symbol, f:symbol, n:symbol) +lcb_wanted_field(s, f, n) :- lcb_entry(_, q, t), t != "", py_lib_callback(q, "self", _, f, n, _, _), f != "", + !contains(".", f), lcb_self_type(t, s). +lcb_wanted_field(t, f, n) :- lcb_arg(site, q, r, a), lcb_entry(site, q, _), py_lib_callback(q, r, _, f, n, _, _), f != "", + !contains(".", f), expr_type("client", a, t). + +// ── the edges ─────────────────────────────────────────────────────────────── +// lcb_hit(Site, Caller, Target, Callee, Root, Path, Member, Via, How): one summary row joined to the client object +// the call actually hands over. +.decl lcb_hit(site:symbol, caller:symbol, m:symbol, q:symbol, r:symbol, f:symbol, n:symbol, via:symbol, how:symbol) +// the receiver itself: a member the client type (or a subclass) defines over the library's +lcb_hit(site, caller, m, q, "self", "", n, via, "receiver") :- + lcb_entry(site, q, t), t != "", site_caller(site, caller), + py_lib_callback(q, "self", _, "", n, via, _), + lcb_self_type(t, s), mro_lookup("client", s, n, m), method_owner("client", _, m). +// a field of the receiver the library constructor filled from a client argument +lcb_hit(site, caller, m, q, "self", f, n, via, "receiver_field") :- + lcb_entry(site, q, t), t != "", site_caller(site, caller), + py_lib_callback(q, "self", _, f, n, via, _), f != "", + lcb_field_holds(t, f, e), lcb_member_of_expr(e, n, m). +// a field of a library object the client holds (built by it, or made by a library decorator around its def) +lcb_hit(site, caller, m, q, "self", f, n, via, "receiver_field") :- + lcb_lobj_entry(site, q, o), site_caller(site, caller), + py_lib_callback(q, "self", _, f, n, via, _), f != "", + lcb_lobj_field_target(o, f, n, m). +lcb_hit(site, caller, m, q, r, f, n, via, "argument_field") :- + lcb_arg(site, q, r, a), site_caller(site, caller), lcb_entry(site, q, _), + py_lib_callback(q, r, _, f, n, via, _), f != "", + lcb_lobj(a, _, o), lcb_lobj_field_target(o, f, n, m). +// a field of the receiver the client itself assigned +lcb_hit(site, caller, m, q, "self", f, n, via, "receiver_field") :- + lcb_entry(site, q, t), t != "", site_caller(site, caller), + py_lib_callback(q, "self", _, f, n, via, _), f != "", + lcb_self_type(t, s), lcb_member_of_field(s, f, n, m). +// an argument, or the callable passed as one +lcb_hit(site, caller, m, q, r, "", n, via, "argument") :- + lcb_arg(site, q, r, a), site_caller(site, caller), + py_lib_callback(q, r, _, "", n, via, _), + lcb_member_of_expr(a, n, m). +// a field of an argument +lcb_hit(site, caller, m, q, r, f, n, via, "argument_field") :- + lcb_arg(site, q, r, a), site_caller(site, caller), + py_lib_callback(q, r, _, f, n, via, _), f != "", + expr_type("client", a, t), lcb_field_holds(t, f, e), lcb_member_of_expr(e, n, m). +lcb_hit(site, caller, m, q, r, f, n, via, "argument_field") :- + lcb_arg(site, q, r, a), site_caller(site, caller), + py_lib_callback(q, r, _, f, n, via, _), f != "", + expr_type("client", a, t), lcb_member_of_field(t, f, n, m). + +// ── WHEN THE OBJECT CANNOT BE TOLD: THE TYPE THE LIBRARY DECLARES ──────────── +// A row whose object the client side could not name (a loader passed through **options, a command a helper +// returns typed only as the library's base) is still a hand-back to SOME object of the type the library declares +// there. The client classes that can be that object are its client subclasses; the one whose member runs is the one +// that defines it. A class-hierarchy answer, used only for a row no client object answered at that site. +.decl lcb_row_hit(site:symbol, q:symbol, r:symbol, f:symbol, n:symbol) +lcb_row_hit(site, q, r, f, n) :- lcb_hit(site, _, _, q, r, f, n, _, _). +.decl lcb_cha_wanted(ty:symbol, n:symbol) +lcb_cha_wanted(ty, n) :- lcb_entry(_, q, _), py_lib_callback(q, r, _, f, n, _, ty), ty != "", (r != "self" ; f != ""). +.decl lcb_cha(ty:symbol, n:symbol, m:symbol) +lcb_cha(ty, n, m) :- lcb_cha_wanted(ty, n), lcb_ext_class(s, ty), mro_lookup("client", s, n, m), method_owner("client", _, m). +lcb_cha(ty, n, m) :- lcb_cha_wanted(ty, n), lcb_ext_class(s, c), py_lib_ancestor(c, ty), + mro_lookup("client", s, n, m), method_owner("client", _, m). + +// lib_callback_edge(Caller, Target, Callee, Member, Via, How): the client method Caller calls the library +// callable Callee, which is summarised (from the library's source) to invoke Member -- written in the library +// function Via -- on an object that resolves to the client method Target. How: receiver | receiver_field | +// argument | argument_field | declared_type. A library's behaviour joined by a member name, not a resolved call. +lib_callback_edge(caller, m, q, n, via, how) :- lcb_hit(_, caller, m, q, _, _, n, via, how). +lib_callback_edge(caller, m, q, n, via, "declared_type") :- + lcb_entry(site, q, _), site_caller(site, caller), + py_lib_callback(q, r, _, f, n, via, ty), ty != "", (r != "self" ; f != ""), + !lcb_row_hit(site, q, r, f, n), lcb_cha(ty, n, m). + +// lib_callback_site(Site, Callee): a client call site the engine left unresolved that enters the library callable +// Callee, which the library's own source declares (a member the client type inherits from its library base, a +// super() call past the client classes, a member of a library object the client built). Not a call to any +// same-named client method, so a consumer must not guess one by name. +lib_callback_site(site, q) :- lcb_entry(site, q, _), q != "". diff --git a/graph/python/engine/resolution/annotations.dl b/graph/python/engine/resolution/annotations.dl index 0d671aa9..2b9087a4 100644 --- a/graph/python/engine/resolution/annotations.dl +++ b/graph/python/engine/resolution/annotations.dl @@ -199,6 +199,25 @@ type_ref_element(p, parentRef, t) :- annotation_owner_module(p, owner, ok, mod), type_name_in_module(p, mod, tn, t). +// ── union_operand(Prov, UnionRefHash, OperandRefHash) — every operand of a union ── +// `|` is LEFT-associative, so `A | B | None` is `(A | B) | None`: A and B are not +// children of the annotation but grandchildren, under a nested UNION_PEP604, and the +// depth-1 clauses above saw only that nested union (which names no type) and None. Every +// rule reading "Optional[X] IS X" therefore lost the members of any union with three or +// more operands, on a parameter, a return, a field or a local alike. +union_operand(p, r, c) :- + type_ref(p, k, _, _, _, r), annotation_optional_kind(k), type_ref_nesting(p, r, _, _, c). +union_operand(p, r, c) :- + union_operand(p, r, u), type_ref(p, "UNION_PEP604", _, _, _, u), type_ref_nesting(p, u, _, _, c). +// the operands BELOW depth 1 (depth 1 is the clauses above), resolved, and by name +type_ref_element(p, r, t) :- + union_operand(p, r, c), type_ref_nesting(p, _, _, d, c), d != "1", type_ref_resolved(p, t, c). +type_ref_element(p, r, t) :- + union_operand(p, r, c), type_ref_nesting(p, _, _, d, c), d != "1", + type_ref(p, _, "GENERIC_ARGUMENT", tn, _, c), tn != "", + type_ref_owner(p, owner, ok, r), annotation_owner_module(p, owner, ok, mod), + type_name_in_module(p, mod, tn, t). + // ── annotation_owner_module(Prov, OwnerHash, OwnerKind, ModuleHash) ────────── // The module an annotation was written in, whatever kind of declaration owns it. annotation_owner_module(p, ph, "METHOD_PARAM", mod) :- diff --git a/graph/python/engine/resolution/attribute-lookup.dl b/graph/python/engine/resolution/attribute-lookup.dl index 3c032d13..7b44fe2d 100644 --- a/graph/python/engine/resolution/attribute-lookup.dl +++ b/graph/python/engine/resolution/attribute-lookup.dl @@ -317,6 +317,15 @@ type_call_target(p, t, m) :- mro_lookup(p, t, "__call__", m). type_property_getter(p, t, n, m) :- mro_lookup(p, t, n, m), method_kind(p, "PROPERTY_GETTER", _, m). +// ── type_property_accessor(Prov, TypeHash, Name, Kind, MethodHash) ─────────── +// `@x.setter` and `@x.deleter` are the other two halves: `obj.x = v` runs the setter and +// `del obj.x` the deleter, and like the getter neither has a call site. They share the +// getter's name and binding, so they are found on the class that WINS the name in the +// receiver's MRO, by their method kind, rather than through mro_lookup's one answer. +type_property_accessor(p, t, n, k, m) :- + mro_winner(p, t, n, c), method_owner(p, c, m), method_decl(p, n, _, _, _, m), + method_kind(p, k, _, m), (k = "PROPERTY_SETTER" ; k = "PROPERTY_DELETER"). + // ── A USER-WRITTEN DATA DESCRIPTOR IS THE SAME PROTOCOL (issue #326) ───────── // `@property` IS a data descriptor; the decorator is sugar over `__get__`/`__set__`. So // the clause above reads one spelling of the protocol and a class-level diff --git a/graph/python/engine/resolution/builtins.dl b/graph/python/engine/resolution/builtins.dl index ca1e14d1..02da0163 100644 --- a/graph/python/engine/resolution/builtins.dl +++ b/graph/python/engine/resolution/builtins.dl @@ -182,6 +182,18 @@ py_copy_module("copy"). py_copy_function("copy"). py_copy_function("deepcopy"). +// ── py_returns_arg(DottedName, Position) — a library call that hands back an argument ── +// Each returns, unchanged, the argument at Position (documented behaviour, not inference): +// functools.update_wrapper(wrapper, wrapped) -> wrapper (copies __name__/__doc__ onto it) +// typing.cast(T, value) -> value (a no-op at run time) +// A decorator written `return update_wrapper(wrapper, f)`, or the typed spelling +// `return t.cast(F, update_wrapper(wrapper, f))`, is therefore the same shape as one +// returning `wrapper` under @functools.wraps. Without these every method it decorates was +// decorator_replaced_target, and no call through the attribute reached anything +// (call-edge-generation/call_chain.dl). Neither module is staged in a client-only run. +py_returns_arg("functools.update_wrapper", "0"). +py_returns_arg("typing.cast", "1"). + // ── py_builtin_dynamic(Name) — the escape hatches ──────────────────────────── // These do not merely lack Python source; they make the PROGRAM unanalysable at that // point. A call to getattr/eval/exec means the engine cannot know what runs, and the diff --git a/graph/python/engine/resolution/value-flow.dl b/graph/python/engine/resolution/value-flow.dl index 14ef74b5..69fb4ae4 100644 --- a/graph/python/engine/resolution/value-flow.dl +++ b/graph/python/engine/resolution/value-flow.dl @@ -97,6 +97,21 @@ param_arg_type(ph, t) :- param_decl("client", kw, _, _, m, ph), expr_type("client", a, t). +// ── a parameter the TEST RUNNER fills ───────────────────────────────────────── +// pytest calls `test_commit(session=)`: the call is the runner's, +// so no call site spells it, but it is an argument reaching a parameter all the same, and +// the parameter is typed exactly as one passed at a call site would be. Which fixture +// serves the parameter is framework-behavior/dispatch.dl's py_fixs_injection (2a), the +// runner's own lookup (same class, same module, nearest conftest, a pytest_plugins +// module), so this is as deterministic as a name the LEGB walk resolves. +// MEASURED across ten public suites: 1,529 method calls on such parameters were +// ambiguous_unknown, and everything derived from them (`tx = session.begin(); tx.commit()`) +// with them; a suite whose fixtures carry annotations resolved its own. +param_arg_type(ph, t) :- + py_fixs_injection(req, fix, pn), + param_decl("client", pn, _, _, req, ph), + py_fixture_value_type(fix, t). + // ── param_arg_method(ParamHash, MethodHash) — a CALLABLE reaching a parameter ── // `Delegator(target)` puts the FUNCTION `target` into the parameter `fn`. Java would // need a functional interface for this; in Python it is an ordinary assignment. diff --git a/graph/python/libsum/libsum.py b/graph/python/libsum/libsum.py new file mode 100644 index 00000000..e910d123 --- /dev/null +++ b/graph/python/libsum/libsum.py @@ -0,0 +1,1090 @@ +#!/usr/bin/env python3 +"""libsum.py -- what an installed Python library does with the objects a client hands it. + + libsum.py --src --out [--site [,...]] [--cache ] + +THE GAP THIS CLOSES. The engine reads a dependency's signatures, never its bodies, so a hop the +LIBRARY makes back into client code is invisible: a test client's `get()` calls the `open()` a +client subclass overrides, a CLI runner calls `main()` on the command it was handed, a test client +built around an application calls `app(environ, start_response)` on it. Every one of those is a +library frame between two client frames, and the call graph stops at the first. + +WHAT A SUMMARY SAYS. One row per hand-back a library callable is known to make, rooted at one of +its parameters (or at `self`, the object it was called on): + + py-lib-callback.csv callee root pos path member via type + callee the library callable, canonical dotted name: `pkg.mod.func`, `pkg.mod.Class.method` + (one row per method a class HAS, its inherited ones included), `pkg.mod.Class` for + constructing it + root the parameter the object arrives in, or `self` + pos where the CLIENT writes that argument: the index among the positional arguments of + the call as written (an instance method's `self` is not written), -1 for `self` + path the attribute chain the library follows from that object first ("" for the object + itself, `application` for `self.application`), at most two long + member the method the library invokes on what it reached: a name it calls, or the dunder a + syntax runs (`__call__` for `x()`, `__enter__` for `with x`, `__iter__` for `for`...) + via the library function in which that invocation is written + type the class the LIBRARY declares that object to be (a parameter annotation, a field's), "" when it + declares none: when the client's own object cannot be told, a client subclass of it is the answer + py-lib-field-init.csv class field param pos + constructing `class` stores the argument `param` (written at `pos`) on the new object as + `field`, so a later `self.` hand-back reaches what the client passed + py-lib-ancestor.csv class ancestor + a library class's library ancestors, for the classes the client can name + py-lib-alias.csv alias canonical + a name a package re-exports (`clikit.Command` is `clikit.core.Command`) + py-lib-decorates.csv decorator form class field + applying the library callable as a decorator -- `@decorator` (DIRECT) or `@decorator(...)` (FACTORY) -- + to a function f yields a new `class` whose `field` holds f; class "" when f itself comes back + +HOW, AND WHY NOT IN THE SOLVE. Library source is just Python, so the facts are DERIVED from it by a +small flow-insensitive pass over each function body (which parameter-rooted value meets which call or +protocol syntax), composed to a fixpoint across the library's own calls: `Client.get` calls +`self.open`, `open` calls `run_app(self.application, ...)`, `run_app` calls `app(...)`, so +`Client.get` hands back `__call__` on `self.application`. Nothing is solved over library bodies at +index time: the pass reads only the packages the client imports (and what they import), is cached by +package and version, and writes three small tables. A library written in C has no body to read and +yields nothing; that is a declared blind spot, not a guess. + +WHERE THE LIBRARY IS. The project's own virtual environment (`.venv`, `venv`, `env` in the repository, +else $VIRTUAL_ENV), its stdlib through pyvenv.cfg, or AXIOMCODE_PY_SITE (a comma list of directories) +when set. No environment found means no summaries: the engine then behaves exactly as before. +""" +import argparse, ast, csv, hashlib, json, os, re, sys + +VERSION = '1' +MAXPATH = 2 +DUNDER_BUILTIN = {'len': '__len__', 'str': '__str__', 'repr': '__repr__', 'iter': '__iter__', 'next': '__next__', + 'hash': '__hash__', 'bool': '__bool__', 'format': '__format__', 'reversed': '__reversed__', + 'abs': '__abs__', 'aiter': '__aiter__', 'anext': '__anext__'} +SKIP_DIRS = {'__pycache__', 'tests', 'test', 'testing_support', 'docs', 'examples', 'benchmarks'} + + +# ───────────────────────────────────────────────────────────────────────────── +# where the library is +# ───────────────────────────────────────────────────────────────────────────── +def find_sites(src): + if os.environ.get('AXIOMCODE_PY_SITE'): + return [d for d in os.environ['AXIOMCODE_PY_SITE'].split(',') if os.path.isdir(d)], None + envs = [os.path.join(src, n) for n in ('.venv', 'venv', 'env', '.env')] + if os.environ.get('VIRTUAL_ENV'): + envs.append(os.environ['VIRTUAL_ENV']) + for e in envs: + cfg = os.path.join(e, 'pyvenv.cfg') + if not os.path.isfile(cfg): + continue + sites = [] + for root in (os.path.join(e, 'lib'), os.path.join(e, 'Lib')): + if not os.path.isdir(root): + continue + if os.path.isdir(os.path.join(root, 'site-packages')): + sites.append(os.path.join(root, 'site-packages')) + for d in sorted(os.listdir(root)): + sp = os.path.join(root, d, 'site-packages') + if d.startswith('python') and os.path.isdir(sp): + sites.append(sp) + stdlib = None + try: + kv = dict(l.split('=', 1) for l in open(cfg, encoding='utf-8') if '=' in l) + kv = {k.strip(): v.strip() for k, v in kv.items()} + home, ver = kv.get('home', ''), (kv.get('version_info') or kv.get('version') or '') + mm = '.'.join(ver.split('.')[:2]) + for cand in (os.path.join(os.path.dirname(home), 'lib', f'python{mm}'), os.path.join(home, 'Lib'), + os.path.join(os.path.dirname(home), 'Lib')): + if mm and os.path.isfile(os.path.join(cand, 'contextlib.py')): + stdlib = cand; break + except Exception: + pass + if sites: + return sites, stdlib + return [], None + + +# ───────────────────────────────────────────────────────────────────────────── +# which modules: the client's imports, then what those import, transitively +# ───────────────────────────────────────────────────────────────────────────── +IMPORT_RE = re.compile(r'^\s*(?:from\s+([A-Za-z_][\w.]*)\s+import|import\s+([A-Za-z_][\w.]*(?:\s*,\s*[A-Za-z_][\w.]*)*))', re.M) + + +def client_imports(src, ir): + """every dotted path the client imports: `from a.b import C` gives a.b.C, `import a.b` gives a.b. From the + parser's import table when the IR is there, else from the source""" + paths = set() + p = os.path.join(ir, 'all-python-imports.csv') if ir else '' + if p and os.path.isfile(p): + try: + with open(p, newline='', encoding='utf-8') as fh: + r = csv.reader(fh, delimiter='\t') + head = next(r) + if 'importedPath' in head: + col = head.index('importedPath') + for row in r: + if len(row) > col and row[col]: + paths.add(row[col]) + return paths + except Exception: + pass + skip = {'.venv', 'venv', 'env', '.env', '.git', 'node_modules', '.tox', '__pycache__', '.axiomcode'} + for d, ds, fs in os.walk(src): + ds[:] = [x for x in ds if x not in skip] + for f in fs: + if not f.endswith('.py'): + continue + try: + tree = ast.parse(open(os.path.join(d, f), encoding='utf-8', errors='replace').read()) + except Exception: + continue + for n in ast.walk(tree): + if isinstance(n, ast.Import): + paths.update(a.name for a in n.names) + elif isinstance(n, ast.ImportFrom) and n.module and not n.level: + paths.update(f'{n.module}.{a.name}' for a in n.names if a.name != '*') + paths.add(n.module) + return paths + + +class Package: + """one importable top-level name: a directory package or a single module file""" + def __init__(self, name, path, base): + self.name, self.path, self.base = name, path, base + + def files(self): + if os.path.isfile(self.path): + yield self.name, self.path + return + for d, ds, fs in os.walk(self.path): + ds[:] = sorted(x for x in ds if x not in SKIP_DIRS and not x.startswith('.')) + for f in sorted(fs): + if not f.endswith('.py'): + continue + rel = os.path.relpath(os.path.join(d, f), self.base)[:-3].replace(os.sep, '.') + if rel.endswith('.__init__'): + rel = rel[:-9] + yield rel, os.path.join(d, f) + + +def locate(name, sites, stdlib): + for base in list(sites) + ([stdlib] if stdlib else []): + p = os.path.join(base, name) + if os.path.isdir(p) and (os.path.isfile(os.path.join(p, '__init__.py')) or base != stdlib): + return Package(name, p, base) + if os.path.isfile(p + '.py'): + return Package(name, p + '.py', base) + return None + + +# ───────────────────────────────────────────────────────────────────────────── +# one module: its namespace, its classes, and the events in each function body +# ───────────────────────────────────────────────────────────────────────────── +class Fn: + __slots__ = ('q', 'params', 'pos', 'kind', 'events', 'cls', 'stores', 'ptypes', 'rets', 'local') + + def __init__(self, q, cls): + self.q, self.cls, self.ptypes, self.rets, self.local = q, cls, {}, [], False + self.params, self.pos, self.kind = [], {}, 'function' + self.events = [] # ('use', val, member) | ('call', callee, selfval, {param: [vals]}, [posvals], {kw: [vals]}) + self.stores = [] # (field, val) written on self + + +class Cls: + __slots__ = ('q', 'bases', 'methods', 'mod', 'ftypes') + + def __init__(self, q, mod): + self.q, self.mod, self.bases, self.methods, self.ftypes = q, mod, [], {}, {} + + +def resolve_rel(mod, is_pkg, level, name): + parts = mod.split('.') + if not is_pkg: + parts = parts[:-1] + if level > 1: + parts = parts[:len(parts) - (level - 1)] + return '.'.join([p for p in parts if p] + ([name] if name else [])) + + +class Module: + def __init__(self, name, path, is_pkg): + self.name, self.path, self.is_pkg = name, path, is_pkg + self.ns = {} # local name -> qualified name (a def, a class, an imported thing) + self.fns, self.classes = {}, {} + self.imports = set() + + def load(self): + try: + tree = ast.parse(open(self.path, encoding='utf-8', errors='replace').read()) + except Exception: + return False + self.tree = tree + for st in tree.body: + self._ns_stmt(st) + return True + + def _ns_stmt(self, st): + if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + self.ns[st.name] = f'{self.name}.{st.name}' + elif isinstance(st, ast.ImportFrom): + base = resolve_rel(self.name, self.is_pkg, st.level, st.module or '') if st.level else (st.module or '') + if base: + self.imports.add(base) + for a in st.names: + if a.name != '*': + self.ns[a.asname or a.name] = f'{base}.{a.name}' if base else a.name + elif isinstance(st, ast.Import): + for a in st.names: + self.imports.add(a.name) + if a.asname: + self.ns[a.asname] = a.name + else: + self.ns.setdefault(a.name.split('.')[0], a.name.split('.')[0]) + elif isinstance(st, ast.Assign) and len(st.targets) == 1 and isinstance(st.targets[0], ast.Name): + v = dotted(st.value) + if v and v.split('.')[0] in self.ns: + self.ns.setdefault(st.targets[0].id, self.ns[v.split('.')[0]] + v[len(v.split('.')[0]):]) + elif isinstance(st, (ast.If, ast.Try)): + for b in [st.body, getattr(st, 'orelse', []), getattr(st, 'finalbody', [])] + [h.body for h in getattr(st, 'handlers', [])]: + for s in b: + self._ns_stmt(s) + + def qual(self, n): + return self.ns.get(n) + + def analyse(self): + for st in self.tree.body: + if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef)): + self._fn(st, None) + elif isinstance(st, ast.ClassDef): + c = Cls(f'{self.name}.{st.name}', self) + c.bases = [b for b in (self.qual_expr(x) for x in st.bases) if b] + self.classes[c.q] = c + for s in st.body: + if isinstance(s, ast.AnnAssign) and isinstance(s.target, ast.Name): + t = self.ann_qual(s.annotation) + if t: + c.ftypes[s.target.id] = t + for s in st.body: + if isinstance(s, (ast.FunctionDef, ast.AsyncFunctionDef)): + f = self._fn(s, c) + c.methods[s.name] = f.q + + def ann_qual(self, ann): + """the class an annotation names: `T`, `pkg.T`, `"T"`, `Optional[T]`, `T | None`""" + if isinstance(ann, ast.Constant) and isinstance(ann.value, str): + try: + ann = ast.parse(ann.value, mode='eval').body + except SyntaxError: + return None + if isinstance(ann, ast.BinOp): + return self.ann_qual(ann.left) + if isinstance(ann, ast.Subscript) and (dotted(ann.value) or '').split('.')[-1] == 'Optional': + return self.ann_qual(ann.slice) + return self.qual_expr(ann) if ann is not None else None + + def qual_expr(self, e): + d = dotted(e.value if isinstance(e, ast.Subscript) else e) + if not d: + return None + head, _, rest = d.partition('.') + q = self.qual(head) + return (q + ('.' + rest if rest else '')) if q else None + + def _fn(self, node, cls): + q = f'{cls.q}.{node.name}' if cls else f'{self.name}.{node.name}' + f = self.signature(node, q, cls) + self.fns[q] = f + Body(self, f, f.params[0] if f.kind in ('method', 'classmethod', 'property') and f.params else None, cls).run(node) + return f + + def signature(self, node, q, cls): + f = Fn(q, cls.q if cls else None) + decos = {dotted(d.func if isinstance(d, ast.Call) else d) for d in node.decorator_list} + a = node.args + names = [x.arg for x in a.posonlyargs + a.args] + if cls is not None: + if 'staticmethod' in decos: + f.kind = 'static' + elif 'classmethod' in decos: + f.kind = 'classmethod' + elif 'property' in decos or any(d and d.endswith(('.setter', '.getter', 'cached_property')) for d in decos): + f.kind = 'property' + else: + f.kind = 'method' + f.params = names + [x.arg for x in a.kwonlyargs] + skip = 1 if f.kind in ('method', 'classmethod', 'property') and names else 0 + for i, n in enumerate(names): + f.pos[n] = i - skip if i >= skip else -1 + for x in a.kwonlyargs: + f.pos[x.arg] = -2 # keyword only: matched by name + for x in a.posonlyargs + a.args + a.kwonlyargs: + if x.annotation is not None: + t = self.ann_qual(x.annotation) + if t: + f.ptypes[x.arg] = t + return f + + +def dotted(e): + if isinstance(e, ast.Name): + return e.id + if isinstance(e, ast.Attribute): + d = dotted(e.value) + return f'{d}.{e.attr}' if d else None + return None + + +class Body: + """a flow-insensitive pass over one function body (nested defs and lambdas included: a closure the + library builds around a parameter is how a decorator or a callback wrapper hands it back)""" + def __init__(self, mod, fn, selfname, cls): + self.mod, self.fn, self.selfname, self.cls = mod, fn, selfname, cls + self.env = {} + self.params = set(fn.params) + + def run(self, node): + for _ in range(2): # twice: a local assigned below its first use + self.fn.events = []; self.fn.stores = []; self.fn.rets = [] + for st in node.body: + self.stmt(st) + return self + + # values: ('p', root, path) | ('ref', qname) | ('new', class, argvals) | ('self', ...) is ('p', 'self', ()) + def val(self, e): + if isinstance(e, ast.Name): + if e.id in self.env: + return self.env[e.id] + if e.id in self.params: + return [('p', 'self' if e.id == self.selfname else e.id, ())] + q = self.mod.qual(e.id) + return [('ref', q)] if q else [] + if isinstance(e, ast.Attribute): + out = [] + for v in self.val(e.value): + if v[0] == 'p' and len(v[2]) < MAXPATH: + out.append(('p', v[1], v[2] + (e.attr,))) + elif v[0] == 'ref': + out.append(('ref', f'{v[1]}.{e.attr}')) + elif v[0] == 'new': + out.append(('field', v, e.attr)) + return out + if isinstance(e, ast.Call): + return self.call(e) + if isinstance(e, (ast.IfExp,)): + self.val(e.test) + return self.val(e.body) + self.val(e.orelse) + if isinstance(e, ast.BoolOp): + out = [] + for x in e.values: + out += self.val(x) + return out + if isinstance(e, ast.NamedExpr): + v = self.val(e.value) + if isinstance(e.target, ast.Name): + self.env[e.target.id] = v + return v + if isinstance(e, ast.Await): + return self.val(e.value) + if isinstance(e, ast.Subscript): + for v in self.val(e.value): + self.use(v, '__getitem__') + self.val(e.slice) + return [] + if isinstance(e, ast.Compare): + self.val(e.left) + for op, c in zip(e.ops, e.comparators): + cv = self.val(c) + if isinstance(op, (ast.In, ast.NotIn)): + for v in cv: + self.use(v, '__contains__') + return [] + if isinstance(e, ast.JoinedStr): + for x in e.values: + if isinstance(x, ast.FormattedValue): + m = {115: '__str__', 114: '__repr__', 97: '__repr__'}.get(x.conversion, '__format__') + for v in self.val(x.value): + self.use(v, m) + return [] + if isinstance(e, ast.Lambda): + self.nested(e.body if isinstance(e.body, list) else [ast.Expr(e.body)], e.args) + return [] + if isinstance(e, (ast.ListComp, ast.SetComp, ast.GeneratorExp, ast.DictComp)): + for g in e.generators: + for v in self.val(g.iter): + self.use(v, '__iter__') + for c in g.ifs: + self.val(c) + for x in ([e.key, e.value] if isinstance(e, ast.DictComp) else [e.elt]): + self.val(x) + return [] + for ch in ast.iter_child_nodes(e): + if isinstance(ch, ast.expr): + self.val(ch) + return [] + + def use(self, v, member): + if member == '__call__' and v == ('p', 'self', ()) and self.fn.kind == 'classmethod': + member = '__init__' # cls(...) constructs the class the method was called on + if v[0] == 'p' or v[0] == 'field': + # what the library itself declares the object to be, where it says: a parameter's annotation, a field's + hint = None + if v[0] == 'p': + base = self.cls.q if v[1] == 'self' and self.cls else self.fn.ptypes.get(v[1]) + if base and not v[2]: + hint = ('t', base) + elif base and len(v[2]) == 1: + hint = ('f', base, v[2][0]) + self.fn.events.append(('use', v, member, hint)) + + def call(self, e): + f = e.func + posv = [self.val(a) if not isinstance(a, ast.Starred) else None for a in e.args] + kwv = {k.arg: self.val(k.value) for k in e.keywords if k.arg} + for k in e.keywords: + if not k.arg: + self.val(k.value) + if isinstance(f, ast.Name) and f.id in DUNDER_BUILTIN and f.id not in self.env and not self.mod.qual(f.id) and posv and posv[0]: + for v in posv[0]: + self.use(v, DUNDER_BUILTIN[f.id]) + return [] + if isinstance(f, ast.Name) and f.id == 'super' and not e.args: + return [('super',)] + fd = dotted(f) or '' + if fd.split('.')[-1] == 'cast' and len(posv) == 2 and posv[1]: + return posv[1] # typing.cast(T, x) is x + if fd == 'type' and len(posv) == 1 and posv[0]: + return posv[0] # type(x).__enter__ is looked up on x's class: x's member + if isinstance(f, ast.Attribute): + out = [] + for rv in self.val(f.value): + if rv[0] in ('p', 'field'): + self.use(rv, f.attr) + self.fn.events.append(('call', ('method', rv, f.attr), posv, kwv)) + elif rv[0] == 'super': + self.fn.events.append(('call', ('super', self.cls.q if self.cls else None, f.attr), posv, kwv)) + elif rv[0] == 'new': + self.fn.events.append(('call', ('method', rv, f.attr), posv, kwv)) + elif rv[0] == 'ref': + self.fn.events.append(('call', ('ref', f'{rv[1]}.{f.attr}'), posv, kwv)) + return out + out = [] + for cv in self.val(f): + if cv[0] in ('p', 'field'): + self.use(cv, '__call__') + if cv[0] == 'p' and cv[2]: + # a member read first and called later (`_enter = type(cm).__enter__; _enter(cm)`) + self.use(('p', cv[1], cv[2][:-1]), cv[2][-1]) + elif cv[0] == 'ref': + self.fn.events.append(('call', ('ref', cv[1]), posv, kwv)) + out.append(('new', cv[1], self.args(posv, kwv))) + elif cv[0] == 'closure': + self.fn.events.append(('call', ('ref', cv[1]), posv, kwv)) + out.append(('res', cv[1], self.args(posv, kwv))) + elif cv[0] in ('new', 'res', 'res2'): + out.append(('res2', cv, self.args(posv, kwv))) # calling what a call returned: `command(...)(f)` + return out + + @staticmethod + def args(posv, kwv): + return (tuple(tuple(p) if p is not None else None for p in posv), tuple(sorted((k, tuple(v)) for k, v in kwv.items()))) + + def nested(self, body, args): + saved = set(self.params) + for a in args.posonlyargs + args.args + args.kwonlyargs + [x for x in (args.vararg, args.kwarg) if x]: + self.params.discard(a.arg); self.env.pop(a.arg, None) + for st in body: + self.stmt(st) + self.params = saved + + def assign(self, t, vals): + if isinstance(t, ast.Name): + # flow-insensitive: a parameter rebound keeps what it arrived as (`callback = other.callback`) + base = self.env.get(t.id) or ([('p', 'self' if t.id == self.selfname else t.id, ())] if t.id in self.params else []) + self.env[t.id] = list(dict.fromkeys(base + vals)) + elif isinstance(t, ast.Attribute): + for ov in self.val(t.value): + if ov == ('p', 'self', ()): + for v in vals: + self.fn.stores.append((t.attr, v)) + if self.cls and v[0] == 'p' and not v[2] and v[1] in self.fn.ptypes: + self.cls.ftypes.setdefault(t.attr, self.fn.ptypes[v[1]]) + elif isinstance(t, (ast.Tuple, ast.List)): + for x in t.elts: + self.assign(x, []) + elif isinstance(t, ast.Subscript): + for v in self.val(t.value): + self.use(v, '__setitem__') + + def stmt(self, st): + if isinstance(st, (ast.FunctionDef, ast.AsyncFunctionDef)): + self.nested(st.body, st.args) + # and as a callable of its own, so what it RETURNS is known: a decorator factory returns this closure, + # and the closure returns the object it wrapped the decorated function in + sq = f'{self.fn.q}..{st.name}' + sub = self.mod.signature(st, sq, None) + sub.local = True + self.mod.fns[sq] = sub + b = Body(self.mod, sub, None, None) + b.env = {k: list(v) for k, v in self.env.items() if k not in set(sub.params)} + b.run(st) + self.env[st.name] = [('closure', sq)] + return + if isinstance(st, ast.Return): + if st.value is not None: + self.fn.rets.extend(self.val(st.value)) + return + if isinstance(st, ast.ClassDef): + return + if isinstance(st, ast.Assign): + v = self.val(st.value) + for t in st.targets: + self.assign(t, v) + return + if isinstance(st, ast.AnnAssign) and isinstance(st.target, ast.Attribute) and self.cls \ + and isinstance(st.target.value, ast.Name) and st.target.value.id == self.selfname: + t = self.mod.ann_qual(st.annotation) + if t: + self.cls.ftypes[st.target.attr] = t + if isinstance(st, (ast.AnnAssign, ast.AugAssign)): + if st.value is not None: + v = self.val(st.value) + self.assign(st.target, v if isinstance(st, ast.AnnAssign) else []) + return + if isinstance(st, (ast.For, ast.AsyncFor)): + for v in self.val(st.iter): + self.use(v, '__aiter__' if isinstance(st, ast.AsyncFor) else '__iter__') + self.assign(st.target, []) + if isinstance(st, (ast.With, ast.AsyncWith)): + a = isinstance(st, ast.AsyncWith) + for it in st.items: + vs = self.val(it.context_expr) + for v in vs: + self.use(v, '__aenter__' if a else '__enter__'); self.use(v, '__aexit__' if a else '__exit__') + if it.optional_vars is not None: + self.assign(it.optional_vars, []) + if isinstance(st, ast.Delete): + for t in st.targets: + if isinstance(t, ast.Subscript): + for v in self.val(t.value): + self.use(v, '__delitem__') + return + for fld, ch in ast.iter_fields(st): + if fld in ('target', 'targets', 'iter', 'items') and isinstance(st, (ast.For, ast.AsyncFor, ast.With, ast.AsyncWith)): + continue + for c in (ch if isinstance(ch, list) else [ch]): + if isinstance(c, ast.stmt): + self.stmt(c) + elif isinstance(c, ast.expr): + self.val(c) + elif isinstance(c, ast.excepthandler): + for s in c.body: + self.stmt(s) + elif isinstance(c, ast.match_case) if hasattr(ast, 'match_case') else False: + for s in c.body: + self.stmt(s) + + +# ───────────────────────────────────────────────────────────────────────────── +# the program: every analysed module, then the fixpoint across the library's own calls +# ───────────────────────────────────────────────────────────────────────────── +class Program: + def __init__(self): + self.mods, self.fns, self.classes = {}, {}, {} + + def add(self, m): + self.mods[m.name] = m + self.fns.update(m.fns); self.classes.update(m.classes) + + def canon(self, q, seen=None): + """follow a module re-export to the declaration: clikit.Command -> clikit.core.Command""" + if q in self.fns or q in self.classes: + return q + seen = seen or set() + if q in seen or not q: + return None + seen.add(q) + parts = q.split('.') + for i in range(len(parts) - 1, 0, -1): + m = self.mods.get('.'.join(parts[:i])) + if m is None: + continue + nxt = m.qual(parts[i]) + if nxt and nxt != '.'.join(parts[:i + 1]): + return self.canon('.'.join([nxt] + parts[i + 1:]), seen) + if i + 1 < len(parts): # Class.method + c = self.canon('.'.join(parts[:i + 1]), seen) + if c in self.classes: + r = self.lookup(c, parts[i + 1]) + return r if len(parts) == i + 2 else None + return None + return None + + def type_of(self, hint): + """the canonical library class a use hint names: ('t', T) the object's own declared type, ('f', B, f) the + declared type of field f on B (annotated in the class, or assigned from an annotated parameter)""" + if not hint: + return '' + if hint[0] == 't': + c = self.canon(hint[1]) + return c if c in self.classes else '' + b = self.canon(hint[1]) + if b not in self.classes: + return '' + for k in self.mro(b): + t = self.classes[k].ftypes.get(hint[2]) + if t: + c = self.canon(t) or self.canon(f'{self.classes[k].mod.name}.{t}') + return c if c in self.classes else '' + return '' + + def mro(self, c, seen=None): + seen = seen if seen is not None else set() + if c in seen: + return [] + seen.add(c) + out = [c] + k = self.classes.get(c) + for b in (k.bases if k else []): + bc = self.canon(b) + if bc in self.classes: + for x in self.mro(bc, seen): + if x not in out: + out.append(x) + return out + + def lookup(self, c, name): + for k in self.mro(c): + m = self.classes[k].methods.get(name) + if m: + return m + return None + + def solve(self): + S = {q: set() for q in self.fns} # (root, path, member, via) + ST = {c: set() for c in self.classes} # class -> (field, init param) + for f in self.fns.values(): + for ev in f.events: + if ev[0] == 'use' and ev[1][0] == 'p' and (ev[1][1] == 'self' or ev[1][1] in f.pos): + S[f.q].add((ev[1][1], ev[1][2], ev[2], f.q, self.type_of(ev[3]))) + if f.cls and f.q.endswith('.__init__'): + for fld, v in f.stores: + if v[0] == 'p' and v[2] == () and v[1] in f.pos: + ST[f.cls].add((fld, v[1])) + + def init_of(c): + return self.lookup(c, '__init__') + + def stores(c): + out = set() + for k in self.mro(c): + out |= ST.get(k, set()) + return out + + def bind(callee, posv, kwv): + """callee parameter -> the caller's values""" + g = self.fns.get(callee) + if not g: + return {} + out = {} + byp = {i: n for n, i in g.pos.items() if i >= 0} + for i, vs in enumerate(posv): + if vs is not None and i in byp: + out.setdefault(byp[i], []).extend(vs) + for k, vs in kwv.items(): + if k in g.pos: + out.setdefault(k, []).extend(vs) + return out + + def lift(v, path, depth=0): + """a value reached in the caller, followed down `path`: param-rooted values only""" + if v[0] == 'p': + p = v[2] + tuple(path) + return [(v[1], p)] if len(p) <= MAXPATH else [] + if v[0] == 'field' and depth < 3: + return [x for w in field_vals(v[1], v[2]) for x in lift(w, path, depth + 1)] + if v[0] == 'new' and path and depth < 3: + return [x for w in field_vals(v, path[0]) for x in lift(w, path[1:], depth + 1)] + return [] + + def field_vals(newv, fld): + c = self.canon(newv[1]) + if c not in self.classes: + return [] + ini = init_of(c) + if not ini: + return [] + posv = [list(x) if x is not None else None for x in newv[2][0]] + kwv = {k: list(v) for k, v in newv[2][1]} + b = bind(ini, posv, kwv) + return [w for f, pn in stores(c) if f == fld for w in b.get(pn, [])] + + changed, rounds = True, 0 + while changed and rounds < 12: + changed, rounds = False, rounds + 1 + for f in self.fns.values(): + acc = S[f.q]; n0 = len(acc) + for ev in f.events: + if ev[0] == 'use' and ev[1][0] == 'field': + for root, path in lift(ev[1], ()): + if root == 'self' or root in f.pos: + acc.add((root, path, ev[2], f.q, '')) + continue + if ev[0] != 'call': + continue + tgt, posv, kwv = ev[1], ev[2], ev[3] + targets = [] # (callee, selfvals) + if tgt[0] == 'ref': + q = self.canon(tgt[1]) + if q in self.fns: + targets.append((q, [])) + elif q in self.classes and init_of(q): + targets.append((init_of(q), [])) + elif tgt[0] == 'method': + rv, name = tgt[1], tgt[2] + if rv == ('p', 'self', ()) and f.cls: + m = self.lookup(f.cls, name) + if m: + targets.append((m, [rv])) + elif rv[0] == 'p' and rv[2] == () and rv[1] in f.ptypes: + c = self.canon(f.ptypes[rv[1]]) + m = self.lookup(c, name) if c in self.classes else None + if m: + targets.append((m, [rv])) + elif rv[0] == 'p' and len(rv[2]) == 1: + # `self.loader.load(...)`: the field's declared type says whose `load` runs + base = f.cls if rv[1] == 'self' else f.ptypes.get(rv[1]) + c = self.type_of(('f', base, rv[2][0])) if base else '' + m = self.lookup(c, name) if c else None + if m: + targets.append((m, [rv])) + elif rv[0] == 'new': + c = self.canon(rv[1]) + if c in self.classes and self.lookup(c, name): + targets.append((self.lookup(c, name), [rv])) + elif tgt[0] == 'super' and tgt[1]: + for k in self.mro(tgt[1])[1:]: + m = self.classes[k].methods.get(tgt[2]) + if m: + targets.append((m, [('p', 'self', ())])); break + for callee, selfv in targets: + b = bind(callee, posv, kwv) + for (r, p, mem, via, ty) in list(S.get(callee, ())): + src = selfv if r == 'self' else b.get(r, []) + for v in src: + for root, path in lift(v, p): + if root == 'self' or root in f.pos: + acc.add((root, path, mem, via, ty)) + if callee.endswith('.__init__'): + g = self.fns[callee] + # super().__init__(x) / Base.__init__(self, x): what the base stores, this class stores + if f.cls and f.q.endswith('.__init__') and (selfv == [('p', 'self', ())] or tgt[0] == 'super'): + for fld, pn in stores(g.cls): + for v in b.get(pn, []): + if v[0] == 'p' and v[2] == () and v[1] in f.pos and (fld, v[1]) not in ST[f.cls]: + ST[f.cls].add((fld, v[1])); changed = True + if len(acc) != n0: + changed = True + self.S, self.ST = S, ST + return rounds + + def returns(self): + """R[f]: what f's return value is, as far as a decorator needs it -- + ('obj', C, field, param) a new C whose `field` holds f's parameter `param` + ('closure', D) the nested function D (a decorator factory's decorator) + ('param', param) f's own parameter, unchanged (an identity / registering decorator)""" + R = {q: set() for q in self.fns} + + def bindv(g, args): + posv = [list(x) if x is not None else None for x in args[0]] + return self._bind(g, posv, {k: list(v) for k, v in args[1]}) + + def own(f, w): + return w[0] == 'p' and w[2] == () and w[1] in f.pos and w[1] != 'self' + + def apply(f, g, args, out): + b = bindv(g, args) + for d in list(R.get(g, ())): + if d[0] == 'closure': + out.add(d) + elif d[0] in ('obj', 'param'): + for w in b.get(d[-1], []): + if own(f, w): + out.add(d[:-1] + (w[1],)) + elif w[0] == 'closure' and d[0] == 'param': + out.add(w) + + def resolve(f, v, out, depth=0): + if depth > 4: + return + if v[0] == 'closure': + out.add(v) + elif own(f, v): + out.add(('param', v[1])) + elif v[0] == 'new': + c = self.canon(v[1]) + if c in self.classes: + ini = self.lookup(c, '__init__') + if ini: + b = bindv(ini, v[2]) + for fld, pn in self._stores(c): + for w in b.get(pn, []): + if own(f, w): + out.add(('obj', c, fld, w[1])) + elif c in self.fns: + apply(f, c, v[2], out) + elif v[0] == 'res': + apply(f, v[1], v[2], out) + elif v[0] == 'res2': + inner = set() + resolve(f, v[1], inner, depth + 1) + for d in inner: + if d[0] == 'closure': + apply(f, d[1], v[2], out) + + for _ in range(8): + changed = False + for f in self.fns.values(): + acc = R[f.q]; n0 = len(acc) + for v in f.rets: + resolve(f, v, acc) + changed |= len(acc) != n0 + if not changed: + break + self.R = R + + def _bind(self, callee, posv, kwv): + g = self.fns.get(callee) + if not g: + return {} + out = {} + byp = {i: n for n, i in g.pos.items() if i >= 0} + for i, vs in enumerate(posv): + if vs is not None and i in byp: + out.setdefault(byp[i], []).extend(vs) + for k, vs in kwv.items(): + if k in g.pos: + out.setdefault(k, []).extend(vs) + return out + + def _stores(self, c): + out = set() + for k in self.mro(c): + out |= self.ST.get(k, set()) + return out + + def decorates(self, g): + """what applying the callable g as a decorator to a function f yields: (form, class, field) -- DIRECT for + `@g`, FACTORY for `@g(...)`; class '' when f itself comes back (a registering decorator)""" + f = self.fns[g] + first = lambda h: next((n for n, i in self.fns[h].pos.items() if i == 0), None) + out = set() + p0 = first(g) + for d in self.R.get(g, ()): + if d[0] == 'obj' and d[3] == p0: + out.add(('DIRECT', d[1], d[2])) + elif d[0] == 'param' and d[1] == p0: + out.add(('DIRECT', '', '')) + elif d[0] == 'closure': + q0 = first(d[1]) + for e in self.R.get(d[1], ()): + if e[0] == 'obj' and e[3] == q0: + out.add(('FACTORY', e[1], e[2])) + elif e[0] == 'param' and e[1] == q0: + out.add(('FACTORY', '', '')) + return out + + def named(self, imports, lib_pkgs): + """the library classes and functions the client can name: what it imports, what a module it imports + (and that module's direct submodules) declares, and the class a decorator it imports wraps a function + in. Summaries are emitted for these only -- the solve reads what the client can reach, not the venv.""" + out = set() + def add(q): + c = self.canon(q) + if c and c.split('.')[0] in lib_pkgs and '' not in c: + out.add(c) + for p in imports: + add(p) + if p in self.mods: + for m in [p] + [x for x in self.mods if x.startswith(p + '.') and '.' not in x[len(p) + 1:]]: + for local in self.mods[m].ns: + add(f'{m}.{local}') + for q in list(out): + if q in self.fns: + for _, c, _ in self.decorates(q): + if c: + out.add(c) + elif q in self.classes: + for n in {n for k in self.mro(q) for n in self.classes[k].methods}: + m = self.lookup(q, n) + if m: + for _, c, _ in self.decorates(m): + if c: + out.add(c) + return out + + def rows(self, keep): + """flattened rows: every method a class HAS (inherited ones included) and every constructor""" + cb, fi, al, de, an = set(), set(), set(), set(), set() + + def emit(callee_name, fq, kind): + g = self.fns[fq] + for (r, p, mem, via, ty) in self.S.get(fq, ()): + if r == 'self': + pos = -1 + else: + pos = g.pos.get(r, -3) + if pos == -3 or (pos == -1 and kind != 'function'): + continue + cb.add((callee_name, r, pos, '.'.join(p), mem, via, ty)) + for q, g in self.fns.items(): + if g.cls is None and not g.local and keep(q): + emit(q, q, 'function') + for form, c, fld in self.decorates(q): + de.add((q, form, c, fld)) + for c in self.classes: + if not keep(c): + continue + names = set() + for k in self.mro(c): + names |= set(self.classes[k].methods) + for n in names: + m = self.lookup(c, n) + if m and self.fns[m].kind != 'property': + emit(f'{c}.{n}', m, 'method' if self.fns[m].kind != 'static' else 'function') + for form, k, fld in self.decorates(m): + de.add((f'{c}.{n}', form, k, fld)) + for a in self.mro(c)[1:]: + an.add((c, a)) + ini = self.lookup(c, '__init__') + if ini: + emit(c, ini, 'ctor') + g = self.fns[ini] + for fld, pn in set().union(*[self.ST.get(k, set()) for k in self.mro(c)]): + p = g.pos.get(pn, -3) + if p != -3: + fi.add((c, fld, pn, p)) + if p == 0: + de.add((c, 'DIRECT', c, fld)) # `@C def f`: C(f) holds f + for mname, m in self.mods.items(): + for local, q in m.ns.items(): + a = f'{mname}.{local}' + if a == q: + continue + c = self.canon(a) + if c and c != a and keep(c): + al.add((a, c)) + return cb, fi, al, de, an + + +def write(path, head, rows): + tmp = path + '.tmp' + with open(tmp, 'w', newline='', encoding='utf-8') as fh: + w = csv.writer(fh, delimiter='\t', lineterminator='\n') + w.writerow(head) + for r in sorted(rows, key=lambda x: tuple(str(c) for c in x)): + w.writerow(r) + os.replace(tmp, path) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--src', required=True); ap.add_argument('--out', required=True) + ap.add_argument('--ir', default=None); ap.add_argument('--stats', action='store_true') + A = ap.parse_args() + os.makedirs(A.out, exist_ok=True) + heads = (('py-lib-callback.csv', ('callee', 'root', 'pos', 'path', 'member', 'via', 'type')), + ('py-lib-field-init.csv', ('class', 'field', 'param', 'pos')), + ('py-lib-alias.csv', ('alias', 'canonical')), + ('py-lib-decorates.csv', ('decorator', 'form', 'class', 'field')), + ('py-lib-ancestor.csv', ('class', 'ancestor'))) + sites, stdlib = find_sites(os.path.abspath(A.src)) + if not sites: + for f, h in heads: + write(os.path.join(A.out, f), h, []) + print('▶ library callbacks: no Python environment found in the repository (.venv, venv, $VIRTUAL_ENV, ' + 'AXIOMCODE_PY_SITE); none summarised') + return + imports = client_imports(A.src, A.ir or A.out) + # CACHED BY WHAT DECIDES THE ANSWER: this file, the client's import paths, and the installed distributions + # (name and version from each *.dist-info) of every environment read. An edit that adds no import -- every + # refresh but a few -- reuses the tables instead of reading the library again. + h = hashlib.sha256() + h.update(open(os.path.abspath(__file__), 'rb').read()) + h.update('\n'.join(sorted(imports)).encode()) + for d in sites + ([stdlib] if stdlib else []): + h.update(d.encode()) + try: + h.update('\n'.join(sorted(x for x in os.listdir(d) if x.endswith(('.dist-info', '.egg-info', '.pth')))).encode()) + except OSError: + pass + cache = os.path.join(os.environ.get('XDG_CACHE_HOME') or os.path.join(os.path.expanduser('~'), '.cache'), + 'axiomcode', 'libsum', h.hexdigest()[:32]) + if all(os.path.isfile(os.path.join(cache, f)) for f, _ in heads): + import shutil + for f, _ in heads: + shutil.copyfile(os.path.join(cache, f), os.path.join(A.out, f)) + try: + os.utime(cache) + print(open(os.path.join(cache, 'summary.txt'), encoding='utf-8').read().strip() + ' (cached)') + except OSError: + print('▶ library callbacks: reused the cached summaries') + return + roots = {p.split('.')[0] for p in imports} + prog, todo, seen, client_pkgs = Program(), sorted(roots), set(), set() + # the client's own package installed in editable mode is not a library: its top-level names in the source tree + for n in os.listdir(A.src): + if os.path.isfile(os.path.join(A.src, n, '__init__.py')) or n.endswith('.py'): + client_pkgs.add(n[:-3] if n.endswith('.py') else n) + for sub in ('src', 'lib'): + d = os.path.join(A.src, sub) + if os.path.isdir(d): + for n in os.listdir(d): + if os.path.isfile(os.path.join(d, n, '__init__.py')): + client_pkgs.add(n) + nfiles = 0 + while todo: + r = todo.pop() + if r in seen or r in client_pkgs: + continue + seen.add(r) + pkg = locate(r, sites, stdlib) + if not pkg: + continue + for modname, path in pkg.files(): + m = Module(modname, path, path.endswith('__init__.py')) + if not m.load(): + continue + nfiles += 1 + m.analyse() + del m.tree + prog.add(m) + for imp in m.imports: + t = imp.split('.')[0] + if t and t not in seen: + todo.append(t) + rounds = prog.solve() + prog.returns() + lib_pkgs = {p for p in seen if p not in client_pkgs} + named = prog.named(imports, lib_pkgs) + keep = lambda q: q in named + cb, fi, al, de, an = prog.rows(keep) + for (f, h), rows in zip(heads, (cb, fi, al, de, an)): + write(os.path.join(A.out, f), h, rows) + line = (f'▶ library callbacks: {len(cb)} hand-backs, {len(fi)} constructor stores, {len(al)} re-exports, ' + f'{len(de)} decorators, from {nfiles} library modules ({len(lib_pkgs)} packages, {rounds} rounds)') + print(line) + try: + import shutil + tmp = cache + f'.tmp{os.getpid()}' + os.makedirs(tmp, exist_ok=True) + for f, _ in heads: + shutil.copyfile(os.path.join(A.out, f), os.path.join(tmp, f)) + open(os.path.join(tmp, 'summary.txt'), 'w', encoding='utf-8').write(line + '\n') + os.replace(tmp, cache) + root = os.path.dirname(cache) # keep the 64 most recently used + old = sorted((os.path.getmtime(os.path.join(root, x)), x) for x in os.listdir(root) if '.tmp' not in x) + for _, x in old[:-64]: + shutil.rmtree(os.path.join(root, x), ignore_errors=True) + except OSError: + pass + + +if __name__ == '__main__': + main() diff --git a/graph/python/souffle/decls_all.dl b/graph/python/souffle/decls_all.dl index d6b069c6..4617c7f1 100644 --- a/graph/python/souffle/decls_all.dl +++ b/graph/python/souffle/decls_all.dl @@ -238,6 +238,7 @@ .decl element_lib_type_of(c0:symbol,c1:symbol) .decl binding_element_lib_type(c0:symbol,c1:symbol) .decl param_declared_type_by_parser(c0:symbol,c1:symbol,c2:symbol) +.decl union_operand(c0:symbol,c1:symbol,c2:symbol) .decl type_ref_element(c0:symbol,c1:symbol,c2:symbol) .decl annotation_owner_module(c0:symbol,c1:symbol,c2:symbol,c3:symbol) .decl annotation_container_kind(c0:symbol) @@ -313,6 +314,9 @@ .decl py_path_join_function(c0:symbol) .decl py_path_join_operator(c0:symbol) .decl py_copy_function(c0:symbol) +.decl py_returns_arg(c0:symbol,c1:symbol) +.decl py_passthrough_arg(c0:symbol,c1:symbol) +.decl py_passthrough_root(c0:symbol,c1:symbol) .decl dict_lookup_method(c0:symbol) .decl builtin_target(c0:symbol,c1:symbol) .decl builtin_object_init_target(c0:symbol) @@ -594,6 +598,8 @@ .decl decorator_hits_lib(c0:symbol,c1:symbol) .decl decorator_hits_builtin(c0:symbol,c1:symbol) .decl property_read_edge(c0:symbol,c1:symbol,c2:symbol) +.decl property_write_edge(c0:symbol,c1:symbol,c2:symbol) +.decl type_property_accessor(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol) .decl with_protocol_edge(c0:symbol,c1:symbol,c2:symbol) .decl iteration_protocol_sync(c0:symbol) .decl iteration_protocol_of(c0:symbol,c1:symbol) @@ -662,3 +668,11 @@ .decl binding_declared_call_protocol(c0:symbol,c1:symbol,c2:symbol) .decl binding_assigned_lambda(c0:symbol,c1:symbol,c2:symbol) .decl instance_attr_method_value(c0:symbol,c1:symbol,c2:symbol,c3:symbol) +// staged beside the client IR by graph/python/libsum/libsum.py (framework-behavior/library-callbacks.dl) +.decl py_lib_callback(callee:symbol, root:symbol, pos:symbol, path:symbol, member:symbol, via:symbol, type:symbol) +.decl py_lib_ancestor(cls:symbol, ancestor:symbol) +.decl py_lib_field_init(cls:symbol, field:symbol, param:symbol, pos:symbol) +.decl py_lib_alias(alias:symbol, canon:symbol) +.decl lib_callback_edge(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol,c5:symbol) +.decl lib_callback_site(c0:symbol,c1:symbol) +.decl py_lib_decorates(decorator:symbol, form:symbol, cls:symbol, field:symbol) diff --git a/graph/python/souffle/export_manifest.tsv b/graph/python/souffle/export_manifest.tsv index 758e6d9e..19aaa3d2 100644 --- a/graph/python/souffle/export_manifest.tsv +++ b/graph/python/souffle/export_manifest.tsv @@ -91,3 +91,5 @@ remote_unserved remote-unserved.csv remote_unsent remote-unsent.csv framework_edge framework-edge.csv framework_unjoined framework-unjoined.csv +lib_callback_edge lib-callback-edge.csv +lib_callback_site lib-callback-site.csv diff --git a/graph/python/templates/client-extra.map b/graph/python/templates/client-extra.map new file mode 100644 index 00000000..18f5076f --- /dev/null +++ b/graph/python/templates/client-extra.map @@ -0,0 +1,12 @@ +# Soufflé input-relation -> CSV basename for facts staged BESIDE the client IR that are not parser facts. +# Same format and staging as client-ir.map (absent file = empty relation, so every project compiles one program), +# but outside the frozen parser schema: no decls_base.dl row, no lib.map counterpart (a library's own summary of +# its dependencies is not staged), declared in decls_all.dl instead. +# +# The library callback summaries graph/python/libsum/libsum.py writes from the project's installed dependencies +# (framework-behavior/library-callbacks.dl). Absent = empty = no library hand-backs, exactly as before. +py_lib_callback py-lib-callback +py_lib_field_init py-lib-field-init +py_lib_alias py-lib-alias +py_lib_decorates py-lib-decorates +py_lib_ancestor py-lib-ancestor diff --git a/graph/test/python/expected/33-url-view-forms.edges b/graph/test/python/expected/33-url-view-forms.edges index f5bb9f3f..c0500600 100644 --- a/graph/test/python/expected/33-url-view-forms.edges +++ b/graph/test/python/expected/33-url-view-forms.edges @@ -1,9 +1,11 @@ ambiguous_unknown DECORATOR_APPLICATION shop.decorators.audited -> - ambiguous_unknown DECORATOR_BARE shop.views. -> - ambiguous_unknown METHOD_CALL shop.urls. -> - -ambiguous_unknown SIMPLE_CALL catalog.urls. -> - -ambiguous_unknown SIMPLE_CALL shop.urls. -> - boundary_lib DECORATOR_CALL shop.decorators.audited -> external:functools.wraps +boundary_lib SIMPLE_CALL catalog.urls. -> external:path +boundary_lib SIMPLE_CALL shop.urls. -> external:include +boundary_lib SIMPLE_CALL shop.urls. -> external:path +boundary_lib SIMPLE_CALL shop.urls. -> external:re_path known_edge DECORATOR_BARE shop.views. -> shop.decorators.audited known_edge SIMPLE_CALL shop.decorators.audited..wrapper -> shop.views.order_history known_edge SIMPLE_CALL shop.views.BaseWidgetView.get -> shop.views._render diff --git a/graph/test/python/expected/33-url-view-forms.tiers b/graph/test/python/expected/33-url-view-forms.tiers index 2713fd47..f05de8d3 100644 --- a/graph/test/python/expected/33-url-view-forms.tiers +++ b/graph/test/python/expected/33-url-view-forms.tiers @@ -1,8 +1,8 @@ distinct call sites emitted: 51 --- by tier: edge ROWS, and the distinct SITES they cover --- - 32 rows 32 sites ambiguous_unknown - 1 rows 1 sites boundary_lib + 6 rows 6 sites ambiguous_unknown + 27 rows 27 sites boundary_lib 18 rows 18 sites known_edge --- edge rows by call kind --- @@ -14,13 +14,13 @@ distinct call sites emitted: 51 --- unresolved reasons --- 1 decorator_factory_result_untyped - 27 no_rule + 1 no_rule 4 unmodelled_decorator --- the engine's own conservation ledger --- 51 _total_sites - 32 ambiguous_unknown - 1 boundary_lib + 6 ambiguous_unknown + 27 boundary_lib 18 known_edge --- reconciling rows against the conserved site count --- diff --git a/graph/test/python/expected/37-model-signals.edges b/graph/test/python/expected/37-model-signals.edges index a50f46f3..67531829 100644 --- a/graph/test/python/expected/37-model-signals.edges +++ b/graph/test/python/expected/37-model-signals.edges @@ -1,6 +1,4 @@ ambiguous_unknown DECORATOR_APPLICATION models. -> - -ambiguous_unknown METHOD_CALL models.Invoice. -> - -ambiguous_unknown METHOD_CALL models.Order. -> - ambiguous_unknown METHOD_CALL views.create_order -> - ambiguous_unknown METHOD_CALL views.drop_invoice -> - ambiguous_unknown METHOD_CALL views.drop_order -> - @@ -9,4 +7,6 @@ ambiguous_unknown METHOD_CALL views.save_draft -> - ambiguous_unknown METHOD_CALL views.touch_invoice -> - boundary_lib DECORATOR_CALL models. -> external:receiver boundary_lib METHOD_CALL models. -> external:post_save.connect +boundary_lib METHOD_CALL models.Invoice. -> external:models.IntegerField +boundary_lib METHOD_CALL models.Order. -> external:models.CharField known_edge METHOD_CALL views.save_cache -> views.Cache.save diff --git a/graph/test/python/expected/37-model-signals.tiers b/graph/test/python/expected/37-model-signals.tiers index a7e08a5c..e87fd3fa 100644 --- a/graph/test/python/expected/37-model-signals.tiers +++ b/graph/test/python/expected/37-model-signals.tiers @@ -1,8 +1,8 @@ distinct call sites emitted: 14 --- by tier: edge ROWS, and the distinct SITES they cover --- - 10 rows 10 sites ambiguous_unknown - 3 rows 3 sites boundary_lib + 8 rows 8 sites ambiguous_unknown + 5 rows 5 sites boundary_lib 1 rows 1 sites known_edge --- edge rows by call kind --- @@ -14,13 +14,12 @@ distinct call sites emitted: 14 2 decorator_factory_result_untyped 4 escape_hatch 1 untyped_receiver:class_attribute_absent - 2 untyped_receiver:local_untyped 1 untyped_receiver:parameter --- the engine's own conservation ledger --- 14 _total_sites - 10 ambiguous_unknown - 3 boundary_lib + 8 ambiguous_unknown + 5 boundary_lib 1 known_edge --- reconciling rows against the conserved site count --- diff --git a/graph/test/python/expected/38-app-wiring.edges b/graph/test/python/expected/38-app-wiring.edges index 89dad32f..39d93d28 100644 --- a/graph/test/python/expected/38-app-wiring.edges +++ b/graph/test/python/expected/38-app-wiring.edges @@ -3,8 +3,6 @@ ambiguous_unknown DECORATOR_ATTRIBUTE main. -> - ambiguous_unknown DECORATOR_BARE main. -> - ambiguous_unknown SIMPLE_CALL main.Timing.dispatch -> - ambiguous_unknown SIMPLE_CALL main.add_header -> - -ambiguous_unknown SIMPLE_CALL shop.jobs.nightly -> - -ambiguous_unknown SIMPLE_CALL shop.jobs.unknown -> - boundary_lib DECORATOR_CALL main. -> external:APIRouter.api_route boundary_lib DECORATOR_CALL main. -> external:FastAPI.exception_handler boundary_lib DECORATOR_CALL main. -> external:FastAPI.middleware @@ -17,6 +15,8 @@ boundary_lib SIMPLE_CALL main. -> external:FastAPI boundary_lib SIMPLE_CALL main. -> external:Route boundary_lib SIMPLE_CALL main. -> external:Starlette boundary_lib SIMPLE_CALL main. -> external:WebSocketRoute +boundary_lib SIMPLE_CALL shop.jobs.nightly -> external:call_command +boundary_lib SIMPLE_CALL shop.jobs.unknown -> external:call_command known_edge METHOD_CALL main. -> main.Registry.add_route known_edge SIMPLE_CALL main. -> main.configure known_edge SIMPLE_CALL shop.management.commands.close_orders.Command.handle -> shop.orders.close_all diff --git a/graph/test/python/expected/38-app-wiring.tiers b/graph/test/python/expected/38-app-wiring.tiers index 4df06d8a..8496a4b9 100644 --- a/graph/test/python/expected/38-app-wiring.tiers +++ b/graph/test/python/expected/38-app-wiring.tiers @@ -1,8 +1,8 @@ distinct call sites emitted: 26 --- by tier: edge ROWS, and the distinct SITES they cover --- - 10 rows 10 sites ambiguous_unknown - 13 rows 13 sites boundary_lib + 8 rows 8 sites ambiguous_unknown + 15 rows 15 sites boundary_lib 3 rows 3 sites known_edge --- edge rows by call kind --- @@ -16,13 +16,12 @@ distinct call sites emitted: 26 --- unresolved reasons --- 2 callee_is_parameter 4 decorator_factory_result_untyped - 2 no_rule 2 unmodelled_decorator --- the engine's own conservation ledger --- 26 _total_sites - 10 ambiguous_unknown - 13 boundary_lib + 8 ambiguous_unknown + 15 boundary_lib 3 known_edge --- reconciling rows against the conserved site count --- diff --git a/graph/test/python/tools/literal_gate.py b/graph/test/python/tools/literal_gate.py index db44b130..0e7af6f7 100644 --- a/graph/test/python/tools/literal_gate.py +++ b/graph/test/python/tools/literal_gate.py @@ -83,7 +83,12 @@ # unresolved_value_callee's second column is `binding`: WHAT the callee of an # unresolved call is ("parameter", "loop_variable", "field", ...), the same words # the JavaScript rules write. Written into the output and never joined on. - "unresolved_value_callee(") + "unresolved_value_callee(", + # lib_callback_edge's How column (receiver / receiver_field / argument / argument_field) says + # where on the call the handed-back object arrived. Written into the output by + # framework-behavior/library-callbacks.dl and never joined on; every name that DOES decide a + # library hand-back comes from the tables graph/python/libsum derives from the library's source. + "lib_callback_edge(", "lcb_hit(") GROUND_FACT = re.compile(r'^[a-z_]+\((?:\s*"[^"]*"\s*,?)+\)\.\s*$') diff --git a/graph/test/python/torture/client/f43_def_rebind.py b/graph/test/python/torture/client/f43_def_rebind.py index 107989af..3a454e7e 100644 --- a/graph/test/python/torture/client/f43_def_rebind.py +++ b/graph/test/python/torture/client/f43_def_rebind.py @@ -89,10 +89,10 @@ def call_branched() -> str: def read_property() -> int: """The getter survives — the control that keeps this rule off property pairs. - EXPECT: miss — `h.value = 5` invokes the SETTER, and the engine emits a - PROPERTY_READ edge for a property read and nothing at all for a property WRITE. - That is a pre-existing gap this fixture happens to expose, not something #383 - changed; the getter edge on the next line is what this family is asserting. + `h.value = 5` invokes the SETTER, and the engine now emits a PROPERTY_WRITE + edge for it beside the PROPERTY_READ for the read; both halves of the pair + are live, which is exactly what this control exists to keep true. The getter + edge on the next line is what this family is asserting. """ h = Holder() h.value = 5 diff --git a/graph/test/python/torture/expected/coverage.txt b/graph/test/python/torture/expected/coverage.txt index 06b88e32..daf7e130 100644 --- a/graph/test/python/torture/expected/coverage.txt +++ b/graph/test/python/torture/expected/coverage.txt @@ -1,4 +1,4 @@ -=== per-family coverage (tier-4, 471 scored sites) === +=== per-family coverage (tier-4, 474 scored sites) === f01 inheritance & MRO links= 10 found= 10 (100.0%) missed= 0 wide= 0 WRONG= 0 f02 callables & closures links= 10 found= 9 (90.0%) missed= 1 wide= 0 WRONG= 0 f03 generics links= 14 found= 14 (100.0%) missed= 0 wide= 0 WRONG= 0 @@ -38,16 +38,16 @@ f40 data descriptor links= 14 found= 14 (100.0%) missed= 0 wide= 0 WRONG= 0 f41 class attribute absent links= 3 found= 3 (100.0%) missed= 0 wide= 0 WRONG= 0 f42 f42 links= 6 found= 6 (100.0%) missed= 0 wide= 0 WRONG= 0 - f43 f43 links= 7 found= 7 (100.0%) missed= 0 wide= 1 WRONG= 0 + f43 f43 links= 10 found= 10 (100.0%) missed= 0 wide= 1 WRONG= 0 nestmod.py nestmod.py links= 1 found= 1 (100.0%) missed= 0 wide= 0 WRONG= 0 pkgmod relative imports (subpackage) links= 3 found= 3 (100.0%) missed= 0 wide= 0 WRONG= 0 - TOTAL links=434 found=428 (98.6%) missed= 6 wide=37 WRONG= 0 - recall 428/434 = 98.6% of the links that actually ran + TOTAL links=437 found=431 (98.6%) missed= 6 wide=37 WRONG= 0 + recall 431/437 = 98.6% of the links that actually ran wide 37 a member of a SOUND SET that did not run on this pass WRONG 0 a single target asserted as certain that never ran - EXPECTED-MISS cases: {'FOUND': 20, 'MISSED': 24} (a CONCRETE here means a known blind spot closed) + EXPECTED-MISS cases: {'FOUND': 18, 'MISSED': 23} (a CONCRETE here means a known blind spot closed) --- non-concrete sites --- f02_callables.py:31 MISSED true=[('lib', 'callables.py', 22)] engine=[] diff --git a/graph/test/python/torture/expected/torture.edges b/graph/test/python/torture/expected/torture.edges index 4d85cc1a..557d3208 100644 --- a/graph/test/python/torture/expected/torture.edges +++ b/graph/test/python/torture/expected/torture.edges @@ -635,6 +635,7 @@ known_edge PROPERTY_READ f40_data_descriptor.via_data_descriptor -> f40_data_des known_edge PROPERTY_READ f40_data_descriptor.via_delete_descriptor -> f40_data_descriptor.Deletable.__get__ known_edge PROPERTY_READ f40_data_descriptor.via_property -> f40_data_descriptor.Prop.slot known_edge PROPERTY_READ f43_def_rebind.read_property -> f43_def_rebind.Holder.value +known_edge PROPERTY_WRITE f43_def_rebind.read_property -> f43_def_rebind.Holder.value known_edge SELF_CALL f02_callables.HoldsCallables.run -> f02_callables.LocalCallable.__call__ known_edge SELF_CALL f28_await.Registry.into_a_field -> f28_await.Registry.build known_edge SELF_CALL f28_await.Registry.via_self_receiver -> f28_await.Registry.build diff --git a/graph/test/python/torture/expected/torture.oracle b/graph/test/python/torture/expected/torture.oracle index 95fd8d90..4267008a 100644 --- a/graph/test/python/torture/expected/torture.oracle +++ b/graph/test/python/torture/expected/torture.oracle @@ -1 +1 @@ -oracle=634 engine=624 agree=587 missing=47 extra=37 +oracle=634 engine=625 agree=588 missing=46 extra=37 diff --git a/parser/src/parsers/python/extractors/python-resolution-linker.ts b/parser/src/parsers/python/extractors/python-resolution-linker.ts index 0d5c5485..f226446d 100644 --- a/parser/src/parsers/python/extractors/python-resolution-linker.ts +++ b/parser/src/parsers/python/extractors/python-resolution-linker.ts @@ -43,6 +43,13 @@ export interface ProjectModuleFacts extends ResolutionInput { * re-export fails to resolve. */ isPackage?: boolean; + /** + * The directory names above this module's top-level package, outermost first: `["repo", "src"]` for + * `repo/src/pkg/mod.py`. An import more qualified than the module (`import myproject.framework` naming a + * `framework` that sits in a `myproject/` directory with no `__init__.py`) is matched by dropping its leading + * segments, and those segments must then be the directories the module actually sits in. + */ + outerDirs?: string[]; } export interface ProjectResolutionStats { @@ -1839,18 +1846,42 @@ export class PythonResolutionLinker { const parts = name.split('.'); for (let drop = 1; drop < parts.length; drop++) { const candidate = parts.slice(drop).join('.'); + const dropped = parts.slice(0, drop); + // THE DROPPED SEGMENTS MUST BE WHERE THE MODULE SITS. Dropping them is for an import more qualified than the + // module; dropping any leading segment turned `from clikit.testing import CliRunner` into the project's own + // `testing` (or `app.testing`, by suffix) — a dependency whose module shares a last segment with one of the + // project's resolved INTO the project, and its import bound nothing. const direct = moduleByQualifiedName.get(candidate); - if (direct) { + if (direct && this.sitsUnder(direct, candidate, dropped)) { return direct; } const suffixed = this.moduleSuffixIndex.get(candidate); - if (suffixed) { + if (suffixed && this.sitsUnder(suffixed, candidate, dropped)) { return suffixed; } } return undefined; } + /** + * Whether `dropped` names what lies directly above `tail` on the module's own path: the part of its qualified + * name before the matched tail, then the directories above its top-level package. A module with no recorded + * directories keeps the old answer. + */ + private sitsUnder(module: ProjectModuleFacts, tail: string, dropped: string[]): boolean { + if (module.outerDirs === undefined) { + return true; + } + const own = module.qualifiedName.split('.'); + const before = own.slice(0, Math.max(0, own.length - tail.split('.').length)); + const chain = [...module.outerDirs, ...before]; + if (chain.length < dropped.length) { + return false; + } + const end = chain.slice(chain.length - dropped.length); + return end.every((segment, i) => segment === dropped[i]); + } + /** * Registers every module under every dotted suffix of its qualified name. * diff --git a/parser/src/workflows/python/python-project-analyzer.ts b/parser/src/workflows/python/python-project-analyzer.ts index 0c07497a..739b8f21 100644 --- a/parser/src/workflows/python/python-project-analyzer.ts +++ b/parser/src/workflows/python/python-project-analyzer.ts @@ -320,6 +320,7 @@ export class PythonProjectAnalyzer { qualifiedName: facts.module.getQualifiedName(), moduleHash: facts.module.getHash(), isPackage: isPythonPackageInitFileName(path.basename(filePath)), + outerDirs: this.outerDirsOf(filePath, facts.module.getQualifiedName()), scopes: facts.scopes, bindings: facts.bindings, types: facts.types, @@ -577,6 +578,15 @@ export class PythonProjectAnalyzer { return path.relative(rootDir, filePath) || path.basename(filePath); } + /** The directory names above a module's top-level package, outermost first (see ProjectModuleFacts.outerDirs). */ + private outerDirsOf(filePath: string, qualifiedName: string): string[] { + const dirs = path.dirname(path.resolve(filePath)).split(path.sep).filter(part => part !== ''); + const inPackage = isPythonPackageInitFileName(path.basename(filePath)) + ? qualifiedName.split('.').length + : qualifiedName.split('.').length - 1; + return dirs.slice(0, Math.max(0, dirs.length - inPackage)); + } + private moduleQualifiedNameFor(rootDir: string, filePath: string): string { const parsed = path.parse(filePath); const segments: string[] = []; diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py index f5283b76..30d4be7d 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_edges.py @@ -12,7 +12,7 @@ + COMPUTED_CALL IIFE_CALL DYNAMIC_IMPORT_CALL DYNAMIC_CODE_CALL TAGGED_TEMPLATE_CALL FUNCTION_CALL_APPLY FUNCTION_CALL_CALL FUNCTION_CALL_BIND python SIMPLE_CALL METHOD_CALL SELF_CALL SUPER_CALL CHAINED_CALL SUBSCRIPT_CALL CONTEXT_MANAGER - PROPERTY_READ METACLASS_CREATION DYNAMIC_CALL UNKNOWN_CALLEE_CALL DECORATOR_{APPLICATION,ATTRIBUTE,BARE,CALL} + PROPERTY_READ PROPERTY_WRITE METACLASS_CREATION DYNAMIC_CALL UNKNOWN_CALLEE_CALL DECORATOR_{APPLICATION,ATTRIBUTE,BARE,CALL} csharp + boundary_generated known_implicit_ctor known_builtin_operator ambiguous_dynamic fan_capped event_dispatch runtime_observed (only with a runtime trace) · new property_read property_write @@ -109,7 +109,7 @@ 'DECORATOR_APPLICATION': 'decorator', 'DECORATOR_ATTRIBUTE': 'decorator', 'DECORATOR_BARE': 'decorator', 'DECORATOR_CALL': 'decorator', # an accessor: written as a field, run as a method - 'property_read': 'property', 'property_write': 'property', 'PROPERTY_READ': 'property', + 'property_read': 'property', 'property_write': 'property', 'PROPERTY_READ': 'property', 'PROPERTY_WRITE': 'property', # the language runs it at a block boundary 'CONTEXT_MANAGER': 'with', # run-time code loading @@ -507,9 +507,14 @@ def nearest(cands): # the candidates enclosing no other candidat def library_receiver_sites(q): - """the ids of the unresolved call sites whose receiver is a package's value. q(sql, params) -> rows.""" - if not list(q("SELECT 1 FROM sqlite_master WHERE name = 'ext_library_receiver'", ())): return set() - return {r[0] for r in q("SELECT DISTINCT c0 FROM ext_library_receiver", ())} + """the ids of the unresolved call sites whose receiver is a package's value, or that the engine knows enter a + library callable (Python ext_lib_callback_site: a member a client type inherits from its library base, whose + source declares it). q(sql, params) -> rows.""" + out = set() + for t in ('ext_library_receiver', 'ext_lib_callback_site'): + if list(q("SELECT 1 FROM sqlite_master WHERE name = ?", (t,))): + out |= {r[0] for r in q(f"SELECT DISTINCT c0 FROM {t}", ())} + return out # ── a MOCKED TYPE: a test class that holds a mock of T never runs T's methods ──────────────────────────────────── diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact index 7a8dc195..871cbdf6 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-impact @@ -806,13 +806,31 @@ class Impact: byfile = set(files) # a module path as each language spells it -> the file it names. Both spellings of a Python package are # kept (app/settings.py and app/settings/__init__.py), because the import statement cannot tell them apart. + # A module is also named from its package root, the first directory above it that is not itself a package: + # in a src layout `src/app/core.py` is imported as `app.core`, and the repository-relative spelling alone + # matches no import of it. mod = {} for f in files: stem = os.path.splitext(f)[0] - mod.setdefault(stem.replace('/', '.'), f) - if stem.endswith('/__init__'): - mod.setdefault(stem[:-len('/__init__')].replace('/', '.'), f) + names = [stem] + if f.endswith(('.py', '.pyi')): + parts = stem.split('/'); k = len(parts) - 1 + while k > 0 and '/'.join(parts[:k]) + '/__init__.py' in byfile: k -= 1 + if k > 0: names.append('/'.join(parts[k:])) + for n in names: + mod.setdefault(n.replace('/', '.'), f) + if n.endswith('/__init__'): + mod.setdefault(n[:-len('/__init__')].replace('/', '.'), f) out = set() + # pytest imports every conftest.py from the rootdir down to a test file's directory before the file itself, + # so whatever a conftest imports is imported for each test file beneath it + conftests = [f for f in files if f.rsplit('/', 1)[-1] == 'conftest.py'] + for f in files: + b = f.rsplit('/', 1)[-1] + if not (b.endswith('.py') and (b.startswith('test_') or b.endswith('_test.py'))): continue + for c in conftests: + cd = os.path.dirname(c) + if not cd or f.startswith(cd + '/'): out.add((f, c)) for f in files: ext = os.path.splitext(f)[1].lower() if not ext: # a python shebang script (#1376) @@ -1321,7 +1339,7 @@ class Impact: W('cs_fixture_type', sorted(x for x in fixt if not x[0].startswith('collection:'))) # ── facts: the graph, exported once (reused while graph.sqlite is unchanged) ──────────────────────────────── - IMPACT_VERSION = '63' # 63: imports_fact resolves an import written with its extension and a workspace package's specifier, and test_method takes no JavaScript / TypeScript helper by its name; 62: state_gate, state_gate_alloc, state_call_alloc, state_call_open, state_world, the callbacks one instance was given and the allocation each caller's receiver may be (JavaScript instance-state.dl); 61: filter_run, a servlet filter a test's context holds (added to HttpSecurity in a configuration the test loads, or a component) and the test methods that send a request through it; 57: a TypeScript object literal key is a ref of entity kind OBJECT_PROPERTY_KEY, kept past a bound access on its line; 56: reg_key_fact carries a handler table's entries (kind table), literal a table key written as a dotted string or through a constant, and test_code; 55: cs_data_source, cs_data_type, cs_fixture_type, the C# test links a runner makes from a data attribute or a class/collection fixture (#1498, #1499); 53: implicit_new, the type a C# `new T()` constructs where T writes no constructor (#1473); 52: test_method holds a method under a composed or derived test marker declared in the repository (a Java annotation meta-annotated @Test, a C# attribute derived from FactAttribute: #1418, #1497; 51 was the C# test-links branch's number, landed as 55); 48: sigtype, a parameter / return position type_use resolves to a type, read before the textuse grep (#1422), and persist_field, the properties a persistence query reads (#1461); 47: test_method / fixture from graph_sql's one classification (a tear-down such as @AfterEach or [TestCleanup] is a fixture, [TestInitialize] is no test, an @Override is no named test: #1417 #1419 #1502), and reg_key_fact drops a string a decoration does not register under (#1413); 46: accessor carries the wither and an isX boolean's setX / withX, and a generated builder or fluent setter the engine resolved is a writer (#1404, #1409); 45: runs_before, a C# set-up an NUnit [SetUpFixture] or an MSTest assembly initializer runs for tests outside its type (#1501), stub rows for a member a Moq Protected() setup names by string (#1540), cs_config_literal for a Section:Key configuration key (#1443), and lex_parent puts a lambda under the declaration on its own line (#1556); 44: a C# MEMBER_ACCESS ref is qualified, so its qualifier decides (#1445); 41: spawns_fact, a test that runs a script by its path (ax_spawn.py); 40: test_method holds a script test's module (a test-tree file run as a program, no framework: graph_sql.script_tests); 39: a chained route link's `calls` row and `registration` label sit on the link's own line, with its own verb and path; 38: reg_key_fact drops a decoration string with a space in it (a description, not a key); 37: via_base / via_site, a caller that reaches a declaration through a base it is override-equivalent to (#1542), and injected_bean, the bean an injection point was wired to (#1384); 36: handoff_at, route_arg, callable_const, init_wrapper, init_alias, returns_fn — a const holding a wrapped handler registered at a route; 35: 0.1.5's 33 (#1598, the defines edges the path export links) joined 0.1.6's 33, two different fact sets under one number; 33 (0.1.6): calls carries the tier "stub" for a call inside a mock's stub or verification, reg_verb / lit_verb join a route by its HTTP method, and a handler's route joins its type's prefix; 32: cert_tier's why is worded per tier (an event_dispatch row says it sends the request or event), and the route facts #1633 changed (#1510), which merged without a bump; 31: event_dispatch edges (a published event reaches its listeners, #1391) and the pytest fixture_injection reading (#1631) change impact's facts; 30: registers, a bean another class's annotation registers (#1396, #1414); 29: the edges it links from the path export changed (#1402), and a cache written before it must not survive; 28: reexport_from, the file an `export *` line re-exports; 27: framework, the engine's framework_edge (#1509); 24: the test* naming convention requires a test class as owner (#1181); 23: owner/member disambiguated by file, two classes of one name no longer merging (#1188); 22: lex_parent, the innermost declaration enclosing each one by span (#1183); 21: cert_tier, the tier -> certainty table the call rules join on (#1131); 20: faccess, the engine's resolved field accesses (#1071); 3: decl_file facts (the import-time test-file rule); 14: the registration-key + IMPACT_VERSION = '67' # 67: libcb, a library summarised to call a client member back (Python library-callbacks.dl), walked as fw_edge "library callback"; 66: imports_fact names a module from its package root (a src layout) and has each test file import the conftest.py files above it; 65: guard_only also reads a conjunctive guard (`cond and __name__ == "__main__"`); 64: guard_only, a module body's call made only under `if __name__ == "__main__":`, which the import walk does not take; 63: imports_fact resolves an import written with its extension and a workspace package's specifier, and test_method takes no JavaScript / TypeScript helper by its name; 62: state_gate, state_gate_alloc, state_call_alloc, state_call_open, state_world, the callbacks one instance was given and the allocation each caller's receiver may be (JavaScript instance-state.dl); 61: filter_run, a servlet filter a test's context holds (added to HttpSecurity in a configuration the test loads, or a component) and the test methods that send a request through it; 57: a TypeScript object literal key is a ref of entity kind OBJECT_PROPERTY_KEY, kept past a bound access on its line; 56: reg_key_fact carries a handler table's entries (kind table), literal a table key written as a dotted string or through a constant, and test_code; 55: cs_data_source, cs_data_type, cs_fixture_type, the C# test links a runner makes from a data attribute or a class/collection fixture (#1498, #1499); 53: implicit_new, the type a C# `new T()` constructs where T writes no constructor (#1473); 52: test_method holds a method under a composed or derived test marker declared in the repository (a Java annotation meta-annotated @Test, a C# attribute derived from FactAttribute: #1418, #1497; 51 was the C# test-links branch's number, landed as 55); 48: sigtype, a parameter / return position type_use resolves to a type, read before the textuse grep (#1422), and persist_field, the properties a persistence query reads (#1461); 47: test_method / fixture from graph_sql's one classification (a tear-down such as @AfterEach or [TestCleanup] is a fixture, [TestInitialize] is no test, an @Override is no named test: #1417 #1419 #1502), and reg_key_fact drops a string a decoration does not register under (#1413); 46: accessor carries the wither and an isX boolean's setX / withX, and a generated builder or fluent setter the engine resolved is a writer (#1404, #1409); 45: runs_before, a C# set-up an NUnit [SetUpFixture] or an MSTest assembly initializer runs for tests outside its type (#1501), stub rows for a member a Moq Protected() setup names by string (#1540), cs_config_literal for a Section:Key configuration key (#1443), and lex_parent puts a lambda under the declaration on its own line (#1556); 44: a C# MEMBER_ACCESS ref is qualified, so its qualifier decides (#1445); 41: spawns_fact, a test that runs a script by its path (ax_spawn.py); 40: test_method holds a script test's module (a test-tree file run as a program, no framework: graph_sql.script_tests); 39: a chained route link's `calls` row and `registration` label sit on the link's own line, with its own verb and path; 38: reg_key_fact drops a decoration string with a space in it (a description, not a key); 37: via_base / via_site, a caller that reaches a declaration through a base it is override-equivalent to (#1542), and injected_bean, the bean an injection point was wired to (#1384); 36: handoff_at, route_arg, callable_const, init_wrapper, init_alias, returns_fn — a const holding a wrapped handler registered at a route; 35: 0.1.5's 33 (#1598, the defines edges the path export links) joined 0.1.6's 33, two different fact sets under one number; 33 (0.1.6): calls carries the tier "stub" for a call inside a mock's stub or verification, reg_verb / lit_verb join a route by its HTTP method, and a handler's route joins its type's prefix; 32: cert_tier's why is worded per tier (an event_dispatch row says it sends the request or event), and the route facts #1633 changed (#1510), which merged without a bump; 31: event_dispatch edges (a published event reaches its listeners, #1391) and the pytest fixture_injection reading (#1631) change impact's facts; 30: registers, a bean another class's annotation registers (#1396, #1414); 29: the edges it links from the path export changed (#1402), and a cache written before it must not survive; 28: reexport_from, the file an `export *` line re-exports; 27: framework, the engine's framework_edge (#1509); 24: the test* naming convention requires a test class as owner (#1181); 23: owner/member disambiguated by file, two classes of one name no longer merging (#1188); 22: lex_parent, the innermost declaration enclosing each one by span (#1183); 21: cert_tier, the tier -> certainty table the call rules join on (#1131); 20: faccess, the engine's resolved field accesses (#1071); 3: decl_file facts (the import-time test-file rule); 14: the registration-key # layer; 15: the registration facts (two 14s landed independently, which is exactly the collision this # guards); 16: regsite folded into ax_registration's reg_key_fact; 20: implements_pair (#1011); 17/18: the tagged-template test registrar # (it.each`…`) and its table span @@ -1530,6 +1548,39 @@ class Impact: W('calls', [(r['caller_id'], r['callee_method_id'], ax_edges.STUB_TIER if r['call_site_id'] in stubs else r['tier'], g.site_file(r['file_path']) if r['file_path'] else '', link_line.get(r['call_site_id'], r['start_line'] or 0)) for r in g.q("SELECT e.call_site_id, e.caller_id, e.callee_method_id, e.tier, s.file_path, s.start_line FROM call_edges e LEFT JOIN call_sites s ON s.id = e.call_site_id WHERE e.callee_provenance = 'client' AND e.callee_method_id IS NOT NULL")] + self.protected_name_stubs()) + # A CALL UNDER `if __name__ == "__main__":` DOES NOT RUN ON IMPORT. The guard is true only when the file is run + # as a script, so a module body's calls inside it are real edges (the script makes them) but no hop for the + # import walk (dl/impact.dl up_running): otherwise a module ending in a demo block reads as breaking at import for + # every file that imports it. + guarded = {} + def guard_ranges(f): + if f not in guarded: + rs = [] + try: + import ast as _ast + def is_main(t): + if isinstance(t, _ast.Compare) and len(t.ops) == 1 and isinstance(t.ops[0], _ast.Eq): + sides = [t.left, t.comparators[0]] + return any(isinstance(x, _ast.Name) and x.id == '__name__' for x in sides) and \ + any(isinstance(x, _ast.Constant) and x.value == '__main__' for x in sides) + # `cond and __name__ == "__main__"`: a conjunct false on import keeps the block from running; + # an `or` would not, so it is not read as a guard + return isinstance(t, _ast.BoolOp) and isinstance(t.op, _ast.And) and any(is_main(v) for v in t.values) + for n in _ast.parse('\n'.join(self.lines(f))).body: + if isinstance(n, _ast.If) and is_main(n.test): + rs.append((n.lineno, n.end_lineno)) + except (SyntaxError, ValueError): + pass + guarded[f] = rs + return guarded[f] + sites_of = collections.defaultdict(list) + for r in g.q("SELECT e.caller_id, e.callee_method_id, s.file_path, s.start_line FROM call_edges e JOIN call_sites s ON s.id = e.call_site_id " + "WHERE e.callee_method_id IS NOT NULL"): + sy = g.sym.get(r['caller_id']) or {} + if sy.get('kind') == 'module' and (r['file_path'] or '').endswith('.py'): + sites_of[(r['caller_id'], r['callee_method_id'])].append((g.site_file(r['file_path']), r['start_line'] or 0)) + W('guard_only', sorted((a, b) for (a, b), ss in sites_of.items() + if all(any(lo <= ln <= hi for lo, hi in guard_ranges(f)) for f, ln in ss))) # a hand-off on EVERY line its call site spans: a chained registration (`router\n .route('/')\n .post(auth(), ctrl.h)`) # records its edges on the statement's first line, while the handler is named on a later one W('handoff_at', sorted({(r['caller_id'], r['callee_method_id'], g.site_file(r['file_path']), l) for r in g.q( @@ -2008,6 +2059,13 @@ class Impact: g.q("SELECT c0, c1, c2, c3, c4 FROM ext_framework_edge") if r[0] and r[1] and r[0] != r[1]}) W('framework', fw) + # A LIBRARY CALLING BACK INTO THE CLIENT (Python framework-behavior/library-callbacks.dl): walked by the closure as + # its own hop, `library callback`, so a test that drives the library reaches the client member it hands back to. + libcb = [] + if g.has('ext_lib_callback_edge'): + libcb = sorted({(r[0], r[1], r[2], r[3]) for r in g.q("SELECT c0, c1, c2, c3 FROM ext_lib_callback_edge") + if r[0] and r[1] and r[0] != r[1]}) + W('libcb', libcb) t = f'{stamp}.{os.getpid()}.tmp'; open(t, 'w').write(want); P.replace_file(t, stamp); return D # ── one query: the target facts, the text-level facts, one Soufflé run ────────────────────────────────────── @@ -2904,7 +2962,10 @@ def main(argv): 'remote': 'remote', # a servlet filter the test's context holds, run on each request the test sends (ax_registration.filter_links): # the container calls it, no call site does, so it is a registration like a callback's - 'filter': 'registered'} + 'filter': 'registered', + # a library summarised (from its source) to call a client member back: a member name on a type the client + # handed over, not a resolved call (framework-behavior/library-callbacks.dl) + 'library callback': 'library callback'} # The numbers that used to stand here (sound 0.645, one of a set 0.414, by key 0.396, fixture 0.233, by name # 0.133) were measured against a truth set that was SHORT: a pytest file which fails to COLLECT is emitted in # junit XML with classname="" and the module in `name`, the oracle read classname alone, and so every @@ -2942,7 +3003,7 @@ def main(argv): # is strong. It sits below `one of a set` all the same, because it is not a resolved call and should not borrow # the word that means one. CERT_RANK = {'sound': 0, 'defines': 1, 'at import': 2, 'one of a set': 3, 'dispatch': 4, 'registered': 5, - 'remote': 6, 'spawns': 7, 'by key': 8, 'decorator by name': 9, 'protocol': 10, 'fixture': 11, + 'library callback': 5, 'remote': 6, 'spawns': 7, 'by key': 8, 'decorator by name': 9, 'protocol': 10, 'fixture': 11, 'by name': 12} # `remote` (#1293) sits just above `by key`: both join two ends on a string, but the engine matched a remote # hop's whole route and verb on both ends, where a key join is this file's own rules matching one literal. Not @@ -3017,7 +3078,7 @@ def main(argv): # `decorator by name` is the rules' own hop from a decorated def to the decorator it names # (dl/impact.dl fw_edge), asserted exactly as `decorator` is; leaving it out printed "✗ not found" on # every answer whose test route passes through a project decorator - if (a, b) not in real and not any(x == b and t in ('dispatch', 'defines', 'by key', 'decorator', 'decorator by name', 'remote', 'spawns', 'filter') for x, t in parent.get(a, [])): bad += 1 + if (a, b) not in real and not any(x == b and t in ('dispatch', 'defines', 'by key', 'decorator', 'decorator by name', 'remote', 'spawns', 'filter', 'library callback') for x, t in parent.get(a, [])): bad += 1 asserted = collections.defaultdict(set) # the callee each direct row claims an edge to for c, callee in {(x[0], x[1]) for x in res['direct_edge']}: asserted[c].add(callee) for c, role, why, cert, loc, n in D: diff --git a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path index 51c36cbd..50267978 100755 --- a/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path +++ b/plugins/axiomcode/skills/axiomcode/scripts/axiomcode-path @@ -1069,6 +1069,12 @@ class G: for a, b, how, where, conf in self.q(f"SELECT c0, c1, c2, c3, c4 FROM {table}"): if a == b or a not in self.sym or b not in self.sym: continue self.OUTSIDE.setdefault((a, b, tier), []).append((how, where, conf)) + # a library summarised (from its source) to call a client member back (Python framework-behavior/ + # library-callbacks.dl): `impact` walks it as the `library callback` hop, so the chain search does too + if self.has('ext_lib_callback_edge'): + for a, b, callee, member, via, how in self.q("SELECT c0, c1, c2, c3, c4, c5 FROM ext_lib_callback_edge"): + if a == b or a not in self.sym or b not in self.sym: continue + self.OUTSIDE.setdefault((a, b, 'framework'), []).append(('library_callback', f'{callee} -> {member}', how)) # a servlet filter a test's context holds, run on each request the test sends (ax_registration.filter_links) try: import ax_registration @@ -1168,6 +1174,8 @@ def outside_hop_holds(g, a, b, t): table = 'ext_remote_edge' if t == 'remote' else 'ext_framework_edge' if g.has(table) and g.q(f"SELECT 1 FROM {table} WHERE c0 = ? AND c1 = ? LIMIT 1", a, b): return True if t != 'framework': return False + if g.has('ext_lib_callback_edge') and g.q("SELECT 1 FROM ext_lib_callback_edge WHERE c0 = ? AND c1 = ? LIMIT 1", a, b): + return True import ax_registration links = getattr(g, '_filter_links', None) if links is None: links = g._filter_links = {(x, y) for x, y, _h in ax_registration.filter_links(g.q)} diff --git a/plugins/axiomcode/skills/axiomcode/scripts/dl/impact.dl b/plugins/axiomcode/skills/axiomcode/scripts/dl/impact.dl index f8d70e36..34dd38b5 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/dl/impact.dl +++ b/plugins/axiomcode/skills/axiomcode/scripts/dl/impact.dl @@ -937,7 +937,9 @@ parent_up(q, a, b, t) :- reach(q, a, d), d > 0, reach(q, b, d1), d1 = d - 1, fw_ // shape — which is the whole argument for keeping cases beside a corpus. .decl up_running(q:symbol, m:symbol, d:number) up_running(q, m, 0) :- seed(q, m). -up_running(q, a, d+1) :- up_running(q, b, d), edge(a, b, "known_edge"), !state_gate(a, b, _), d < 40. +// a module body's call made only under `if __name__ == "__main__":` runs when the file is a script, never on import +.decl guard_only(a:symbol, b:symbol) .input guard_only +up_running(q, a, d+1) :- up_running(q, b, d), edge(a, b, "known_edge"), !state_gate(a, b, _), !guard_only(a, b), d < 40. // the same gate as `up` (above): a caller that leaves the instance code keeps the route only on an instance given f .decl up_running_g(q:symbol, m:symbol, t:symbol, f:symbol, d:number) up_running_g(q, a, t, f, d+1) :- up_running(q, f, d), state_gate(a, f, t), edge(a, f, "known_edge"), d < 40. @@ -1080,6 +1082,16 @@ fw_edge(a, b, "remote") :- remote(a, b, _, _, _), a != b. // the test methods that send a request get the hop; its own rung, `registered`, never a call chain. .decl filter_run(a:symbol, b:symbol, how:symbol) .input filter_run fw_edge(a, b, "filter") :- filter_run(a, b, _), a != b. +// A LIBRARY CALLING BACK INTO THE CLIENT. The engine's lib_callback_edge (Python framework-behavior/library-callbacks.dl): +// a call into an installed library that the library's own source is summarised to hand back to a client member -- a +// test client's get() running the open() a client subclass overrides, a runner running the command it was handed. +// No call site names the client member, so without this hop a test that drives the library never reached it. Its own +// rung, `library callback`: a library's behaviour joined by a member name, never a resolved call. +.decl libcb(a:symbol, b:symbol, callee:symbol, member:symbol) .input libcb +fw_edge(a, b, "library callback") :- libcb(a, b, _, _), a != b. +// and the member's own answer names who reaches it that way, worded as a framework hop +direct(q, c, "uses", cat("framework-mediated, not a call: a library it calls hands this back (", cat(callee, cat(" calls ", cat(member, ")")))), "framework", "", 0) + :- target(q, "method", m, _), libcb(c, m, callee, member), c != m. // and the filter's own answer names who runs it, worded as a framework hop (the `framework` rule above) direct(q, c, "uses", cat("framework-mediated, not a call: servlet_filter via ", cat(how, " [registered]")), "framework", "", 0) :- target(q, "method", m, _), filter_run(c, m, how), c != m. @@ -1092,6 +1104,16 @@ direct(q, c, "uses", cat("framework-mediated, not a call: servlet_filter via ", // twice, which is the shape that made an override count as both contract and reached. .decl protocol_member(t:symbol, m:symbol) protocol_member(t, m) :- member(t, m, n, _), match("__.*__", n), n != "__init__". +// Showing a value (`__repr__`, `__str__`, `__format__`), pickling or copying it (`__getstate__`, `__setstate__`, +// `__reduce__`, ...) and deleting from it (`__delitem__`, `__delattr__`, `__del__`) happen only when some code asks, +// so a constructor deep inside other code says nothing about whether they run, and taking the hop from every one of +// them named every test above it. A TEST that builds the object is the one that asks: for these the hop is taken +// from test code only. +.decl protocol_on_demand(n:symbol) +protocol_on_demand("__repr__"). protocol_on_demand("__str__"). protocol_on_demand("__format__"). +protocol_on_demand("__getstate__"). protocol_on_demand("__setstate__"). protocol_on_demand("__getnewargs__"). +protocol_on_demand("__getnewargs_ex__"). protocol_on_demand("__reduce__"). protocol_on_demand("__reduce_ex__"). +protocol_on_demand("__delitem__"). protocol_on_demand("__delattr__"). protocol_on_demand("__del__"). .decl constructs(c:symbol, t:symbol) constructs(c, t) :- member(t, i, "__init__", _), edge(c, i, _), c != i. // The hop is taken only where the CHANGE ITSELF is a protocol member, not wherever one turns up in the closure. @@ -1100,7 +1122,9 @@ constructs(c, t) :- member(t, i, "__init__", _), edge(c, i, _), c != i. // that cost 115 false pairs for 46 true ones. Narrowed to the seed it keeps what it was written for — a // declaration with NO call site by design, whose callers are whoever built the object — and drops the rest. .decl protocol_hop(q:symbol, c:symbol, m:symbol) -protocol_hop(q, c, m) :- seed(q, m), protocol_member(t, m), constructs(c, t), c != m. +protocol_hop(q, c, m) :- seed(q, m), protocol_member(t, m), member(t, m, n, _), !protocol_on_demand(n), constructs(c, t), c != m. +protocol_hop(q, c, m) :- seed(q, m), protocol_member(t, m), member(t, m, n, _), protocol_on_demand(n), constructs(c, t), c != m, + decl_file(c, f), is_test_file(f). up(q, c, 1) :- protocol_hop(q, c, _). parent_up(q, a, b, "protocol") :- reach(q, a, d), d > 0, reach(q, b, d1), d1 = d - 1, protocol_hop(q, a, b). // WHAT RUNS AT IMPORT. `settings = Settings()` at the top level of a module is executed by the import statement, diff --git a/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py b/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py index ecf87c75..94bc824e 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/graph_sql.py @@ -3561,6 +3561,9 @@ def _has_framework_hops(q, at=None, site_file=None): try: if _has(q, 'ext_decorated_name_target') and q("SELECT 1 FROM ext_decorated_name_target WHERE c0 <> c1 LIMIT 1"): return True + # fw_edge "library callback": a library summarised to call a client member back (library-callbacks.dl) + if _has(q, 'ext_lib_callback_edge') and q("SELECT 1 FROM ext_lib_callback_edge WHERE c0 <> c1 LIMIT 1"): + return True if _has(q, 'symbols') and q("SELECT 1 FROM symbols WHERE file LIKE '%conftest.py' AND method_id IS NOT NULL LIMIT 1"): return True # a fixture injected by name anywhere, conftest or not: the rules credit it only to the tests that request diff --git a/tests/cases/python/decorator-returning-update-wrapper/case.json b/tests/cases/python/decorator-returning-update-wrapper/case.json new file mode 100644 index 00000000..58328af4 --- /dev/null +++ b/tests/cases/python/decorator-returning-update-wrapper/case.json @@ -0,0 +1,13 @@ +{"lang": "python", "src": ".", + "checks": [ + {"why": "a decorator returning functools.update_wrapper(wrapper, f) hands back `wrapper`: a call to the decorated method resolves to the wrapper, and the caller reaches the method through it rather than by name", + "run": ["impact", "Scaffold.other"], + "want": ["2 hop(s) App.use"], + "avoid": ["[by name] App.use"]}, + {"why": "the typed spelling `return t.cast(F, update_wrapper(wrapper, f))`, typing imported as an alias, is the same shape", + "run": ["impact", "Scaffold.route"], + "want": ["2 hop(s) App.get"], + "avoid": ["[by name] App.get"]}, + {"why": "CONTROL: an undecorated method keeps its resolved caller", + "run": ["impact", "Scaffold.add"], + "want": ["[resolved] Scaffold.route"]}]} diff --git a/tests/cases/python/decorator-returning-update-wrapper/pkg/__init__.py b/tests/cases/python/decorator-returning-update-wrapper/pkg/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/decorator-returning-update-wrapper/pkg/scaffold.py b/tests/cases/python/decorator-returning-update-wrapper/pkg/scaffold.py new file mode 100644 index 00000000..08c14f15 --- /dev/null +++ b/tests/cases/python/decorator-returning-update-wrapper/pkg/scaffold.py @@ -0,0 +1,45 @@ +import typing as t +from functools import update_wrapper + +F = t.TypeVar("F", bound=t.Callable[..., t.Any]) + + +def setupmethod(f: F) -> F: + f_name = f.__name__ + + def wrapper_func(self, *args, **kwargs): + self._check(f_name) + return f(self, *args, **kwargs) + + return t.cast(F, update_wrapper(wrapper_func, f)) + + +def plain(f): + def inner(self, *args): + return f(self, *args) + + return update_wrapper(inner, f) + + +class Scaffold: + def _check(self, n): + return n + + @setupmethod + def route(self, rule): + return self.add(rule) + + @plain + def other(self, x): + return x + + def add(self, rule): + return rule + + +class App(Scaffold): + def get(self, rule): + return self.route(rule) + + def use(self): + return self.other(1) diff --git a/tests/cases/python/fixture-value-types-the-parameter/app/__init__.py b/tests/cases/python/fixture-value-types-the-parameter/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/fixture-value-types-the-parameter/app/core.py b/tests/cases/python/fixture-value-types-the-parameter/app/core.py new file mode 100644 index 00000000..687b3245 --- /dev/null +++ b/tests/cases/python/fixture-value-types-the-parameter/app/core.py @@ -0,0 +1,26 @@ +class Session: + def __init__(self, name): + self.name = name + + def begin(self): + return Transaction(self) + + def close(self): + return None + + +class Transaction: + def __init__(self, session): + self.session = session + + def commit(self): + return validate(self.session.name) + + +def validate(name): + return bool(name) + + +class Ledger: + def post(self, amount): + return amount diff --git a/tests/cases/python/fixture-value-types-the-parameter/case.json b/tests/cases/python/fixture-value-types-the-parameter/case.json new file mode 100644 index 00000000..828a6f45 --- /dev/null +++ b/tests/cases/python/fixture-value-types-the-parameter/case.json @@ -0,0 +1,17 @@ +{"lang": "python", "src": ".", + "checks": [ + {"why": "a test parameter the runner fills from a yield fixture is typed by what the fixture yields, so a method called on it resolves and the test reaches the change", + "run": ["impact", "Session.begin", "--tests"], + "want": ["test_tx.py"], + "avoid": ["test_ledger.py"]}, + {"why": "the type carries through a local derived from the parameter (`tx = session.begin(); tx.commit()`), two calls below the fixture", + "run": ["impact", "validate", "--tests"], + "want": ["test_tx.py"], + "avoid": ["test_ledger.py"]}, + {"why": "a return fixture types its parameter the same way", + "run": ["impact", "Ledger.post", "--tests"], + "want": ["test_ledger.py"], + "avoid": ["test_tx.py"]}, + {"why": "CONTROL: a method no test calls on a fixture value reaches no test file through it", + "run": ["impact", "Session.close", "--tests"], + "avoid": ["test_ledger.py"]}]} diff --git a/tests/cases/python/fixture-value-types-the-parameter/tests/conftest.py b/tests/cases/python/fixture-value-types-the-parameter/tests/conftest.py new file mode 100644 index 00000000..12d7146d --- /dev/null +++ b/tests/cases/python/fixture-value-types-the-parameter/tests/conftest.py @@ -0,0 +1,15 @@ +import pytest + +from app.core import Ledger, Session + + +@pytest.fixture +def session(): + s = Session("db") + yield s + s.close() + + +@pytest.fixture +def ledger(): + return Ledger() diff --git a/tests/cases/python/fixture-value-types-the-parameter/tests/test_ledger.py b/tests/cases/python/fixture-value-types-the-parameter/tests/test_ledger.py new file mode 100644 index 00000000..54302900 --- /dev/null +++ b/tests/cases/python/fixture-value-types-the-parameter/tests/test_ledger.py @@ -0,0 +1,2 @@ +def test_post(ledger): + assert ledger.post(3) == 3 diff --git a/tests/cases/python/fixture-value-types-the-parameter/tests/test_tx.py b/tests/cases/python/fixture-value-types-the-parameter/tests/test_tx.py new file mode 100644 index 00000000..54cfaa2e --- /dev/null +++ b/tests/cases/python/fixture-value-types-the-parameter/tests/test_tx.py @@ -0,0 +1,3 @@ +def test_commit(session): + tx = session.begin() + assert tx.commit() diff --git a/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/clikit/__init__.py b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/clikit/__init__.py new file mode 100644 index 00000000..031798a8 --- /dev/null +++ b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/clikit/__init__.py @@ -0,0 +1 @@ +from .core import Command, command diff --git a/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/clikit/core.py b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/clikit/core.py new file mode 100644 index 00000000..b376dd56 --- /dev/null +++ b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/clikit/core.py @@ -0,0 +1,28 @@ +class Context: + def __init__(self, command): + self.command = command + + def invoke(self, callback): + return callback() + + +class Command: + def __init__(self, name, callback=None): + self.name = name + self.callback = callback + + def make_context(self, args): + return Context(self) + + def invoke(self, ctx: Context): + return ctx.invoke(self.callback) + + def main(self, args): + ctx = self.make_context(args) + return self.invoke(ctx) + + +def command(name=None): + def decorator(f): + return Command(name or f.__name__, callback=f) + return decorator diff --git a/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/clikit/testing.py b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/clikit/testing.py new file mode 100644 index 00000000..0f5f711e --- /dev/null +++ b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/clikit/testing.py @@ -0,0 +1,6 @@ +class CliRunner: + def __init__(self, charset="utf-8"): + self.charset = charset + + def invoke(self, cli, args=None): + return cli.main(args or []) diff --git a/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/hooks/__init__.py b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/hooks/__init__.py new file mode 100644 index 00000000..09edbf15 --- /dev/null +++ b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/hooks/__init__.py @@ -0,0 +1,6 @@ +def fire(listener): + listener.on_event("x") + + +def keep(listener): + return [listener] diff --git a/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/tmpl/__init__.py b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/tmpl/__init__.py new file mode 100644 index 00000000..397ea642 --- /dev/null +++ b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/tmpl/__init__.py @@ -0,0 +1,15 @@ +class BaseLoader: + def load(self, env, name): + return self.get_source(env, name) + + def get_source(self, env, name): + raise NotImplementedError + + +class Environment: + def __init__(self, loader: "BaseLoader" = None, **options): + self.loader = loader + self.options = options + + def get_template(self, name): + return self.loader.load(self, name) diff --git a/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/webkit/__init__.py b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/webkit/__init__.py new file mode 100644 index 00000000..2d2fd002 --- /dev/null +++ b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/webkit/__init__.py @@ -0,0 +1 @@ +from .testing import Client diff --git a/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/webkit/testing.py b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/webkit/testing.py new file mode 100644 index 00000000..10ef5bbe --- /dev/null +++ b/tests/cases/python/library-calls-back/.venv/lib/python3.12/site-packages/webkit/testing.py @@ -0,0 +1,14 @@ +def run_app(app, environ): + return app(environ, None) + + +class Client: + def __init__(self, application, use_cookies=True): + self.application = application + self.use_cookies = use_cookies + + def open(self, path): + return run_app(self.application, {"PATH": path}) + + def get(self, path): + return self.open(path) diff --git a/tests/cases/python/library-calls-back/.venv/pyvenv.cfg b/tests/cases/python/library-calls-back/.venv/pyvenv.cfg new file mode 100644 index 00000000..5c46911c --- /dev/null +++ b/tests/cases/python/library-calls-back/.venv/pyvenv.cfg @@ -0,0 +1,2 @@ +home = /nonexistent/bin +version_info = 3.12.0 diff --git a/tests/cases/python/library-calls-back/app/__init__.py b/tests/cases/python/library-calls-back/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/library-calls-back/app/cli.py b/tests/cases/python/library-calls-back/app/cli.py new file mode 100644 index 00000000..3199a2c3 --- /dev/null +++ b/tests/cases/python/library-calls-back/app/cli.py @@ -0,0 +1,12 @@ +import clikit +from clikit.core import Command + + +class AppGroup(Command): + def make_context(self, args): + return super().make_context(args) + + +@clikit.command() +def greet(): + return "hi" diff --git a/tests/cases/python/library-calls-back/app/events.py b/tests/cases/python/library-calls-back/app/events.py new file mode 100644 index 00000000..992ff07d --- /dev/null +++ b/tests/cases/python/library-calls-back/app/events.py @@ -0,0 +1,22 @@ +from hooks import fire, keep + + +class Listener: + def on_event(self, e): + return e + + +class Kept: + def on_event(self, e): + return e + + +class Other: + def on_event(self, e): + return e + + +def emit(): + fire(Listener()) + keep(Kept()) + return Other() diff --git a/tests/cases/python/library-calls-back/app/render.py b/tests/cases/python/library-calls-back/app/render.py new file mode 100644 index 00000000..4f89f726 --- /dev/null +++ b/tests/cases/python/library-calls-back/app/render.py @@ -0,0 +1,20 @@ +from tmpl import BaseLoader, Environment + + +class AppLoader(BaseLoader): + def get_source(self, env, name): + return name + + +class OtherLoader: + def get_source(self, env, name): + return name + + +def make_env(opts): + return Environment(**opts) + + +def render(opts): + env = make_env(opts) + return env.get_template("index") diff --git a/tests/cases/python/library-calls-back/app/testing.py b/tests/cases/python/library-calls-back/app/testing.py new file mode 100644 index 00000000..6e2e205b --- /dev/null +++ b/tests/cases/python/library-calls-back/app/testing.py @@ -0,0 +1,4 @@ +# a project module whose name is also the last segment of a dependency's module (`clikit.testing`): an import of the +# dependency's module must not resolve here +def make_args(): + return [] diff --git a/tests/cases/python/library-calls-back/app/web.py b/tests/cases/python/library-calls-back/app/web.py new file mode 100644 index 00000000..86a22ceb --- /dev/null +++ b/tests/cases/python/library-calls-back/app/web.py @@ -0,0 +1,22 @@ +from webkit import Client + + +class App: + def __call__(self, environ, start): + return self.dispatch(environ) + + def dispatch(self, environ): + return "ok" + + def test_client(self): + return AppClient(self) + + +class AppClient(Client): + def open(self, path): + return super().open(path) + + +class Unrelated: + def open(self, path): + return path diff --git a/tests/cases/python/library-calls-back/case.json b/tests/cases/python/library-calls-back/case.json new file mode 100644 index 00000000..91f0c6af --- /dev/null +++ b/tests/cases/python/library-calls-back/case.json @@ -0,0 +1,132 @@ +{ + "lang": "python", + "src": ".", + "checks": [ + { + "why": "a test client's get() runs the open() the client subclass overrides: the library calls it back, no call site names it", + "run": [ + "impact", + "AppClient.open", + "--tests" + ], + "want": [ + "tests/test_web.py", + "library callback" + ] + }, + { + "why": "the test client then calls the application it was built around (stored by the library's constructor, called as app(environ, start))", + "run": [ + "impact", + "App.dispatch", + "--tests" + ], + "want": [ + "tests/test_web.py" + ] + }, + { + "why": "CONTROL: an open() on a class no library calls back is reached only by its own test", + "run": [ + "impact", + "Unrelated.open", + "--tests" + ], + "want": [ + "test_unrelated" + ], + "avoid": [ + "test_index" + ] + }, + { + "why": "a CLI runner's invoke(cli) calls main() on the command, whose make_context() the client overrides", + "run": [ + "impact", + "AppGroup.make_context", + "--tests" + ], + "want": [ + "tests/test_cli.py", + "test_group" + ] + }, + { + "why": "a library function calls a member on the object handed to it", + "run": [ + "impact", + "Listener.on_event", + "--tests" + ], + "want": [ + "tests/test_events.py" + ] + }, + { + "why": "CONTROL: an object handed to a library function that never calls that member gains nothing", + "run": [ + "impact", + "Kept.on_event", + "--tests" + ], + "avoid": [ + "tests/test_events.py" + ] + }, + { + "why": "CONTROL: an object with the same member name that is never handed to the library gains nothing", + "run": [ + "impact", + "Other.on_event", + "--tests" + ], + "avoid": [ + "tests/test_events.py" + ] + }, + { + "why": "a function a library decorator wraps into a command object is called back when a runner invokes that object", + "run": [ + "impact", + "greet", + "--tests" + ], + "want": [ + "test_greet" + ] + }, + { + "why": "a field the library declares as its base type, filled where the client cannot be followed (**options): the client subclass of that type that defines the member is the one called back", + "run": [ + "impact", + "AppLoader.get_source", + "--tests" + ], + "want": [ + "tests/test_render.py" + ] + }, + { + "why": "CONTROL: a class with the same member that is not a subclass of the declared library type gains nothing", + "run": [ + "impact", + "OtherLoader.get_source", + "--tests" + ], + "avoid": [ + "tests/test_render.py" + ] + }, + { + "why": "path walks the library's hand-back too: from the test to the application the test client calls", + "run": [ + "path", + "test_index", + "App.dispatch" + ], + "want": [ + "library_callback via webkit.testing.Client.get -> __call__" + ] + } + ] +} diff --git a/tests/cases/python/library-calls-back/tests/__init__.py b/tests/cases/python/library-calls-back/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/library-calls-back/tests/test_cli.py b/tests/cases/python/library-calls-back/tests/test_cli.py new file mode 100644 index 00000000..e76e6998 --- /dev/null +++ b/tests/cases/python/library-calls-back/tests/test_cli.py @@ -0,0 +1,18 @@ +import pytest +from clikit.testing import CliRunner + +from app.cli import AppGroup, greet + + +@pytest.fixture +def runner(): + return CliRunner() + + +def test_group(runner): + cli = AppGroup("app") + runner.invoke(cli, []) + + +def test_greet(runner): + runner.invoke(greet) diff --git a/tests/cases/python/library-calls-back/tests/test_events.py b/tests/cases/python/library-calls-back/tests/test_events.py new file mode 100644 index 00000000..90947356 --- /dev/null +++ b/tests/cases/python/library-calls-back/tests/test_events.py @@ -0,0 +1,5 @@ +from app.events import emit + + +def test_emit(): + emit() diff --git a/tests/cases/python/library-calls-back/tests/test_render.py b/tests/cases/python/library-calls-back/tests/test_render.py new file mode 100644 index 00000000..f6d8a1e3 --- /dev/null +++ b/tests/cases/python/library-calls-back/tests/test_render.py @@ -0,0 +1,5 @@ +from app.render import AppLoader, render + + +def test_render(): + assert render({"loader": AppLoader()}) == "index" diff --git a/tests/cases/python/library-calls-back/tests/test_web.py b/tests/cases/python/library-calls-back/tests/test_web.py new file mode 100644 index 00000000..c6e4d193 --- /dev/null +++ b/tests/cases/python/library-calls-back/tests/test_web.py @@ -0,0 +1,16 @@ +import pytest + +from app.web import App, Unrelated + + +@pytest.fixture +def client(): + return App().test_client() + + +def test_index(client): + assert client.get("/") == "ok" + + +def test_unrelated(): + assert Unrelated().open("/") == "/" diff --git a/tests/cases/python/main-guard-does-not-run-on-import/case.json b/tests/cases/python/main-guard-does-not-run-on-import/case.json new file mode 100644 index 00000000..4d1ef066 --- /dev/null +++ b/tests/cases/python/main-guard-does-not-run-on-import/case.json @@ -0,0 +1,46 @@ +{ + "lang": "python", + "src": ".", + "checks": [ + { + "why": "a call made only under `if __name__ == \"__main__\":` runs when the file is a script, not on import: breaking `show` breaks no file that merely imports the module", + "run": [ + "impact", + "show", + "--tests" + ], + "want": [ + "0 of 2 test method(s) reach the change" + ], + "avoid": [ + "at import" + ] + }, + { + "why": "a guard written as a conjunction (`cond and __name__ == \"__main__\"`) is still false on import", + "run": [ + "impact", + "banner", + "--tests" + ], + "want": [ + "0 of 2 test method(s) reach the change" + ], + "avoid": [ + "at import" + ] + }, + { + "why": "CONTROL: a call the module body makes OUTSIDE the guard does run on import, so an importer breaks with `render`", + "run": [ + "impact", + "render", + "--tests" + ], + "want": [ + "test_other.", + "1 at import" + ] + } + ] +} \ No newline at end of file diff --git a/tests/cases/python/main-guard-does-not-run-on-import/pkg/__init__.py b/tests/cases/python/main-guard-does-not-run-on-import/pkg/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/main-guard-does-not-run-on-import/pkg/demo.py b/tests/cases/python/main-guard-does-not-run-on-import/pkg/demo.py new file mode 100644 index 00000000..96beb1b6 --- /dev/null +++ b/tests/cases/python/main-guard-does-not-run-on-import/pkg/demo.py @@ -0,0 +1,21 @@ +def render(x): + return str(x) + + +def show(): + return render(1) + + +TABLE = render("header") + + +if __name__ == "__main__": + print(show()) + + +def banner(): + return render("b") + + +if render and __name__ == "__main__": + banner() diff --git a/tests/cases/python/main-guard-does-not-run-on-import/tests/test_demo.py b/tests/cases/python/main-guard-does-not-run-on-import/tests/test_demo.py new file mode 100644 index 00000000..1a974c9c --- /dev/null +++ b/tests/cases/python/main-guard-does-not-run-on-import/tests/test_demo.py @@ -0,0 +1,5 @@ +from pkg import demo + + +def test_render(): + assert demo.render(2) == "2" diff --git a/tests/cases/python/main-guard-does-not-run-on-import/tests/test_other.py b/tests/cases/python/main-guard-does-not-run-on-import/tests/test_other.py new file mode 100644 index 00000000..5e8f6973 --- /dev/null +++ b/tests/cases/python/main-guard-does-not-run-on-import/tests/test_other.py @@ -0,0 +1,5 @@ +import pkg.demo + + +def test_nothing(): + assert True diff --git a/tests/cases/python/on-demand-dunder-hop-from-tests/case.json b/tests/cases/python/on-demand-dunder-hop-from-tests/case.json new file mode 100644 index 00000000..8ca71978 --- /dev/null +++ b/tests/cases/python/on-demand-dunder-hop-from-tests/case.json @@ -0,0 +1,9 @@ +{"lang": "python", "src": ".", + "checks": [ + {"why": "showing a value is on demand: a test that builds the object is taken for `__repr__`, a constructor deep in the source is not", + "run": ["impact", "Point.__repr__", "--tests"], + "want": ["test_point.py"], + "avoid": ["test_use.py"]}, + {"why": "CONTROL: a protocol a constructed object does not avoid (`__eq__`) keeps the hop from every constructor", + "run": ["impact", "Point.__eq__", "--tests"], + "want": ["test_use.py", "test_point.py"]}]} diff --git a/tests/cases/python/on-demand-dunder-hop-from-tests/pkg/__init__.py b/tests/cases/python/on-demand-dunder-hop-from-tests/pkg/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/on-demand-dunder-hop-from-tests/pkg/model.py b/tests/cases/python/on-demand-dunder-hop-from-tests/pkg/model.py new file mode 100644 index 00000000..c37360f8 --- /dev/null +++ b/tests/cases/python/on-demand-dunder-hop-from-tests/pkg/model.py @@ -0,0 +1,17 @@ +class Point: + def __init__(self, x, y): + self.x, self.y = x, y + + def __repr__(self): + return f"Point({self.x}, {self.y})" + + def __eq__(self, other): + return (self.x, self.y) == (other.x, other.y) + + +def make(): + return Point(1, 2) + + +def use(): + return make() diff --git a/tests/cases/python/on-demand-dunder-hop-from-tests/tests/test_point.py b/tests/cases/python/on-demand-dunder-hop-from-tests/tests/test_point.py new file mode 100644 index 00000000..c0b95782 --- /dev/null +++ b/tests/cases/python/on-demand-dunder-hop-from-tests/tests/test_point.py @@ -0,0 +1,5 @@ +from pkg.model import Point + + +def test_repr(): + assert repr(Point(1, 2)) == "Point(1, 2)" diff --git a/tests/cases/python/on-demand-dunder-hop-from-tests/tests/test_use.py b/tests/cases/python/on-demand-dunder-hop-from-tests/tests/test_use.py new file mode 100644 index 00000000..17baa6db --- /dev/null +++ b/tests/cases/python/on-demand-dunder-hop-from-tests/tests/test_use.py @@ -0,0 +1,5 @@ +from pkg.model import use + + +def test_use(): + assert use() is not None diff --git a/tests/cases/python/optional-and-union-annotations/case.json b/tests/cases/python/optional-and-union-annotations/case.json new file mode 100644 index 00000000..5747f72b --- /dev/null +++ b/tests/cases/python/optional-and-union-annotations/case.json @@ -0,0 +1,12 @@ +{"lang": "python", "src": ".", + "checks": [ + {"why": "a local annotated `Optional[Foo]` is a Foo, as the same annotation on a parameter already was", + "run": ["impact", "Foo.go"], + "want": ["[resolved] l_optional"]}, + {"why": "a three-operand PEP 604 union nests left-associatively; every operand counts, on a parameter and on a local", + "run": ["impact", "Bar.go"], + "want": ["p_pipe3", "l_pipe"], + "avoid": ["[by name] p_pipe3", "[by name] l_pipe"]}, + {"why": "CONTROL: a two-operand union and a plain annotation resolve as before", + "run": ["impact", "Foo.go"], + "want": ["[resolved] p_pipe", "[resolved] p_plain"]}]} diff --git a/tests/cases/python/optional-and-union-annotations/pkg/__init__.py b/tests/cases/python/optional-and-union-annotations/pkg/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/optional-and-union-annotations/pkg/m.py b/tests/cases/python/optional-and-union-annotations/pkg/m.py new file mode 100644 index 00000000..71acab0a --- /dev/null +++ b/tests/cases/python/optional-and-union-annotations/pkg/m.py @@ -0,0 +1,46 @@ +import typing as t +from typing import Optional, Union + + +class Foo: + def go(self): + return 1 + + +class Bar: + def go(self): + return 2 + + +def p_plain(x: Foo): + return x.go() + + +def p_optional(x: Optional[Foo]): + return x.go() + + +def p_t_optional(x: t.Optional[Foo]): + return x.go() + + +def p_pipe(x: Foo | None): + return x.go() + + +def p_union(x: Union[Foo, Bar]): + return x.go() + + +def l_optional(y): + z: Optional[Foo] = y + return z.go() + + +def l_pipe(y): + z: Foo | Bar | None = y + return z.go() + + +def p_pipe3(x: Foo | Bar | None): + return x.go() diff --git a/tests/cases/python/property-setter-is-a-call/case.json b/tests/cases/python/property-setter-is-a-call/case.json new file mode 100644 index 00000000..ef9c8fbb --- /dev/null +++ b/tests/cases/python/property-setter-is-a-call/case.json @@ -0,0 +1,16 @@ +{"lang": "python", "src": ".", + "checks": [ + {"why": "assigning `obj.x = v` runs x's @x.setter: a test that assigns reaches a change to the setter, one that only reads does not", + "run": ["impact", "pkg/m.py:10", "--tests"], + "want": ["test_write.py"], + "avoid": ["test_read.py"]}, + {"why": "`del obj.x` runs x's @x.deleter", + "run": ["impact", "pkg/m.py:14", "--tests"], + "want": ["test_delete.py"], + "avoid": ["test_read.py"]}, + {"why": "a setter's own calls are reached through the write too", + "run": ["impact", "check", "--tests"], + "want": ["test_write.py"]}, + {"why": "CONTROL: the getter keeps its readers", + "run": ["impact", "pkg/m.py:6", "--tests"], + "want": ["test_read.py"]}]} diff --git a/tests/cases/python/property-setter-is-a-call/pkg/__init__.py b/tests/cases/python/property-setter-is-a-call/pkg/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/python/property-setter-is-a-call/pkg/m.py b/tests/cases/python/property-setter-is-a-call/pkg/m.py new file mode 100644 index 00000000..49ecf51e --- /dev/null +++ b/tests/cases/python/property-setter-is-a-call/pkg/m.py @@ -0,0 +1,31 @@ +class Box: + def __init__(self): + self._v = 0 + + @property + def value(self): + return self._v + + @value.setter + def value(self, v): + self._v = check(v) + + @value.deleter + def value(self): + self._v = None + + +def check(v): + return v + + +def fill(b: Box): + b.value = 3 + + +def clear(b: Box): + del b.value + + +def read(b: Box): + return b.value diff --git a/tests/cases/python/property-setter-is-a-call/tests/test_delete.py b/tests/cases/python/property-setter-is-a-call/tests/test_delete.py new file mode 100644 index 00000000..fd8cf303 --- /dev/null +++ b/tests/cases/python/property-setter-is-a-call/tests/test_delete.py @@ -0,0 +1,7 @@ +from pkg.m import Box + + +def test_delete(): + b = Box() + del b.value + assert b.value is None diff --git a/tests/cases/python/property-setter-is-a-call/tests/test_read.py b/tests/cases/python/property-setter-is-a-call/tests/test_read.py new file mode 100644 index 00000000..843cb79f --- /dev/null +++ b/tests/cases/python/property-setter-is-a-call/tests/test_read.py @@ -0,0 +1,5 @@ +from pkg.m import Box + + +def test_read_only(): + assert Box().value == 0 diff --git a/tests/cases/python/property-setter-is-a-call/tests/test_write.py b/tests/cases/python/property-setter-is-a-call/tests/test_write.py new file mode 100644 index 00000000..b88149b7 --- /dev/null +++ b/tests/cases/python/property-setter-is-a-call/tests/test_write.py @@ -0,0 +1,7 @@ +from pkg.m import Box + + +def test_assign(): + b = Box() + b.value = 4 + assert b.value == 4 diff --git a/tests/cases/python/src-layout-conftest-import-time/case.json b/tests/cases/python/src-layout-conftest-import-time/case.json new file mode 100644 index 00000000..2efeb4d3 --- /dev/null +++ b/tests/cases/python/src-layout-conftest-import-time/case.json @@ -0,0 +1,11 @@ +{"lang": "python", "src": ".", + "checks": [ + {"why": "a src-layout package is imported by its package name: `from app import lookup` in a test is an import of src/app/__init__.py, so the test file reaches what that import runs", + "run": ["impact", "src/app/registry.py:4", "--tests"], + "want": ["test_lookup.py"]}, + {"why": "pytest imports tests/conftest.py before every test file under it, so a function that runs while the package is imported (normalise, through the module-level @register) breaks a test file that never imports the package itself", + "run": ["impact", "src/app/registry.py:4", "--tests"], + "want": ["test_plain.py"]}, + {"why": "CONTROL: describe runs at no import, so a test file that does not call it is not reached by the import walk", + "run": ["impact", "src/app/registry.py:19", "--tests"], + "avoid": ["test_plain.py"]}]} diff --git a/tests/cases/python/src-layout-conftest-import-time/pyproject.toml b/tests/cases/python/src-layout-conftest-import-time/pyproject.toml new file mode 100644 index 00000000..6d1bd6e7 --- /dev/null +++ b/tests/cases/python/src-layout-conftest-import-time/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "app" +version = "0.0.0" + +[tool.pytest.ini_options] +testpaths = "tests" diff --git a/tests/cases/python/src-layout-conftest-import-time/src/app/__init__.py b/tests/cases/python/src-layout-conftest-import-time/src/app/__init__.py new file mode 100644 index 00000000..3df798e6 --- /dev/null +++ b/tests/cases/python/src-layout-conftest-import-time/src/app/__init__.py @@ -0,0 +1 @@ +from .registry import register, lookup diff --git a/tests/cases/python/src-layout-conftest-import-time/src/app/registry.py b/tests/cases/python/src-layout-conftest-import-time/src/app/registry.py new file mode 100644 index 00000000..43ce0e0a --- /dev/null +++ b/tests/cases/python/src-layout-conftest-import-time/src/app/registry.py @@ -0,0 +1,25 @@ +_TABLE = {} + + +def normalise(name): + return name.strip().lower() + + +def register(name): + def deco(fn): + _TABLE[normalise(name)] = fn + return fn + return deco + + +def lookup(name): + return _TABLE[normalise(name)] + + +def describe(name): + return f"handler {normalise(name)}" + + +@register("Default") +def default_handler(): + return "default" diff --git a/tests/cases/python/src-layout-conftest-import-time/tests/conftest.py b/tests/cases/python/src-layout-conftest-import-time/tests/conftest.py new file mode 100644 index 00000000..aa144041 --- /dev/null +++ b/tests/cases/python/src-layout-conftest-import-time/tests/conftest.py @@ -0,0 +1,8 @@ +import pytest + +import app + + +@pytest.fixture +def table(): + return app diff --git a/tests/cases/python/src-layout-conftest-import-time/tests/test_lookup.py b/tests/cases/python/src-layout-conftest-import-time/tests/test_lookup.py new file mode 100644 index 00000000..539bd9a1 --- /dev/null +++ b/tests/cases/python/src-layout-conftest-import-time/tests/test_lookup.py @@ -0,0 +1,5 @@ +from app import lookup + + +def test_lookup(): + assert lookup("default")() == "default" diff --git a/tests/cases/python/src-layout-conftest-import-time/tests/test_plain.py b/tests/cases/python/src-layout-conftest-import-time/tests/test_plain.py new file mode 100644 index 00000000..2733328f --- /dev/null +++ b/tests/cases/python/src-layout-conftest-import-time/tests/test_plain.py @@ -0,0 +1,2 @@ +def test_arithmetic(): + assert 1 + 1 == 2