diff --git a/src/bundle/SCHEMA.md b/src/bundle/SCHEMA.md index 307c43977..7a1ab5880 100644 --- a/src/bundle/SCHEMA.md +++ b/src/bundle/SCHEMA.md @@ -26,12 +26,13 @@ Identifiers are the parser's hashes and are opaque; join them to `methods` / `ty 1. This is a call graph of one codebase, derived by a type-directed Datalog engine. Start with `SELECT value FROM run WHERE key='language'` — every language-specific fact below is keyed on it. 2. The graph is `call_edges`: one row per (call site, possible target). Rows join to `methods` (names, files, lines) on `caller_id` / `callee_method_id`, and to `call_sites` on `call_site_id` for where the call is written. Identifiers are opaque hashes — never parse them, always join. 3. Trust is explicit. `tier` says what kind of claim a row is: `known_edge` (one resolved target), `multi_inferred` (a sound set — every row of the set is a real possibility), `boundary_lib` (leaves the client; not expanded further), `ambiguous_*` (a declared unknown: callee is NULL). Pick the tiers your question tolerates and filter on them; never treat an `ambiguous_*` row as an edge. -4. Before answering "nothing calls X" or "X cannot reach Y", check `unresolved_sites` for the methods on the path: a caller listed there has a call the engine could not resolve, so the answer is a lower bound and should say so. -5. Library targets (`callee_provenance = lib`) are named in `methods` with `provenance = lib` but their bodies were not analysed; a Python `builtin`/`external` target has no methods row and lives in `callee_label`. -6. `schema_vocab` lists every value a column can hold FOR THIS LANGUAGE with its meaning — filter on `language = (SELECT value FROM run WHERE key='language')`. `schema_notes` lists the caveats for this language (empty tables, what an id may point at). Read both before interpreting `kind`, `tier` or an empty table. -7. `schema_queries` holds tested SQL for the common questions (callers, callees, blast radius, entry reachability, the method at a file:line, the blind spots of a method). Bind the named parameters and run. -8. Tables named `ext_` are the language's raw engine relations with positional columns c0…cN; `schema_tables` carries each one's description lifted from its rule. Use them only when a core table does not hold what you need. -9. When you report a result, carry the tier and the unresolved count with it. A consumer who cannot see the confidence of an edge cannot use it. +4. `call_edges` is what the engine CONCLUDED; `dispatch_candidates` is what the hierarchy ADMITTED. Read the second when you need an upper bound rather than a best answer — a candidate whose owner is absent from `type_instantiated` is admitted by the hierarchy but never constructed in this run, which is how you narrow it yourself. `basis` separates a declared relationship from a shape match. +5. Before answering "nothing calls X" or "X cannot reach Y", check `unresolved_sites` for the methods on the path: a caller listed there has a call the engine could not resolve, so the answer is a lower bound and should say so. +6. Library targets (`callee_provenance = lib`) are named in `methods` with `provenance = lib` but their bodies were not analysed; a Python `builtin`/`external` target has no methods row and lives in `callee_label`. +7. `schema_vocab` lists every value a column can hold FOR THIS LANGUAGE with its meaning — filter on `language = (SELECT value FROM run WHERE key='language')`. `schema_notes` lists the caveats for this language (empty tables, what an id may point at). Read both before interpreting `kind`, `tier` or an empty table. +8. `schema_queries` holds tested SQL for the common questions (callers, callees, blast radius, entry reachability, the method at a file:line, the blind spots of a method, the dispatch envelope of a method). Bind the named parameters and run. +9. Tables named `ext_` are the language's raw engine relations with positional columns c0…cN; `schema_tables` carries each one's description lifted from its rule. Use them only when a core table does not hold what you need. +10. When you report a result, carry the tier and the unresolved count with it. A consumer who cannot see the confidence of an edge cannot use it. ## Canonical queries (`schema_queries`) @@ -62,7 +63,7 @@ WHERE caller.qualified_name = :qualified_name ORDER BY s.start_line, target ``` -**`blast_radius`** — If this method changes, which methods are transitively affected, up to :depth hops, through sound edges only (known_edge and multi_inferred)? _(:qualified_name, :depth)_ +**`blast_radius`** — If this method changes, which methods are transitively affected, up to :depth hops, through RESOLVED client edges only (a declared unknown is not traversed, and the count of them is returned alongside)? _(:qualified_name, :depth)_ ```sql WITH RECURSIVE up(id, depth) AS ( @@ -70,7 +71,8 @@ WITH RECURSIVE up(id, depth) AS ( UNION SELECT e.caller_id, up.depth + 1 FROM call_edges e JOIN up ON e.callee_method_id = up.id - WHERE e.tier IN ('known_edge', 'multi_inferred') AND up.depth < :depth + WHERE e.tier IN ('known_edge', 'multi_inferred', 'ambient_terminal', 'intrinsic_terminal') + AND up.depth < :depth ) SELECT MIN(up.depth) AS depth, m.qualified_name, m.file_path, m.start_line, (SELECT count(*) FROM unresolved_sites u WHERE u.caller_id = m.id) AS unresolved_calls_inside @@ -108,6 +110,18 @@ WHERE m.qualified_name = :qualified_name ORDER BY s.start_line ``` +**`dispatch_envelope_of`** — What else might actually run at a call that resolves to this method — the set the graph narrowed from, and whether each candidate is a declaration or a shape match? _(:qualified_name)_ + +```sql +SELECT cand.qualified_name AS candidate, cand.file_path, cand.start_line, d.basis, + EXISTS (SELECT 1 FROM type_instantiated i WHERE i.type_id = cand.owner_type_id) AS owner_instantiated +FROM dispatch_candidates d +JOIN methods base ON base.id = d.base_method_id +JOIN methods cand ON cand.id = d.candidate_method_id +WHERE base.qualified_name = :qualified_name +ORDER BY d.basis, cand.qualified_name +``` + **`subtypes_of`** — Which types extend or implement this type (transitively)? _(:qualified_name)_ ```sql @@ -340,7 +354,7 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h | value | languages | meaning | |---|---|---| | `known_edge` | all | Exactly one target resolved. The strongest claim. | -| `multi_inferred` | all | A sound SET of possible targets (virtual dispatch over instantiated subtypes); each member is one row. The set over-approximates; no member is a guess. | +| `multi_inferred` | all | A sound SET of possible targets; each member is one row. The set over-approximates — every member is a real possibility, but not every member runs. HOW WIDE the set is differs by language: see the per-language notes on this table for whether the fan is narrowed by the instantiation set. | | `boundary_lib` | all | The target is outside the client (library, builtin, or unstaged external). The chain is not expanded past it here. | | `ambiguous_unknown` | all | Declared blind spot: the engine could not resolve the site (unresolved receiver, missing type, reflection…). callee is NULL. Never dropped. | | `ambiguous_anon` | java | Known structural gap: an anonymous-class creation has no candidate rule yet. callee is NULL. | @@ -400,6 +414,10 @@ THE GRAPH. One row per (site, resolved target). A site with N possible targets h - **python** — A `boundary_lib` edge may point at a builtin (callee_provenance builtin, callee_label `builtin:NAME`) or at an unstaged import path (callee_provenance external) — neither has a methods row. - **python** — The reason a site is ambiguous_unknown is exported per site in ext_call_site_unresolved (site, caller, reason, detail). +- **all** — THE TRUST LINE, and it is not the same set of tiers in every language. RESOLVED (callee_method_id is set): known_edge, multi_inferred, boundary_lib, and in TypeScript ALSO ambient_terminal and intrinsic_terminal. BLIND SPOT (callee is NULL): ambiguous_unknown, and in Java ALSO ambiguous_anon. A filter written as `tier IN (known_edge, multi_inferred)` therefore drops resolved edges in TypeScript and nowhere else — derive the set from this note or from unresolved_sites, never from a hardcoded list. +- **java** — A multi_inferred fan is CHA-wide: it is every override the hierarchy admits, bounded only by the dispatch cap. type_instantiated is computed and exported but NOT read by any rule, so the fan is not narrowed to types the program constructs. Narrow it yourself by joining dispatch_candidates to type_instantiated — see the dispatch_envelope_of query. +- **typescript** — A multi_inferred fan is CHA-wide, as in Java: type_instantiated is computed and exported but NOT read by any rule. The fan also has sources that are not virtual dispatch at all — an overload set or a union-typed receiver produces one too. +- **python** — A multi_inferred fan IS narrowed by the instantiation set: type_instantiated_reachable (the constructed classes and their bases) bounds dispatch in resolution/dispatch.dl. Python is the only front end where that narrowing is applied, so a fan here is tighter than the same shape would be in Java or TypeScript. - **all** — The raw relation has a seventh column, ToExpr, that is always `-` (reserved). It is dropped here. - **all** — An unresolved site (tier ambiguous_*) has NULL callee_method_id, callee_label and callee_provenance. The raw relation writes `-` in those slots. @@ -412,9 +430,27 @@ Transitive supertype closure: (type, ancestor) for every ancestor reachable thro | 0 | `type_id` 🔑 | TEXT | | FK → types.id. | | 1 | `ancestor_type_id` 🔑 | TEXT | | FK → types.id. | +### `dispatch_candidates` + +THE DISPATCH ENVELOPE: (base method, method that may run instead) for every call that statically resolves to the base. This is the set `call_edges` narrowed FROM — the difference between "these are the targets" and "these are the targets, out of these possibilities". Populated in every language; `basis` says what admitted the pair, because the three front ends admit by different means. + +| # | column | type | null | meaning | +|---|---|---|---|---| +| 0 | `base_method_id` 🔑 | TEXT | | FK → methods.id — the method a call resolves to statically. | +| 1 | `candidate_method_id` 🔑 | TEXT | | FK → methods.id — a method that may run instead at such a call. | +| 2 | `basis` 🔑 | TEXT | | What admitted the pair — see vocabulary. Filter on it to trust only declarations. | + +**`dispatch_candidates.basis` values** + +| value | languages | meaning | +|---|---|---| +| `nominal` | java, typescript | A written extends/implements reaches the candidate's owner from the base's owner. The strongest evidence there is: the author declared the relationship. | +| `structural` | typescript | No declaration; the candidate's owner satisfies the base's owner by SHAPE. Emitted only for supertypes with no nominal implementor at all, so it never competes with a declared answer — but it is a heuristic, and a consumer that wants declarations only filters it out. | +| `mro` | python | The subtype's C3 linearisation picks the candidate for that attribute name. Not merely "the subtype declares this name" — a name a sibling base wins is attributed to that sibling. | + ### `overrides` -Virtual-dispatch pairs: (base method, overriding method) wherever a call to the base may run the override. Java only today — see notes for what the other front ends offer instead. +Virtual-dispatch pairs: (base method, overriding method) wherever a call to the base may run the override. Java only, and kept for compatibility — it is exactly `dispatch_candidates` filtered to `basis = nominal`. Prefer `dispatch_candidates`, which is populated in every language. | # | column | type | null | meaning | |---|---|---|---|---| @@ -423,8 +459,8 @@ Virtual-dispatch pairs: (base method, overriding method) wherever a call to the **Notes** -- **typescript** — EMPTY. TypeScript dispatch is captured directly as multi_inferred edges; the structural and nominal implementor sets are in ext_implementors, ext_structural_implementor and ext_type_satisfies. -- **python** — EMPTY. Python method lookup is by MRO, exported positionally in ext_mro_position (type, ancestor, …, position). +- **typescript** — EMPTY — this table is Java-shaped. The TypeScript dispatch envelope is in dispatch_candidates, with basis `nominal` or `structural`. +- **python** — EMPTY — this table is Java-shaped. The Python dispatch envelope is in dispatch_candidates with basis `mro`; the raw linearisation is in ext_mro_position. ### `entry_points` @@ -473,7 +509,7 @@ The blind spots, attributed to the code that contains them: (caller, site) for e ### `type_instantiated` -Types the client actually creates an instance of — the rapid-type-analysis set that bounds virtual dispatch. (A subtype nothing instantiates cannot receive a dispatched call.) +Types this run creates an instance of — the rapid-type-analysis set that bounds virtual dispatch. (A subtype nothing instantiates cannot receive a dispatched call.) Deliberately an over-approximation: narrowing it on evidence the run does not have would lose real edges. Populated in every language. | # | column | type | null | meaning | |---|---|---|---|---| @@ -484,13 +520,13 @@ Types the client actually creates an instance of — the rapid-type-analysis set | value | languages | meaning | |---|---|---| -| `new` | java, python | A constructor call in the client. | +| `new` | all | A constructor call — `new C()` / `C()`. | | `anonymous` | java | An anonymous class exists only by being instantiated. | | `enum_constant` | java | An enum's constants are its instances. | **Notes** -- **typescript** — EMPTY. The TypeScript rule set does not export an instantiation set. +- **typescript** — Every row has how = `new`. Not restricted to client provenance: a type the library constructs is still a type that exists at run time, and dropping it would narrow the envelope unsoundly. - **python** — Every row has how = `new`: the rule set records that some client call constructs the class, not which form. ## Extended tables — `ext_` diff --git a/src/bundle/build.ts b/src/bundle/build.ts index edf818b0a..a3707e32e 100644 --- a/src/bundle/build.ts +++ b/src/bundle/build.ts @@ -23,6 +23,7 @@ export interface CoreTables { call_sites: Row[]; call_edges: Row[]; type_ancestors: Row[]; + dispatch_candidates: Row[]; overrides: Row[]; entry_points: Row[]; entry_reachable: Row[]; @@ -124,6 +125,7 @@ export async function buildCore(inp: BuildInputs): Promise { const rawEdges = await readSource(rawDir, A.raw.callEdges); // site, caller, callee, prov, tier, kind const rawAncestors = A.raw.typeAncestors ? await readSource(rawDir, A.raw.typeAncestors) : []; const rawOverrides = A.raw.overrides ? await readSource(rawDir, A.raw.overrides) : []; + const rawDispatch = A.raw.dispatchCandidates ? await readSource(rawDir, A.raw.dispatchCandidates) : []; const rawEntry = A.raw.entryPoints ? await readSource(rawDir, A.raw.entryPoints) : []; const rawReach = A.raw.entryReachable ? await readSource(rawDir, A.raw.entryReachable) : []; const rawInst = A.raw.typeInstantiated ? await readSource(rawDir, A.raw.typeInstantiated) : []; @@ -304,6 +306,7 @@ export async function buildCore(inp: BuildInputs): Promise { call_sites: [...sites.values()], call_edges, type_ancestors: dedupe(rawAncestors), + dispatch_candidates: dedupe(rawDispatch), overrides: dedupe(rawOverrides), entry_points: dedupe(rawEntry), entry_reachable: dedupe(rawReach), diff --git a/src/bundle/languages.ts b/src/bundle/languages.ts index 1fc2f2d44..a1e685433 100644 --- a/src/bundle/languages.ts +++ b/src/bundle/languages.ts @@ -63,6 +63,7 @@ export interface LanguageAdapter { callEdges: RawSource; typeAncestors?: RawSource; overrides?: RawSource; + dispatchCandidates?: RawSource; entryPoints?: RawSource; entryReachable?: RawSource; typeInstantiated?: RawSource; @@ -88,6 +89,7 @@ const JAVA: LanguageAdapter = { callEdges: CALL_EDGES, typeAncestors: { file: 'resolution-type-ancestor.csv', columns: [0, 1] }, overrides: { file: 'resolution-virtual-override.csv', columns: [0, 1] }, + dispatchCandidates: { file: 'dispatch-candidates.csv', columns: [0, 1, 2] }, entryPoints: { file: 'entry-point.csv', columns: [0, 1] }, entryReachable: { file: 'entry-reachable.csv', columns: [0] }, typeInstantiated: { file: 'type-instantiated.csv', columns: [0, 1] }, @@ -121,6 +123,9 @@ const TYPESCRIPT: LanguageAdapter = { typeAncestors: { file: 'resolution-type-ancestor.csv', columns: [0, 1] }, entryPoints: { file: 'entry-point.csv', columns: [0, 1] }, entryReachable: { file: 'entry-reachable.csv', columns: [0] }, + dispatchCandidates: { file: 'dispatch-candidates.csv', columns: [0, 1, 2] }, + // (type, how) — `new` is the only form TypeScript emits + typeInstantiated: { file: 'resolution-type-instantiated.csv', columns: [0, 1] }, }, ir: { methods: { @@ -155,6 +160,7 @@ const PYTHON: LanguageAdapter = { typeAncestors: { file: 'resolution-type-ancestor.csv', columns: [1, 2] }, // (prov, type) — no "how"; every row is a constructor call typeInstantiated: { file: 'resolution-type-instantiated.csv', columns: [1], constant: 'new' }, + dispatchCandidates: { file: 'dispatch-candidates.csv', columns: [0, 1, 2] }, }, ir: { methods: { diff --git a/src/bundle/schema.ts b/src/bundle/schema.ts index 63fc59e03..2507b4e74 100644 --- a/src/bundle/schema.ts +++ b/src/bundle/schema.ts @@ -130,9 +130,18 @@ export const CORE_TABLES: readonly TableSpec[] = [ { name: 'ancestor_type_id', type: 'TEXT', key: true, indexed: true, description: 'FK → types.id.' }, ], }, + { + name: 'dispatch_candidates', + description: 'THE DISPATCH ENVELOPE: (base method, method that may run instead) for every call that statically resolves to the base. This is the set `call_edges` narrowed FROM — the difference between "these are the targets" and "these are the targets, out of these possibilities". Populated in every language; `basis` says what admitted the pair, because the three front ends admit by different means.', + columns: [ + { name: 'base_method_id', type: 'TEXT', key: true, indexed: true, description: 'FK → methods.id — the method a call resolves to statically.' }, + { name: 'candidate_method_id', type: 'TEXT', key: true, indexed: true, description: 'FK → methods.id — a method that may run instead at such a call.' }, + { name: 'basis', type: 'TEXT', key: true, description: 'What admitted the pair — see vocabulary. Filter on it to trust only declarations.' }, + ], + }, { name: 'overrides', - description: 'Virtual-dispatch pairs: (base method, overriding method) wherever a call to the base may run the override. Java only today — see notes for what the other front ends offer instead.', + description: 'Virtual-dispatch pairs: (base method, overriding method) wherever a call to the base may run the override. Java only, and kept for compatibility — it is exactly `dispatch_candidates` filtered to `basis = nominal`. Prefer `dispatch_candidates`, which is populated in every language.', columns: [ { name: 'method_id', type: 'TEXT', key: true, indexed: true, description: 'FK → methods.id — the base (declared) method.' }, { name: 'overriding_method_id', type: 'TEXT', key: true, indexed: true, description: 'FK → methods.id — the override in a subtype.' }, @@ -163,7 +172,7 @@ export const CORE_TABLES: readonly TableSpec[] = [ }, { name: 'type_instantiated', - description: 'Types the client actually creates an instance of — the rapid-type-analysis set that bounds virtual dispatch. (A subtype nothing instantiates cannot receive a dispatched call.)', + description: 'Types this run creates an instance of — the rapid-type-analysis set that bounds virtual dispatch. (A subtype nothing instantiates cannot receive a dispatched call.) Deliberately an over-approximation: narrowing it on evidence the run does not have would lose real edges. Populated in every language.', columns: [ { name: 'type_id', type: 'TEXT', key: true, indexed: true, description: 'FK → types.id.' }, { name: 'how', type: 'TEXT', key: true, description: 'What creates the instance — see vocabulary.' }, @@ -341,7 +350,7 @@ export const VOCAB: readonly VocabSpec[] = [ // call_edges.tier { table: 'call_edges', column: 'tier', value: 'known_edge', languages: 'all', meaning: 'Exactly one target resolved. The strongest claim.' }, - { table: 'call_edges', column: 'tier', value: 'multi_inferred', languages: 'all', meaning: 'A sound SET of possible targets (virtual dispatch over instantiated subtypes); each member is one row. The set over-approximates; no member is a guess.' }, + { table: 'call_edges', column: 'tier', value: 'multi_inferred', languages: 'all', meaning: 'A sound SET of possible targets; each member is one row. The set over-approximates — every member is a real possibility, but not every member runs. HOW WIDE the set is differs by language: see the per-language notes on this table for whether the fan is narrowed by the instantiation set.' }, { table: 'call_edges', column: 'tier', value: 'boundary_lib', languages: 'all', meaning: 'The target is outside the client (library, builtin, or unstaged external). The chain is not expanded past it here.' }, { table: 'call_edges', column: 'tier', value: 'ambiguous_unknown', languages: 'all', meaning: 'Declared blind spot: the engine could not resolve the site (unresolved receiver, missing type, reflection…). callee is NULL. Never dropped.' }, { table: 'call_edges', column: 'tier', value: 'ambiguous_anon', languages: J, meaning: 'Known structural gap: an anonymous-class creation has no candidate rule yet. callee is NULL.' }, @@ -405,8 +414,13 @@ export const VOCAB: readonly VocabSpec[] = [ { table: 'entry_points', column: 'reason', value: 'scheduled', languages: J, meaning: 'A `@Scheduled` method.' }, { table: 'entry_points', column: 'reason', value: 'unimported_module', languages: T, meaning: 'The initializer of a module nothing imports — a script or a bundle root.' }, + // dispatch_candidates.basis + { table: 'dispatch_candidates', column: 'basis', value: 'nominal', languages: ['java', 'typescript'], meaning: 'A written extends/implements reaches the candidate\'s owner from the base\'s owner. The strongest evidence there is: the author declared the relationship.' }, + { table: 'dispatch_candidates', column: 'basis', value: 'structural', languages: T, meaning: 'No declaration; the candidate\'s owner satisfies the base\'s owner by SHAPE. Emitted only for supertypes with no nominal implementor at all, so it never competes with a declared answer — but it is a heuristic, and a consumer that wants declarations only filters it out.' }, + { table: 'dispatch_candidates', column: 'basis', value: 'mro', languages: P, meaning: 'The subtype\'s C3 linearisation picks the candidate for that attribute name. Not merely "the subtype declares this name" — a name a sibling base wins is attributed to that sibling.' }, + // type_instantiated.how - { table: 'type_instantiated', column: 'how', value: 'new', languages: ['java', 'python'], meaning: 'A constructor call in the client.' }, + { table: 'type_instantiated', column: 'how', value: 'new', languages: 'all', meaning: 'A constructor call — `new C()` / `C()`.' }, { table: 'type_instantiated', column: 'how', value: 'anonymous', languages: J, meaning: 'An anonymous class exists only by being instantiated.' }, { table: 'type_instantiated', column: 'how', value: 'enum_constant', languages: J, meaning: 'An enum\'s constants are its instances.' }, ]; @@ -420,15 +434,19 @@ export const NOTES: readonly NoteSpec[] = [ { language: 'java', table: 'call_sites', note: 'callee_name for `new X()` is the class name written at the site; NULL for ctor_delegate (`this(…)`/`super(…)`), anon_new, and record_accessor.' }, { language: 'java', table: 'call_sites', note: 'A record_accessor site is the RECORD_PATTERN expression, positioned where the pattern is written.' }, { language: 'typescript', table: 'call_sites', note: 'end_line / end_column come from the expression row; the call-site row itself records only the start.' }, - { language: 'typescript', table: 'overrides', note: 'EMPTY. TypeScript dispatch is captured directly as multi_inferred edges; the structural and nominal implementor sets are in ext_implementors, ext_structural_implementor and ext_type_satisfies.' }, - { language: 'typescript', table: 'type_instantiated', note: 'EMPTY. The TypeScript rule set does not export an instantiation set.' }, + { language: 'typescript', table: 'overrides', note: 'EMPTY — this table is Java-shaped. The TypeScript dispatch envelope is in dispatch_candidates, with basis `nominal` or `structural`.' }, + { language: 'typescript', table: 'type_instantiated', note: 'Every row has how = `new`. Not restricted to client provenance: a type the library constructs is still a type that exists at run time, and dropping it would narrow the envelope unsoundly.' }, { language: 'python', table: 'call_sites', note: 'PROPERTY_READ, CONTEXT_MANAGER and ITERATION_PROTOCOL rows are protocol edges with no written call: their site is the expression that triggers the protocol, and callee_name is NULL because nothing was written. Filter them out with kind NOT IN (…) when counting calls.' }, { language: 'python', table: 'call_sites', note: 'The id is an EXPRESSION hash for a written call; a DECORATOR hash (PY_DECORATOR_…) for DECORATOR_APPLICATION and DECORATOR_* sites, positioned at the decorator line; and the class\'s TYPE hash for METACLASS_CREATION, positioned at the class declaration.' }, { language: 'python', table: 'call_edges', note: 'A `boundary_lib` edge may point at a builtin (callee_provenance builtin, callee_label `builtin:NAME`) or at an unstaged import path (callee_provenance external) — neither has a methods row.' }, { language: 'python', table: 'call_edges', note: 'The reason a site is ambiguous_unknown is exported per site in ext_call_site_unresolved (site, caller, reason, detail).' }, { language: 'python', table: 'entry_points', note: 'EMPTY. The Python rule set does not derive entry points; entry_reachable is therefore empty too.' }, - { language: 'python', table: 'overrides', note: 'EMPTY. Python method lookup is by MRO, exported positionally in ext_mro_position (type, ancestor, …, position).' }, + { language: 'python', table: 'overrides', note: 'EMPTY — this table is Java-shaped. The Python dispatch envelope is in dispatch_candidates with basis `mro`; the raw linearisation is in ext_mro_position.' }, { language: 'python', table: 'type_instantiated', note: 'Every row has how = `new`: the rule set records that some client call constructs the class, not which form.' }, + { language: 'all', table: 'call_edges', note: 'THE TRUST LINE, and it is not the same set of tiers in every language. RESOLVED (callee_method_id is set): known_edge, multi_inferred, boundary_lib, and in TypeScript ALSO ambient_terminal and intrinsic_terminal. BLIND SPOT (callee is NULL): ambiguous_unknown, and in Java ALSO ambiguous_anon. A filter written as `tier IN (known_edge, multi_inferred)` therefore drops resolved edges in TypeScript and nowhere else — derive the set from this note or from unresolved_sites, never from a hardcoded list.' }, + { language: 'java', table: 'call_edges', note: 'A multi_inferred fan is CHA-wide: it is every override the hierarchy admits, bounded only by the dispatch cap. type_instantiated is computed and exported but NOT read by any rule, so the fan is not narrowed to types the program constructs. Narrow it yourself by joining dispatch_candidates to type_instantiated — see the dispatch_envelope_of query.' }, + { language: 'typescript', table: 'call_edges', note: 'A multi_inferred fan is CHA-wide, as in Java: type_instantiated is computed and exported but NOT read by any rule. The fan also has sources that are not virtual dispatch at all — an overload set or a union-typed receiver produces one too.' }, + { language: 'python', table: 'call_edges', note: 'A multi_inferred fan IS narrowed by the instantiation set: type_instantiated_reachable (the constructed classes and their bases) bounds dispatch in resolution/dispatch.dl. Python is the only front end where that narrowing is applied, so a fan here is tighter than the same shape would be in Java or TypeScript.' }, { language: 'all', table: 'call_edges', note: 'The raw relation has a seventh column, ToExpr, that is always `-` (reserved). It is dropped here.' }, { language: 'all', table: 'call_edges', note: 'An unresolved site (tier ambiguous_*) has NULL callee_method_id, callee_label and callee_provenance. The raw relation writes `-` in those slots.' }, ]; @@ -439,10 +457,11 @@ export const GUIDE: readonly string[] = [ 'This is a call graph of one codebase, derived by a type-directed Datalog engine. Start with `SELECT value FROM run WHERE key=\'language\'` — every language-specific fact below is keyed on it.', 'The graph is `call_edges`: one row per (call site, possible target). Rows join to `methods` (names, files, lines) on `caller_id` / `callee_method_id`, and to `call_sites` on `call_site_id` for where the call is written. Identifiers are opaque hashes — never parse them, always join.', 'Trust is explicit. `tier` says what kind of claim a row is: `known_edge` (one resolved target), `multi_inferred` (a sound set — every row of the set is a real possibility), `boundary_lib` (leaves the client; not expanded further), `ambiguous_*` (a declared unknown: callee is NULL). Pick the tiers your question tolerates and filter on them; never treat an `ambiguous_*` row as an edge.', + '`call_edges` is what the engine CONCLUDED; `dispatch_candidates` is what the hierarchy ADMITTED. Read the second when you need an upper bound rather than a best answer — a candidate whose owner is absent from `type_instantiated` is admitted by the hierarchy but never constructed in this run, which is how you narrow it yourself. `basis` separates a declared relationship from a shape match.', 'Before answering "nothing calls X" or "X cannot reach Y", check `unresolved_sites` for the methods on the path: a caller listed there has a call the engine could not resolve, so the answer is a lower bound and should say so.', 'Library targets (`callee_provenance = lib`) are named in `methods` with `provenance = lib` but their bodies were not analysed; a Python `builtin`/`external` target has no methods row and lives in `callee_label`.', '`schema_vocab` lists every value a column can hold FOR THIS LANGUAGE with its meaning — filter on `language = (SELECT value FROM run WHERE key=\'language\')`. `schema_notes` lists the caveats for this language (empty tables, what an id may point at). Read both before interpreting `kind`, `tier` or an empty table.', - '`schema_queries` holds tested SQL for the common questions (callers, callees, blast radius, entry reachability, the method at a file:line, the blind spots of a method). Bind the named parameters and run.', + '`schema_queries` holds tested SQL for the common questions (callers, callees, blast radius, entry reachability, the method at a file:line, the blind spots of a method, the dispatch envelope of a method). Bind the named parameters and run.', 'Tables named `ext_` are the language\'s raw engine relations with positional columns c0…cN; `schema_tables` carries each one\'s description lifted from its rule. Use them only when a core table does not hold what you need.', 'When you report a result, carry the tier and the unresolved count with it. A consumer who cannot see the confidence of an edge cannot use it.', ]; @@ -478,14 +497,15 @@ ORDER BY s.start_line, target`, }, { name: 'blast_radius', - question: 'If this method changes, which methods are transitively affected, up to :depth hops, through sound edges only (known_edge and multi_inferred)?', + question: 'If this method changes, which methods are transitively affected, up to :depth hops, through RESOLVED client edges only (a declared unknown is not traversed, and the count of them is returned alongside)?', params: ':qualified_name, :depth', sql: `WITH RECURSIVE up(id, depth) AS ( SELECT id, 0 FROM methods WHERE qualified_name = :qualified_name UNION SELECT e.caller_id, up.depth + 1 FROM call_edges e JOIN up ON e.callee_method_id = up.id - WHERE e.tier IN ('known_edge', 'multi_inferred') AND up.depth < :depth + WHERE e.tier IN ('known_edge', 'multi_inferred', 'ambient_terminal', 'intrinsic_terminal') + AND up.depth < :depth ) SELECT MIN(up.depth) AS depth, m.qualified_name, m.file_path, m.start_line, (SELECT count(*) FROM unresolved_sites u WHERE u.caller_id = m.id) AS unresolved_calls_inside @@ -521,6 +541,18 @@ JOIN methods m ON m.id = u.caller_id LEFT JOIN call_sites s ON s.id = u.call_site_id WHERE m.qualified_name = :qualified_name ORDER BY s.start_line`, + }, + { + name: 'dispatch_envelope_of', + question: 'What else might actually run at a call that resolves to this method — the set the graph narrowed from, and whether each candidate is a declaration or a shape match?', + params: ':qualified_name', + sql: `SELECT cand.qualified_name AS candidate, cand.file_path, cand.start_line, d.basis, + EXISTS (SELECT 1 FROM type_instantiated i WHERE i.type_id = cand.owner_type_id) AS owner_instantiated +FROM dispatch_candidates d +JOIN methods base ON base.id = d.base_method_id +JOIN methods cand ON cand.id = d.candidate_method_id +WHERE base.qualified_name = :qualified_name +ORDER BY d.basis, cand.qualified_name`, }, { name: 'subtypes_of', diff --git a/src/java/engine/resolution/virtual-dispatch.dl b/src/java/engine/resolution/virtual-dispatch.dl index fdd02b9ad..febf5ad40 100644 --- a/src/java/engine/resolution/virtual-dispatch.dl +++ b/src/java/engine/resolution/virtual-dispatch.dl @@ -125,3 +125,17 @@ virtual_override(base, override) :- client_overridden_lib_type(baseType), virtual_override(base, override) :- java_method(name, _, _, _, _, _, _, enum, _, _, _, _, _, _, _, _, "ABSTRACT_METHOD", pc, _, _, _, base), name != "", java_method(name, _, _, _, _, _, _, enum, _, _, _, _, _, _, _, _, "ENUM_CONSTANT_METHOD", pc, _, _, _, override). + +// ── method_dispatch_candidate(BaseMethod, CandidateMethod, Basis) ─────────── +// THE DISPATCH ENVELOPE, in the shape every front end exports (#471). A call that +// statically resolves to BaseMethod may at run time execute CandidateMethod instead; +// Basis says which relation admitted it, because the three languages admit by +// different means and a consumer that cannot tell them apart cannot judge the set. +// +// Java's basis is always `nominal` — the hierarchy is declared, so virtual_override is +// already the answer and this is a rename, not a new analysis. TypeScript adds +// `structural`, Python answers by `mro`. Keeping one relation name across the three is +// the entire point: `overrides` was a core table of the output bundle that came back +// empty in two of three languages while the data sat in per-language ext_ relations of +// three different shapes. +method_dispatch_candidate(base, override, "nominal") :- virtual_override(base, override). diff --git a/src/java/souffle/decls_all.dl b/src/java/souffle/decls_all.dl index 8c5c5bf80..ec89bf327 100644 --- a/src/java/souffle/decls_all.dl +++ b/src/java/souffle/decls_all.dl @@ -211,6 +211,7 @@ .decl member_type(c0:symbol,c1:symbol,c2:symbol) .decl method_decl(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol) .decl method_declared_in(c0:symbol,c1:symbol,c2:symbol) +.decl method_dispatch_candidate(c0:symbol,c1:symbol,c2:symbol) .decl method_has_params(c0:symbol) .decl method_has_unknown_call(c0:symbol,c1:symbol) .decl method_is_varargs(c0:symbol) diff --git a/src/java/souffle/export_manifest.tsv b/src/java/souffle/export_manifest.tsv index bd4728c9d..b3250e235 100644 --- a/src/java/souffle/export_manifest.tsv +++ b/src/java/souffle/export_manifest.tsv @@ -19,3 +19,4 @@ config_entry_point config-entry-point.csv config_key_ref config-key-ref.csv config_unresolved config-unresolved.csv type_instantiated type-instantiated.csv +method_dispatch_candidate dispatch-candidates.csv diff --git a/src/python/engine/resolution/attribute-lookup.dl b/src/python/engine/resolution/attribute-lookup.dl index 6ca1b7bf8..355c7ed5f 100644 --- a/src/python/engine/resolution/attribute-lookup.dl +++ b/src/python/engine/resolution/attribute-lookup.dl @@ -331,3 +331,38 @@ type_descriptor_getter(p, t, n, g) :- field_type(p, f, dt), type_is_data_descriptor(p, dt), descriptor_get_slot(gs), mro_lookup(p, dt, gs, g). + +// ── py_init_name(Name) ────────────────────────────────────────────────────── +// The two names that initialise an instance of the class they are written in. +py_init_name("__init__"). +py_init_name("__new__"). + +// ── method_dispatch_candidate(BaseMethod, CandidateMethod, Basis) ─────────── +// THE DISPATCH ENVELOPE, in the shape every front end exports (#471). A call that +// statically resolves to BaseMethod may at run time execute CandidateMethod instead. +// +// Python's basis is `mro`, and that is not a formality: the candidate is not "whatever +// the subtype declares with this name" but whatever the subtype's LINEARISATION picks, +// so a name a sibling base wins is attributed to that sibling and not to the subtype. +// mro_lookup is already that answer per (type, name) — this clause only turns the +// receiver types back into the base method they may displace. +// +// Reading mro_lookup rather than type_defines_method is what makes the set honest in +// both directions: a subtype that does NOT redefine the name yields no row (mro_lookup +// returns the base's own method, and cand != base drops it), and a subtype whose winner +// is a monkey-patched value or an unrelated co-base yields the method that really runs. +// A name shadowed by a data attribute yields nothing here, which is correct — that path +// is not a method call at all (see mro_lookup_data_shadow). +// +// AN INITIALISER IS NOT DISPATCHED FROM ITS OWN CLASS. `Base(...)` runs Base.__init__ and +// never Sub.__init__, however Sub subclasses Base — the constructed class is written at +// the site. mro_lookup does not know that, because to Python's attribute machinery +// `__init__` is an ordinary attribute found through the MRO, and it genuinely is one when +// the receiver is a Sub. The table's claim is about what a call RESOLVING TO THE BASE may +// run, so the pair does not belong in it. `__new__` is excluded for the same reason. +method_dispatch_candidate(base, cand, "mro") :- + type_defines_method(_, sup, n, base), + !py_init_name(n), + type_ancestor(p, sub, sup), + mro_lookup(p, sub, n, cand), + cand != base. diff --git a/src/python/souffle/decls_all.dl b/src/python/souffle/decls_all.dl index 62daccf36..2c06f072d 100644 --- a/src/python/souffle/decls_all.dl +++ b/src/python/souffle/decls_all.dl @@ -78,6 +78,8 @@ .decl method_scope(c0:symbol,c1:symbol,c2:symbol) .decl method_declaring_binding(c0:symbol,c1:symbol,c2:symbol) .decl method_start_line(c0:symbol,c1:symbol,c2:symbol) +.decl method_dispatch_candidate(c0:symbol,c1:symbol,c2:symbol) +.decl py_init_name(c0:symbol) .decl def_separated_by_block(c0:symbol,c1:symbol,c2:symbol) .decl def_rebound_by_later_def(c0:symbol,c1:symbol) .decl method_file(c0:symbol,c1:symbol,c2:symbol) diff --git a/src/python/souffle/export_manifest.tsv b/src/python/souffle/export_manifest.tsv index d6743e0aa..2580f86ca 100644 --- a/src/python/souffle/export_manifest.tsv +++ b/src/python/souffle/export_manifest.tsv @@ -82,3 +82,4 @@ param_arg_dict_entry resolution-param-dict-entry.csv mro_lookup_decorated resolution-mro-lookup-decorated.csv param_type resolution-param-type.csv param_flow_outside_declaration assumption-flow-outside-declaration.csv +method_dispatch_candidate dispatch-candidates.csv diff --git a/src/typescript/engine/resolution/reference-types.dl b/src/typescript/engine/resolution/reference-types.dl index 0abd40000..607f70f70 100644 --- a/src/typescript/engine/resolution/reference-types.dl +++ b/src/typescript/engine/resolution/reference-types.dl @@ -517,3 +517,19 @@ var_type(v, prov, et) :- var_binding("client", _, "INDEX", _, e, v), var_shape(v, sh) :- var_binding("client", _, "INDEX", _, e, v), e != "", expr_element_shape(e, sh). + +// ── type_instantiated(TypeHash, How) ──────────────────────────────────────── +// THE RTA SET (#471): the types this run actually constructs, which is what bounds +// virtual dispatch — a subtype nothing instantiates cannot receive a dispatched call. +// Java and Python both exported one; TypeScript exported nothing, so the bundle's +// core `type_instantiated` table was empty for TypeScript alone while every clause of +// new_expression_type above was already computing the answer. +// +// `How` mirrors Java's column so the three tables read the same. `new` is the only +// value TypeScript emits today: an enum member is not a construction, and a +// class expression is instantiated through a `new` like any other. +// +// NOT restricted to client provenance. RTA is an over-approximation and must stay one: +// dropping a type the library constructs would narrow the envelope on evidence the run +// does not have, and a narrowed envelope is unsound in the direction that loses edges. +type_instantiated(t, "new") :- new_expression_type(_, _, t). diff --git a/src/typescript/engine/resolution/type-hierarchy.dl b/src/typescript/engine/resolution/type-hierarchy.dl index f313dc366..6a133bf3a 100644 --- a/src/typescript/engine/resolution/type-hierarchy.dl +++ b/src/typescript/engine/resolution/type-hierarchy.dl @@ -177,3 +177,47 @@ category_is_shape_only("TYPE_ALIAS_TYPE"). class_type(t) :- type_decl(_, _, _, _, categ, _, t), category_is_class(categ). interface_type(t) :- type_decl(_, _, _, _, categ, _, t), category_is_shape_only(categ). instantiable_type(t) :- type_decl(_, _, _, _, categ, _, t), category_is_instantiable(categ). + +// ── method_is_constructor(MethodHash) ─────────────────────────────────────── +// A constructor by any of TypeScript's four spellings; see member-lookup.dl. +method_is_constructor(m) :- method_kind(_, k, _, m), method_kind_is_constructor(k). + +// ── method_dispatch_candidate(BaseMethod, CandidateMethod, Basis) ─────────── +// THE DISPATCH ENVELOPE, in the shape every front end exports (#471). A call that +// statically resolves to BaseMethod may at run time execute CandidateMethod instead. +// +// TypeScript already resolves dispatch straight into multi_inferred edges, so this +// relation is not what the resolver reads — it is what a CONSUMER reads to see the set +// those edges were narrowed FROM. Without it the bundle's `overrides` table came back +// empty for TypeScript while the type-level fan sat in ext_implementors and +// ext_structural_implementor, two relations of a shape Java's consumer never sees. +// +// The join is implementor x member: the fan is over TYPES, and the member name carries +// it down to methods. IsStatic is bound on both sides — a static of the same name is a +// different member, and dispatching a virtual call into it would be a fabricated edge. +// +// Two bases, kept apart because they are not equally good evidence. `nominal` follows a +// written extends/implements. `structural` follows a shape match, and +// structural_implementor already restricts itself to targets with no nominal +// implementor at all (see structural-satisfaction.dl) — so the two never both fire for +// one supertype, and a consumer that trusts only declarations can filter on the basis. +// +// A CONSTRUCTOR IS NOT DISPATCHED. `new Shape()` runs Shape's constructor and never +// Circle's, however Circle extends Shape — the constructed type is written at the site. +// Without this guard the join pairs them anyway, because a subclass constructor is a +// member of the subclass with the same escaped name: MEASURED on 01-class-dispatch-and- +// super, 2 of 5 pairs were `Shape. -> Circle.` and +// `-> Square.`, both of them edges no run can take. +method_dispatch_candidate(base, cand, "nominal") :- + declared_method(sup, n, st, base), + !method_is_constructor(base), + implementors(sup, sub), + declared_method(sub, n, st, cand), + cand != base. +method_dispatch_candidate(base, cand, "structural") :- + declared_method(sup, n, st, base), + !method_is_constructor(base), + structural_implementor(sup, sub), + sub != sup, + declared_method(sub, n, st, cand), + cand != base. diff --git a/src/typescript/souffle/decls_all.dl b/src/typescript/souffle/decls_all.dl index bb7531b9b..1895df1be 100644 --- a/src/typescript/souffle/decls_all.dl +++ b/src/typescript/souffle/decls_all.dl @@ -257,6 +257,8 @@ .decl method_flags(c0:symbol,c1:symbol,c2:symbol,c3:symbol,c4:symbol) .decl method_group(c0:symbol,c1:symbol,c2:symbol) .decl method_declares_type_param(c0:symbol) +.decl method_dispatch_candidate(c0:symbol,c1:symbol,c2:symbol) +.decl method_is_constructor(c0:symbol) .decl method_group_of(c0:symbol,c1:symbol) .decl method_has_rest(c0:symbol) .decl method_has_unknown_call(c0:symbol,c1:symbol) @@ -437,6 +439,7 @@ .decl type_group(c0:symbol,c1:symbol,c2:symbol) .decl type_inherits(c0:symbol,c1:symbol) .decl type_inherits_star(c0:symbol,c1:symbol) +.decl type_instantiated(c0:symbol,c1:symbol) .decl type_is_type_only(c0:symbol,c1:symbol) .decl type_lines(c0:symbol,c1:symbol,c2:symbol,c3:symbol) .decl type_module(c0:symbol,c1:symbol,c2:symbol) diff --git a/src/typescript/souffle/export_manifest.tsv b/src/typescript/souffle/export_manifest.tsv index f45a6c8f1..10daf008d 100644 --- a/src/typescript/souffle/export_manifest.tsv +++ b/src/typescript/souffle/export_manifest.tsv @@ -41,3 +41,5 @@ expr_shape diag-expr-shape.csv call_signature_in_scope diag-call-sig.csv ref_shape_target diag-ref-shape.csv alias_step diag-alias-step.csv +method_dispatch_candidate dispatch-candidates.csv +type_instantiated resolution-type-instantiated.csv diff --git a/test/java/expected/03-overloads-and-receivers.envelope b/test/java/expected/03-overloads-and-receivers.envelope new file mode 100644 index 000000000..1b869db70 --- /dev/null +++ b/test/java/expected/03-overloads-and-receivers.envelope @@ -0,0 +1 @@ +nominal testcases.engine.Sink.accept -> testcases.engine.ComplexDispatch$anon:Sink.accept diff --git a/test/java/expected/08-cha-interface-fanout.envelope b/test/java/expected/08-cha-interface-fanout.envelope new file mode 100644 index 000000000..133dba3a7 --- /dev/null +++ b/test/java/expected/08-cha-interface-fanout.envelope @@ -0,0 +1,4 @@ +nominal testcases.engine.Box.set -> testcases.engine.SBox.set +nominal testcases.engine.Codec.encode -> testcases.engine.UpperCodec.encode +nominal testcases.engine.Sink.accept -> testcases.engine.EdgeCHA$anon:Sink.accept +nominal testcases.engine.Sink.accept -> testcases.engine.NamedSink.accept diff --git a/test/java/expected/09-cha-interface-injection.envelope b/test/java/expected/09-cha-interface-injection.envelope new file mode 100644 index 000000000..eea982a63 --- /dev/null +++ b/test/java/expected/09-cha-interface-injection.envelope @@ -0,0 +1,2 @@ +nominal testcases.engine.Handler.handle -> testcases.engine.FastHandler.handle +nominal testcases.engine.Handler.handle -> testcases.engine.SlowHandler.handle diff --git a/test/java/expected/10-super-invocations.envelope b/test/java/expected/10-super-invocations.envelope new file mode 100644 index 000000000..4688a9940 --- /dev/null +++ b/test/java/expected/10-super-invocations.envelope @@ -0,0 +1,2 @@ +nominal testcases.engine.Base.sound -> testcases.engine.SuperInvocations.sound +nominal testcases.engine.Named.name -> testcases.engine.SuperInvocations.name diff --git a/test/java/expected/11-cha-inherited-into-implementor.envelope b/test/java/expected/11-cha-inherited-into-implementor.envelope new file mode 100644 index 000000000..9bc44c341 --- /dev/null +++ b/test/java/expected/11-cha-inherited-into-implementor.envelope @@ -0,0 +1,5 @@ +nominal Handler.handle -> Another.handle +nominal Handler.handle -> Base.handle +nominal Handler.handle -> Direct.handle +nominal Parent.run -> Child.run +nominal Parent.run -> Sibling.run diff --git a/test/java/expected/15-cross-file-same-package.envelope b/test/java/expected/15-cross-file-same-package.envelope new file mode 100644 index 000000000..ce17c27c7 --- /dev/null +++ b/test/java/expected/15-cross-file-same-package.envelope @@ -0,0 +1 @@ +nominal shop.Discount.apply -> shop.PercentDiscount.apply diff --git a/test/java/expected/16-cross-package-imports.envelope b/test/java/expected/16-cross-package-imports.envelope new file mode 100644 index 000000000..9671642a8 --- /dev/null +++ b/test/java/expected/16-cross-package-imports.envelope @@ -0,0 +1 @@ +nominal core.Handler.handle -> core.FastHandler.handle diff --git a/test/java/expected/18-enum-record-sealed.envelope b/test/java/expected/18-enum-record-sealed.envelope new file mode 100644 index 000000000..eeecffa3c --- /dev/null +++ b/test/java/expected/18-enum-record-sealed.envelope @@ -0,0 +1,4 @@ +nominal Op.apply@10 -> Op.apply@8 +nominal Op.apply@10 -> Op.apply@9 +nominal Shape.area -> Circle.area +nominal Shape.area -> Square.area diff --git a/test/java/expected/19-receiver-forms.envelope b/test/java/expected/19-receiver-forms.envelope new file mode 100644 index 000000000..abbf55da0 --- /dev/null +++ b/test/java/expected/19-receiver-forms.envelope @@ -0,0 +1,2 @@ +nominal Node.tag -> Leaf.tag +nominal Node.tag -> Twig.tag diff --git a/test/java/expected/25-di-narrowing.envelope b/test/java/expected/25-di-narrowing.envelope new file mode 100644 index 000000000..d6ee663a6 --- /dev/null +++ b/test/java/expected/25-di-narrowing.envelope @@ -0,0 +1,4 @@ +nominal testcases.config.Codec.encode -> testcases.config.JsonCodec.encode +nominal testcases.config.Codec.encode -> testcases.config.XmlCodec.encode +nominal testcases.config.Store.read -> testcases.config.DbStore.read +nominal testcases.config.Store.read -> testcases.config.InMemoryStore.read diff --git a/test/java/expected/26-spring-oracle.envelope b/test/java/expected/26-spring-oracle.envelope new file mode 100644 index 000000000..3d595d8e2 --- /dev/null +++ b/test/java/expected/26-spring-oracle.envelope @@ -0,0 +1,6 @@ +nominal testcases.springoracle.AuditSink.record -> testcases.springoracle.LoggingAuditSink.record +nominal testcases.springoracle.Clock.now -> testcases.springoracle.SystemClock.now +nominal testcases.springoracle.Notifier.notifyOf -> testcases.springoracle.EmailNotifier.notifyOf +nominal testcases.springoracle.Notifier.notifyOf -> testcases.springoracle.SmsNotifier.notifyOf +nominal testcases.springoracle.OrderRepo.find -> testcases.springoracle.JdbcOrderRepo.find +nominal testcases.springoracle.OrderRepo.find -> testcases.springoracle.NoopOrderRepo.find diff --git a/test/java/expected/27-messaging-and-grpc.envelope b/test/java/expected/27-messaging-and-grpc.envelope new file mode 100644 index 000000000..06d101e64 --- /dev/null +++ b/test/java/expected/27-messaging-and-grpc.envelope @@ -0,0 +1 @@ +nominal testcases.config.GreeterImplBase.sayHello -> testcases.config.GreeterService.sayHello diff --git a/test/java/expected/30-dispatch-param-types.envelope b/test/java/expected/30-dispatch-param-types.envelope new file mode 100644 index 000000000..925e30396 --- /dev/null +++ b/test/java/expected/30-dispatch-param-types.envelope @@ -0,0 +1 @@ +nominal probe.Base.put -> probe.Impl.put diff --git a/test/java/expected/31-service-loader.envelope b/test/java/expected/31-service-loader.envelope new file mode 100644 index 000000000..9b56cfe07 --- /dev/null +++ b/test/java/expected/31-service-loader.envelope @@ -0,0 +1,3 @@ +nominal probe.Codec.encode -> probe.ReverseCodec.encode +nominal probe.Codec.encode -> probe.Unregistered.encode +nominal probe.Codec.encode -> probe.UpperCodec.encode diff --git a/test/java/expected/34-object-members.envelope b/test/java/expected/34-object-members.envelope new file mode 100644 index 000000000..398e10093 --- /dev/null +++ b/test/java/expected/34-object-members.envelope @@ -0,0 +1 @@ +nominal probe.Base.clone -> probe.Sub.clone diff --git a/test/java/expected/39-library-supertype-dispatch.envelope b/test/java/expected/39-library-supertype-dispatch.envelope new file mode 100644 index 000000000..39bcd8ff8 --- /dev/null +++ b/test/java/expected/39-library-supertype-dispatch.envelope @@ -0,0 +1,8 @@ +nominal app.LocalStrategy.run -> app.LocImplA.run +nominal app.LocalStrategy.run -> app.LocImplB.run +nominal app.LocalTask.run -> app.LocSubA.run +nominal app.LocalTask.run -> app.LocSubB.run +nominal lib:dep.Strategy.run -> app.ExtImplA.run +nominal lib:dep.Strategy.run -> app.ExtImplB.run +nominal lib:dep.Task.run -> app.ExtSubA.run +nominal lib:dep.Task.run -> app.ExtSubB.run diff --git a/test/java/run-tests.sh b/test/java/run-tests.sh index 975497f84..ef43d8db7 100755 --- a/test/java/run-tests.sh +++ b/test/java/run-tests.sh @@ -306,6 +306,28 @@ for dir in "$HERE"/cases/*/; do cfg_summary=" [config: ${cfg_rows} rows]" else cfg_summary=""; fi + + # ── DISPATCH-ENVELOPE golden ────────────────────────────────────────────── + # The edge golden records what the engine CONCLUDED. dispatch_candidates records what + # the hierarchy ADMITTED — the set those edges were narrowed from, and the only table + # in the output bundle that answers "what ELSE might run here". Nothing downstream + # consumes it, so a rule that stopped emitting it would move no other golden and no + # test would notice; it was Java-only for the whole life of the bundle for exactly that + # reason (#471). A case with envelope rows and NO golden fails, and a golden with no + # rows fails too. + python3 "$HERE/../tools/envelope_report.py" "$w/ir" "$w/out/raw" all-methods.csv methodRegistryUniqueHash --library "$case_lib" > "$w/actual.envelope" 2>"$w/envelope.log" || { + echo "FAIL (envelope report — see $w/envelope.log)"; fail=$((fail+1)); failed+=("$name"); continue; } + env_rows=$(wc -l < "$w/actual.envelope" | tr -d ' ') + eexp="$HERE/expected/$name.envelope" + if [ "$BLESS" = "1" ]; then + if [ "${env_rows:-0}" -gt 0 ]; then cp "$w/actual.envelope" "$eexp"; else rm -f "$eexp"; fi + elif [ -f "$eexp" ] || [ "${env_rows:-0}" -gt 0 ]; then + if [ ! -f "$eexp" ]; then + echo "FAIL (envelope rows but no golden — run with --bless)"; fail=$((fail+1)); failed+=("$name"); continue; fi + if ! diff -q "$eexp" "$w/actual.envelope" >/dev/null; then + echo "FAIL (dispatch envelope changed)"; diff -u "$eexp" "$w/actual.envelope" | sed 's/^/ /' | head -40 + fail=$((fail+1)); failed+=("$name"); continue; fi + fi # ── LIVE SPRING CONTEXT oracle (opt-in, and only for cases that declare one) ── # cases//spring-oracle.conf holds: [key=value ...] sconf="$dir/spring-oracle.conf" diff --git a/test/python/expected/03-single-inheritance.envelope b/test/python/expected/03-single-inheritance.envelope new file mode 100644 index 000000000..d9707dc68 --- /dev/null +++ b/test/python/expected/03-single-inheritance.envelope @@ -0,0 +1 @@ +mro main.Base.step -> main.Derived.step diff --git a/test/python/expected/04-mro-diamond.envelope b/test/python/expected/04-mro-diamond.envelope new file mode 100644 index 000000000..dcf6273f3 --- /dev/null +++ b/test/python/expected/04-mro-diamond.envelope @@ -0,0 +1,4 @@ +mro main.A.shared -> main.B.shared +mro main.A.shared -> main.C.shared +mro main.A.who -> main.C.who +mro main.C.shared -> main.B.shared diff --git a/test/python/expected/05-super.envelope b/test/python/expected/05-super.envelope new file mode 100644 index 000000000..e19db19bc --- /dev/null +++ b/test/python/expected/05-super.envelope @@ -0,0 +1,5 @@ +mro main.Left.greet -> main.Both.greet +mro main.Right.greet -> main.Both.greet +mro main.Root.greet -> main.Both.greet +mro main.Root.greet -> main.Left.greet +mro main.Root.greet -> main.Right.greet diff --git a/test/python/run-tests.sh b/test/python/run-tests.sh index 53603ff12..00d194199 100755 --- a/test/python/run-tests.sh +++ b/test/python/run-tests.sh @@ -333,6 +333,28 @@ for dir in "$HERE"/cases/*/; do fail=$((fail+1)); failed+=("$name"); continue fi + + # ── DISPATCH-ENVELOPE golden ────────────────────────────────────────────── + # The edge golden records what the engine CONCLUDED. dispatch_candidates records what + # the hierarchy ADMITTED — the set those edges were narrowed from, and the only table + # in the output bundle that answers "what ELSE might run here". Nothing downstream + # consumes it, so a rule that stopped emitting it would move no other golden and no + # test would notice; it was Java-only for the whole life of the bundle for exactly that + # reason (#471). A case with envelope rows and NO golden fails, and a golden with no + # rows fails too. + "$PY" "$HERE/../tools/envelope_report.py" "$w/ir" "$w/out/raw" all-python-methods.csv pyMethodUniqueHash > "$w/actual.envelope" 2>"$w/envelope.log" || { + echo "FAIL (envelope report — see $w/envelope.log)"; fail=$((fail+1)); failed+=("$name"); continue; } + env_rows=$(wc -l < "$w/actual.envelope" | tr -d ' ') + eexp="$HERE/expected/$name.envelope" + if [ "$BLESS" = "1" ]; then + if [ "${env_rows:-0}" -gt 0 ]; then cp "$w/actual.envelope" "$eexp"; else rm -f "$eexp"; fi + elif [ -f "$eexp" ] || [ "${env_rows:-0}" -gt 0 ]; then + if [ ! -f "$eexp" ]; then + echo "FAIL (envelope rows but no golden — run with --bless)"; fail=$((fail+1)); failed+=("$name"); continue; fi + if ! diff -q "$eexp" "$w/actual.envelope" >/dev/null; then + echo "FAIL (dispatch envelope changed)"; diff -u "$eexp" "$w/actual.envelope" | sed 's/^/ /' | head -40 + fail=$((fail+1)); failed+=("$name"); continue; fi + fi exp="$HERE/expected/$name.edges" if [ "$BLESS" = "1" ]; then if [ -f "$exp" ] && ! diff -q "$exp" "$w/actual.edges" >/dev/null; then diff --git a/test/python/tools/literal_gate.py b/test/python/tools/literal_gate.py index 19dd31e0b..a7bb69a80 100644 --- a/test/python/tools/literal_gate.py +++ b/test/python/tools/literal_gate.py @@ -55,7 +55,13 @@ DUNDER = re.compile(r"^__\w+__$") # a label only ever produced, never joined on REASON_HEADS = ("call_unresolvable(", "site_reason(", "expr_type_untypable(", - "call_chain_summary(") + "call_chain_summary(", + # method_dispatch_candidate's third column is `basis`: which relation + # admitted the pair (`mro` here, `nominal`/`structural` in the other front + # ends). It is written into the output and never joined on, so it is the + # same category as the reasons above -- an output vocabulary term, not a + # literal the resolution could be fitted to. + "method_dispatch_candidate(") GROUND_FACT = re.compile(r'^[a-z_]+\((?:\s*"[^"]*"\s*,?)+\)\.\s*$') diff --git a/test/tools/bundle-test.sh b/test/tools/bundle-test.sh index f63e4c516..2869261e8 100755 --- a/test/tools/bundle-test.sh +++ b/test/tools/bundle-test.sh @@ -34,20 +34,23 @@ mk_java(){ local d="$1"; mkdir -p "$d/ir" "$d/raw" printf 'name\tsignature\tqualifiedName\tfilePath\tstartLine\tendLine\ttypeRegistryLinkHash\townerQualifiedName\tmethodKind\tmethodRegistryUniqueHash\n' > "$d/ir/all-methods.csv" printf 'main\tmain(String[])\tapp.Main.main\tsrc/Main.java\t3\t7\tTYPE_REGISTRY_t1\tapp.Main\tSTATIC_METHOD\tMETHOD_REGISTRY_m1\n' >> "$d/ir/all-methods.csv" printf 'render\trender()\tapp.Widget.render\tsrc/Widget.java\t4\t9\tTYPE_REGISTRY_t2\tapp.Widget\tINSTANCE_METHOD\tMETHOD_REGISTRY_m2\n' >> "$d/ir/all-methods.csv" + printf 'render\trender()\tapp.FancyWidget.render\tsrc/FancyWidget.java\t3\t6\tTYPE_REGISTRY_t3\tapp.FancyWidget\tINSTANCE_METHOD\tMETHOD_REGISTRY_m3\n' >> "$d/ir/all-methods.csv" printf 'name\tqualifiedName\ttypeCategory\tfilePath\tstartLine\tendLine\ttypeRegistryUniqueHash\n' > "$d/ir/all-types.csv" - printf 'Main\tapp.Main\tCLASS_TYPE\tsrc/Main.java\t1\t9\tTYPE_REGISTRY_t1\nWidget\tapp.Widget\tCLASS_TYPE\tsrc/Widget.java\t1\t12\tTYPE_REGISTRY_t2\n' >> "$d/ir/all-types.csv" + printf 'Main\tapp.Main\tCLASS_TYPE\tsrc/Main.java\t1\t9\tTYPE_REGISTRY_t1\nWidget\tapp.Widget\tCLASS_TYPE\tsrc/Widget.java\t1\t12\tTYPE_REGISTRY_t2\nFancyWidget\tapp.FancyWidget\tCLASS_TYPE\tsrc/FancyWidget.java\t1\t8\tTYPE_REGISTRY_t3\n' >> "$d/ir/all-types.csv" printf 'kind\tliteralValue\ttypeRegistryLinkHash\tstartLine\tstartColumn\tendLine\tendColumn\texpressionUniqueHash\n' > "$d/ir/all-expressions.csv" printf 'METHOD_INVOCATION\trender\tTYPE_REGISTRY_t1\t5\t9\t5\t20\tEXPRESSION_REFERENCE_e1\nMETHOD_INVOCATION\tmystery\tTYPE_REGISTRY_t1\t6\t9\t6\t22\tEXPRESSION_REFERENCE_e2\n' >> "$d/ir/all-expressions.csv" printf 'EXPRESSION_REFERENCE_e1\tMETHOD_REGISTRY_m1\t-\tMETHOD_REGISTRY_m2\tclient\tknown_edge\tmethod\n' > "$d/raw/call-chain-edges.csv" printf 'EXPRESSION_REFERENCE_e2\tMETHOD_REGISTRY_m1\t-\t-\t-\tambiguous_unknown\tmethod\n' >> "$d/raw/call-chain-edges.csv" printf 'METHOD_REGISTRY_m1\tmain\n' > "$d/raw/entry-point.csv" - printf 'TYPE_REGISTRY_t2\tnew\nTYPE_REGISTRY_t2\tmade_up_how\n' > "$d/raw/type-instantiated.csv" + printf 'TYPE_REGISTRY_t2\tnew\nTYPE_REGISTRY_t2\tmade_up_how\nTYPE_REGISTRY_t3\tnew\n' > "$d/raw/type-instantiated.csv" + printf 'METHOD_REGISTRY_m2\tMETHOD_REGISTRY_m3\tnominal\n' > "$d/raw/dispatch-candidates.csv" } mk_typescript(){ local d="$1"; mkdir -p "$d/ir" "$d/raw" printf 'name\tsignature\tqualifiedName\tfilePath\tstartLine\tendLine\ttsTypeLinkHash\townerQualifiedName\tmethodKind\ttsMethodUniqueHash\n' > "$d/ir/all-typescript-methods.csv" printf 'main\tmain()\tapp#main\tsrc/app.ts\t3\t7\t\t\tFUNCTION_DECLARATION\tTS_METHOD_m1\nrender\trender()\twidget#Widget.render\tsrc/widget.ts\t4\t9\tTS_TYPE_t2\twidget#Widget\tMETHOD_DECLARATION\tTS_METHOD_m2\n' >> "$d/ir/all-typescript-methods.csv" + printf 'render\trender()\twidget#FancyWidget.render\tsrc/fancy.ts\t3\t6\tTS_TYPE_t3\twidget#FancyWidget\tMETHOD_DECLARATION\tTS_METHOD_m3\nrender\trender()\tduck#Duck.render\tsrc/duck.ts\t2\t4\tTS_TYPE_t4\tduck#Duck\tMETHOD_DECLARATION\tTS_METHOD_m4\n' >> "$d/ir/all-typescript-methods.csv" printf 'name\tqualifiedName\ttypeCategory\tfilePath\tstartLine\tendLine\ttsTypeUniqueHash\n' > "$d/ir/all-typescript-types.csv" - printf 'Widget\twidget#Widget\tCLASS_TYPE\tsrc/widget.ts\t1\t12\tTS_TYPE_t2\n' >> "$d/ir/all-typescript-types.csv" + printf 'Widget\twidget#Widget\tCLASS_TYPE\tsrc/widget.ts\t1\t12\tTS_TYPE_t2\nFancyWidget\twidget#FancyWidget\tCLASS_TYPE\tsrc/fancy.ts\t1\t8\tTS_TYPE_t3\nDuck\tduck#Duck\tCLASS_TYPE\tsrc/duck.ts\t1\t5\tTS_TYPE_t4\n' >> "$d/ir/all-typescript-types.csv" printf 'tsModuleUniqueHash\tfilePath\n' > "$d/ir/all-typescript-modules.csv" printf 'TS_MODULE_a\tsrc/app.ts\n' >> "$d/ir/all-typescript-modules.csv" printf 'callKind\tcalleeName\ttsExpressionLinkHash\ttsModuleLinkHash\tstartLine\tstartColumn\n' > "$d/ir/all-typescript-call-sites.csv" @@ -57,12 +60,15 @@ mk_typescript(){ local d="$1"; mkdir -p "$d/ir" "$d/raw" printf 'TS_EXPRESSION_e1\tTS_METHOD_m1\t-\tTS_METHOD_m2\tclient\tknown_edge\tMETHOD_CALL\n' > "$d/raw/call-chain-edges.csv" printf 'TS_EXPRESSION_e2\tTS_METHOD_m1\t-\t-\t-\tambiguous_unknown\tFUNCTION_CALL\n' >> "$d/raw/call-chain-edges.csv" printf 'TS_METHOD_m1\tunimported_module\n' > "$d/raw/entry-point.csv" + printf 'TS_METHOD_m2\tTS_METHOD_m3\tnominal\nTS_METHOD_m2\tTS_METHOD_m4\tstructural\n' > "$d/raw/dispatch-candidates.csv" + printf 'TS_TYPE_t2\tnew\nTS_TYPE_t3\tnew\n' > "$d/raw/resolution-type-instantiated.csv" } mk_python(){ local d="$1"; mkdir -p "$d/ir" "$d/raw" printf 'name\tsignature\tqualifiedName\tfilePath\tstartLine\tendLine\tpyTypeLinkHash\townerQualifiedName\tmethodKind\tpyMethodUniqueHash\n' > "$d/ir/all-python-methods.csv" printf 'main\tmain()\tapp.main\tapp.py\t3\t7\t\t\tFUNCTION\tPY_METHOD_m1\nrender\trender(self)\twidget.Widget.render\twidget.py\t4\t9\tPY_TYPE_t2\twidget.Widget\tINSTANCE_METHOD\tPY_METHOD_m2\n' >> "$d/ir/all-python-methods.csv" + printf 'render\trender(self)\tfancy.FancyWidget.render\tfancy.py\t3\t6\tPY_TYPE_t3\tfancy.FancyWidget\tINSTANCE_METHOD\tPY_METHOD_m3\n' >> "$d/ir/all-python-methods.csv" printf 'name\tqualifiedName\ttypeCategory\tfilePath\tstartLine\tendLine\tpyTypeUniqueHash\n' > "$d/ir/all-python-types.csv" - printf 'Widget\twidget.Widget\tCLASS_TYPE\twidget.py\t1\t12\tPY_TYPE_t2\n' >> "$d/ir/all-python-types.csv" + printf 'Widget\twidget.Widget\tCLASS_TYPE\twidget.py\t1\t12\tPY_TYPE_t2\nFancyWidget\tfancy.FancyWidget\tCLASS_TYPE\tfancy.py\t1\t8\tPY_TYPE_t3\n' >> "$d/ir/all-python-types.csv" printf 'pyModuleUniqueHash\tfilePath\n' > "$d/ir/all-python-modules.csv" printf 'PY_MODULE_a\tapp.py\n' >> "$d/ir/all-python-modules.csv" printf 'callKind\tcalleeName\tpyExpressionLinkHash\tpyModuleLinkHash\tstartLine\tstartColumn\tendLine\n' > "$d/ir/all-python-call-sites.csv" @@ -73,7 +79,8 @@ mk_python(){ local d="$1"; mkdir -p "$d/ir" "$d/raw" printf 'PY_EXPRESSION_e1\tPY_METHOD_m1\t-\tPY_METHOD_m2\tclient\tknown_edge\tMETHOD_CALL\n' > "$d/raw/call-chain-edges.csv" printf 'PY_EXPRESSION_e2\tPY_METHOD_m1\t-\t-\t-\tambiguous_unknown\tSIMPLE_CALL\n' >> "$d/raw/call-chain-edges.csv" printf 'PY_EXPRESSION_e3\tPY_METHOD_m1\t-\tbuiltin:print\tbuiltin\tboundary_lib\tSIMPLE_CALL\n' >> "$d/raw/call-chain-edges.csv" - printf 'client\tPY_TYPE_t2\n' > "$d/raw/resolution-type-instantiated.csv" + printf 'client\tPY_TYPE_t2\nclient\tPY_TYPE_t3\n' > "$d/raw/resolution-type-instantiated.csv" + printf 'PY_METHOD_m2\tPY_METHOD_m3\tmro\n' > "$d/raw/dispatch-candidates.csv" } # column N (1-based) of the row whose first column is $3, in headered TSV $1 @@ -87,7 +94,7 @@ for lang in java typescript python; do fi G="$d/graph" # 1. every core table, with the declared header - for t in run methods types call_sites call_edges type_ancestors overrides entry_points entry_reachable unresolved_sites type_instantiated; do + for t in run methods types call_sites call_edges type_ancestors dispatch_candidates overrides entry_points entry_reachable unresolved_sites type_instantiated; do [ -f "$G/$t.csv" ] || bad "$lang: graph/$t.csv missing" done [ "$(header "$G/call_edges.csv")" = "$(printf 'call_site_id\tcaller_id\tcallee_method_id\tcallee_label\tcallee_provenance\ttier\tkind')" ] || bad "$lang: call_edges header is $(header "$G/call_edges.csv")" @@ -105,6 +112,20 @@ for lang in java typescript python; do unres="$(awk -F'\t' '$6=="ambiguous_unknown"{print $1"|"$3"|"$4"|"$5}' "$G/call_edges.csv")" case "$unres" in *_e2\|\|\|) ;; *) bad "$lang: unresolved row is '$unres' — expected empty callee/label/provenance";; esac [ "$(wc -l < "$G/unresolved_sites.csv" | tr -d ' ')" = "2" ] || bad "$lang: unresolved_sites has $(($(wc -l < "$G/unresolved_sites.csv")-1)) rows, expected 1" + # 3b. the DISPATCH ENVELOPE and the RTA set, IN EVERY LANGUAGE (#471). + # Asserted inside the per-language loop and not once outside it: the whole defect + # was that these two tables were populated in Java and empty in the other two, and + # an assertion that runs once passes on Java and never looks. + [ "$(header "$G/dispatch_candidates.csv")" = "$(printf 'base_method_id\tcandidate_method_id\tbasis')" ] || bad "$lang: dispatch_candidates header is $(header "$G/dispatch_candidates.csv")" + nd=$(($(wc -l < "$G/dispatch_candidates.csv") - 1)) + [ "$nd" -ge 1 ] || bad "$lang: dispatch_candidates is EMPTY — the dispatch envelope is not language-neutral" + ni=$(($(wc -l < "$G/type_instantiated.csv") - 1)) + [ "$ni" -ge 1 ] || bad "$lang: type_instantiated is EMPTY — no RTA set for this language" + # the pair is the base render -> the overriding render, and it joins to real methods + base="$(awk -F'\t' 'NR>1{print $1; exit}' "$G/dispatch_candidates.csv")" + cand="$(awk -F'\t' 'NR>1{print $2; exit}' "$G/dispatch_candidates.csv")" + case "$(cell "$G/methods.csv" 3 "$base")" in *Widget.render) ;; *) bad "$lang: envelope base is '$(cell "$G/methods.csv" 3 "$base")', not a Widget.render";; esac + case "$(cell "$G/methods.csv" 3 "$cand")" in *FancyWidget.render) ;; *) bad "$lang: envelope candidate is '$(cell "$G/methods.csv" 3 "$cand")', not FancyWidget.render";; esac # 4. the in-database catalog if [ "$HAVE_SQLITE" = 1 ]; then DB="$d/graph.sqlite"; [ -f "$DB" ] || { bad "$lang: graph.sqlite missing"; continue; } @@ -115,6 +136,10 @@ for lang in java typescript python; do [ "$(SQL "$DB" "SELECT count(*) FROM ext_call_chain_edge")" = "$(wc -l < "$d/raw/call-chain-edges.csv" | tr -d ' ')" ] || bad "$lang: ext_call_chain_edge row count differs from raw" joined="$(SQL "$DB" "SELECT cm.qualified_name||' -> '||tm.qualified_name||' @ '||s.file_path||':'||s.start_line FROM call_edges e JOIN call_sites s ON s.id=e.call_site_id JOIN methods cm ON cm.id=e.caller_id JOIN methods tm ON tm.id=e.callee_method_id WHERE e.tier='known_edge'")" case "$joined" in *"-> "*"Widget.render @ "*":5") ;; *) bad "$lang: the SQL join gave '$joined'";; esac + # every basis this language emits is one the schema documents FOR THIS LANGUAGE — an + # undocumented value here means the vocabulary and the rules disagree about the envelope + undoc="$(SQL "$DB" "SELECT DISTINCT d.basis FROM dispatch_candidates d WHERE NOT EXISTS (SELECT 1 FROM schema_vocab v WHERE v.table_name='dispatch_candidates' AND v.column_name='basis' AND v.value=d.basis AND v.language='$lang' AND v.meaning NOT LIKE 'undocumented%')")" + [ -z "$undoc" ] || bad "$lang: dispatch_candidates.basis emits '$undoc', which the schema does not document for $lang" case $lang in java) [ "$(SQL "$DB" "SELECT count(*) FROM schema_vocab WHERE table_name='type_instantiated' AND value='made_up_how' AND meaning LIKE 'undocumented%'")" = 1 ] || bad "java: an unauthored value was not recorded as undocumented";; python) [ "$(SQL "$DB" "SELECT callee_label||'/'||callee_provenance FROM call_edges WHERE tier='boundary_lib'")" = "builtin:print/builtin" ] || bad "python: builtin target not carried as a label";; @@ -148,6 +173,16 @@ if [ "$HAVE_SQLITE" = 1 ]; then want=app.Widget.render; [ "$lang" = typescript ] && want='widget#Widget.render'; [ "$lang" = python ] && want=widget.Widget.render got="$(sqlite3 "$DB" ".parameter set :qualified_name '$want'" "$(SQL "$DB" "SELECT sql FROM schema_queries WHERE name='callers_of'")" | cut -d'|' -f1,3,4)" case "$got" in *"main|5|known_edge"*) ;; *) bad "$lang: callers_of on the fixture gave '$got'";; esac + env="$(sqlite3 "$DB" ".parameter set :qualified_name '$want'" "$(SQL "$DB" "SELECT sql FROM schema_queries WHERE name='dispatch_envelope_of'")")" + case "$env" in *FancyWidget.render*) ;; *) bad "$lang: dispatch_envelope_of on the fixture gave '$env'";; esac + # the fixture instantiates the nominal override's owner, so the query must say so. + # The column that lets a consumer narrow the envelope itself is worth nothing if it is + # constant, so TypeScript — whose fixture also carries a duck type nothing constructs — + # must return BOTH values. That is the discriminating half of this assertion. + printf '%s\n' "$env" | grep -q '|1$' || bad "$lang: dispatch_envelope_of never marks an instantiated owner (got '$env')" + if [ "$lang" = typescript ]; then + printf '%s\n' "$env" | grep -q '|0$' || bad "typescript: owner_instantiated is 1 for the never-constructed duck type — the column does not discriminate" + fi done fi diff --git a/test/tools/envelope_report.py b/test/tools/envelope_report.py new file mode 100755 index 000000000..5defbd149 --- /dev/null +++ b/test/tools/envelope_report.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""The dispatch envelope of one case, in a form a human can check against the source. + +WHY THIS EXISTS. `dispatch_candidates` is the set `call_edges` was narrowed FROM, and it is +the only table in the bundle that answers "what ELSE might run here". It was Java-only for +the whole life of the output bundle: the relation was computed in every language and +projected in one, so a consumer got a full answer from a Java bundle and silence from the +other two, with no error (#471). A relation that can be empty without anything failing is a +relation that can be silently reverted, so it needs a test that reads it. + +The report is hash-free on purpose. Every id in the raw relation derives from baseMservPath, +so a golden written in hashes would only ever pass on the machine that blessed it; the +qualified names come from the IR and are stable across checkouts. + +usage: envelope_report.py [--library ] +""" +import csv +import os +import sys + +csv.field_size_limit(10 ** 9) + + +def rows(path, rfc=True): + """Raw relations are TSV written by souffle; IR entity tables are RFC4180 CSV-in-TSV.""" + if not os.path.exists(path): + return [] + with open(path, newline='', encoding='utf-8', errors='replace') as fh: + return list(csv.reader(fh, delimiter='\t')) if rfc else \ + list(csv.reader(fh, delimiter='\t', quoting=csv.QUOTE_NONE)) + + +def read_methods(ir_dir, methods_csv, hash_col, into, prefix=''): + """hash -> qualified name, from one IR's method table. Returns its rows (or []).""" + m = rows(os.path.join(ir_dir, methods_csv)) + if not m: + return [], {} + ix = {c: i for i, c in enumerate(m[0])} + if hash_col not in ix: + sys.stderr.write(f'{methods_csv} has no column {hash_col}; header is {m[0]}\n') + raise SystemExit(2) + for r in m[1:]: + if len(r) <= ix[hash_col]: + continue + h = r[ix[hash_col]] + qn = r[ix['qualifiedName']] if 'qualifiedName' in ix and len(r) > ix['qualifiedName'] else '' + if not qn and 'name' in ix and len(r) > ix['name']: + qn = r[ix['name']] + if h and h not in into: + into[h] = (prefix + qn) if qn else h + return m, ix + + +def main() -> int: + ir_dir, raw_dir, methods_csv, hash_col = sys.argv[1:5] + lib_dirs = [sys.argv[i + 1] for i, a in enumerate(sys.argv) if a == '--library' and i + 1 < len(sys.argv)] + + # method hash -> qualified name (falling back to the simple name, then the hash) + name = {} + m, ix = read_methods(ir_dir, methods_csv, hash_col, name) + # A LIBRARY-DECLARED BASE IS THE INTERESTING HALF. `Runnable.run -> MyTask.run` is the + # shape a client most often writes, and with only the client IR loaded the base printed + # as a bare `lib:?` -- so a golden could not tell one library base from another, and a + # rule that started fanning a DIFFERENT library method would not move it. The library + # IR names them, by qualified name rather than by hash, so the golden stays portable. + for d in lib_dirs: + if os.path.isdir(d): + read_methods(d, methods_csv, hash_col, name, prefix='lib:') + + # A qualified name is NOT unique. Java's enum-constant fan pairs an abstract method + # with each constant's body, and every one of those is `.` — three + # distinct hashes printing as one line, so a golden could not tell a fan of three from + # a fan of one. Where a name is shared, the declaration line disambiguates it. + _vals = list(name.values()) + shared = {q for q in _vals if _vals.count(q) > 1} + line = {} + if m: + for r in m[1:]: + if len(r) > ix[hash_col] and r[ix[hash_col]] and 'startLine' in ix and len(r) > ix['startLine']: + line[r[ix[hash_col]]] = r[ix['startLine']] + + def label(h): + # A LIBRARY method has no row in the client IR. Naming it `lib:` would put a + # machine-specific hash in the golden, so it is named by what it is instead: the + # pair still records that the envelope crosses the boundary, which is the fact + # worth regressing on. + q = name.get(h) + if not q: + return 'lib:?' + return f'{q}@{line.get(h, "?")}' if q in shared else q + + out = set() + for r in rows(os.path.join(raw_dir, 'dispatch-candidates.csv'), rfc=False): + if len(r) < 3: + continue + base, cand, basis = r[0], r[1], r[2] + out.add(f'{basis}\t{label(base)} -> {label(cand)}') + for s in sorted(out): + print(s) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/typescript/expected/01-class-dispatch-and-super.envelope b/test/typescript/expected/01-class-dispatch-and-super.envelope new file mode 100644 index 000000000..33930a2e3 --- /dev/null +++ b/test/typescript/expected/01-class-dispatch-and-super.envelope @@ -0,0 +1,3 @@ +nominal shapes#Shape.area -> shapes#Circle.area +nominal shapes#Shape.area -> shapes#Square.area +nominal shapes#Shape.prefix -> shapes#Circle.prefix diff --git a/test/typescript/expected/01-class-dispatch-and-super.lib.envelope b/test/typescript/expected/01-class-dispatch-and-super.lib.envelope new file mode 100644 index 000000000..9cf22fa2b --- /dev/null +++ b/test/typescript/expected/01-class-dispatch-and-super.lib.envelope @@ -0,0 +1,4 @@ +nominal lib:geometry#Drawable.draw -> shapes#LibCircle.draw +nominal shapes#Shape.area -> shapes#Circle.area +nominal shapes#Shape.area -> shapes#Square.area +nominal shapes#Shape.prefix -> shapes#Circle.prefix diff --git a/test/typescript/expected/02-interface-fanout.envelope b/test/typescript/expected/02-interface-fanout.envelope new file mode 100644 index 000000000..df6694f3b --- /dev/null +++ b/test/typescript/expected/02-interface-fanout.envelope @@ -0,0 +1,3 @@ +nominal handlers#Handler.handle -> handlers#LowerHandler.handle +nominal handlers#Handler.handle -> handlers#NeverBuiltHandler.handle +nominal handlers#Handler.handle -> handlers#UpperHandler.handle diff --git a/test/typescript/expected/02-interface-fanout.lib.envelope b/test/typescript/expected/02-interface-fanout.lib.envelope new file mode 100644 index 000000000..f09c56602 --- /dev/null +++ b/test/typescript/expected/02-interface-fanout.lib.envelope @@ -0,0 +1,5 @@ +nominal handlers#Handler.handle -> handlers#LowerHandler.handle +nominal handlers#Handler.handle -> handlers#NeverBuiltHandler.handle +nominal handlers#Handler.handle -> handlers#UpperHandler.handle +nominal lib:contracts#Sink.accept -> handlers#ClientSink.accept +nominal lib:contracts#Sink.accept -> lib:contracts#ConsoleSink.accept diff --git a/test/typescript/expected/03-structural-satisfaction.envelope b/test/typescript/expected/03-structural-satisfaction.envelope new file mode 100644 index 000000000..224a7991e --- /dev/null +++ b/test/typescript/expected/03-structural-satisfaction.envelope @@ -0,0 +1,2 @@ +structural duck#Reader.read -> duck#FileReader.read +structural duck#Reader.read -> duck#NetReader.read diff --git a/test/typescript/expected/03-structural-satisfaction.lib.envelope b/test/typescript/expected/03-structural-satisfaction.lib.envelope new file mode 100644 index 000000000..595d139aa --- /dev/null +++ b/test/typescript/expected/03-structural-satisfaction.lib.envelope @@ -0,0 +1,3 @@ +structural duck#Reader.read -> duck#FileReader.read +structural duck#Reader.read -> duck#NetReader.read +structural lib:io#Closeable.close -> duck#FileReader.close diff --git a/test/typescript/expected/05-declaration-merging.envelope b/test/typescript/expected/05-declaration-merging.envelope new file mode 100644 index 000000000..2c9ce376e --- /dev/null +++ b/test/typescript/expected/05-declaration-merging.envelope @@ -0,0 +1,2 @@ +nominal merged#Config.host -> merged#AppConfig.host +nominal merged#Config.port -> merged#AppConfig.port diff --git a/test/typescript/expected/05-declaration-merging.lib.envelope b/test/typescript/expected/05-declaration-merging.lib.envelope new file mode 100644 index 000000000..2c9ce376e --- /dev/null +++ b/test/typescript/expected/05-declaration-merging.lib.envelope @@ -0,0 +1,2 @@ +nominal merged#Config.host -> merged#AppConfig.host +nominal merged#Config.port -> merged#AppConfig.port diff --git a/test/typescript/expected/15-type-only-and-erasure.envelope b/test/typescript/expected/15-type-only-and-erasure.envelope new file mode 100644 index 000000000..cacb0fd0f --- /dev/null +++ b/test/typescript/expected/15-type-only-and-erasure.envelope @@ -0,0 +1 @@ +nominal types#Spec.check -> types#Impl.check diff --git a/test/typescript/expected/15-type-only-and-erasure.lib.envelope b/test/typescript/expected/15-type-only-and-erasure.lib.envelope new file mode 100644 index 000000000..fb1132f38 --- /dev/null +++ b/test/typescript/expected/15-type-only-and-erasure.lib.envelope @@ -0,0 +1,2 @@ +nominal lib:api#Contract.verify -> lib:api#Verifier.verify +nominal types#Spec.check -> types#Impl.check diff --git a/test/typescript/expected/18-heritage-across-boundary.lib.envelope b/test/typescript/expected/18-heritage-across-boundary.lib.envelope new file mode 100644 index 000000000..89e099a10 --- /dev/null +++ b/test/typescript/expected/18-heritage-across-boundary.lib.envelope @@ -0,0 +1,3 @@ +nominal lib:base#Base.describe -> derived#ClientChild.describe +nominal lib:base#Closeable.close -> derived#ClientSink.close +nominal lib:base#Sink.write -> derived#ClientSink.write diff --git a/test/typescript/expected/21-callable-function-members.envelope b/test/typescript/expected/21-callable-function-members.envelope new file mode 100644 index 000000000..e914caf7d --- /dev/null +++ b/test/typescript/expected/21-callable-function-members.envelope @@ -0,0 +1,3 @@ +nominal globals#Function.apply -> globals#CallableFunction.apply +nominal globals#Function.bind -> globals#CallableFunction.bind +nominal globals#Function.call -> globals#CallableFunction.call diff --git a/test/typescript/expected/22-loose-bind-call-apply.envelope b/test/typescript/expected/22-loose-bind-call-apply.envelope new file mode 100644 index 000000000..e914caf7d --- /dev/null +++ b/test/typescript/expected/22-loose-bind-call-apply.envelope @@ -0,0 +1,3 @@ +nominal globals#Function.apply -> globals#CallableFunction.apply +nominal globals#Function.bind -> globals#CallableFunction.bind +nominal globals#Function.call -> globals#CallableFunction.call diff --git a/test/typescript/expected/24-hedged-overload-return-union.lib.envelope b/test/typescript/expected/24-hedged-overload-return-union.lib.envelope new file mode 100644 index 000000000..9cfead5dd --- /dev/null +++ b/test/typescript/expected/24-hedged-overload-return-union.lib.envelope @@ -0,0 +1,2 @@ +nominal lib:widgets#BaseWidget.attach -> lib:widgets#SourceWidget.attach +nominal lib:widgets#BaseWidget.attach -> lib:widgets#VideoWidget.attach diff --git a/test/typescript/expected/25-qualified-type-names.envelope b/test/typescript/expected/25-qualified-type-names.envelope new file mode 100644 index 000000000..248f37339 --- /dev/null +++ b/test/typescript/expected/25-qualified-type-names.envelope @@ -0,0 +1,2 @@ +nominal shapes#geo.Point.move -> shapes#geo.Origin.move +nominal shapes#geo.deep.Box.fit -> shapes#geo.deep.Impl.fit diff --git a/test/typescript/expected/25-qualified-type-names.lib.envelope b/test/typescript/expected/25-qualified-type-names.lib.envelope new file mode 100644 index 000000000..248f37339 --- /dev/null +++ b/test/typescript/expected/25-qualified-type-names.lib.envelope @@ -0,0 +1,2 @@ +nominal shapes#geo.Point.move -> shapes#geo.Origin.move +nominal shapes#geo.deep.Box.fit -> shapes#geo.deep.Impl.fit diff --git a/test/typescript/expected/30-super-into-construct-signature.lib.envelope b/test/typescript/expected/30-super-into-construct-signature.lib.envelope new file mode 100644 index 000000000..792e78eae --- /dev/null +++ b/test/typescript/expected/30-super-into-construct-signature.lib.envelope @@ -0,0 +1,2 @@ +nominal lib:base#LibClass.describe -> app#AppFromClass.describe +nominal lib:base#LibErr.describe -> app#AppError.describe diff --git a/test/typescript/expected/52-cross-module-library-heritage.lib.envelope b/test/typescript/expected/52-cross-module-library-heritage.lib.envelope new file mode 100644 index 000000000..ce68f0a05 --- /dev/null +++ b/test/typescript/expected/52-cross-module-library-heritage.lib.envelope @@ -0,0 +1,2 @@ +nominal lib:core#Shape.area -> lib:core#Rect.area +nominal lib:core#Shape.area -> lib:ext#Circle.area diff --git a/test/typescript/run-tests.sh b/test/typescript/run-tests.sh index f367a37aa..05d6f75fe 100755 --- a/test/typescript/run-tests.sh +++ b/test/typescript/run-tests.sh @@ -380,7 +380,34 @@ for dir in "$HERE"/cases/*/; do [ $ok -eq 1 ] || { fail=$((fail+1)); failed+=("$name"); continue; } fi + # ── DISPATCH-ENVELOPE golden ────────────────────────────────────────────── + # The edge goldens record what the engine CONCLUDED. dispatch_candidates records what + # the hierarchy ADMITTED — the set those edges were narrowed from, and the only table + # in the output bundle that answers "what ELSE might run here". Nothing downstream + # consumes it, so a rule that stopped emitting it would move no other golden and no + # test would notice; the relation was Java-only for the whole life of the bundle for + # exactly that reason (#471). Both passes are scored: the library pass is where a + # library-declared base gains a client override, which the client-only pass cannot see. + # A case with envelope rows and NO golden fails, and a golden with no rows fails too. + envelope_golden() { + local raw="$1" exp="$2" out="$3" label="$4" + python3 "$HERE/../tools/envelope_report.py" "$w/ir" "$raw" all-typescript-methods.csv tsMethodUniqueHash \ + --library "$w/libir" > "$out" 2>"$w/envelope.log" || { echo "FAIL ($label report — see $w/envelope.log)"; return 1; } + local n; n=$(wc -l < "$out" | tr -d ' ') + if [ "$BLESS" = "1" ]; then + if [ "${n:-0}" -gt 0 ]; then cp "$out" "$exp"; else rm -f "$exp"; fi; return 0 + fi + [ -f "$exp" ] || [ "${n:-0}" -gt 0 ] || return 0 + [ -f "$exp" ] || { echo "FAIL ($label rows but no golden — run with --bless)"; return 1; } + diff -q "$exp" "$out" >/dev/null && return 0 + echo "FAIL ($label changed)"; diff -u "$exp" "$out" | sed 's/^/ /' | head -40; return 1 + } + bad=0 + envelope_golden "$w/plain/out/raw" "$HERE/expected/$name.envelope" "$w/actual.envelope" "envelope" || bad=1 + if [ "$HAS_LIB" = "1" ]; then + envelope_golden "$w/withlib/out/raw" "$HERE/expected/$name.lib.envelope" "$w/actual.lib.envelope" "lib-envelope" || bad=1 + fi check_golden "$w/actual.edges" "$HERE/expected/$name.edges" "edges" || bad=1 if [ "$HAS_LIB" = "1" ]; then check_golden "$w/actual.lib.edges" "$HERE/expected/$name.lib.edges" "lib-edges" || bad=1