diff --git a/graphify/extract.py b/graphify/extract.py index 516cd374b..eae7653a7 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -45,6 +45,7 @@ from graphify.extractors.fortran import _cpp_preprocess, extract_fortran # noqa: F401 from graphify.extractors.go import extract_go # noqa: F401 from graphify.extractors.json_config import extract_json # noqa: F401 +from graphify.extractors.kotlin import _resolve_cross_file_kotlin_imports # kotlin-cross-file from graphify.extractors.markdown import extract_markdown # noqa: F401 from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 @@ -2087,8 +2088,8 @@ def _resolve_swift_member_calls( all_nodes: list[dict], all_edges: list[dict], ) -> None: - """Resolve cross-file Swift member calls (``recv.method()``) to the real - definition of the receiver's type (#1356). + """Resolve cross-file receiver-typed member calls (``recv.method()``) to the + real definition of the receiver's type (#1356). The shared cross-file call pass drops every ``is_member_call`` because a bare method name (``update``) collides across the corpus and inflates god-nodes @@ -2098,7 +2099,17 @@ def _resolve_swift_member_calls( exactly one definition. A type-qualified call (``Type.staticMethod()``) is EXTRACTED (the type is named explicitly in source); an instance call typed via local inference (``obj.method()``) is INFERRED. The shared-pass member-call drop - stays intact: this is purely additive and fires only on receiver-typed Swift calls. + stays intact: this is purely additive and fires only on receiver-typed calls. + + Despite the name, this pass is language-agnostic once a + ``(file path -> {name: TypeName})`` table exists for a file: Kotlin reuses it + via its own ``kotlin_type_table`` (same shape, populated by the Kotlin + property/class-parameter/local-var branches in engine.py) instead of + duplicating this whole resolver -- the receiver-typing logic below doesn't + care which language produced the table. The resolver is registered for both + ``.swift`` and ``.kt``/``.kts`` suffixes (see ``register_language_resolver`` + below) so it activates for a Kotlin-only corpus with zero Swift files too. + # kotlin-cross-file Must run after id-disambiguation so node ids and caller_nids are final. """ @@ -2107,6 +2118,9 @@ def _resolve_swift_member_calls( tt = result.get("swift_type_table") if tt and tt.get("path"): type_table_by_file[tt["path"]] = tt.get("table", {}) + ktt = result.get("kotlin_type_table") + if ktt and ktt.get("path"): + type_table_by_file[ktt["path"]] = ktt.get("table", {}) if not type_table_by_file: return @@ -2975,7 +2989,15 @@ def _key(label: str) -> str: # by adding one register() call below — no edits to extract()'s body. Order # preserved from the prior inlined wiring: Swift (#1356) before Python (#1446). register_language_resolver( - LanguageResolver("swift_member_calls", frozenset({".swift"}), _resolve_swift_member_calls) + LanguageResolver( + "swift_member_calls", + # Also gates Kotlin (#kotlin-cross-file): the resolver itself is + # language-agnostic once a per-file type table exists, see its + # docstring. Without .kt/.kts here the pass never runs for a + # Kotlin-only corpus (no .swift files present to trip the suffix gate). + frozenset({".swift", ".kt", ".kts"}), + _resolve_swift_member_calls, + ) ) register_language_resolver( LanguageResolver("python_member_calls", frozenset({".py"}), _resolve_python_member_calls) @@ -5139,6 +5161,23 @@ def _learn(e: dict) -> None: import logging logging.getLogger(__name__).warning("Java type-reference resolution failed, skipping: %s", exc) + # Cross-file Kotlin import + same-package resolution. `_import_kotlin` + # (engine.py) emits a per-file `imports` edge whose target id is a bare + # module-name hash that almost never matches a real (file-qualified) Kotlin + # symbol id, so it's dropped as dangling by build.py; this pass re-parses + # each Kotlin file and resolves imports (plus Kotlin's no-import-needed + # same-package visibility) against the real corpus-wide symbol index, + # mirroring the Java cross-file import pass above. + # kotlin-cross-file + kotlin_paths = [p for p in paths if p.suffix in (".kt", ".kts")] + if kotlin_paths: + kotlin_results = [r for r, p in zip(per_file, paths) if p.suffix in (".kt", ".kts")] + try: + all_edges.extend(_resolve_cross_file_kotlin_imports(kotlin_results, kotlin_paths)) + except Exception as exc: + import logging + logging.getLogger(__name__).warning("Kotlin cross-file import resolution failed, skipping: %s", exc) + # Cross-file C# type-reference resolution: re-point dangling inherits/implements/ # references edges left on shadow stubs, disambiguating same-named types by the # referencing file's `using` directives + enclosing namespace (mirrors Java #1318). diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index ab6c1657c..684e86de2 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -743,6 +743,93 @@ def _kotlin_function_return_type_node(func_node): return c return None +# kotlin-cross-file +def _kotlin_property_name_node(property_node): + """Find the name `identifier` within a Kotlin property_declaration (the + binding name, not any identifier that's part of its type).""" + for c in property_node.children: + if c.type == "variable_declaration": + for sub in c.children: + if sub.type == "identifier": + return sub + return None + if c.type == "identifier": + return c + return None + +def _kotlin_constructor_call_type(property_node, source: bytes) -> str | None: + """Infer a property's type from a bare initializer when there's no + explicit type annotation: a constructor call (`val vault = + SovereignKeyVault()`), a static-factory call (`val storage = + BCryptoSecureStorage.getInstance()` -- the standard Kotlin/Android + singleton idiom, ubiquitous alongside/instead of a real constructor call), + or a bare static-member access (`val s = Storage.shared`). In every case + only an upper-cased receiver/callee counts as a type (a lower-cased one is + a local factory function or builder, not reliably the property's own + type) -- mirrors Swift's #1356/#1604 `_swift_constructor_type` and its + `Type.shared` singleton-property inference. + # kotlin-cross-file + """ + for child in property_node.children: + if child.type == "call_expression": + ctor = child.children[0] if child.children else None + if ctor is not None and ctor.type in ("identifier", "type_identifier"): + ctor_name = _read_text(ctor, source) + if ctor_name and ctor_name[:1].isupper(): + return ctor_name + elif ctor is not None and ctor.type == "navigation_expression": + recv = ctor.children[0] if ctor.children else None + if recv is not None and recv.type in ("identifier", "type_identifier"): + recv_name = _read_text(recv, source) + if recv_name and recv_name[:1].isupper(): + return recv_name + break + if child.type == "navigation_expression": + recv = child.children[0] if child.children else None + if recv is not None and recv.type in ("identifier", "type_identifier"): + recv_name = _read_text(recv, source) + if recv_name and recv_name[:1].isupper(): + return recv_name + break + return None + +def _kotlin_property_declared_type(node, source: bytes) -> str | None: + """Return the single flat type name of a Kotlin property_declaration: its + explicit type annotation if present, else a constructor-call inference.""" + type_node = _kotlin_property_type_node(node) + if type_node is not None: + refs: list[tuple[str, str]] = [] + _kotlin_collect_type_refs(type_node, source, False, refs) + for ref_name, role in refs: + if role == "type": + return ref_name + return None + return _kotlin_constructor_call_type(node, source) + +def _kotlin_local_var_types(body_node, source: bytes, type_table: dict[str, str]) -> None: + """Type local `val x: Foo` / `val x = Foo()` bindings inside a Kotlin + function body into the shared file-scoped type_table (mirrors Swift's + #1604 `_swift_local_var_types`) so a locally-declared receiver's member + call resolves cross-file. Conservative: does not descend into a nested + function/lambda/class body, since a raw call is only method-scoped (no + lexical-scope info survives to the cross-file pass), so a nested scope's + own locals could shadow an outer binding with the same name.""" + if body_node is None: + return + stack = [body_node] + while stack: + n = stack.pop() + if n.type == "property_declaration": + name_node = _kotlin_property_name_node(n) + if name_node is not None: + prop_type = _kotlin_property_declared_type(n, source) + if prop_type: + type_table[_read_text(name_node, source)] = prop_type + continue + if n.type in ("function_declaration", "lambda_literal", "anonymous_function", "class_declaration", "object_declaration"): + continue + stack.extend(n.children) + def _swift_declaration_keyword(node) -> str | None: """Return the leading kind token for a Swift class_declaration: class/struct/enum/extension/actor.""" for c in node.children: @@ -2588,6 +2675,64 @@ def _php_emit_base(base_name: str, rel: str, at_line: int) -> None: add_edge(class_nid, target, "references", line, context="generic_arg") + # Kotlin primary-constructor `val`/`var` parameters + # (`class Foo(private val bar: Bar)`) are real class properties -- + # the extremely common constructor-injection idiom in Android/DI + # code -- but were previously invisible: no `references` edge to + # their type, and no type_table binding for member-call receiver + # typing (`bar.something()` inside the class). A plain constructor + # parameter without `val`/`var` is NOT a property (only in scope + # for property initializers/init blocks, not ordinary methods), so + # it's excluded from both the edge and the type_table binding. + # kotlin-cross-file + if config.ts_module == "tree_sitter_kotlin": + for child in node.children: + if child.type != "primary_constructor": + continue + for cparams in child.children: + if cparams.type != "class_parameters": + continue + for cp in cparams.children: + if cp.type != "class_parameter": + continue + # #1356-equivalent: a class_parameter's default-value + # expression (`storage: Storage = Storage.getInstance()`) + # runs at construction time regardless of whether the + # parameter is promoted to a property (`val`/`var`) -- + # collect any call in it for the same deferred walk + # Swift/property initializers already get, so a + # companion-object factory call used as a constructor + # default isn't silently invisible to the call graph. + # kotlin-cross-file + for sub in cp.children: + if sub.type in config.call_types: + initializer_nodes.append((class_nid, sub)) + if not any(sub.type in ("val", "var") for sub in cp.children): + continue + cp_name_node = None + cp_type_node = None + for sub in cp.children: + if sub.type == "identifier" and cp_name_node is None: + cp_name_node = sub + elif sub.type in ("user_type", "nullable_type", "type_reference"): + cp_type_node = sub + if cp_type_node is None: + continue + cp_line = cp.start_point[0] + 1 + refs: list[tuple[str, str]] = [] + _kotlin_collect_type_refs(cp_type_node, source, False, refs) + prop_type: str | None = None + for ref_name, role in refs: + ctx = "generic_arg" if role == "generic_arg" else "field" + target_nid = ensure_named_node(ref_name, cp_line) + if target_nid != class_nid: + add_edge(class_nid, target_nid, "references", + cp_line, context=ctx) + if prop_type is None and role == "type": + prop_type = ref_name + if cp_name_node is not None and prop_type: + type_table[_read_text(cp_name_node, source)] = prop_type + # Ruby: `class Dog < Animal` puts the base class in the `superclass` # field (a `<` token followed by a constant or scope_resolution). # There was no Ruby branch, so every Ruby inherits edge was dropped. @@ -3057,6 +3202,34 @@ def _emit_java_parent_type(type_node, rel: str, at_line: int) -> None: target_nid = ensure_named_node(ref_name, line) if target_nid != parent_class_nid: add_edge(parent_class_nid, target_nid, "references", line, context=ctx) + # Receiver-typed member-call resolution (mirrors Swift #1356/#1604): + # bind this property's name to its declared type (explicit + # annotation, or a bare constructor-call initializer when + # untyped -- `private val vault = SovereignKeyVault()` is the common + # idiom and has no type_node above) into the shared file-scoped + # type_table so `vault.encrypt()` elsewhere in the class resolves + # cross-file via _resolve_swift_member_calls / kotlin_type_table. + # kotlin-cross-file + name_node = _kotlin_property_name_node(node) + if name_node is not None: + prop_type = _kotlin_property_declared_type(node, source) + if prop_type: + type_table[_read_text(name_node, source)] = prop_type + # `by lazy { ... }` (and other property delegates) hold their + # initializer in a `property_delegate` child, not a direct + # call_expression/navigation_expression sibling -- so a call inside + # it (`private val identity: Foo by lazy { Foo(Bar.getInstance()) }`, + # the standard deferred-init idiom, ubiquitous in DI-heavy Android + # code) was invisible to the call graph entirely. walk_calls + # recurses through the delegate node on its own once queued the + # same way as any other initializer (#1356 pattern) -- no special + # unwrapping needed, it naturally finds both the `lazy(...)` call + # and any nested calls inside the lambda body. + # kotlin-cross-file + for child in node.children: + if child.type == "property_delegate": + initializer_nodes.append((parent_class_nid, child)) + break return if (config.ts_module == "tree_sitter_swift" @@ -3685,6 +3858,32 @@ def _emit_java_parent_type(type_node, rel: str, at_line: int) -> None: walk(child, parent_class_nid=parent_class_nid) return + # Kotlin `companion object { ... }` is a nested block whose members + # (`ClassName.staticMethod()`) are semantically the OUTER class's + # members, not a separate type. The default recurse below would clear + # parent_class_nid, so a companion-object function/property was + # extracted with no class ownership at all -- no `method`/`references` + # edge from the outer class, meaning a receiver-typed cross-file call + # to `ClassName.staticFactory()` (the common Kotlin singleton/factory + # idiom, e.g. `Foo.getInstance()`) could never resolve past the class + # itself (a `references` edge, not `calls`) even when the receiver + # type resolved correctly. Treat it as a transparent wrapper -- but + # unlike the decorated_definition pattern above, recurse into the + # companion object's OWN body's children (mirroring exactly how the + # class_types branch recurses via `_find_body`/`body.children` + # further below), not into companion_object's direct children: those + # include the nested `class_body` node itself, which -- if walked + # directly -- isn't a class_types/function_types/companion_object + # match, so it falls through to the default recurse and clears + # parent_class_nid right back to None one level down. + # kotlin-cross-file + if t == "companion_object": + body = _find_body(node, config) + if body: + for child in body.children: + walk(child, parent_class_nid=parent_class_nid) + return + # Default: recurse for child in node.children: walk(child, parent_class_nid=None) @@ -3954,6 +4153,20 @@ def walk_calls( if child.type in ("simple_identifier", "identifier"): callee_name = _read_text(child, source) break + # Receiver-typed member-call resolution (mirrors Swift + # #1356): capture the depth-1 receiver so + # _resolve_swift_member_calls (reused for Kotlin via + # kotlin_type_table) can bind it through the file's local + # type table. Only a direct identifier or `this` receiver + # is captured; a chained/nested receiver (`a.b().c()`) is + # left untyped rather than guessed. + # kotlin-cross-file + recv_node = first.children[0] if first.children else None + if recv_node is not None: + if recv_node.type in ("simple_identifier", "identifier"): + member_receiver = _read_text(recv_node, source) + elif recv_node.type == "this_expression": + member_receiver = "this" elif config.ts_module == "tree_sitter_scala": # Scala: first child first = node.children[0] if node.children else None @@ -4460,6 +4673,15 @@ def walk_calls( for _caller_nid, body_node in function_bodies: _swift_local_var_types(body_node, source, type_table) + # Kotlin: type local `val x: Foo` / `val x = Foo()` bindings inside function + # bodies (mirrors Swift #1604 above) -- class-level properties and + # primary-constructor `val`/`var` params are typed in the walk, method-body + # locals were not. + # kotlin-cross-file + if config.ts_module == "tree_sitter_kotlin": + for _caller_nid, body_node in function_bodies: + _kotlin_local_var_types(body_node, source, type_table) + # JS/TS: bodies already walked with their own caller_nid (const-assigned # arrows, methods). An INLINE/returned arrow or function-expression that is # NOT separately tracked (e.g. `return () => svc.doThing()`) is otherwise @@ -4600,6 +4822,12 @@ def _scan_js_module_dispatch(n) -> None: result["ts_type_table"] = {"path": str_path, "table": type_table} elif config.ts_module == "tree_sitter_cpp": result["cpp_type_table"] = {"path": str_path, "table": type_table} + elif config.ts_module == "tree_sitter_kotlin": + # Read by _resolve_swift_member_calls, which is language-agnostic + # once a (path -> {name: TypeName}) table exists -- see its + # docstring. No separate Kotlin resolver needed. + # kotlin-cross-file + result["kotlin_type_table"] = {"path": str_path, "table": type_table} # C#: a file-wide receiver type table (field/property/param/local -> Type) for # _resolve_csharp_member_calls (#1609). Built from the whole tree, not just # function bodies, so class-level fields/properties are in scope for every method. diff --git a/graphify/extractors/kotlin.py b/graphify/extractors/kotlin.py new file mode 100644 index 000000000..bc31b1bed --- /dev/null +++ b/graphify/extractors/kotlin.py @@ -0,0 +1,187 @@ +"""Kotlin cross-file resolution: imports + implicit same-package visibility. + +Companion to the Kotlin-specific branches inside ``extractors/engine.py`` (call +walk, property/class-parameter type refs) and the receiver-typed member-call +reuse of ``_resolve_swift_member_calls`` in ``extract.py``. This module owns the +piece neither of those covers: turning Kotlin ``import`` statements (and Kotlin's +no-import-needed same-package visibility) into real ``imports`` edges so the +shared cross-file `calls` pass in extract.py has import evidence to promote a +same-name match from INFERRED to EXTRACTED and to disambiguate an otherwise +ambiguous short name (#1219-style). + +Before this module: ``_import_kotlin`` (engine.py) emits one `imports` edge per +Kotlin `import` line, but its target id is a bare `_make_id(module_name)` that +almost never matches a real node id (Kotlin symbol ids are qualified by their +containing file/class, e.g. `crypto_messagecrypto_messagecrypto`, not the bare +class name) -- so `build.py`'s dangling-edge filter silently drops nearly all of +them. Measured on a ~1170-file Kotlin corpus: 14616 source `import` lines -> 691 +surviving graph edges. This module fixes that with the same two-pass strategy +Java already uses (`_resolve_cross_file_java_imports`), plus same-package +resolution Java doesn't need (Java requires an explicit import even within a +package; Kotlin does not). + +# kotlin-cross-file +""" + +from __future__ import annotations + +from pathlib import Path + +from graphify.extractors.base import _make_id, _read_text + + +def _kotlin_package_name(root, source: bytes) -> str | None: + """Return the dotted package name from a Kotlin file's `package_header`, if any.""" + for c in root.children: + if c.type == "package_header": + for sub in c.children: + if sub.type == "qualified_identifier": + text = _read_text(sub, source) + return text or None + return None + + +def _kotlin_import_names(root, source: bytes) -> list[tuple[str, bool]]: + """Return ``(last_segment, is_wildcard)`` for every top-level `import` in a + Kotlin file. A wildcard import (`import com.foo.bar.*`) names no single + symbol, so callers should skip it rather than guess.""" + out: list[tuple[str, bool]] = [] + for c in root.children: + if c.type != "import": + continue + qid = None + wildcard = False + for sub in c.children: + if sub.type == "qualified_identifier": + qid = sub + elif sub.type == "*": + wildcard = True + if qid is None: + continue + raw = _read_text(qid, source) + parts = raw.split(".") + if not parts or not parts[-1]: + continue + out.append((parts[-1], wildcard)) + return out + + +def _resolve_cross_file_kotlin_imports( + per_file: list[dict], + paths: list[Path], +) -> list[dict]: + """Two-pass Kotlin import + same-package resolution. + + Pass 1: build a global {Name: [node_id, ...]} index across every Kotlin node + (classes/objects/interfaces/enums AND top-level functions -- Kotlin, unlike + Java, allows top-level `fun`/`val`/`var` outside any class). + + Pass 2: re-parse each Kotlin file (imports/package aren't threaded out of the + main per-file extraction the way `swift_type_table` is, so this mirrors + Java's approach of a second lightweight parse rather than plumbing new state + through the shared multi-language extractor). For every `import a.b.C`, + resolve C against the index and emit a symbol-level EXTRACTED `imports` edge + (wildcard imports produce no edge -- ambiguous, same policy as Java's + `import a.b.*`). For every pair of files declaring the same `package`, emit + one file-to-file EXTRACTED `imports_from` edge: Kotlin needs no `import` for + same-package visibility, so without this pass the majority of same-package + call sites (the common case in a typical Android module) carry zero import + evidence and the shared cross-file `calls` pass either can't promote them + past INFERRED or drops them outright when the name is ambiguous elsewhere in + the corpus. File-to-file (not file-to-every-symbol) because the fact being + recorded is package-level visibility, not that this file names any specific + symbol -- and it's exactly the granularity the shared pass's `_has_import_evidence` + module-import check already consumes. + """ + try: + import tree_sitter_kotlin as tsk + from tree_sitter import Language, Parser + except ImportError: + return [] + + language = Language(tsk.language()) + parser = Parser(language) + + # Pass 1: name -> node_id index (internal, non-file symbols only), for + # explicit `import a.b.C` resolution below. + name_to_ids: dict[str, list[str]] = {} + for file_result in per_file: + for node in file_result.get("nodes", []): + label = node.get("label", "") + nid = node.get("id", "") + src = node.get("source_file", "") + if not label or not nid or not src: + continue + if label.endswith(")") or label.endswith(".kt") or label.endswith(".kts"): + continue + if not label[0].isalpha(): + continue + name_to_ids.setdefault(label, []).append(nid) + + new_edges: list[dict] = [] + seen_pairs: set[tuple[str, str, str]] = set() + + def _emit(file_nid: str, tgt_nid: str, source_file: str, at_line: int, + relation: str = "imports") -> None: + if tgt_nid == file_nid: + return + key = (file_nid, tgt_nid, relation) + if key in seen_pairs: + return + seen_pairs.add(key) + new_edges.append({ + "source": file_nid, + "target": tgt_nid, + "relation": relation, + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": source_file, + "source_location": f"L{at_line}", + "weight": 1.0, + }) + + # Pass 2: re-parse for package + import statements. + file_nid_to_package: dict[str, str] = {} + package_to_file_nids: dict[str, list[str]] = {} + file_nid_to_path: dict[str, Path] = {} + + for path in paths: + file_nid = _make_id(str(path)) + file_nid_to_path[file_nid] = path + try: + source = path.read_bytes() + tree = parser.parse(source) + except Exception: + continue + root = tree.root_node + + pkg = _kotlin_package_name(root, source) + if pkg: + file_nid_to_package[file_nid] = pkg + package_to_file_nids.setdefault(pkg, []).append(file_nid) + + for last, wildcard in _kotlin_import_names(root, source): + if wildcard: + continue + for tgt_nid in name_to_ids.get(last, []): + _emit(file_nid, tgt_nid, str(path), 1) + + # Same-package implicit visibility: one file-to-file `imports_from` edge per + # same-package pair (not one edge per symbol in the other file) -- Kotlin's + # no-import-needed visibility is a file/package-level fact, not a claim that + # this file names every specific symbol the other file happens to define. + # `imports_from` is exactly what the shared cross-file `calls` pass already + # reads as "module import" evidence (see extract.py's `_has_import_evidence`: + # a candidate's containing file being in `imported_modules` is enough to + # promote an otherwise-INFERRED same-name match to EXTRACTED), so this stays + # fully equivalent for that purpose while cutting edge volume roughly 3x + # (measured: ~3 symbols/file average on a real ~1200-file Android module). + for file_nid, pkg in file_nid_to_package.items(): + path = file_nid_to_path.get(file_nid) + source_file = str(path) if path is not None else "" + for other_nid in package_to_file_nids.get(pkg, []): + if other_nid == file_nid: + continue + _emit(file_nid, other_nid, source_file, 1, relation="imports_from") + + return new_edges