From 386dab132fb1f6b95f66bbe0ee8352f30ef00fc3 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:33:02 -0400 Subject: [PATCH 01/11] feat(extract): add Common Lisp extractor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an imperative tree-sitter-commonlisp walker that extracts packages, classes, functions, methods, macros, generic-function dispatch, and calls from Common Lisp source. Registered alongside the other custom extractors so homoiconic CL — where every defXXX form is the same list_lit node — is classified by reading definer symbols rather than tree-sitter node types. Co-authored-by: jansaasman --- graphify/extract.py | 500 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 500 insertions(+) diff --git a/graphify/extract.py b/graphify/extract.py index b834fb42c..8ce1ad2f5 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4036,6 +4036,506 @@ def add_existing_edge(edge: dict) -> None: # block defined in the corpus (count.index, each.key, self.*, path.module, ...). +# ── Common Lisp ────────────────────────────────────────────────────────────── + +# Standard CL definer forms that introduce data/type/variable bindings +# (not callable, no body to walk for calls) +_CL_DATA_DEFINERS = frozenset({ + "defstruct", "deftype", "define-condition", + "defvar", "defparameter", "defconstant", + "define-symbol-macro", +}) + +# Subset of data definers that take a VALUE as their second argument, which +# may itself be a string literal. For these, we must not mistake the value +# for a docstring. +_CL_VALUE_DEFINERS = frozenset({"defvar", "defparameter", "defconstant"}) + +# Standard CL macro-style definers +_CL_MACRO_DEFINERS = frozenset({ + "define-modify-macro", "define-compiler-macro", + "define-setf-expander", "defsetf", +}) + +# Symbols that start with "def" but are NOT definitions (denylist for the +# def-prefix heuristic that catches custom definers like definline-maybe) +_CL_NOT_DEFINERS = frozenset({ + "default", "default-value", "defaults", "define", + "defer", "deferred", "deflate", "deftest", +}) + +# CL special forms / macros that should not count as "calls" in the graph +_CL_SPECIAL_FORMS = frozenset({ + "let", "let*", "flet", "labels", "macrolet", + "if", "when", "unless", "cond", "case", "ecase", "typecase", "etypecase", + "progn", "prog1", "prog2", "block", "return-from", "tagbody", "go", + "lambda", "function", "funcall", "apply", + "setf", "setq", "psetf", "psetq", "incf", "decf", "push", "pop", + "loop", "do", "do*", "dolist", "dotimes", "map", "mapcar", "mapc", + "format", "print", "princ", "prin1", "write", "terpri", + "and", "or", "not", "null", + "values", "multiple-value-bind", "multiple-value-setq", + "declare", "the", "locally", "eval-when", + "quote", "backquote", + "error", "signal", "warn", "assert", "check-type", + "handler-case", "handler-bind", "restart-case", "ignore-errors", + "unwind-protect", "catch", "throw", + "with-open-file", "with-output-to-string", "with-input-from-string", + "with-slots", "with-accessors", + "defvar", "defparameter", "defconstant", + "in-package", "require", "provide", "use-package", + "t", "nil", +}) + + +def extract_commonlisp(path: Path) -> dict: + """Extract packages, classes, functions, methods, macros, and calls from a Common Lisp file.""" + try: + import tree_sitter_commonlisp as tscl + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree-sitter-commonlisp not installed"} + + try: + language = Language(tscl.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = path.stem + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + function_bodies: list[tuple[str, object]] = [] # (nid, body_nodes) + current_package: str | None = None + + def _text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + # CL symbols can contain operator chars like = < > ? ! + * / that the + # generic _make_id strips. Map them to readable suffixes so upi=, upi<, + # upi> become distinct ids. + _CL_CHAR_MAP = { + '=': '_eq', '<': '_lt', '>': '_gt', '?': '_p', '!': '_bang', + '+': '_plus', '*': '_star', '/': '_slash', '%': '_pct', '&': '_amp', + } + + def _cl_id(*parts: str) -> str: + normalized = [] + for p in parts: + normalized.append(''.join(_CL_CHAR_MAP.get(c, c) for c in p)) + return _make_id(*normalized) + + def add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({ + "id": nid, + "label": label, + "file_type": "code", + "source_file": str_path, + "source_location": f"L{line}", + }) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", weight: float = 1.0) -> None: + edges.append({ + "source": src, + "target": tgt, + "relation": relation, + "confidence": confidence, + "source_file": str_path, + "source_location": f"L{line}", + "weight": weight, + }) + + file_nid = _cl_id(stem) + add_node(file_nid, path.name, 1) + + def _first_sym(node) -> str | None: + """Get the first sym_lit text from a list_lit's children.""" + for child in node.children: + if child.type == "sym_lit": + return _text(child) + return None + + def _kwd_text(node) -> str: + """Extract the symbol name from a kwd_lit, stripping the leading colon.""" + t = _text(node) + return t.lstrip(":").lstrip("#:") + + def _handle_defpackage(node) -> None: + nonlocal current_package + children = node.children + pkg_name = None + for child in children: + if child.type == "kwd_lit" and pkg_name is None: + pkg_name = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "defpackage": + pkg_name = _text(child) + break + if not pkg_name: + return + current_package = pkg_name + pkg_nid = _cl_id(stem, pkg_name) + line = node.start_point[0] + 1 + add_node(pkg_nid, pkg_name, line) + add_edge(file_nid, pkg_nid, "contains", line) + + # Extract :use clauses as imports + for child in children: + if child.type == "list_lit": + first = _first_sym(child) + if first is None: + # Could be (:use ...) with kwd_lit + for gc in child.children: + if gc.type == "kwd_lit" and _kwd_text(gc) == "use": + for uc in child.children: + if uc.type == "kwd_lit" and uc != gc: + mod_name = _kwd_text(uc) + if mod_name != "use": + tgt_nid = _cl_id(mod_name) + add_edge(pkg_nid, tgt_nid, "imports", + child.start_point[0] + 1) + break + + def _handle_defclass(node) -> None: + children = node.children + # Find class name: second sym_lit after "defclass" + sym_lits = [c for c in children if c.type == "sym_lit"] + if len(sym_lits) < 2: + return + class_name = _text(sym_lits[1]) + line = node.start_point[0] + 1 + class_nid = _cl_id(stem, class_name) + add_node(class_nid, class_name, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, class_nid, "contains", line) + + # Superclasses (third element, a list) + list_lits = [c for c in children if c.type == "list_lit"] + if list_lits: + superclass_list = list_lits[0] + for sc in superclass_list.children: + if sc.type == "sym_lit": + sc_name = _text(sc) + sc_nid = _cl_id(stem, sc_name) + add_edge(class_nid, sc_nid, "inherits", + superclass_list.start_point[0] + 1) + + def _handle_defun_node(node) -> None: + """Handle a defun AST node (covers defun, defmethod, defgeneric, defmacro).""" + header = None + for child in node.children: + if child.type == "defun_header": + header = child + break + if not header: + return + + # Determine keyword type + keyword_type = "defun" + func_name = None + for child in header.children: + if child.type == "defun_keyword": + for kc in child.children: + if kc.type in ("defun", "defmethod", "defgeneric", "defmacro"): + keyword_type = kc.type + break + if child.type == "sym_lit" and func_name is None: + func_name = _text(child) + + if not func_name: + return + + line = node.start_point[0] + 1 + func_nid = _cl_id(stem, func_name) + + if keyword_type == "defmethod": + label = f".{func_name}()" + elif keyword_type == "defmacro": + label = f"{func_name} (macro)" + elif keyword_type == "defgeneric": + label = f"{func_name} (generic)" + else: + label = f"{func_name}()" + + add_node(func_nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + if keyword_type == "defmethod": + add_edge(parent_nid, func_nid, "method", line) + else: + add_edge(parent_nid, func_nid, "contains", line) + + # Method specializers: (defmethod name ((param class) ...)) + if keyword_type == "defmethod": + for child in header.children: + if child.type == "list_lit": + # This is the parameter list + for param in child.children: + if param.type == "list_lit": + syms = [c for c in param.children if c.type == "sym_lit"] + if len(syms) >= 2: + specializer_name = _text(syms[1]) + spec_nid = _cl_id(stem, specializer_name) + add_edge(func_nid, spec_nid, "specializes", + param.start_point[0] + 1) + break + + # Docstring + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(func_nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1) + add_edge(doc_nid, func_nid, "rationale_for", + child.start_point[0] + 1) + break + + # Collect body for call extraction + body_nodes = [c for c in node.children + if c.type not in ("(", ")", "defun_header", "str_lit")] + if body_nodes: + function_bodies.append((func_nid, body_nodes)) + + def _extract_def_name(node, def_keyword: str) -> str | None: + """Get the name being defined. May be a sym_lit or a list_lit whose + first sym_lit is the name (e.g. (defstruct (foo :conc-name "BAR-") ...)).""" + seen_keyword = False + for child in node.children: + if not seen_keyword: + if child.type == "sym_lit" and _text(child).lower() == def_keyword: + seen_keyword = True + continue + if child.type == "sym_lit": + return _text(child) + if child.type == "list_lit": + for gc in child.children: + if gc.type == "sym_lit": + return _text(gc) + return None + return None + + def _collect_def_body(node, def_keyword: str) -> list: + """Collect body forms from (DEFKEYWORD name (params) [doc] body...). + Skips the def keyword, the name, the params list (if present), and + a leading docstring str_lit.""" + children = [c for c in node.children if c.type not in ("(", ")")] + idx = 0 + # Skip def keyword + if idx < len(children) and children[idx].type == "sym_lit" \ + and _text(children[idx]).lower() == def_keyword: + idx += 1 + # Skip name (sym_lit or list_lit) + if idx < len(children) and children[idx].type in ("sym_lit", "list_lit"): + idx += 1 + # Skip params list (the next list_lit, if any) + if idx < len(children) and children[idx].type == "list_lit": + idx += 1 + # Skip leading docstring + if idx < len(children) and children[idx].type == "str_lit": + idx += 1 + return children[idx:] + + def _handle_def_form(node, def_keyword: str) -> None: + """Handle a generic (DEFKEYWORD name ...) form. Covers standard CL + definers (defstruct, deftype, defvar, define-condition, etc.) and + custom def-prefixed macros (definline, definline-maybe, etc.).""" + name = _extract_def_name(node, def_keyword) + if not name: + return + line = node.start_point[0] + 1 + nid = _cl_id(stem, name) + + kw = def_keyword.lower() + if kw in _CL_DATA_DEFINERS: + label = name + is_callable = False + elif kw in _CL_MACRO_DEFINERS or "macro" in kw: + label = f"{name} (macro)" + is_callable = True + else: + # Custom def-prefixed: assume function-like + label = f"{name}()" + is_callable = True + + add_node(nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, nid, "contains", line) + + # Docstring. Skip for defvar/defparameter/defconstant — their second + # argument is a value (possibly a string literal) which would be + # wrongly captured as a docstring. + if kw not in _CL_VALUE_DEFINERS: + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1) + add_edge(doc_nid, nid, "rationale_for", + child.start_point[0] + 1) + break + + if is_callable: + body_nodes = _collect_def_body(node, def_keyword) + if body_nodes: + function_bodies.append((nid, body_nodes)) + + def _is_def_prefixed(sym: str) -> bool: + """Heuristic: does this symbol look like a definition macro?""" + if sym in _CL_NOT_DEFINERS: + return False + return sym.startswith("def") or sym.startswith("define-") + + def _process_form(top) -> bool: + """Dispatch on a single list_lit form. Returns True if it was + recognized as a definition or package directive (caller must NOT + recurse into it). Returns False if unrecognized, so the caller can + recurse to find defs nested inside wrapper macros like + (optimizing ...), (eval-when ...), (progn ...), etc.""" + nonlocal current_package + + # Check for defun node type inside list_lit + for child in top.children: + if child.type == "defun": + _handle_defun_node(child) + return True + + first = _first_sym(top) + if not first: + return False + first_lower = first.lower() + + if first_lower == "defpackage": + _handle_defpackage(top) + return True + if first_lower == "in-package": + for child in top.children: + if child.type == "kwd_lit": + current_package = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "in-package": + current_package = _text(child) + break + return True + if first_lower == "defclass": + _handle_defclass(top) + return True + if first_lower in ("require", "ql:quickload"): + for child in top.children: + if child.type in ("kwd_lit", "str_lit"): + mod_name = _kwd_text(child) if child.type == "kwd_lit" else _text(child).strip('"') + if mod_name: + tgt_nid = _cl_id(mod_name) + add_edge(file_nid, tgt_nid, "imports", + top.start_point[0] + 1) + break + return True + if first_lower in _CL_DATA_DEFINERS or first_lower in _CL_MACRO_DEFINERS: + _handle_def_form(top, first_lower) + return True + if _is_def_prefixed(first_lower): + # Custom definer (definline, definline-maybe, defcomponent, etc.) + _handle_def_form(top, first_lower) + return True + + return False + + def _walk_forms(parent) -> None: + """Walk list_lit children of `parent`, processing each. If a form + isn't recognized, recurse into it — many CL codebases wrap + definitions in macros like (optimizing ...), (eval-when ...), or + (progn ...) that aren't themselves definers but contain defs. + Also descends into reader conditionals (#+feature / #-feature), + which wrap their guarded form in an include_reader_macro node.""" + for top in parent.children: + if top.type == "list_lit": + if not _process_form(top): + _walk_forms(top) + elif top.type == "include_reader_macro": + _walk_forms(top) + + _walk_forms(root) + + # Call extraction pass + label_to_nid: dict[str, str] = {} + for n in nodes: + raw = n["label"] + normalised = raw.replace(" (macro)", "").replace(" (generic)", "").strip("()").lstrip(".") + label_to_nid[normalised.lower()] = n["id"] + + seen_call_pairs: set[tuple[str, str]] = set() + + def walk_calls(node, caller_nid: str) -> None: + if node.type == "defun": + return + if node.type == "list_lit": + callee = _first_sym(node) + if callee and callee.lower() not in _CL_SPECIAL_FORMS: + tgt_nid = label_to_nid.get(callee.lower()) + if tgt_nid and tgt_nid != caller_nid: + pair = (caller_nid, tgt_nid) + if pair not in seen_call_pairs: + seen_call_pairs.add(pair) + add_edge(caller_nid, tgt_nid, "calls", + node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0) + for child in node.children: + walk_calls(child, caller_nid) + + for caller_nid, body_nodes in function_bodies: + for body_node in body_nodes: + walk_calls(body_node, caller_nid) + + clean_edges = [e for e in edges if e["source"] in seen_ids and + (e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))] + return {"nodes": nodes, "edges": clean_edges} + + +# ── Main extract and collect_files ──────────────────────────────────────────── + + +def _check_tree_sitter_version() -> None: + """Raise a clear error if tree-sitter is too old for the new Language API.""" + try: + from tree_sitter import LANGUAGE_VERSION + except ImportError: + raise ImportError( + "tree-sitter is not installed. Run: pip install 'tree-sitter>=0.23.0'" + ) + # Language API v2 starts at LANGUAGE_VERSION 14 + if LANGUAGE_VERSION < 14: + import tree_sitter as _ts + raise RuntimeError( + f"tree-sitter {getattr(_ts, '__version__', 'unknown')} is too old. " + f"graphify requires tree-sitter >= 0.23.0 (Language API v2). " + f"Run: pip install --upgrade tree-sitter" + ) + + + + _DISPATCH: dict[str, Any] = { ".py": extract_python, ".js": extract_js, From 4ff26aedea82da6e529c7cafa4cbc964fd423910 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:33:45 -0400 Subject: [PATCH 02/11] fix(extract): type Common Lisp rationale nodes so they don't leak Emit docstring-derived rationale nodes with file_type "rationale" instead of "code" so the cross-file resolution filter excludes them, keeping docstrings out of import and symbol-resolution edges. Co-authored-by: jansaasman --- graphify/extract.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 8ce1ad2f5..a02a6f9b7 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4130,13 +4130,13 @@ def _cl_id(*parts: str) -> str: normalized.append(''.join(_CL_CHAR_MAP.get(c, c) for c in p)) return _make_id(*normalized) - def add_node(nid: str, label: str, line: int) -> None: + def add_node(nid: str, label: str, line: int, file_type: str = "code") -> None: if nid not in seen_ids: seen_ids.add(nid) nodes.append({ "id": nid, "label": label, - "file_type": "code", + "file_type": file_type, "source_file": str_path, "source_location": f"L{line}", }) @@ -4303,7 +4303,7 @@ def _handle_defun_node(node) -> None: doc_text = _text(child).strip('"') if doc_text: doc_nid = _cl_id(func_nid, "rationale") - add_node(doc_nid, doc_text[:120], child.start_point[0] + 1) + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") add_edge(doc_nid, func_nid, "rationale_for", child.start_point[0] + 1) break @@ -4393,7 +4393,7 @@ def _handle_def_form(node, def_keyword: str) -> None: doc_text = _text(child).strip('"') if doc_text: doc_nid = _cl_id(nid, "rationale") - add_node(doc_nid, doc_text[:120], child.start_point[0] + 1) + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") add_edge(doc_nid, nid, "rationale_for", child.start_point[0] + 1) break From 73881c16baeb2a6a98acb27df417c989e10c2c1c Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:33:57 -0400 Subject: [PATCH 03/11] feat(detect): dispatch Common Lisp source files Classify .lisp/.cl/.lsp/.asd as code and route them to the Common Lisp extractor so ASDF systems and source files build into the graph. Co-authored-by: jansaasman --- graphify/detect.py | 2 +- graphify/extract.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/graphify/detect.py b/graphify/detect.py index 0b569e4b7..7393805a6 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -28,7 +28,7 @@ class FileType(str, Enum): _MANIFEST_PATH = str(out_path("manifest.json")) -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.lisp', '.cl', '.lsp', '.asd', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} diff --git a/graphify/extract.py b/graphify/extract.py index a02a6f9b7..7edefa995 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4597,6 +4597,10 @@ def _check_tree_sitter_version() -> None: ".sv": extract_verilog, ".svh": extract_verilog, ".sql": extract_sql, + ".lisp": extract_commonlisp, + ".cl": extract_commonlisp, + ".lsp": extract_commonlisp, + ".asd": extract_commonlisp, ".md": extract_markdown, ".mdx": extract_markdown, ".qmd": extract_markdown, From 980ef93f158b010d44c642a21b77307ce586cc2e Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:34:11 -0400 Subject: [PATCH 04/11] build: add tree-sitter-commonlisp as an optional extra Declare the Common Lisp grammar under optional-dependencies, matching the policy for other niche grammars, so the core install stays lean and Lisp users opt in with the commonlisp extra. Co-authored-by: jansaasman --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5a0dce38b..eebd2577b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,12 +76,13 @@ sql = ["tree-sitter-sql"] # absent (#781), so this stays optional. Unlike tree-sitter-dm below, it ships # prebuilt wheels for every platform (win/macOS/Linux), so no C toolchain needed. pascal = ["tree-sitter-pascal"] +commonlisp = ["tree-sitter-commonlisp"] # tree-sitter-dm (BYOND DreamMaker) ships only a Windows wheel, so on Linux/Mac it # must compile from source (needs a C toolchain + python3-dev). Keeping it optional # avoids breaking the default `uv tool install graphifyy` for everyone (#1104). dm = ["tree-sitter-dm"] terraform = ["tree-sitter-hcl"] -all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal"] +all = ["mcp", "starlette>=1.3.1", "neo4j", "falkordb", "pypdf>=6.12.0", "markdownify", "watchdog", "graspologic; python_version < '3.13'", "python-docx", "openpyxl", "faster-whisper; python_version >= '3.11'", "yt-dlp>=2026.6.9", "matplotlib", "numpy>=2.0; python_version >= '3.13'", "openai", "tiktoken", "boto3", "anthropic", "tree-sitter-sql", "jieba", "tree-sitter-dm", "tree-sitter-hcl", "tree-sitter-pascal", "tree-sitter-commonlisp"] [project.scripts] graphify = "graphify.__main__:main" From af7b9d1e0ee009e986479d90370db6420b62e1ec Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:35:49 -0400 Subject: [PATCH 05/11] test: port Common Lisp extraction suite Add the sample.lisp fixture and the Common Lisp extraction tests covering packages, classes, defuns, generics, macros, method specializers, inheritance, imports, custom definers, reader conditionals, and docstring rationale typing. Co-authored-by: jansaasman --- tests/fixtures/sample.lisp | 70 ++++++++++ tests/test_multilang.py | 256 ++++++++++++++++++++++++++++++++++++- 2 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/sample.lisp diff --git a/tests/fixtures/sample.lisp b/tests/fixtures/sample.lisp new file mode 100644 index 000000000..4dc2dc417 --- /dev/null +++ b/tests/fixtures/sample.lisp @@ -0,0 +1,70 @@ +(defpackage :http-server + (:use :cl :alexandria) + (:export #:make-server #:start #:stop)) + +(in-package :http-server) + +(deftype port-number () '(integer 1 65535)) + +(defstruct request-stats + bytes-in + bytes-out + duration-ms) + +(defstruct (connection (:conc-name conn-)) + id + socket + state) + +(defvar *active-connections* nil) +(defparameter *default-port* 8080) +(defconstant +max-headers+ 100) + +(define-condition server-error (error) + ((reason :initarg :reason :reader error-reason))) + +(defclass server () + ((host :initarg :host :accessor server-host) + (port :initarg :port :accessor server-port) + (handler :initarg :handler :accessor server-handler))) + +(defclass ssl-server (server) + ((cert-path :initarg :cert-path :accessor ssl-cert-path))) + +(defgeneric process-request (server request)) + +(defmethod process-request ((srv server) (req string)) + "Process an incoming HTTP request." + (let ((parsed (parse-headers req))) + (funcall (server-handler srv) parsed))) + +;; Custom definer (Franz-style) — should be picked up by the def-prefix heuristic +(definline-maybe header= (a b) + "Fast header equality." + (string-equal a b)) + +(definline header< (a b) + (string< a b)) + +(defun make-server (host port handler) + "Create a new server instance." + (make-instance 'server :host host :port port :handler handler)) + +(defun start (server) + "Start the server listening on its configured port." + (format t "Starting server on ~a:~a~%" (server-host server) (server-port server)) + (process-request server "GET / HTTP/1.1")) + +(defun stop (server) + (format t "Stopping server~%")) + +(defun compare-headers (h1 h2) + "Compare two headers using the custom definers." + (or (header= h1 h2) (header< h1 h2))) + +(defmacro with-server ((var host port handler) &body body) + "Execute body with a running server bound to var." + `(let ((,var (make-server ,host ,port ,handler))) + (unwind-protect + (progn (start ,var) ,@body) + (stop ,var)))) diff --git a/tests/test_multilang.py b/tests/test_multilang.py index 4bac41bc4..d9828552e 100644 --- a/tests/test_multilang.py +++ b/tests/test_multilang.py @@ -3,7 +3,7 @@ import shutil from pathlib import Path import pytest -from graphify.extract import extract_js, extract_go, extract_rust, extract, extract_sql +from graphify.extract import extract_js, extract_go, extract_rust, extract, extract_sql, extract_commonlisp FIXTURES = Path(__file__).parent / "fixtures" @@ -586,3 +586,257 @@ def test_sql_quoted_plpgsql_file_stays_clean(): contains_targets = {e["target"] for e in r["edges"] if e["relation"] == "contains"} fn_ids = {n["id"] for n in r["nodes"] if n["label"].endswith("()")} assert fn_ids <= contains_targets + +# ── Common Lisp ────────────────────────────────────────────────────────────── + +def test_cl_finds_package(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "error" not in r + assert "http-server" in _labels(r) + +def test_cl_finds_class(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server" in _labels(r) + assert "ssl-server" in _labels(r) + +def test_cl_finds_defun(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert any("make-server" in l for l in labels) + assert any("start" in l for l in labels) + assert any("stop" in l for l in labels) + +def test_cl_finds_generic(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("process-request" in l for l in _labels(r)) + +def test_cl_finds_macro(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("with-server" in l and "macro" in l for l in _labels(r)) + +def test_cl_emits_calls(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _call_pairs(r) + # start() calls process-request + assert any("start" in src and "process-request" in tgt for src, tgt in calls) + # with-server macro calls make-server and start + assert any("with-server" in src and "make-server" in tgt for src, tgt in calls) + assert any("with-server" in src and "start" in tgt for src, tgt in calls) + +def test_cl_calls_are_extracted(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + for e in r["edges"]: + if e["relation"] == "calls": + assert e["confidence"] == "EXTRACTED" + +def test_cl_no_dangling_edges(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + if e["relation"] in ("contains", "method", "calls"): + assert e["source"] in node_ids + +def test_cl_docstrings(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + rationale_edges = [e for e in r["edges"] if e["relation"] == "rationale_for"] + assert len(rationale_edges) >= 3 # make-server, start, process-request have docstrings + labels = _labels(r) + assert any("Process an incoming" in l for l in labels) + +def test_cl_method_specializers(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + spec_edges = [e for e in r["edges"] if e["relation"] == "specializes"] + assert len(spec_edges) >= 1 + # process-request specializes on server + assert any("process_request" in e["source"] and "server" in e["target"] for e in spec_edges) + +def test_cl_inherits(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + inherit_edges = [e for e in r["edges"] if e["relation"] == "inherits"] + assert len(inherit_edges) >= 1 + assert any("ssl_server" in e["source"] and "server" in e["target"] for e in inherit_edges) + +def test_cl_imports(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + import_edges = [e for e in r["edges"] if e["relation"] == "imports"] + targets = {e["target"] for e in import_edges} + assert "cl" in targets + assert "alexandria" in targets + +def test_cl_finds_deftype(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "port-number" in _labels(r) + +def test_cl_finds_defstruct(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "request-stats" in labels + # defstruct with options form: (defstruct (name ...) ...) + assert "connection" in labels + +def test_cl_finds_defvar_defparameter_defconstant(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "*active-connections*" in labels + assert "*default-port*" in labels + assert "+max-headers+" in labels + +def test_cl_finds_define_condition(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server-error" in _labels(r) + +def test_cl_finds_custom_definer(): + """The def-prefix heuristic should catch definline / definline-maybe.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + # definline-maybe and definline should produce function-style nodes + assert "header=()" in labels + assert "header<()" in labels + +def test_cl_custom_definer_in_call_graph(): + """Functions defined via custom definers should appear in the call graph.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _call_pairs(r) + # compare-headers calls header= and header< (defined via definline-maybe / definline) + assert any("compare-headers" in src and "header=" in tgt for src, tgt in calls) + assert any("compare-headers" in src and "header<" in tgt for src, tgt in calls) + +def test_cl_operator_names_disambiguated(): + """upi=, upi<, upi> must produce distinct ids (operator chars matter).""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + # header= and header< must have different ids + eq_ids = [n["id"] for n in r["nodes"] if n["label"] == "header=()"] + lt_ids = [n["id"] for n in r["nodes"] if n["label"] == "header<()"] + assert len(eq_ids) == 1 + assert len(lt_ids) == 1 + assert eq_ids[0] != lt_ids[0] + +def test_cl_default_value_not_treated_as_definition(): + """The def-prefix heuristic must not match denylisted symbols.""" + import tempfile + code = "(in-package :cl-user)\n(default-value foo)\n" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + # default-value is denylisted, shouldn't create a "foo" node + assert "foo" not in labels + assert "foo()" not in labels + finally: + path.unlink() + +def test_cl_defs_inside_wrapper_macro(): + """Definitions nested inside wrapper macros like (optimizing ...) or + (eval-when ...) must be extracted. Many CL codebases wrap hot-path + inline functions in application-specific macros.""" + import tempfile + code = """(in-package :cl-user) +(optimizing + (definline-maybe packet-type (p) (aref p 0)) + (definline-maybe set-packet-type (p v) (setf (aref p 0) v))) +(eval-when (:compile-toplevel :load-toplevel :execute) + (defun helper () 42)) +(progn + (defun progn-def () 'ok)) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "packet-type()" in labels + assert "set-packet-type()" in labels + assert "helper()" in labels + assert "progn-def()" in labels + finally: + path.unlink() + +def test_cl_defs_inside_reader_conditional(): + """#+feature / #-feature reader conditionals wrap their guarded form + in an include_reader_macro AST node, which the walker must descend + into to find the nested definition.""" + import tempfile + code = """(in-package :cl-user) +#+little-endian +(definline-maybe byte-hash= (a b) (eq a b)) +#-sbcl +(defun only-on-non-sbcl () 1) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "byte-hash=()" in labels + assert "only-on-non-sbcl()" in labels + finally: + path.unlink() + +def test_cl_defparameter_string_value_not_docstring(): + """For defvar/defparameter/defconstant, a string literal in the VALUE + position must not be wrongly captured as a docstring node.""" + import tempfile + code = '''(in-package :cl-user) +(defparameter *config-path* "/etc/app/config") +(defvar *greeting* "hello world") +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "*config-path*" in labels + assert "*greeting*" in labels + # The string VALUES must not show up as rationale nodes + assert not any("/etc/app/config" in l for l in labels) + assert not any("hello world" in l for l in labels) + finally: + path.unlink() + + +# ── extract() dispatch ──────────────────────────────────────────────────────── + +def test_extract_dispatches_all_languages(): + files = [ + FIXTURES / "sample.py", + FIXTURES / "sample.ts", + FIXTURES / "sample.go", + FIXTURES / "sample.rs", + FIXTURES / "sample.lisp", + ] + r = extract(files) + source_files = {n["source_file"] for n in r["nodes"] if n["source_file"]} + assert any("sample.py" in f for f in source_files) + assert any("sample.ts" in f for f in source_files) + assert any("sample.go" in f for f in source_files) + assert any("sample.rs" in f for f in source_files) + assert any("sample.lisp" in f for f in source_files) + + +# ── Cache ───────────────────────────────────────────────────────────────────── + +def test_cache_hit_returns_same_result(tmp_path): + src = FIXTURES / "sample.py" + dst = tmp_path / "sample.py" + dst.write_bytes(src.read_bytes()) + + r1 = extract([dst]) + r2 = extract([dst]) + assert len(r1["nodes"]) == len(r2["nodes"]) + assert len(r1["edges"]) == len(r2["edges"]) + +def test_cache_miss_after_file_change(tmp_path): + dst = tmp_path / "a.py" + dst.write_text("def foo(): pass\n") + r1 = extract([dst]) + + dst.write_text("def foo(): pass\ndef bar(): pass\n") + r2 = extract([dst]) + # bar() should appear in the second result + labels2 = [n["label"] for n in r2["nodes"]] + assert any("bar" in l for l in labels2) From 5330297bb25633e7245dcd0ed1b2b53c3fae1f94 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:49:40 -0400 Subject: [PATCH 06/11] fix(extract): silence the Common Lisp grammar's deprecation warning The latest tree-sitter-commonlisp release returns an int pointer that the tree-sitter runtime only accepts via a deprecated path, so extracting Lisp spewed a DeprecationWarning per file. Suppress that one warning at the construction site so Lisp extraction runs clean. Co-authored-by: jansaasman --- graphify/extract.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/graphify/extract.py b/graphify/extract.py index 7edefa995..f0e804195 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -7,6 +7,7 @@ import os import re import sys +import warnings import textwrap from collections import Counter from dataclasses import dataclass, field @@ -4097,7 +4098,17 @@ def extract_commonlisp(path: Path) -> dict: return {"nodes": [], "edges": [], "error": "tree-sitter-commonlisp not installed"} try: - language = Language(tscl.language()) + # tree-sitter-commonlisp 0.4.1 (latest) ships the old binding that returns + # an int pointer from language(); tree_sitter.Language only accepts that + # int via a deprecated path. Silence that one warning at the call site + # until the grammar ships a PyCapsule binding. + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="int argument support is deprecated", + category=DeprecationWarning, + ) + language = Language(tscl.language()) parser = Parser(language) source = path.read_bytes() tree = parser.parse(source) From 6dcdbad1ecc560ccb52de75b8ad69f641b567ea1 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:58:00 -0400 Subject: [PATCH 07/11] test: house Common Lisp tests with the other language extractors Move the Common Lisp extraction tests into the language-extractor suite where the project keeps per-language coverage, so they sit with their peers rather than in the general multi-language file. Co-authored-by: jansaasman --- tests/test_languages.py | 214 ++++++++++++++++++++++++++++++++- tests/test_multilang.py | 256 +--------------------------------------- 2 files changed, 214 insertions(+), 256 deletions(-) diff --git a/tests/test_languages.py b/tests/test_languages.py index aff689d2a..79430bb6e 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -8,7 +8,7 @@ extract_swift, extract_go, extract_julia, extract_js, extract_fortran, extract_groovy, extract_sln, extract_csproj, extract_xaml, extract_razor, extract_dm, extract_dmi, extract_dmm, extract_dmf, - extract_powershell, extract_apex, extract_verilog, + extract_powershell, extract_apex, extract_commonlisp, extract_verilog, extract_powershell_manifest, ) @@ -2999,3 +2999,215 @@ def test_decldef_merge_does_not_merge_same_name_same_dir_distinct_files(): r = _corpus("cpp_samedir/Alpha.h", "cpp_samedir/Beta.h") dups = _nodes_with_label(r, "Dup") assert len(dups) == 2, f"same-dir distinct Dups must stay distinct, got {[n['id'] for n in dups]}" + +# ── Common Lisp ────────────────────────────────────────────────────────────── + +def test_cl_finds_package(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "error" not in r + assert "http-server" in _labels(r) + +def test_cl_finds_class(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server" in _labels(r) + assert "ssl-server" in _labels(r) + +def test_cl_finds_defun(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert any("make-server" in l for l in labels) + assert any("start" in l for l in labels) + assert any("stop" in l for l in labels) + +def test_cl_finds_generic(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("process-request" in l for l in _labels(r)) + +def test_cl_finds_macro(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert any("with-server" in l and "macro" in l for l in _labels(r)) + +def test_cl_emits_calls(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _calls(r) + # start() calls process-request + assert any("start" in src and "process-request" in tgt for src, tgt in calls) + # with-server macro calls make-server and start + assert any("with-server" in src and "make-server" in tgt for src, tgt in calls) + assert any("with-server" in src and "start" in tgt for src, tgt in calls) + +def test_cl_calls_are_extracted(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + for e in r["edges"]: + if e["relation"] == "calls": + assert e["confidence"] == "EXTRACTED" + +def test_cl_no_dangling_edges(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + node_ids = {n["id"] for n in r["nodes"]} + for e in r["edges"]: + if e["relation"] in ("contains", "method", "calls"): + assert e["source"] in node_ids + +def test_cl_docstrings(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + rationale_edges = [e for e in r["edges"] if e["relation"] == "rationale_for"] + assert len(rationale_edges) >= 3 # make-server, start, process-request have docstrings + labels = _labels(r) + assert any("Process an incoming" in l for l in labels) + +def test_cl_method_specializers(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + spec_edges = [e for e in r["edges"] if e["relation"] == "specializes"] + assert len(spec_edges) >= 1 + # process-request specializes on server + assert any("process_request" in e["source"] and "server" in e["target"] for e in spec_edges) + +def test_cl_inherits(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + inherit_edges = [e for e in r["edges"] if e["relation"] == "inherits"] + assert len(inherit_edges) >= 1 + assert any("ssl_server" in e["source"] and "server" in e["target"] for e in inherit_edges) + +def test_cl_imports(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + import_edges = [e for e in r["edges"] if e["relation"] == "imports"] + targets = {e["target"] for e in import_edges} + assert "cl" in targets + assert "alexandria" in targets + +def test_cl_finds_deftype(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "port-number" in _labels(r) + +def test_cl_finds_defstruct(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "request-stats" in labels + # defstruct with options form: (defstruct (name ...) ...) + assert "connection" in labels + +def test_cl_finds_defvar_defparameter_defconstant(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + assert "*active-connections*" in labels + assert "*default-port*" in labels + assert "+max-headers+" in labels + +def test_cl_finds_define_condition(): + r = extract_commonlisp(FIXTURES / "sample.lisp") + assert "server-error" in _labels(r) + +def test_cl_finds_custom_definer(): + """The def-prefix heuristic should catch definline / definline-maybe.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + labels = _labels(r) + # definline-maybe and definline should produce function-style nodes + assert "header=()" in labels + assert "header<()" in labels + +def test_cl_custom_definer_in_call_graph(): + """Functions defined via custom definers should appear in the call graph.""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + calls = _calls(r) + # compare-headers calls header= and header< (defined via definline-maybe / definline) + assert any("compare-headers" in src and "header=" in tgt for src, tgt in calls) + assert any("compare-headers" in src and "header<" in tgt for src, tgt in calls) + +def test_cl_operator_names_disambiguated(): + """upi=, upi<, upi> must produce distinct ids (operator chars matter).""" + r = extract_commonlisp(FIXTURES / "sample.lisp") + # header= and header< must have different ids + eq_ids = [n["id"] for n in r["nodes"] if n["label"] == "header=()"] + lt_ids = [n["id"] for n in r["nodes"] if n["label"] == "header<()"] + assert len(eq_ids) == 1 + assert len(lt_ids) == 1 + assert eq_ids[0] != lt_ids[0] + +def test_cl_default_value_not_treated_as_definition(): + """The def-prefix heuristic must not match denylisted symbols.""" + import tempfile + code = "(in-package :cl-user)\n(default-value foo)\n" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + # default-value is denylisted, shouldn't create a "foo" node + assert "foo" not in labels + assert "foo()" not in labels + finally: + path.unlink() + +def test_cl_defs_inside_wrapper_macro(): + """Definitions nested inside wrapper macros like (optimizing ...) or + (eval-when ...) must be extracted. Many CL codebases wrap hot-path + inline functions in application-specific macros.""" + import tempfile + code = """(in-package :cl-user) +(optimizing + (definline-maybe packet-type (p) (aref p 0)) + (definline-maybe set-packet-type (p v) (setf (aref p 0) v))) +(eval-when (:compile-toplevel :load-toplevel :execute) + (defun helper () 42)) +(progn + (defun progn-def () 'ok)) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "packet-type()" in labels + assert "set-packet-type()" in labels + assert "helper()" in labels + assert "progn-def()" in labels + finally: + path.unlink() + +def test_cl_defs_inside_reader_conditional(): + """#+feature / #-feature reader conditionals wrap their guarded form + in an include_reader_macro AST node, which the walker must descend + into to find the nested definition.""" + import tempfile + code = """(in-package :cl-user) +#+little-endian +(definline-maybe byte-hash= (a b) (eq a b)) +#-sbcl +(defun only-on-non-sbcl () 1) +""" + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "byte-hash=()" in labels + assert "only-on-non-sbcl()" in labels + finally: + path.unlink() + +def test_cl_defparameter_string_value_not_docstring(): + """For defvar/defparameter/defconstant, a string literal in the VALUE + position must not be wrongly captured as a docstring node.""" + import tempfile + code = '''(in-package :cl-user) +(defparameter *config-path* "/etc/app/config") +(defvar *greeting* "hello world") +''' + with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: + f.write(code) + path = Path(f.name) + try: + r = extract_commonlisp(path) + labels = _labels(r) + assert "*config-path*" in labels + assert "*greeting*" in labels + # The string VALUES must not show up as rationale nodes + assert not any("/etc/app/config" in l for l in labels) + assert not any("hello world" in l for l in labels) + finally: + path.unlink() + diff --git a/tests/test_multilang.py b/tests/test_multilang.py index d9828552e..4bac41bc4 100644 --- a/tests/test_multilang.py +++ b/tests/test_multilang.py @@ -3,7 +3,7 @@ import shutil from pathlib import Path import pytest -from graphify.extract import extract_js, extract_go, extract_rust, extract, extract_sql, extract_commonlisp +from graphify.extract import extract_js, extract_go, extract_rust, extract, extract_sql FIXTURES = Path(__file__).parent / "fixtures" @@ -586,257 +586,3 @@ def test_sql_quoted_plpgsql_file_stays_clean(): contains_targets = {e["target"] for e in r["edges"] if e["relation"] == "contains"} fn_ids = {n["id"] for n in r["nodes"] if n["label"].endswith("()")} assert fn_ids <= contains_targets - -# ── Common Lisp ────────────────────────────────────────────────────────────── - -def test_cl_finds_package(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert "error" not in r - assert "http-server" in _labels(r) - -def test_cl_finds_class(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert "server" in _labels(r) - assert "ssl-server" in _labels(r) - -def test_cl_finds_defun(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - labels = _labels(r) - assert any("make-server" in l for l in labels) - assert any("start" in l for l in labels) - assert any("stop" in l for l in labels) - -def test_cl_finds_generic(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert any("process-request" in l for l in _labels(r)) - -def test_cl_finds_macro(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert any("with-server" in l and "macro" in l for l in _labels(r)) - -def test_cl_emits_calls(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - calls = _call_pairs(r) - # start() calls process-request - assert any("start" in src and "process-request" in tgt for src, tgt in calls) - # with-server macro calls make-server and start - assert any("with-server" in src and "make-server" in tgt for src, tgt in calls) - assert any("with-server" in src and "start" in tgt for src, tgt in calls) - -def test_cl_calls_are_extracted(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - for e in r["edges"]: - if e["relation"] == "calls": - assert e["confidence"] == "EXTRACTED" - -def test_cl_no_dangling_edges(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - node_ids = {n["id"] for n in r["nodes"]} - for e in r["edges"]: - if e["relation"] in ("contains", "method", "calls"): - assert e["source"] in node_ids - -def test_cl_docstrings(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - rationale_edges = [e for e in r["edges"] if e["relation"] == "rationale_for"] - assert len(rationale_edges) >= 3 # make-server, start, process-request have docstrings - labels = _labels(r) - assert any("Process an incoming" in l for l in labels) - -def test_cl_method_specializers(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - spec_edges = [e for e in r["edges"] if e["relation"] == "specializes"] - assert len(spec_edges) >= 1 - # process-request specializes on server - assert any("process_request" in e["source"] and "server" in e["target"] for e in spec_edges) - -def test_cl_inherits(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - inherit_edges = [e for e in r["edges"] if e["relation"] == "inherits"] - assert len(inherit_edges) >= 1 - assert any("ssl_server" in e["source"] and "server" in e["target"] for e in inherit_edges) - -def test_cl_imports(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - import_edges = [e for e in r["edges"] if e["relation"] == "imports"] - targets = {e["target"] for e in import_edges} - assert "cl" in targets - assert "alexandria" in targets - -def test_cl_finds_deftype(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert "port-number" in _labels(r) - -def test_cl_finds_defstruct(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - labels = _labels(r) - assert "request-stats" in labels - # defstruct with options form: (defstruct (name ...) ...) - assert "connection" in labels - -def test_cl_finds_defvar_defparameter_defconstant(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - labels = _labels(r) - assert "*active-connections*" in labels - assert "*default-port*" in labels - assert "+max-headers+" in labels - -def test_cl_finds_define_condition(): - r = extract_commonlisp(FIXTURES / "sample.lisp") - assert "server-error" in _labels(r) - -def test_cl_finds_custom_definer(): - """The def-prefix heuristic should catch definline / definline-maybe.""" - r = extract_commonlisp(FIXTURES / "sample.lisp") - labels = _labels(r) - # definline-maybe and definline should produce function-style nodes - assert "header=()" in labels - assert "header<()" in labels - -def test_cl_custom_definer_in_call_graph(): - """Functions defined via custom definers should appear in the call graph.""" - r = extract_commonlisp(FIXTURES / "sample.lisp") - calls = _call_pairs(r) - # compare-headers calls header= and header< (defined via definline-maybe / definline) - assert any("compare-headers" in src and "header=" in tgt for src, tgt in calls) - assert any("compare-headers" in src and "header<" in tgt for src, tgt in calls) - -def test_cl_operator_names_disambiguated(): - """upi=, upi<, upi> must produce distinct ids (operator chars matter).""" - r = extract_commonlisp(FIXTURES / "sample.lisp") - # header= and header< must have different ids - eq_ids = [n["id"] for n in r["nodes"] if n["label"] == "header=()"] - lt_ids = [n["id"] for n in r["nodes"] if n["label"] == "header<()"] - assert len(eq_ids) == 1 - assert len(lt_ids) == 1 - assert eq_ids[0] != lt_ids[0] - -def test_cl_default_value_not_treated_as_definition(): - """The def-prefix heuristic must not match denylisted symbols.""" - import tempfile - code = "(in-package :cl-user)\n(default-value foo)\n" - with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: - f.write(code) - path = Path(f.name) - try: - r = extract_commonlisp(path) - labels = _labels(r) - # default-value is denylisted, shouldn't create a "foo" node - assert "foo" not in labels - assert "foo()" not in labels - finally: - path.unlink() - -def test_cl_defs_inside_wrapper_macro(): - """Definitions nested inside wrapper macros like (optimizing ...) or - (eval-when ...) must be extracted. Many CL codebases wrap hot-path - inline functions in application-specific macros.""" - import tempfile - code = """(in-package :cl-user) -(optimizing - (definline-maybe packet-type (p) (aref p 0)) - (definline-maybe set-packet-type (p v) (setf (aref p 0) v))) -(eval-when (:compile-toplevel :load-toplevel :execute) - (defun helper () 42)) -(progn - (defun progn-def () 'ok)) -""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: - f.write(code) - path = Path(f.name) - try: - r = extract_commonlisp(path) - labels = _labels(r) - assert "packet-type()" in labels - assert "set-packet-type()" in labels - assert "helper()" in labels - assert "progn-def()" in labels - finally: - path.unlink() - -def test_cl_defs_inside_reader_conditional(): - """#+feature / #-feature reader conditionals wrap their guarded form - in an include_reader_macro AST node, which the walker must descend - into to find the nested definition.""" - import tempfile - code = """(in-package :cl-user) -#+little-endian -(definline-maybe byte-hash= (a b) (eq a b)) -#-sbcl -(defun only-on-non-sbcl () 1) -""" - with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: - f.write(code) - path = Path(f.name) - try: - r = extract_commonlisp(path) - labels = _labels(r) - assert "byte-hash=()" in labels - assert "only-on-non-sbcl()" in labels - finally: - path.unlink() - -def test_cl_defparameter_string_value_not_docstring(): - """For defvar/defparameter/defconstant, a string literal in the VALUE - position must not be wrongly captured as a docstring node.""" - import tempfile - code = '''(in-package :cl-user) -(defparameter *config-path* "/etc/app/config") -(defvar *greeting* "hello world") -''' - with tempfile.NamedTemporaryFile(mode='w', suffix='.lisp', delete=False) as f: - f.write(code) - path = Path(f.name) - try: - r = extract_commonlisp(path) - labels = _labels(r) - assert "*config-path*" in labels - assert "*greeting*" in labels - # The string VALUES must not show up as rationale nodes - assert not any("/etc/app/config" in l for l in labels) - assert not any("hello world" in l for l in labels) - finally: - path.unlink() - - -# ── extract() dispatch ──────────────────────────────────────────────────────── - -def test_extract_dispatches_all_languages(): - files = [ - FIXTURES / "sample.py", - FIXTURES / "sample.ts", - FIXTURES / "sample.go", - FIXTURES / "sample.rs", - FIXTURES / "sample.lisp", - ] - r = extract(files) - source_files = {n["source_file"] for n in r["nodes"] if n["source_file"]} - assert any("sample.py" in f for f in source_files) - assert any("sample.ts" in f for f in source_files) - assert any("sample.go" in f for f in source_files) - assert any("sample.rs" in f for f in source_files) - assert any("sample.lisp" in f for f in source_files) - - -# ── Cache ───────────────────────────────────────────────────────────────────── - -def test_cache_hit_returns_same_result(tmp_path): - src = FIXTURES / "sample.py" - dst = tmp_path / "sample.py" - dst.write_bytes(src.read_bytes()) - - r1 = extract([dst]) - r2 = extract([dst]) - assert len(r1["nodes"]) == len(r2["nodes"]) - assert len(r1["edges"]) == len(r2["edges"]) - -def test_cache_miss_after_file_change(tmp_path): - dst = tmp_path / "a.py" - dst.write_text("def foo(): pass\n") - r1 = extract([dst]) - - dst.write_text("def foo(): pass\ndef bar(): pass\n") - r2 = extract([dst]) - # bar() should appear in the second result - labels2 = [n["label"] for n in r2["nodes"]] - assert any("bar" in l for l in labels2) From 03b8fe484ca908825fd66593032137c6fa014624 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Sun, 7 Jun 2026 15:58:01 -0400 Subject: [PATCH 08/11] docs: list Common Lisp in the README language and extras tables Surface Common Lisp in the supported-extensions table and add the commonlisp optional-extra row so users know the grammar exists and how to install it. Co-authored-by: jansaasman --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a4f188f46..c75d0ee90 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,7 @@ Codex users also need `multi_agent = true` under `[features]` in `~/.codex/confi | `sql` | SQL schema extraction | `uv tool install "graphifyy[sql]"` | | `postgres` | Live PostgreSQL introspection (`--postgres DSN`) | `uv tool install "graphifyy[postgres]"` | | `dm` | BYOND DreamMaker `.dm`/`.dme` AST extraction (may need a C compiler + `python3-dev` if no wheel matches your platform) | `uv tool install "graphifyy[dm]"` | +| `commonlisp` | Common Lisp `.lisp`/`.cl`/`.lsp`/`.asd` AST extraction | `uv tool install "graphifyy[commonlisp]"` | | `terraform` | Terraform / HCL `.tf`/`.tfvars`/`.hcl` AST extraction | `uv tool install "graphifyy[terraform]"` | | `pascal` | Pascal / Delphi `.pas`/`.dpr`/`.dpk`/`.inc` AST extraction (more accurate `calls`/`inherits` edges; falls back to a regex extractor when absent) | `uv tool install "graphifyy[pascal]"` | | `chinese` | Chinese query segmentation (jieba) | `uv tool install "graphifyy[chinese]"` | @@ -331,7 +332,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Type | Extensions | |------|-----------| -| Code (36 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | +| Code (37 tree-sitter grammars) | `.py .ts .mts .cts .js .jsx .tsx .mjs .go .rs .java .c .cpp .cc .cxx .h .hpp .cu .cuh .metal .rb .cs .kt .kts .scala .php .swift .lua .luau .toc .zig .ps1 .psm1 .psd1 .ex .exs .m .mm .jl .lisp .cl .lsp .asd .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` (`.dm`/`.dme` requires `uv tool install graphifyy[dm]`; `.lisp`/`.cl`/`.lsp`/`.asd` requires `uv tool install graphifyy[commonlisp]`; `.mts`/`.cts` reuse the TypeScript grammar, `.cc`/`.cxx` and CUDA `.cu`/`.cuh` and Metal `.metal` reuse the C++ grammar) | | Salesforce Apex | `.cls .trigger` (regex-based; classes, interfaces, enums, methods, triggers, SOQL/DML edges) | | Terraform / HCL | `.tf .tfvars .hcl` (requires `uv tool install graphifyy[terraform]`) | | MCP configs | `.mcp.json` `mcp.json` `mcp_servers.json` `claude_desktop_config.json` — extracts server nodes, package refs, env var requirements | From 0f510801c62324010048eb36fc0b9ec7e38a1559 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Tue, 28 Jul 2026 11:13:33 -0400 Subject: [PATCH 09/11] refactor(extract): house the Common Lisp extractor in extractors/ extract.py is being decomposed one language per module (#1212), so a newly added language belongs in the package rather than adding another 500 lines to the file being emptied. The facade re-export and the LANGUAGE_EXTRACTORS entry keep every existing importer and the registry identity guard working unchanged. The extractor body is unchanged; only the import plumbing moved. --- graphify/extract.py | 489 +---------------------------- graphify/extractors/__init__.py | 2 + graphify/extractors/commonlisp.py | 492 ++++++++++++++++++++++++++++++ 3 files changed, 495 insertions(+), 488 deletions(-) create mode 100644 graphify/extractors/commonlisp.py diff --git a/graphify/extract.py b/graphify/extract.py index f0e804195..392527f89 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -7,7 +7,6 @@ import os import re import sys -import warnings import textwrap from collections import Counter from dataclasses import dataclass, field @@ -35,6 +34,7 @@ from graphify.extractors.apex import extract_apex # noqa: F401 from graphify.extractors.bash import extract_bash # noqa: F401 from graphify.extractors.blade import extract_blade # noqa: F401 +from graphify.extractors.commonlisp import extract_commonlisp # noqa: F401 from graphify.extractors.csharp import ( CsharpNameResolver, _resolve_cross_file_csharp_imports, @@ -4037,493 +4037,6 @@ def add_existing_edge(edge: dict) -> None: # block defined in the corpus (count.index, each.key, self.*, path.module, ...). -# ── Common Lisp ────────────────────────────────────────────────────────────── - -# Standard CL definer forms that introduce data/type/variable bindings -# (not callable, no body to walk for calls) -_CL_DATA_DEFINERS = frozenset({ - "defstruct", "deftype", "define-condition", - "defvar", "defparameter", "defconstant", - "define-symbol-macro", -}) - -# Subset of data definers that take a VALUE as their second argument, which -# may itself be a string literal. For these, we must not mistake the value -# for a docstring. -_CL_VALUE_DEFINERS = frozenset({"defvar", "defparameter", "defconstant"}) - -# Standard CL macro-style definers -_CL_MACRO_DEFINERS = frozenset({ - "define-modify-macro", "define-compiler-macro", - "define-setf-expander", "defsetf", -}) - -# Symbols that start with "def" but are NOT definitions (denylist for the -# def-prefix heuristic that catches custom definers like definline-maybe) -_CL_NOT_DEFINERS = frozenset({ - "default", "default-value", "defaults", "define", - "defer", "deferred", "deflate", "deftest", -}) - -# CL special forms / macros that should not count as "calls" in the graph -_CL_SPECIAL_FORMS = frozenset({ - "let", "let*", "flet", "labels", "macrolet", - "if", "when", "unless", "cond", "case", "ecase", "typecase", "etypecase", - "progn", "prog1", "prog2", "block", "return-from", "tagbody", "go", - "lambda", "function", "funcall", "apply", - "setf", "setq", "psetf", "psetq", "incf", "decf", "push", "pop", - "loop", "do", "do*", "dolist", "dotimes", "map", "mapcar", "mapc", - "format", "print", "princ", "prin1", "write", "terpri", - "and", "or", "not", "null", - "values", "multiple-value-bind", "multiple-value-setq", - "declare", "the", "locally", "eval-when", - "quote", "backquote", - "error", "signal", "warn", "assert", "check-type", - "handler-case", "handler-bind", "restart-case", "ignore-errors", - "unwind-protect", "catch", "throw", - "with-open-file", "with-output-to-string", "with-input-from-string", - "with-slots", "with-accessors", - "defvar", "defparameter", "defconstant", - "in-package", "require", "provide", "use-package", - "t", "nil", -}) - - -def extract_commonlisp(path: Path) -> dict: - """Extract packages, classes, functions, methods, macros, and calls from a Common Lisp file.""" - try: - import tree_sitter_commonlisp as tscl - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree-sitter-commonlisp not installed"} - - try: - # tree-sitter-commonlisp 0.4.1 (latest) ships the old binding that returns - # an int pointer from language(); tree_sitter.Language only accepts that - # int via a deprecated path. Silence that one warning at the call site - # until the grammar ships a PyCapsule binding. - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - message="int argument support is deprecated", - category=DeprecationWarning, - ) - language = Language(tscl.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - stem = path.stem - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - function_bodies: list[tuple[str, object]] = [] # (nid, body_nodes) - current_package: str | None = None - - def _text(node) -> str: - return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") - - # CL symbols can contain operator chars like = < > ? ! + * / that the - # generic _make_id strips. Map them to readable suffixes so upi=, upi<, - # upi> become distinct ids. - _CL_CHAR_MAP = { - '=': '_eq', '<': '_lt', '>': '_gt', '?': '_p', '!': '_bang', - '+': '_plus', '*': '_star', '/': '_slash', '%': '_pct', '&': '_amp', - } - - def _cl_id(*parts: str) -> str: - normalized = [] - for p in parts: - normalized.append(''.join(_CL_CHAR_MAP.get(c, c) for c in p)) - return _make_id(*normalized) - - def add_node(nid: str, label: str, line: int, file_type: str = "code") -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": label, - "file_type": file_type, - "source_file": str_path, - "source_location": f"L{line}", - }) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0) -> None: - edges.append({ - "source": src, - "target": tgt, - "relation": relation, - "confidence": confidence, - "source_file": str_path, - "source_location": f"L{line}", - "weight": weight, - }) - - file_nid = _cl_id(stem) - add_node(file_nid, path.name, 1) - - def _first_sym(node) -> str | None: - """Get the first sym_lit text from a list_lit's children.""" - for child in node.children: - if child.type == "sym_lit": - return _text(child) - return None - - def _kwd_text(node) -> str: - """Extract the symbol name from a kwd_lit, stripping the leading colon.""" - t = _text(node) - return t.lstrip(":").lstrip("#:") - - def _handle_defpackage(node) -> None: - nonlocal current_package - children = node.children - pkg_name = None - for child in children: - if child.type == "kwd_lit" and pkg_name is None: - pkg_name = _kwd_text(child) - break - if child.type == "sym_lit" and _text(child) != "defpackage": - pkg_name = _text(child) - break - if not pkg_name: - return - current_package = pkg_name - pkg_nid = _cl_id(stem, pkg_name) - line = node.start_point[0] + 1 - add_node(pkg_nid, pkg_name, line) - add_edge(file_nid, pkg_nid, "contains", line) - - # Extract :use clauses as imports - for child in children: - if child.type == "list_lit": - first = _first_sym(child) - if first is None: - # Could be (:use ...) with kwd_lit - for gc in child.children: - if gc.type == "kwd_lit" and _kwd_text(gc) == "use": - for uc in child.children: - if uc.type == "kwd_lit" and uc != gc: - mod_name = _kwd_text(uc) - if mod_name != "use": - tgt_nid = _cl_id(mod_name) - add_edge(pkg_nid, tgt_nid, "imports", - child.start_point[0] + 1) - break - - def _handle_defclass(node) -> None: - children = node.children - # Find class name: second sym_lit after "defclass" - sym_lits = [c for c in children if c.type == "sym_lit"] - if len(sym_lits) < 2: - return - class_name = _text(sym_lits[1]) - line = node.start_point[0] + 1 - class_nid = _cl_id(stem, class_name) - add_node(class_nid, class_name, line) - - parent_nid = file_nid - if current_package: - pkg_nid = _cl_id(stem, current_package) - if pkg_nid in seen_ids: - parent_nid = pkg_nid - add_edge(parent_nid, class_nid, "contains", line) - - # Superclasses (third element, a list) - list_lits = [c for c in children if c.type == "list_lit"] - if list_lits: - superclass_list = list_lits[0] - for sc in superclass_list.children: - if sc.type == "sym_lit": - sc_name = _text(sc) - sc_nid = _cl_id(stem, sc_name) - add_edge(class_nid, sc_nid, "inherits", - superclass_list.start_point[0] + 1) - - def _handle_defun_node(node) -> None: - """Handle a defun AST node (covers defun, defmethod, defgeneric, defmacro).""" - header = None - for child in node.children: - if child.type == "defun_header": - header = child - break - if not header: - return - - # Determine keyword type - keyword_type = "defun" - func_name = None - for child in header.children: - if child.type == "defun_keyword": - for kc in child.children: - if kc.type in ("defun", "defmethod", "defgeneric", "defmacro"): - keyword_type = kc.type - break - if child.type == "sym_lit" and func_name is None: - func_name = _text(child) - - if not func_name: - return - - line = node.start_point[0] + 1 - func_nid = _cl_id(stem, func_name) - - if keyword_type == "defmethod": - label = f".{func_name}()" - elif keyword_type == "defmacro": - label = f"{func_name} (macro)" - elif keyword_type == "defgeneric": - label = f"{func_name} (generic)" - else: - label = f"{func_name}()" - - add_node(func_nid, label, line) - - parent_nid = file_nid - if current_package: - pkg_nid = _cl_id(stem, current_package) - if pkg_nid in seen_ids: - parent_nid = pkg_nid - if keyword_type == "defmethod": - add_edge(parent_nid, func_nid, "method", line) - else: - add_edge(parent_nid, func_nid, "contains", line) - - # Method specializers: (defmethod name ((param class) ...)) - if keyword_type == "defmethod": - for child in header.children: - if child.type == "list_lit": - # This is the parameter list - for param in child.children: - if param.type == "list_lit": - syms = [c for c in param.children if c.type == "sym_lit"] - if len(syms) >= 2: - specializer_name = _text(syms[1]) - spec_nid = _cl_id(stem, specializer_name) - add_edge(func_nid, spec_nid, "specializes", - param.start_point[0] + 1) - break - - # Docstring - for child in node.children: - if child.type == "str_lit": - doc_text = _text(child).strip('"') - if doc_text: - doc_nid = _cl_id(func_nid, "rationale") - add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") - add_edge(doc_nid, func_nid, "rationale_for", - child.start_point[0] + 1) - break - - # Collect body for call extraction - body_nodes = [c for c in node.children - if c.type not in ("(", ")", "defun_header", "str_lit")] - if body_nodes: - function_bodies.append((func_nid, body_nodes)) - - def _extract_def_name(node, def_keyword: str) -> str | None: - """Get the name being defined. May be a sym_lit or a list_lit whose - first sym_lit is the name (e.g. (defstruct (foo :conc-name "BAR-") ...)).""" - seen_keyword = False - for child in node.children: - if not seen_keyword: - if child.type == "sym_lit" and _text(child).lower() == def_keyword: - seen_keyword = True - continue - if child.type == "sym_lit": - return _text(child) - if child.type == "list_lit": - for gc in child.children: - if gc.type == "sym_lit": - return _text(gc) - return None - return None - - def _collect_def_body(node, def_keyword: str) -> list: - """Collect body forms from (DEFKEYWORD name (params) [doc] body...). - Skips the def keyword, the name, the params list (if present), and - a leading docstring str_lit.""" - children = [c for c in node.children if c.type not in ("(", ")")] - idx = 0 - # Skip def keyword - if idx < len(children) and children[idx].type == "sym_lit" \ - and _text(children[idx]).lower() == def_keyword: - idx += 1 - # Skip name (sym_lit or list_lit) - if idx < len(children) and children[idx].type in ("sym_lit", "list_lit"): - idx += 1 - # Skip params list (the next list_lit, if any) - if idx < len(children) and children[idx].type == "list_lit": - idx += 1 - # Skip leading docstring - if idx < len(children) and children[idx].type == "str_lit": - idx += 1 - return children[idx:] - - def _handle_def_form(node, def_keyword: str) -> None: - """Handle a generic (DEFKEYWORD name ...) form. Covers standard CL - definers (defstruct, deftype, defvar, define-condition, etc.) and - custom def-prefixed macros (definline, definline-maybe, etc.).""" - name = _extract_def_name(node, def_keyword) - if not name: - return - line = node.start_point[0] + 1 - nid = _cl_id(stem, name) - - kw = def_keyword.lower() - if kw in _CL_DATA_DEFINERS: - label = name - is_callable = False - elif kw in _CL_MACRO_DEFINERS or "macro" in kw: - label = f"{name} (macro)" - is_callable = True - else: - # Custom def-prefixed: assume function-like - label = f"{name}()" - is_callable = True - - add_node(nid, label, line) - - parent_nid = file_nid - if current_package: - pkg_nid = _cl_id(stem, current_package) - if pkg_nid in seen_ids: - parent_nid = pkg_nid - add_edge(parent_nid, nid, "contains", line) - - # Docstring. Skip for defvar/defparameter/defconstant — their second - # argument is a value (possibly a string literal) which would be - # wrongly captured as a docstring. - if kw not in _CL_VALUE_DEFINERS: - for child in node.children: - if child.type == "str_lit": - doc_text = _text(child).strip('"') - if doc_text: - doc_nid = _cl_id(nid, "rationale") - add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") - add_edge(doc_nid, nid, "rationale_for", - child.start_point[0] + 1) - break - - if is_callable: - body_nodes = _collect_def_body(node, def_keyword) - if body_nodes: - function_bodies.append((nid, body_nodes)) - - def _is_def_prefixed(sym: str) -> bool: - """Heuristic: does this symbol look like a definition macro?""" - if sym in _CL_NOT_DEFINERS: - return False - return sym.startswith("def") or sym.startswith("define-") - - def _process_form(top) -> bool: - """Dispatch on a single list_lit form. Returns True if it was - recognized as a definition or package directive (caller must NOT - recurse into it). Returns False if unrecognized, so the caller can - recurse to find defs nested inside wrapper macros like - (optimizing ...), (eval-when ...), (progn ...), etc.""" - nonlocal current_package - - # Check for defun node type inside list_lit - for child in top.children: - if child.type == "defun": - _handle_defun_node(child) - return True - - first = _first_sym(top) - if not first: - return False - first_lower = first.lower() - - if first_lower == "defpackage": - _handle_defpackage(top) - return True - if first_lower == "in-package": - for child in top.children: - if child.type == "kwd_lit": - current_package = _kwd_text(child) - break - if child.type == "sym_lit" and _text(child) != "in-package": - current_package = _text(child) - break - return True - if first_lower == "defclass": - _handle_defclass(top) - return True - if first_lower in ("require", "ql:quickload"): - for child in top.children: - if child.type in ("kwd_lit", "str_lit"): - mod_name = _kwd_text(child) if child.type == "kwd_lit" else _text(child).strip('"') - if mod_name: - tgt_nid = _cl_id(mod_name) - add_edge(file_nid, tgt_nid, "imports", - top.start_point[0] + 1) - break - return True - if first_lower in _CL_DATA_DEFINERS or first_lower in _CL_MACRO_DEFINERS: - _handle_def_form(top, first_lower) - return True - if _is_def_prefixed(first_lower): - # Custom definer (definline, definline-maybe, defcomponent, etc.) - _handle_def_form(top, first_lower) - return True - - return False - - def _walk_forms(parent) -> None: - """Walk list_lit children of `parent`, processing each. If a form - isn't recognized, recurse into it — many CL codebases wrap - definitions in macros like (optimizing ...), (eval-when ...), or - (progn ...) that aren't themselves definers but contain defs. - Also descends into reader conditionals (#+feature / #-feature), - which wrap their guarded form in an include_reader_macro node.""" - for top in parent.children: - if top.type == "list_lit": - if not _process_form(top): - _walk_forms(top) - elif top.type == "include_reader_macro": - _walk_forms(top) - - _walk_forms(root) - - # Call extraction pass - label_to_nid: dict[str, str] = {} - for n in nodes: - raw = n["label"] - normalised = raw.replace(" (macro)", "").replace(" (generic)", "").strip("()").lstrip(".") - label_to_nid[normalised.lower()] = n["id"] - - seen_call_pairs: set[tuple[str, str]] = set() - - def walk_calls(node, caller_nid: str) -> None: - if node.type == "defun": - return - if node.type == "list_lit": - callee = _first_sym(node) - if callee and callee.lower() not in _CL_SPECIAL_FORMS: - tgt_nid = label_to_nid.get(callee.lower()) - if tgt_nid and tgt_nid != caller_nid: - pair = (caller_nid, tgt_nid) - if pair not in seen_call_pairs: - seen_call_pairs.add(pair) - add_edge(caller_nid, tgt_nid, "calls", - node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0) - for child in node.children: - walk_calls(child, caller_nid) - - for caller_nid, body_nodes in function_bodies: - for body_node in body_nodes: - walk_calls(body_node, caller_nid) - - clean_edges = [e for e in edges if e["source"] in seen_ids and - (e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))] - return {"nodes": nodes, "edges": clean_edges} - - # ── Main extract and collect_files ──────────────────────────────────────────── diff --git a/graphify/extractors/__init__.py b/graphify/extractors/__init__.py index ada517094..68ff3340c 100644 --- a/graphify/extractors/__init__.py +++ b/graphify/extractors/__init__.py @@ -13,6 +13,7 @@ from graphify.extractors.apex import extract_apex from graphify.extractors.bash import extract_bash from graphify.extractors.blade import extract_blade +from graphify.extractors.commonlisp import extract_commonlisp from graphify.extractors.dart import extract_dart from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm from graphify.extractors.elixir import extract_elixir @@ -37,6 +38,7 @@ "apex": extract_apex, "bash": extract_bash, "blade": extract_blade, + "commonlisp": extract_commonlisp, "dart": extract_dart, "delphi_form": extract_delphi_form, "dm": extract_dm, diff --git a/graphify/extractors/commonlisp.py b/graphify/extractors/commonlisp.py new file mode 100644 index 000000000..150880903 --- /dev/null +++ b/graphify/extractors/commonlisp.py @@ -0,0 +1,492 @@ +"""Common Lisp extractor for .lisp/.cl/.lsp/.asd, backed by tree-sitter-commonlisp.""" +from __future__ import annotations + +import warnings +from pathlib import Path + +from graphify.extractors.base import _make_id + + +# Standard CL definer forms that introduce data/type/variable bindings +# (not callable, no body to walk for calls) +_CL_DATA_DEFINERS = frozenset({ + "defstruct", "deftype", "define-condition", + "defvar", "defparameter", "defconstant", + "define-symbol-macro", +}) + +# Subset of data definers that take a VALUE as their second argument, which +# may itself be a string literal. For these, we must not mistake the value +# for a docstring. +_CL_VALUE_DEFINERS = frozenset({"defvar", "defparameter", "defconstant"}) + +# Standard CL macro-style definers +_CL_MACRO_DEFINERS = frozenset({ + "define-modify-macro", "define-compiler-macro", + "define-setf-expander", "defsetf", +}) + +# Symbols that start with "def" but are NOT definitions (denylist for the +# def-prefix heuristic that catches custom definers like definline-maybe) +_CL_NOT_DEFINERS = frozenset({ + "default", "default-value", "defaults", "define", + "defer", "deferred", "deflate", "deftest", +}) + +# CL special forms / macros that should not count as "calls" in the graph +_CL_SPECIAL_FORMS = frozenset({ + "let", "let*", "flet", "labels", "macrolet", + "if", "when", "unless", "cond", "case", "ecase", "typecase", "etypecase", + "progn", "prog1", "prog2", "block", "return-from", "tagbody", "go", + "lambda", "function", "funcall", "apply", + "setf", "setq", "psetf", "psetq", "incf", "decf", "push", "pop", + "loop", "do", "do*", "dolist", "dotimes", "map", "mapcar", "mapc", + "format", "print", "princ", "prin1", "write", "terpri", + "and", "or", "not", "null", + "values", "multiple-value-bind", "multiple-value-setq", + "declare", "the", "locally", "eval-when", + "quote", "backquote", + "error", "signal", "warn", "assert", "check-type", + "handler-case", "handler-bind", "restart-case", "ignore-errors", + "unwind-protect", "catch", "throw", + "with-open-file", "with-output-to-string", "with-input-from-string", + "with-slots", "with-accessors", + "defvar", "defparameter", "defconstant", + "in-package", "require", "provide", "use-package", + "t", "nil", +}) + + +def extract_commonlisp(path: Path) -> dict: + """Extract packages, classes, functions, methods, macros, and calls from a Common Lisp file.""" + try: + import tree_sitter_commonlisp as tscl + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree-sitter-commonlisp not installed"} + + try: + # tree-sitter-commonlisp 0.4.1 (latest) ships the old binding that returns + # an int pointer from language(); tree_sitter.Language only accepts that + # int via a deprecated path. Silence that one warning at the call site + # until the grammar ships a PyCapsule binding. + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="int argument support is deprecated", + category=DeprecationWarning, + ) + language = Language(tscl.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = path.stem + str_path = str(path) + nodes: list[dict] = [] + edges: list[dict] = [] + seen_ids: set[str] = set() + function_bodies: list[tuple[str, object]] = [] # (nid, body_nodes) + current_package: str | None = None + + def _text(node) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + + # CL symbols can contain operator chars like = < > ? ! + * / that the + # generic _make_id strips. Map them to readable suffixes so upi=, upi<, + # upi> become distinct ids. + _CL_CHAR_MAP = { + '=': '_eq', '<': '_lt', '>': '_gt', '?': '_p', '!': '_bang', + '+': '_plus', '*': '_star', '/': '_slash', '%': '_pct', '&': '_amp', + } + + def _cl_id(*parts: str) -> str: + normalized = [] + for p in parts: + normalized.append(''.join(_CL_CHAR_MAP.get(c, c) for c in p)) + return _make_id(*normalized) + + def add_node(nid: str, label: str, line: int, file_type: str = "code") -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({ + "id": nid, + "label": label, + "file_type": file_type, + "source_file": str_path, + "source_location": f"L{line}", + }) + + def add_edge(src: str, tgt: str, relation: str, line: int, + confidence: str = "EXTRACTED", weight: float = 1.0) -> None: + edges.append({ + "source": src, + "target": tgt, + "relation": relation, + "confidence": confidence, + "source_file": str_path, + "source_location": f"L{line}", + "weight": weight, + }) + + file_nid = _cl_id(stem) + add_node(file_nid, path.name, 1) + + def _first_sym(node) -> str | None: + """Get the first sym_lit text from a list_lit's children.""" + for child in node.children: + if child.type == "sym_lit": + return _text(child) + return None + + def _kwd_text(node) -> str: + """Extract the symbol name from a kwd_lit, stripping the leading colon.""" + t = _text(node) + return t.lstrip(":").lstrip("#:") + + def _handle_defpackage(node) -> None: + nonlocal current_package + children = node.children + pkg_name = None + for child in children: + if child.type == "kwd_lit" and pkg_name is None: + pkg_name = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "defpackage": + pkg_name = _text(child) + break + if not pkg_name: + return + current_package = pkg_name + pkg_nid = _cl_id(stem, pkg_name) + line = node.start_point[0] + 1 + add_node(pkg_nid, pkg_name, line) + add_edge(file_nid, pkg_nid, "contains", line) + + # Extract :use clauses as imports + for child in children: + if child.type == "list_lit": + first = _first_sym(child) + if first is None: + # Could be (:use ...) with kwd_lit + for gc in child.children: + if gc.type == "kwd_lit" and _kwd_text(gc) == "use": + for uc in child.children: + if uc.type == "kwd_lit" and uc != gc: + mod_name = _kwd_text(uc) + if mod_name != "use": + tgt_nid = _cl_id(mod_name) + add_edge(pkg_nid, tgt_nid, "imports", + child.start_point[0] + 1) + break + + def _handle_defclass(node) -> None: + children = node.children + # Find class name: second sym_lit after "defclass" + sym_lits = [c for c in children if c.type == "sym_lit"] + if len(sym_lits) < 2: + return + class_name = _text(sym_lits[1]) + line = node.start_point[0] + 1 + class_nid = _cl_id(stem, class_name) + add_node(class_nid, class_name, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, class_nid, "contains", line) + + # Superclasses (third element, a list) + list_lits = [c for c in children if c.type == "list_lit"] + if list_lits: + superclass_list = list_lits[0] + for sc in superclass_list.children: + if sc.type == "sym_lit": + sc_name = _text(sc) + sc_nid = _cl_id(stem, sc_name) + add_edge(class_nid, sc_nid, "inherits", + superclass_list.start_point[0] + 1) + + def _handle_defun_node(node) -> None: + """Handle a defun AST node (covers defun, defmethod, defgeneric, defmacro).""" + header = None + for child in node.children: + if child.type == "defun_header": + header = child + break + if not header: + return + + # Determine keyword type + keyword_type = "defun" + func_name = None + for child in header.children: + if child.type == "defun_keyword": + for kc in child.children: + if kc.type in ("defun", "defmethod", "defgeneric", "defmacro"): + keyword_type = kc.type + break + if child.type == "sym_lit" and func_name is None: + func_name = _text(child) + + if not func_name: + return + + line = node.start_point[0] + 1 + func_nid = _cl_id(stem, func_name) + + if keyword_type == "defmethod": + label = f".{func_name}()" + elif keyword_type == "defmacro": + label = f"{func_name} (macro)" + elif keyword_type == "defgeneric": + label = f"{func_name} (generic)" + else: + label = f"{func_name}()" + + add_node(func_nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + if keyword_type == "defmethod": + add_edge(parent_nid, func_nid, "method", line) + else: + add_edge(parent_nid, func_nid, "contains", line) + + # Method specializers: (defmethod name ((param class) ...)) + if keyword_type == "defmethod": + for child in header.children: + if child.type == "list_lit": + # This is the parameter list + for param in child.children: + if param.type == "list_lit": + syms = [c for c in param.children if c.type == "sym_lit"] + if len(syms) >= 2: + specializer_name = _text(syms[1]) + spec_nid = _cl_id(stem, specializer_name) + add_edge(func_nid, spec_nid, "specializes", + param.start_point[0] + 1) + break + + # Docstring + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(func_nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") + add_edge(doc_nid, func_nid, "rationale_for", + child.start_point[0] + 1) + break + + # Collect body for call extraction + body_nodes = [c for c in node.children + if c.type not in ("(", ")", "defun_header", "str_lit")] + if body_nodes: + function_bodies.append((func_nid, body_nodes)) + + def _extract_def_name(node, def_keyword: str) -> str | None: + """Get the name being defined. May be a sym_lit or a list_lit whose + first sym_lit is the name (e.g. (defstruct (foo :conc-name "BAR-") ...)).""" + seen_keyword = False + for child in node.children: + if not seen_keyword: + if child.type == "sym_lit" and _text(child).lower() == def_keyword: + seen_keyword = True + continue + if child.type == "sym_lit": + return _text(child) + if child.type == "list_lit": + for gc in child.children: + if gc.type == "sym_lit": + return _text(gc) + return None + return None + + def _collect_def_body(node, def_keyword: str) -> list: + """Collect body forms from (DEFKEYWORD name (params) [doc] body...). + Skips the def keyword, the name, the params list (if present), and + a leading docstring str_lit.""" + children = [c for c in node.children if c.type not in ("(", ")")] + idx = 0 + # Skip def keyword + if idx < len(children) and children[idx].type == "sym_lit" \ + and _text(children[idx]).lower() == def_keyword: + idx += 1 + # Skip name (sym_lit or list_lit) + if idx < len(children) and children[idx].type in ("sym_lit", "list_lit"): + idx += 1 + # Skip params list (the next list_lit, if any) + if idx < len(children) and children[idx].type == "list_lit": + idx += 1 + # Skip leading docstring + if idx < len(children) and children[idx].type == "str_lit": + idx += 1 + return children[idx:] + + def _handle_def_form(node, def_keyword: str) -> None: + """Handle a generic (DEFKEYWORD name ...) form. Covers standard CL + definers (defstruct, deftype, defvar, define-condition, etc.) and + custom def-prefixed macros (definline, definline-maybe, etc.).""" + name = _extract_def_name(node, def_keyword) + if not name: + return + line = node.start_point[0] + 1 + nid = _cl_id(stem, name) + + kw = def_keyword.lower() + if kw in _CL_DATA_DEFINERS: + label = name + is_callable = False + elif kw in _CL_MACRO_DEFINERS or "macro" in kw: + label = f"{name} (macro)" + is_callable = True + else: + # Custom def-prefixed: assume function-like + label = f"{name}()" + is_callable = True + + add_node(nid, label, line) + + parent_nid = file_nid + if current_package: + pkg_nid = _cl_id(stem, current_package) + if pkg_nid in seen_ids: + parent_nid = pkg_nid + add_edge(parent_nid, nid, "contains", line) + + # Docstring. Skip for defvar/defparameter/defconstant — their second + # argument is a value (possibly a string literal) which would be + # wrongly captured as a docstring. + if kw not in _CL_VALUE_DEFINERS: + for child in node.children: + if child.type == "str_lit": + doc_text = _text(child).strip('"') + if doc_text: + doc_nid = _cl_id(nid, "rationale") + add_node(doc_nid, doc_text[:120], child.start_point[0] + 1, file_type="rationale") + add_edge(doc_nid, nid, "rationale_for", + child.start_point[0] + 1) + break + + if is_callable: + body_nodes = _collect_def_body(node, def_keyword) + if body_nodes: + function_bodies.append((nid, body_nodes)) + + def _is_def_prefixed(sym: str) -> bool: + """Heuristic: does this symbol look like a definition macro?""" + if sym in _CL_NOT_DEFINERS: + return False + return sym.startswith("def") or sym.startswith("define-") + + def _process_form(top) -> bool: + """Dispatch on a single list_lit form. Returns True if it was + recognized as a definition or package directive (caller must NOT + recurse into it). Returns False if unrecognized, so the caller can + recurse to find defs nested inside wrapper macros like + (optimizing ...), (eval-when ...), (progn ...), etc.""" + nonlocal current_package + + # Check for defun node type inside list_lit + for child in top.children: + if child.type == "defun": + _handle_defun_node(child) + return True + + first = _first_sym(top) + if not first: + return False + first_lower = first.lower() + + if first_lower == "defpackage": + _handle_defpackage(top) + return True + if first_lower == "in-package": + for child in top.children: + if child.type == "kwd_lit": + current_package = _kwd_text(child) + break + if child.type == "sym_lit" and _text(child) != "in-package": + current_package = _text(child) + break + return True + if first_lower == "defclass": + _handle_defclass(top) + return True + if first_lower in ("require", "ql:quickload"): + for child in top.children: + if child.type in ("kwd_lit", "str_lit"): + mod_name = _kwd_text(child) if child.type == "kwd_lit" else _text(child).strip('"') + if mod_name: + tgt_nid = _cl_id(mod_name) + add_edge(file_nid, tgt_nid, "imports", + top.start_point[0] + 1) + break + return True + if first_lower in _CL_DATA_DEFINERS or first_lower in _CL_MACRO_DEFINERS: + _handle_def_form(top, first_lower) + return True + if _is_def_prefixed(first_lower): + # Custom definer (definline, definline-maybe, defcomponent, etc.) + _handle_def_form(top, first_lower) + return True + + return False + + def _walk_forms(parent) -> None: + """Walk list_lit children of `parent`, processing each. If a form + isn't recognized, recurse into it — many CL codebases wrap + definitions in macros like (optimizing ...), (eval-when ...), or + (progn ...) that aren't themselves definers but contain defs. + Also descends into reader conditionals (#+feature / #-feature), + which wrap their guarded form in an include_reader_macro node.""" + for top in parent.children: + if top.type == "list_lit": + if not _process_form(top): + _walk_forms(top) + elif top.type == "include_reader_macro": + _walk_forms(top) + + _walk_forms(root) + + # Call extraction pass + label_to_nid: dict[str, str] = {} + for n in nodes: + raw = n["label"] + normalised = raw.replace(" (macro)", "").replace(" (generic)", "").strip("()").lstrip(".") + label_to_nid[normalised.lower()] = n["id"] + + seen_call_pairs: set[tuple[str, str]] = set() + + def walk_calls(node, caller_nid: str) -> None: + if node.type == "defun": + return + if node.type == "list_lit": + callee = _first_sym(node) + if callee and callee.lower() not in _CL_SPECIAL_FORMS: + tgt_nid = label_to_nid.get(callee.lower()) + if tgt_nid and tgt_nid != caller_nid: + pair = (caller_nid, tgt_nid) + if pair not in seen_call_pairs: + seen_call_pairs.add(pair) + add_edge(caller_nid, tgt_nid, "calls", + node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0) + for child in node.children: + walk_calls(child, caller_nid) + + for caller_nid, body_nodes in function_bodies: + for body_node in body_nodes: + walk_calls(body_node, caller_nid) + + clean_edges = [e for e in edges if e["source"] in seen_ids and + (e["target"] in seen_ids or e["relation"] in ("imports", "imports_from"))] + return {"nodes": nodes, "edges": clean_edges} From 704368b97fd1f5b3cd35fbca3712029d90b644ad Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Tue, 28 Jul 2026 11:17:39 -0400 Subject: [PATCH 10/11] fix(extract): drop a duplicated tree-sitter version guard The Common Lisp branch carried a second copy of the "Main extract and collect_files" banner and _check_tree_sitter_version. Python let it pass because the two definitions are identical, so nothing failed, but the file should define it once. --- graphify/extract.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 392527f89..6b9dde503 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4037,29 +4037,6 @@ def add_existing_edge(edge: dict) -> None: # block defined in the corpus (count.index, each.key, self.*, path.module, ...). -# ── Main extract and collect_files ──────────────────────────────────────────── - - -def _check_tree_sitter_version() -> None: - """Raise a clear error if tree-sitter is too old for the new Language API.""" - try: - from tree_sitter import LANGUAGE_VERSION - except ImportError: - raise ImportError( - "tree-sitter is not installed. Run: pip install 'tree-sitter>=0.23.0'" - ) - # Language API v2 starts at LANGUAGE_VERSION 14 - if LANGUAGE_VERSION < 14: - import tree_sitter as _ts - raise RuntimeError( - f"tree-sitter {getattr(_ts, '__version__', 'unknown')} is too old. " - f"graphify requires tree-sitter >= 0.23.0 (Language API v2). " - f"Run: pip install --upgrade tree-sitter" - ) - - - - _DISPATCH: dict[str, Any] = { ".py": extract_python, ".js": extract_js, From 74aea31cb4986c96566102332af836e6b8871725 Mon Sep 17 00:00:00 2001 From: Brian O'Reilly Date: Tue, 28 Jul 2026 11:25:00 -0400 Subject: [PATCH 11/11] test: skip the Common Lisp tests when its grammar is absent The grammar is an optional extra, so a default install does not have it, and these tests called the extractor unguarded: absent grammar meant a hard failure rather than a skip. Every other optional-grammar language already guards this way. Matters for a fresh checkout and for any lane that installs from the lock file, where the extra is not present at all. --- tests/test_languages.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_languages.py b/tests/test_languages.py index 79430bb6e..d2e4005b3 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -22,6 +22,10 @@ _ilu.find_spec("tree_sitter_dm") is None, reason="tree-sitter-dm not installed (optional [dm] extra)", ) +_needs_commonlisp = pytest.mark.skipif( + _ilu.find_spec("tree_sitter_commonlisp") is None, + reason="tree-sitter-commonlisp not installed (optional [commonlisp] extra)", +) def _labels(r): @@ -3002,16 +3006,19 @@ def test_decldef_merge_does_not_merge_same_name_same_dir_distinct_files(): # ── Common Lisp ────────────────────────────────────────────────────────────── +@_needs_commonlisp def test_cl_finds_package(): r = extract_commonlisp(FIXTURES / "sample.lisp") assert "error" not in r assert "http-server" in _labels(r) +@_needs_commonlisp def test_cl_finds_class(): r = extract_commonlisp(FIXTURES / "sample.lisp") assert "server" in _labels(r) assert "ssl-server" in _labels(r) +@_needs_commonlisp def test_cl_finds_defun(): r = extract_commonlisp(FIXTURES / "sample.lisp") labels = _labels(r) @@ -3019,14 +3026,17 @@ def test_cl_finds_defun(): assert any("start" in l for l in labels) assert any("stop" in l for l in labels) +@_needs_commonlisp def test_cl_finds_generic(): r = extract_commonlisp(FIXTURES / "sample.lisp") assert any("process-request" in l for l in _labels(r)) +@_needs_commonlisp def test_cl_finds_macro(): r = extract_commonlisp(FIXTURES / "sample.lisp") assert any("with-server" in l and "macro" in l for l in _labels(r)) +@_needs_commonlisp def test_cl_emits_calls(): r = extract_commonlisp(FIXTURES / "sample.lisp") calls = _calls(r) @@ -3036,12 +3046,14 @@ def test_cl_emits_calls(): assert any("with-server" in src and "make-server" in tgt for src, tgt in calls) assert any("with-server" in src and "start" in tgt for src, tgt in calls) +@_needs_commonlisp def test_cl_calls_are_extracted(): r = extract_commonlisp(FIXTURES / "sample.lisp") for e in r["edges"]: if e["relation"] == "calls": assert e["confidence"] == "EXTRACTED" +@_needs_commonlisp def test_cl_no_dangling_edges(): r = extract_commonlisp(FIXTURES / "sample.lisp") node_ids = {n["id"] for n in r["nodes"]} @@ -3049,6 +3061,7 @@ def test_cl_no_dangling_edges(): if e["relation"] in ("contains", "method", "calls"): assert e["source"] in node_ids +@_needs_commonlisp def test_cl_docstrings(): r = extract_commonlisp(FIXTURES / "sample.lisp") rationale_edges = [e for e in r["edges"] if e["relation"] == "rationale_for"] @@ -3056,6 +3069,7 @@ def test_cl_docstrings(): labels = _labels(r) assert any("Process an incoming" in l for l in labels) +@_needs_commonlisp def test_cl_method_specializers(): r = extract_commonlisp(FIXTURES / "sample.lisp") spec_edges = [e for e in r["edges"] if e["relation"] == "specializes"] @@ -3063,12 +3077,14 @@ def test_cl_method_specializers(): # process-request specializes on server assert any("process_request" in e["source"] and "server" in e["target"] for e in spec_edges) +@_needs_commonlisp def test_cl_inherits(): r = extract_commonlisp(FIXTURES / "sample.lisp") inherit_edges = [e for e in r["edges"] if e["relation"] == "inherits"] assert len(inherit_edges) >= 1 assert any("ssl_server" in e["source"] and "server" in e["target"] for e in inherit_edges) +@_needs_commonlisp def test_cl_imports(): r = extract_commonlisp(FIXTURES / "sample.lisp") import_edges = [e for e in r["edges"] if e["relation"] == "imports"] @@ -3076,10 +3092,12 @@ def test_cl_imports(): assert "cl" in targets assert "alexandria" in targets +@_needs_commonlisp def test_cl_finds_deftype(): r = extract_commonlisp(FIXTURES / "sample.lisp") assert "port-number" in _labels(r) +@_needs_commonlisp def test_cl_finds_defstruct(): r = extract_commonlisp(FIXTURES / "sample.lisp") labels = _labels(r) @@ -3087,6 +3105,7 @@ def test_cl_finds_defstruct(): # defstruct with options form: (defstruct (name ...) ...) assert "connection" in labels +@_needs_commonlisp def test_cl_finds_defvar_defparameter_defconstant(): r = extract_commonlisp(FIXTURES / "sample.lisp") labels = _labels(r) @@ -3094,10 +3113,12 @@ def test_cl_finds_defvar_defparameter_defconstant(): assert "*default-port*" in labels assert "+max-headers+" in labels +@_needs_commonlisp def test_cl_finds_define_condition(): r = extract_commonlisp(FIXTURES / "sample.lisp") assert "server-error" in _labels(r) +@_needs_commonlisp def test_cl_finds_custom_definer(): """The def-prefix heuristic should catch definline / definline-maybe.""" r = extract_commonlisp(FIXTURES / "sample.lisp") @@ -3106,6 +3127,7 @@ def test_cl_finds_custom_definer(): assert "header=()" in labels assert "header<()" in labels +@_needs_commonlisp def test_cl_custom_definer_in_call_graph(): """Functions defined via custom definers should appear in the call graph.""" r = extract_commonlisp(FIXTURES / "sample.lisp") @@ -3114,6 +3136,7 @@ def test_cl_custom_definer_in_call_graph(): assert any("compare-headers" in src and "header=" in tgt for src, tgt in calls) assert any("compare-headers" in src and "header<" in tgt for src, tgt in calls) +@_needs_commonlisp def test_cl_operator_names_disambiguated(): """upi=, upi<, upi> must produce distinct ids (operator chars matter).""" r = extract_commonlisp(FIXTURES / "sample.lisp") @@ -3124,6 +3147,7 @@ def test_cl_operator_names_disambiguated(): assert len(lt_ids) == 1 assert eq_ids[0] != lt_ids[0] +@_needs_commonlisp def test_cl_default_value_not_treated_as_definition(): """The def-prefix heuristic must not match denylisted symbols.""" import tempfile @@ -3140,6 +3164,7 @@ def test_cl_default_value_not_treated_as_definition(): finally: path.unlink() +@_needs_commonlisp def test_cl_defs_inside_wrapper_macro(): """Definitions nested inside wrapper macros like (optimizing ...) or (eval-when ...) must be extracted. Many CL codebases wrap hot-path @@ -3167,6 +3192,7 @@ def test_cl_defs_inside_wrapper_macro(): finally: path.unlink() +@_needs_commonlisp def test_cl_defs_inside_reader_conditional(): """#+feature / #-feature reader conditionals wrap their guarded form in an include_reader_macro AST node, which the walker must descend @@ -3189,6 +3215,7 @@ def test_cl_defs_inside_reader_conditional(): finally: path.unlink() +@_needs_commonlisp def test_cl_defparameter_string_value_not_docstring(): """For defvar/defparameter/defconstant, a string literal in the VALUE position must not be wrongly captured as a docstring node."""