from __future__ import annotations import os from collections import defaultdict from functools import cmp_to_key from typing import Callable from mypy.build import State from mypy.messages import format_type from mypy.modulefinder import PYTHON_EXTENSIONS from mypy.nodes import ( LDEF, Decorator, Expression, FuncBase, MemberExpr, MypyFile, Node, OverloadedFuncDef, RefExpr, SymbolNode, TypeInfo, Var, ) from mypy.server.update import FineGrainedBuildManager from mypy.traverser import ExtendedTraverserVisitor from mypy.typeops import tuple_fallback from mypy.types import ( FunctionLike, Instance, LiteralType, ProperType, TupleType, TypedDictType, TypeVarType, UnionType, get_proper_type, ) from mypy.typevars import fill_typevars_with_any def node_starts_after(o: Node, line: int, column: int) -> bool: return o.line > line or o.line == line and o.column > column def node_ends_before(o: Node, line: int, column: int) -> bool: # Unfortunately, end positions for some statements are a mess, # e.g. overloaded functions, so we return False when we don't know. if o.end_line is not None and o.end_column is not None: if o.end_line < line or o.end_line == line and o.end_column < column: return True return False def expr_span(expr: Expression) -> str: """Format expression span as in mypy error messages.""" return f"{expr.line}:{expr.column + 1}:{expr.end_line}:{expr.end_column}" def get_instance_fallback(typ: ProperType) -> list[Instance]: """Returns the Instance fallback for this type if one exists or None.""" if isinstance(typ, Instance): return [typ] elif isinstance(typ, TupleType): return [tuple_fallback(typ)] elif isinstance(typ, TypedDictType): return [typ.fallback] elif isinstance(typ, FunctionLike): return [typ.fallback] elif isinstance(typ, LiteralType): return [typ.fallback] elif isinstance(typ, TypeVarType): if typ.values: res = [] for t in typ.values: res.extend(get_instance_fallback(get_proper_type(t))) return res return get_instance_fallback(get_proper_type(typ.upper_bound)) elif isinstance(typ, UnionType): res = [] for t in typ.items: res.extend(get_instance_fallback(get_proper_type(t))) return res return [] def find_node(name: str, info: TypeInfo) -> Var | FuncBase | None: """Find the node defining member 'name' in given TypeInfo.""" # TODO: this code shares some logic with checkmember.py method = info.get_method(name) if method: if isinstance(method, Decorator): return method.var if method.is_property: assert isinstance(method, OverloadedFuncDef) dec = method.items[0] assert isinstance(dec, Decorator) return dec.var return method else: # don't have such method, maybe variable? node = info.get(name) v = node.node if node else None if isinstance(v, Var): return v return None def find_module_by_fullname(fullname: str, modules: dict[str, State]) -> State | None: """Find module by a node fullname. This logic mimics the one we use in fixup, so should be good enough. """ head = fullname # Special case: a module symbol is considered to be defined in itself, not in enclosing # package, since this is what users want when clicking go to definition on a module. if head in modules: return modules[head] while True: if "." not in head: return None head, tail = head.rsplit(".", maxsplit=1) mod = modules.get(head) if mod is not None: return mod class SearchVisitor(ExtendedTraverserVisitor): """Visitor looking for an expression whose span matches given one exactly.""" def __init__(self, line: int, column: int, end_line: int, end_column: int) -> None: self.line = line self.column = column self.end_line = end_line self.end_column = end_column self.result: Expression | None = None def visit(self, o: Node) -> bool: if node_starts_after(o, self.line, self.column): return False if node_ends_before(o, self.end_line, self.end_column): return False if ( o.line == self.line and o.end_line == self.end_line and o.column == self.column and o.end_column == self.end_column ): if isinstance(o, Expression): self.result = o return self.result is None def find_by_location( tree: MypyFile, line: int, column: int, end_line: int, end_column: int ) -> Expression | None: """Find an expression matching given span, or None if not found.""" if end_line < line: raise ValueError('"end_line" must not be before "line"') if end_line == line and end_column <= column: raise ValueError('"end_column" must be after "column"') visitor = SearchVisitor(line, column, end_line, end_column) tree.accept(visitor) return visitor.result class SearchAllVisitor(ExtendedTraverserVisitor): """Visitor looking for all expressions whose spans enclose given position.""" def __init__(self, line: int, column: int) -> None: self.line = line self.column = column self.result: list[Expression] = [] def visit(self, o: Node) -> bool: if node_starts_after(o, self.line, self.column): return False if node_ends_before(o, self.line, self.column): return False if isinstance(o, Expression): self.result.append(o) return True def find_all_by_location(tree: MypyFile, line: int, column: int) -> list[Expression]: """Find all expressions enclosing given position starting from innermost.""" visitor = SearchAllVisitor(line, column) tree.accept(visitor) return list(reversed(visitor.result)) class InspectionEngine: """Engine for locating and statically inspecting expressions.""" def __init__( self, fg_manager: FineGrainedBuildManager, *, verbosity: int = 0, limit: int = 0, include_span: bool = False, include_kind: bool = False, include_object_attrs: bool = False, union_attrs: bool = False, force_reload: bool = False, ) -> None: self.fg_manager = fg_manager self.verbosity = verbosity self.limit = limit self.include_span = include_span self.include_kind = include_kind self.include_object_attrs = include_object_attrs self.union_attrs = union_attrs self.force_reload = force_reload # Module for which inspection was requested. self.module: State | None = None def reload_module(self, state: State) -> None: """Reload given module while temporary exporting types.""" old = self.fg_manager.manager.options.export_types self.fg_manager.manager.options.export_types = True try: self.fg_manager.flush_cache() assert state.path is not None self.fg_manager.update([(state.id, state.path)], []) finally: self.fg_manager.manager.options.export_types = old def expr_type(self, expression: Expression) -> tuple[str, bool]: """Format type for an expression using current options. If type is known, second item returned is True. If type is not known, an error message is returned instead, and second item returned is False. """ expr_type = self.fg_manager.manager.all_types.get(expression) if expr_type is None: return self.missing_type(expression), False type_str = format_type( expr_type, self.fg_manager.manager.options, verbosity=self.verbosity ) return self.add_prefixes(type_str, expression), True def object_type(self) -> Instance: builtins = self.fg_manager.graph["builtins"].tree assert builtins is not None object_node = builtins.names["object"].node assert isinstance(object_node, TypeInfo) return Instance(object_node, []) def collect_attrs(self, instances: list[Instance]) -> dict[TypeInfo, list[str]]: """Collect attributes from all union/typevar variants.""" def item_attrs(attr_dict: dict[TypeInfo, list[str]]) -> set[str]: attrs = set() for base in attr_dict: attrs |= set(attr_dict[base]) return attrs def cmp_types(x: TypeInfo, y: TypeInfo) -> int: if x in y.mro: return 1 if y in x.mro: return -1 return 0 # First gather all attributes for every union variant. assert instances all_attrs = [] for instance in instances: attrs = {} mro = instance.type.mro if not self.include_object_attrs: mro = mro[:-1] for base in mro: attrs[base] = sorted(base.names) all_attrs.append(attrs) # Find attributes valid for all variants in a union or type variable. intersection = item_attrs(all_attrs[0]) for item in all_attrs[1:]: intersection &= item_attrs(item) # Combine attributes from all variants into a single dict while # also removing invalid attributes (unless using --union-attrs). combined_attrs = defaultdict(list) for item in all_attrs: for base in item: if base in combined_attrs: continue for name in item[base]: if self.union_attrs or name in intersection: combined_attrs[base].append(name) # Sort bases by MRO, unrelated will appear in the order they appeared as union variants. sorted_bases = sorted(combined_attrs.keys(), key=cmp_to_key(cmp_types)) result = {} for base in sorted_bases: if not combined_attrs[base]: # Skip bases where everytihng was filtered out. continue result[base] = combined_attrs[base] return result def _fill_from_dict( self, attrs_strs: list[str], attrs_dict: dict[TypeInfo, list[str]] ) -> None: for base in attrs_dict: cls_name = base.name if self.verbosity < 1 else base.fullname attrs = [f'"{attr}"' for attr in attrs_dict[base]] attrs_strs.append(f'"{cls_name}": [{", ".join(attrs)}]') def expr_attrs(self, expression: Expression) -> tuple[str, bool]: """Format attributes that are valid for a given expression. If expression type is not an Instance, try using fallback. Attributes are returned as a JSON (ordered by MRO) that maps base class name to list of attributes. Attributes may appear in multiple bases if overridden (we simply follow usual mypy logic for creating new Vars etc). """ expr_type = self.fg_manager.manager.all_types.get(expression) if expr_type is None: return self.missing_type(expression), False expr_type = get_proper_type(expr_type) instances = get_instance_fallback(expr_type) if not instances: # Everything is an object in Python. instances = [self.object_type()] attrs_dict = self.collect_attrs(instances) # Special case: modules have names apart from those from ModuleType. if isinstance(expression, RefExpr) and isinstance(expression.node, MypyFile): node = expression.node names = sorted(node.names) if "__builtins__" in names: # This is just to make tests stable. No one will really need this name. names.remove("__builtins__") mod_dict = {f'"<{node.fullname}>"': [f'"{name}"' for name in names]} else: mod_dict = {} # Special case: for class callables, prepend with the class attributes. # TODO: also handle cases when such callable appears in a union. if isinstance(expr_type, FunctionLike) and expr_type.is_type_obj(): template = fill_typevars_with_any(expr_type.type_object()) class_dict = self.collect_attrs(get_instance_fallback(template)) else: class_dict = {} # We don't use JSON dump to be sure keys order is always preserved. base_attrs = [] if mod_dict: for mod in mod_dict: base_attrs.append(f'{mod}: [{", ".join(mod_dict[mod])}]') self._fill_from_dict(base_attrs, class_dict) self._fill_from_dict(base_attrs, attrs_dict) return self.add_prefixes(f'{{{", ".join(base_attrs)}}}', expression), True def format_node(self, module: State, node: FuncBase | SymbolNode) -> str: return f"{module.path}:{node.line}:{node.column + 1}:{node.name}" def collect_nodes(self, expression: RefExpr) -> list[FuncBase | SymbolNode]: """Collect nodes that can be referred to by an expression. Note: it can be more than one for example in case of a union attribute. """ node: FuncBase | SymbolNode | None = expression.node nodes: list[FuncBase | SymbolNode] if node is None: # Tricky case: instance attribute if isinstance(expression, MemberExpr) and expression.kind is None: base_type = self.fg_manager.manager.all_types.get(expression.expr) if base_type is None: return [] # Now we use the base type to figure out where the attribute is defined. base_type = get_proper_type(base_type) instances = get_instance_fallback(base_type) nodes = [] for instance in instances: node = find_node(expression.name, instance.type) if node: nodes.append(node) if not nodes: # Try checking class namespace if attribute is on a class object. if isinstance(base_type, FunctionLike) and base_type.is_type_obj(): instances = get_instance_fallback( fill_typevars_with_any(base_type.type_object()) ) for instance in instances: node = find_node(expression.name, instance.type) if node: nodes.append(node) else: # Still no luck, give up. return [] else: return [] else: # Easy case: a module-level definition nodes = [node] return nodes def modules_for_nodes( self, nodes: list[FuncBase | SymbolNode], expression: RefExpr ) -> tuple[dict[FuncBase | SymbolNode, State], bool]: """Gather modules where given nodes where defined. Also check if they need to be refreshed (cached nodes may have lines/columns missing). """ modules = {} reload_needed = False for node in nodes: module = find_module_by_fullname(node.fullname, self.fg_manager.graph) if not module: if expression.kind == LDEF and self.module: module = self.module else: continue modules[node] = module if not module.tree or module.tree.is_cache_skeleton or self.force_reload: reload_needed |= not module.tree or module.tree.is_cache_skeleton self.reload_module(module) return modules, reload_needed def expression_def(self, expression: Expression) -> tuple[str, bool]: """Find and format definition location for an expression. If it is not a RefExpr, it is effectively skipped by returning an empty result. """ if not isinstance(expression, RefExpr): # If there are no suitable matches at all, we return error later. return "", True nodes = self.collect_nodes(expression) if not nodes: return self.missing_node(expression), False modules, reload_needed = self.modules_for_nodes(nodes, expression) if reload_needed: # TODO: line/column are not stored in cache for vast majority of symbol nodes. # Adding them will make thing faster, but will have visible memory impact. nodes = self.collect_nodes(expression) modules, reload_needed = self.modules_for_nodes(nodes, expression) assert not reload_needed result = [] for node in modules: result.append(self.format_node(modules[node], node)) if not result: return self.missing_node(expression), False return self.add_prefixes(", ".join(result), expression), True def missing_type(self, expression: Expression) -> str: alt_suggestion = "" if not self.force_reload: alt_suggestion = " or try --force-reload" return ( f'No known type available for "{type(expression).__name__}"' f" (maybe unreachable{alt_suggestion})" ) def missing_node(self, expression: Expression) -> str: return ( f'Cannot find definition for "{type(expression).__name__}" at {expr_span(expression)}' ) def add_prefixes(self, result: str, expression: Expression) -> str: prefixes = [] if self.include_kind: prefixes.append(f"{type(expression).__name__}") if self.include_span: prefixes.append(expr_span(expression)) if prefixes: prefix = ":".join(prefixes) + " -> " else: prefix = "" return prefix + result def run_inspection_by_exact_location( self, tree: MypyFile, line: int, column: int, end_line: int, end_column: int, method: Callable[[Expression], tuple[str, bool]], ) -> dict[str, object]: """Get type of an expression matching a span. Type or error is returned as a standard daemon response dict. """ try: expression = find_by_location(tree, line, column - 1, end_line, end_column) except ValueError as err: return {"error": str(err)} if expression is None: span = f"{line}:{column}:{end_line}:{end_column}" return {"out": f"Can't find expression at span {span}", "err": "", "status": 1} inspection_str, success = method(expression) return {"out": inspection_str, "err": "", "status": 0 if success else 1} def run_inspection_by_position( self, tree: MypyFile, line: int, column: int, method: Callable[[Expression], tuple[str, bool]], ) -> dict[str, object]: """Get types of all expressions enclosing a position. Types and/or errors are returned as a standard daemon response dict. """ expressions = find_all_by_location(tree, line, column - 1) if not expressions: position = f"{line}:{column}" return { "out": f"Can't find any expressions at position {position}", "err": "", "status": 1, } inspection_strs = [] status = 0 for expression in expressions: inspection_str, success = method(expression) if not success: status = 1 if inspection_str: inspection_strs.append(inspection_str) if self.limit: inspection_strs = inspection_strs[: self.limit] return {"out": "\n".join(inspection_strs), "err": "", "status": status} def find_module(self, file: str) -> tuple[State | None, dict[str, object]]: """Find module by path, or return a suitable error message. Note we don't use exceptions to simplify handling 1 vs 2 statuses. """ if not any(file.endswith(ext) for ext in PYTHON_EXTENSIONS): return None, {"error": "Source file is not a Python file"} # We are using a bit slower but robust way to find a module by path, # to be sure that namespace packages are handled properly. abs_path = os.path.abspath(file) state = next((s for s in self.fg_manager.graph.values() if s.abspath == abs_path), None) self.module = state return ( state, {"out": f"Unknown module: {file}", "err": "", "status": 1} if state is None else {}, ) def run_inspection( self, location: str, method: Callable[[Expression], tuple[str, bool]] ) -> dict[str, object]: """Top-level logic to inspect expression(s) at a location. This can be reused by various simple inspections. """ try: file, pos = parse_location(location) except ValueError as err: return {"error": str(err)} state, err_dict = self.find_module(file) if state is None: assert err_dict return err_dict # Force reloading to load from cache, account for any edits, etc. if not state.tree or state.tree.is_cache_skeleton or self.force_reload: self.reload_module(state) assert state.tree is not None if len(pos) == 4: # Full span, return an exact match only. line, column, end_line, end_column = pos return self.run_inspection_by_exact_location( state.tree, line, column, end_line, end_column, method ) assert len(pos) == 2 # Inexact location, return all expressions. line, column = pos return self.run_inspection_by_position(state.tree, line, column, method) def get_type(self, location: str) -> dict[str, object]: """Get types of expression(s) at a location.""" return self.run_inspection(location, self.expr_type) def get_attrs(self, location: str) -> dict[str, object]: """Get attributes of expression(s) at a location.""" return self.run_inspection(location, self.expr_attrs) def get_definition(self, location: str) -> dict[str, object]: """Get symbol definitions of expression(s) at a location.""" result = self.run_inspection(location, self.expression_def) if "out" in result and not result["out"]: # None of the expressions found turns out to be a RefExpr. _, location = location.split(":", maxsplit=1) result["out"] = f"No name or member expressions at {location}" result["status"] = 1 return result def parse_location(location: str) -> tuple[str, list[int]]: if location.count(":") < 2: raise ValueError("Format should be file:line:column[:end_line:end_column]") parts = location.rsplit(":", maxsplit=2) start, *rest = parts # Note: we must allow drive prefix like `C:` on Windows. if start.count(":") < 2: return start, [int(p) for p in rest] parts = start.rsplit(":", maxsplit=2) start, *start_rest = parts if start.count(":") < 2: return start, [int(p) for p in start_rest + rest] raise ValueError("Format should be file:line:column[:end_line:end_column]")