New AutoImport implementation (early-merge of @bageljrkhanofemus's changes)

.github/workflows/main.yml

@@ -19,7 +19,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip pytest pytest-timeout
+        python -m pip install -e .[dev] 
     - name: Test with pytest
       run: |
         pytest -v

CHANGELOG.md

@@ -1,8 +1,6 @@
 # **Upcoming release**
-
-## XXX
-
-- XXX
+## New feature
+- #464 Improve autoimport code to use a sqllite3 database, cache all available modules quickly, search for names and produce import statements, sort import statements.
 
 # Release 1.0.0
 

docs/library.rst

@@ -836,6 +836,34 @@ returns the list of modules with the given global name.
 global name that starts with the given prefix.
 
 
+There are currently two implementations of autoimport in rope, a deprecated
+implementation that uses pickle-based storage
+(rope.contrib.autoimport.pickle.AutoImport) and a new, experimental one that
+uses sqlite3 database (rope.contrib.autoimport.sqlite.AutoImport). New and
+existing integrations should migrate to the sqlite3 storage as the pickle-based
+autoimport will be removed in the future.
+
+By default, the sqlite3-based only stores autoimport cache in an in-memory
+sqlite3 database, you can make it write the import cache to persistent storage
+by passing memory=False to AutoImport constructor.
+
+It must be closed when done with the ```AutoImport.close()``` method.
+
+AutoImport can search for a name from both modules and statements you can import from them.
+
+.. code-block:: python
+
+  from rope.base.project import Project
+  from rope.contrib.autoimport import AutoImport
+
+  project = Project("/path/to/project")
+  autoimport = AutoImport(project, memory=False)
+  autoimport.generate_resource_cache()  # Generates a cache of the local modules, from the project you're working on
+  autoimport.generate_modules_cache()  # Generates a cache of external modules
+  print(autoimport.search("Dict"))
+  autoimport.close()
+
+
 Cross-Project Refactorings
 --------------------------
 

rope/contrib/autoimport/__init__.py

@@ -0,0 +1,8 @@
+"""AutoImport module for rope."""
+from .pickle import AutoImport as _PickleAutoImport
+from .sqlite import AutoImport as _SqliteAutoImport
+
+
+AutoImport = _PickleAutoImport
+
+__all__ = ["AutoImport"]

rope/contrib/autoimport/defs.py

@@ -0,0 +1,109 @@
+"""Definitions of types for the Autoimport program."""
+import pathlib
+from enum import Enum
+from typing import NamedTuple, Optional
+
+
+class Source(Enum):
+    """Describes the source of the package, for sorting purposes."""
+
+    PROJECT = 0  # Obviously any project packages come first
+    MANUAL = 1  # Placeholder since Autoimport classifies manually added modules
+    BUILTIN = 2
+    STANDARD = 3  # We want to favor standard library items
+    SITE_PACKAGE = 4
+    UNKNOWN = 5
+
+    # modified_time
+
+
+class ModuleInfo(NamedTuple):
+    """Descriptor of information to get names from a module."""
+
+    filepath: Optional[pathlib.Path]
+    modname: str
+    underlined: bool
+    process_imports: bool = False
+
+
+class ModuleFile(ModuleInfo):
+    """Descriptor of information to get names from a file using ast."""
+
+    filepath: pathlib.Path
+    modname: str
+    underlined: bool
+    process_imports: bool = False
+
+
+class ModuleCompiled(ModuleInfo):
+    """Descriptor of information to get names using imports."""
+
+    filepath = None
+    modname: str
+    underlined: bool
+    process_imports: bool = False
+
+
+class PackageType(Enum):
+    """Describes the type of package, to determine how to get the names from it."""
+
+    BUILTIN = 0  # No file exists, compiled into python. IE: Sys
+    STANDARD = 1  # Just a folder
+    COMPILED = 2  # .so module
+    SINGLE_FILE = 3  # a .py file
+
+
+class NameType(Enum):
+    """Describes the type of Name for lsp completions. Taken from python lsp server."""
+
+    Text = 1
+    Method = 2
+    Function = 3
+    Constructor = 4
+    Field = 5
+    Variable = 6
+    Class = 7
+    Interface = 8
+    Module = 9
+    Property = 10
+    Unit = 11
+    Value = 12
+    Enum = 13
+    Keyword = 14
+    Snippet = 15
+    Color = 16
+    File = 17
+    Reference = 18
+    Folder = 19
+    EnumMember = 20
+    Constant = 21
+    Struct = 22
+    Event = 23
+    Operator = 24
+    TypeParameter = 25
+
+
+class Package(NamedTuple):
+    """Attributes of a package."""
+
+    name: str
+    source: Source
+    path: Optional[pathlib.Path]
+    type: PackageType
+
+
+class Name(NamedTuple):
+    """A Name to be added to the database."""
+
+    name: str
+    modname: str
+    package: str
+    source: Source
+    name_type: NameType
+
+
+class PartialName(NamedTuple):
+    """Partial information of a Name."""
+
+    name: str
+    name_type: NameType

rope/contrib/autoimport/parse.py

@@ -0,0 +1,137 @@
+"""
+Functions to find importable names.
+
+Can extract names from source code of a python file, .so object, or builtin module.
+"""
+
+import ast
+import inspect
+import logging
+import pathlib
+from importlib import import_module
+from typing import Generator, List
+
+from .defs import (
+    ModuleCompiled,
+    ModuleFile,
+    ModuleInfo,
+    Name,
+    NameType,
+    Package,
+    PartialName,
+    Source,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def get_type_ast(node: ast.AST) -> NameType:
+    """Get the lsp type of a node."""
+    if isinstance(node, ast.ClassDef):
+        return NameType.Class
+    if isinstance(node, ast.FunctionDef):
+        return NameType.Function
+    if isinstance(node, ast.Assign):
+        return NameType.Variable
+    return NameType.Text  # default value
+
+
+def get_names_from_file(
+    module: pathlib.Path,
+    underlined: bool = False,
+) -> Generator[PartialName, None, None]:
+    """Get all the names from a given file using ast."""
+
+    try:
+        root_node = ast.parse(module.read_bytes())
+    except SyntaxError as error:
+        print(error)
+        return
+    for node in ast.iter_child_nodes(root_node):
+        if isinstance(node, ast.Assign):
+            for target in node.targets:
+                try:
+                    assert isinstance(target, ast.Name)
+                    if underlined or not target.id.startswith("_"):
+                        yield PartialName(
+                            target.id,
+                            get_type_ast(node),
+                        )
+                except (AttributeError, AssertionError):
+                    # TODO handle tuple assignment
+                    pass
+        elif isinstance(node, (ast.FunctionDef, ast.ClassDef)):
+            if underlined or not node.name.startswith("_"):
+                yield PartialName(
+                    node.name,
+                    get_type_ast(node),
+                )
+
+
+def get_type_object(imported_object) -> NameType:
+    """Determine the type of an object."""
+    if inspect.isclass(imported_object):
+        return NameType.Class
+    if inspect.isfunction(imported_object) or inspect.isbuiltin(imported_object):
+        return NameType.Function
+    return NameType.Constant
+
+
+def get_names(module: ModuleInfo, package: Package) -> List[Name]:
+    """Get all names from a module and package."""
+    if isinstance(module, ModuleCompiled):
+        return list(
+            get_names_from_compiled(package.name, package.source, module.underlined)
+        )
+    if isinstance(module, ModuleFile):
+        return [
+            combine(package, module, partial_name)
+            for partial_name in get_names_from_file(module.filepath, module.underlined)
+        ]
+    return []
+
+
+def get_names_from_compiled(
+    package: str,
+    source: Source,
+    underlined: bool = False,
+) -> Generator[Name, None, None]:
+    """
+    Get the names from a compiled module.
+
+    Instead of using ast, it imports the module.
+    Parameters
+    ----------
+    package : str
+        package to import. Must be in sys.path
+    underlined : bool
+        include underlined names
+    """
+    # builtins is banned because you never have to import it
+    # python_crun is banned because it crashes python
+    banned = ["builtins", "python_crun"]
+    if package in banned or (package.startswith("_") and not underlined):
+        return  # Builtins is redundant since you don't have to import it.
+    if source not in (Source.BUILTIN, Source.STANDARD):
+        return
+    try:
+        module = import_module(str(package))
+    except ImportError:
+        logger.error(f"{package} could not be imported for autoimport analysis")
+        return
+    else:
+        for name, value in inspect.getmembers(module):
+            if underlined or not name.startswith("_"):
+                if (
+                    inspect.isclass(value)
+                    or inspect.isfunction(value)
+                    or inspect.isbuiltin(value)
+                ):
+                    yield Name(
+                        str(name), package, package, source, get_type_object(value)
+                    )
+
+
+def combine(package: Package, module: ModuleFile, name: PartialName) -> Name:
+    """Combine information to form a full name."""
+    return Name(name.name, module.modname, package.name, package.source, name.name_type)

rope/contrib/autoimport.py → rope/contrib/autoimport/pickle.py

@@ -1,3 +1,15 @@
+"""
+IMPORTANT: This is a deprecated implementation of autoimport using pickle-based
+storage.
+
+This pickle-based autoimport is provided only for backwards compatibility
+purpose and will be removed and the sqlite backend will be the new default
+implementation in the future.
+
+If you are still using this module, you should migrate to the new and improved
+sqlite-based storage backend (rope.contrib.autoimport.sqlite.AutoImport).
+"""
+
 import re
 
 from rope.base import builtins