Autoimport refactor

.github/workflows/main.yml

@@ -4,17 +4,18 @@ on: [push, pull_request]
 
 jobs:
   test:
-    name: Test - ${{ matrix.python-version }}
-    runs-on: ubuntu-latest
+    name: Test - ${{ matrix.python-version }} - ${{matrix.os}}
+    runs-on: ${{matrix.os}}
 
     strategy:
       matrix:
         python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
+        os: [ubuntu-latest, windows-latest, macos-latest]
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

rope/contrib/autoimport/sqlite.py

@@ -6,13 +6,16 @@
 from collections import OrderedDict
 from concurrent.futures import Future, ProcessPoolExecutor, as_completed
 from itertools import chain
+from pathlib import Path
 from typing import Generator, Iterable, List, Optional, Set, Tuple
 
 from rope.base import exceptions, libutils, resourceobserver, taskhandle
 from rope.base.project import Project
 from rope.base.resources import Resource
+from rope.base.utils import deprecated
 from rope.contrib.autoimport.defs import (
     ModuleFile,
+    ModuleInfo,
     Name,
     NameType,
     Package,
@@ -31,17 +34,16 @@
 from rope.refactor import importutils
 
 
-def get_future_names(
-    packages: List[Package], underlined: bool, job_set: taskhandle.JobSet
+def _get_future_names(
+    to_index: List[Tuple[ModuleInfo, Package]],
+    underlined: bool,
+    job_set: taskhandle.JobSet,
 ) -> Generator[Future, None, None]:
     """Get all names as futures."""
     with ProcessPoolExecutor() as executor:
-        for package in packages:
-            for module in get_files(package, underlined):
-                job_set.started_job(module.modname)
-                if not isinstance(job_set, taskhandle.NullJobSet):
-                    job_set.count += 1
-                yield executor.submit(get_names, module, package)
+        for module, package in to_index:
+            job_set.started_job(module.modname)
+            yield executor.submit(get_names, module, package)
 
 
 def filter_packages(
@@ -121,6 +123,7 @@ def _setup_db(self):
         self.connection.execute("CREATE INDEX IF NOT EXISTS package on names(package)")
         self.connection.commit()
 
+    @deprecated("Use search or search_full")
     def import_assist(self, starting: str):
         """
         Find modules that have a global name that starts with `starting`.
@@ -247,13 +250,15 @@ def _search_module(
                 f"import {module}", module, source, NameType.Module.value
             )
 
+    @deprecated("Use search or search_full")
     def get_modules(self, name) -> List[str]:
         """Get the list of modules that have global `name`."""
         results = self.connection.execute(
             "SELECT module, source FROM names WHERE name LIKE (?)", (name,)
         ).fetchall()
         return sort_and_deduplicate(results)
 
+    @deprecated("Use search or search_full")
     def get_all_names(self) -> List[str]:
         """Get the list of all cached global names."""
         results = self.connection.execute("select name from names").fetchall()
@@ -277,24 +282,13 @@ def generate_cache(
         those files are searched; otherwise all python modules in the
         project are cached.
         """
+
         if resources is None:
             resources = self.project.get_python_files()
-        job_set = task_handle.create_jobset(
-            "Generating autoimport cache", len(resources)
+        files = [Path(resource.real_path) for resource in resources]
+        self._generate_cache(
+            files=files, task_handle=task_handle, underlined=underlined
         )
-        self.connection.execute(
-            "delete from names where package = ?", (self.project_package.name,)
-        )
-        futures = []
-        with ProcessPoolExecutor() as executor:
-            for file in resources:
-                job_set.started_job(f"Working on {file.path}")
-                module = self._resource_to_module(file, underlined)
-                futures.append(executor.submit(get_names, module, self.project_package))
-        for future in as_completed(futures):
-            self._add_names(future.result())
-            job_set.finished_job()
-        self.connection.commit()
 
     def generate_modules_cache(
         self,
@@ -311,33 +305,68 @@ def generate_modules_cache(
         Do not use this for generating your own project's internal names,
         use generate_resource_cache for that instead.
         """
-        packages: List[Package] = []
+        self._generate_cache(
+            package_names=modules,
+            task_handle=task_handle,
+            single_thread=single_thread,
+            underlined=underlined,
+        )
+
+    # TODO: Update to use Task Handle ABC class
+    def _generate_cache(
+        self,
+        package_names: Optional[List[str]] = None,
+        files: Optional[List[Path]] = None,
+        underlined: bool = False,
+        task_handle=None,
+        single_thread: bool = False,
+        remove_extras: bool = False,
+    ):
+        """
+        This will work under 3 modes:
+        1. packages or files are specified. Autoimport will only index these.
+        2. PEP 621 is configured. Only these dependencies are indexed.
+        3. Index only standard library modules.
+        """
         if self.underlined:
             underlined = True
+        if task_handle is None:
+            task_handle = taskhandle.NullTaskHandle()
+        packages: List[Package] = []
         existing = self._get_existing()
-        if modules is None:
-            packages = self._get_available_packages()
+        to_index: List[Tuple[ModuleInfo, Package]] = []
+        if files is not None:
+            assert package_names is None  # Cannot have both package_names and files.
+            for file in files:
+                to_index.append((self._path_to_module(file, underlined), self.project))
         else:
-            for modname in modules:
-                package = self._find_package_path(modname)
-                if package is None:
-                    continue
-                packages.append(package)
-        packages = list(filter_packages(packages, underlined, existing))
-        if len(packages) == 0:
-            return
-        self._add_packages(packages)
-        job_set = task_handle.create_jobset("Generating autoimport cache", 0)
-        if single_thread:
+            if package_names is None:
+                packages = self._get_available_packages()
+            else:
+                for modname in package_names:
+                    package = self._find_package_path(modname)
+                    if package is None:
+                        continue
+                    packages.append(package)
+            packages = list(filter_packages(packages, underlined, existing))
             for package in packages:
                 for module in get_files(package, underlined):
-                    job_set.started_job(module.modname)
-                    for name in get_names(module, package):
-                        self._add_name(name)
-                        job_set.finished_job()
+                    to_index.append((module, package))
+            self._add_packages(packages)
+        if len(to_index) == 0:
+            return
+        job_set = task_handle.create_jobset(
+            "Generating autoimport cache", len(to_index)
+        )
+        if single_thread:
+            for module, package in to_index:
+                job_set.started_job(module.modname)
+                for name in get_names(module, package):
+                    self._add_name(name)
+                    job_set.finished_job()
         else:
             for future_name in as_completed(
-                get_future_names(packages, underlined, job_set)
+                _get_future_names(to_index, underlined, job_set)
             ):
                 self._add_names(future_name.result())
                 job_set.finished_job()
@@ -406,27 +435,25 @@ def find_insertion_line(self, code):
         lineno = code.count("\n", 0, offset) + 1
         return lineno
 
-    def update_resource(
-        self, resource: Resource, underlined: bool = False, commit: bool = True
-    ):
+    def update_resource(self, resource: Resource, underlined: bool = False):
         """Update the cache for global names in `resource`."""
         underlined = underlined if underlined else self.underlined
-        module = self._resource_to_module(resource, underlined)
+        path = Path(resource.real_path)
+        module = self._path_to_module(path, underlined)
         self._del_if_exist(module_name=module.modname, commit=False)
-        for name in get_names(module, self.project_package):
-            self._add_name(name)
-        if commit:
-            self.connection.commit()
+        self._generate_cache(files=[path], underlined=underlined)
 
     def _changed(self, resource):
         if not resource.is_folder():
             self.update_resource(resource)
 
     def _moved(self, resource: Resource, newresource: Resource):
         if not resource.is_folder():
-            modname = self._resource_to_module(resource).modname
+            path = Path(resource.real_path)
+            modname = self._path_to_module(path).modname
             self._del_if_exist(modname)
-            self.update_resource(newresource)
+            new_path = Path(newresource.real_path)
+            self._generate_cache(files=[new_path])
 
     def _del_if_exist(self, module_name, commit: bool = True):
         self.connection.execute("delete from names where module = ?", (module_name,))
@@ -468,7 +495,8 @@ def _get_existing(self) -> List[str]:
 
     def _removed(self, resource):
         if not resource.is_folder():
-            modname = self._resource_to_module(resource).modname
+            path = Path(resource.real_path)
+            modname = self._path_to_module(path).modname
             self._del_if_exist(modname)
 
     def _add_future_names(self, names: Future):
@@ -504,21 +532,18 @@ def _find_package_path(self, target_name: str) -> Optional[Package]:
 
         return None
 
-    def _resource_to_module(
-        self, resource: Resource, underlined: bool = False
-    ) -> ModuleFile:
+    def _path_to_module(self, path: Path, underlined: bool = False) -> ModuleFile:
         assert self.project_package.path
         underlined = underlined if underlined else self.underlined
-        resource_path: pathlib.Path = pathlib.Path(resource.real_path)
         # The project doesn't need its name added to the path,
         # since the standard python file layout accounts for that
         # so we set add_package_name to False
         resource_modname: str = get_modname_from_path(
-            resource_path, self.project_package.path, add_package_name=False
+            path, self.project_package.path, add_package_name=False
         )
         return ModuleFile(
-            resource_path,
+            path,
             resource_modname,
             underlined,
-            resource_path.name == "__init__.py",
+            path.name == "__init__.py",
         )

rope/contrib/autoimport/utils.py

@@ -19,34 +19,42 @@ def get_package_tuple(
     Returns None if not a viable package.
     """
     package_name = package_path.name
-    package_source = get_package_source(package_path, project)
+    package_type: PackageType
     if package_name.startswith(".") or package_name == "__pycache__":
         return None
-    if package_path.is_file():
-        if package_name.endswith(".so"):
-            name = package_name.split(".")[0]
-            return Package(name, package_source, package_path, PackageType.COMPILED)
-        if package_name.endswith(".py"):
-            stripped_name = package_path.stem
-            return Package(
-                stripped_name, package_source, package_path, PackageType.SINGLE_FILE
-            )
-        return None
     if package_name.endswith((".egg-info", ".dist-info")):
         return None
-    return Package(package_name, package_source, package_path, PackageType.STANDARD)
+    if package_path.is_file():
+        if package_name.endswith(".so"):
+            package_name = package_name.split(".")[0]
+            package_type = PackageType.COMPILED
+        elif package_name.endswith(".py"):
+            package_name = package_path.stem
+            package_type = PackageType.SINGLE_FILE
+        else:
+            return None
+    else:
+        package_type = PackageType.STANDARD
+    package_source: Source = get_package_source(package_path, project, package_name)
+    return Package(package_name, package_source, package_path, package_type)
 
 
 def get_package_source(
-    package: pathlib.Path, project: Optional[Project] = None
+    package: pathlib.Path, project: Optional[Project], name: str
 ) -> Source:
     """Detect the source of a given package. Rudimentary implementation."""
+    if name in sys.builtin_module_names:
+        return Source.BUILTIN
     if project is not None and project.address in str(package):
         return Source.PROJECT
     if "site-packages" in package.parts:
         return Source.SITE_PACKAGE
-    if package.as_posix().startswith(sys.prefix):
-        return Source.STANDARD
+    if sys.version_info < (3, 10, 0):
+        if str(package).startswith(sys.prefix):
+            return Source.STANDARD
+    else:
+        if name in sys.stdlib_module_names:
+            return Source.STANDARD
     return Source.UNKNOWN
 
 

ropetest/contrib/autoimport/utilstest.py

@@ -1,28 +1,27 @@
 """Tests for autoimport utility functions, written in pytest"""
-import pathlib
 
 from rope.contrib.autoimport import utils
 from rope.contrib.autoimport.defs import Package, PackageType, Source
 
 
 def test_get_package_source(mod1_path, project):
-    assert utils.get_package_source(mod1_path, project) == Source.PROJECT
+    assert utils.get_package_source(mod1_path, project, "") == Source.PROJECT
 
 
 def test_get_package_source_not_project(mod1_path):
-    assert utils.get_package_source(mod1_path) == Source.UNKNOWN
+    assert utils.get_package_source(mod1_path, None, "") == Source.UNKNOWN
 
 
 def test_get_package_source_pytest(build_path):
     # pytest is not installed as part of the standard library
     # but should be installed into site_packages,
     # so it should return Source.SITE_PACKAGE
-    assert utils.get_package_source(build_path) == Source.SITE_PACKAGE
+    assert utils.get_package_source(build_path, None, "build") == Source.SITE_PACKAGE
 
 
 def test_get_package_source_typing(typing_path):
 
-    assert utils.get_package_source(typing_path) == Source.STANDARD
+    assert utils.get_package_source(typing_path, None, "typing") == Source.STANDARD
 
 
 def test_get_modname_project_no_add(mod1_path, project_path):