diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e4ee512ba..adf41dc32 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,17 +4,18 @@ on: [push, pull_request] jobs: test: - name: Test - ${{ matrix.python-version }} - runs-on: ubuntu-latest + name: Test - ${{ matrix.python-version }} - ${{matrix.os}} + runs-on: ${{matrix.os}} strategy: matrix: python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + os: [ubuntu-latest, windows-latest, macos-latest] fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/rope/contrib/autoimport/sqlite.py b/rope/contrib/autoimport/sqlite.py index b21e5fa0e..b18fcb245 100644 --- a/rope/contrib/autoimport/sqlite.py +++ b/rope/contrib/autoimport/sqlite.py @@ -6,13 +6,16 @@ from collections import OrderedDict from concurrent.futures import Future, ProcessPoolExecutor, as_completed from itertools import chain +from pathlib import Path from typing import Generator, Iterable, List, Optional, Set, Tuple from rope.base import exceptions, libutils, resourceobserver, taskhandle from rope.base.project import Project from rope.base.resources import Resource +from rope.base.utils import deprecated from rope.contrib.autoimport.defs import ( ModuleFile, + ModuleInfo, Name, NameType, Package, @@ -31,17 +34,16 @@ from rope.refactor import importutils -def get_future_names( - packages: List[Package], underlined: bool, job_set: taskhandle.JobSet +def _get_future_names( + to_index: List[Tuple[ModuleInfo, Package]], + underlined: bool, + job_set: taskhandle.JobSet, ) -> Generator[Future, None, None]: """Get all names as futures.""" with ProcessPoolExecutor() as executor: - for package in packages: - for module in get_files(package, underlined): - job_set.started_job(module.modname) - if not isinstance(job_set, taskhandle.NullJobSet): - job_set.count += 1 - yield executor.submit(get_names, module, package) + for module, package in to_index: + job_set.started_job(module.modname) + yield executor.submit(get_names, module, package) def filter_packages( @@ -121,6 +123,7 @@ def _setup_db(self): self.connection.execute("CREATE INDEX IF NOT EXISTS package on names(package)") self.connection.commit() + @deprecated("Use search or search_full") def import_assist(self, starting: str): """ Find modules that have a global name that starts with `starting`. @@ -247,6 +250,7 @@ def _search_module( f"import {module}", module, source, NameType.Module.value ) + @deprecated("Use search or search_full") def get_modules(self, name) -> List[str]: """Get the list of modules that have global `name`.""" results = self.connection.execute( @@ -254,6 +258,7 @@ def get_modules(self, name) -> List[str]: ).fetchall() return sort_and_deduplicate(results) + @deprecated("Use search or search_full") def get_all_names(self) -> List[str]: """Get the list of all cached global names.""" results = self.connection.execute("select name from names").fetchall() @@ -277,24 +282,13 @@ def generate_cache( those files are searched; otherwise all python modules in the project are cached. """ + if resources is None: resources = self.project.get_python_files() - job_set = task_handle.create_jobset( - "Generating autoimport cache", len(resources) + files = [Path(resource.real_path) for resource in resources] + self._generate_cache( + files=files, task_handle=task_handle, underlined=underlined ) - self.connection.execute( - "delete from names where package = ?", (self.project_package.name,) - ) - futures = [] - with ProcessPoolExecutor() as executor: - for file in resources: - job_set.started_job(f"Working on {file.path}") - module = self._resource_to_module(file, underlined) - futures.append(executor.submit(get_names, module, self.project_package)) - for future in as_completed(futures): - self._add_names(future.result()) - job_set.finished_job() - self.connection.commit() def generate_modules_cache( self, @@ -311,33 +305,68 @@ def generate_modules_cache( Do not use this for generating your own project's internal names, use generate_resource_cache for that instead. """ - packages: List[Package] = [] + self._generate_cache( + package_names=modules, + task_handle=task_handle, + single_thread=single_thread, + underlined=underlined, + ) + + # TODO: Update to use Task Handle ABC class + def _generate_cache( + self, + package_names: Optional[List[str]] = None, + files: Optional[List[Path]] = None, + underlined: bool = False, + task_handle=None, + single_thread: bool = False, + remove_extras: bool = False, + ): + """ + This will work under 3 modes: + 1. packages or files are specified. Autoimport will only index these. + 2. PEP 621 is configured. Only these dependencies are indexed. + 3. Index only standard library modules. + """ if self.underlined: underlined = True + if task_handle is None: + task_handle = taskhandle.NullTaskHandle() + packages: List[Package] = [] existing = self._get_existing() - if modules is None: - packages = self._get_available_packages() + to_index: List[Tuple[ModuleInfo, Package]] = [] + if files is not None: + assert package_names is None # Cannot have both package_names and files. + for file in files: + to_index.append((self._path_to_module(file, underlined), self.project)) else: - for modname in modules: - package = self._find_package_path(modname) - if package is None: - continue - packages.append(package) - packages = list(filter_packages(packages, underlined, existing)) - if len(packages) == 0: - return - self._add_packages(packages) - job_set = task_handle.create_jobset("Generating autoimport cache", 0) - if single_thread: + if package_names is None: + packages = self._get_available_packages() + else: + for modname in package_names: + package = self._find_package_path(modname) + if package is None: + continue + packages.append(package) + packages = list(filter_packages(packages, underlined, existing)) for package in packages: for module in get_files(package, underlined): - job_set.started_job(module.modname) - for name in get_names(module, package): - self._add_name(name) - job_set.finished_job() + to_index.append((module, package)) + self._add_packages(packages) + if len(to_index) == 0: + return + job_set = task_handle.create_jobset( + "Generating autoimport cache", len(to_index) + ) + if single_thread: + for module, package in to_index: + job_set.started_job(module.modname) + for name in get_names(module, package): + self._add_name(name) + job_set.finished_job() else: for future_name in as_completed( - get_future_names(packages, underlined, job_set) + _get_future_names(to_index, underlined, job_set) ): self._add_names(future_name.result()) job_set.finished_job() @@ -406,17 +435,13 @@ def find_insertion_line(self, code): lineno = code.count("\n", 0, offset) + 1 return lineno - def update_resource( - self, resource: Resource, underlined: bool = False, commit: bool = True - ): + def update_resource(self, resource: Resource, underlined: bool = False): """Update the cache for global names in `resource`.""" underlined = underlined if underlined else self.underlined - module = self._resource_to_module(resource, underlined) + path = Path(resource.real_path) + module = self._path_to_module(path, underlined) self._del_if_exist(module_name=module.modname, commit=False) - for name in get_names(module, self.project_package): - self._add_name(name) - if commit: - self.connection.commit() + self._generate_cache(files=[path], underlined=underlined) def _changed(self, resource): if not resource.is_folder(): @@ -424,9 +449,11 @@ def _changed(self, resource): def _moved(self, resource: Resource, newresource: Resource): if not resource.is_folder(): - modname = self._resource_to_module(resource).modname + path = Path(resource.real_path) + modname = self._path_to_module(path).modname self._del_if_exist(modname) - self.update_resource(newresource) + new_path = Path(newresource.real_path) + self._generate_cache(files=[new_path]) def _del_if_exist(self, module_name, commit: bool = True): self.connection.execute("delete from names where module = ?", (module_name,)) @@ -468,7 +495,8 @@ def _get_existing(self) -> List[str]: def _removed(self, resource): if not resource.is_folder(): - modname = self._resource_to_module(resource).modname + path = Path(resource.real_path) + modname = self._path_to_module(path).modname self._del_if_exist(modname) def _add_future_names(self, names: Future): @@ -504,21 +532,18 @@ def _find_package_path(self, target_name: str) -> Optional[Package]: return None - def _resource_to_module( - self, resource: Resource, underlined: bool = False - ) -> ModuleFile: + def _path_to_module(self, path: Path, underlined: bool = False) -> ModuleFile: assert self.project_package.path underlined = underlined if underlined else self.underlined - resource_path: pathlib.Path = pathlib.Path(resource.real_path) # The project doesn't need its name added to the path, # since the standard python file layout accounts for that # so we set add_package_name to False resource_modname: str = get_modname_from_path( - resource_path, self.project_package.path, add_package_name=False + path, self.project_package.path, add_package_name=False ) return ModuleFile( - resource_path, + path, resource_modname, underlined, - resource_path.name == "__init__.py", + path.name == "__init__.py", ) diff --git a/rope/contrib/autoimport/utils.py b/rope/contrib/autoimport/utils.py index bb49b6a49..c54038dbd 100644 --- a/rope/contrib/autoimport/utils.py +++ b/rope/contrib/autoimport/utils.py @@ -19,34 +19,42 @@ def get_package_tuple( Returns None if not a viable package. """ package_name = package_path.name - package_source = get_package_source(package_path, project) + package_type: PackageType if package_name.startswith(".") or package_name == "__pycache__": return None - if package_path.is_file(): - if package_name.endswith(".so"): - name = package_name.split(".")[0] - return Package(name, package_source, package_path, PackageType.COMPILED) - if package_name.endswith(".py"): - stripped_name = package_path.stem - return Package( - stripped_name, package_source, package_path, PackageType.SINGLE_FILE - ) - return None if package_name.endswith((".egg-info", ".dist-info")): return None - return Package(package_name, package_source, package_path, PackageType.STANDARD) + if package_path.is_file(): + if package_name.endswith(".so"): + package_name = package_name.split(".")[0] + package_type = PackageType.COMPILED + elif package_name.endswith(".py"): + package_name = package_path.stem + package_type = PackageType.SINGLE_FILE + else: + return None + else: + package_type = PackageType.STANDARD + package_source: Source = get_package_source(package_path, project, package_name) + return Package(package_name, package_source, package_path, package_type) def get_package_source( - package: pathlib.Path, project: Optional[Project] = None + package: pathlib.Path, project: Optional[Project], name: str ) -> Source: """Detect the source of a given package. Rudimentary implementation.""" + if name in sys.builtin_module_names: + return Source.BUILTIN if project is not None and project.address in str(package): return Source.PROJECT if "site-packages" in package.parts: return Source.SITE_PACKAGE - if package.as_posix().startswith(sys.prefix): - return Source.STANDARD + if sys.version_info < (3, 10, 0): + if str(package).startswith(sys.prefix): + return Source.STANDARD + else: + if name in sys.stdlib_module_names: + return Source.STANDARD return Source.UNKNOWN diff --git a/ropetest/contrib/autoimport/utilstest.py b/ropetest/contrib/autoimport/utilstest.py index 423c0e754..345237c5f 100644 --- a/ropetest/contrib/autoimport/utilstest.py +++ b/ropetest/contrib/autoimport/utilstest.py @@ -1,28 +1,27 @@ """Tests for autoimport utility functions, written in pytest""" -import pathlib from rope.contrib.autoimport import utils from rope.contrib.autoimport.defs import Package, PackageType, Source def test_get_package_source(mod1_path, project): - assert utils.get_package_source(mod1_path, project) == Source.PROJECT + assert utils.get_package_source(mod1_path, project, "") == Source.PROJECT def test_get_package_source_not_project(mod1_path): - assert utils.get_package_source(mod1_path) == Source.UNKNOWN + assert utils.get_package_source(mod1_path, None, "") == Source.UNKNOWN def test_get_package_source_pytest(build_path): # pytest is not installed as part of the standard library # but should be installed into site_packages, # so it should return Source.SITE_PACKAGE - assert utils.get_package_source(build_path) == Source.SITE_PACKAGE + assert utils.get_package_source(build_path, None, "build") == Source.SITE_PACKAGE def test_get_package_source_typing(typing_path): - assert utils.get_package_source(typing_path) == Source.STANDARD + assert utils.get_package_source(typing_path, None, "typing") == Source.STANDARD def test_get_modname_project_no_add(mod1_path, project_path):