diff --git a/CHANGELOG.md b/CHANGELOG.md index ec01b75abde..168944d8f4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Change Log +## [1.1.4] - 2020-10-23 + +### Added + +- Added `installer.parallel` boolean flag (defaults to `true`) configuration to enable/disable parallel execution of operations when using the new installer. ([#3088](https://github.com/python-poetry/poetry/pull/3088)) + +### Changed + +- When using system environments as an unprivileged user, user site and bin directories are created if they do not already exist. ([#3107](https://github.com/python-poetry/poetry/pull/3107)) + +### Fixed + +- Fixed editable installation of poetry projects when using system environments. ([#3107](https://github.com/python-poetry/poetry/pull/3107)) +- Fixed locking of nested extra activations. If you were affected by this issue, you will need to regenerate the lock file using `poetry lock --no-update`. ([#3229](https://github.com/python-poetry/poetry/pull/3229)) +- Fixed prioritisation of non-default custom package sources. ([#3251](https://github.com/python-poetry/poetry/pull/3251)) +- Fixed detection of installed editable packages when non-poetry managed `.pth` file exists. ([#3210](https://github.com/python-poetry/poetry/pull/3210)) +- Fixed scripts generated by editable builder to use valid import statements. ([#3214](https://github.com/python-poetry/poetry/pull/3214)) +- Fixed recursion error when locked dependencies contain cyclic dependencies. ([#3237](https://github.com/python-poetry/poetry/pull/3237)) +- Fixed propagation of editable flag for VCS dependencies. ([#3264](https://github.com/python-poetry/poetry/pull/3264)) + ## [1.1.3] - 2020-10-14 ### Changed @@ -1063,7 +1083,8 @@ Initial release -[Unreleased]: https://github.com/python-poetry/poetry/compare/1.1.3...master +[Unreleased]: https://github.com/python-poetry/poetry/compare/1.1.4...master +[1.1.3]: https://github.com/python-poetry/poetry/compare/1.1.4 [1.1.3]: https://github.com/python-poetry/poetry/compare/1.1.3 [1.1.2]: https://github.com/python-poetry/poetry/releases/tag/1.1.2 [1.1.1]: https://github.com/python-poetry/poetry/releases/tag/1.1.1 diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index 782c2f9ac87..72310886ca7 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -103,6 +103,15 @@ Defaults to one of the following directories: - Windows: `C:\Users\\AppData\Local\pypoetry\Cache` - Unix: `~/.cache/pypoetry` +### `installer.parallel`: boolean + +Use parallel execution when using the new (`>=1.1.0`) installer. +Defaults to `true`. + +!!!note: + This configuration will be ignored, and parallel execution disabled when running + Python 2.7 under Windows. + ### `virtualenvs.create`: boolean Create a new virtual environment if one doesn't already exist. diff --git a/poetry/config/config.py b/poetry/config/config.py index d1fb421e53c..6f18127e8ff 100644 --- a/poetry/config/config.py +++ b/poetry/config/config.py @@ -39,6 +39,7 @@ class Config(object): "options": {"always-copy": False}, }, "experimental": {"new-installer": True}, + "installer": {"parallel": True}, } def __init__( @@ -140,6 +141,7 @@ def _get_validator(self, name): # type: (str) -> Callable "virtualenvs.create", "virtualenvs.in-project", "virtualenvs.options.always-copy", + "installer.parallel", }: return boolean_validator @@ -151,6 +153,7 @@ def _get_normalizer(self, name): # type: (str) -> Callable "virtualenvs.create", "virtualenvs.in-project", "virtualenvs.options.always-copy", + "installer.parallel", }: return boolean_normalizer diff --git a/poetry/console/commands/config.py b/poetry/console/commands/config.py index 52e1cf125ab..934b2c818eb 100644 --- a/poetry/console/commands/config.py +++ b/poetry/console/commands/config.py @@ -69,6 +69,7 @@ def unique_config_values(self): boolean_normalizer, False, ), + "installer.parallel": (boolean_validator, boolean_normalizer, True,), } return unique_config_values diff --git a/poetry/factory.py b/poetry/factory.py index fd863ba5991..08a6faca170 100644 --- a/poetry/factory.py +++ b/poetry/factory.py @@ -70,7 +70,8 @@ def create_poetry( ) # Configuring sources - for source in poetry.local_config.get("source", []): + sources = poetry.local_config.get("source", []) + for source in sources: repository = self.create_legacy_repository(source, config) is_default = source.get("default", False) is_secondary = source.get("secondary", False) @@ -90,7 +91,8 @@ def create_poetry( # Always put PyPI last to prefer private repositories # but only if we have no other default source if not poetry.pool.has_default(): - poetry.pool.add_repository(PyPiRepository(), True) + has_sources = bool(sources) + poetry.pool.add_repository(PyPiRepository(), not has_sources, has_sources) else: if io.is_debug(): io.write_line("Deactivating the PyPI repository") diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index a3184b718d5..a65dbb4a0e1 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -32,7 +32,7 @@ class Executor(object): - def __init__(self, env, pool, config, io, parallel=True): + def __init__(self, env, pool, config, io, parallel=None): self._env = env self._io = io self._dry_run = False @@ -42,6 +42,9 @@ def __init__(self, env, pool, config, io, parallel=True): self._chef = Chef(config, self._env) self._chooser = Chooser(pool, self._env) + if parallel is None: + parallel = config.get("installer.parallel", True) + if parallel and not (PY2 and WINDOWS): # This should be directly handled by ThreadPoolExecutor # however, on some systems the number of CPUs cannot be determined diff --git a/poetry/installation/pip_installer.py b/poetry/installation/pip_installer.py index f5de6642f2d..e44d7a61297 100644 --- a/poetry/installation/pip_installer.py +++ b/poetry/installation/pip_installer.py @@ -113,7 +113,9 @@ def remove(self, package): raise # This is a workaround for https://github.com/pypa/pip/issues/4176 - nspkg_pth_file = self._env.site_packages / "{}-nspkg.pth".format(package.name) + nspkg_pth_file = self._env.site_packages.path / "{}-nspkg.pth".format( + package.name + ) if nspkg_pth_file.exists(): nspkg_pth_file.unlink() diff --git a/poetry/masonry/builders/editable.py b/poetry/masonry/builders/editable.py index 99136ecbd3e..74d1f69c886 100644 --- a/poetry/masonry/builders/editable.py +++ b/poetry/masonry/builders/editable.py @@ -13,6 +13,7 @@ from poetry.utils._compat import WINDOWS from poetry.utils._compat import Path from poetry.utils._compat import decode +from poetry.utils.helpers import is_dir_writable SCRIPT_TEMPLATE = """\ @@ -94,7 +95,6 @@ def _setup_build(self): os.remove(str(setup)) def _add_pth(self): - pth_file = Path(self._module.name).with_suffix(".pth") paths = set() for include in self._module.includes: if isinstance(include, PackageInclude) and ( @@ -106,34 +106,40 @@ def _add_pth(self): for path in paths: content += decode(path + os.linesep) - for site_package in [self._env.site_packages, self._env.usersite]: - if not site_package: - continue - - try: - site_package.mkdir(parents=True, exist_ok=True) - path = site_package.joinpath(pth_file) - self._debug( - " - Adding {} to {} for {}".format( - path.name, site_package, self._poetry.file.parent - ) + pth_file = Path(self._module.name).with_suffix(".pth") + try: + pth_file = self._env.site_packages.write_text( + pth_file, content, encoding="utf-8" + ) + self._debug( + " - Adding {} to {} for {}".format( + pth_file.name, pth_file.parent, self._poetry.file.parent ) - path.write_text(content, encoding="utf-8") - return [path] - except PermissionError: - self._debug("- {} is not writable trying next available site") - - self._io.error_line( - " - Failed to create {} for {}".format( - pth_file.name, self._poetry.file.parent ) - ) - return [] + return [pth_file] + except OSError: + # TODO: Replace with PermissionError + self._io.error_line( + " - Failed to create {} for {}".format( + pth_file.name, self._poetry.file.parent + ) + ) + return [] def _add_scripts(self): added = [] entry_points = self.convert_entry_points() - scripts_path = Path(self._env.paths["scripts"]) + + for scripts_path in self._env.script_dirs: + if is_dir_writable(path=scripts_path, create=True): + break + else: + self._io.error_line( + " - Failed to find a suitable script installation directory for {}".format( + self._poetry.file.parent + ) + ) + return [] scripts = entry_points.get("console_scripts", []) for script in scripts: @@ -151,7 +157,7 @@ def _add_scripts(self): f.write( decode( SCRIPT_TEMPLATE.format( - python=self._env._bin("python"), + python=self._env.python, module=module, callable_holder=callable_holder, callable_=callable_, @@ -165,9 +171,7 @@ def _add_scripts(self): if WINDOWS: cmd_script = script_file.with_suffix(".cmd") - cmd = WINDOWS_CMD_TEMPLATE.format( - python=self._env._bin("python"), script=name - ) + cmd = WINDOWS_CMD_TEMPLATE.format(python=self._env.python, script=name) self._debug( " - Adding the {} script wrapper to {}".format( cmd_script.name, scripts_path @@ -187,19 +191,27 @@ def _add_dist_info(self, added_files): added_files = added_files[:] builder = WheelBuilder(self._poetry) - dist_info = self._env.site_packages.joinpath(builder.dist_info) + + dist_info_path = Path(builder.dist_info) + for dist_info in self._env.site_packages.find( + dist_info_path, writable_only=True + ): + if dist_info.exists(): + self._debug( + " - Removing existing {} directory from {}".format( + dist_info.name, dist_info.parent + ) + ) + shutil.rmtree(str(dist_info)) + + dist_info = self._env.site_packages.mkdir(dist_info_path) self._debug( " - Adding the {} directory to {}".format( - dist_info.name, self._env.site_packages + dist_info.name, dist_info.parent ) ) - if dist_info.exists(): - shutil.rmtree(str(dist_info)) - - dist_info.mkdir() - with dist_info.joinpath("METADATA").open("w", encoding="utf-8") as f: builder._write_metadata_file(f) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 9dd75e66519..f1637407068 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -24,12 +24,14 @@ import poetry.repositories +from poetry.core.packages import dependency_from_pep_508 from poetry.core.packages.package import Dependency from poetry.core.packages.package import Package from poetry.core.semver import parse_constraint from poetry.core.semver.version import Version from poetry.core.toml.file import TOMLFile from poetry.core.version.markers import parse_marker +from poetry.core.version.requirements import InvalidRequirement from poetry.packages import DependencyPackage from poetry.utils._compat import OrderedDict from poetry.utils._compat import Path @@ -142,11 +144,18 @@ def locked_repository( package.extras[name] = [] for dep in deps: - m = re.match(r"^(.+?)(?:\s+\((.+)\))?$", dep) - dep_name = m.group(1) - constraint = m.group(2) or "*" - - package.extras[name].append(Dependency(dep_name, constraint)) + try: + dependency = dependency_from_pep_508(dep) + except InvalidRequirement: + # handle lock files with invalid PEP 508 + m = re.match(r"^(.+?)(?:\[(.+?)])?(?:\s+\((.+)\))?$", dep) + dep_name = m.group(1) + extras = m.group(2) or "" + constraint = m.group(3) or "*" + dependency = Dependency( + dep_name, constraint, extras=extras.split(",") + ) + package.extras[name].append(dependency) if "marker" in info: package.marker = parse_marker(info["marker"]) @@ -217,45 +226,43 @@ def __walk_dependency_level( next_level_dependencies = [] for requirement in dependencies: + key = (requirement.name, requirement.pretty_constraint) locked_package = cls.__get_locked_package(requirement, packages_by_name) if locked_package: - for require in locked_package.requires: - if require.marker.is_empty(): - require.marker = requirement.marker - else: - require.marker = require.marker.intersect(requirement.marker) + # create dependency from locked package to retain dependency metadata + # if this is not done, we can end-up with incorrect nested dependencies + marker = requirement.marker + requirement = locked_package.to_dependency() + requirement.marker = requirement.marker.intersect(marker) + + key = (requirement.name, requirement.pretty_constraint) + + if pinned_versions: + requirement.set_constraint( + locked_package.to_dependency().constraint + ) + + if key not in nested_dependencies: + for require in locked_package.requires: + if require.marker.is_empty(): + require.marker = requirement.marker + else: + require.marker = require.marker.intersect( + requirement.marker + ) - require.marker = require.marker.intersect(locked_package.marker) - next_level_dependencies.append(require) + require.marker = require.marker.intersect(locked_package.marker) + next_level_dependencies.append(require) if requirement.name in project_level_dependencies and level == 0: # project level dependencies take precedence continue - if locked_package: - # create dependency from locked package to retain dependency metadata - # if this is not done, we can end-up with incorrect nested dependencies - marker = requirement.marker - requirement = locked_package.to_dependency() - requirement.marker = requirement.marker.intersect(marker) - else: + if not locked_package: # we make a copy to avoid any side-effects requirement = deepcopy(requirement) - if pinned_versions: - requirement.set_constraint( - cls.__get_locked_package(requirement, packages_by_name) - .to_dependency() - .constraint - ) - - # dependencies use extra to indicate that it was activated via parent - # package's extras, this is not required for nested exports as we assume - # the resolver already selected this dependency - requirement.marker = requirement.marker.without_extras() - - key = (requirement.name, requirement.pretty_constraint) if key not in nested_dependencies: nested_dependencies[key] = requirement else: @@ -543,8 +550,10 @@ def _dump_package(self, package): # type: (Package) -> dict if package.extras: extras = {} for name, deps in package.extras.items(): + # TODO: This should use dep.to_pep_508() once this is fixed + # https://github.com/python-poetry/poetry-core/pull/102 extras[name] = [ - str(dep) if not dep.constraint.is_any() else dep.name + dep.base_pep_508_name if not dep.constraint.is_any() else dep.name for dep in deps ] @@ -573,7 +582,7 @@ def _dump_package(self, package): # type: (Package) -> dict if package.source_resolved_reference: data["source"]["resolved_reference"] = package.source_resolved_reference - if package.source_type == "directory": + if package.source_type in ["directory", "git"]: data["develop"] = package.develop return data diff --git a/poetry/puzzle/provider.py b/poetry/puzzle/provider.py index e2838370a79..c05efbd6844 100644 --- a/poetry/puzzle/provider.py +++ b/poetry/puzzle/provider.py @@ -168,6 +168,7 @@ def search_for_vcs(self, dependency): # type: (VCSDependency) -> List[Package] rev=dependency.rev, name=dependency.name, ) + package.develop = dependency.develop dependency._constraint = package.version dependency._pretty_constraint = package.version.text diff --git a/poetry/repositories/installed_repository.py b/poetry/repositories/installed_repository.py index a0630116f53..1320fdd6698 100644 --- a/poetry/repositories/installed_repository.py +++ b/poetry/repositories/installed_repository.py @@ -138,14 +138,22 @@ def load(cls, env): # type: (Env) -> InstalledRepository if path.name.endswith(".dist-info"): paths = cls.get_package_paths(env=env, name=package.pretty_name) if paths: + is_editable_package = False for src in paths: if cls.is_vcs_package(src, env): cls.set_package_vcs_properties(package, env) break + + if not ( + is_editable_package + or env.is_path_relative_to_lib(src) + ): + is_editable_package = True else: # TODO: handle multiple source directories? - package._source_type = "directory" - package._source_url = paths.pop().as_posix() + if is_editable_package: + package._source_type = "directory" + package._source_url = paths.pop().as_posix() continue if cls.is_vcs_package(path, env): diff --git a/poetry/utils/env.py b/poetry/utils/env.py index 0e3df320d6a..c247bf014f7 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -39,6 +39,8 @@ from poetry.utils._compat import encode from poetry.utils._compat import list_to_shell_command from poetry.utils._compat import subprocess +from poetry.utils.helpers import is_dir_writable +from poetry.utils.helpers import paths_csv GET_ENVIRONMENT_INFO = """\ @@ -143,6 +145,120 @@ def _version_nodot(version): """ +class SitePackages: + def __init__( + self, path, fallbacks=None, skip_write_checks=False + ): # type: (Path, List[Path], bool) -> None + self._path = path + self._fallbacks = fallbacks or [] + self._skip_write_checks = skip_write_checks + self._candidates = [self._path] + self._fallbacks + self._writable_candidates = None if not skip_write_checks else self._candidates + + @property + def path(self): # type: () -> Path + return self._path + + @property + def candidates(self): # type: () -> List[Path] + return self._candidates + + @property + def writable_candidates(self): # type: () -> List[Path] + if self._writable_candidates is not None: + return self._writable_candidates + + self._writable_candidates = [] + for candidate in self._candidates: + if not is_dir_writable(path=candidate, create=True): + continue + self._writable_candidates.append(candidate) + + return self._writable_candidates + + def make_candidates( + self, path, writable_only=False + ): # type: (Path, bool) -> List[Path] + candidates = self._candidates if not writable_only else self.writable_candidates + if path.is_absolute(): + for candidate in candidates: + try: + path.relative_to(candidate) + return [path] + except ValueError: + pass + else: + raise ValueError( + "{} is not relative to any discovered {}sites".format( + path, "writable " if writable_only else "" + ) + ) + + return [candidate / path for candidate in candidates if candidate] + + def _path_method_wrapper( + self, path, method, *args, **kwargs + ): # type: (Path, str, *Any, **Any) -> Union[Tuple[Path, Any], List[Tuple[Path, Any]]] + + # TODO: Move to parameters after dropping Python 2.7 + return_first = kwargs.pop("return_first", True) + writable_only = kwargs.pop("writable_only", False) + + candidates = self.make_candidates(path, writable_only=writable_only) + + if not candidates: + raise RuntimeError( + 'Unable to find a suitable destination for "{}" in {}'.format( + str(path), paths_csv(self._candidates) + ) + ) + + results = [] + + for candidate in candidates: + try: + result = candidate, getattr(candidate, method)(*args, **kwargs) + if return_first: + return result + else: + results.append(result) + except (IOError, OSError): + # TODO: Replace with PermissionError + pass + + if results: + return results + + raise OSError("Unable to access any of {}".format(paths_csv(candidates))) + + def write_text(self, path, *args, **kwargs): # type: (Path, *Any, **Any) -> Path + return self._path_method_wrapper(path, "write_text", *args, **kwargs)[0] + + def mkdir(self, path, *args, **kwargs): # type: (Path, *Any, **Any) -> Path + return self._path_method_wrapper(path, "mkdir", *args, **kwargs)[0] + + def exists(self, path): # type: (Path) -> bool + return any( + value[-1] + for value in self._path_method_wrapper(path, "exists", return_first=False) + ) + + def find(self, path, writable_only=False): # type: (Path, bool) -> List[Path] + return [ + value[0] + for value in self._path_method_wrapper( + path, "exists", return_first=False, writable_only=writable_only + ) + if value[-1] is True + ] + + def __getattr__(self, item): + try: + return super(SitePackages, self).__getattribute__(item) + except AttributeError: + return getattr(self.path, item) + + class EnvError(Exception): pass @@ -771,6 +887,7 @@ def __init__(self, path, base=None): # type: (Path, Optional[Path]) -> None self._supported_tags = None self._purelib = None self._platlib = None + self._script_dirs = None @property def path(self): # type: () -> Path @@ -825,9 +942,13 @@ def pip_version(self): return self._pip_version @property - def site_packages(self): # type: () -> Path + def site_packages(self): # type: () -> SitePackages if self._site_packages is None: - self._site_packages = self.purelib + # we disable write checks if no user site exist + fallbacks = [self.usersite] if self.usersite else [] + self._site_packages = SitePackages( + self.purelib, fallbacks, skip_write_checks=False if fallbacks else True + ) return self._site_packages @property @@ -835,6 +956,11 @@ def usersite(self): # type: () -> Optional[Path] if "usersite" in self.paths: return Path(self.paths["usersite"]) + @property + def userbase(self): # type: () -> Optional[Path] + if "userbase" in self.paths: + return Path(self.paths["userbase"]) + @property def purelib(self): # type: () -> Path if self._purelib is None: @@ -981,6 +1107,18 @@ def execute(self, bin, *args, **kwargs): def is_venv(self): # type: () -> bool raise NotImplementedError() + @property + def script_dirs(self): # type: () -> List[Path] + if self._script_dirs is None: + self._script_dirs = ( + [Path(self.paths["scripts"])] + if "scripts" in self.paths + else self._bin_dir + ) + if self.userbase: + self._script_dirs.append(self.userbase / self._script_dirs[0].name) + return self._script_dirs + def _bin(self, bin): # type: (str) -> str """ Return path to the given executable. @@ -1016,6 +1154,10 @@ class SystemEnv(Env): A system (i.e. not a virtualenv) Python environment. """ + @property + def python(self): # type: () -> str + return sys.executable + @property def sys_path(self): # type: () -> List[str] return sys.path @@ -1056,6 +1198,7 @@ def get_paths(self): # type: () -> Dict[str, str] if site.check_enableusersite() and hasattr(obj, "install_usersite"): paths["usersite"] = getattr(obj, "install_usersite") + paths["userbase"] = getattr(obj, "install_userbase") return paths @@ -1191,7 +1334,7 @@ def is_venv(self): # type: () -> bool def is_sane(self): # A virtualenv is considered sane if both "python" and "pip" exist. - return os.path.exists(self._bin("python")) and os.path.exists(self._bin("pip")) + return os.path.exists(self.python) and os.path.exists(self._bin("pip")) def _run(self, cmd, **kwargs): with self.temp_environ(): diff --git a/poetry/utils/helpers.py b/poetry/utils/helpers.py index 180a90d50f3..232e65b7d44 100644 --- a/poetry/utils/helpers.py +++ b/poetry/utils/helpers.py @@ -5,6 +5,7 @@ import tempfile from contextlib import contextmanager +from typing import List from typing import Optional import requests @@ -113,3 +114,22 @@ def get_package_version_display_string( ) return package.full_pretty_version + + +def paths_csv(paths): # type: (List[Path]) -> str + return ", ".join('"{}"'.format(str(c)) for c in paths) + + +def is_dir_writable(path, create=False): # type: (Path, bool) -> bool + try: + if not path.exists(): + if not create: + return False + path.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryFile(dir=str(path)): + pass + except (IOError, OSError): + return False + else: + return True diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 07373ade0b4..4bd0cd048da 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -1,16 +1,29 @@ import os +import pytest -def test_config_get_default_value(config): - assert config.get("virtualenvs.create") is True + +@pytest.mark.parametrize( + ("name", "value"), [("installer.parallel", True), ("virtualenvs.create", True)] +) +def test_config_get_default_value(config, name, value): + assert config.get(name) is value def test_config_get_processes_depended_on_values(config): assert os.path.join("/foo", "virtualenvs") == config.get("virtualenvs.path") -def test_config_get_from_environment_variable(config, environ): - assert config.get("virtualenvs.create") - - os.environ["POETRY_VIRTUALENVS_CREATE"] = "false" - assert not config.get("virtualenvs.create") +@pytest.mark.parametrize( + ("name", "env_value", "value"), + [ + ("installer.parallel", "true", True), + ("installer.parallel", "false", False), + ("virtualenvs.create", "true", True), + ("virtualenvs.create", "false", False), + ], +) +def test_config_get_from_environment_variable(config, environ, name, env_value, value): + env_var = "POETRY_{}".format("_".join(k.upper() for k in name.split("."))) + os.environ[env_var] = env_value + assert config.get(name) is value diff --git a/tests/console/commands/test_config.py b/tests/console/commands/test_config.py index 9923f1b77af..4d0b9ada0c8 100644 --- a/tests/console/commands/test_config.py +++ b/tests/console/commands/test_config.py @@ -6,6 +6,8 @@ from poetry.config.config_source import ConfigSource from poetry.core.pyproject import PyProjectException from poetry.factory import Factory +from poetry.utils._compat import PY2 +from poetry.utils._compat import WINDOWS @pytest.fixture() @@ -28,6 +30,7 @@ def test_list_displays_default_value_if_not_set(tester, config): expected = """cache-dir = "/foo" experimental.new-installer = true +installer.parallel = true virtualenvs.create = true virtualenvs.in-project = null virtualenvs.options.always-copy = false @@ -46,6 +49,7 @@ def test_list_displays_set_get_setting(tester, config): expected = """cache-dir = "/foo" experimental.new-installer = true +installer.parallel = true virtualenvs.create = false virtualenvs.in-project = null virtualenvs.options.always-copy = false @@ -86,6 +90,7 @@ def test_list_displays_set_get_local_setting(tester, config): expected = """cache-dir = "/foo" experimental.new-installer = true +installer.parallel = true virtualenvs.create = false virtualenvs.in-project = null virtualenvs.options.always-copy = false @@ -122,3 +127,25 @@ def test_set_cert(tester, auth_config_source, mocker): tester.execute("certificates.foo.cert path/to/ca.pem") assert "path/to/ca.pem" == auth_config_source.config["certificates"]["foo"]["cert"] + + +def test_config_installer_parallel(tester, command_tester_factory): + serial_enforced = PY2 and WINDOWS + + tester.execute("--local installer.parallel") + assert tester.io.fetch_output().strip() == "true" + + workers = command_tester_factory( + "install" + )._command._installer._executor._max_workers + assert workers > 1 or (serial_enforced and workers == 1) + + tester.io.clear_output() + tester.execute("--local installer.parallel false") + tester.execute("--local installer.parallel") + assert tester.io.fetch_output().strip() == "false" + + workers = command_tester_factory( + "install" + )._command._installer._executor._max_workers + assert workers == 1 diff --git a/tests/fixtures/with_non_default_source/pyproject.toml b/tests/fixtures/with_non_default_source/pyproject.toml new file mode 100644 index 00000000000..d36abb55a25 --- /dev/null +++ b/tests/fixtures/with_non_default_source/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "my-package" +version = "1.2.3" +description = "Some description." +authors = [ + "Your Name " +] +license = "MIT" + +# Requirements +[tool.poetry.dependencies] +python = "~2.7 || ^3.6" + +[tool.poetry.dev-dependencies] + +[[tool.poetry.source]] +name = "foo" +url = "https://foo.bar/simple/" diff --git a/tests/installation/fixtures/with-dependencies-extras.test b/tests/installation/fixtures/with-dependencies-extras.test index 63560bb4793..042e29670e1 100644 --- a/tests/installation/fixtures/with-dependencies-extras.test +++ b/tests/installation/fixtures/with-dependencies-extras.test @@ -18,7 +18,7 @@ python-versions = "*" C = {version = "^1.0", optional = true} [package.extras] -foo = ["C (^1.0)"] +foo = ["C (>=1.0,<2.0)"] [[package]] name = "C" diff --git a/tests/installation/fixtures/with-dependencies-nested-extras.test b/tests/installation/fixtures/with-dependencies-nested-extras.test new file mode 100644 index 00000000000..48a22a7c7f3 --- /dev/null +++ b/tests/installation/fixtures/with-dependencies-nested-extras.test @@ -0,0 +1,45 @@ +[[package]] +name = "A" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +B = {version = "^1.0", optional = true, extras = ["C"]} + +[package.extras] +B = ["B[C] (>=1.0,<2.0)"] + +[[package]] +name = "B" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +C = {version = "^1.0", optional = true} + +[package.extras] +C = ["C (>=1.0,<2.0)"] + +[[package]] +name = "C" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[metadata] +python-versions = "*" +lock-version = "1.1" +content-hash = "123456789" + +[metadata.files] +"A" = [] +"B" = [] +"C" = [] diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index 7a1660670ee..106efde6e9c 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -639,6 +639,35 @@ def test_run_with_dependencies_extras(installer, locker, repo, package): assert locker.written_data == expected +def test_run_with_dependencies_nested_extras(installer, locker, repo, package): + package_a = get_package("A", "1.0") + package_b = get_package("B", "1.0") + package_c = get_package("C", "1.0") + + dependency_c = Factory.create_dependency("C", {"version": "^1.0", "optional": True}) + dependency_b = Factory.create_dependency( + "B", {"version": "^1.0", "optional": True, "extras": ["C"]} + ) + dependency_a = Factory.create_dependency("A", {"version": "^1.0", "extras": ["B"]}) + + package_b.extras = {"C": [dependency_c]} + package_b.add_dependency(dependency_c) + + package_a.add_dependency(dependency_b) + package_a.extras = {"B": [dependency_b]} + + repo.add_package(package_a) + repo.add_package(package_b) + repo.add_package(package_c) + + package.add_dependency(dependency_a) + + installer.run() + expected = fixture("with-dependencies-nested-extras") + + assert locker.written_data == expected + + def test_run_does_not_install_extras_if_not_requested(installer, locker, repo, package): package.extras["foo"] = [get_dependency("D")] package_a = get_package("A", "1.0") diff --git a/tests/installation/test_pip_installer.py b/tests/installation/test_pip_installer.py index 206bcce59cc..d0e2e5a4dcd 100644 --- a/tests/installation/test_pip_installer.py +++ b/tests/installation/test_pip_installer.py @@ -189,7 +189,9 @@ def test_uninstall_git_package_nspkg_pth_cleanup(mocker, tmp_venv, pool): ) # we do this here because the virtual env might not be usable if failure case is triggered - pth_file_candidate = tmp_venv.site_packages / "{}-nspkg.pth".format(package.name) + pth_file_candidate = tmp_venv.site_packages.path / "{}-nspkg.pth".format( + package.name + ) # in order to reproduce the scenario where the git source is removed prior to proper # clean up of nspkg.pth file, we need to make sure the fixture is copied and not diff --git a/tests/masonry/builders/test_editable_builder.py b/tests/masonry/builders/test_editable_builder.py index 3aee74e7c77..daeff0e7777 100644 --- a/tests/masonry/builders/test_editable_builder.py +++ b/tests/masonry/builders/test_editable_builder.py @@ -76,14 +76,14 @@ def test_builder_installs_proper_files_for_standard_packages(simple_poetry, tmp_ builder.build() assert tmp_venv._bin_dir.joinpath("foo").exists() - assert tmp_venv.site_packages.joinpath("simple_project.pth").exists() - assert simple_poetry.file.parent.resolve().as_posix() == tmp_venv.site_packages.joinpath( + assert tmp_venv.site_packages.path.joinpath("simple_project.pth").exists() + assert simple_poetry.file.parent.resolve().as_posix() == tmp_venv.site_packages.path.joinpath( "simple_project.pth" ).read_text().strip( os.linesep ) - dist_info = tmp_venv.site_packages.joinpath("simple_project-1.2.3.dist-info") + dist_info = tmp_venv.site_packages.path.joinpath("simple_project-1.2.3.dist-info") assert dist_info.exists() assert dist_info.joinpath("INSTALLER").exists() assert dist_info.joinpath("METADATA").exists() @@ -130,7 +130,7 @@ def test_builder_installs_proper_files_for_standard_packages(simple_poetry, tmp_ assert metadata == dist_info.joinpath("METADATA").read_text(encoding="utf-8") records = dist_info.joinpath("RECORD").read_text() - assert str(tmp_venv.site_packages.joinpath("simple_project.pth")) in records + assert str(tmp_venv.site_packages.path.joinpath("simple_project.pth")) in records assert str(tmp_venv._bin_dir.joinpath("foo")) in records assert str(tmp_venv._bin_dir.joinpath("baz")) in records assert str(dist_info.joinpath("METADATA")) in records @@ -202,7 +202,7 @@ def test_builder_installs_proper_files_when_packages_configured( builder = EditableBuilder(project_with_include, tmp_venv, NullIO()) builder.build() - pth_file = tmp_venv.site_packages.joinpath("with_include.pth") + pth_file = tmp_venv.site_packages.path.joinpath("with_include.pth") assert pth_file.is_file() paths = set() diff --git a/tests/packages/test_locker.py b/tests/packages/test_locker.py index 35c584f2767..7fa89a44962 100644 --- a/tests/packages/test_locker.py +++ b/tests/packages/test_locker.py @@ -73,6 +73,7 @@ def test_lock_file_data_is_ordered(locker, root): category = "main" optional = false python-versions = "*" +develop = true [package.source] type = "git" @@ -142,6 +143,136 @@ def test_locker_properly_loads_extras(locker): assert lockfile_dep.name == "lockfile" +def test_locker_properly_loads_nested_extras(locker): + content = """\ +[[package]] +name = "a" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +b = {version = "^1.0", optional = true, extras = "c"} + +[package.extras] +b = ["b[c] (>=1.0,<2.0)"] + +[[package]] +name = "b" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +c = {version = "^1.0", optional = true} + +[package.extras] +c = ["c (>=1.0,<2.0)"] + +[[package]] +name = "c" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[metadata] +python-versions = "*" +lock-version = "1.1" +content-hash = "123456789" + +[metadata.files] +"a" = [] +"b" = [] +"c" = [] +""" + + locker.lock.write(tomlkit.parse(content)) + + repository = locker.locked_repository() + assert 3 == len(repository.packages) + + packages = repository.find_packages(get_dependency("a", "1.0")) + assert len(packages) == 1 + + package = packages[0] + assert len(package.requires) == 1 + assert len(package.extras) == 1 + + dependency_b = package.extras["b"][0] + assert dependency_b.name == "b" + assert dependency_b.extras == frozenset({"c"}) + + packages = repository.find_packages(dependency_b) + assert len(packages) == 1 + + package = packages[0] + assert len(package.requires) == 1 + assert len(package.extras) == 1 + + dependency_c = package.extras["c"][0] + assert dependency_c.name == "c" + assert dependency_c.extras == frozenset() + + packages = repository.find_packages(dependency_c) + assert len(packages) == 1 + + +def test_locker_properly_loads_extras_legacy(locker): + content = """\ +[[package]] +name = "a" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +b = {version = "^1.0", optional = true} + +[package.extras] +b = ["b (^1.0)"] + +[[package]] +name = "b" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" + +[metadata] +python-versions = "*" +lock-version = "1.1" +content-hash = "123456789" + +[metadata.files] +"a" = [] +"b" = [] +""" + + locker.lock.write(tomlkit.parse(content)) + + repository = locker.locked_repository() + assert 2 == len(repository.packages) + + packages = repository.find_packages(get_dependency("a", "1.0")) + assert len(packages) == 1 + + package = packages[0] + assert len(package.requires) == 1 + assert len(package.extras) == 1 + + dependency_b = package.extras["b"][0] + assert dependency_b.name == "b" + + def test_lock_packages_with_null_description(locker, root): package_a = get_package("A", "1.0.0") package_a.description = None diff --git a/tests/puzzle/test_provider.py b/tests/puzzle/test_provider.py index 693470156f4..ecab7f3ab2d 100644 --- a/tests/puzzle/test_provider.py +++ b/tests/puzzle/test_provider.py @@ -47,6 +47,15 @@ def provider(root, pool): return Provider(root, pool, NullIO()) +@pytest.mark.parametrize("value", [True, False]) +def test_search_for_vcs_retains_develop_flag(provider, value): + dependency = VCSDependency( + "demo", "git", "https://github.com/demo/demo.git", develop=value + ) + package = provider.search_for_vcs(dependency)[0] + assert package.develop == value + + def test_search_for_vcs_setup_egg_info(provider): dependency = VCSDependency("demo", "git", "https://github.com/demo/demo.git") diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard-1.2.3.dist-info/METADATA b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard-1.2.3.dist-info/METADATA new file mode 100644 index 00000000000..245121d496d --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard-1.2.3.dist-info/METADATA @@ -0,0 +1,22 @@ +Metadata-Version: 2.1 +Name: standard +Version: 1.2.3 +Summary: Standard description. +License: MIT +Keywords: cli,commands +Author: Foo Bar +Author-email: foo@bar.com +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Description-Content-Type: text/x-rst + +Editable +#### diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard.pth b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard.pth new file mode 100644 index 00000000000..aa0bc074b62 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/standard.pth @@ -0,0 +1 @@ +standard \ No newline at end of file diff --git a/tests/repositories/test_installed_repository.py b/tests/repositories/test_installed_repository.py index 79ac262671c..3caa702a09c 100644 --- a/tests/repositories/test_installed_repository.py +++ b/tests/repositories/test_installed_repository.py @@ -26,6 +26,7 @@ zipp.Path(str(SITE_PURELIB / "foo-0.1.0-py3.8.egg"), "EGG-INFO") ), metadata.PathDistribution(VENDOR_DIR / "attrs-19.3.0.dist-info"), + metadata.PathDistribution(SITE_PURELIB / "standard-1.2.3.dist-info"), metadata.PathDistribution(SITE_PURELIB / "editable-2.3.4.dist-info"), metadata.PathDistribution(SITE_PURELIB / "editable-with-import-2.3.4.dist-info"), metadata.PathDistribution(SITE_PLATLIB / "lib64-2.3.4.dist-info"), @@ -158,3 +159,13 @@ def test_load_editable_with_import_package(repository): assert editable.version.text == "2.3.4" assert editable.source_type is None assert editable.source_url is None + + +def test_load_standard_package_with_pth_file(repository): + # test standard packages with .pth file is not treated as editable + standard = get_package_from_repository("standard", repository) + assert standard is not None + assert standard.name == "standard" + assert standard.version.text == "1.2.3" + assert standard.source_type is None + assert standard.source_url is None diff --git a/tests/test_factory.py b/tests/test_factory.py index d7758e8deaa..b2c232b20e7 100644 --- a/tests/test_factory.py +++ b/tests/test_factory.py @@ -6,6 +6,8 @@ from poetry.core.toml.file import TOMLFile from poetry.factory import Factory +from poetry.repositories.legacy_repository import LegacyRepository +from poetry.repositories.pypi_repository import PyPiRepository from poetry.utils._compat import PY2 from poetry.utils._compat import Path @@ -150,6 +152,31 @@ def test_poetry_with_default_source(): assert 1 == len(poetry.pool.repositories) +def test_poetry_with_non_default_source(): + poetry = Factory().create_poetry(fixtures_dir / "with_non_default_source") + + assert len(poetry.pool.repositories) == 2 + + assert not poetry.pool.has_default() + + assert poetry.pool.repositories[0].name == "foo" + assert isinstance(poetry.pool.repositories[0], LegacyRepository) + + assert poetry.pool.repositories[1].name == "PyPI" + assert isinstance(poetry.pool.repositories[1], PyPiRepository) + + +def test_poetry_with_no_default_source(): + poetry = Factory().create_poetry(fixtures_dir / "sample_project") + + assert len(poetry.pool.repositories) == 1 + + assert poetry.pool.has_default() + + assert poetry.pool.repositories[0].name == "PyPI" + assert isinstance(poetry.pool.repositories[0], PyPiRepository) + + def test_poetry_with_two_default_sources(): with pytest.raises(ValueError) as e: Factory().create_poetry(fixtures_dir / "with_two_default_sources") diff --git a/tests/utils/test_env.py b/tests/utils/test_env.py index 8c816a2d32a..8934ae43427 100644 --- a/tests/utils/test_env.py +++ b/tests/utils/test_env.py @@ -866,7 +866,7 @@ def test_system_env_has_correct_paths(): assert paths.get("purelib") is not None assert paths.get("platlib") is not None assert paths.get("scripts") is not None - assert env.site_packages == Path(paths["purelib"]) + assert env.site_packages.path == Path(paths["purelib"]) @pytest.mark.parametrize( @@ -886,4 +886,4 @@ def test_venv_has_correct_paths(tmp_venv): assert paths.get("purelib") is not None assert paths.get("platlib") is not None assert paths.get("scripts") is not None - assert tmp_venv.site_packages == Path(paths["purelib"]) + assert tmp_venv.site_packages.path == Path(paths["purelib"]) diff --git a/tests/utils/test_env_site.py b/tests/utils/test_env_site.py new file mode 100644 index 00000000000..f25e2142193 --- /dev/null +++ b/tests/utils/test_env_site.py @@ -0,0 +1,43 @@ +import uuid + +from poetry.utils._compat import Path +from poetry.utils._compat import decode +from poetry.utils.env import SitePackages + + +def test_env_site_simple(tmp_dir, mocker): + # emulate permission error when creating directory + mocker.patch("poetry.utils._compat.Path.mkdir", side_effect=OSError()) + site_packages = SitePackages(Path("/non-existent"), fallbacks=[Path(tmp_dir)]) + candidates = site_packages.make_candidates(Path("hello.txt"), writable_only=True) + hello = Path(tmp_dir) / "hello.txt" + + assert len(candidates) == 1 + assert candidates[0].as_posix() == hello.as_posix() + + content = decode(str(uuid.uuid4())) + site_packages.write_text(Path("hello.txt"), content, encoding="utf-8") + + assert hello.read_text(encoding="utf-8") == content + + assert not (site_packages.path / "hello.txt").exists() + + +def test_env_site_select_first(tmp_dir): + path = Path(tmp_dir) + fallback = path / "fallback" + fallback.mkdir(parents=True) + + site_packages = SitePackages(path, fallbacks=[fallback]) + candidates = site_packages.make_candidates(Path("hello.txt"), writable_only=True) + + assert len(candidates) == 2 + assert len(site_packages.find(Path("hello.txt"))) == 0 + + content = decode(str(uuid.uuid4())) + site_packages.write_text(Path("hello.txt"), content, encoding="utf-8") + + assert (site_packages.path / "hello.txt").exists() + assert not (fallback / "hello.txt").exists() + + assert len(site_packages.find(Path("hello.txt"))) == 1 diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index c5ce0214ff7..1e660ce69cd 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -696,6 +696,62 @@ def test_exporter_can_export_requirements_txt_with_nested_packages(tmp_dir, poet assert expected == content +def test_exporter_can_export_requirements_txt_with_nested_packages_cyclic( + tmp_dir, poetry +): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "foo", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + "dependencies": {"bar": {"version": "4.5.6"}}, + }, + { + "name": "bar", + "version": "4.5.6", + "category": "main", + "optional": False, + "python-versions": "*", + "dependencies": {"baz": {"version": "7.8.9"}}, + }, + { + "name": "baz", + "version": "7.8.9", + "category": "main", + "optional": False, + "python-versions": "*", + "dependencies": {"foo": {"version": "1.2.3"}}, + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"foo": [], "bar": [], "baz": []}, + }, + } + ) + set_package_requires(poetry, skip={"bar", "baz"}) + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + expected = """\ +bar==4.5.6 +baz==7.8.9 +foo==1.2.3 +""" + + assert expected == content + + def test_exporter_can_export_requirements_txt_with_git_packages_and_markers( tmp_dir, poetry ):