From 2810d2b203becd608dacf654b3381d40933b84aa Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Thu, 8 Oct 2020 16:46:23 +0200 Subject: [PATCH 01/12] inspection: ad each extra package as a dependency Resolves: #3129 (cherry picked from commit 38fddf05b36ec64647e66bf459578f9e0b41442d) --- poetry/inspection/info.py | 7 ++++++- tests/repositories/test_legacy_repository.py | 4 ++-- tests/repositories/test_pypi_repository.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/poetry/inspection/info.py b/poetry/inspection/info.py index 73034de16d3..251dac95f91 100644 --- a/poetry/inspection/info.py +++ b/poetry/inspection/info.py @@ -165,6 +165,8 @@ def to_package( package.requires = poetry_package.requires return package + seen_requirements = set() + for req in self.requires_dist or []: try: # Attempt to parse the PEP-508 requirement string @@ -191,8 +193,11 @@ def to_package( package.extras[extra].append(dependency) - if dependency not in package.requires: + req = dependency.to_pep_508(with_extras=True) + + if req not in seen_requirements: package.requires.append(dependency) + seen_requirements.add(req) return package diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py index 7fd131c70e4..47ccc1052f9 100644 --- a/tests/repositories/test_legacy_repository.py +++ b/tests/repositories/test_legacy_repository.py @@ -117,7 +117,7 @@ def test_get_package_information_skips_dependencies_with_invalid_constraints(): package.description == "Python Language Server for the Language Server Protocol" ) - assert 19 == len(package.requires) + assert 25 == len(package.requires) assert sorted( [r for r in package.requires if not r.is_optional()], key=lambda r: r.name ) == [ @@ -216,7 +216,7 @@ def test_get_package_from_both_py2_and_py3_specific_wheels(): assert "ipython" == package.name assert "5.7.0" == package.version.text assert "*" == package.python_versions - assert 26 == len(package.requires) + assert 41 == len(package.requires) expected = [ Dependency("appnope", "*"), diff --git a/tests/repositories/test_pypi_repository.py b/tests/repositories/test_pypi_repository.py index ef094f3f1d5..55afdd39485 100644 --- a/tests/repositories/test_pypi_repository.py +++ b/tests/repositories/test_pypi_repository.py @@ -165,7 +165,7 @@ def test_pypi_repository_supports_reading_bz2_files(): package = repo.package("twisted", "18.9.0") assert package.name == "twisted" - assert 28 == len(package.requires) + assert 71 == len(package.requires) assert sorted( [r for r in package.requires if not r.is_optional()], key=lambda r: r.name ) == [ From 8d0afdc7d7e13a00c88107d00be2ce52c576f72e Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Wed, 7 Oct 2020 15:36:30 +0200 Subject: [PATCH 02/12] show: ignore dependency source when finding package Resolves: #3116 (cherry picked from commit 50e06289a40819afe68f9b9cfecd7ccf08763476) --- poetry/console/commands/show.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry/console/commands/show.py b/poetry/console/commands/show.py index 808122e64dc..f6f42f42654 100644 --- a/poetry/console/commands/show.py +++ b/poetry/console/commands/show.py @@ -68,7 +68,7 @@ def handle(self): table = self.table(style="compact") # table.style.line_vc_char = "" locked_packages = locked_repo.packages - pool = Pool() + pool = Pool(ignore_repository_names=True) pool.add_repository(locked_repo) solver = Solver( self.poetry.package, From b30dab75b2becbeeb886ce91136689bd4dc50399 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Wed, 7 Oct 2020 15:51:26 +0200 Subject: [PATCH 03/12] locker: move export logic into locker for reuse This change, moves common functionality from exporter into the locker to allow for sharing of logic that can generate `DependencyPackage` instances given a a list of requirements using the lock data. (cherry picked from commit c2adb3283c0b76e290a8f25053bf035500b47cb2) --- poetry/packages/locker.py | 44 +++++++++++++++++++++++++++++++++++++-- poetry/utils/exporter.py | 34 +++++++----------------------- 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 2283a2373d2..a486bf1c5d6 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -6,9 +6,11 @@ from copy import deepcopy from hashlib import sha256 -from typing import Any +from typing import Iterator from typing import List from typing import Optional +from typing import Sequence +from typing import Union from tomlkit import array from tomlkit import document @@ -25,8 +27,10 @@ from poetry.core.semver.version import Version from poetry.core.toml.file import TOMLFile from poetry.core.version.markers import parse_marker +from poetry.packages import DependencyPackage from poetry.utils._compat import OrderedDict from poetry.utils._compat import Path +from poetry.utils.extras import get_extra_package_names logger = logging.getLogger(__name__) @@ -183,7 +187,7 @@ def locked_repository( def get_project_dependencies( self, project_requires, pinned_versions=False, with_nested=False, with_dev=False - ): # type: (List[Dependency], bool, bool, bool) -> Any + ): # type: (List[Dependency], bool, bool, bool) -> List[Dependency] packages = self.locked_repository(with_dev).packages # group packages entries by name, this is required because requirement might use @@ -263,6 +267,42 @@ def __get_locked_package( key=lambda x: x.name.lower(), ) + def get_project_dependency_packages( + self, project_requires, dev=False, extras=None + ): # type: (List[Dependency], bool, Optional[Union[bool, Sequence[str]]]) -> Iterator[DependencyPackage] + repository = self.locked_repository(with_dev_reqs=dev) + + # Build a set of all packages required by our selected extras + extra_package_names = ( + None if (isinstance(extras, bool) and extras is True) else () + ) + + if extra_package_names is not None: + extra_package_names = set( + get_extra_package_names( + repository.packages, self.lock_data.get("extras", {}), extras or (), + ) + ) + + for dependency in self.get_project_dependencies( + project_requires=project_requires, with_nested=True, with_dev=dev, + ): + try: + package = repository.find_packages(dependency=dependency)[0] + except IndexError: + continue + + # If a package is optional and we haven't opted in to it, continue + if extra_package_names is not None and ( + package.optional and package.name not in extra_package_names + ): + continue + + for extra in dependency.extras: + package.requires_extras.append(extra) + + yield DependencyPackage(dependency=dependency, package=package) + def set_lock_data(self, root, packages): # type: (...) -> bool files = table() packages = self._lock_packages(packages) diff --git a/poetry/utils/exporter.py b/poetry/utils/exporter.py index 4200abd146d..ae85c29cc0b 100644 --- a/poetry/utils/exporter.py +++ b/poetry/utils/exporter.py @@ -1,3 +1,5 @@ +from typing import Optional +from typing import Sequence from typing import Union from clikit.api.io import IO @@ -5,7 +7,6 @@ from poetry.poetry import Poetry from poetry.utils._compat import Path from poetry.utils._compat import decode -from poetry.utils.extras import get_extra_package_names class Exporter(object): @@ -51,38 +52,19 @@ def _export_requirements_txt( dev=False, extras=None, with_credentials=False, - ): # type: (Path, Union[IO, str], bool, bool, bool) -> None + ): # type: (Path, Union[IO, str], bool, bool, Optional[Union[bool, Sequence[str]]], bool) -> None indexes = set() content = "" - repository = self._poetry.locker.locked_repository(dev) - - # Build a set of all packages required by our selected extras - extra_package_names = set( - get_extra_package_names( - repository.packages, - self._poetry.locker.lock_data.get("extras", {}), - extras or (), - ) - ) - dependency_lines = set() - for dependency in self._poetry.locker.get_project_dependencies( - project_requires=self._poetry.package.all_requires, - with_nested=True, - with_dev=dev, + for dependency_package in self._poetry.locker.get_project_dependency_packages( + project_requires=self._poetry.package.all_requires, dev=dev, extras=extras ): - try: - package = repository.find_packages(dependency=dependency)[0] - except IndexError: - continue - - # If a package is optional and we haven't opted in to it, continue - if package.optional and package.name not in extra_package_names: - continue - line = "" + dependency = dependency_package.dependency + package = dependency_package.package + if package.develop: line += "-e " From 2974d402d243ed8d030ae61a1b52b2dbf47fa4c9 Mon Sep 17 00:00:00 2001 From: "Yury V. Zaytsev" Date: Wed, 7 Oct 2020 17:54:40 +0200 Subject: [PATCH 04/12] locker: reuse locked metadata for nested deps Resolves: #3115 (cherry picked from commit 04967db7ab20558b593e6b47ef5258e95a1b642f) --- poetry/packages/locker.py | 11 +++++-- tests/utils/test_exporter.py | 60 ++++++++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index a486bf1c5d6..2a82fef816a 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -234,8 +234,15 @@ def __get_locked_package( # project level dependencies take precedence continue - # we make a copy to avoid any side-effects - requirement = deepcopy(requirement) + locked_package = __get_locked_package(requirement) + if locked_package: + # create dependency from locked package to retain dependency metadata + # if this is not done, we can end-up with incorrect nested dependencies + requirement = locked_package.to_dependency() + else: + # we make a copy to avoid any side-effects + requirement = deepcopy(requirement) + requirement._category = pkg.category if pinned_versions: diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index e26f448f6de..a75fb3da502 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -59,13 +59,18 @@ def poetry(fixture_dir, locker): return p -def set_package_requires(poetry): +def set_package_requires(poetry, skip=None): + skip = skip or set() packages = poetry.locker.locked_repository(with_dev_reqs=True).packages poetry.package.requires = [ - pkg.to_dependency() for pkg in packages if pkg.category == "main" + pkg.to_dependency() + for pkg in packages + if pkg.category == "main" and pkg.name not in skip ] poetry.package.dev_requires = [ - pkg.to_dependency() for pkg in packages if pkg.category == "dev" + pkg.to_dependency() + for pkg in packages + if pkg.category == "dev" and pkg.name not in skip ] @@ -503,6 +508,55 @@ def test_exporter_can_export_requirements_txt_with_git_packages(tmp_dir, poetry) assert expected == content +def test_exporter_can_export_requirements_txt_with_nested_packages(tmp_dir, poetry): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "foo", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + "source": { + "type": "git", + "url": "https://github.com/foo/foo.git", + "reference": "123456", + }, + }, + { + "name": "bar", + "version": "4.5.6", + "category": "main", + "optional": False, + "python-versions": "*", + "dependencies": {"foo": "rev 123456"}, + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"foo": [], "bar": []}, + }, + } + ) + set_package_requires(poetry, skip={"foo"}) + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + expected = """\ +bar==4.5.6 +foo @ git+https://github.com/foo/foo.git@123456 +""" + + assert expected == content + + def test_exporter_can_export_requirements_txt_with_git_packages_and_markers( tmp_dir, poetry ): From 88b13134daaac3f36c81e741dc858e979841e444 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 9 Oct 2020 06:34:43 +0200 Subject: [PATCH 05/12] locker: propagate cumulative markers to nested deps This change ensures that markers are propagated from top level dependencies to the deepest level by walking top to bottom instead of iterating over all available packages. In addition, we also compress any dependencies with the same name and constraint to provide a more concise representation. Resolves: #3112 #3160 (cherry picked from commit e78a67ba139c70ee1856834711ddaf14de0c926a) --- poetry/packages/locker.py | 86 ++++++++++++++------- tests/utils/test_exporter.py | 140 +++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 27 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 2a82fef816a..26c2b584504 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -215,10 +215,18 @@ def __get_locked_package( for dependency in project_requires: dependency = deepcopy(dependency) - if pinned_versions: - locked_package = __get_locked_package(dependency) - if locked_package: - dependency.set_constraint(locked_package.to_dependency().constraint) + locked_package = __get_locked_package(dependency) + if locked_package: + locked_dependency = locked_package.to_dependency() + locked_dependency.marker = dependency.marker.intersect( + locked_package.marker + ) + + if not pinned_versions: + locked_dependency.set_constraint(dependency.constraint) + + dependency = locked_dependency + project_level_dependencies.add(dependency.name) dependencies.append(dependency) @@ -226,19 +234,43 @@ def __get_locked_package( # return only with project level dependencies return dependencies - nested_dependencies = list() + nested_dependencies = dict() - for pkg in packages: # type: Package - for requirement in pkg.requires: # type: Dependency - if requirement.name in project_level_dependencies: + def __walk_level( + __dependencies, __level + ): # type: (List[Dependency], int) -> None + if not __dependencies: + return + + __next_level = [] + + for requirement in __dependencies: + __locked_package = __get_locked_package(requirement) + + if __locked_package: + for require in __locked_package.requires: + if require.marker.is_empty(): + require.marker = requirement.marker + else: + require.marker = require.marker.intersect( + requirement.marker + ) + + require.marker = require.marker.intersect( + __locked_package.marker + ) + __next_level.append(require) + + if requirement.name in project_level_dependencies and __level == 0: # project level dependencies take precedence continue - locked_package = __get_locked_package(requirement) - if locked_package: + if __locked_package: # create dependency from locked package to retain dependency metadata # if this is not done, we can end-up with incorrect nested dependencies - requirement = locked_package.to_dependency() + marker = requirement.marker + requirement = __locked_package.to_dependency() + requirement.marker = requirement.marker.intersect(marker) else: # we make a copy to avoid any side-effects requirement = deepcopy(requirement) @@ -251,26 +283,26 @@ def __get_locked_package( ) # dependencies use extra to indicate that it was activated via parent - # package's extras - marker = requirement.marker.without_extras() - for project_requirement in project_requires: - if ( - pkg.name == project_requirement.name - and project_requirement.constraint.allows(pkg.version) - ): - requirement.marker = marker.intersect( - project_requirement.marker - ) - break + # package's extras, this is not required for nested exports as we assume + # the resolver already selected this dependency + requirement.marker = requirement.marker.without_extras().intersect( + pkg.marker + ) + + key = (requirement.name, requirement.pretty_constraint) + if key not in nested_dependencies: + nested_dependencies[key] = requirement else: - # this dependency was not from a project requirement - requirement.marker = marker.intersect(pkg.marker) + nested_dependencies[key].marker = nested_dependencies[ + key + ].marker.intersect(requirement.marker) + + return __walk_level(__next_level, __level + 1) - if requirement not in nested_dependencies: - nested_dependencies.append(requirement) + __walk_level(dependencies, 0) return sorted( - itertools.chain(dependencies, nested_dependencies), + itertools.chain(dependencies, nested_dependencies.values()), key=lambda x: x.name.lower(), ) diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index a75fb3da502..66b9e81ff53 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -2,6 +2,7 @@ import pytest +from poetry.core.packages import dependency_from_pep_508 from poetry.core.toml.file import TOMLFile from poetry.factory import Factory from poetry.packages import Locker as BaseLocker @@ -175,6 +176,145 @@ def test_exporter_can_export_requirements_txt_with_standard_packages_and_markers assert expected == content +def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers( + tmp_dir, poetry +): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "a", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "python_version < '3.7'", + "dependencies": {"b": ">=0.0.0", "c": ">=0.0.0"}, + }, + { + "name": "b", + "version": "4.5.6", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "platform_system == 'Windows'", + "dependencies": {"d": ">=0.0.0"}, + }, + { + "name": "c", + "version": "7.8.9", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "sys_platform == 'win32'", + "dependencies": {"d": ">=0.0.0"}, + }, + { + "name": "d", + "version": "0.0.1", + "category": "main", + "optional": False, + "python-versions": "*", + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"a": [], "b": [], "c": [], "d": []}, + }, + } + ) + set_package_requires(poetry, skip={"b", "c", "d"}) + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + expected = { + "a": dependency_from_pep_508("a==1.2.3; python_version < '3.7'"), + "b": dependency_from_pep_508( + "b==4.5.6; platform_system == 'Windows' and python_version < '3.7'" + ), + "c": dependency_from_pep_508( + "c==7.8.9; sys_platform == 'win32' and python_version < '3.7'" + ), + "d": dependency_from_pep_508( + "d==0.0.1; python_version < '3.7' and platform_system == 'Windows' and sys_platform == 'win32'" + ), + } + + for line in content.strip().split("\n"): + dependency = dependency_from_pep_508(line) + assert dependency.name in expected + expected_dependency = expected.pop(dependency.name) + assert dependency == expected_dependency + assert dependency.marker == expected_dependency.marker + + assert expected == {} + + +def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_any( + tmp_dir, poetry +): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "a", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + }, + { + "name": "b", + "version": "4.5.6", + "category": "dev", + "optional": False, + "python-versions": "*", + "dependencies": {"a": ">=1.2.3"}, + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"a": [], "b": []}, + }, + } + ) + + poetry.package.requires = [ + Factory.create_dependency( + name="a", constraint=dict(version="^1.2.3", python="<3.8") + ), + ] + poetry.package.dev_requires = [ + Factory.create_dependency( + name="b", constraint=dict(version="^4.5.6"), category="dev" + ), + Factory.create_dependency(name="a", constraint=dict(version="^1.2.3")), + ] + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True) + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + assert ( + content + == """\ +a==1.2.3 +a==1.2.3; python_version < "3.8" +b==4.5.6 +""" + ) + + def test_exporter_can_export_requirements_txt_with_standard_packages_and_hashes( tmp_dir, poetry ): From b4fd58b87fcdc07b67f9eebc30b52bf5b9249f05 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Oct 2020 02:44:14 +0200 Subject: [PATCH 06/12] utils/exporter: fix type hint for export() (cherry picked from commit b29450c19034cf34856cf6e2a610507411833e36) --- poetry/utils/exporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry/utils/exporter.py b/poetry/utils/exporter.py index ae85c29cc0b..5f2d6303817 100644 --- a/poetry/utils/exporter.py +++ b/poetry/utils/exporter.py @@ -31,7 +31,7 @@ def export( dev=False, extras=None, with_credentials=False, - ): # type: (str, Path, Union[IO, str], bool, bool, bool) -> None + ): # type: (str, Path, Union[IO, str], bool, bool, Optional[Union[bool, Sequence[str]]], bool) -> None if fmt not in self.ACCEPTED_FORMATS: raise ValueError("Invalid export format: {}".format(fmt)) From 445be5bdf9259be2039cdc917b09ad059ada6bc6 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Oct 2020 02:46:38 +0200 Subject: [PATCH 07/12] locker: remove redundant lock data processing (cherry picked from commit a7d66767a089c5604803a0a2d96ddc5fcb41c043) --- poetry/packages/locker.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 26c2b584504..8eccacd0c97 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -185,15 +185,14 @@ def locked_repository( return packages + @classmethod def get_project_dependencies( - self, project_requires, pinned_versions=False, with_nested=False, with_dev=False - ): # type: (List[Dependency], bool, bool, bool) -> List[Dependency] - packages = self.locked_repository(with_dev).packages - + cls, project_requires, locked_packages, pinned_versions=False, with_nested=False + ): # type: (List[Dependency], List[Package], bool, bool) -> List[Dependency] # group packages entries by name, this is required because requirement might use # different constraints packages_by_name = {} - for pkg in packages: + for pkg in locked_packages: if pkg.name not in packages_by_name: packages_by_name[pkg.name] = [] packages_by_name[pkg.name].append(pkg) @@ -275,8 +274,6 @@ def __walk_level( # we make a copy to avoid any side-effects requirement = deepcopy(requirement) - requirement._category = pkg.category - if pinned_versions: requirement.set_constraint( __get_locked_package(requirement).to_dependency().constraint @@ -285,9 +282,7 @@ def __walk_level( # dependencies use extra to indicate that it was activated via parent # package's extras, this is not required for nested exports as we assume # the resolver already selected this dependency - requirement.marker = requirement.marker.without_extras().intersect( - pkg.marker - ) + requirement.marker = requirement.marker.without_extras() key = (requirement.name, requirement.pretty_constraint) if key not in nested_dependencies: @@ -324,7 +319,9 @@ def get_project_dependency_packages( ) for dependency in self.get_project_dependencies( - project_requires=project_requires, with_nested=True, with_dev=dev, + project_requires=project_requires, + locked_packages=repository.packages, + with_nested=True, ): try: package = repository.find_packages(dependency=dependency)[0] From 68577b055fdabfb286319c787e8abadcb774c783 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Oct 2020 02:53:56 +0200 Subject: [PATCH 08/12] locker: ensure correct handling of extras export Previously, when determining nested dependencies, the check for activated extras/features of top level dependencies were done after the nested dependencies were processed. This lead to exports containing in active extras. This change resolves this by pre-selecting top level packages prior to identifying nested dependencies. (cherry picked from commit 2b6f82e9494b99bdb04d0dac90f87eb23548e4a5) --- poetry/packages/locker.py | 22 ++++++++++++++++------ tests/utils/test_exporter.py | 27 +++++++++++++++------------ 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 8eccacd0c97..50074a4417a 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -318,20 +318,30 @@ def get_project_dependency_packages( ) ) - for dependency in self.get_project_dependencies( - project_requires=project_requires, - locked_packages=repository.packages, - with_nested=True, - ): + # If a package is optional and we haven't opted in to it, do not select + selected = [] + for dependency in project_requires: try: package = repository.find_packages(dependency=dependency)[0] except IndexError: continue - # If a package is optional and we haven't opted in to it, continue if extra_package_names is not None and ( package.optional and package.name not in extra_package_names ): + # a package is locked as optional, but is not activated via extras + continue + + selected.append(dependency) + + for dependency in self.get_project_dependencies( + project_requires=selected, + locked_packages=repository.packages, + with_nested=True, + ): + try: + package = repository.find_packages(dependency=dependency)[0] + except IndexError: continue for extra in dependency.extras: diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index 66b9e81ff53..0cf6f9a202e 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -544,8 +544,17 @@ def test_exporter_exports_requirements_txt_without_optional_packages(tmp_dir, po assert expected == content -def test_exporter_exports_requirements_txt_with_optional_packages_if_opted_in( - tmp_dir, poetry +@pytest.mark.parametrize( + "extras,lines", + [ + (None, ["foo==1.2.3"]), + (False, ["foo==1.2.3"]), + (True, ["bar==4.5.6", "foo==1.2.3", "spam==0.1.0"]), + (["feature_bar"], ["bar==4.5.6", "foo==1.2.3", "spam==0.1.0"]), + ], +) +def test_exporter_exports_requirements_txt_with_optional_packages( + tmp_dir, poetry, extras, lines ): poetry.locker.mock_lock_data( { @@ -590,22 +599,16 @@ def test_exporter_exports_requirements_txt_with_optional_packages_if_opted_in( Path(tmp_dir), "requirements.txt", dev=True, - extras=["feature_bar"], + with_hashes=False, + extras=extras, ) with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: content = f.read() - expected = """\ -bar==4.5.6 \\ - --hash=sha256:67890 -foo==1.2.3 \\ - --hash=sha256:12345 -spam==0.1.0 \\ - --hash=sha256:abcde -""" + expected = "\n".join(lines) - assert expected == content + assert content.strip() == expected def test_exporter_can_export_requirements_txt_with_git_packages(tmp_dir, poetry): From 9c0576cf116f8cbec4c1be19677a3505ddd2009f Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Oct 2020 20:01:18 +0200 Subject: [PATCH 09/12] locker: unify duplicate dependencies on export (cherry picked from commit 733736cd1c2ea6189a77e34ab640059dbff2be23) --- poetry/packages/locker.py | 19 +++++++++++++------ tests/utils/test_exporter.py | 18 +++++++----------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 50074a4417a..9113a303673 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -1,4 +1,3 @@ -import itertools import json import logging import os @@ -6,6 +5,7 @@ from copy import deepcopy from hashlib import sha256 +from typing import Iterable from typing import Iterator from typing import List from typing import Optional @@ -188,7 +188,7 @@ def locked_repository( @classmethod def get_project_dependencies( cls, project_requires, locked_packages, pinned_versions=False, with_nested=False - ): # type: (List[Dependency], List[Package], bool, bool) -> List[Dependency] + ): # type: (List[Dependency], List[Package], bool, bool) -> Iterable[Dependency] # group packages entries by name, this is required because requirement might use # different constraints packages_by_name = {} @@ -296,10 +296,17 @@ def __walk_level( __walk_level(dependencies, 0) - return sorted( - itertools.chain(dependencies, nested_dependencies.values()), - key=lambda x: x.name.lower(), - ) + # Merge same dependencies using marker union + for requirement in dependencies: + key = (requirement.name, requirement.pretty_constraint) + if key not in nested_dependencies: + nested_dependencies[key] = requirement + else: + nested_dependencies[key].marker = nested_dependencies[key].marker.union( + requirement.marker + ) + + return sorted(nested_dependencies.values(), key=lambda x: x.name.lower()) def get_project_dependency_packages( self, project_requires, dev=False, extras=None diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index 0cf6f9a202e..d810bb8b08b 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -256,8 +256,12 @@ def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers( assert expected == {} +@pytest.mark.parametrize( + "dev,lines", + [(False, ['a==1.2.3; python_version < "3.8"']), (True, ["a==1.2.3", "b==4.5.6"])], +) def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_any( - tmp_dir, poetry + tmp_dir, poetry, dev, lines ): poetry.locker.mock_lock_data( { @@ -295,24 +299,16 @@ def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_a Factory.create_dependency( name="b", constraint=dict(version="^4.5.6"), category="dev" ), - Factory.create_dependency(name="a", constraint=dict(version="^1.2.3")), ] exporter = Exporter(poetry) - exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True) + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=dev) with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: content = f.read() - assert ( - content - == """\ -a==1.2.3 -a==1.2.3; python_version < "3.8" -b==4.5.6 -""" - ) + assert content.strip() == "\n".join(lines) def test_exporter_can_export_requirements_txt_with_standard_packages_and_hashes( From c37b68a15bd9d9934cc01e3b276e18e803719d05 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Oct 2020 20:34:42 +0200 Subject: [PATCH 10/12] locker: refactor to reduce code complexity (cherry picked from commit ed43d94b423a7bf53dfb530a549b686d016ab891) --- poetry/packages/locker.py | 174 +++++++++++++++++++++----------------- 1 file changed, 97 insertions(+), 77 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 9113a303673..9dd75e66519 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -5,11 +5,14 @@ from copy import deepcopy from hashlib import sha256 +from typing import Dict from typing import Iterable from typing import Iterator from typing import List from typing import Optional from typing import Sequence +from typing import Set +from typing import Tuple from typing import Union from tomlkit import array @@ -185,36 +188,107 @@ def locked_repository( return packages + @staticmethod + def __get_locked_package( + _dependency, packages_by_name + ): # type: (Dependency, Dict[str, List[Package]]) -> Optional[Package] + """ + Internal helper to identify corresponding locked package using dependency + version constraints. + """ + for _package in packages_by_name.get(_dependency.name, []): + if _dependency.constraint.allows(_package.version): + return _package + return None + + @classmethod + def __walk_dependency_level( + cls, + dependencies, + level, + pinned_versions, + packages_by_name, + project_level_dependencies, + nested_dependencies, + ): # type: (List[Dependency], int, bool, Dict[str, List[Package]], Set[str], Dict[Tuple[str, str], Dependency]) -> Dict[Tuple[str, str], Dependency] + if not dependencies: + return nested_dependencies + + next_level_dependencies = [] + + for requirement in dependencies: + locked_package = cls.__get_locked_package(requirement, packages_by_name) + + if locked_package: + for require in locked_package.requires: + if require.marker.is_empty(): + require.marker = requirement.marker + else: + require.marker = require.marker.intersect(requirement.marker) + + require.marker = require.marker.intersect(locked_package.marker) + next_level_dependencies.append(require) + + if requirement.name in project_level_dependencies and level == 0: + # project level dependencies take precedence + continue + + if locked_package: + # create dependency from locked package to retain dependency metadata + # if this is not done, we can end-up with incorrect nested dependencies + marker = requirement.marker + requirement = locked_package.to_dependency() + requirement.marker = requirement.marker.intersect(marker) + else: + # we make a copy to avoid any side-effects + requirement = deepcopy(requirement) + + if pinned_versions: + requirement.set_constraint( + cls.__get_locked_package(requirement, packages_by_name) + .to_dependency() + .constraint + ) + + # dependencies use extra to indicate that it was activated via parent + # package's extras, this is not required for nested exports as we assume + # the resolver already selected this dependency + requirement.marker = requirement.marker.without_extras() + + key = (requirement.name, requirement.pretty_constraint) + if key not in nested_dependencies: + nested_dependencies[key] = requirement + else: + nested_dependencies[key].marker = nested_dependencies[ + key + ].marker.intersect(requirement.marker) + + return cls.__walk_dependency_level( + dependencies=next_level_dependencies, + level=level + 1, + pinned_versions=pinned_versions, + packages_by_name=packages_by_name, + project_level_dependencies=project_level_dependencies, + nested_dependencies=nested_dependencies, + ) + @classmethod def get_project_dependencies( cls, project_requires, locked_packages, pinned_versions=False, with_nested=False ): # type: (List[Dependency], List[Package], bool, bool) -> Iterable[Dependency] - # group packages entries by name, this is required because requirement might use - # different constraints + # group packages entries by name, this is required because requirement might use different constraints packages_by_name = {} for pkg in locked_packages: if pkg.name not in packages_by_name: packages_by_name[pkg.name] = [] packages_by_name[pkg.name].append(pkg) - def __get_locked_package( - _dependency, - ): # type: (Dependency) -> Optional[Package] - """ - Internal helper to identify corresponding locked package using dependency - version constraints. - """ - for _package in packages_by_name.get(_dependency.name, []): - if _dependency.constraint.allows(_package.version): - return _package - return None - project_level_dependencies = set() dependencies = [] for dependency in project_requires: dependency = deepcopy(dependency) - locked_package = __get_locked_package(dependency) + locked_package = cls.__get_locked_package(dependency, packages_by_name) if locked_package: locked_dependency = locked_package.to_dependency() locked_dependency.marker = dependency.marker.intersect( @@ -233,68 +307,14 @@ def __get_locked_package( # return only with project level dependencies return dependencies - nested_dependencies = dict() - - def __walk_level( - __dependencies, __level - ): # type: (List[Dependency], int) -> None - if not __dependencies: - return - - __next_level = [] - - for requirement in __dependencies: - __locked_package = __get_locked_package(requirement) - - if __locked_package: - for require in __locked_package.requires: - if require.marker.is_empty(): - require.marker = requirement.marker - else: - require.marker = require.marker.intersect( - requirement.marker - ) - - require.marker = require.marker.intersect( - __locked_package.marker - ) - __next_level.append(require) - - if requirement.name in project_level_dependencies and __level == 0: - # project level dependencies take precedence - continue - - if __locked_package: - # create dependency from locked package to retain dependency metadata - # if this is not done, we can end-up with incorrect nested dependencies - marker = requirement.marker - requirement = __locked_package.to_dependency() - requirement.marker = requirement.marker.intersect(marker) - else: - # we make a copy to avoid any side-effects - requirement = deepcopy(requirement) - - if pinned_versions: - requirement.set_constraint( - __get_locked_package(requirement).to_dependency().constraint - ) - - # dependencies use extra to indicate that it was activated via parent - # package's extras, this is not required for nested exports as we assume - # the resolver already selected this dependency - requirement.marker = requirement.marker.without_extras() - - key = (requirement.name, requirement.pretty_constraint) - if key not in nested_dependencies: - nested_dependencies[key] = requirement - else: - nested_dependencies[key].marker = nested_dependencies[ - key - ].marker.intersect(requirement.marker) - - return __walk_level(__next_level, __level + 1) - - __walk_level(dependencies, 0) + nested_dependencies = cls.__walk_dependency_level( + dependencies=dependencies, + level=0, + pinned_versions=pinned_versions, + packages_by_name=packages_by_name, + project_level_dependencies=project_level_dependencies, + nested_dependencies=dict(), + ) # Merge same dependencies using marker union for requirement in dependencies: From d5377c7de2027d1b798b574983bd15c360935291 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Wed, 14 Oct 2020 18:33:08 +0200 Subject: [PATCH 11/12] Update CHANGELOG.md --- CHANGELOG.md | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e375aeeffb6..ec01b75abde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Change Log +## [1.1.3] - 2020-10-14 + +### Changed + +- Python version support deprecation warning is now written to `stderr`. ([#3131](https://github.com/python-poetry/poetry/pull/3131)) + +### Fixed + +- Fixed `KeyError` when `PATH` is not defined in environment variables. ([#3159](https://github.com/python-poetry/poetry/pull/3159)) +- Fixed error when using `config` command in a directory with an existing `pyproject.toml` without any Poetry configuration. ([#3172](https://github.com/python-poetry/poetry/pull/3172)) +- Fixed incorrect inspection of package requirements when same dependency is specified multiple times with unique markers. ([#3147](https://github.com/python-poetry/poetry/pull/3147)) +- Fixed `show` command to use already resolved package metadata. ([#3117](https://github.com/python-poetry/poetry/pull/3117)) +- Fixed multiple issues with `export` command output when using `requirements.txt` format. ([#3119](https://github.com/python-poetry/poetry/pull/3119)) + +## [1.1.2] - 2020-10-06 + +### Changed +- Dependency installation of editable packages and all uninstall operations are now performed serially within their corresponding priority groups. ([#3099](https://github.com/python-poetry/poetry/pull/3099)) +- Improved package metadata inspection of nested poetry projects within project path dependencies. ([#3105](https://github.com/python-poetry/poetry/pull/3105)) + +### Fixed + +- Fixed export of `requirements.txt` when project dependency contains git dependencies. ([#3100](https://github.com/python-poetry/poetry/pull/3100)) + +## [1.1.1] - 2020-10-05 + +### Added + +- Added `--no-update` option to `lock` command. ([#3034](https://github.com/python-poetry/poetry/pull/3034)) + +### Fixed + +- Fixed resolution of packages with missing required extras. ([#3035](https://github.com/python-poetry/poetry/pull/3035)) +- Fixed export of `requirements.txt` dependencies to include development dependencies. ([#3024](https://github.com/python-poetry/poetry/pull/3024)) +- Fixed incorrect selection of unsupported binary distribution formats when selecting a package artifact to install. ([#3058](https://github.com/python-poetry/poetry/pull/3058)) +- Fixed incorrect use of system executable when building package distributions via `build` command. ([#3056](https://github.com/python-poetry/poetry/pull/3056)) +- Fixed errors in `init` command when specifying `--dependency` in non-interactive mode when a `pyproject.toml` file already exists. ([#3076](https://github.com/python-poetry/poetry/pull/3076)) +- Fixed incorrect selection of configured source url when a publish repository url configuration with the same name already exists. ([#3047](https://github.com/python-poetry/poetry/pull/3047)) +- Fixed dependency resolution issues when the same package is specified in multiple dependency extras. ([#3046](https://github.com/python-poetry/poetry/pull/3046)) + ## [1.1.0] - 2020-10-01 ### Changed @@ -1023,7 +1063,10 @@ Initial release -[Unreleased]: https://github.com/python-poetry/poetry/compare/1.1.0...master +[Unreleased]: https://github.com/python-poetry/poetry/compare/1.1.3...master +[1.1.3]: https://github.com/python-poetry/poetry/compare/1.1.3 +[1.1.2]: https://github.com/python-poetry/poetry/releases/tag/1.1.2 +[1.1.1]: https://github.com/python-poetry/poetry/releases/tag/1.1.1 [1.1.0]: https://github.com/python-poetry/poetry/releases/tag/1.1.0 [1.1.0rc1]: https://github.com/python-poetry/poetry/releases/tag/1.1.0rc1 [1.1.0b4]: https://github.com/python-poetry/poetry/releases/tag/1.1.0b4 From 433cb2c24b9a3f614794bcc65cc21944cadd0cb7 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Wed, 14 Oct 2020 18:35:34 +0200 Subject: [PATCH 12/12] Bump version to 1.2.0a0 --- poetry/__version__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry/__version__.py b/poetry/__version__.py index 6849410aae0..bc0be1b6fa5 100644 --- a/poetry/__version__.py +++ b/poetry/__version__.py @@ -1 +1 @@ -__version__ = "1.1.0" +__version__ = "1.2.0a0" diff --git a/pyproject.toml b/pyproject.toml index f099a4cac80..59c49f74cf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "poetry" -version = "1.1.0" +version = "1.2.0a0" description = "Python dependency management and packaging made easy." authors = [ "Sébastien Eustace "