From 8b67d161b3e9a9dc7b85cae36b2be44bca8464a2 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Wed, 7 Oct 2020 15:51:26 +0200 Subject: [PATCH 1/8] locker: move export logic into locker for reuse This change, moves common functionality from exporter into the locker to allow for sharing of logic that can generate `DependencyPackage` instances given a a list of requirements using the lock data. --- poetry/packages/locker.py | 44 +++++++++++++++++++++++++++++++++++++-- poetry/utils/exporter.py | 34 +++++++----------------------- 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 2283a2373d2..a486bf1c5d6 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -6,9 +6,11 @@ from copy import deepcopy from hashlib import sha256 -from typing import Any +from typing import Iterator from typing import List from typing import Optional +from typing import Sequence +from typing import Union from tomlkit import array from tomlkit import document @@ -25,8 +27,10 @@ from poetry.core.semver.version import Version from poetry.core.toml.file import TOMLFile from poetry.core.version.markers import parse_marker +from poetry.packages import DependencyPackage from poetry.utils._compat import OrderedDict from poetry.utils._compat import Path +from poetry.utils.extras import get_extra_package_names logger = logging.getLogger(__name__) @@ -183,7 +187,7 @@ def locked_repository( def get_project_dependencies( self, project_requires, pinned_versions=False, with_nested=False, with_dev=False - ): # type: (List[Dependency], bool, bool, bool) -> Any + ): # type: (List[Dependency], bool, bool, bool) -> List[Dependency] packages = self.locked_repository(with_dev).packages # group packages entries by name, this is required because requirement might use @@ -263,6 +267,42 @@ def __get_locked_package( key=lambda x: x.name.lower(), ) + def get_project_dependency_packages( + self, project_requires, dev=False, extras=None + ): # type: (List[Dependency], bool, Optional[Union[bool, Sequence[str]]]) -> Iterator[DependencyPackage] + repository = self.locked_repository(with_dev_reqs=dev) + + # Build a set of all packages required by our selected extras + extra_package_names = ( + None if (isinstance(extras, bool) and extras is True) else () + ) + + if extra_package_names is not None: + extra_package_names = set( + get_extra_package_names( + repository.packages, self.lock_data.get("extras", {}), extras or (), + ) + ) + + for dependency in self.get_project_dependencies( + project_requires=project_requires, with_nested=True, with_dev=dev, + ): + try: + package = repository.find_packages(dependency=dependency)[0] + except IndexError: + continue + + # If a package is optional and we haven't opted in to it, continue + if extra_package_names is not None and ( + package.optional and package.name not in extra_package_names + ): + continue + + for extra in dependency.extras: + package.requires_extras.append(extra) + + yield DependencyPackage(dependency=dependency, package=package) + def set_lock_data(self, root, packages): # type: (...) -> bool files = table() packages = self._lock_packages(packages) diff --git a/poetry/utils/exporter.py b/poetry/utils/exporter.py index 4200abd146d..ae85c29cc0b 100644 --- a/poetry/utils/exporter.py +++ b/poetry/utils/exporter.py @@ -1,3 +1,5 @@ +from typing import Optional +from typing import Sequence from typing import Union from clikit.api.io import IO @@ -5,7 +7,6 @@ from poetry.poetry import Poetry from poetry.utils._compat import Path from poetry.utils._compat import decode -from poetry.utils.extras import get_extra_package_names class Exporter(object): @@ -51,38 +52,19 @@ def _export_requirements_txt( dev=False, extras=None, with_credentials=False, - ): # type: (Path, Union[IO, str], bool, bool, bool) -> None + ): # type: (Path, Union[IO, str], bool, bool, Optional[Union[bool, Sequence[str]]], bool) -> None indexes = set() content = "" - repository = self._poetry.locker.locked_repository(dev) - - # Build a set of all packages required by our selected extras - extra_package_names = set( - get_extra_package_names( - repository.packages, - self._poetry.locker.lock_data.get("extras", {}), - extras or (), - ) - ) - dependency_lines = set() - for dependency in self._poetry.locker.get_project_dependencies( - project_requires=self._poetry.package.all_requires, - with_nested=True, - with_dev=dev, + for dependency_package in self._poetry.locker.get_project_dependency_packages( + project_requires=self._poetry.package.all_requires, dev=dev, extras=extras ): - try: - package = repository.find_packages(dependency=dependency)[0] - except IndexError: - continue - - # If a package is optional and we haven't opted in to it, continue - if package.optional and package.name not in extra_package_names: - continue - line = "" + dependency = dependency_package.dependency + package = dependency_package.package + if package.develop: line += "-e " From 98545064eb5d85167ddedc8d637fcc46e29748ec Mon Sep 17 00:00:00 2001 From: "Yury V. Zaytsev" Date: Wed, 7 Oct 2020 17:54:40 +0200 Subject: [PATCH 2/8] locker: reuse locked metadata for nested deps Resolves: #3115 --- poetry/packages/locker.py | 11 +++++-- tests/utils/test_exporter.py | 60 ++++++++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index a486bf1c5d6..2a82fef816a 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -234,8 +234,15 @@ def __get_locked_package( # project level dependencies take precedence continue - # we make a copy to avoid any side-effects - requirement = deepcopy(requirement) + locked_package = __get_locked_package(requirement) + if locked_package: + # create dependency from locked package to retain dependency metadata + # if this is not done, we can end-up with incorrect nested dependencies + requirement = locked_package.to_dependency() + else: + # we make a copy to avoid any side-effects + requirement = deepcopy(requirement) + requirement._category = pkg.category if pinned_versions: diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index e26f448f6de..a75fb3da502 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -59,13 +59,18 @@ def poetry(fixture_dir, locker): return p -def set_package_requires(poetry): +def set_package_requires(poetry, skip=None): + skip = skip or set() packages = poetry.locker.locked_repository(with_dev_reqs=True).packages poetry.package.requires = [ - pkg.to_dependency() for pkg in packages if pkg.category == "main" + pkg.to_dependency() + for pkg in packages + if pkg.category == "main" and pkg.name not in skip ] poetry.package.dev_requires = [ - pkg.to_dependency() for pkg in packages if pkg.category == "dev" + pkg.to_dependency() + for pkg in packages + if pkg.category == "dev" and pkg.name not in skip ] @@ -503,6 +508,55 @@ def test_exporter_can_export_requirements_txt_with_git_packages(tmp_dir, poetry) assert expected == content +def test_exporter_can_export_requirements_txt_with_nested_packages(tmp_dir, poetry): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "foo", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + "source": { + "type": "git", + "url": "https://github.com/foo/foo.git", + "reference": "123456", + }, + }, + { + "name": "bar", + "version": "4.5.6", + "category": "main", + "optional": False, + "python-versions": "*", + "dependencies": {"foo": "rev 123456"}, + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"foo": [], "bar": []}, + }, + } + ) + set_package_requires(poetry, skip={"foo"}) + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + expected = """\ +bar==4.5.6 +foo @ git+https://github.com/foo/foo.git@123456 +""" + + assert expected == content + + def test_exporter_can_export_requirements_txt_with_git_packages_and_markers( tmp_dir, poetry ): From 48caa171e18de53dea46bfb688b7308c0cd07aab Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 9 Oct 2020 06:34:43 +0200 Subject: [PATCH 3/8] locker: propagate cumulative markers to nested deps This change ensures that markers are propagated from top level dependencies to the deepest level by walking top to bottom instead of iterating over all available packages. In addition, we also compress any dependencies with the same name and constraint to provide a more concise representation. Resolves: #3112 #3160 --- poetry/packages/locker.py | 86 ++++++++++++++------- tests/utils/test_exporter.py | 140 +++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 27 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 2a82fef816a..26c2b584504 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -215,10 +215,18 @@ def __get_locked_package( for dependency in project_requires: dependency = deepcopy(dependency) - if pinned_versions: - locked_package = __get_locked_package(dependency) - if locked_package: - dependency.set_constraint(locked_package.to_dependency().constraint) + locked_package = __get_locked_package(dependency) + if locked_package: + locked_dependency = locked_package.to_dependency() + locked_dependency.marker = dependency.marker.intersect( + locked_package.marker + ) + + if not pinned_versions: + locked_dependency.set_constraint(dependency.constraint) + + dependency = locked_dependency + project_level_dependencies.add(dependency.name) dependencies.append(dependency) @@ -226,19 +234,43 @@ def __get_locked_package( # return only with project level dependencies return dependencies - nested_dependencies = list() + nested_dependencies = dict() - for pkg in packages: # type: Package - for requirement in pkg.requires: # type: Dependency - if requirement.name in project_level_dependencies: + def __walk_level( + __dependencies, __level + ): # type: (List[Dependency], int) -> None + if not __dependencies: + return + + __next_level = [] + + for requirement in __dependencies: + __locked_package = __get_locked_package(requirement) + + if __locked_package: + for require in __locked_package.requires: + if require.marker.is_empty(): + require.marker = requirement.marker + else: + require.marker = require.marker.intersect( + requirement.marker + ) + + require.marker = require.marker.intersect( + __locked_package.marker + ) + __next_level.append(require) + + if requirement.name in project_level_dependencies and __level == 0: # project level dependencies take precedence continue - locked_package = __get_locked_package(requirement) - if locked_package: + if __locked_package: # create dependency from locked package to retain dependency metadata # if this is not done, we can end-up with incorrect nested dependencies - requirement = locked_package.to_dependency() + marker = requirement.marker + requirement = __locked_package.to_dependency() + requirement.marker = requirement.marker.intersect(marker) else: # we make a copy to avoid any side-effects requirement = deepcopy(requirement) @@ -251,26 +283,26 @@ def __get_locked_package( ) # dependencies use extra to indicate that it was activated via parent - # package's extras - marker = requirement.marker.without_extras() - for project_requirement in project_requires: - if ( - pkg.name == project_requirement.name - and project_requirement.constraint.allows(pkg.version) - ): - requirement.marker = marker.intersect( - project_requirement.marker - ) - break + # package's extras, this is not required for nested exports as we assume + # the resolver already selected this dependency + requirement.marker = requirement.marker.without_extras().intersect( + pkg.marker + ) + + key = (requirement.name, requirement.pretty_constraint) + if key not in nested_dependencies: + nested_dependencies[key] = requirement else: - # this dependency was not from a project requirement - requirement.marker = marker.intersect(pkg.marker) + nested_dependencies[key].marker = nested_dependencies[ + key + ].marker.intersect(requirement.marker) + + return __walk_level(__next_level, __level + 1) - if requirement not in nested_dependencies: - nested_dependencies.append(requirement) + __walk_level(dependencies, 0) return sorted( - itertools.chain(dependencies, nested_dependencies), + itertools.chain(dependencies, nested_dependencies.values()), key=lambda x: x.name.lower(), ) diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index a75fb3da502..66b9e81ff53 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -2,6 +2,7 @@ import pytest +from poetry.core.packages import dependency_from_pep_508 from poetry.core.toml.file import TOMLFile from poetry.factory import Factory from poetry.packages import Locker as BaseLocker @@ -175,6 +176,145 @@ def test_exporter_can_export_requirements_txt_with_standard_packages_and_markers assert expected == content +def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers( + tmp_dir, poetry +): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "a", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "python_version < '3.7'", + "dependencies": {"b": ">=0.0.0", "c": ">=0.0.0"}, + }, + { + "name": "b", + "version": "4.5.6", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "platform_system == 'Windows'", + "dependencies": {"d": ">=0.0.0"}, + }, + { + "name": "c", + "version": "7.8.9", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "sys_platform == 'win32'", + "dependencies": {"d": ">=0.0.0"}, + }, + { + "name": "d", + "version": "0.0.1", + "category": "main", + "optional": False, + "python-versions": "*", + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"a": [], "b": [], "c": [], "d": []}, + }, + } + ) + set_package_requires(poetry, skip={"b", "c", "d"}) + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + expected = { + "a": dependency_from_pep_508("a==1.2.3; python_version < '3.7'"), + "b": dependency_from_pep_508( + "b==4.5.6; platform_system == 'Windows' and python_version < '3.7'" + ), + "c": dependency_from_pep_508( + "c==7.8.9; sys_platform == 'win32' and python_version < '3.7'" + ), + "d": dependency_from_pep_508( + "d==0.0.1; python_version < '3.7' and platform_system == 'Windows' and sys_platform == 'win32'" + ), + } + + for line in content.strip().split("\n"): + dependency = dependency_from_pep_508(line) + assert dependency.name in expected + expected_dependency = expected.pop(dependency.name) + assert dependency == expected_dependency + assert dependency.marker == expected_dependency.marker + + assert expected == {} + + +def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_any( + tmp_dir, poetry +): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "a", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + }, + { + "name": "b", + "version": "4.5.6", + "category": "dev", + "optional": False, + "python-versions": "*", + "dependencies": {"a": ">=1.2.3"}, + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"a": [], "b": []}, + }, + } + ) + + poetry.package.requires = [ + Factory.create_dependency( + name="a", constraint=dict(version="^1.2.3", python="<3.8") + ), + ] + poetry.package.dev_requires = [ + Factory.create_dependency( + name="b", constraint=dict(version="^4.5.6"), category="dev" + ), + Factory.create_dependency(name="a", constraint=dict(version="^1.2.3")), + ] + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True) + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + assert ( + content + == """\ +a==1.2.3 +a==1.2.3; python_version < "3.8" +b==4.5.6 +""" + ) + + def test_exporter_can_export_requirements_txt_with_standard_packages_and_hashes( tmp_dir, poetry ): From 8bc93d9fdfe860f1c5f592fe6a04942d14e3be62 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Oct 2020 02:44:14 +0200 Subject: [PATCH 4/8] utils/exporter: fix type hint for export() --- poetry/utils/exporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry/utils/exporter.py b/poetry/utils/exporter.py index ae85c29cc0b..5f2d6303817 100644 --- a/poetry/utils/exporter.py +++ b/poetry/utils/exporter.py @@ -31,7 +31,7 @@ def export( dev=False, extras=None, with_credentials=False, - ): # type: (str, Path, Union[IO, str], bool, bool, bool) -> None + ): # type: (str, Path, Union[IO, str], bool, bool, Optional[Union[bool, Sequence[str]]], bool) -> None if fmt not in self.ACCEPTED_FORMATS: raise ValueError("Invalid export format: {}".format(fmt)) From 665afb8e93686f488474a6e4006759d8480fed11 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Oct 2020 02:46:38 +0200 Subject: [PATCH 5/8] locker: remove redundant lock data processing --- poetry/packages/locker.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 26c2b584504..8eccacd0c97 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -185,15 +185,14 @@ def locked_repository( return packages + @classmethod def get_project_dependencies( - self, project_requires, pinned_versions=False, with_nested=False, with_dev=False - ): # type: (List[Dependency], bool, bool, bool) -> List[Dependency] - packages = self.locked_repository(with_dev).packages - + cls, project_requires, locked_packages, pinned_versions=False, with_nested=False + ): # type: (List[Dependency], List[Package], bool, bool) -> List[Dependency] # group packages entries by name, this is required because requirement might use # different constraints packages_by_name = {} - for pkg in packages: + for pkg in locked_packages: if pkg.name not in packages_by_name: packages_by_name[pkg.name] = [] packages_by_name[pkg.name].append(pkg) @@ -275,8 +274,6 @@ def __walk_level( # we make a copy to avoid any side-effects requirement = deepcopy(requirement) - requirement._category = pkg.category - if pinned_versions: requirement.set_constraint( __get_locked_package(requirement).to_dependency().constraint @@ -285,9 +282,7 @@ def __walk_level( # dependencies use extra to indicate that it was activated via parent # package's extras, this is not required for nested exports as we assume # the resolver already selected this dependency - requirement.marker = requirement.marker.without_extras().intersect( - pkg.marker - ) + requirement.marker = requirement.marker.without_extras() key = (requirement.name, requirement.pretty_constraint) if key not in nested_dependencies: @@ -324,7 +319,9 @@ def get_project_dependency_packages( ) for dependency in self.get_project_dependencies( - project_requires=project_requires, with_nested=True, with_dev=dev, + project_requires=project_requires, + locked_packages=repository.packages, + with_nested=True, ): try: package = repository.find_packages(dependency=dependency)[0] From 410df9928228a884b2075ede62f6f31e32cd4865 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Oct 2020 02:53:56 +0200 Subject: [PATCH 6/8] locker: ensure correct handling of extras export Previously, when determining nested dependencies, the check for activated extras/features of top level dependencies were done after the nested dependencies were processed. This lead to exports containing in active extras. This change resolves this by pre-selecting top level packages prior to identifying nested dependencies. --- poetry/packages/locker.py | 22 ++++++++++++++++------ tests/utils/test_exporter.py | 27 +++++++++++++++------------ 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 8eccacd0c97..50074a4417a 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -318,20 +318,30 @@ def get_project_dependency_packages( ) ) - for dependency in self.get_project_dependencies( - project_requires=project_requires, - locked_packages=repository.packages, - with_nested=True, - ): + # If a package is optional and we haven't opted in to it, do not select + selected = [] + for dependency in project_requires: try: package = repository.find_packages(dependency=dependency)[0] except IndexError: continue - # If a package is optional and we haven't opted in to it, continue if extra_package_names is not None and ( package.optional and package.name not in extra_package_names ): + # a package is locked as optional, but is not activated via extras + continue + + selected.append(dependency) + + for dependency in self.get_project_dependencies( + project_requires=selected, + locked_packages=repository.packages, + with_nested=True, + ): + try: + package = repository.find_packages(dependency=dependency)[0] + except IndexError: continue for extra in dependency.extras: diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index 66b9e81ff53..0cf6f9a202e 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -544,8 +544,17 @@ def test_exporter_exports_requirements_txt_without_optional_packages(tmp_dir, po assert expected == content -def test_exporter_exports_requirements_txt_with_optional_packages_if_opted_in( - tmp_dir, poetry +@pytest.mark.parametrize( + "extras,lines", + [ + (None, ["foo==1.2.3"]), + (False, ["foo==1.2.3"]), + (True, ["bar==4.5.6", "foo==1.2.3", "spam==0.1.0"]), + (["feature_bar"], ["bar==4.5.6", "foo==1.2.3", "spam==0.1.0"]), + ], +) +def test_exporter_exports_requirements_txt_with_optional_packages( + tmp_dir, poetry, extras, lines ): poetry.locker.mock_lock_data( { @@ -590,22 +599,16 @@ def test_exporter_exports_requirements_txt_with_optional_packages_if_opted_in( Path(tmp_dir), "requirements.txt", dev=True, - extras=["feature_bar"], + with_hashes=False, + extras=extras, ) with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: content = f.read() - expected = """\ -bar==4.5.6 \\ - --hash=sha256:67890 -foo==1.2.3 \\ - --hash=sha256:12345 -spam==0.1.0 \\ - --hash=sha256:abcde -""" + expected = "\n".join(lines) - assert expected == content + assert content.strip() == expected def test_exporter_can_export_requirements_txt_with_git_packages(tmp_dir, poetry): From 6edf15f65c3ddd2cfe27eea9621ea6ea209cf212 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Oct 2020 20:01:18 +0200 Subject: [PATCH 7/8] locker: unify duplicate dependencies on export --- poetry/packages/locker.py | 19 +++++++++++++------ tests/utils/test_exporter.py | 18 +++++++----------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 50074a4417a..9113a303673 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -1,4 +1,3 @@ -import itertools import json import logging import os @@ -6,6 +5,7 @@ from copy import deepcopy from hashlib import sha256 +from typing import Iterable from typing import Iterator from typing import List from typing import Optional @@ -188,7 +188,7 @@ def locked_repository( @classmethod def get_project_dependencies( cls, project_requires, locked_packages, pinned_versions=False, with_nested=False - ): # type: (List[Dependency], List[Package], bool, bool) -> List[Dependency] + ): # type: (List[Dependency], List[Package], bool, bool) -> Iterable[Dependency] # group packages entries by name, this is required because requirement might use # different constraints packages_by_name = {} @@ -296,10 +296,17 @@ def __walk_level( __walk_level(dependencies, 0) - return sorted( - itertools.chain(dependencies, nested_dependencies.values()), - key=lambda x: x.name.lower(), - ) + # Merge same dependencies using marker union + for requirement in dependencies: + key = (requirement.name, requirement.pretty_constraint) + if key not in nested_dependencies: + nested_dependencies[key] = requirement + else: + nested_dependencies[key].marker = nested_dependencies[key].marker.union( + requirement.marker + ) + + return sorted(nested_dependencies.values(), key=lambda x: x.name.lower()) def get_project_dependency_packages( self, project_requires, dev=False, extras=None diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index 0cf6f9a202e..d810bb8b08b 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -256,8 +256,12 @@ def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers( assert expected == {} +@pytest.mark.parametrize( + "dev,lines", + [(False, ['a==1.2.3; python_version < "3.8"']), (True, ["a==1.2.3", "b==4.5.6"])], +) def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_any( - tmp_dir, poetry + tmp_dir, poetry, dev, lines ): poetry.locker.mock_lock_data( { @@ -295,24 +299,16 @@ def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_a Factory.create_dependency( name="b", constraint=dict(version="^4.5.6"), category="dev" ), - Factory.create_dependency(name="a", constraint=dict(version="^1.2.3")), ] exporter = Exporter(poetry) - exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True) + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=dev) with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: content = f.read() - assert ( - content - == """\ -a==1.2.3 -a==1.2.3; python_version < "3.8" -b==4.5.6 -""" - ) + assert content.strip() == "\n".join(lines) def test_exporter_can_export_requirements_txt_with_standard_packages_and_hashes( From 36d74aeefed382b14803f88191ca95476698da0c Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 13 Oct 2020 20:34:42 +0200 Subject: [PATCH 8/8] locker: refactor to reduce code complexity --- poetry/packages/locker.py | 174 +++++++++++++++++++++----------------- 1 file changed, 97 insertions(+), 77 deletions(-) diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 9113a303673..9dd75e66519 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -5,11 +5,14 @@ from copy import deepcopy from hashlib import sha256 +from typing import Dict from typing import Iterable from typing import Iterator from typing import List from typing import Optional from typing import Sequence +from typing import Set +from typing import Tuple from typing import Union from tomlkit import array @@ -185,36 +188,107 @@ def locked_repository( return packages + @staticmethod + def __get_locked_package( + _dependency, packages_by_name + ): # type: (Dependency, Dict[str, List[Package]]) -> Optional[Package] + """ + Internal helper to identify corresponding locked package using dependency + version constraints. + """ + for _package in packages_by_name.get(_dependency.name, []): + if _dependency.constraint.allows(_package.version): + return _package + return None + + @classmethod + def __walk_dependency_level( + cls, + dependencies, + level, + pinned_versions, + packages_by_name, + project_level_dependencies, + nested_dependencies, + ): # type: (List[Dependency], int, bool, Dict[str, List[Package]], Set[str], Dict[Tuple[str, str], Dependency]) -> Dict[Tuple[str, str], Dependency] + if not dependencies: + return nested_dependencies + + next_level_dependencies = [] + + for requirement in dependencies: + locked_package = cls.__get_locked_package(requirement, packages_by_name) + + if locked_package: + for require in locked_package.requires: + if require.marker.is_empty(): + require.marker = requirement.marker + else: + require.marker = require.marker.intersect(requirement.marker) + + require.marker = require.marker.intersect(locked_package.marker) + next_level_dependencies.append(require) + + if requirement.name in project_level_dependencies and level == 0: + # project level dependencies take precedence + continue + + if locked_package: + # create dependency from locked package to retain dependency metadata + # if this is not done, we can end-up with incorrect nested dependencies + marker = requirement.marker + requirement = locked_package.to_dependency() + requirement.marker = requirement.marker.intersect(marker) + else: + # we make a copy to avoid any side-effects + requirement = deepcopy(requirement) + + if pinned_versions: + requirement.set_constraint( + cls.__get_locked_package(requirement, packages_by_name) + .to_dependency() + .constraint + ) + + # dependencies use extra to indicate that it was activated via parent + # package's extras, this is not required for nested exports as we assume + # the resolver already selected this dependency + requirement.marker = requirement.marker.without_extras() + + key = (requirement.name, requirement.pretty_constraint) + if key not in nested_dependencies: + nested_dependencies[key] = requirement + else: + nested_dependencies[key].marker = nested_dependencies[ + key + ].marker.intersect(requirement.marker) + + return cls.__walk_dependency_level( + dependencies=next_level_dependencies, + level=level + 1, + pinned_versions=pinned_versions, + packages_by_name=packages_by_name, + project_level_dependencies=project_level_dependencies, + nested_dependencies=nested_dependencies, + ) + @classmethod def get_project_dependencies( cls, project_requires, locked_packages, pinned_versions=False, with_nested=False ): # type: (List[Dependency], List[Package], bool, bool) -> Iterable[Dependency] - # group packages entries by name, this is required because requirement might use - # different constraints + # group packages entries by name, this is required because requirement might use different constraints packages_by_name = {} for pkg in locked_packages: if pkg.name not in packages_by_name: packages_by_name[pkg.name] = [] packages_by_name[pkg.name].append(pkg) - def __get_locked_package( - _dependency, - ): # type: (Dependency) -> Optional[Package] - """ - Internal helper to identify corresponding locked package using dependency - version constraints. - """ - for _package in packages_by_name.get(_dependency.name, []): - if _dependency.constraint.allows(_package.version): - return _package - return None - project_level_dependencies = set() dependencies = [] for dependency in project_requires: dependency = deepcopy(dependency) - locked_package = __get_locked_package(dependency) + locked_package = cls.__get_locked_package(dependency, packages_by_name) if locked_package: locked_dependency = locked_package.to_dependency() locked_dependency.marker = dependency.marker.intersect( @@ -233,68 +307,14 @@ def __get_locked_package( # return only with project level dependencies return dependencies - nested_dependencies = dict() - - def __walk_level( - __dependencies, __level - ): # type: (List[Dependency], int) -> None - if not __dependencies: - return - - __next_level = [] - - for requirement in __dependencies: - __locked_package = __get_locked_package(requirement) - - if __locked_package: - for require in __locked_package.requires: - if require.marker.is_empty(): - require.marker = requirement.marker - else: - require.marker = require.marker.intersect( - requirement.marker - ) - - require.marker = require.marker.intersect( - __locked_package.marker - ) - __next_level.append(require) - - if requirement.name in project_level_dependencies and __level == 0: - # project level dependencies take precedence - continue - - if __locked_package: - # create dependency from locked package to retain dependency metadata - # if this is not done, we can end-up with incorrect nested dependencies - marker = requirement.marker - requirement = __locked_package.to_dependency() - requirement.marker = requirement.marker.intersect(marker) - else: - # we make a copy to avoid any side-effects - requirement = deepcopy(requirement) - - if pinned_versions: - requirement.set_constraint( - __get_locked_package(requirement).to_dependency().constraint - ) - - # dependencies use extra to indicate that it was activated via parent - # package's extras, this is not required for nested exports as we assume - # the resolver already selected this dependency - requirement.marker = requirement.marker.without_extras() - - key = (requirement.name, requirement.pretty_constraint) - if key not in nested_dependencies: - nested_dependencies[key] = requirement - else: - nested_dependencies[key].marker = nested_dependencies[ - key - ].marker.intersect(requirement.marker) - - return __walk_level(__next_level, __level + 1) - - __walk_level(dependencies, 0) + nested_dependencies = cls.__walk_dependency_level( + dependencies=dependencies, + level=0, + pinned_versions=pinned_versions, + packages_by_name=packages_by_name, + project_level_dependencies=project_level_dependencies, + nested_dependencies=dict(), + ) # Merge same dependencies using marker union for requirement in dependencies: