From a74310374e4d932a8565db07619672dca08f32b0 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 6 Oct 2020 11:38:04 +0200 Subject: [PATCH 1/2] executor: execute parallel unsafe ops serially Resolves: #2658 #3086 --- poetry/installation/executor.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index 9effaf3ebeb..8c1d621af03 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -107,14 +107,33 @@ def execute(self, operations): # type: (Operation) -> int self._sections = OrderedDict() for _, group in groups: tasks = [] + serial_operations = [] for operation in group: if self._shutdown: break + # Some operations are unsafe, we mus execute them serially in a group + # https://github.com/python-poetry/poetry/issues/3086 + # https://github.com/python-poetry/poetry/issues/2658 + # + # We need to explicitly check source type here, see: + # https://github.com/python-poetry/poetry-core/pull/98 + is_parallel_unsafe = operation.job_type == "uninstall" or ( + operation.package.develop + and operation.package.source_type in {"directory", "git"} + ) + if not operation.skipped and is_parallel_unsafe: + serial_operations.append(operation) + continue + tasks.append(self._executor.submit(self._execute_operation, operation)) try: wait(tasks) + + for operation in serial_operations: + wait([self._executor.submit(self._execute_operation, operation)]) + except KeyboardInterrupt: self._shutdown = True From 19411e4b9628b50ad7eb4028a7ec2c76ae2112b1 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Tue, 6 Oct 2020 17:19:45 +0200 Subject: [PATCH 2/2] info: enhance local path poetry project support This change ensures that package inspection will handle rich core metadata available for poetry managed projects. This will allow, nested local path dependencies to use "develop" mode packages. Resolves: #3098 --- poetry/inspection/info.py | 9 ++++ tests/conftest.py | 9 +++- .../bar/pyproject.toml | 11 ++++ .../foo/pyproject.toml | 11 ++++ .../project_with_nested_local/pyproject.toml | 12 +++++ .../quix/pyproject.toml | 10 ++++ .../with-directory-dependency-poetry.test | 2 +- tests/installation/test_installer.py | 35 ++++++------ tests/installation/test_installer_old.py | 33 ++++++------ tests/puzzle/test_solver.py | 54 +++++++++++++++++++ 10 files changed, 147 insertions(+), 39 deletions(-) create mode 100644 tests/fixtures/project_with_nested_local/bar/pyproject.toml create mode 100644 tests/fixtures/project_with_nested_local/foo/pyproject.toml create mode 100644 tests/fixtures/project_with_nested_local/pyproject.toml create mode 100644 tests/fixtures/project_with_nested_local/quix/pyproject.toml diff --git a/poetry/inspection/info.py b/poetry/inspection/info.py index a2d3cbeb545..73034de16d3 100644 --- a/poetry/inspection/info.py +++ b/poetry/inspection/info.py @@ -156,6 +156,15 @@ def to_package( package.python_versions = self.requires_python or "*" package.files = self.files + if root_dir or (self._source_type in {"directory"} and self._source_url): + # this is a local poetry project, this means we can extract "richer" requirement information + # eg: development requirements etc. + poetry_package = self._get_poetry_package(path=root_dir or self._source_url) + if poetry_package: + package.extras = poetry_package.extras + package.requires = poetry_package.requires + return package + for req in self.requires_dist or []: try: # Attempt to parse the PEP-508 requirement string diff --git a/tests/conftest.py b/tests/conftest.py index 15f7b56e567..e2b73936924 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -141,9 +141,14 @@ def http(): @pytest.fixture -def fixture_dir(): +def fixture_base(): + return Path(__file__).parent / "fixtures" + + +@pytest.fixture +def fixture_dir(fixture_base): def _fixture_dir(name): - return Path(__file__).parent / "fixtures" / name + return fixture_base / name return _fixture_dir diff --git a/tests/fixtures/project_with_nested_local/bar/pyproject.toml b/tests/fixtures/project_with_nested_local/bar/pyproject.toml new file mode 100644 index 00000000000..bf058b8813a --- /dev/null +++ b/tests/fixtures/project_with_nested_local/bar/pyproject.toml @@ -0,0 +1,11 @@ +[tool.poetry] +name = "bar" +version = "1.2.3" +description = "Some description." +authors = ["Poetry Maintainer "] +license = "MIT" + +# Requirements +[tool.poetry.dependencies] +python = "~2.7 || ^3.4" +quix = { path = "../quix", develop = true } diff --git a/tests/fixtures/project_with_nested_local/foo/pyproject.toml b/tests/fixtures/project_with_nested_local/foo/pyproject.toml new file mode 100644 index 00000000000..1aba06effe1 --- /dev/null +++ b/tests/fixtures/project_with_nested_local/foo/pyproject.toml @@ -0,0 +1,11 @@ +[tool.poetry] +name = "foo" +version = "1.2.3" +description = "Some description." +authors = ["Poetry Maintainer "] +license = "MIT" + +# Requirements +[tool.poetry.dependencies] +python = "~2.7 || ^3.4" +bar = { path = "../bar", develop = true } diff --git a/tests/fixtures/project_with_nested_local/pyproject.toml b/tests/fixtures/project_with_nested_local/pyproject.toml new file mode 100644 index 00000000000..c6eb2c6b46c --- /dev/null +++ b/tests/fixtures/project_with_nested_local/pyproject.toml @@ -0,0 +1,12 @@ +[tool.poetry] +name = "project-with-nested-local" +version = "1.2.3" +description = "Some description." +authors = ["Poetry Maintainer "] +license = "MIT" + +# Requirements +[tool.poetry.dependencies] +python = "~2.7 || ^3.4" +foo = { path = "./foo", develop = true } +bar = { path = "./bar", develop = true } diff --git a/tests/fixtures/project_with_nested_local/quix/pyproject.toml b/tests/fixtures/project_with_nested_local/quix/pyproject.toml new file mode 100644 index 00000000000..a90d9bcc41f --- /dev/null +++ b/tests/fixtures/project_with_nested_local/quix/pyproject.toml @@ -0,0 +1,10 @@ +[tool.poetry] +name = "quix" +version = "1.2.3" +description = "Some description." +authors = ["Poetry Maintainer "] +license = "MIT" + +# Requirements +[tool.poetry.dependencies] +python = "~2.7 || ^3.4" diff --git a/tests/installation/fixtures/with-directory-dependency-poetry.test b/tests/installation/fixtures/with-directory-dependency-poetry.test index 32530cb97f0..12431f62185 100644 --- a/tests/installation/fixtures/with-directory-dependency-poetry.test +++ b/tests/installation/fixtures/with-directory-dependency-poetry.test @@ -16,7 +16,7 @@ python-versions = "*" version = "1.2.3" [package.dependencies] -pendulum = {version = ">=1.4.4", optional = true, markers = "extra == \"extras_a\""} +pendulum = {version = ">=1.4.4", optional = true} [package.extras] extras_a = ["pendulum (>=1.4.4)"] diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index 077f6dcab40..7a1660670ee 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -29,9 +29,6 @@ from tests.repositories.test_pypi_repository import MockRepository -fixtures_dir = Path("tests/fixtures") - - class Installer(BaseInstaller): def _get_installer(self): return NoopInstaller() @@ -779,8 +776,8 @@ def test_installer_with_pypi_repository(package, locker, installed, config): assert locker.written_data == expected -def test_run_installs_with_local_file(installer, locker, repo, package): - file_path = fixtures_dir / "distributions/demo-0.1.0-py2.py3-none-any.whl" +def test_run_installs_with_local_file(installer, locker, repo, package, fixture_dir): + file_path = fixture_dir("distributions/demo-0.1.0-py2.py3-none-any.whl") package.add_dependency(Factory.create_dependency("demo", {"file": str(file_path)})) repo.add_package(get_package("pendulum", "1.4.4")) @@ -793,9 +790,11 @@ def test_run_installs_with_local_file(installer, locker, repo, package): assert 2 == installer.executor.installations_count -def test_run_installs_wheel_with_no_requires_dist(installer, locker, repo, package): - file_path = ( - fixtures_dir / "wheel_with_no_requires_dist/demo-0.1.0-py2.py3-none-any.whl" +def test_run_installs_wheel_with_no_requires_dist( + installer, locker, repo, package, fixture_dir +): + file_path = fixture_dir( + "wheel_with_no_requires_dist/demo-0.1.0-py2.py3-none-any.whl" ) package.add_dependency(Factory.create_dependency("demo", {"file": str(file_path)})) @@ -809,31 +808,29 @@ def test_run_installs_wheel_with_no_requires_dist(installer, locker, repo, packa def test_run_installs_with_local_poetry_directory_and_extras( - installer, locker, repo, package, tmpdir + installer, locker, repo, package, tmpdir, fixture_dir ): - file_path = fixtures_dir / "project_with_extras" + file_path = fixture_dir("project_with_extras") package.add_dependency( Factory.create_dependency( "project-with-extras", {"path": str(file_path), "extras": ["extras_a"]} ) ) - print(package.requires[0].develop) repo.add_package(get_package("pendulum", "1.4.4")) installer.run() expected = fixture("with-directory-dependency-poetry") - assert locker.written_data == expected assert 2 == installer.executor.installations_count def test_run_installs_with_local_poetry_directory_transitive( - installer, locker, repo, package, tmpdir + installer, locker, repo, package, tmpdir, fixture_dir ): - root_dir = fixtures_dir.joinpath("directory") + root_dir = fixture_dir("directory") package.root_dir = root_dir locker.set_lock_path(root_dir) directory = root_dir.joinpath("project_with_transitive_directory_dependencies") @@ -858,12 +855,12 @@ def test_run_installs_with_local_poetry_directory_transitive( def test_run_installs_with_local_poetry_file_transitive( - installer, locker, repo, package, tmpdir + installer, locker, repo, package, tmpdir, fixture_dir ): - root_dir = fixtures_dir.joinpath("directory") + root_dir = fixture_dir("directory") package.root_dir = root_dir locker.set_lock_path(root_dir) - directory = fixtures_dir.joinpath("directory").joinpath( + directory = fixture_dir("directory").joinpath( "project_with_transitive_file_dependencies" ) package.add_dependency( @@ -887,9 +884,9 @@ def test_run_installs_with_local_poetry_file_transitive( def test_run_installs_with_local_setuptools_directory( - installer, locker, repo, package, tmpdir + installer, locker, repo, package, tmpdir, fixture_dir ): - file_path = fixtures_dir / "project_with_setup/" + file_path = fixture_dir("project_with_setup/") package.add_dependency( Factory.create_dependency("project-with-setup", {"path": str(file_path)}) ) diff --git a/tests/installation/test_installer_old.py b/tests/installation/test_installer_old.py index f8f5f62a469..b92bdce460e 100644 --- a/tests/installation/test_installer_old.py +++ b/tests/installation/test_installer_old.py @@ -27,9 +27,6 @@ from tests.repositories.test_pypi_repository import MockRepository -fixtures_dir = Path("tests/fixtures") - - class Installer(BaseInstaller): def _get_installer(self): return NoopInstaller() @@ -749,8 +746,8 @@ def test_installer_with_pypi_repository(package, locker, installed, config): assert locker.written_data == expected -def test_run_installs_with_local_file(installer, locker, repo, package): - file_path = fixtures_dir / "distributions/demo-0.1.0-py2.py3-none-any.whl" +def test_run_installs_with_local_file(installer, locker, repo, package, fixture_dir): + file_path = fixture_dir("distributions/demo-0.1.0-py2.py3-none-any.whl") package.add_dependency(Factory.create_dependency("demo", {"file": str(file_path)})) repo.add_package(get_package("pendulum", "1.4.4")) @@ -764,9 +761,11 @@ def test_run_installs_with_local_file(installer, locker, repo, package): assert len(installer.installer.installs) == 2 -def test_run_installs_wheel_with_no_requires_dist(installer, locker, repo, package): - file_path = ( - fixtures_dir / "wheel_with_no_requires_dist/demo-0.1.0-py2.py3-none-any.whl" +def test_run_installs_wheel_with_no_requires_dist( + installer, locker, repo, package, fixture_dir +): + file_path = fixture_dir( + "wheel_with_no_requires_dist/demo-0.1.0-py2.py3-none-any.whl" ) package.add_dependency(Factory.create_dependency("demo", {"file": str(file_path)})) @@ -780,9 +779,9 @@ def test_run_installs_wheel_with_no_requires_dist(installer, locker, repo, packa def test_run_installs_with_local_poetry_directory_and_extras( - installer, locker, repo, package, tmpdir + installer, locker, repo, package, tmpdir, fixture_dir ): - file_path = fixtures_dir / "project_with_extras" + file_path = fixture_dir("project_with_extras") package.add_dependency( Factory.create_dependency( "project-with-extras", {"path": str(file_path), "extras": ["extras_a"]} @@ -801,9 +800,9 @@ def test_run_installs_with_local_poetry_directory_and_extras( def test_run_installs_with_local_poetry_directory_transitive( - installer, locker, repo, package, tmpdir + installer, locker, repo, package, tmpdir, fixture_dir ): - root_dir = fixtures_dir.joinpath("directory") + root_dir = fixture_dir("directory") package.root_dir = root_dir locker.set_lock_path(root_dir) directory = root_dir.joinpath("project_with_transitive_directory_dependencies") @@ -828,16 +827,16 @@ def test_run_installs_with_local_poetry_directory_transitive( def test_run_installs_with_local_poetry_file_transitive( - installer, locker, repo, package, tmpdir + installer, locker, repo, package, tmpdir, fixture_dir ): - root_dir = fixtures_dir.joinpath("directory") + root_dir = fixture_dir("directory") package.root_dir = root_dir locker.set_lock_path(root_dir) directory = root_dir.joinpath("project_with_transitive_file_dependencies") package.add_dependency( Factory.create_dependency( "project-with-transitive-file-dependencies", - {"path": str(directory.relative_to(fixtures_dir.joinpath("directory")))}, + {"path": str(directory.relative_to(fixture_dir("directory")))}, root_dir=root_dir, ) ) @@ -855,9 +854,9 @@ def test_run_installs_with_local_poetry_file_transitive( def test_run_installs_with_local_setuptools_directory( - installer, locker, repo, package, tmpdir + installer, locker, repo, package, tmpdir, fixture_dir ): - file_path = fixtures_dir / "project_with_setup/" + file_path = fixture_dir("project_with_setup/") package.add_dependency( Factory.create_dependency("project-with-setup", {"path": str(file_path)}) ) diff --git a/tests/puzzle/test_solver.py b/tests/puzzle/test_solver.py index f1ab2197898..1393b13f1ef 100644 --- a/tests/puzzle/test_solver.py +++ b/tests/puzzle/test_solver.py @@ -1672,6 +1672,60 @@ def test_solver_can_resolve_directory_dependencies(solver, repo, package): assert op.package.source_url == path +def test_solver_can_resolve_directory_dependencies_nested_editable( + solver, repo, pool, installed, locked, io +): + base = Path(__file__).parent.parent / "fixtures" / "project_with_nested_local" + poetry = Factory().create_poetry(cwd=base) + package = poetry.package + + solver = Solver( + package, pool, installed, locked, io, provider=Provider(package, pool, io) + ) + + ops = solver.solve() + + check_solver_result( + ops, + [ + { + "job": "install", + "package": Package( + "quix", + "1.2.3", + source_type="directory", + source_url=(base / "quix").as_posix(), + ), + "skipped": False, + }, + { + "job": "install", + "package": Package( + "bar", + "1.2.3", + source_type="directory", + source_url=(base / "bar").as_posix(), + ), + "skipped": False, + }, + { + "job": "install", + "package": Package( + "foo", + "1.2.3", + source_type="directory", + source_url=(base / "foo").as_posix(), + ), + "skipped": False, + }, + ], + ) + + for op in ops: + assert op.package.source_type == "directory" + assert op.package.develop is True + + def test_solver_can_resolve_directory_dependencies_with_extras(solver, repo, package): pendulum = get_package("pendulum", "2.0.3") cleo = get_package("cleo", "1.0.0")