From b70e5ef7a2197cd9ca820e03450de930f11d1239 Mon Sep 17 00:00:00 2001 From: finswimmer Date: Tue, 30 Nov 2021 08:44:06 +0100 Subject: [PATCH 1/5] proof of concept to detect current running python through pyenv --- src/poetry/console/application.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/poetry/console/application.py b/src/poetry/console/application.py index 158b7bed604..b7d910965ea 100644 --- a/src/poetry/console/application.py +++ b/src/poetry/console/application.py @@ -1,8 +1,11 @@ +import contextlib import logging import re +import subprocess from contextlib import suppress from importlib import import_module +from subprocess import CalledProcessError from typing import TYPE_CHECKING from typing import Any from typing import Callable @@ -21,6 +24,8 @@ from poetry.__version__ import __version__ from poetry.console.command_loader import CommandLoader from poetry.console.commands.command import Command +from poetry.utils._compat import decode +from poetry.utils._compat import list_to_shell_command if TYPE_CHECKING: @@ -281,9 +286,18 @@ def configure_env( io = event.io poetry = command.poetry + executable = None + + with contextlib.suppress(CalledProcessError): + executable = decode( + subprocess.check_output( + list_to_shell_command(["pyenv", "which", "python"]), + shell=True, + ).strip() + ) env_manager = EnvManager(poetry) - env = env_manager.create_venv(io) + env = env_manager.create_venv(io, executable=executable) if env.is_venv() and io.is_verbose(): io.write_line(f"Using virtualenv: {env.path}") From c5a4e6ca4b78681ddc77f8f1f8d717e7333b05bd Mon Sep 17 00:00:00 2001 From: finswimmer Date: Tue, 30 Nov 2021 11:27:44 +0100 Subject: [PATCH 2/5] alternative generic solution by returning sys.executable --- src/poetry/console/application.py | 16 ++++++++++++++-- src/poetry/utils/env.py | 3 ++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/poetry/console/application.py b/src/poetry/console/application.py index b7d910965ea..80cde5dcfb7 100644 --- a/src/poetry/console/application.py +++ b/src/poetry/console/application.py @@ -286,18 +286,30 @@ def configure_env( io = event.io poetry = command.poetry + executable = None + find_compatible = None + # add on option to trigger this with contextlib.suppress(CalledProcessError): executable = decode( subprocess.check_output( - list_to_shell_command(["pyenv", "which", "python"]), + list_to_shell_command( + [ + "python", + "-c", + '"import sys; print(sys.executable)"', + ] + ), shell=True, ).strip() ) + find_compatible = True env_manager = EnvManager(poetry) - env = env_manager.create_venv(io, executable=executable) + env = env_manager.create_venv( + io, executable=executable, find_compatible=find_compatible + ) if env.is_venv() and io.is_verbose(): io.write_line(f"Using virtualenv: {env.path}") diff --git a/src/poetry/utils/env.py b/src/poetry/utils/env.py index 5eadf3cdfd2..a66bd56af71 100644 --- a/src/poetry/utils/env.py +++ b/src/poetry/utils/env.py @@ -767,6 +767,7 @@ def create_venv( name: Optional[str] = None, executable: Optional[str] = None, force: bool = False, + find_compatible: Optional[bool] = None, ) -> Union["SystemEnv", "VirtualEnv"]: if self._env is not None and not force: return self._env @@ -820,7 +821,7 @@ def create_venv( # If an executable has been specified, we stop there # and notify the user of the incompatibility. # Otherwise, we try to find a compatible Python version. - if executable: + if executable and not find_compatible: raise NoCompatiblePythonVersionFound( self._poetry.package.python_versions, python_patch ) From ee11f76ee645f968f9516b4c47e1411a520f2f9a Mon Sep 17 00:00:00 2001 From: finswimmer Date: Tue, 30 Nov 2021 14:35:30 +0100 Subject: [PATCH 3/5] add option `virtualenvs.prefer-shell-python` --- src/poetry/config/config.py | 2 + src/poetry/console/application.py | 53 +++++++++++++++++---------- src/poetry/console/commands/config.py | 5 +++ tests/console/commands/test_config.py | 3 ++ 4 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/poetry/config/config.py b/src/poetry/config/config.py index 636fa2097dd..86a863b0e7a 100644 --- a/src/poetry/config/config.py +++ b/src/poetry/config/config.py @@ -38,6 +38,7 @@ class Config: "in-project": None, "path": os.path.join("{cache-dir}", "virtualenvs"), "options": {"always-copy": False, "system-site-packages": False}, + "prefer-shell-python": False, }, "experimental": {"new-installer": True}, "installer": {"parallel": True, "max-workers": None}, @@ -138,6 +139,7 @@ def _get_normalizer(name: str) -> Callable: "virtualenvs.in-project", "virtualenvs.options.always-copy", "virtualenvs.options.system-site-packages", + "virtualenvs.options.prefer-shell-python", "experimental.new-installer", "installer.parallel", }: diff --git a/src/poetry/console/application.py b/src/poetry/console/application.py index 80cde5dcfb7..94adddd1fba 100644 --- a/src/poetry/console/application.py +++ b/src/poetry/console/application.py @@ -1,4 +1,3 @@ -import contextlib import logging import re import subprocess @@ -19,6 +18,7 @@ from cleo.exceptions import CleoException from cleo.formatters.style import Style from cleo.io.inputs.argv_input import ArgvInput +from cleo.io.outputs.output import Verbosity from poetry.core.utils._compat import PY37 from poetry.__version__ import __version__ @@ -229,6 +229,30 @@ def _configure_io(self, io: "IO") -> None: return super()._configure_io(io) + def _detect_active_python(self, io: "IO") -> str: + executable = None + + try: + io.write_line( + "Trying to detect current active python executable as specified in the config.", + verbosity=Verbosity.VERBOSE, + ) + executable = decode( + subprocess.check_output( + list_to_shell_command( + ["python", "-c", '"import sys; print(sys.executable)"'] + ), + shell=True, + ).strip() + ) + io.write_line(f"Found: {executable}", verbosity=Verbosity.VERBOSE) + except CalledProcessError: + io.write_line( + "Unable to detect the current active python executable. Falling back to default.", + verbosity=Verbosity.VERBOSE, + ) + return executable + def register_command_loggers( self, event: "ConsoleCommandEvent", event_name: str, _: Any ) -> None: @@ -287,28 +311,17 @@ def configure_env( io = event.io poetry = command.poetry - executable = None - find_compatible = None - - # add on option to trigger this - with contextlib.suppress(CalledProcessError): - executable = decode( - subprocess.check_output( - list_to_shell_command( - [ - "python", - "-c", - '"import sys; print(sys.executable)"', - ] - ), - shell=True, - ).strip() - ) - find_compatible = True + executable = ( + self._detect_active_python(io) + if poetry.config.get("virtualenvs.prefer-shell-python") + else None + ) env_manager = EnvManager(poetry) env = env_manager.create_venv( - io, executable=executable, find_compatible=find_compatible + io, + executable=executable, + find_compatible=True if executable else None, ) if env.is_venv() and io.is_verbose(): diff --git a/src/poetry/console/commands/config.py b/src/poetry/console/commands/config.py index 96df3cd4514..a8f78867757 100644 --- a/src/poetry/console/commands/config.py +++ b/src/poetry/console/commands/config.py @@ -78,6 +78,11 @@ def unique_config_values(self) -> Dict[str, Tuple[Any, Any, Any]]: lambda val: str(Path(val)), str(Path(CACHE_DIR) / "virtualenvs"), ), + "virtualenvs.prefer-shell-python": ( + boolean_validator, + boolean_normalizer, + False, + ), "experimental.new-installer": ( boolean_validator, boolean_normalizer, diff --git a/tests/console/commands/test_config.py b/tests/console/commands/test_config.py index 3278b19a170..9aca2b3d78a 100644 --- a/tests/console/commands/test_config.py +++ b/tests/console/commands/test_config.py @@ -56,6 +56,7 @@ def test_list_displays_default_value_if_not_set( virtualenvs.options.always-copy = false virtualenvs.options.system-site-packages = false virtualenvs.path = {venv_path} # {config_cache_dir / 'virtualenvs'} +virtualenvs.prefer-shell-python = false """ assert expected == tester.io.fetch_output() @@ -79,6 +80,7 @@ def test_list_displays_set_get_setting( virtualenvs.options.always-copy = false virtualenvs.options.system-site-packages = false virtualenvs.path = {venv_path} # {config_cache_dir / 'virtualenvs'} +virtualenvs.prefer-shell-python = false """ assert config.set_config_source.call_count == 0 @@ -126,6 +128,7 @@ def test_list_displays_set_get_local_setting( virtualenvs.options.always-copy = false virtualenvs.options.system-site-packages = false virtualenvs.path = {venv_path} # {config_cache_dir / 'virtualenvs'} +virtualenvs.prefer-shell-python = false """ assert config.set_config_source.call_count == 1 From 411dc4960acd0de0c22bdd681fd017c134c27d3a Mon Sep 17 00:00:00 2001 From: finswimmer Date: Wed, 1 Dec 2021 06:04:28 +0100 Subject: [PATCH 4/5] update docs --- docs/configuration.md | 8 ++++++++ docs/managing-environments.md | 17 ++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index c93322310a1..3887446006c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -48,6 +48,7 @@ virtualenvs.in-project = null virtualenvs.options.always-copy = true virtualenvs.options.system-site-packages = false virtualenvs.path = "{cache-dir}/virtualenvs" # /path/to/cache/directory/virtualenvs +virtualenvs.prefer-shell-python = false ``` ## Displaying a single configuration setting @@ -188,6 +189,13 @@ Give the virtual environment access to the system site-packages directory. Applies on virtualenv creation. Defaults to `false`. +### `virtualenvs.prefer-shell-python` + +**Type**: boolean + +Use currently activated Python version to create a new venv. +Defaults to `false`, which means Python version used during Poetry installation is used. + ### `repositories.` **Type**: string diff --git a/docs/managing-environments.md b/docs/managing-environments.md index 632db282917..0c7a63fecf6 100644 --- a/docs/managing-environments.md +++ b/docs/managing-environments.md @@ -18,7 +18,7 @@ To achieve this, it will first check if it's currently running inside a virtual If it is, it will use it directly without creating a new one. But if it's not, it will use one that it has already created or create a brand new one for you. -By default, Poetry will try to use the currently activated Python version +By default, Poetry will try to use the Python version used during Poetry's installation to create the virtual environment for the current project. However, for various reasons, this Python version might not be compatible @@ -26,6 +26,21 @@ with the `python` requirement of the project. In this case, Poetry will try to find one that is and use it. If it's unable to do so then you will be prompted to activate one explicitly, see [Switching environments](#switching-between-environments). +{{% note %}} +To easily switch between Python versions, it is recommended to +use [pyenv](https://github.com/pyenv/pyenv) or similar tools. + +For instance, if your project requires a newer Python than is available with +your system, a standard workflow would be: + +```bash +pyenv install 3.9.8 +pyenv local 3.9.8 # Activate Python 3.9 for the current project +poetry install +``` +This requires setting the `virtualenvs.prefer-shell-python` option to `true`. +{{% /note %}} + ## Switching between environments Sometimes this might not be feasible for your system, especially Windows where `pyenv` From bfe848c5ad5833a64c2b15a7ec6f6194baebb61e Mon Sep 17 00:00:00 2001 From: finswimmer Date: Wed, 1 Dec 2021 06:39:56 +0100 Subject: [PATCH 5/5] rename option `virtualenvs.prefer-active-python` and move whole logic to EnvManager.create_venv() --- docs/configuration.md | 4 +-- docs/managing-environments.md | 6 ++-- src/poetry/config/config.py | 4 +-- src/poetry/console/application.py | 41 +-------------------------- src/poetry/console/commands/config.py | 2 +- src/poetry/utils/env.py | 34 ++++++++++++++++++++-- tests/console/commands/test_config.py | 6 ++-- 7 files changed, 44 insertions(+), 53 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3887446006c..b81edd26513 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -48,7 +48,7 @@ virtualenvs.in-project = null virtualenvs.options.always-copy = true virtualenvs.options.system-site-packages = false virtualenvs.path = "{cache-dir}/virtualenvs" # /path/to/cache/directory/virtualenvs -virtualenvs.prefer-shell-python = false +virtualenvs.prefer-active-python = false ``` ## Displaying a single configuration setting @@ -189,7 +189,7 @@ Give the virtual environment access to the system site-packages directory. Applies on virtualenv creation. Defaults to `false`. -### `virtualenvs.prefer-shell-python` +### `virtualenvs.prefer-active-python` (experimental) **Type**: boolean diff --git a/docs/managing-environments.md b/docs/managing-environments.md index 0c7a63fecf6..605ef9940cc 100644 --- a/docs/managing-environments.md +++ b/docs/managing-environments.md @@ -27,8 +27,9 @@ to find one that is and use it. If it's unable to do so then you will be prompte to activate one explicitly, see [Switching environments](#switching-between-environments). {{% note %}} -To easily switch between Python versions, it is recommended to -use [pyenv](https://github.com/pyenv/pyenv) or similar tools. +If you use a tool like [pyenv](https://github.com/pyenv/pyenv) to manage different Python versions, +you can set the experimental `virtualenvs.prefer-active-python` option to `true`. Poetry +than will try to find the current `python` of your shell. For instance, if your project requires a newer Python than is available with your system, a standard workflow would be: @@ -38,7 +39,6 @@ pyenv install 3.9.8 pyenv local 3.9.8 # Activate Python 3.9 for the current project poetry install ``` -This requires setting the `virtualenvs.prefer-shell-python` option to `true`. {{% /note %}} ## Switching between environments diff --git a/src/poetry/config/config.py b/src/poetry/config/config.py index 86a863b0e7a..bf4898bfdd5 100644 --- a/src/poetry/config/config.py +++ b/src/poetry/config/config.py @@ -38,7 +38,7 @@ class Config: "in-project": None, "path": os.path.join("{cache-dir}", "virtualenvs"), "options": {"always-copy": False, "system-site-packages": False}, - "prefer-shell-python": False, + "prefer-active-python": False, }, "experimental": {"new-installer": True}, "installer": {"parallel": True, "max-workers": None}, @@ -139,7 +139,7 @@ def _get_normalizer(name: str) -> Callable: "virtualenvs.in-project", "virtualenvs.options.always-copy", "virtualenvs.options.system-site-packages", - "virtualenvs.options.prefer-shell-python", + "virtualenvs.options.prefer-active-python", "experimental.new-installer", "installer.parallel", }: diff --git a/src/poetry/console/application.py b/src/poetry/console/application.py index 94adddd1fba..158b7bed604 100644 --- a/src/poetry/console/application.py +++ b/src/poetry/console/application.py @@ -1,10 +1,8 @@ import logging import re -import subprocess from contextlib import suppress from importlib import import_module -from subprocess import CalledProcessError from typing import TYPE_CHECKING from typing import Any from typing import Callable @@ -18,14 +16,11 @@ from cleo.exceptions import CleoException from cleo.formatters.style import Style from cleo.io.inputs.argv_input import ArgvInput -from cleo.io.outputs.output import Verbosity from poetry.core.utils._compat import PY37 from poetry.__version__ import __version__ from poetry.console.command_loader import CommandLoader from poetry.console.commands.command import Command -from poetry.utils._compat import decode -from poetry.utils._compat import list_to_shell_command if TYPE_CHECKING: @@ -229,30 +224,6 @@ def _configure_io(self, io: "IO") -> None: return super()._configure_io(io) - def _detect_active_python(self, io: "IO") -> str: - executable = None - - try: - io.write_line( - "Trying to detect current active python executable as specified in the config.", - verbosity=Verbosity.VERBOSE, - ) - executable = decode( - subprocess.check_output( - list_to_shell_command( - ["python", "-c", '"import sys; print(sys.executable)"'] - ), - shell=True, - ).strip() - ) - io.write_line(f"Found: {executable}", verbosity=Verbosity.VERBOSE) - except CalledProcessError: - io.write_line( - "Unable to detect the current active python executable. Falling back to default.", - verbosity=Verbosity.VERBOSE, - ) - return executable - def register_command_loggers( self, event: "ConsoleCommandEvent", event_name: str, _: Any ) -> None: @@ -311,18 +282,8 @@ def configure_env( io = event.io poetry = command.poetry - executable = ( - self._detect_active_python(io) - if poetry.config.get("virtualenvs.prefer-shell-python") - else None - ) - env_manager = EnvManager(poetry) - env = env_manager.create_venv( - io, - executable=executable, - find_compatible=True if executable else None, - ) + env = env_manager.create_venv(io) if env.is_venv() and io.is_verbose(): io.write_line(f"Using virtualenv: {env.path}") diff --git a/src/poetry/console/commands/config.py b/src/poetry/console/commands/config.py index a8f78867757..785abe361fe 100644 --- a/src/poetry/console/commands/config.py +++ b/src/poetry/console/commands/config.py @@ -78,7 +78,7 @@ def unique_config_values(self) -> Dict[str, Tuple[Any, Any, Any]]: lambda val: str(Path(val)), str(Path(CACHE_DIR) / "virtualenvs"), ), - "virtualenvs.prefer-shell-python": ( + "virtualenvs.prefer-active-python": ( boolean_validator, boolean_normalizer, False, diff --git a/src/poetry/utils/env.py b/src/poetry/utils/env.py index a66bd56af71..3c18aaa62e1 100644 --- a/src/poetry/utils/env.py +++ b/src/poetry/utils/env.py @@ -30,6 +30,7 @@ import tomlkit import virtualenv +from cleo.io.outputs.output import Verbosity from packaging.tags import Tag from packaging.tags import interpreter_name from packaging.tags import interpreter_version @@ -456,6 +457,30 @@ class EnvManager: def __init__(self, poetry: "Poetry") -> None: self._poetry = poetry + def _detect_active_python(self, io: "IO") -> str: + executable = None + + try: + io.write_line( + "Trying to detect current active python executable as specified in the config.", + verbosity=Verbosity.VERBOSE, + ) + executable = decode( + subprocess.check_output( + list_to_shell_command( + ["python", "-c", '"import sys; print(sys.executable)"'] + ), + shell=True, + ).strip() + ) + io.write_line(f"Found: {executable}", verbosity=Verbosity.VERBOSE) + except CalledProcessError: + io.write_line( + "Unable to detect the current active python executable. Falling back to default.", + verbosity=Verbosity.VERBOSE, + ) + return executable + def activate(self, python: str, io: "IO") -> "Env": venv_path = self._poetry.config.get("virtualenvs.path") if venv_path is None: @@ -767,7 +792,6 @@ def create_venv( name: Optional[str] = None, executable: Optional[str] = None, force: bool = False, - find_compatible: Optional[bool] = None, ) -> Union["SystemEnv", "VirtualEnv"]: if self._env is not None and not force: return self._env @@ -785,6 +809,12 @@ def create_venv( create_venv = self._poetry.config.get("virtualenvs.create") root_venv = self._poetry.config.get("virtualenvs.in-project") venv_path = self._poetry.config.get("virtualenvs.path") + prefer_active_python = self._poetry.config.get( + "virtualenvs.prefer-active-python" + ) + + if not executable and prefer_active_python: + executable = self._detect_active_python(io) if root_venv: venv_path = cwd / ".venv" @@ -821,7 +851,7 @@ def create_venv( # If an executable has been specified, we stop there # and notify the user of the incompatibility. # Otherwise, we try to find a compatible Python version. - if executable and not find_compatible: + if executable and not prefer_active_python: raise NoCompatiblePythonVersionFound( self._poetry.package.python_versions, python_patch ) diff --git a/tests/console/commands/test_config.py b/tests/console/commands/test_config.py index 9aca2b3d78a..f515eb4fc67 100644 --- a/tests/console/commands/test_config.py +++ b/tests/console/commands/test_config.py @@ -56,7 +56,7 @@ def test_list_displays_default_value_if_not_set( virtualenvs.options.always-copy = false virtualenvs.options.system-site-packages = false virtualenvs.path = {venv_path} # {config_cache_dir / 'virtualenvs'} -virtualenvs.prefer-shell-python = false +virtualenvs.prefer-active-python = false """ assert expected == tester.io.fetch_output() @@ -80,7 +80,7 @@ def test_list_displays_set_get_setting( virtualenvs.options.always-copy = false virtualenvs.options.system-site-packages = false virtualenvs.path = {venv_path} # {config_cache_dir / 'virtualenvs'} -virtualenvs.prefer-shell-python = false +virtualenvs.prefer-active-python = false """ assert config.set_config_source.call_count == 0 @@ -128,7 +128,7 @@ def test_list_displays_set_get_local_setting( virtualenvs.options.always-copy = false virtualenvs.options.system-site-packages = false virtualenvs.path = {venv_path} # {config_cache_dir / 'virtualenvs'} -virtualenvs.prefer-shell-python = false +virtualenvs.prefer-active-python = false """ assert config.set_config_source.call_count == 1