From 6a7958c9ead78c605015e70220717a6e0d7277bf Mon Sep 17 00:00:00 2001 From: jakkdl Date: Tue, 25 Feb 2025 17:29:37 +0100 Subject: [PATCH 1/8] add --disable-plugin-autoload --- doc/en/how-to/plugins.rst | 23 ++++++++++++++- doc/en/reference/reference.rst | 7 +++-- src/_pytest/config/__init__.py | 26 ++++++++++++----- src/_pytest/helpconfig.py | 9 +++++- testing/test_assertion.py | 53 +++++++++++++++++++++++++++++----- testing/test_config.py | 52 +++++++++++++++++++++++++-------- 6 files changed, 140 insertions(+), 30 deletions(-) diff --git a/doc/en/how-to/plugins.rst b/doc/en/how-to/plugins.rst index 7d5bcd85a31..b94f14a2f85 100644 --- a/doc/en/how-to/plugins.rst +++ b/doc/en/how-to/plugins.rst @@ -133,4 +133,25 @@ CI server), you can set ``PYTEST_ADDOPTS`` environment variable to See :ref:`findpluginname` for how to obtain the name of a plugin. -.. _`builtin plugins`: +.. _`disable_plugin_autoload`: + +Disabling plugins from autoloading +---------------------------------- + +If you want to disable plugins from loading automatically, requiring you to +manually specify each plugin with ``-p`` or :envvar:`PYTEST_PLUGINS`, you can use ``--disable-plugin-autoload`` or :envvar:`PYTEST_DISABLE_PLUGIN_AUTOLOAD`. + +.. code-block:: bash + + export PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 + export PYTEST_PLUGINS=NAME + pytest + +.. code-block:: bash + + pytest --disable-plugin-autoload -p NAME,NAME2 + +.. code-block:: ini + + [pytest] + addopts = --disable-plugin-autoload -p NAME,NAME2 diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 809e97b4747..3eb6812df8b 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -1161,8 +1161,9 @@ as discussed in :ref:`temporary directory location and retention`. .. envvar:: PYTEST_DISABLE_PLUGIN_AUTOLOAD When set, disables plugin auto-loading through :std:doc:`entry point packaging -metadata `. Only explicitly -specified plugins will be loaded. +metadata `. Only plugins +explicitly specified in :envvar:`PYTEST_PLUGINS` or with ``-p`` will be loaded. +See also :ref:`--disable-plugin-autoload `. .. envvar:: PYTEST_PLUGINS @@ -1172,6 +1173,8 @@ Contains comma-separated list of modules that should be loaded as plugins: export PYTEST_PLUGINS=mymodule.plugin,xdist +See also ``-p``. + .. envvar:: PYTEST_THEME Sets a `pygment style `_ to use for the code output. diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 02da5cf9229..aacf2c12c06 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -70,6 +70,7 @@ if TYPE_CHECKING: + from _pytest.assertions.rewrite import AssertionRewritingHook from _pytest.cacheprovider import Cache from _pytest.terminal import TerminalReporter @@ -1271,6 +1272,10 @@ def _consider_importhook(self, args: Sequence[str]) -> None: """ ns, unknown_args = self._parser.parse_known_and_unknown_args(args) mode = getattr(ns, "assertmode", "plain") + + disable_autoload = getattr(ns, "disable_plugin_autoload", False) | bool( + os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + ) if mode == "rewrite": import _pytest.assertion @@ -1279,16 +1284,18 @@ def _consider_importhook(self, args: Sequence[str]) -> None: except SystemError: mode = "plain" else: - self._mark_plugins_for_rewrite(hook) + self._mark_plugins_for_rewrite(hook, disable_autoload) self._warn_about_missing_assertion(mode) - def _mark_plugins_for_rewrite(self, hook) -> None: + def _mark_plugins_for_rewrite( + self, hook: AssertionRewritingHook, disable_autoload: bool + ) -> None: """Given an importhook, mark for rewrite any top-level modules or packages in the distribution package for all pytest plugins.""" self.pluginmanager.rewrite_hook = hook - if os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"): + if disable_autoload: # We don't autoload from distribution package entry points, # no need to continue. return @@ -1393,10 +1400,15 @@ def _preparse(self, args: list[str], addopts: bool = True) -> None: self._consider_importhook(args) self._configure_python_path() self.pluginmanager.consider_preparse(args, exclude_only=False) - if not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"): - # Don't autoload from distribution package entry point. Only - # explicitly specified plugins are going to be loaded. + if ( + not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + and not self.known_args_namespace.disable_plugin_autoload + ): + # Autoloading from distribution package entry point has + # not been disabled. self.pluginmanager.load_setuptools_entrypoints("pytest11") + # Otherwise only plugins explicitly specified in PYTEST_PLUGINS + # are going to be loaded. self.pluginmanager.consider_env() self.known_args_namespace = self._parser.parse_known_args( @@ -1419,7 +1431,7 @@ def _preparse(self, args: list[str], addopts: bool = True) -> None: except ConftestImportFailure as e: if self.known_args_namespace.help or self.known_args_namespace.version: # we don't want to prevent --help/--version to work - # so just let is pass and print a warning at the end + # so just let it pass and print a warning at the end self.issue_config_time_warning( PytestConfigWarning(f"could not load initial conftests: {e.path}"), stacklevel=2, diff --git a/src/_pytest/helpconfig.py b/src/_pytest/helpconfig.py index 2b377c70f7b..d96ae19f6cc 100644 --- a/src/_pytest/helpconfig.py +++ b/src/_pytest/helpconfig.py @@ -70,7 +70,14 @@ def pytest_addoption(parser: Parser) -> None: metavar="name", help="Early-load given plugin module name or entry point (multi-allowed). " "To avoid loading of plugins, use the `no:` prefix, e.g. " - "`no:doctest`.", + "`no:doctest`. See also --disable-plugin-autoload", + ) + group.addoption( + "--disable-plugin-autoload", + action="store_true", + default=False, + help="Disable plugin auto-loading through entry point packaging metadata. " + "Only plugins explicitly specified in -p or env var PYTEST_PLUGINS will be loaded.", ) group.addoption( "--traceconfig", diff --git a/testing/test_assertion.py b/testing/test_assertion.py index e3d45478466..04284e0a55d 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -218,10 +218,40 @@ def test_foo(pytestconfig): assert result.ret == 0 @pytest.mark.parametrize("mode", ["plain", "rewrite"]) + @pytest.mark.parametrize("disable_plugin_autoload", ["env_var", "cli", ""]) + @pytest.mark.parametrize("explicit_specify", ["env_var", "cli", ""]) def test_installed_plugin_rewrite( - self, pytester: Pytester, mode, monkeypatch + self, + pytester: Pytester, + mode: str, + monkeypatch: pytest.MonkeyPatch, + disable_plugin_autoload: str, + explicit_specify: str, ) -> None: - monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + args = ["mainwrapper.py", "-s", f"--assert={mode}"] + if disable_plugin_autoload == "env_var": + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + elif disable_plugin_autoload == "cli": + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + args.append("--disable-plugin-autoload") + else: + assert disable_plugin_autoload == "" + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + + # FIXME: if it's already loaded then you get a ValueError: "Plugin already + # registered under a different name." + # vaguely related to https://github.com/pytest-dev/pytest/issues/5661 + name = "spamplugin" if disable_plugin_autoload else "spam" + # is there a single dotted name that can be used either way? idk + + if explicit_specify == "env_var": + monkeypatch.setenv("PYTEST_PLUGINS", name) + elif explicit_specify == "cli": + args.append("-p") + args.append(name) + else: + assert explicit_specify == "" + # Make sure the hook is installed early enough so that plugins # installed via distribution package are rewritten. pytester.mkdir("hampkg") @@ -275,20 +305,29 @@ def test(check_first): check_first([10, 30], 30) def test2(check_first2): - check_first([10, 30], 30) + check_first2([10, 30], 30) """, } pytester.makepyfile(**contents) - result = pytester.run( - sys.executable, "mainwrapper.py", "-s", f"--assert={mode}" - ) + result = pytester.run(sys.executable, *args) if mode == "plain": expected = "E AssertionError" elif mode == "rewrite": expected = "*assert 10 == 30*" else: assert 0 - result.stdout.fnmatch_lines([expected]) + + if not disable_plugin_autoload or explicit_specify: + result.assert_outcomes(failed=2) + result.stdout.fnmatch_lines([expected, expected]) + else: + result.assert_outcomes(errors=2) + result.stdout.fnmatch_lines( + [ + "E fixture 'check_first' not found", + "E fixture 'check_first2' not found", + ] + ) def test_rewrite_ast(self, pytester: Pytester) -> None: pytester.mkdir("pkg") diff --git a/testing/test_config.py b/testing/test_config.py index de07141238c..daec49eda5c 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -1314,14 +1314,13 @@ def distributions(): ) -@pytest.mark.parametrize( - "parse_args,should_load", [(("-p", "mytestplugin"), True), ((), False)] -) +@pytest.mark.parametrize("disable_plugin_method", ["env_var", "flag", ""]) +@pytest.mark.parametrize("enable_plugin_method", ["env_var", "flag", ""]) def test_disable_plugin_autoload( pytester: Pytester, monkeypatch: MonkeyPatch, - parse_args: tuple[str, str] | tuple[()], - should_load: bool, + enable_plugin_method: str, + disable_plugin_method: str, ) -> None: class DummyEntryPoint: project_name = name = "mytestplugin" @@ -1342,23 +1341,52 @@ class PseudoPlugin: attrs_used = [] def __getattr__(self, name): - assert name == "__loader__" + assert name in ("__loader__", "__spec__") self.attrs_used.append(name) return object() def distributions(): return (Distribution(),) - monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + parse_args: list[str] = [] + + if disable_plugin_method == "env_var": + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + elif disable_plugin_method == "flag": + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + parse_args.append("--disable-plugin-autoload") + else: + assert disable_plugin_method == "" + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + + if enable_plugin_method == "env_var": + monkeypatch.setenv("PYTEST_PLUGINS", "mytestplugin") + elif enable_plugin_method == "flag": + parse_args.extend(["-p", "mytestplugin"]) + else: + assert enable_plugin_method == "" + monkeypatch.setattr(importlib.metadata, "distributions", distributions) monkeypatch.setitem(sys.modules, "mytestplugin", PseudoPlugin()) config = pytester.parseconfig(*parse_args) + has_loaded = config.pluginmanager.get_plugin("mytestplugin") is not None - assert has_loaded == should_load - if should_load: - assert PseudoPlugin.attrs_used == ["__loader__"] - else: - assert PseudoPlugin.attrs_used == [] + # it should load if it's enabled, or we haven't disabled autoloading + assert has_loaded == bool(enable_plugin_method) or not disable_plugin_method + + # __loader__ is accessed in mark_rewrite + # ...?? + assert ("__loader__" in PseudoPlugin.attrs_used) == bool( + enable_plugin_method == "flag" + or (enable_plugin_method == "env_var" and disable_plugin_method) + ) + + # Config._preparse explicitly loads plugins in PYTEST_PLUGINS + # but if autoloading has been disabled it needs to inspect __spec__ when loading + assert ("__spec__" in PseudoPlugin.attrs_used) == bool( + enable_plugin_method == "env_var" and disable_plugin_method + ) + # why doesn't that happen with -p? dunno def test_plugin_loading_order(pytester: Pytester) -> None: From bfd049e60c734b4daf46feff763a14c03288849e Mon Sep 17 00:00:00 2001 From: jakkdl Date: Thu, 27 Feb 2025 13:11:57 +0100 Subject: [PATCH 2/8] update comments in test, don't check __spec__ on pypy (????), add changelog --- changelog/13253.feature.rst | 1 + testing/test_config.py | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 11 deletions(-) create mode 100644 changelog/13253.feature.rst diff --git a/changelog/13253.feature.rst b/changelog/13253.feature.rst new file mode 100644 index 00000000000..e497c207223 --- /dev/null +++ b/changelog/13253.feature.rst @@ -0,0 +1 @@ +New flag: :ref:`--disable-plugin-autoload ` which works as an alternative to :envvar:`PYTEST_DISABLE_PLUGIN_AUTOLOAD` when setting environment variables is inconvenient; and allows setting it in config files with :confval:`addopts`. diff --git a/testing/test_config.py b/testing/test_config.py index daec49eda5c..9317ba398ff 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -6,6 +6,7 @@ import importlib.metadata import os from pathlib import Path +import platform import re import sys import textwrap @@ -1374,19 +1375,26 @@ def distributions(): # it should load if it's enabled, or we haven't disabled autoloading assert has_loaded == bool(enable_plugin_method) or not disable_plugin_method - # __loader__ is accessed in mark_rewrite - # ...?? - assert ("__loader__" in PseudoPlugin.attrs_used) == bool( - enable_plugin_method == "flag" - or (enable_plugin_method == "env_var" and disable_plugin_method) + # The reason for the discrepancy between 'has_loaded' and __loader__ being accessed + # appears to be the monkeypatching of importlib.metadata.distributions; where + # files being empty means that _mark_plugins_for_rewrite doesn't find the plugin. + # But enable_method==flag ends up in mark_rewrite being called and __loader__ + # being accessed. + assert ("__loader__" in PseudoPlugin.attrs_used) == has_loaded and not ( + enable_plugin_method in ("env_var", "") and not disable_plugin_method ) - # Config._preparse explicitly loads plugins in PYTEST_PLUGINS - # but if autoloading has been disabled it needs to inspect __spec__ when loading - assert ("__spec__" in PseudoPlugin.attrs_used) == bool( - enable_plugin_method == "env_var" and disable_plugin_method - ) - # why doesn't that happen with -p? dunno + # __spec__ is accessed in AssertionRewritingHook.exec_module, which would be + # eventually called if we did a full pytest run; but it's only accessed with + # enable_plugin_method=="env_var" because that will early-load it. + # Except when autoloads aren't disabled, in which case PytestPluginManager.import_plugin + # bails out before importing it.. because it knows it'll be loaded later? + # The above seems a bit weird, but I *think* it's true. + if platform.python_implementation() != "PyPy": + assert ("__spec__" in PseudoPlugin.attrs_used) == bool( + enable_plugin_method == "env_var" and disable_plugin_method + ) + # __spec__ is present when testing locally on pypy, but not in CI ???? def test_plugin_loading_order(pytester: Pytester) -> None: From 60becec6e3ead2195d6a9e01279460220ac73ade Mon Sep 17 00:00:00 2001 From: jakkdl Date: Thu, 27 Feb 2025 16:02:20 +0100 Subject: [PATCH 3/8] pemdas except not --- testing/test_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/test_config.py b/testing/test_config.py index 9317ba398ff..52fdaadbec7 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -1380,8 +1380,9 @@ def distributions(): # files being empty means that _mark_plugins_for_rewrite doesn't find the plugin. # But enable_method==flag ends up in mark_rewrite being called and __loader__ # being accessed. - assert ("__loader__" in PseudoPlugin.attrs_used) == has_loaded and not ( - enable_plugin_method in ("env_var", "") and not disable_plugin_method + assert ("__loader__" in PseudoPlugin.attrs_used) == ( + has_loaded + and not (enable_plugin_method in ("env_var", "") and not disable_plugin_method) ) # __spec__ is accessed in AssertionRewritingHook.exec_module, which would be From 48f4e2a8005484bdd6edd64b2a226df4e6de0ebd Mon Sep 17 00:00:00 2001 From: John Litborn <11260241+jakkdl@users.noreply.github.com> Date: Fri, 28 Feb 2025 16:42:12 +0100 Subject: [PATCH 4/8] Apply suggestions from code review Co-authored-by: Florian Bruhin --- doc/en/how-to/plugins.rst | 2 +- src/_pytest/helpconfig.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/en/how-to/plugins.rst b/doc/en/how-to/plugins.rst index b94f14a2f85..8048ea076d2 100644 --- a/doc/en/how-to/plugins.rst +++ b/doc/en/how-to/plugins.rst @@ -144,7 +144,7 @@ manually specify each plugin with ``-p`` or :envvar:`PYTEST_PLUGINS`, you can us .. code-block:: bash export PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 - export PYTEST_PLUGINS=NAME + export PYTEST_PLUGINS=NAME,NAME2 pytest .. code-block:: bash diff --git a/src/_pytest/helpconfig.py b/src/_pytest/helpconfig.py index d96ae19f6cc..b5ac0e6a50c 100644 --- a/src/_pytest/helpconfig.py +++ b/src/_pytest/helpconfig.py @@ -70,7 +70,7 @@ def pytest_addoption(parser: Parser) -> None: metavar="name", help="Early-load given plugin module name or entry point (multi-allowed). " "To avoid loading of plugins, use the `no:` prefix, e.g. " - "`no:doctest`. See also --disable-plugin-autoload", + "`no:doctest`. See also --disable-plugin-autoload.", ) group.addoption( "--disable-plugin-autoload", From 98f940c5c5e00d3e938556edf22f798fd5d6b35c Mon Sep 17 00:00:00 2001 From: jakkdl Date: Fri, 28 Feb 2025 16:45:09 +0100 Subject: [PATCH 5/8] add parens --- testing/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test_config.py b/testing/test_config.py index 52fdaadbec7..bb08c40fef4 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -1373,7 +1373,7 @@ def distributions(): has_loaded = config.pluginmanager.get_plugin("mytestplugin") is not None # it should load if it's enabled, or we haven't disabled autoloading - assert has_loaded == bool(enable_plugin_method) or not disable_plugin_method + assert has_loaded == (bool(enable_plugin_method) or not disable_plugin_method) # The reason for the discrepancy between 'has_loaded' and __loader__ being accessed # appears to be the monkeypatching of importlib.metadata.distributions; where From b157cefedf39fa1115dcab4ccfbf4a81fe7339ca Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 1 Mar 2025 10:21:08 -0300 Subject: [PATCH 6/8] Simplify plugin name in test_installed_plugin_rewrite --- testing/test_assertion.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/testing/test_assertion.py b/testing/test_assertion.py index 04284e0a55d..2c2830eb929 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -238,11 +238,7 @@ def test_installed_plugin_rewrite( assert disable_plugin_autoload == "" monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) - # FIXME: if it's already loaded then you get a ValueError: "Plugin already - # registered under a different name." - # vaguely related to https://github.com/pytest-dev/pytest/issues/5661 - name = "spamplugin" if disable_plugin_autoload else "spam" - # is there a single dotted name that can be used either way? idk + name = "spamplugin" if explicit_specify == "env_var": monkeypatch.setenv("PYTEST_PLUGINS", name) @@ -280,7 +276,7 @@ def check(values, value): import pytest class DummyEntryPoint(object): - name = 'spam' + name = 'spamplugin' module_name = 'spam.py' group = 'pytest11' From 40d8ceb8621f1485f52d8cbd0766ade37bb2e298 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 1 Mar 2025 10:25:05 -0300 Subject: [PATCH 7/8] Apply suggestions from code review --- doc/en/how-to/plugins.rst | 6 +++++- src/_pytest/config/__init__.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/en/how-to/plugins.rst b/doc/en/how-to/plugins.rst index 8048ea076d2..0fe97d34241 100644 --- a/doc/en/how-to/plugins.rst +++ b/doc/en/how-to/plugins.rst @@ -138,7 +138,7 @@ See :ref:`findpluginname` for how to obtain the name of a plugin. Disabling plugins from autoloading ---------------------------------- -If you want to disable plugins from loading automatically, requiring you to +If you want to disable plugins from loading automatically, instead of requiring you to manually specify each plugin with ``-p`` or :envvar:`PYTEST_PLUGINS`, you can use ``--disable-plugin-autoload`` or :envvar:`PYTEST_DISABLE_PLUGIN_AUTOLOAD`. .. code-block:: bash @@ -155,3 +155,7 @@ manually specify each plugin with ``-p`` or :envvar:`PYTEST_PLUGINS`, you can us [pytest] addopts = --disable-plugin-autoload -p NAME,NAME2 + +.. versionadded:: 8.4 + + The ``--disable-plugin-autoload`` command-line flag. diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index aacf2c12c06..56b04719641 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1273,7 +1273,7 @@ def _consider_importhook(self, args: Sequence[str]) -> None: ns, unknown_args = self._parser.parse_known_and_unknown_args(args) mode = getattr(ns, "assertmode", "plain") - disable_autoload = getattr(ns, "disable_plugin_autoload", False) | bool( + disable_autoload = getattr(ns, "disable_plugin_autoload", False) or bool( os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD") ) if mode == "rewrite": From e7403ac2a9d173e99759cff08f25f894c0309a96 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 1 Mar 2025 13:25:26 +0000 Subject: [PATCH 8/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/en/how-to/plugins.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/en/how-to/plugins.rst b/doc/en/how-to/plugins.rst index 0fe97d34241..fca8ab54e63 100644 --- a/doc/en/how-to/plugins.rst +++ b/doc/en/how-to/plugins.rst @@ -155,7 +155,7 @@ manually specify each plugin with ``-p`` or :envvar:`PYTEST_PLUGINS`, you can us [pytest] addopts = --disable-plugin-autoload -p NAME,NAME2 - + .. versionadded:: 8.4 The ``--disable-plugin-autoload`` command-line flag.