diff --git a/CHANGES.rst b/CHANGES.rst index 2afab31d5..bf75e0be6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ Changelog ========= +Unreleased +----------- +- Add options to control free variables in Examples and step definitions + + 5.0.0 ----- This release introduces breaking changes, please refer to the :ref:`Migration from 4.x.x`. diff --git a/README.rst b/README.rst index 7bf302d5e..576d68581 100644 --- a/README.rst +++ b/README.rst @@ -696,7 +696,85 @@ With a parametrized.feature file: The significant downside of this approach is inability to see the test table from the feature file. +It's possible to disallow steps free parameters substitution from fixtures (so test case will fail): +.. code-block:: python + + @pytest.mark.parametrize( + ["start", "eat", "left"], + [(12, 5, 7)], + ) + @scenario( + "parametrized.feature", + "Parametrized given, when, thens", + allow_step_free_variables=False, + ) + def test_parametrized(start, eat, left): + """We don't need to do anything here, everything will be managed by the scenario decorator.""" + +Sometimes you want leave a column not used in steps for specific reason in examples section: + +.. code-block:: gherkin + + Feature: Scenario outlines + Scenario Outline: Outlined given, when, thens + Given there are cucumbers + When I eat cucumbers + Then I should have cucumbers + + Examples: + | start | eat | left | comment | + | 12 | 5 | 7 | sweet cucumbers!| + + +.. code-block:: python + + from pytest_bdd import given, when, then, scenario + + + @scenario( + "outline.feature", + "Outlined given, when, thens", + allow_example_free_variables=True, + ) + def test_outlined(): + pass + +Or leave some parameter as is without substitution: + +.. code-block:: gherkin + + Feature: Outline + Scenario Outline: Outlined with wrong examples + Given there are cucumbers + When I eat cucumbers + Then I should have cucumbers in my bucket + + Examples: + | start | eat | left | + | 12 | 5 | 7 | + +.. code-block:: python + + @scenario( + "outline.feature", + "Outlined with wrong examples", + allow_step_free_variables=False + ) + def test_outline(request): + pass + + @then(parsers.parse('I should have {left} cucumbers in my bucket')) + def stepdef(left): + pass + + +Also you could grant such possibility for whole session using pytest.ini (or any other same config) + +.. code-block:: ini + + [pytest] + bdd_allow_step_free_variables=false Organizing your scenarios ------------------------- diff --git a/pytest_bdd/parser.py b/pytest_bdd/parser.py index 8cbfbb145..a39e37868 100644 --- a/pytest_bdd/parser.py +++ b/pytest_bdd/parser.py @@ -232,6 +232,9 @@ def __init__(self, feature: Feature, name: str, line_number: int, tags=None): self.line_number = line_number self.tags = tags or set() + self.allow_example_free_variables = None + self.allow_step_free_variables = None + def add_step(self, step): """Add step to the scenario. @@ -258,18 +261,57 @@ def render(self, context: typing.Mapping[str, typing.Any]) -> "Scenario": ] return Scenario(feature=self.feature, name=self.name, line_number=self.line_number, steps=steps, tags=self.tags) + @property + def params(self): + return frozenset(sum((list(step.params) for step in self.steps), [])) + + def get_example_params(self): + return set(self.examples.example_params + self.feature.examples.example_params) + def validate(self): """Validate the scenario. :raises ScenarioValidationError: when scenario is not valid """ - params = frozenset(sum((list(step.params) for step in self.steps), [])) - example_params = set(self.examples.example_params + self.feature.examples.example_params) - if params and example_params and params != example_params: + if self.params or self.get_example_params(): + self._validate_example_free_variables() + self._validate_step_free_variables() + + def _validate_example_free_variables(self): + params = self.params + example_params = self.get_example_params() + if self.allow_example_free_variables or example_params.issubset(params): + return + else: raise exceptions.ScenarioExamplesNotValidError( - """Scenario "{}" in the feature "{}" has not valid examples. """ - """Set of step parameters {} should match set of example values {}.""".format( - self.name, self.feature.filename, sorted(params), sorted(example_params) + ( + """Scenario "{}" in the feature "{}" does not have valid examples. """ + """Set of example parameters {} should be a subset of step """ + """parameters {} if examples free variables are not allowed""" + ).format( + self.name, + self.feature.filename, + sorted(example_params), + sorted(params), + ) + ) + + def _validate_step_free_variables(self): + params = self.params + example_params = self.get_example_params() + if self.allow_step_free_variables or params.issubset(example_params): + return + else: + raise exceptions.ScenarioExamplesNotValidError( + ( + """Scenario "{}" in the feature "{}" does not have valid examples. """ + """Set of step parameters {} should be a subset of example """ + """parameters {} if steps free variables are not allowed""" + ).format( + self.name, + self.feature.filename, + sorted(params), + sorted(example_params), ) ) @@ -359,7 +401,11 @@ def params(self): def render(self, context: typing.Mapping[str, typing.Any]): def replacer(m: typing.Match): varname = m.group(1) - return str(context[varname]) + try: + return str(context[varname]) + except KeyError: + # Unavailability of varname is handled by bdd_allow_step_free_variables ini option + return f"<{varname}>" return STEP_PARAM_RE.sub(replacer, self.name) diff --git a/pytest_bdd/plugin.py b/pytest_bdd/plugin.py index c6d42e8f9..460bce393 100644 --- a/pytest_bdd/plugin.py +++ b/pytest_bdd/plugin.py @@ -44,7 +44,19 @@ def pytest_addoption(parser): def add_bdd_ini(parser): - parser.addini("bdd_features_base_dir", "Base features directory.") + parser.addini(name="bdd_features_base_dir", help="Base features directory.") + parser.addini( + name="bdd_allow_step_free_variables", + help="Allow use not defined in examples. They will be skipped during parametrization", + type="bool", + default=False, + ) + parser.addini( + name="bdd_allow_example_free_variables", + help="Allow use not defined in steps. They will be skipped during parametrization", + type="bool", + default=True, + ) @pytest.mark.trylast diff --git a/pytest_bdd/scenario.py b/pytest_bdd/scenario.py index c9446e38e..90c875b0f 100644 --- a/pytest_bdd/scenario.py +++ b/pytest_bdd/scenario.py @@ -13,7 +13,9 @@ import collections import os import re +import sys import typing +from pathlib import Path import pytest from _pytest.fixtures import FixtureLookupError @@ -215,12 +217,23 @@ def collect_example_parametrizations( return [pytest.param(context, id="-".join(context.values())) for context in contexts] -def scenario(feature_name: str, scenario_name: str, encoding: str = "utf-8", features_base_dir=None): +def scenario( + feature_name: str, + scenario_name: str, + *, + allow_example_free_variables: typing.Optional[typing.Any] = None, + allow_step_free_variables: typing.Optional[typing.Any] = None, + encoding: str = "utf-8", + features_base_dir: typing.Optional[typing.Union[str, Path]] = None, +): """Scenario decorator. - :param str feature_name: Feature file name. Absolute or relative to the configured feature base path. - :param str scenario_name: Scenario name. - :param str encoding: Feature file encoding. + :param feature_name: Feature file name. Absolute or relative to the configured feature base path. + :param scenario_name: Scenario name. + :param allow_example_free_variables: Examples could contain free(unused) variables + :param allow_step_free_variables: Steps could contain free(unused) variables which could be taken from fixtures + :param encoding: Feature file encoding. + :param features_base_dir: Base directory to build features path """ scenario_name = str(scenario_name) @@ -229,11 +242,22 @@ def scenario(feature_name: str, scenario_name: str, encoding: str = "utf-8", fea # Get the feature if features_base_dir is None: features_base_dir = get_features_base_dir(caller_module_path) - feature = get_feature(features_base_dir, feature_name, encoding=encoding) + feature = get_feature(str(features_base_dir), feature_name, encoding=encoding) # Get the scenario try: scenario = feature.scenarios[scenario_name] + scenario.allow_example_free_variables = ( + allow_example_free_variables + if allow_example_free_variables is not None + else get_from_ini("bdd_allow_example_free_variables") + ) + scenario.allow_step_free_variables = ( + allow_step_free_variables + if allow_step_free_variables is not None + else get_from_ini("bdd_allow_step_free_variables") + ) + except KeyError: feature_name = feature.name or "[Empty]" raise exceptions.ScenarioNotFound( @@ -253,7 +277,7 @@ def get_features_base_dir(caller_module_path): return get_from_ini("bdd_features_base_dir", default_base_dir) -def get_from_ini(key, default): +def get_from_ini(key, default=None): """Get value from ini config. Return default if value has not been set. Use if the default value is dynamic. Otherwise set default on addini call. @@ -294,10 +318,16 @@ def get_name(): suffix = f"_{index}" -def scenarios(*feature_paths, **kwargs): +class ScenarioKwargs(typing.TypedDict if sys.version_info >= (3, 8) else object): + allow_example_free_variables: typing.Optional[typing.Any] + allow_step_free_variables: typing.Optional[typing.Any] + features_base_dir: typing.Optional[typing.Union[str, Path]] + + +def scenarios(*feature_paths: typing.Union[str, Path], **kwargs: ScenarioKwargs): """Parse features from the paths and put all found scenarios in the caller module. - :param *feature_paths: feature file paths to use for scenarios + :param feature_paths: feature file paths to use for scenarios """ caller_locals = get_caller_module_locals() caller_path = get_caller_module_path() @@ -309,7 +339,7 @@ def scenarios(*feature_paths, **kwargs): abs_feature_paths = [] for path in feature_paths: if not os.path.isabs(path): - path = os.path.abspath(os.path.join(features_base_dir, path)) + path = os.path.abspath(os.path.join(str(features_base_dir), path)) abs_feature_paths.append(path) found = False diff --git a/tests/feature/test_outline.py b/tests/feature/test_outline.py index 3628db8d7..6c0c0ca41 100644 --- a/tests/feature/test_outline.py +++ b/tests/feature/test_outline.py @@ -81,7 +81,7 @@ def test_outline(request): # fmt: on -def test_wrongly_outlined(testdir): +def test_disallow_free_example_params(testdir): """Test parametrized scenario when the test function lacks parameters.""" testdir.makefile( @@ -103,6 +103,60 @@ def test_wrongly_outlined(testdir): ) testdir.makeconftest(textwrap.dedent(STEPS)) + testdir.makepyfile( + textwrap.dedent( + """\ + from pytest_bdd import scenario + + @scenario( + "outline.feature", + "Outlined with wrong examples", + allow_example_free_variables=False + ) + def test_outline(request): + pass + """ + ) + ) + result = testdir.runpytest() + assert_outcomes(result, errors=1) + result.stdout.fnmatch_lines( + '*ScenarioExamplesNotValidError: Scenario "Outlined with wrong examples"*does not have valid examples*' + ) + result.stdout.fnmatch_lines( + "*Set of example parameters [[]'eat', 'left', 'start', 'unknown_param'[]] should be " + "a subset of step parameters [[]'eat', 'left', 'start'[]]*" + ) + + +def test_disallow_free_example_params_by_ini(testdir): + """Test parametrized scenario when the test function lacks parameters.""" + testdir.makeini( + """ + [pytest] + bdd_allow_example_free_variables=false + """ + ) + + testdir.makefile( + ".feature", + outline=textwrap.dedent( + """\ + Feature: Outline + Scenario Outline: Outlined with wrong examples + Given there are cucumbers + When I eat cucumbers + Then I should have cucumbers + + Examples: + | start | eat | left | unknown_param | + | 12 | 5 | 7 | value | + + """ + ), + ) + testdir.makeconftest(textwrap.dedent(STEPS)) + testdir.makepyfile( textwrap.dedent( """\ @@ -117,9 +171,290 @@ def test_outline(request): result = testdir.runpytest() assert_outcomes(result, errors=1) result.stdout.fnmatch_lines( - '*ScenarioExamplesNotValidError: Scenario "Outlined with wrong examples"*has not valid examples*', + '*ScenarioExamplesNotValidError: Scenario "Outlined with wrong examples"*does not have valid examples*' + ) + result.stdout.fnmatch_lines( + "*Set of example parameters [[]'eat', 'left', 'start', 'unknown_param'[]] should be " + "a subset of step parameters [[]'eat', 'left', 'start'[]]*" ) - result.stdout.fnmatch_lines("*should match set of example values [[]'eat', 'left', 'start', 'unknown_param'[]].*") + + +def test_allow_free_example_params(testdir): + """Test parametrized scenario when the test function has a subset of the parameters of the examples.""" + + testdir.makefile( + ".feature", + outline=textwrap.dedent( + """\ + Feature: Outline + Scenario Outline: Outlined with subset of examples + Given there are cucumbers + When I eat cucumbers + Then I should have cucumbers + + Examples: + | start | eat | left | notes | + | 12 | 5 | 7 | Should be ignored | + + """ + ), + ) + testdir.makeconftest(textwrap.dedent(STEPS)) + + testdir.makepyfile( + textwrap.dedent( + """\ + from pytest_bdd import scenario + + @scenario( + "outline.feature", + "Outlined with subset of examples", + allow_example_free_variables=True + ) + def test_outline(request): + pass + """ + ) + ) + result = testdir.runpytest() + assert_outcomes(result, passed=1) + + +def test_allow_free_example_params_by_ini(testdir): + """Test parametrized scenario when the test function has a subset of the parameters of the examples.""" + + testdir.makeini( + """ + [pytest] + bdd_allow_example_free_variables=true + """ + ) + + testdir.makefile( + ".feature", + outline=textwrap.dedent( + """\ + Feature: Outline + Scenario Outline: Outlined with subset of examples + Given there are cucumbers + When I eat cucumbers + Then I should have cucumbers + + Examples: + | start | eat | left | notes | + | 12 | 5 | 7 | Should be ignored | + + """ + ), + ) + testdir.makeconftest(textwrap.dedent(STEPS)) + + testdir.makepyfile( + textwrap.dedent( + """\ + from pytest_bdd import scenario + + @scenario("outline.feature", "Outlined with subset of examples") + def test_outline(request): + pass + """ + ) + ) + result = testdir.runpytest() + assert_outcomes(result, passed=1) + + +def test_disallow_outlined_parameters_not_a_subset_of_examples(testdir): + """Test parametrized scenario when the test function has a parameter set + which is not a subset of those in the examples table.""" + + testdir.makefile( + ".feature", + outline=textwrap.dedent( + """\ + Feature: Outline + Scenario Outline: Outlined with wrong examples + Given there are cucumbers + When I eat cucumbers + Then I should have cucumbers in my bucket + + Examples: + | start | eat | left | + | 12 | 5 | 7 | + + """ + ), + ) + testdir.makeconftest(textwrap.dedent(STEPS)) + + testdir.makepyfile( + textwrap.dedent( + """\ + from pytest_bdd import scenario, then + import pytest_bdd.parsers as parsers + + @scenario("outline.feature", "Outlined with wrong examples", allow_step_free_variables=False) + def test_outline(request): + pass + + @then(parsers.parse('I should have {left} cucumbers in my bucket')) + def stepdef(left): + pass + """ + ) + ) + result = testdir.runpytest() + assert_outcomes(result, errors=1) + result.stdout.fnmatch_lines( + '*ScenarioExamplesNotValidError: Scenario "Outlined with wrong examples"*does not have valid examples*', + ) + result.stdout.fnmatch_lines("*should be a subset of example parameters [[]'eat', 'left', 'start'[]]*") + + +def test_disallow_outlined_parameters_not_a_subset_of_examples_by_ini(testdir): + """Test parametrized scenario when the test function has a parameter set + which is not a subset of those in the examples table.""" + testdir.makeini( + """ + [pytest] + bdd_allow_step_free_variables=false + """ + ) + + testdir.makefile( + ".feature", + outline=textwrap.dedent( + """\ + Feature: Outline + Scenario Outline: Outlined with wrong examples + Given there are cucumbers + When I eat cucumbers + Then I should have cucumbers in my bucket + + Examples: + | start | eat | left | + | 12 | 5 | 7 | + + """ + ), + ) + testdir.makeconftest(textwrap.dedent(STEPS)) + + testdir.makepyfile( + textwrap.dedent( + """\ + from pytest_bdd import scenario, then + import pytest_bdd.parsers as parsers + + @scenario("outline.feature", "Outlined with wrong examples") + def test_outline(request): + pass + + @then(parsers.parse('I should have {left} cucumbers in my bucket')) + def stepdef(left): + pass + """ + ) + ) + result = testdir.runpytest() + assert_outcomes(result, errors=1) + result.stdout.fnmatch_lines( + '*ScenarioExamplesNotValidError: Scenario "Outlined with wrong examples"*does not have valid examples*', + ) + result.stdout.fnmatch_lines("*should be a subset of example parameters [[]'eat', 'left', 'start'[]]*") + + +def test_allow_outlined_parameters_not_a_subset_of_examples(testdir): + """Test parametrized scenario when the test function has a parameter set + which is not a subset of those in the examples table.""" + + testdir.makefile( + ".feature", + outline=textwrap.dedent( + """\ + Feature: Outline + Scenario Outline: Outlined with wrong examples + Given there are cucumbers + When I eat cucumbers + Then I should have cucumbers in my bucket + + Examples: + | start | eat | left | + | 12 | 5 | 7 | + + """ + ), + ) + testdir.makeconftest(textwrap.dedent(STEPS)) + + testdir.makepyfile( + textwrap.dedent( + """\ + from pytest_bdd import scenario, then + import pytest_bdd.parsers as parsers + + @scenario("outline.feature", "Outlined with wrong examples", allow_step_free_variables=True) + def test_outline(request): + pass + + @then(parsers.parse('I should have {left} cucumbers in my bucket')) + def stepdef(left): + pass + """ + ) + ) + result = testdir.runpytest() + assert_outcomes(result, passed=1) + + +def test_allow_outlined_parameters_not_a_subset_of_examples_by_ini(testdir): + """Test parametrized scenario when the test function has a parameter set + which is not a subset of those in the examples table.""" + + testdir.makeini( + """ + [pytest] + bdd_allow_step_free_variables=true + """ + ) + + testdir.makefile( + ".feature", + outline=textwrap.dedent( + """\ + Feature: Outline + Scenario Outline: Outlined with wrong examples + Given there are cucumbers + When I eat cucumbers + Then I should have cucumbers in my bucket + + Examples: + | start | eat | left | + | 12 | 5 | 7 | + + """ + ), + ) + testdir.makeconftest(textwrap.dedent(STEPS)) + + testdir.makepyfile( + textwrap.dedent( + """\ + from pytest_bdd import scenario, then + import pytest_bdd.parsers as parsers + + @scenario("outline.feature", "Outlined with wrong examples") + def test_outline(request): + pass + + @then(parsers.parse('I should have {left} cucumbers in my bucket')) + def stepdef(left): + pass + """ + ) + ) + result = testdir.runpytest() + assert_outcomes(result, passed=1) def test_wrong_vertical_examples_scenario(testdir):