From fdd74c029fc24ca3e1e862f8966ae54ca95a6110 Mon Sep 17 00:00:00 2001 From: Jeremy Hicks Date: Thu, 17 Sep 2020 15:01:38 +0100 Subject: [PATCH 1/8] Fix issue: https://github.com/pytest-dev/pytest-bdd/issues/390 Changed validation of a scenario against its examples table so that the list of parameters defined for the scenario does not have to be the same as the list of parameters defined in the examples table, but can be a subset. This allows columns to be specified in the examples table that are not used in the scenario, but are there for future use or purely for documentation purposes. --- pytest_bdd/parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytest_bdd/parser.py b/pytest_bdd/parser.py index a8ee644e9..46b70a334 100644 --- a/pytest_bdd/parser.py +++ b/pytest_bdd/parser.py @@ -283,10 +283,10 @@ def validate(self): """ params = self.params example_params = self.get_example_params() - if params and example_params and params != example_params: + if params and example_params and not params.issubset(example_params): raise exceptions.ScenarioExamplesNotValidError( - """Scenario "{0}" in the feature "{1}" has not valid examples. """ - """Set of step parameters {2} should match set of example values {3}.""".format( + """Scenario "{0}" in the feature "{1}" does not have valid examples. """ + """Set of step parameters {2} should be a subset of example values {3}.""".format( self.name, self.feature.filename, sorted(params), sorted(example_params) ) ) From f9d775f5ad13155795e43a93eadd2f21c8f4935a Mon Sep 17 00:00:00 2001 From: Jeremy Hicks Date: Sat, 19 Sep 2020 11:44:51 +0100 Subject: [PATCH 2/8] Update unit tests --- tests/feature/test_outline.py | 56 ++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/tests/feature/test_outline.py b/tests/feature/test_outline.py index 6a89aef12..cfc29aecd 100644 --- a/tests/feature/test_outline.py +++ b/tests/feature/test_outline.py @@ -78,22 +78,22 @@ def test_outline(request): result.assert_outcomes(passed=2) -def test_wrongly_outlined(testdir): - """Test parametrized scenario when the test function lacks parameters.""" +def test_outline_has_subset_of_parameters(testdir): + """Test parametrized scenario when the test function has a subset of the parameters of the examples.""" testdir.makefile( ".feature", outline=textwrap.dedent( """\ Feature: Outline - Scenario Outline: Outlined with wrong examples + Scenario Outline: Outlined with subset of examples Given there are cucumbers When I eat cucumbers Then I should have cucumbers Examples: - | start | eat | left | unknown_param | - | 12 | 5 | 7 | value | + | start | eat | left | notes | + | 12 | 5 | 7 | Should be ignored | """ ), @@ -105,18 +105,60 @@ def test_wrongly_outlined(testdir): """\ from pytest_bdd import scenario + @scenario("outline.feature", "Outlined with subset of examples", + example_converters=dict(start=int, eat=float, left=str)) + def test_outline(request): + pass + """ + ) + ) + result = testdir.runpytest() + assert_outcomes(result, passed=1) + + +def test_wrongly_outlined_parameters_not_a_subset_of_examples(testdir): + """Test parametrized scenario when the test function has a parameter set which is not a subset of those in the examples table.""" + + testdir.makefile( + ".feature", + outline=textwrap.dedent( + """\ + Feature: Outline + Scenario Outline: Outlined with wrong examples + Given there are cucumbers + When I eat cucumbers + Then I should have cucumbers in my bucket + + Examples: + | start | eat | left | + | 12 | 5 | 7 | + + """ + ), + ) + testdir.makeconftest(textwrap.dedent(STEPS)) + + testdir.makepyfile( + textwrap.dedent( + """\ + from pytest_bdd import scenario, then + @scenario("outline.feature", "Outlined with wrong examples") def test_outline(request): pass + + @then(parsers.parse('I should have cucumbers in my bucket')) + def stepdef(left, right): + pass """ ) ) result = testdir.runpytest() assert_outcomes(result, errors=1) result.stdout.fnmatch_lines( - '*ScenarioExamplesNotValidError: Scenario "Outlined with wrong examples"*has not valid examples*', + '*ScenarioExamplesNotValidError: Scenario "Outlined with wrong examples"*does not have valid examples*', ) - result.stdout.fnmatch_lines("*should match set of example values [[]'eat', 'left', 'start', 'unknown_param'[]].*") + result.stdout.fnmatch_lines("*should be a subset of example values [[]'eat', 'left', 'start'[]].*") def test_wrong_vertical_examples_scenario(testdir): From d3a4d7f814047db5d961cba23df067c01845e349 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 1 Oct 2021 17:41:11 +0000 Subject: [PATCH 3/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/feature/test_outline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/feature/test_outline.py b/tests/feature/test_outline.py index 69e50e148..f5f191e52 100644 --- a/tests/feature/test_outline.py +++ b/tests/feature/test_outline.py @@ -149,7 +149,7 @@ def test_wrongly_outlined_parameters_not_a_subset_of_examples(testdir): @scenario("outline.feature", "Outlined with wrong examples") def test_outline(request): pass - + @then(parsers.parse('I should have cucumbers in my bucket')) def stepdef(left, right): pass From 0d52fa3671829d01dbe098fd0b27312da8dbcaf4 Mon Sep 17 00:00:00 2001 From: Alessio Bogon Date: Fri, 1 Oct 2021 19:50:42 +0200 Subject: [PATCH 4/8] Simplify test --- tests/feature/test_outline.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/feature/test_outline.py b/tests/feature/test_outline.py index f5f191e52..5573a3782 100644 --- a/tests/feature/test_outline.py +++ b/tests/feature/test_outline.py @@ -119,8 +119,9 @@ def test_outline(request): assert_outcomes(result, passed=1) -def test_wrongly_outlined_parameters_not_a_subset_of_examples(testdir): - """Test parametrized scenario when the test function has a parameter set which is not a subset of those in the examples table.""" +def test_not_enough_examples(testdir): + """Test parametrized scenario when the test function has a parameter set which is not a subset of those in the + examples table.""" testdir.makefile( ".feature", @@ -130,11 +131,11 @@ def test_wrongly_outlined_parameters_not_a_subset_of_examples(testdir): Scenario Outline: Outlined with wrong examples Given there are cucumbers When I eat cucumbers - Then I should have cucumbers in my bucket + Then I should have cucumbers Examples: - | start | eat | left | - | 12 | 5 | 7 | + | start | eat | + | 12 | 5 | """ ), @@ -144,15 +145,9 @@ def test_wrongly_outlined_parameters_not_a_subset_of_examples(testdir): testdir.makepyfile( textwrap.dedent( """\ - from pytest_bdd import scenario, then - - @scenario("outline.feature", "Outlined with wrong examples") - def test_outline(request): - pass + from pytest_bdd import scenarios - @then(parsers.parse('I should have cucumbers in my bucket')) - def stepdef(left, right): - pass + scenarios("outline.feature") """ ) ) @@ -161,7 +156,10 @@ def stepdef(left, right): result.stdout.fnmatch_lines( '*ScenarioExamplesNotValidError: Scenario "Outlined with wrong examples"*does not have valid examples*', ) - result.stdout.fnmatch_lines("*should be a subset of example values [[]'eat', 'left', 'start'[]].*") + expected_msg = ( + "Set of step parameters ['eat', 'left', 'start'] " "should be a subset of example values ['eat', 'start']" + ) + assert expected_msg in result.stdout.str() def test_wrong_vertical_examples_scenario(testdir): From 9c72041c1c885fe48863e6915c09889f75b47d75 Mon Sep 17 00:00:00 2001 From: Alessio Bogon Date: Fri, 1 Oct 2021 19:52:14 +0200 Subject: [PATCH 5/8] Simplify test --- tests/feature/test_outline.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/feature/test_outline.py b/tests/feature/test_outline.py index 5573a3782..0bc56b7d3 100644 --- a/tests/feature/test_outline.py +++ b/tests/feature/test_outline.py @@ -81,8 +81,8 @@ def test_outline(request): # fmt: on -def test_outline_has_subset_of_parameters(testdir): - """Test parametrized scenario when the test function has a subset of the parameters of the examples.""" +def test_extra_examples_are_ignored(testdir): + """Test that the Examples section can have more examples than the ones used by the scenario.""" testdir.makefile( ".feature", @@ -108,10 +108,7 @@ def test_outline_has_subset_of_parameters(testdir): """\ from pytest_bdd import scenario - @scenario("outline.feature", "Outlined with subset of examples", - example_converters=dict(start=int, eat=float, left=str)) - def test_outline(request): - pass + scenarios("outline.feature") """ ) ) From 76e5333aaee7364636c004b8f27185410feb61dd Mon Sep 17 00:00:00 2001 From: Alessio Bogon Date: Fri, 1 Oct 2021 19:53:00 +0200 Subject: [PATCH 6/8] Improve naming --- tests/feature/test_outline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/feature/test_outline.py b/tests/feature/test_outline.py index 0bc56b7d3..acb3f1278 100644 --- a/tests/feature/test_outline.py +++ b/tests/feature/test_outline.py @@ -89,7 +89,7 @@ def test_extra_examples_are_ignored(testdir): outline=textwrap.dedent( """\ Feature: Outline - Scenario Outline: Outlined with subset of examples + Scenario Outline: Outlined with extra examples Given there are cucumbers When I eat cucumbers Then I should have cucumbers From 3b9aa4a7846144bd1419d69ba72965a1e87235b7 Mon Sep 17 00:00:00 2001 From: Alessio Bogon Date: Fri, 1 Oct 2021 19:54:03 +0200 Subject: [PATCH 7/8] Update release notes --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 1c90f501b..570eec289 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,7 @@ Unreleased ----------- This release introduces breaking changes, please refer to the :ref:`Migration from 4.x.x`. +- Unused examples are now ignored. Previously an error was raised. - Rewrite the logic to parse Examples for Scenario Outlines. Now the substitution of the examples is done during the parsing of Gherkin feature files. You won't need to define the steps twice like ``@given("there are cucumbers")`` and ``@given(parsers.parse("there are {start} cucumbers"))``. The latter will be enough. - Removed ``example_converters`` from ``scenario(...)`` signature. You should now use just the ``converters`` parameter for ``given``, ``when``, ``then``. - Removed ``--cucumberjson-expanded`` and ``--cucumber-json-expanded`` options. Now the JSON report is always expanded. From b33fdc3088ea5e557b4810131c4ca2fed0b43aa8 Mon Sep 17 00:00:00 2001 From: Alessio Bogon Date: Fri, 1 Oct 2021 19:55:55 +0200 Subject: [PATCH 8/8] Fix test --- tests/feature/test_outline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/feature/test_outline.py b/tests/feature/test_outline.py index acb3f1278..e411bc3ff 100644 --- a/tests/feature/test_outline.py +++ b/tests/feature/test_outline.py @@ -106,7 +106,7 @@ def test_extra_examples_are_ignored(testdir): testdir.makepyfile( textwrap.dedent( """\ - from pytest_bdd import scenario + from pytest_bdd import scenarios scenarios("outline.feature") """