diff --git a/README.md b/README.md index b9ba012..7287311 100644 --- a/README.md +++ b/README.md @@ -163,13 +163,55 @@ You can use **directory** option to specify directory to use for test file struc pytest --testomatio sync --directory imported_tests ``` Note: **keep-structure** option takes precedence over **directory** option. If both are used **keep-structure** will be used. -#### Filter tests by id -You can filter tests by testomat.io id, using **test-id** option. You pass single or multiple ids to this option. Use this option with **report** command: - ```bash -pytest --testomatio report --test-id "Tc0880217|Tfd1c595c" +#### Filter tests +You can filter tests that will be reported, using **testomatio-filter** option. Filter format: *filter_type=value*. Use this option with **report** command. + +**Note**: Only one filter can be applied at a time + +**Filter types**: + - **test_id**. Filter test by Testomat.io id. You can pass single or multiple ids for this filter using *|* as separator. Ex: "test_id=@T3h2r432|T2e34e342|b234fr254" + - **plan**. Filter by plan id. + - **jira**. Filter by jira issue id. + - **tag**. Filter by tag name. + - **label**. Filter by label name or label-id. + +**Examples**: + +**Filter by test id** +```bash +pytest --testomatio report --testomatio-filter="test_id=Tc0880217|Tfd1c595c" +``` +**Filter by tag** + +If your test have '@smoke' tag on testomat.io, then value for this filter == smoke +```bash +pytest --testomatio report --testomatio-filter="tag=smoke" +``` + +**Filter by label** + +```bash +# by label name +pytest --testomatio report --testomatio-filter="label=important" + +# by label id +pytest --testomatio report --testomatio-filter="label=important-f435-e" + +# based on Severity type +pytest --testomatio report --testomatio-filter="label=severity-f124r-3:⚠️ Critical" ``` -Note: Test id should be started from letter "T" +**Filter by plan** + +```bash +pytest --testomatio report --testomatio-filter="plan=ca34gf3t" +``` + +**Filter by Jira Issue** + +```bash +pytest --testomatio report --testomatio-filter="jira=TES1" +``` ### Configuration with environment variables You can use environment variable to control certain features of testomat.io diff --git a/pyproject.toml b/pyproject.toml index 5cae11c..0a272f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ version_provider = "pep621" update_changelog_on_bump = false [project] name = "pytestomatio" -version = "2.10.2" +version = "2.11.0b1" dependencies = [ diff --git a/pytestomatio/connect/connector.py b/pytestomatio/connect/connector.py index ee78515..9ecd35e 100644 --- a/pytestomatio/connect/connector.py +++ b/pytestomatio/connect/connector.py @@ -120,6 +120,23 @@ def get_tests(self, test_metadata: list[TestItem]) -> dict: response = self.session.get(f'{self.base_url}/api/test_data?api_key={self.api_key}') return response.json() + def get_filtered_tests(self, filter_type, filter_value): + """ + Returns list of filtered tests from Testomat.io + """ + # TODO: add retry logic + url = f'{self.base_url}/api/test_grep?api_key={self.api_key}&type={filter_type}&id={filter_value}' + try: + response = self.session.get(url) + if response.status_code < 400: + log.info(f'Received tests filtered by {filter_type}={filter_value} from {self.base_url}') + return response.json() + else: + log.error(f'Failed to receive tests from {self.base_url}. Status code: {response.status_code}') + except Exception as e: + log.error(f'An unexpected exception occurred. Please report an issue: {e}') + return + def create_test_run(self, access_event: str, title: str, group_title, env: str, label: str, shared_run: bool, shared_run_timeout: str, parallel, ci_build_url: str) -> dict | None: request = { diff --git a/pytestomatio/testomatio/filter_plugin.py b/pytestomatio/testomatio/filter_plugin.py index 6797a3b..92f08a3 100644 --- a/pytestomatio/testomatio/filter_plugin.py +++ b/pytestomatio/testomatio/filter_plugin.py @@ -1,34 +1,90 @@ +import logging import pytest +log = logging.getLogger('pytestomatio') + + class TestomatioFilterPlugin: - @pytest.hookimpl(trylast=True) - def pytest_collection_modifyitems(self, session, config, items): - # By now all other filters (like -m, -k, name-based) have been applied - # and `items` is the filtered set after all their conditions. - test_ids_str = config.getoption("test_id") - if not test_ids_str: - # No custom IDs specified, nothing to do - return + # todo: check multiple values for filter, apply several filters + allowed_filters = {'test_id', 'jira', 'label', 'plan', 'tag'} - test_ids = test_ids_str.split("|") + def get_matched_test_ids(self, f_type, f_value) -> list: + """ + Returns test ids that matches given filter + :param f_type: filter type(test_id, label) + :param f_value: filter value + """ + if f_type != 'test_id': + test_ids = self.filter_by_testomatio_related_fields(f_type, f_value) + else: + test_ids = f_value.split("|") # Remove "@" from the start of test IDs if present - test_ids = [test_id.lstrip("@T") for test_id in test_ids] - if not test_ids: - return + cleared_ids = [test_id.lstrip("@T") for test_id in test_ids] + return cleared_ids - # Now let's find all tests that match these test IDs from the original full list. - # We use the originally collected tests to avoid losing tests filtered out by others. - original_items = session._pytestomatio_original_collected_items - testomatio_matched = [] - - for item in original_items: + def filter_by_testomatio_related_fields(self, filter_type, filter_value): + """Get ids of tests that matches filter from Testomatio. Used for Testomatio related filters""" + connector = pytest.testomatio.connector + tests = connector.get_filtered_tests(filter_type, filter_value) + return tests.get('tests') if tests else [] + + def match_tests_by_id(self, test_ids, items) -> list: + matched_tests = [] + + for item in items: # Check for testomatio marker for marker in item.iter_markers(name="testomatio"): marker_id = marker.args[0].lstrip("@T") # Strip "@" from the marker argument if marker_id in test_ids: - testomatio_matched.append(item) - break + matched_tests.append(item) + break + return matched_tests + + def filter_tests(self, filter_opts, original_items): + try: + f_type, f_value = filter_opts.split('=') + if not (f_type and f_value): + log.error(f'Failed to retrieve filter data. Filter type: {f_type} Filter value: {f_value}') + return + + if f_type not in self.allowed_filters: + log.error(f"Filter '{f_type}' not allowed. Choose of these filters: {self.allowed_filters}") + return + + log.info(f"Filtering tests using the '{f_type}' filter with '{f_value}' value") + test_ids = self.get_matched_test_ids(f_type, f_value) + if not test_ids: + return + + return self.match_tests_by_id(test_ids, original_items) + except ValueError as e: + log.error(f"Incorrect filter format. Filter must be in type=value format. Received: {filter_opts}") + + @pytest.hookimpl(trylast=True) + def pytest_collection_modifyitems(self, session, config, items): + testomatio_option = config.getoption('testomatio') + if testomatio_option is None or testomatio_option != 'report': + return + + filter_opts = config.getoption('testomatio_filter') or config.getoption('test_id') + if not filter_opts: + return + + # compatibility with deprecated filter option + if config.getoption('test_id') and not config.getoption('testomatio_filter'): + filter_opts = f'test_id={filter_opts}' + log.warning('--test-id filter option is deprecated. Use --testomatio-filter instead') + + log.info('Testomatio Filter enabled') + # By now all other filters (like -m, -k, name-based) have been applied + # and `items` is the filtered set after all their conditions. + # We use the originally collected tests to avoid losing tests filtered out by others. + original_items = session._pytestomatio_original_collected_items + testomatio_matched = self.filter_tests(filter_opts, original_items) + if not testomatio_matched: + log.info('No tests were found matching the filter provided. Filtering is skipped') + return # We'll check common filters: -k, -m and a few others. # If they are empty or None, they are not active. diff --git a/pytestomatio/utils/parser_setup.py b/pytestomatio/utils/parser_setup.py index 35dba0d..11aaf12 100644 --- a/pytestomatio/utils/parser_setup.py +++ b/pytestomatio/utils/parser_setup.py @@ -67,11 +67,18 @@ def parser_options(parser: Parser, testomatio='testomatio') -> None: Note: --structure option takes precedence over --directory option. If both are used --structure will be used. """ ) + group.addoption('--testomatio-filter', + default=None, + dest="testomatio_filter", + help=""" + help="Filter tests by Test IDs (e.g., single test id 'T00C73028' or multiply 'T00C73028|T00C73029'), Labels, Tags, Plan or Jira issue ids + """ + ) group.addoption('--test-id', default=None, dest="test_id", help=""" - help="Filter tests by Test IDs (e.g., single test id 'T00C73028' or multiply 'T00C73028|T00C73029') + help="DEPRECATED. Filter tests by Test IDs (e.g., single test id 'T00C73028' or multiply 'T00C73028|T00C73029') """ ) parser.addini('testomatio_url', 'testomat.io base url') diff --git a/tests/test_testomatio/test_filter_plugin.py b/tests/test_testomatio/test_filter_plugin.py index dbba0e0..a8f8434 100644 --- a/tests/test_testomatio/test_filter_plugin.py +++ b/tests/test_testomatio/test_filter_plugin.py @@ -46,19 +46,76 @@ def create_mock_item_without_marker(self, nodeid): item.iter_markers.return_value = iter([]) return item - def test_no_test_id_option(self, plugin, mock_session, mock_config): - """Test when no test_id option""" + def test_no_testomatio_filter_option(self, plugin, mock_session, mock_config): + """Test when no testomatio-filter option""" + mock_config.getoption = Mock(return_value='report') items = [Mock(), Mock()] original_items = items.copy() plugin.pytest_collection_modifyitems(mock_session, mock_config, items) assert items == original_items - mock_config.getoption.assert_called_once_with("test_id") + mock_config.getoption.assert_called_with("testomatio_filter") + + @patch('pytestomatio.testomatio.filter_plugin.TestomatioFilterPlugin.get_matched_test_ids') + def test_filter_incorrect_format(self, get_ids_mock, plugin, mock_session, mock_config, caplog): + option_values = { + 'testomatio': 'report', + 'testomatio_filter': 'test_id' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) + matched_item = self.create_mock_item_with_marker("test1", "@T12345678") + unmatched_item = self.create_mock_item_with_marker("test2", "@T87654321") + + mock_session._pytestomatio_original_collected_items = [matched_item, unmatched_item] + + items = [] + plugin.pytest_collection_modifyitems(mock_session, mock_config, items) + assert 'Incorrect filter format. Filter must be in type=value format.' in caplog.text + assert get_ids_mock.call_count == 0 + + @patch('pytestomatio.testomatio.filter_plugin.TestomatioFilterPlugin.get_matched_test_ids') + def test_filter_one_of_values_empty(self, get_ids_mock, plugin, mock_session, mock_config, caplog): + option_values = { + 'testomatio': 'report', + 'testomatio_filter': 'test_id=' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) + matched_item = self.create_mock_item_with_marker("test1", "@T12345678") + unmatched_item = self.create_mock_item_with_marker("test2", "@T87654321") + + mock_session._pytestomatio_original_collected_items = [matched_item, unmatched_item] + + items = [] + plugin.pytest_collection_modifyitems(mock_session, mock_config, items) + assert 'Failed to retrieve filter data.' in caplog.text + assert get_ids_mock.call_count == 0 + + @patch('pytestomatio.testomatio.filter_plugin.TestomatioFilterPlugin.get_matched_test_ids') + def test_filter_not_allowed(self, get_ids_mock, plugin, mock_session, mock_config, caplog): + f_type = 'random' + option_values = { + 'testomatio': 'report', + 'testomatio_filter': f'{f_type}=T12345678' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) + matched_item = self.create_mock_item_with_marker("test1", "@T12345678") + unmatched_item = self.create_mock_item_with_marker("test2", "@T87654321") + + mock_session._pytestomatio_original_collected_items = [matched_item, unmatched_item] + + items = [] + plugin.pytest_collection_modifyitems(mock_session, mock_config, items) + assert f"Filter '{f_type}' not allowed." in caplog.text + assert get_ids_mock.call_count == 0 def test_single_test_id_match(self, plugin, mock_session, mock_config): """Test with one matched test_id""" - mock_config.getoption.return_value = "@T12345678" + option_values = { + 'testomatio': 'report', + 'testomatio_filter': 'test_id=@T12345678' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) matched_item = self.create_mock_item_with_marker("test1", "@T12345678") unmatched_item = self.create_mock_item_with_marker("test2", "@T87654321") @@ -72,7 +129,11 @@ def test_single_test_id_match(self, plugin, mock_session, mock_config): def test_multiple_test_ids_with_pipe_separator(self, plugin, mock_session, mock_config): """Test with multiple test_id""" - mock_config.getoption.return_value = "@T12345678|@T87654321" + option_values = { + 'testomatio': 'report', + 'testomatio_filter': 'test_id=@T12345678|@T87654321' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) item1 = self.create_mock_item_with_marker("test1", "@T12345678") item2 = self.create_mock_item_with_marker("test2", "@T87654321") @@ -91,7 +152,11 @@ def test_multiple_test_ids_with_pipe_separator(self, plugin, mock_session, mock_ def test_test_id_without_at_t_prefix(self, plugin, mock_session, mock_config): """Test test_id without @T prefix""" - mock_config.getoption.return_value = "12345678" # без @T + option_values = { + 'testomatio': 'report', + 'testomatio_filter': 'test_id=12345678' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) item = self.create_mock_item_with_marker("test1", "@T12345678") mock_session._pytestomatio_original_collected_items = [item] @@ -102,9 +167,50 @@ def test_test_id_without_at_t_prefix(self, plugin, mock_session, mock_config): assert items == [item] + def test_deprecated_test_id_option_supported(self, plugin, mock_session, mock_config): + """Test deprecated filter option --test-id supported""" + option_values = { + 'testomatio': 'report', + 'test_id': '@T12345678' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) + + matched_item = self.create_mock_item_with_marker("test1", "@T12345678") + unmatched_item = self.create_mock_item_with_marker("test2", "@T87654321") + + mock_session._pytestomatio_original_collected_items = [matched_item, unmatched_item] + + items = [] + plugin.pytest_collection_modifyitems(mock_session, mock_config, items) + + assert items == [matched_item] + + def test_deprecated_test_id_option_priority(self, plugin, mock_session, mock_config): + """Test deprecated filter option --test-id have lower priority than --testomatio-filter if they both set""" + option_values = { + 'testomatio': 'report', + 'testomatio_filter': 'test_id=@T87654321', + 'test_id': '@T12345678' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) + + unmatched_item = self.create_mock_item_with_marker("test1", "@T12345678") + matched_item = self.create_mock_item_with_marker("test2", "@T87654321") + + mock_session._pytestomatio_original_collected_items = [matched_item, unmatched_item] + + items = [] + plugin.pytest_collection_modifyitems(mock_session, mock_config, items) + + assert items == [matched_item] + def test_no_matching_items(self, plugin, mock_session, mock_config): """Test when no matching items""" - mock_config.getoption.return_value = "@T12345678" + option_values = { + 'testomatio': 'report', + 'testomatio_filter': 'test_id=@T12345678' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) item1 = self.create_mock_item_with_marker("test1", "@T99999999") item2 = self.create_mock_item_without_marker("test2") @@ -119,8 +225,12 @@ def test_no_matching_items(self, plugin, mock_session, mock_config): def test_with_keyword_filter_active(self, plugin, mock_session, mock_config): """Test with active -k filter""" - mock_config.getoption.return_value = "@T12345678" - mock_config.option.keyword = "test_login" # -k фільтр активний + option_values = { + 'testomatio': 'report', + 'testomatio_filter': 'test_id=@T12345678' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) + mock_config.option.keyword = "test_login" # -k filter is active keyword_item = Mock() keyword_item.nodeid = "test_login" @@ -139,7 +249,11 @@ def test_with_keyword_filter_active(self, plugin, mock_session, mock_config): def test_with_markexpr_filter_active(self, plugin, mock_session, mock_config): """Test with -m filter""" - mock_config.getoption.return_value = "@T12345678" + option_values = { + 'testomatio': 'report', + 'testomatio_filter': 'test_id=@T12345678' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) mock_config.option.markexpr = "smoke" mock_config.option.keyword = "" @@ -159,7 +273,11 @@ def test_with_markexpr_filter_active(self, plugin, mock_session, mock_config): def test_with_not_keyword_filter(self, plugin, mock_session, mock_config): """Test with "not" in keyword filter (exclusion logic)""" - mock_config.getoption.return_value = "@T12345678" + option_values = { + 'testomatio': 'report', + 'testomatio_filter': 'test_id=@T12345678' + } + mock_config.getoption.side_effect = lambda x: option_values.get(x) mock_config.option.keyword = "not slow" passed_item = Mock() diff --git a/tests/test_utils/test_parser_setup.py b/tests/test_utils/test_parser_setup.py index 7300ac8..a04e828 100644 --- a/tests/test_utils/test_parser_setup.py +++ b/tests/test_utils/test_parser_setup.py @@ -157,14 +157,31 @@ def test_parser_options_adds_directory_option(self, mock_parser): help=expected_help ) + def test_parser_options_adds_testomatio_filter_option(self, mock_parser): + """Test --testomatio-filter option is added""" + mock_group = mock_parser.getgroup.return_value + + parser_options(mock_parser) + + expected_help = """ + help="Filter tests by Test IDs (e.g., single test id 'T00C73028' or multiply 'T00C73028|T00C73029'), Labels, Tags, Plan or Jira issue ids + """ + + mock_group.addoption.assert_any_call( + '--testomatio-filter', + default=None, + dest="testomatio_filter", + help=expected_help + ) + def test_parser_options_adds_test_id_option(self, mock_parser): - """Test --test-id option is added""" + """Test deprecate --test-id option is added""" mock_group = mock_parser.getgroup.return_value parser_options(mock_parser) expected_help = """ - help="Filter tests by Test IDs (e.g., single test id 'T00C73028' or multiply 'T00C73028|T00C73029') + help="DEPRECATED. Filter tests by Test IDs (e.g., single test id 'T00C73028' or multiply 'T00C73028|T00C73029') """ mock_group.addoption.assert_any_call( @@ -189,7 +206,7 @@ def test_parser_options_call_count(self, mock_parser): parser_options(mock_parser) - assert mock_group.addoption.call_count == 8 + assert mock_group.addoption.call_count == 9 assert mock_parser.addini.call_count == 1