diff --git a/azdev/help.py b/azdev/help.py index e03ceca59..1d9b8984b 100644 --- a/azdev/help.py +++ b/azdev/help.py @@ -110,9 +110,6 @@ populator-commands: - pytest -h examples: - - name: Run all tests. - text: azdev test --ci - - name: Run tests for specific modules. text: azdev test {mod1} {mod2} diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py index d6b85ed83..17c76041f 100644 --- a/azdev/operations/tests/__init__.py +++ b/azdev/operations/tests/__init__.py @@ -22,7 +22,9 @@ COMMAND_MODULE_PREFIX, EXTENSION_PREFIX, make_dirs, get_azdev_config_dir, get_path_table, require_virtual_env, get_name_index) - +from .pytest_runner import get_test_runner +from .profile_context import ProfileContext, current_profile +from .incremental_strategy import CLIAzureDevOpsContext logger = get_logger(__name__) @@ -30,22 +32,20 @@ # pylint: disable=too-many-statements def run_tests(tests, xml_path=None, discover=False, in_series=False, run_live=False, profile=None, last_failed=False, pytest_args=None, - git_source=None, git_target=None, git_repo=None): + git_source=None, git_target=None, git_repo=None, + cli_ci=False): require_virtual_env() DEFAULT_RESULT_FILE = 'test_results.xml' DEFAULT_RESULT_PATH = os.path.join(get_azdev_config_dir(), DEFAULT_RESULT_FILE) - from .pytest_runner import get_test_runner - heading('Run Tests') - original_profile = _get_profile(profile) - if not profile: - profile = original_profile path_table = get_path_table() - test_index = _get_test_index(profile, discover) + + test_index = _get_test_index(profile or current_profile(), discover) + if not tests: tests = list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys()) if tests == ['CLI']: @@ -54,10 +54,13 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False, tests = list(path_table['ext'].keys()) # filter out tests whose modules haven't changed - tests = _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo) + modified_mods = _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo) + if modified_mods: + display('\nTest on modules: {}\n'.format(', '.join(modified_mods))) - if tests: - display('\nTESTS: {}\n'.format(', '.join(tests))) + if cli_ci is True: + ctx = CLIAzureDevOpsContext(git_repo, git_source, git_target) + modified_mods = ctx.filter(test_index) # resolve the path at which to dump the XML results xml_path = xml_path or DEFAULT_RESULT_PATH @@ -73,6 +76,7 @@ def _find_test(index, name): name_comps = name.split('.') num_comps = len(name_comps) key_error = KeyError() + for i in range(num_comps): check_name = '.'.join(name_comps[(-1 - i):]) try: @@ -87,7 +91,7 @@ def _find_test(index, name): # lookup test paths from index test_paths = [] - for t in tests: + for t in modified_mods: try: test_path = os.path.normpath(_find_test(test_index, t)) test_paths.append(test_path) @@ -99,23 +103,17 @@ def _find_test(index, name): if not test_paths: raise CLIError('No tests selected to run.') - runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed) - exit_code = runner(test_paths=test_paths, pytest_args=pytest_args) - _summarize_test_results(xml_path) - - # attempt to restore the original profile - if profile != original_profile: - result = raw_cmd('az cloud update --profile {}'.format(original_profile), - "Restoring profile '{}'.".format(original_profile)) - if result.exit_code != 0: - logger.warning("Failed to restore profile '%s'.", original_profile) + exit_code = 0 + with ProfileContext(profile): + runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed) + exit_code = runner(test_paths=test_paths, pytest_args=pytest_args) sys.exit(0 if not exit_code else 1) def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo): from azdev.utilities import diff_branches, extract_module_name - from azdev.utilities.git_util import _summarize_changed_mods + from azdev.utilities.git_util import summarize_changed_mods if not any([git_source, git_target, git_repo]): return tests @@ -124,7 +122,7 @@ def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo): raise CLIError('usage error: [--src NAME] --tgt NAME --repo PATH') files_changed = diff_branches(git_repo, git_target, git_source) - mods_changed = _summarize_changed_mods(files_changed) + mods_changed = summarize_changed_mods(files_changed) repo_path = str(os.path.abspath(git_repo)).lower() to_remove = [] @@ -145,30 +143,6 @@ def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo): return tests -def _get_profile(profile): - import colorama - colorama.init(autoreset=True) - try: - fore_red = colorama.Fore.RED if not IS_WINDOWS else '' - fore_reset = colorama.Fore.RESET if not IS_WINDOWS else '' - original_profile = raw_cmd('az cloud show --query profile -otsv', show_stderr=False).result - if not profile or original_profile == profile: - profile = original_profile - display('The tests are set to run against current profile {}.' - .format(fore_red + original_profile + fore_reset)) - elif original_profile != profile: - display('The tests are set to run against profile {} but the current az cloud profile is {}.' - .format(fore_red + profile + fore_reset, fore_red + original_profile + fore_reset)) - result = raw_cmd('az cloud update --profile {}'.format(profile), - 'SWITCHING TO PROFILE {}.'.format(fore_red + profile + fore_reset)) - if result.exit_code != 0: - raise CLIError(result.error.output) - # returns the original profile so we can switch back if need be - return original_profile - except CalledProcessError: - raise CLIError('Failed to retrieve current az profile') - - def _discover_module_tests(mod_name, mod_data): # get the list of test files in each module @@ -353,32 +327,3 @@ def _get_test_index(profile, discover): f.write(json.dumps(test_index)) display('\ntest index created: {}'.format(test_index_path)) return test_index - - -def _summarize_test_results(xml_path): - import xml.etree.ElementTree as ElementTree - - subheading('Results') - - root = ElementTree.parse(xml_path).getroot() - summary = { - 'time': root.get('time'), - 'tests': root.get('tests'), - 'skips': root.get('skips'), - 'failures': root.get('failures'), - 'errors': root.get('errors') - } - display('Time: {time} sec\tTests: {tests}\tSkipped: {skips}\tFailures: {failures}\tErrors: {errors}'.format( - **summary)) - - failed = [] - for item in root.findall('testcase'): - if item.findall('failure'): - file_and_class = '.'.join(item.get('classname').split('.')[-2:]) - failed.append('{}.{}'.format(file_and_class, item.get('name'))) - - if failed: - subheading('FAILURES') - for name in failed: - display(name) - display('') diff --git a/azdev/operations/tests/incremental_strategy.py b/azdev/operations/tests/incremental_strategy.py new file mode 100644 index 000000000..f8c010e1b --- /dev/null +++ b/azdev/operations/tests/incremental_strategy.py @@ -0,0 +1,73 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +import abc +from knack.util import CLIError + +from azdev.utilities import get_path_table, git_util + + +# @wrapt.decorator +# def cli_release_scenario(wrapped, _, args, kwargs): +# """ +# Filter out those files in Azure CLI release stage +# """ +# # TODO +# # if instance.resolved: +# # return instance + +# return wrapped(*args, **kwargs) + + +class AzureDevOpsContext(abc.ABC): + def __init__(self, git_repo, git_source, git_target): + """ + :param git_source: could be commit id, branch name or any valid value for git diff + :param git_target: could be commit id, branch name or any valid value for git diff + """ + self.git_repo = git_repo + self.git_source = git_source + self.git_target = git_target + + @abc.abstractmethod + def filter(self, test_index): + pass + + +class CLIAzureDevOpsContext(AzureDevOpsContext): + """ + Assemble strategy of incremental test on Azure DevOps Environment for Azure CLI + """ + def __init__(self, git_repo, git_source, git_target): + super().__init__(git_repo, git_source, git_target) + + if not any([self.git_source, self.git_target, self.git_repo]): + raise CLIError('usage error: [--src NAME] --tgt NAME --repo PATH --cli-ci') + + if not all([self.git_target, self.git_repo]): + raise CLIError('usage error: [--src NAME] --tgt NAME --repo PATH --cli-ci') + + @property + def modified_files(self): + modified_files = git_util.diff_branches(self.git_repo, self.git_source, self.git_target) + return [f for f in modified_files if f.startswith('src/')] + + def filter(self, test_index): + """ + Strategy on Azure CLI pull request verification stage. + + :return: a list of modified packages + """ + + modified_packages = git_util.summarize_changed_mods(self.modified_files) + + if any(core_package in modified_packages for core_package in ['core', 'testsdk', 'telemetry']): + path_table = get_path_table() + + # tests under all packages + return list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys()) + + return modified_packages diff --git a/azdev/operations/tests/profile_context.py b/azdev/operations/tests/profile_context.py new file mode 100644 index 000000000..dec958b7d --- /dev/null +++ b/azdev/operations/tests/profile_context.py @@ -0,0 +1,45 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +import traceback + +from knack.log import get_logger +from knack.util import CLIError + +from azdev.utilities import call, cmd +from azdev.utilities import display + + +logger = get_logger(__name__) + + +class ProfileContext: + def __init__(self, profile_name=None): + self.target_profile = profile_name + + self.origin_profile = current_profile() + + def __enter__(self): + if self.target_profile is None or self.target_profile == self.origin_profile: + display('The tests are set to run against current profile "{}"'.format(self.origin_profile)) + else: + result = cmd('az cloud update --profile {}'.format(self.target_profile), + 'Switching to target profile "{}"...'.format(self.target_profile)) + if result.exit_code != 0: + raise CLIError(result.error.output.decode('utf-8')) + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.target_profile is not None and self.target_profile != self.origin_profile: + display('Switching back to origin profile "{}"...'.format(self.origin_profile)) + call('az cloud update --profile {}'.format(self.origin_profile)) + + if exc_tb: + display('') + traceback.print_exception(exc_type, exc_val, exc_tb) + + +def current_profile(): + return cmd('az cloud show --query profile -otsv', show_stderr=False).result diff --git a/azdev/operations/tests/pytest_runner.py b/azdev/operations/tests/pytest_runner.py index 063cca345..e853a68fd 100644 --- a/azdev/operations/tests/pytest_runner.py +++ b/azdev/operations/tests/pytest_runner.py @@ -4,16 +4,24 @@ # license information. # ----------------------------------------------------------------------------- +import os + +from knack.log import get_logger + +from azdev.utilities import call + def get_test_runner(parallel, log_path, last_failed): """Create a pytest execution method""" def _run(test_paths, pytest_args): - from azdev.utilities import call - from knack.log import get_logger logger = get_logger(__name__) - arguments = ['-p', 'no:warnings', '--no-print-logs', '--junit-xml', log_path] + if os.name == 'posix': + arguments = ['-x', '-v', '--boxed', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path] + else: + arguments = ['-x', '-v', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path] + arguments.extend(test_paths) if parallel: arguments += ['-n', 'auto'] diff --git a/azdev/operations/tests/tests/__init__.py b/azdev/operations/tests/tests/__init__.py new file mode 100644 index 000000000..99c0f28cd --- /dev/null +++ b/azdev/operations/tests/tests/__init__.py @@ -0,0 +1,5 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- diff --git a/azdev/operations/tests/tests/test_profile_context.py b/azdev/operations/tests/tests/test_profile_context.py new file mode 100644 index 000000000..60b780d3f --- /dev/null +++ b/azdev/operations/tests/tests/test_profile_context.py @@ -0,0 +1,33 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + +import unittest + +from knack.util import CLIError + +from azdev.operations.tests.profile_context import ProfileContext + + +class TestProfileContext(unittest.TestCase): + + def test_profile_ok(self): + target_profiles = ['latest', '2017-03-09-profile', '2018-03-01-hybrid', '2019-03-01-hybrid'] + + for profile in target_profiles: + with ProfileContext(profile): + self.assertEqual(1, 1) + + def test_unsupported_profile(self): + unknown_profile = 'unknown-profile' + + with self.assertRaises(CLIError): + with ProfileContext(unknown_profile): + pass + + def test_raise_inner_exception(self): + with self.assertRaises(Exception): + with ProfileContext('latest'): + raise Exception('inner Exception') diff --git a/azdev/params.py b/azdev/params.py index 7994cc2f0..6d2c90402 100644 --- a/azdev/params.py +++ b/azdev/params.py @@ -51,6 +51,12 @@ def load_arguments(self, _): c.argument('pytest_args', nargs=argparse.REMAINDER, options_list=['--pytest-args', '-a'], help='Denotes the remaining args will be passed to pytest.') c.argument('last_failed', options_list='--lf', action='store_true', help='Re-run the last tests that failed.') + # CI parameters + c.argument('cli_ci', + action='store_true', + arg_group='Continuous Integration', + help='Apply incremental test strategy to Azure CLI on Azure DevOps') + with ArgumentsContext(self, 'coverage') as c: c.argument('prefix', type=str, help='Filter analysis by command prefix.') c.argument('report', action='store_true', help='Display results as a report.') diff --git a/azdev/utilities/git_util.py b/azdev/utilities/git_util.py index 8be06910d..817135a50 100644 --- a/azdev/utilities/git_util.py +++ b/azdev/utilities/git_util.py @@ -20,7 +20,7 @@ def filter_by_git_diff(selected_modules, git_source, git_target, git_repo): raise CLIError('usage error: [--src NAME] --tgt NAME --repo PATH') files_changed = diff_branches(git_repo, git_target, git_source) - mods_changed = _summarize_changed_mods(files_changed) + mods_changed = summarize_changed_mods(files_changed) repo_path = str(os.path.abspath(git_repo)).lower() to_remove = {'mod': [], 'core': [], 'ext': []} @@ -43,7 +43,7 @@ def filter_by_git_diff(selected_modules, git_source, git_target, git_repo): return selected_modules -def _summarize_changed_mods(files_changed): +def summarize_changed_mods(files_changed): from azdev.utilities import extract_module_name mod_set = set() diff --git a/setup.py b/setup.py index 096863366..7017d0b13 100644 --- a/setup.py +++ b/setup.py @@ -66,8 +66,8 @@ 'jinja2', 'knack', 'mock', - 'pytest~=4.4.0', - 'pytest-xdist', + 'pytest>=4.4.0', + 'pytest-xdist', # depends on pytest-forked 'pyyaml', 'requests', 'sphinx==1.6.7',