Enable azdev for Azure/azure-cli CI automation test

azdev/help.py

@@ -110,9 +110,6 @@
           populator-commands:
             - pytest -h
     examples:
-        - name: Run all tests.
-          text: azdev test --ci
-
         - name: Run tests for specific modules.
           text: azdev test {mod1} {mod2}
 

azdev/operations/tests/__init__.py

@@ -22,30 +22,30 @@
     COMMAND_MODULE_PREFIX, EXTENSION_PREFIX,
     make_dirs, get_azdev_config_dir,
     get_path_table, require_virtual_env, get_name_index)
-
+from .pytest_runner import get_test_runner
+from .profile_context import ProfileContext, current_profile
+from .incremental_strategy import CLIAzureDevOpsContext
 
 logger = get_logger(__name__)
 
 
 # pylint: disable=too-many-statements
 def run_tests(tests, xml_path=None, discover=False, in_series=False,
               run_live=False, profile=None, last_failed=False, pytest_args=None,
-              git_source=None, git_target=None, git_repo=None):
+              git_source=None, git_target=None, git_repo=None,
+              cli_ci=False):
 
     require_virtual_env()
 
     DEFAULT_RESULT_FILE = 'test_results.xml'
     DEFAULT_RESULT_PATH = os.path.join(get_azdev_config_dir(), DEFAULT_RESULT_FILE)
 
-    from .pytest_runner import get_test_runner
-
     heading('Run Tests')
 
-    original_profile = _get_profile(profile)
-    if not profile:
-        profile = original_profile
     path_table = get_path_table()
-    test_index = _get_test_index(profile, discover)
+
+    test_index = _get_test_index(profile or current_profile(), discover)
+
     if not tests:
         tests = list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys())
     if tests == ['CLI']:
@@ -54,10 +54,13 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
         tests = list(path_table['ext'].keys())
 
     # filter out tests whose modules haven't changed
-    tests = _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo)
+    modified_mods = _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo)
+    if modified_mods:
+        display('\nTest on modules: {}\n'.format(', '.join(modified_mods)))
 
-    if tests:
-        display('\nTESTS: {}\n'.format(', '.join(tests)))
+    if cli_ci is True:
+        ctx = CLIAzureDevOpsContext(git_repo, git_source, git_target)
+        modified_mods = ctx.filter(test_index)
 
     # resolve the path at which to dump the XML results
     xml_path = xml_path or DEFAULT_RESULT_PATH
@@ -73,6 +76,7 @@ def _find_test(index, name):
         name_comps = name.split('.')
         num_comps = len(name_comps)
         key_error = KeyError()
+
         for i in range(num_comps):
             check_name = '.'.join(name_comps[(-1 - i):])
             try:
@@ -87,7 +91,7 @@ def _find_test(index, name):
 
     # lookup test paths from index
     test_paths = []
-    for t in tests:
+    for t in modified_mods:
         try:
             test_path = os.path.normpath(_find_test(test_index, t))
             test_paths.append(test_path)
@@ -99,23 +103,17 @@ def _find_test(index, name):
     if not test_paths:
         raise CLIError('No tests selected to run.')
 
-    runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
-    exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
-    _summarize_test_results(xml_path)
-
-    # attempt to restore the original profile
-    if profile != original_profile:
-        result = raw_cmd('az cloud update --profile {}'.format(original_profile),
-                         "Restoring profile '{}'.".format(original_profile))
-        if result.exit_code != 0:
-            logger.warning("Failed to restore profile '%s'.", original_profile)
+    exit_code = 0
+    with ProfileContext(profile):
+        runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
+        exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
 
     sys.exit(0 if not exit_code else 1)
 
 
 def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo):
     from azdev.utilities import diff_branches, extract_module_name
-    from azdev.utilities.git_util import _summarize_changed_mods
+    from azdev.utilities.git_util import summarize_changed_mods
 
     if not any([git_source, git_target, git_repo]):
         return tests
@@ -124,7 +122,7 @@ def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo):
         raise CLIError('usage error: [--src NAME]  --tgt NAME --repo PATH')
 
     files_changed = diff_branches(git_repo, git_target, git_source)
-    mods_changed = _summarize_changed_mods(files_changed)
+    mods_changed = summarize_changed_mods(files_changed)
 
     repo_path = str(os.path.abspath(git_repo)).lower()
     to_remove = []
@@ -145,30 +143,6 @@ def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo):
     return tests
 
 
-def _get_profile(profile):
-    import colorama
-    colorama.init(autoreset=True)
-    try:
-        fore_red = colorama.Fore.RED if not IS_WINDOWS else ''
-        fore_reset = colorama.Fore.RESET if not IS_WINDOWS else ''
-        original_profile = raw_cmd('az cloud show --query profile -otsv', show_stderr=False).result
-        if not profile or original_profile == profile:
-            profile = original_profile
-            display('The tests are set to run against current profile {}.'
-                    .format(fore_red + original_profile + fore_reset))
-        elif original_profile != profile:
-            display('The tests are set to run against profile {} but the current az cloud profile is {}.'
-                    .format(fore_red + profile + fore_reset, fore_red + original_profile + fore_reset))
-            result = raw_cmd('az cloud update --profile {}'.format(profile),
-                             'SWITCHING TO PROFILE {}.'.format(fore_red + profile + fore_reset))
-            if result.exit_code != 0:
-                raise CLIError(result.error.output)
-        # returns the original profile so we can switch back if need be
-        return original_profile
-    except CalledProcessError:
-        raise CLIError('Failed to retrieve current az profile')
-
-
 def _discover_module_tests(mod_name, mod_data):
 
     # get the list of test files in each module
@@ -353,32 +327,3 @@ def _get_test_index(profile, discover):
             f.write(json.dumps(test_index))
         display('\ntest index created: {}'.format(test_index_path))
     return test_index
-
-
-def _summarize_test_results(xml_path):
-    import xml.etree.ElementTree as ElementTree
-
-    subheading('Results')
-
-    root = ElementTree.parse(xml_path).getroot()
-    summary = {
-        'time': root.get('time'),
-        'tests': root.get('tests'),
-        'skips': root.get('skips'),
-        'failures': root.get('failures'),
-        'errors': root.get('errors')
-    }
-    display('Time: {time} sec\tTests: {tests}\tSkipped: {skips}\tFailures: {failures}\tErrors: {errors}'.format(
-        **summary))
-
-    failed = []
-    for item in root.findall('testcase'):
-        if item.findall('failure'):
-            file_and_class = '.'.join(item.get('classname').split('.')[-2:])
-            failed.append('{}.{}'.format(file_and_class, item.get('name')))
-
-    if failed:
-        subheading('FAILURES')
-        for name in failed:
-            display(name)
-    display('')

azdev/operations/tests/incremental_strategy.py

@@ -0,0 +1,73 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+import abc
+from knack.util import CLIError
+
+from azdev.utilities import get_path_table, git_util
+
+
+# @wrapt.decorator
+# def cli_release_scenario(wrapped, _, args, kwargs):
+#     """
+#     Filter out those files in Azure CLI release stage
+#     """
+#     # TODO
+#     # if instance.resolved:
+#     #     return instance
+
+#     return wrapped(*args, **kwargs)
+
+
+class AzureDevOpsContext(abc.ABC):
+    def __init__(self, git_repo, git_source, git_target):
+        """
+        :param git_source: could be commit id, branch name or any valid value for git diff
+        :param git_target: could be commit id, branch name or any valid value for git diff
+        """
+        self.git_repo = git_repo
+        self.git_source = git_source
+        self.git_target = git_target
+
+    @abc.abstractmethod
+    def filter(self, test_index):
+        pass
+
+
+class CLIAzureDevOpsContext(AzureDevOpsContext):
+    """
+    Assemble strategy of incremental test on Azure DevOps Environment for Azure CLI
+    """
+    def __init__(self, git_repo, git_source, git_target):
+        super().__init__(git_repo, git_source, git_target)
+
+        if not any([self.git_source, self.git_target, self.git_repo]):
+            raise CLIError('usage error: [--src NAME]  --tgt NAME --repo PATH --cli-ci')
+
+        if not all([self.git_target, self.git_repo]):
+            raise CLIError('usage error: [--src NAME]  --tgt NAME --repo PATH --cli-ci')
+
+    @property
+    def modified_files(self):
+        modified_files = git_util.diff_branches(self.git_repo, self.git_source, self.git_target)
+        return [f for f in modified_files if f.startswith('src/')]
+
+    def filter(self, test_index):
+        """
+        Strategy on Azure CLI pull request verification stage.
+
+        :return: a list of modified packages
+        """
+
+        modified_packages = git_util.summarize_changed_mods(self.modified_files)
+
+        if any(core_package in modified_packages for core_package in ['core', 'testsdk', 'telemetry']):
+            path_table = get_path_table()
+
+            # tests under all packages
+            return list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys())
+
+        return modified_packages

azdev/operations/tests/profile_context.py

@@ -0,0 +1,45 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+import traceback
+
+from knack.log import get_logger
+from knack.util import CLIError
+
+from azdev.utilities import call, cmd
+from azdev.utilities import display
+
+
+logger = get_logger(__name__)
+
+
+class ProfileContext:
+    def __init__(self, profile_name=None):
+        self.target_profile = profile_name
+
+        self.origin_profile = current_profile()
+
+    def __enter__(self):
+        if self.target_profile is None or self.target_profile == self.origin_profile:
+            display('The tests are set to run against current profile "{}"'.format(self.origin_profile))
+        else:
+            result = cmd('az cloud update --profile {}'.format(self.target_profile),
+                         'Switching to target profile "{}"...'.format(self.target_profile))
+            if result.exit_code != 0:
+                raise CLIError(result.error.output.decode('utf-8'))
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.target_profile is not None and self.target_profile != self.origin_profile:
+            display('Switching back to origin profile "{}"...'.format(self.origin_profile))
+            call('az cloud update --profile {}'.format(self.origin_profile))
+
+        if exc_tb:
+            display('')
+            traceback.print_exception(exc_type, exc_val, exc_tb)
+
+
+def current_profile():
+    return cmd('az cloud show --query profile -otsv', show_stderr=False).result

azdev/operations/tests/pytest_runner.py

@@ -4,16 +4,24 @@
 # license information.
 # -----------------------------------------------------------------------------
 
+import os
+
+from knack.log import get_logger
+
+from azdev.utilities import call
+
 
 def get_test_runner(parallel, log_path, last_failed):
     """Create a pytest execution method"""
     def _run(test_paths, pytest_args):
-        from azdev.utilities import call
-        from knack.log import get_logger
 
         logger = get_logger(__name__)
 
-        arguments = ['-p', 'no:warnings', '--no-print-logs', '--junit-xml', log_path]
+        if os.name == 'posix':
+            arguments = ['-x', '-v', '--boxed', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path]
+        else:
+            arguments = ['-x', '-v', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path]
+
         arguments.extend(test_paths)
         if parallel:
             arguments += ['-n', 'auto']

azdev/operations/tests/tests/__init__.py

@@ -0,0 +1,5 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------

azdev/operations/tests/tests/test_profile_context.py

@@ -0,0 +1,33 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+import unittest
+
+from knack.util import CLIError
+
+from azdev.operations.tests.profile_context import ProfileContext
+
+
+class TestProfileContext(unittest.TestCase):
+
+    def test_profile_ok(self):
+        target_profiles = ['latest', '2017-03-09-profile', '2018-03-01-hybrid', '2019-03-01-hybrid']
+
+        for profile in target_profiles:
+            with ProfileContext(profile):
+                self.assertEqual(1, 1)
+
+    def test_unsupported_profile(self):
+        unknown_profile = 'unknown-profile'
+
+        with self.assertRaises(CLIError):
+            with ProfileContext(unknown_profile):
+                pass
+
+    def test_raise_inner_exception(self):
+        with self.assertRaises(Exception):
+            with ProfileContext('latest'):
+                raise Exception('inner Exception')

azdev/params.py

@@ -51,6 +51,12 @@ def load_arguments(self, _):
         c.argument('pytest_args', nargs=argparse.REMAINDER, options_list=['--pytest-args', '-a'], help='Denotes the remaining args will be passed to pytest.')
         c.argument('last_failed', options_list='--lf', action='store_true', help='Re-run the last tests that failed.')
 
+        # CI parameters
+        c.argument('cli_ci',
+                   action='store_true',
+                   arg_group='Continuous Integration',
+                   help='Apply incremental test strategy to Azure CLI on Azure DevOps')
+
     with ArgumentsContext(self, 'coverage') as c:
         c.argument('prefix', type=str, help='Filter analysis by command prefix.')
         c.argument('report', action='store_true', help='Display results as a report.')