From 6ca899701818ca50a4d41e6b4a7deb84ea63fc9c Mon Sep 17 00:00:00 2001
From: Jianhui Harold <haroldrandom@gmail.com>
Date: Wed, 8 Apr 2020 18:06:53 +0800
Subject: [PATCH 01/27] test automation test

---
 azdev/operations/tests/__init__.py      | 19 ++++++-----
 azdev/operations/tests/pytest_runner.py |  7 ++--
 azure-pipelines.yml                     | 44 +++++++++++++++++++++++++
 setup.py                                |  2 +-
 4 files changed, 59 insertions(+), 13 deletions(-)

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index d6b85ed83..47bc13e88 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -22,7 +22,7 @@
     COMMAND_MODULE_PREFIX, EXTENSION_PREFIX,
     make_dirs, get_azdev_config_dir,
     get_path_table, require_virtual_env, get_name_index)
-
+from .pytest_runner import get_test_runner
 
 logger = get_logger(__name__)
 
@@ -37,27 +37,28 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
     DEFAULT_RESULT_FILE = 'test_results.xml'
     DEFAULT_RESULT_PATH = os.path.join(get_azdev_config_dir(), DEFAULT_RESULT_FILE)
 
-    from .pytest_runner import get_test_runner
-
     heading('Run Tests')
 
     original_profile = _get_profile(profile)
     if not profile:
         profile = original_profile
     path_table = get_path_table()
+
     test_index = _get_test_index(profile, discover)
+
+    mods_abbr = []
     if not tests:
-        tests = list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys())
+        mods_abbr = list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys())
     if tests == ['CLI']:
-        tests = list(path_table['mod'].keys()) + list(path_table['core'].keys())
+        mods_abbr = list(path_table['mod'].keys()) + list(path_table['core'].keys())
     elif tests == ['EXT']:
-        tests = list(path_table['ext'].keys())
+        mods_abbr = list(path_table['ext'].keys())
 
     # filter out tests whose modules haven't changed
-    tests = _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo)
+    modified_mods = _filter_by_git_diff(mods_abbr, test_index, git_source, git_target, git_repo)
 
-    if tests:
-        display('\nTESTS: {}\n'.format(', '.join(tests)))
+    if modified_mods:
+        display('\nTest on modules: {}\n'.format(', '.join(modified_mods)))
 
     # resolve the path at which to dump the XML results
     xml_path = xml_path or DEFAULT_RESULT_PATH
diff --git a/azdev/operations/tests/pytest_runner.py b/azdev/operations/tests/pytest_runner.py
index 063cca345..e80b9066d 100644
--- a/azdev/operations/tests/pytest_runner.py
+++ b/azdev/operations/tests/pytest_runner.py
@@ -4,16 +4,17 @@
 # license information.
 # -----------------------------------------------------------------------------
 
+from azdev.utilities import call
+from knack.log import get_logger
+
 
 def get_test_runner(parallel, log_path, last_failed):
     """Create a pytest execution method"""
     def _run(test_paths, pytest_args):
-        from azdev.utilities import call
-        from knack.log import get_logger
 
         logger = get_logger(__name__)
 
-        arguments = ['-p', 'no:warnings', '--no-print-logs', '--junit-xml', log_path]
+        arguments = ['-v', '--junit-xml', log_path]
         arguments.extend(test_paths)
         if parallel:
             arguments += ['-n', 'auto']
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 37b5f7b5b..f2b9b6d6e 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -267,3 +267,47 @@ jobs:
       # verify azdev style works
       azdev style redis
     displayName: 'Test azdev style'
+
+- job: CLIAutomationTest
+  condition: succeeded()
+  timeoutInMinutes: 20
+
+  pool:
+    vmImage: 'ubuntu-16.04'
+  strategy:
+    matrix:
+      Python36:
+        python.version: '3.6'
+      Python37:
+        python.version: '3.7'
+      Python38:
+        python.version: '3.8'
+  steps:
+    - task: UsePythonVersion@0
+      displayName: 'Use Python $(python.version)'
+      inputs:
+        versionSpec: '$(python.version)'
+    - bash: |
+        # clone azure-cli
+        git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli
+
+      displayName: Prepare CLI
+    - bash: |
+        set -ev
+
+        python -m venv env
+        chmod +x ./env/bin/active
+        source ./env/bin/active
+
+
+        # install azdev
+        pip install .
+
+        azdev setup -c ../azure-cli
+
+        azdev --version
+        az --version
+
+        azdev test --repo ../azure-cli --src=dev --tgt=HEAD~2
+
+      displayName: Run Automation Test on CLI
diff --git a/setup.py b/setup.py
index 096863366..4e35ddefb 100644
--- a/setup.py
+++ b/setup.py
@@ -66,7 +66,7 @@
         'jinja2',
         'knack',
         'mock',
-        'pytest~=4.4.0',
+        'pytest>=4.4.0',
         'pytest-xdist',
         'pyyaml',
         'requests',

From f2978f61e5d130f2b5ff2797ba6d8df441f83c4e Mon Sep 17 00:00:00 2001
From: Jianhui Harold <haroldrandom@gmail.com>
Date: Wed, 8 Apr 2020 18:13:19 +0800
Subject: [PATCH 02/27] fix bug

---
 azure-pipelines.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index f2b9b6d6e..caa2308d7 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -296,9 +296,8 @@ jobs:
         set -ev
 
         python -m venv env
-        chmod +x ./env/bin/active
-        source ./env/bin/active
-
+        chmod +x ./env/bin/activate
+        source ./env/bin/activate
 
         # install azdev
         pip install .

From e08fd5346c97718e1305608c36dad9edba871418 Mon Sep 17 00:00:00 2001
From: Jianhui Harold <haroldrandom@gmail.com>
Date: Wed, 8 Apr 2020 18:16:33 +0800
Subject: [PATCH 03/27] fix import order error

---
 azdev/operations/tests/pytest_runner.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/azdev/operations/tests/pytest_runner.py b/azdev/operations/tests/pytest_runner.py
index e80b9066d..6a3e48693 100644
--- a/azdev/operations/tests/pytest_runner.py
+++ b/azdev/operations/tests/pytest_runner.py
@@ -4,9 +4,10 @@
 # license information.
 # -----------------------------------------------------------------------------
 
-from azdev.utilities import call
 from knack.log import get_logger
 
+from azdev.utilities import call
+
 
 def get_test_runner(parallel, log_path, last_failed):
     """Create a pytest execution method"""

From b2e598cd87a7ef52518cc71efc5e05538e7bf52c Mon Sep 17 00:00:00 2001
From: Jianhui Harold <haroldrandom@gmail.com>
Date: Wed, 8 Apr 2020 18:41:36 +0800
Subject: [PATCH 04/27] fix bug that no tests to run

---
 azdev/operations/tests/__init__.py      | 2 +-
 azdev/operations/tests/pytest_runner.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index 47bc13e88..a2abc7624 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -88,7 +88,7 @@ def _find_test(index, name):
 
     # lookup test paths from index
     test_paths = []
-    for t in tests:
+    for t in modified_mods:
         try:
             test_path = os.path.normpath(_find_test(test_index, t))
             test_paths.append(test_path)
diff --git a/azdev/operations/tests/pytest_runner.py b/azdev/operations/tests/pytest_runner.py
index 6a3e48693..7c69fd5d7 100644
--- a/azdev/operations/tests/pytest_runner.py
+++ b/azdev/operations/tests/pytest_runner.py
@@ -15,7 +15,7 @@ def _run(test_paths, pytest_args):
 
         logger = get_logger(__name__)
 
-        arguments = ['-v', '--junit-xml', log_path]
+        arguments = ['-x', '-v', '--junit-xml', log_path]
         arguments.extend(test_paths)
         if parallel:
             arguments += ['-n', 'auto']

From 82479092c6eab460822bd03201d151d7faa4f4ba Mon Sep 17 00:00:00 2001
From: Jianhui Harold <haroldrandom@gmail.com>
Date: Wed, 8 Apr 2020 18:56:17 +0800
Subject: [PATCH 05/27] fix bug that tests will miss origin parameter value

---
 azdev/operations/tests/__init__.py |  9 ++++-----
 azure-pipelines.yml                | 15 ++-------------
 2 files changed, 6 insertions(+), 18 deletions(-)

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index a2abc7624..f5b47de31 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -46,16 +46,15 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
 
     test_index = _get_test_index(profile, discover)
 
-    mods_abbr = []
     if not tests:
-        mods_abbr = list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys())
+        tests = list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys())
     if tests == ['CLI']:
-        mods_abbr = list(path_table['mod'].keys()) + list(path_table['core'].keys())
+        tests = list(path_table['mod'].keys()) + list(path_table['core'].keys())
     elif tests == ['EXT']:
-        mods_abbr = list(path_table['ext'].keys())
+        tests = list(path_table['ext'].keys())
 
     # filter out tests whose modules haven't changed
-    modified_mods = _filter_by_git_diff(mods_abbr, test_index, git_source, git_target, git_repo)
+    modified_mods = _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo)
 
     if modified_mods:
         display('\nTest on modules: {}\n'.format(', '.join(modified_mods)))
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index caa2308d7..ba01de99e 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -287,25 +287,14 @@ jobs:
       displayName: 'Use Python $(python.version)'
       inputs:
         versionSpec: '$(python.version)'
-    - bash: |
-        # clone azure-cli
-        git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli
-
-      displayName: Prepare CLI
     - bash: |
         set -ev
 
-        python -m venv env
-        chmod +x ./env/bin/activate
-        source ./env/bin/activate
-
-        # install azdev
-        pip install .
+        . scripts/ci/install.sh
 
-        azdev setup -c ../azure-cli
+        az --version
 
         azdev --version
-        az --version
 
         azdev test --repo ../azure-cli --src=dev --tgt=HEAD~2
 

From 3f057de502ad76c89c3764feb20bfd97b1fc6234 Mon Sep 17 00:00:00 2001
From: Jianhui Harold <haroldrandom@gmail.com>
Date: Wed, 8 Apr 2020 20:43:02 +0800
Subject: [PATCH 06/27] fix bug

---
 azure-pipelines.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index ba01de99e..c99eaa224 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -269,6 +269,7 @@ jobs:
     displayName: 'Test azdev style'
 
 - job: CLIAutomationTest
+  dependsOn: BuildPythonWheel
   condition: succeeded()
   timeoutInMinutes: 20
 

From 2f8be71ac3796bc742adc540d1793ddeb46aabfd Mon Sep 17 00:00:00 2001
From: Jianhui Harold <haroldrandom@gmail.com>
Date: Wed, 8 Apr 2020 22:02:51 +0800
Subject: [PATCH 07/27] Add ProfileContext management

---
 azdev/operations/tests/__init__.py            | 100 ++++++++++--------
 azdev/operations/tests/profile_context.py     |  41 +++++++
 azdev/operations/tests/tests/__init__.py      |   5 +
 .../tests/tests/test_profile_context.py       |  33 ++++++
 4 files changed, 132 insertions(+), 47 deletions(-)
 create mode 100644 azdev/operations/tests/profile_context.py
 create mode 100644 azdev/operations/tests/tests/__init__.py
 create mode 100644 azdev/operations/tests/tests/test_profile_context.py

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index f5b47de31..325384947 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -23,6 +23,7 @@
     make_dirs, get_azdev_config_dir,
     get_path_table, require_virtual_env, get_name_index)
 from .pytest_runner import get_test_runner
+from .profile_context import ProfileContext
 
 logger = get_logger(__name__)
 
@@ -34,6 +35,11 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
 
     require_virtual_env()
 
+    from .profile_context import ProfileContext
+
+    with ProfileContext('latest'):
+        raise CLIError('the hell?!?')
+
     DEFAULT_RESULT_FILE = 'test_results.xml'
     DEFAULT_RESULT_PATH = os.path.join(get_azdev_config_dir(), DEFAULT_RESULT_FILE)
 
@@ -64,53 +70,53 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
     if not xml_path.endswith('.xml'):
         xml_path = os.path.join(xml_path, DEFAULT_RESULT_FILE)
 
-    # process environment variables
-    if run_live:
-        logger.warning('RUNNING TESTS LIVE')
-        os.environ[ENV_VAR_TEST_LIVE] = 'True'
-
-    def _find_test(index, name):
-        name_comps = name.split('.')
-        num_comps = len(name_comps)
-        key_error = KeyError()
-        for i in range(num_comps):
-            check_name = '.'.join(name_comps[(-1 - i):])
-            try:
-                match = index[check_name]
-                if check_name != name:
-                    logger.info("Test found using just '%s'. The rest of the name was ignored.\n", check_name)
-                return match
-            except KeyError as ex:
-                key_error = ex
-                continue
-        raise key_error
-
-    # lookup test paths from index
-    test_paths = []
-    for t in modified_mods:
-        try:
-            test_path = os.path.normpath(_find_test(test_index, t))
-            test_paths.append(test_path)
-        except KeyError:
-            logger.warning("'%s' not found. If newly added, re-run with --discover", t)
-            continue
-
-    # Tests have been collected. Now run them.
-    if not test_paths:
-        raise CLIError('No tests selected to run.')
-
-    runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
-    exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
-    _summarize_test_results(xml_path)
-
-    # attempt to restore the original profile
-    if profile != original_profile:
-        result = raw_cmd('az cloud update --profile {}'.format(original_profile),
-                         "Restoring profile '{}'.".format(original_profile))
-        if result.exit_code != 0:
-            logger.warning("Failed to restore profile '%s'.", original_profile)
-
-    sys.exit(0 if not exit_code else 1)
+    # # process environment variables
+    # if run_live:
+    #     logger.warning('RUNNING TESTS LIVE')
+    #     os.environ[ENV_VAR_TEST_LIVE] = 'True'
+    #
+    # def _find_test(index, name):
+    #     name_comps = name.split('.')
+    #     num_comps = len(name_comps)
+    #     key_error = KeyError()
+    #     for i in range(num_comps):
+    #         check_name = '.'.join(name_comps[(-1 - i):])
+    #         try:
+    #             match = index[check_name]
+    #             if check_name != name:
+    #                 logger.info("Test found using just '%s'. The rest of the name was ignored.\n", check_name)
+    #             return match
+    #         except KeyError as ex:
+    #             key_error = ex
+    #             continue
+    #     raise key_error
+    #
+    # # lookup test paths from index
+    # test_paths = []
+    # for t in modified_mods:
+    #     try:
+    #         test_path = os.path.normpath(_find_test(test_index, t))
+    #         test_paths.append(test_path)
+    #     except KeyError:
+    #         logger.warning("'%s' not found. If newly added, re-run with --discover", t)
+    #         continue
+    #
+    # # Tests have been collected. Now run them.
+    # if not test_paths:
+    #     raise CLIError('No tests selected to run.')
+    #
+    # runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
+    # exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
+    # _summarize_test_results(xml_path)
+    #
+    # # attempt to restore the original profile
+    # if profile != original_profile:
+    #     result = raw_cmd('az cloud update --profile {}'.format(original_profile),
+    #                      "Restoring profile '{}'.".format(original_profile))
+    #     if result.exit_code != 0:
+    #         logger.warning("Failed to restore profile '%s'.", original_profile)
+    #
+    # sys.exit(0 if not exit_code else 1)
 
 
 def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo):
diff --git a/azdev/operations/tests/profile_context.py b/azdev/operations/tests/profile_context.py
new file mode 100644
index 000000000..e3c6e6a42
--- /dev/null
+++ b/azdev/operations/tests/profile_context.py
@@ -0,0 +1,41 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+import traceback
+
+from knack.log import get_logger
+from knack.util import CLIError
+
+from azdev.utilities import call, cmd
+from azdev.utilities import display
+
+
+logger = get_logger(__name__)
+
+
+class ProfileContext:
+    def __init__(self, profile_name=None):
+        self.target_profile = profile_name
+
+        self.origin_profile = cmd('az cloud show --query profile -otsv', show_stderr=False).result
+
+    def __enter__(self):
+        if self.target_profile is not None and self.target_profile == self.origin_profile:
+            display('The tests are set to run against current profile "{}"'.format(self.origin_profile))
+        else:
+            result = cmd('az cloud update --profile {}'.format(self.target_profile),
+                         'Switching to target profile "{}"...'.format(self.target_profile))
+            if result.exit_code != 0:
+                raise CLIError(result.error.output.decode('utf-8'))
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.target_profile is not None and self.target_profile != self.origin_profile:
+            display('Switching back to origin profile "{}"...'.format(self.origin_profile))
+            call('az cloud update --profile {}'.format(self.origin_profile))
+
+        if exc_tb:
+            display('')
+            traceback.print_exception(exc_type, exc_val, exc_tb)
diff --git a/azdev/operations/tests/tests/__init__.py b/azdev/operations/tests/tests/__init__.py
new file mode 100644
index 000000000..99c0f28cd
--- /dev/null
+++ b/azdev/operations/tests/tests/__init__.py
@@ -0,0 +1,5 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
diff --git a/azdev/operations/tests/tests/test_profile_context.py b/azdev/operations/tests/tests/test_profile_context.py
new file mode 100644
index 000000000..06241fa52
--- /dev/null
+++ b/azdev/operations/tests/tests/test_profile_context.py
@@ -0,0 +1,33 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+import unittest
+
+from knack.util import CLIError
+
+from ..profile_context import ProfileContext
+
+
+class TestProfileContext(unittest.TestCase):
+
+    def test_profile_ok(self):
+        target_profiles = ['latest', '2017-03-09-profile', '2018-03-01-hybrid', '2019-03-01-hybrid']
+
+        for profile in target_profiles:
+            with ProfileContext(profile):
+                pass
+
+    def test_unsupported_profile(self):
+        unknown_profile = 'unknown-profile'
+
+        with self.assertRaises(CLIError):
+            with ProfileContext(unknown_profile):
+                pass
+
+    def test_raise_inner_exception(self):
+        with self.assertRaises(Exception):
+            with ProfileContext('latest'):
+                raise Exception('inner Exception')

From 11059f4df9f975df450585ec854326b0634af885 Mon Sep 17 00:00:00 2001
From: Jianhui Harold <haroldrandom@gmail.com>
Date: Wed, 8 Apr 2020 22:12:42 +0800
Subject: [PATCH 08/27] update pipeline

---
 azure-pipelines.yml | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index c99eaa224..a86cfa2eb 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -288,15 +288,23 @@ jobs:
       displayName: 'Use Python $(python.version)'
       inputs:
         versionSpec: '$(python.version)'
+    - bash: |
+        # clone azure-cli
+        git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli
     - bash: |
         set -ev
 
-        . scripts/ci/install.sh
+        python -m venv env
+        chmod +x ./env/bin/active
+        source ./env/bin/active
 
-        az --version
+        # install azdev
+        pip install .
 
+        # setup
+        azdev setup -c ../azure-cli
         azdev --version
+        az --version
 
         azdev test --repo ../azure-cli --src=dev --tgt=HEAD~2
-
       displayName: Run Automation Test on CLI

From 29e42f76d4fc138ce32bdff309df1f2056b2935e Mon Sep 17 00:00:00 2001
From: Jianhui Harold <haroldrandom@gmail.com>
Date: Wed, 8 Apr 2020 22:20:30 +0800
Subject: [PATCH 09/27] fix bug

---
 azdev/operations/tests/__init__.py | 94 +++++++++++++++---------------
 azure-pipelines.yml                |  4 +-
 2 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index 325384947..b02acca5e 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -70,53 +70,53 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
     if not xml_path.endswith('.xml'):
         xml_path = os.path.join(xml_path, DEFAULT_RESULT_FILE)
 
-    # # process environment variables
-    # if run_live:
-    #     logger.warning('RUNNING TESTS LIVE')
-    #     os.environ[ENV_VAR_TEST_LIVE] = 'True'
-    #
-    # def _find_test(index, name):
-    #     name_comps = name.split('.')
-    #     num_comps = len(name_comps)
-    #     key_error = KeyError()
-    #     for i in range(num_comps):
-    #         check_name = '.'.join(name_comps[(-1 - i):])
-    #         try:
-    #             match = index[check_name]
-    #             if check_name != name:
-    #                 logger.info("Test found using just '%s'. The rest of the name was ignored.\n", check_name)
-    #             return match
-    #         except KeyError as ex:
-    #             key_error = ex
-    #             continue
-    #     raise key_error
-    #
-    # # lookup test paths from index
-    # test_paths = []
-    # for t in modified_mods:
-    #     try:
-    #         test_path = os.path.normpath(_find_test(test_index, t))
-    #         test_paths.append(test_path)
-    #     except KeyError:
-    #         logger.warning("'%s' not found. If newly added, re-run with --discover", t)
-    #         continue
-    #
-    # # Tests have been collected. Now run them.
-    # if not test_paths:
-    #     raise CLIError('No tests selected to run.')
-    #
-    # runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
-    # exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
-    # _summarize_test_results(xml_path)
-    #
-    # # attempt to restore the original profile
-    # if profile != original_profile:
-    #     result = raw_cmd('az cloud update --profile {}'.format(original_profile),
-    #                      "Restoring profile '{}'.".format(original_profile))
-    #     if result.exit_code != 0:
-    #         logger.warning("Failed to restore profile '%s'.", original_profile)
-    #
-    # sys.exit(0 if not exit_code else 1)
+    # process environment variables
+    if run_live:
+        logger.warning('RUNNING TESTS LIVE')
+        os.environ[ENV_VAR_TEST_LIVE] = 'True'
+
+    def _find_test(index, name):
+        name_comps = name.split('.')
+        num_comps = len(name_comps)
+        key_error = KeyError()
+        for i in range(num_comps):
+            check_name = '.'.join(name_comps[(-1 - i):])
+            try:
+                match = index[check_name]
+                if check_name != name:
+                    logger.info("Test found using just '%s'. The rest of the name was ignored.\n", check_name)
+                return match
+            except KeyError as ex:
+                key_error = ex
+                continue
+        raise key_error
+
+    # lookup test paths from index
+    test_paths = []
+    for t in modified_mods:
+        try:
+            test_path = os.path.normpath(_find_test(test_index, t))
+            test_paths.append(test_path)
+        except KeyError:
+            logger.warning("'%s' not found. If newly added, re-run with --discover", t)
+            continue
+
+    # Tests have been collected. Now run them.
+    if not test_paths:
+        raise CLIError('No tests selected to run.')
+
+    runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
+    exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
+    _summarize_test_results(xml_path)
+
+    # attempt to restore the original profile
+    if profile != original_profile:
+        result = raw_cmd('az cloud update --profile {}'.format(original_profile),
+                         "Restoring profile '{}'.".format(original_profile))
+        if result.exit_code != 0:
+            logger.warning("Failed to restore profile '%s'.", original_profile)
+
+    sys.exit(0 if not exit_code else 1)
 
 
 def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo):
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index a86cfa2eb..6a094bc7d 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -295,8 +295,8 @@ jobs:
         set -ev
 
         python -m venv env
-        chmod +x ./env/bin/active
-        source ./env/bin/active
+        chmod +x ./env/bin/activate
+        . env/bin/activate
 
         # install azdev
         pip install .

From 48a8023fc0916f3b0238c430d6bfd73b87437c9a Mon Sep 17 00:00:00 2001
From: Jianhui Harold <haroldrandom@gmail.com>
Date: Wed, 8 Apr 2020 22:26:38 +0800
Subject: [PATCH 10/27] fix style error

---
 azdev/operations/tests/__init__.py                   | 6 ------
 azdev/operations/tests/tests/test_profile_context.py | 2 +-
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index b02acca5e..f5b47de31 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -23,7 +23,6 @@
     make_dirs, get_azdev_config_dir,
     get_path_table, require_virtual_env, get_name_index)
 from .pytest_runner import get_test_runner
-from .profile_context import ProfileContext
 
 logger = get_logger(__name__)
 
@@ -35,11 +34,6 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
 
     require_virtual_env()
 
-    from .profile_context import ProfileContext
-
-    with ProfileContext('latest'):
-        raise CLIError('the hell?!?')
-
     DEFAULT_RESULT_FILE = 'test_results.xml'
     DEFAULT_RESULT_PATH = os.path.join(get_azdev_config_dir(), DEFAULT_RESULT_FILE)
 
diff --git a/azdev/operations/tests/tests/test_profile_context.py b/azdev/operations/tests/tests/test_profile_context.py
index 06241fa52..dfbc4d169 100644
--- a/azdev/operations/tests/tests/test_profile_context.py
+++ b/azdev/operations/tests/tests/test_profile_context.py
@@ -18,7 +18,7 @@ def test_profile_ok(self):
 
         for profile in target_profiles:
             with ProfileContext(profile):
-                pass
+                self.assertEqual(1, 1)
 
     def test_unsupported_profile(self):
         unknown_profile = 'unknown-profile'

From 04ea4da3ccd047271a1201d7c97fc5f271842f69 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 9 Apr 2020 01:54:07 +0000
Subject: [PATCH 11/27] fix bug

---
 azdev/operations/tests/profile_context.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azdev/operations/tests/profile_context.py b/azdev/operations/tests/profile_context.py
index e3c6e6a42..ef360958c 100644
--- a/azdev/operations/tests/profile_context.py
+++ b/azdev/operations/tests/profile_context.py
@@ -23,7 +23,7 @@ def __init__(self, profile_name=None):
         self.origin_profile = cmd('az cloud show --query profile -otsv', show_stderr=False).result
 
     def __enter__(self):
-        if self.target_profile is not None and self.target_profile == self.origin_profile:
+        if self.target_profile is None or self.target_profile == self.origin_profile:
             display('The tests are set to run against current profile "{}"'.format(self.origin_profile))
         else:
             result = cmd('az cloud update --profile {}'.format(self.target_profile),

From 2d869bef3ad93b54acd4abc0485c05237463d268 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 9 Apr 2020 02:59:09 +0000
Subject: [PATCH 12/27] apply ProfileContext

---
 azdev/operations/tests/__init__.py            | 53 ++++---------------
 ...ofile_context.py => profile_management.py} |  4 ++
 azure-pipelines.yml                           |  4 +-
 3 files changed, 15 insertions(+), 46 deletions(-)
 rename azdev/operations/tests/{profile_context.py => profile_management.py} (94%)

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index f5b47de31..a25030a73 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -23,6 +23,7 @@
     make_dirs, get_azdev_config_dir,
     get_path_table, require_virtual_env, get_name_index)
 from .pytest_runner import get_test_runner
+from .profile_management import ProfileContext, current_profile
 
 logger = get_logger(__name__)
 
@@ -30,7 +31,8 @@
 # pylint: disable=too-many-statements
 def run_tests(tests, xml_path=None, discover=False, in_series=False,
               run_live=False, profile=None, last_failed=False, pytest_args=None,
-              git_source=None, git_target=None, git_repo=None):
+              git_source=None, git_target=None, git_repo=None,
+              cli_ci=False, ext_ci=False):
 
     require_virtual_env()
 
@@ -39,12 +41,9 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
 
     heading('Run Tests')
 
-    original_profile = _get_profile(profile)
-    if not profile:
-        profile = original_profile
     path_table = get_path_table()
 
-    test_index = _get_test_index(profile, discover)
+    test_index = _get_test_index(profile or current_profile(), discover)
 
     if not tests:
         tests = list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys())
@@ -73,6 +72,7 @@ def _find_test(index, name):
         name_comps = name.split('.')
         num_comps = len(name_comps)
         key_error = KeyError()
+
         for i in range(num_comps):
             check_name = '.'.join(name_comps[(-1 - i):])
             try:
@@ -99,16 +99,10 @@ def _find_test(index, name):
     if not test_paths:
         raise CLIError('No tests selected to run.')
 
-    runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
-    exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
-    _summarize_test_results(xml_path)
-
-    # attempt to restore the original profile
-    if profile != original_profile:
-        result = raw_cmd('az cloud update --profile {}'.format(original_profile),
-                         "Restoring profile '{}'.".format(original_profile))
-        if result.exit_code != 0:
-            logger.warning("Failed to restore profile '%s'.", original_profile)
+    exit_code = 0
+    with ProfileContext(profile):
+        runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
+        exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
 
     sys.exit(0 if not exit_code else 1)
 
@@ -353,32 +347,3 @@ def _get_test_index(profile, discover):
             f.write(json.dumps(test_index))
         display('\ntest index created: {}'.format(test_index_path))
     return test_index
-
-
-def _summarize_test_results(xml_path):
-    import xml.etree.ElementTree as ElementTree
-
-    subheading('Results')
-
-    root = ElementTree.parse(xml_path).getroot()
-    summary = {
-        'time': root.get('time'),
-        'tests': root.get('tests'),
-        'skips': root.get('skips'),
-        'failures': root.get('failures'),
-        'errors': root.get('errors')
-    }
-    display('Time: {time} sec\tTests: {tests}\tSkipped: {skips}\tFailures: {failures}\tErrors: {errors}'.format(
-        **summary))
-
-    failed = []
-    for item in root.findall('testcase'):
-        if item.findall('failure'):
-            file_and_class = '.'.join(item.get('classname').split('.')[-2:])
-            failed.append('{}.{}'.format(file_and_class, item.get('name')))
-
-    if failed:
-        subheading('FAILURES')
-        for name in failed:
-            display(name)
-    display('')
diff --git a/azdev/operations/tests/profile_context.py b/azdev/operations/tests/profile_management.py
similarity index 94%
rename from azdev/operations/tests/profile_context.py
rename to azdev/operations/tests/profile_management.py
index ef360958c..bd533705f 100644
--- a/azdev/operations/tests/profile_context.py
+++ b/azdev/operations/tests/profile_management.py
@@ -39,3 +39,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         if exc_tb:
             display('')
             traceback.print_exception(exc_type, exc_val, exc_tb)
+
+
+def current_profile():
+    return cmd('az cloud show --query profile -otsv', show_stderr=False).result
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 6a094bc7d..cdd1a66e3 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -41,7 +41,7 @@ jobs:
     inputs:
       targetType: 'filePath'
       filePath: scripts/ci/run_tox.sh
-      
+
 - job: Tox38
   displayName: 'Tox: Python 3.8'
   condition: succeeded()
@@ -306,5 +306,5 @@ jobs:
         azdev --version
         az --version
 
-        azdev test --repo ../azure-cli --src=dev --tgt=HEAD~2
+        azdev test --repo ../azure-cli --src=dev --tgt=HEAD~2 --series
       displayName: Run Automation Test on CLI

From a50137ceeca314818856b083762696bbb5d3bfcc Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 9 Apr 2020 09:10:19 +0000
Subject: [PATCH 13/27] Init incremental strategy on ADO for CLI

---
 azdev/help.py                                 |  3 -
 azdev/operations/tests/__init__.py            | 68 ++++++---------
 .../operations/tests/incremental_strategy.py  | 87 +++++++++++++++++++
 azdev/params.py                               | 10 +++
 azdev/utilities/git_util.py                   |  4 +-
 setup.py                                      |  1 +
 6 files changed, 126 insertions(+), 47 deletions(-)
 create mode 100644 azdev/operations/tests/incremental_strategy.py

diff --git a/azdev/help.py b/azdev/help.py
index e03ceca59..1d9b8984b 100644
--- a/azdev/help.py
+++ b/azdev/help.py
@@ -110,9 +110,6 @@
           populator-commands:
             - pytest -h
     examples:
-        - name: Run all tests.
-          text: azdev test --ci
-
         - name: Run tests for specific modules.
           text: azdev test {mod1} {mod2}
 
diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index a25030a73..4d99b44ad 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -24,6 +24,7 @@
     get_path_table, require_virtual_env, get_name_index)
 from .pytest_runner import get_test_runner
 from .profile_management import ProfileContext, current_profile
+from .incremental_strategy import CLIAzureDevOpsContext
 
 logger = get_logger(__name__)
 
@@ -43,6 +44,8 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
 
     path_table = get_path_table()
 
+    from pprint import pprint
+
     test_index = _get_test_index(profile or current_profile(), discover)
 
     if not tests:
@@ -54,10 +57,15 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
 
     # filter out tests whose modules haven't changed
     modified_mods = _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo)
-
     if modified_mods:
         display('\nTest on modules: {}\n'.format(', '.join(modified_mods)))
 
+    if cli_ci is True:
+        ctx = CLIAzureDevOpsContext(git_repo, git_source, git_target)
+        modified_mods = ctx.filter(test_index)
+    if ext_ci is True:
+        pass
+
     # resolve the path at which to dump the XML results
     xml_path = xml_path or DEFAULT_RESULT_PATH
     if not xml_path.endswith('.xml'):
@@ -85,31 +93,31 @@ def _find_test(index, name):
                 continue
         raise key_error
 
-    # lookup test paths from index
-    test_paths = []
-    for t in modified_mods:
-        try:
-            test_path = os.path.normpath(_find_test(test_index, t))
-            test_paths.append(test_path)
-        except KeyError:
-            logger.warning("'%s' not found. If newly added, re-run with --discover", t)
-            continue
+    # # lookup test paths from index
+    # test_paths = []
+    # for t in modified_mods:
+    #     try:
+    #         test_path = os.path.normpath(_find_test(test_index, t))
+    #         test_paths.append(test_path)
+    #     except KeyError:
+    #         logger.warning("'%s' not found. If newly added, re-run with --discover", t)
+    #         continue
 
-    # Tests have been collected. Now run them.
-    if not test_paths:
-        raise CLIError('No tests selected to run.')
+    # # Tests have been collected. Now run them.
+    # if not test_paths:
+    #     raise CLIError('No tests selected to run.')
 
     exit_code = 0
-    with ProfileContext(profile):
-        runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
-        exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
+    # with ProfileContext(profile):
+    #     runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
+    #     exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
 
     sys.exit(0 if not exit_code else 1)
 
 
 def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo):
     from azdev.utilities import diff_branches, extract_module_name
-    from azdev.utilities.git_util import _summarize_changed_mods
+    from azdev.utilities.git_util import summarize_changed_mods
 
     if not any([git_source, git_target, git_repo]):
         return tests
@@ -118,7 +126,7 @@ def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo):
         raise CLIError('usage error: [--src NAME]  --tgt NAME --repo PATH')
 
     files_changed = diff_branches(git_repo, git_target, git_source)
-    mods_changed = _summarize_changed_mods(files_changed)
+    mods_changed = summarize_changed_mods(files_changed)
 
     repo_path = str(os.path.abspath(git_repo)).lower()
     to_remove = []
@@ -139,30 +147,6 @@ def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo):
     return tests
 
 
-def _get_profile(profile):
-    import colorama
-    colorama.init(autoreset=True)
-    try:
-        fore_red = colorama.Fore.RED if not IS_WINDOWS else ''
-        fore_reset = colorama.Fore.RESET if not IS_WINDOWS else ''
-        original_profile = raw_cmd('az cloud show --query profile -otsv', show_stderr=False).result
-        if not profile or original_profile == profile:
-            profile = original_profile
-            display('The tests are set to run against current profile {}.'
-                    .format(fore_red + original_profile + fore_reset))
-        elif original_profile != profile:
-            display('The tests are set to run against profile {} but the current az cloud profile is {}.'
-                    .format(fore_red + profile + fore_reset, fore_red + original_profile + fore_reset))
-            result = raw_cmd('az cloud update --profile {}'.format(profile),
-                             'SWITCHING TO PROFILE {}.'.format(fore_red + profile + fore_reset))
-            if result.exit_code != 0:
-                raise CLIError(result.error.output)
-        # returns the original profile so we can switch back if need be
-        return original_profile
-    except CalledProcessError:
-        raise CLIError('Failed to retrieve current az profile')
-
-
 def _discover_module_tests(mod_name, mod_data):
 
     # get the list of test files in each module
diff --git a/azdev/operations/tests/incremental_strategy.py b/azdev/operations/tests/incremental_strategy.py
new file mode 100644
index 000000000..f0a54433f
--- /dev/null
+++ b/azdev/operations/tests/incremental_strategy.py
@@ -0,0 +1,87 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# -----------------------------------------------------------------------------
+
+import abc
+import wrapt
+from knack.util import CLIError
+
+from azdev.utilities import get_path_table, git_util
+
+
+@wrapt.decorator
+def cli_release_scenario(wrapped, _, args, kwargs):
+    """
+    Filter out those files in Azure CLI release stage
+    """
+    # TODO
+    # if instance.resolved:
+    #     return instance
+
+    return wrapped(*args, **kwargs)
+
+
+@wrapt.decorator
+def cli_pull_request_scenario(wrapped, instance, args, kwargs):
+    """
+    Strategy on Azure CLI pull request verification stage.
+
+    :return: a list of modified packages
+    """
+
+    wrapped(*args, **kwargs)
+
+    modified_packages = git_util.summarize_changed_mods(instance.modified_files)
+
+    if any(core_package in modified_packages for core_package in ['core', 'testsdk', 'telemetry']):
+        path_table = get_path_table()
+
+        # tests under all packages
+        return list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys())
+
+    return modified_packages
+
+
+class AzureDevOpsContext(abc.ABC):
+    def __init__(self, git_repo, git_source, git_target):
+        """
+        :param git_source: could be commit id, branch name or any valid value for git diff
+        :param git_target: could be commit id, branch name or any valid value for git diff
+        """
+        self.git_repo = git_repo
+        self.git_source = git_source
+        self.git_target = git_target
+
+    @abc.abstractmethod
+    def filter(self, test_index):
+        pass
+
+
+class CLIAzureDevOpsContext(AzureDevOpsContext):
+    """
+    Assemble strategy of incremental test on Azure DevOps Environment for Azure CLI
+    """
+    def __init__(self, git_repo, git_source, git_target):
+        super().__init__(git_repo, git_source, git_target)
+
+        if not any([self.git_source, self.git_target, self.git_repo]):
+            raise CLIError('usage error: [--src NAME]  --tgt NAME --repo PATH --cli-ci')
+
+        if not all([self.git_target, self.git_repo]):
+            raise CLIError('usage error: [--src NAME]  --tgt NAME --repo PATH --cli-ci')
+
+        self.modified_files = self._modified_files()
+
+    def _modified_files(self):
+        modified_files = git_util.diff_branches(self.git_repo, self.git_source, self.git_target)
+        return [f for f in modified_files if f.startswith('src/')]
+
+    @cli_pull_request_scenario
+    @cli_release_scenario
+    def filter(self, test_index):
+        """
+        :retuen: a list of modified packages
+        """
+        return []
diff --git a/azdev/params.py b/azdev/params.py
index 7994cc2f0..afc3c4cc6 100644
--- a/azdev/params.py
+++ b/azdev/params.py
@@ -51,6 +51,16 @@ def load_arguments(self, _):
         c.argument('pytest_args', nargs=argparse.REMAINDER, options_list=['--pytest-args', '-a'], help='Denotes the remaining args will be passed to pytest.')
         c.argument('last_failed', options_list='--lf', action='store_true', help='Re-run the last tests that failed.')
 
+        # CI parameters
+        c.argument('cli_ci',
+                   action='store_true',
+                   arg_group='Continuous Integration',
+                   help='Apply incremental test strategy to Azure CLI on Azure DevOps')
+        c.argument('ext_ci',
+                   action='store_true',
+                   arg_group='Continuous Integration',
+                   help='Apply incremental test strategy to Azure CLI Extension CI on Azure DevOps')
+
     with ArgumentsContext(self, 'coverage') as c:
         c.argument('prefix', type=str, help='Filter analysis by command prefix.')
         c.argument('report', action='store_true', help='Display results as a report.')
diff --git a/azdev/utilities/git_util.py b/azdev/utilities/git_util.py
index 8be06910d..817135a50 100644
--- a/azdev/utilities/git_util.py
+++ b/azdev/utilities/git_util.py
@@ -20,7 +20,7 @@ def filter_by_git_diff(selected_modules, git_source, git_target, git_repo):
         raise CLIError('usage error: [--src NAME]  --tgt NAME --repo PATH')
 
     files_changed = diff_branches(git_repo, git_target, git_source)
-    mods_changed = _summarize_changed_mods(files_changed)
+    mods_changed = summarize_changed_mods(files_changed)
 
     repo_path = str(os.path.abspath(git_repo)).lower()
     to_remove = {'mod': [], 'core': [], 'ext': []}
@@ -43,7 +43,7 @@ def filter_by_git_diff(selected_modules, git_source, git_target, git_repo):
     return selected_modules
 
 
-def _summarize_changed_mods(files_changed):
+def summarize_changed_mods(files_changed):
     from azdev.utilities import extract_module_name
 
     mod_set = set()
diff --git a/setup.py b/setup.py
index 4e35ddefb..78cba1a27 100644
--- a/setup.py
+++ b/setup.py
@@ -73,6 +73,7 @@
         'sphinx==1.6.7',
         'tox',
         'wheel==0.30.0',
+        'wrapt',
         'azure-storage-blob>=1.3.1,<2.0.0',
     ],
     extras_require={

From 98f0a4243f41b0765d95c629965ece97680eb53d Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 9 Apr 2020 09:15:07 +0000
Subject: [PATCH 14/27] fix style error

---
 azdev/operations/tests/__init__.py | 2 --
 setup.py                           | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index 4d99b44ad..3221b457e 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -44,8 +44,6 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
 
     path_table = get_path_table()
 
-    from pprint import pprint
-
     test_index = _get_test_index(profile or current_profile(), discover)
 
     if not tests:
diff --git a/setup.py b/setup.py
index 78cba1a27..537e2ebd7 100644
--- a/setup.py
+++ b/setup.py
@@ -61,6 +61,7 @@
     ],
     install_requires=[
         'docutils',
+        'wrapt',
         'flake8',
         'gitpython',
         'jinja2',
@@ -73,7 +74,6 @@
         'sphinx==1.6.7',
         'tox',
         'wheel==0.30.0',
-        'wrapt',
         'azure-storage-blob>=1.3.1,<2.0.0',
     ],
     extras_require={

From 9c6b277826e1a08fe1d5590371f3222de5ec3f3b Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 9 Apr 2020 09:20:18 +0000
Subject: [PATCH 15/27] fix import error

---
 azdev/operations/tests/__init__.py            | 32 +++++++++----------
 ...ofile_management.py => profile_context.py} |  0
 2 files changed, 16 insertions(+), 16 deletions(-)
 rename azdev/operations/tests/{profile_management.py => profile_context.py} (100%)

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index 3221b457e..765df2ed6 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -23,7 +23,7 @@
     make_dirs, get_azdev_config_dir,
     get_path_table, require_virtual_env, get_name_index)
 from .pytest_runner import get_test_runner
-from .profile_management import ProfileContext, current_profile
+from .profile_context import ProfileContext, current_profile
 from .incremental_strategy import CLIAzureDevOpsContext
 
 logger = get_logger(__name__)
@@ -91,24 +91,24 @@ def _find_test(index, name):
                 continue
         raise key_error
 
-    # # lookup test paths from index
-    # test_paths = []
-    # for t in modified_mods:
-    #     try:
-    #         test_path = os.path.normpath(_find_test(test_index, t))
-    #         test_paths.append(test_path)
-    #     except KeyError:
-    #         logger.warning("'%s' not found. If newly added, re-run with --discover", t)
-    #         continue
+    # lookup test paths from index
+    test_paths = []
+    for t in modified_mods:
+        try:
+            test_path = os.path.normpath(_find_test(test_index, t))
+            test_paths.append(test_path)
+        except KeyError:
+            logger.warning("'%s' not found. If newly added, re-run with --discover", t)
+            continue
 
-    # # Tests have been collected. Now run them.
-    # if not test_paths:
-    #     raise CLIError('No tests selected to run.')
+    # Tests have been collected. Now run them.
+    if not test_paths:
+        raise CLIError('No tests selected to run.')
 
     exit_code = 0
-    # with ProfileContext(profile):
-    #     runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
-    #     exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
+    with ProfileContext(profile):
+        runner = get_test_runner(parallel=not in_series, log_path=xml_path, last_failed=last_failed)
+        exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
 
     sys.exit(0 if not exit_code else 1)
 
diff --git a/azdev/operations/tests/profile_management.py b/azdev/operations/tests/profile_context.py
similarity index 100%
rename from azdev/operations/tests/profile_management.py
rename to azdev/operations/tests/profile_context.py

From 02ba680af564f3becb4bd3ca7a7f878129df8526 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 9 Apr 2020 09:49:01 +0000
Subject: [PATCH 16/27] pin wrapt==1.11.* to align with pylint 2.3.0

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 537e2ebd7..f4439fb87 100644
--- a/setup.py
+++ b/setup.py
@@ -61,7 +61,7 @@
     ],
     install_requires=[
         'docutils',
-        'wrapt',
+        'wrapt==1.11.*',    # align with pylint==2.3.0
         'flake8',
         'gitpython',
         'jinja2',

From 8031b15184f69baff982139b12c023e57083f3ba Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 9 Apr 2020 10:03:47 +0000
Subject: [PATCH 17/27] test past 100 commits

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index cdd1a66e3..bcfbac2ef 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -306,5 +306,5 @@ jobs:
         azdev --version
         az --version
 
-        azdev test --repo ../azure-cli --src=dev --tgt=HEAD~2 --series
+        azdev test --repo ../azure-cli --src=dev --tgt=HEAD~100 --series
       displayName: Run Automation Test on CLI

From f008e2de717a7bd423998ea942b8d7a621b7046b Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Wed, 22 Apr 2020 16:18:39 +0800
Subject: [PATCH 18/27] trival test

---
 azdev/operations/tests/__init__.py        | 6 +++---
 azdev/operations/tests/profile_context.py | 2 +-
 azdev/operations/tests/pytest_runner.py   | 3 ++-
 setup.py                                  | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index 765df2ed6..7b0fdaf98 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -33,7 +33,7 @@
 def run_tests(tests, xml_path=None, discover=False, in_series=False,
               run_live=False, profile=None, last_failed=False, pytest_args=None,
               git_source=None, git_target=None, git_repo=None,
-              cli_ci=False, ext_ci=False):
+              cli_ci=False):
 
     require_virtual_env()
 
@@ -61,8 +61,8 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
     if cli_ci is True:
         ctx = CLIAzureDevOpsContext(git_repo, git_source, git_target)
         modified_mods = ctx.filter(test_index)
-    if ext_ci is True:
-        pass
+    # if ext_ci is True:
+    #     pass
 
     # resolve the path at which to dump the XML results
     xml_path = xml_path or DEFAULT_RESULT_PATH
diff --git a/azdev/operations/tests/profile_context.py b/azdev/operations/tests/profile_context.py
index bd533705f..dec958b7d 100644
--- a/azdev/operations/tests/profile_context.py
+++ b/azdev/operations/tests/profile_context.py
@@ -20,7 +20,7 @@ class ProfileContext:
     def __init__(self, profile_name=None):
         self.target_profile = profile_name
 
-        self.origin_profile = cmd('az cloud show --query profile -otsv', show_stderr=False).result
+        self.origin_profile = current_profile()
 
     def __enter__(self):
         if self.target_profile is None or self.target_profile == self.origin_profile:
diff --git a/azdev/operations/tests/pytest_runner.py b/azdev/operations/tests/pytest_runner.py
index 7c69fd5d7..147662875 100644
--- a/azdev/operations/tests/pytest_runner.py
+++ b/azdev/operations/tests/pytest_runner.py
@@ -15,7 +15,8 @@ def _run(test_paths, pytest_args):
 
         logger = get_logger(__name__)
 
-        arguments = ['-x', '-v', '--junit-xml', log_path]
+        # arguments = ['-x', '-v', '--junit-xml', log_path]
+        arguments = ['-x', '-v', '--boxed', '-p no:warnings', '--log-level=WARN']
         arguments.extend(test_paths)
         if parallel:
             arguments += ['-n', 'auto']
diff --git a/setup.py b/setup.py
index f4439fb87..29145919b 100644
--- a/setup.py
+++ b/setup.py
@@ -68,7 +68,7 @@
         'knack',
         'mock',
         'pytest>=4.4.0',
-        'pytest-xdist',
+        'pytest-xdist',     # include pytest-forked
         'pyyaml',
         'requests',
         'sphinx==1.6.7',

From 31fc669e99f6c47b5ca1968a286a59492201ed55 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Wed, 22 Apr 2020 16:27:19 +0800
Subject: [PATCH 19/27] timeout limit increase to 120

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index bcfbac2ef..fe4e8af7a 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -271,7 +271,7 @@ jobs:
 - job: CLIAutomationTest
   dependsOn: BuildPythonWheel
   condition: succeeded()
-  timeoutInMinutes: 20
+  timeoutInMinutes: 120
 
   pool:
     vmImage: 'ubuntu-16.04'

From ea24493cf6b111e9b192b0bfb0ca57e8ccbfca96 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 23 Apr 2020 14:12:46 +0800
Subject: [PATCH 20/27] test against release

---
 azure-pipelines.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index fe4e8af7a..42c67cc7f 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -291,6 +291,11 @@ jobs:
     - bash: |
         # clone azure-cli
         git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli
+
+        cd ../azure-cli
+
+        git fetch origin release
+        git checkout release
     - bash: |
         set -ev
 
@@ -306,5 +311,5 @@ jobs:
         azdev --version
         az --version
 
-        azdev test --repo ../azure-cli --src=dev --tgt=HEAD~100 --series
+        azdev test --repo ../azure-cli --src=dev --tgt=origin/release --series
       displayName: Run Automation Test on CLI

From cb36c1beadbd2c38cc36e9c3f723cd2d17f401e5 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 23 Apr 2020 14:17:10 +0800
Subject: [PATCH 21/27] fix

---
 azure-pipelines.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 42c67cc7f..9dfbb739c 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -289,13 +289,15 @@ jobs:
       inputs:
         versionSpec: '$(python.version)'
     - bash: |
+        set -ev
+
         # clone azure-cli
         git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli
 
         cd ../azure-cli
 
         git fetch origin release
-        git checkout release
+        git checkout -b release origin/release
     - bash: |
         set -ev
 

From 8e4ccd723c5b335b7f4507ba2302046659524279 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 23 Apr 2020 14:30:55 +0800
Subject: [PATCH 22/27] fix

---
 azure-pipelines.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 9dfbb739c..2da451c63 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -297,6 +297,8 @@ jobs:
         cd ../azure-cli
 
         git fetch origin release
+        git branch -v
+        git status
         git checkout -b release origin/release
     - bash: |
         set -ev

From dc44b41c71a0bf3b1a7305d0ba8c3600edc89bc0 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 23 Apr 2020 14:35:50 +0800
Subject: [PATCH 23/27] fix

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 2da451c63..4dae6c956 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -292,7 +292,7 @@ jobs:
         set -ev
 
         # clone azure-cli
-        git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli
+        git clone -q -b dev https://github.com/Azure/azure-cli.git ../azure-cli
 
         cd ../azure-cli
 

From 367e6b2f47a9fcbaf8fed5394a86bd78d0e1a7a0 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 23 Apr 2020 14:54:47 +0800
Subject: [PATCH 24/27] fix

---
 .../operations/tests/incremental_strategy.py  | 62 +++++++------------
 azure-pipelines.yml                           |  3 -
 setup.py                                      |  1 -
 3 files changed, 24 insertions(+), 42 deletions(-)

diff --git a/azdev/operations/tests/incremental_strategy.py b/azdev/operations/tests/incremental_strategy.py
index f0a54433f..f8c010e1b 100644
--- a/azdev/operations/tests/incremental_strategy.py
+++ b/azdev/operations/tests/incremental_strategy.py
@@ -5,43 +5,21 @@
 # -----------------------------------------------------------------------------
 
 import abc
-import wrapt
 from knack.util import CLIError
 
 from azdev.utilities import get_path_table, git_util
 
 
-@wrapt.decorator
-def cli_release_scenario(wrapped, _, args, kwargs):
-    """
-    Filter out those files in Azure CLI release stage
-    """
-    # TODO
-    # if instance.resolved:
-    #     return instance
-
-    return wrapped(*args, **kwargs)
-
-
-@wrapt.decorator
-def cli_pull_request_scenario(wrapped, instance, args, kwargs):
-    """
-    Strategy on Azure CLI pull request verification stage.
-
-    :return: a list of modified packages
-    """
-
-    wrapped(*args, **kwargs)
+# @wrapt.decorator
+# def cli_release_scenario(wrapped, _, args, kwargs):
+#     """
+#     Filter out those files in Azure CLI release stage
+#     """
+#     # TODO
+#     # if instance.resolved:
+#     #     return instance
 
-    modified_packages = git_util.summarize_changed_mods(instance.modified_files)
-
-    if any(core_package in modified_packages for core_package in ['core', 'testsdk', 'telemetry']):
-        path_table = get_path_table()
-
-        # tests under all packages
-        return list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys())
-
-    return modified_packages
+#     return wrapped(*args, **kwargs)
 
 
 class AzureDevOpsContext(abc.ABC):
@@ -72,16 +50,24 @@ def __init__(self, git_repo, git_source, git_target):
         if not all([self.git_target, self.git_repo]):
             raise CLIError('usage error: [--src NAME]  --tgt NAME --repo PATH --cli-ci')
 
-        self.modified_files = self._modified_files()
-
-    def _modified_files(self):
+    @property
+    def modified_files(self):
         modified_files = git_util.diff_branches(self.git_repo, self.git_source, self.git_target)
         return [f for f in modified_files if f.startswith('src/')]
 
-    @cli_pull_request_scenario
-    @cli_release_scenario
     def filter(self, test_index):
         """
-        :retuen: a list of modified packages
+        Strategy on Azure CLI pull request verification stage.
+
+        :return: a list of modified packages
         """
-        return []
+
+        modified_packages = git_util.summarize_changed_mods(self.modified_files)
+
+        if any(core_package in modified_packages for core_package in ['core', 'testsdk', 'telemetry']):
+            path_table = get_path_table()
+
+            # tests under all packages
+            return list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys())
+
+        return modified_packages
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 4dae6c956..b95f3b695 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -297,9 +297,6 @@ jobs:
         cd ../azure-cli
 
         git fetch origin release
-        git branch -v
-        git status
-        git checkout -b release origin/release
     - bash: |
         set -ev
 
diff --git a/setup.py b/setup.py
index 29145919b..ebb6b754f 100644
--- a/setup.py
+++ b/setup.py
@@ -61,7 +61,6 @@
     ],
     install_requires=[
         'docutils',
-        'wrapt==1.11.*',    # align with pylint==2.3.0
         'flake8',
         'gitpython',
         'jinja2',

From 1a1731d004f5962e55de7ce1fd28d38eb2109f77 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 23 Apr 2020 14:57:52 +0800
Subject: [PATCH 25/27] fix

---
 azure-pipelines.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index b95f3b695..24ca31e03 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -305,7 +305,7 @@ jobs:
         . env/bin/activate
 
         # install azdev
-        pip install .
+        pip install -q .
 
         # setup
         azdev setup -c ../azure-cli

From 158148509c58db1e730407e221c6dd376a6cb28b Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 23 Apr 2020 15:08:23 +0800
Subject: [PATCH 26/27] revert CI parameters

---
 azdev/operations/tests/__init__.py |  3 +--
 azdev/params.py                    | 10 ----------
 2 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index 7b0fdaf98..83e53cebd 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -32,8 +32,7 @@
 # pylint: disable=too-many-statements
 def run_tests(tests, xml_path=None, discover=False, in_series=False,
               run_live=False, profile=None, last_failed=False, pytest_args=None,
-              git_source=None, git_target=None, git_repo=None,
-              cli_ci=False):
+              git_source=None, git_target=None, git_repo=None):
 
     require_virtual_env()
 
diff --git a/azdev/params.py b/azdev/params.py
index afc3c4cc6..7994cc2f0 100644
--- a/azdev/params.py
+++ b/azdev/params.py
@@ -51,16 +51,6 @@ def load_arguments(self, _):
         c.argument('pytest_args', nargs=argparse.REMAINDER, options_list=['--pytest-args', '-a'], help='Denotes the remaining args will be passed to pytest.')
         c.argument('last_failed', options_list='--lf', action='store_true', help='Re-run the last tests that failed.')
 
-        # CI parameters
-        c.argument('cli_ci',
-                   action='store_true',
-                   arg_group='Continuous Integration',
-                   help='Apply incremental test strategy to Azure CLI on Azure DevOps')
-        c.argument('ext_ci',
-                   action='store_true',
-                   arg_group='Continuous Integration',
-                   help='Apply incremental test strategy to Azure CLI Extension CI on Azure DevOps')
-
     with ArgumentsContext(self, 'coverage') as c:
         c.argument('prefix', type=str, help='Filter analysis by command prefix.')
         c.argument('report', action='store_true', help='Display results as a report.')

From affca4afe71ab31679b14595b2bd7d56bc725347 Mon Sep 17 00:00:00 2001
From: Harold Zeng <haroldrandom@gmail.com>
Date: Thu, 23 Apr 2020 15:21:52 +0800
Subject: [PATCH 27/27] add OS platform awareness to pytest runner

---
 azdev/operations/tests/__init__.py      | 6 ------
 azdev/operations/tests/pytest_runner.py | 9 +++++++--
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/azdev/operations/tests/__init__.py b/azdev/operations/tests/__init__.py
index 83e53cebd..6ba146d50 100644
--- a/azdev/operations/tests/__init__.py
+++ b/azdev/operations/tests/__init__.py
@@ -57,12 +57,6 @@ def run_tests(tests, xml_path=None, discover=False, in_series=False,
     if modified_mods:
         display('\nTest on modules: {}\n'.format(', '.join(modified_mods)))
 
-    if cli_ci is True:
-        ctx = CLIAzureDevOpsContext(git_repo, git_source, git_target)
-        modified_mods = ctx.filter(test_index)
-    # if ext_ci is True:
-    #     pass
-
     # resolve the path at which to dump the XML results
     xml_path = xml_path or DEFAULT_RESULT_PATH
     if not xml_path.endswith('.xml'):
diff --git a/azdev/operations/tests/pytest_runner.py b/azdev/operations/tests/pytest_runner.py
index 147662875..e853a68fd 100644
--- a/azdev/operations/tests/pytest_runner.py
+++ b/azdev/operations/tests/pytest_runner.py
@@ -4,6 +4,8 @@
 # license information.
 # -----------------------------------------------------------------------------
 
+import os
+
 from knack.log import get_logger
 
 from azdev.utilities import call
@@ -15,8 +17,11 @@ def _run(test_paths, pytest_args):
 
         logger = get_logger(__name__)
 
-        # arguments = ['-x', '-v', '--junit-xml', log_path]
-        arguments = ['-x', '-v', '--boxed', '-p no:warnings', '--log-level=WARN']
+        if os.name == 'posix':
+            arguments = ['-x', '-v', '--boxed', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path]
+        else:
+            arguments = ['-x', '-v', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path]
+
         arguments.extend(test_paths)
         if parallel:
             arguments += ['-n', 'auto']