From ae7e1fad475ad99550d60f07718cbfe55b8b3aac Mon Sep 17 00:00:00 2001 From: Jianhui Harold Date: Wed, 18 Mar 2020 18:40:06 +0800 Subject: [PATCH 1/9] Init --- scripts/ci/test_automation.sh | 3 ++ tools/automation/tests/__init__.py | 74 ++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/scripts/ci/test_automation.sh b/scripts/ci/test_automation.sh index 080b3e118fc..8f0377017b2 100755 --- a/scripts/ci/test_automation.sh +++ b/scripts/ci/test_automation.sh @@ -53,5 +53,8 @@ fi az -v az -h +# for the test below +export ADO_PULL_REQUEST_TARGET_BRANCH=$(System.PullRequest.TargetBranch) + title 'Running tests' python -m automation test --ci --profile $target_profile diff --git a/tools/automation/tests/__init__.py b/tools/automation/tests/__init__.py index 44688dad7bb..5be1117991c 100644 --- a/tools/automation/tests/__init__.py +++ b/tools/automation/tests/__init__.py @@ -19,6 +19,66 @@ TEST_INDEX_FORMAT = 'testIndex_{}.json' +def _separator_line(): + print('-' * 100) + + +def _extract_module_name(path): + _CORE_NAME_REGEX = re.compile(r'azure-cli-(?P[^/\\]+)[/\\]azure[/\\]cli') + _MOD_NAME_REGEX = re.compile(r'azure-cli[/\\]azure[/\\]cli[/\\]command_modules[/\\](?P[^/\\]+)') + _EXT_NAME_REGEX = re.compile(r'.*(?Pazext_[^/\\]+).*') + + for expression in [_MOD_NAME_REGEX, _CORE_NAME_REGEX, _EXT_NAME_REGEX]: + match = re.search(expression, path) + if not match: + continue + return match.groupdict().get('name') + raise ValueError('unexpected error: unable to extract name from path: {}'.format(path)) + + +def _extract_modified_files(target_branch=None): + if target_branch is None: + ado_pr_target_branch = os.environ.get('ADO_PULL_REQUEST_TARGET_BRANCH') + qualified_target_branch = 'origin/{}'.format(ado_pr_target_branch) + else: + qualified_target_branch = target_branch + + cmd_tpl = 'git --no-pager diff --name-only --diff-filter=ACMRT {} -- src/' + cmd = cmd_tpl.format(qualified_target_branch) + + modified_files = check_output(cmd, shell=True).decode('utf-8').split('\n') + modified_files = [f for f in modified_files if len(f) > 0] + + _separator_line() + if modified_files: + print('modified files in src/ :', '\n'.join(modified_files)) + else: + print('no modified files in src/') + + return modified_files + + +def _extract_top_level_modified_modules(): + modified_modules = set() + + modified_files = _extract_modified_files() + + for file_path in modified_files: + try: + mod = _extract_module_name(file_path) + modified_modules.add(mod) + except ValueError: + continue + + _separator_line() + if modified_modules: + print('related top level modules:', list(modified_modules)) + else: + print('mo related top level modules.') + + return modified_modules + + def extract_module_name(path): mod_name_regex = re.compile(r'azure[/\\]cli[/\\]([^/\\]+)') ext_name_regex = re.compile(r'.*(azext_[^/\\]+).*') @@ -42,6 +102,20 @@ def execute(args): output('Running in CI Mode') selected_modules = [('All modules', 'azure.cli', 'azure.cli'), ('CLI Linter', 'automation.cli_linter', 'automation.cli_linter')] + + modified_modules = _extract_top_level_modified_modules() + if any(base_mod in modified_modules for base_mod in ['core', 'testsdk', 'telemetry']): + pass + else: + test_paths = [] + for mod in modified_modules: + try: + test_paths.append(os.path.normpath(test_index[mod])) + except KeyError: + display("no tests found in module: {}".format(mod)) + args.tests = test_paths + + selected_modules = filter_user_selected_modules_with_tests(modified_modules, args.profile) elif not (args.tests or args.src_file): # Default is to run with modules (possibly via environment variable) if os.environ.get('AZURE_CLI_TEST_MODULES', None): From 4a3c1a753c4cca8cc5e7fbb857bbd27cf5728788 Mon Sep 17 00:00:00 2001 From: Jianhui Harold Date: Wed, 18 Mar 2020 18:44:54 +0800 Subject: [PATCH 2/9] echo --- scripts/ci/test_automation.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ci/test_automation.sh b/scripts/ci/test_automation.sh index 8f0377017b2..613d9de1bfe 100755 --- a/scripts/ci/test_automation.sh +++ b/scripts/ci/test_automation.sh @@ -55,6 +55,7 @@ az -h # for the test below export ADO_PULL_REQUEST_TARGET_BRANCH=$(System.PullRequest.TargetBranch) +echo "${ADO_PULL_REQUEST_TARGET_BRANCH}" title 'Running tests' python -m automation test --ci --profile $target_profile From 6678dee77b7d037816681232ba3fac8e513b0138 Mon Sep 17 00:00:00 2001 From: Jianhui Harold Date: Thu, 19 Mar 2020 11:08:58 +0800 Subject: [PATCH 3/9] Add env in azure-pipeline.yml --- azure-pipelines.yml | 8 +++++++- scripts/ci/test_automation.sh | 4 ---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e9facefa8b4..edf1f7f8e38 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -424,6 +424,7 @@ jobs: filePath: ./scripts/ci/test_automation.sh env: REDUCE_SDK: 'True' + ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - job: RunAutomationReduced20190301 displayName: Run Automation Reduced Python 3.6, Profile 2019-03-01 @@ -451,6 +452,7 @@ jobs: env: REDUCE_SDK: 'True' AZURE_CLI_TEST_TARGET_PROFILE: '2019-03-01' + ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - job: RunAutomation20180301 displayName: Run Automation, Profile 2018-03-01 @@ -477,6 +479,7 @@ jobs: filePath: ./scripts/ci/test_automation.sh env: AZURE_CLI_TEST_TARGET_PROFILE: '2018-03-01' + ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - job: RunAutomationReducedPython3_8 displayName: Run Automation Reduced Python 3.8 @@ -504,6 +507,7 @@ jobs: filePath: ./scripts/ci/test_automation.sh env: REDUCE_SDK: 'True' + ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - job: RunAutomationReduced20190301Python3_8 displayName: Run Automation Reduced Python 3.8, Profile 2019-03-01 @@ -531,6 +535,7 @@ jobs: env: REDUCE_SDK: 'True' AZURE_CLI_TEST_TARGET_PROFILE: '2019-03-01' + ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - job: TestExtensionsLoading displayName: Test Extensions Loading @@ -578,7 +583,8 @@ jobs: targetType: 'filePath' filePath: ./scripts/ci/test_automation.sh env: - AZURE_CLI_TEST_TARGET_PROFILE: '2018-03-01' + AZURE_CLI_TEST_TARGET_PROFILE: '2018-03-01' + ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - job: BuildHomebrewFormula displayName: Build Homebrew Formula diff --git a/scripts/ci/test_automation.sh b/scripts/ci/test_automation.sh index 613d9de1bfe..080b3e118fc 100755 --- a/scripts/ci/test_automation.sh +++ b/scripts/ci/test_automation.sh @@ -53,9 +53,5 @@ fi az -v az -h -# for the test below -export ADO_PULL_REQUEST_TARGET_BRANCH=$(System.PullRequest.TargetBranch) -echo "${ADO_PULL_REQUEST_TARGET_BRANCH}" - title 'Running tests' python -m automation test --ci --profile $target_profile From 5f1c49f7d3bf263b079a0a6564e555e56e824c69 Mon Sep 17 00:00:00 2001 From: Jianhui Harold Date: Thu, 19 Mar 2020 11:37:48 +0800 Subject: [PATCH 4/9] exit 0 when no tests --- tools/automation/tests/__init__.py | 6 +++--- tools/automation/tests/main.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/automation/tests/__init__.py b/tools/automation/tests/__init__.py index 5be1117991c..cd4a03d2071 100644 --- a/tools/automation/tests/__init__.py +++ b/tools/automation/tests/__init__.py @@ -53,7 +53,7 @@ def _extract_modified_files(target_branch=None): if modified_files: print('modified files in src/ :', '\n'.join(modified_files)) else: - print('no modified files in src/') + print('no modified files in src/, no need to run automation test') return modified_files @@ -74,7 +74,7 @@ def _extract_top_level_modified_modules(): if modified_modules: print('related top level modules:', list(modified_modules)) else: - print('mo related top level modules.') + print('no related top level modules modified, no need to run automation test') return modified_modules @@ -115,7 +115,7 @@ def execute(args): display("no tests found in module: {}".format(mod)) args.tests = test_paths - selected_modules = filter_user_selected_modules_with_tests(modified_modules, args.profile) + selected_modules = [] elif not (args.tests or args.src_file): # Default is to run with modules (possibly via environment variable) if os.environ.get('AZURE_CLI_TEST_MODULES', None): diff --git a/tools/automation/tests/main.py b/tools/automation/tests/main.py index 35b6d583ab7..41e7bbcb9f2 100644 --- a/tools/automation/tests/main.py +++ b/tools/automation/tests/main.py @@ -16,7 +16,7 @@ def run_tests(modules, parallel, run_live, tests): if not modules and not tests: display('No tests set to run.') - sys.exit(1) + sys.exit(0) display(""" ============= From 267079aec505edf5954246a04c2c3f983a98b2cf Mon Sep 17 00:00:00 2001 From: Jianhui Harold Date: Thu, 19 Mar 2020 13:34:09 +0800 Subject: [PATCH 5/9] test to remove REDUCE=True --- azure-pipelines.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index edf1f7f8e38..3d2b4d56c88 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -398,7 +398,7 @@ jobs: - bash: ./scripts/ci/test_profile_integration.sh displayName: 'Run Integration Test against Profiles' -- job: RunAutomationReducedPython3_6 +- job: RunAutomationPython3_6 displayName: Run Automation Reduced Python 3.6 dependsOn: BuildPythonWheel timeoutInMinutes: 90 @@ -423,7 +423,6 @@ jobs: targetType: 'filePath' filePath: ./scripts/ci/test_automation.sh env: - REDUCE_SDK: 'True' ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - job: RunAutomationReduced20190301 @@ -481,7 +480,7 @@ jobs: AZURE_CLI_TEST_TARGET_PROFILE: '2018-03-01' ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) -- job: RunAutomationReducedPython3_8 +- job: RunAutomationPython3_8 displayName: Run Automation Reduced Python 3.8 dependsOn: BuildPythonWheel timeoutInMinutes: 90 @@ -506,7 +505,6 @@ jobs: targetType: 'filePath' filePath: ./scripts/ci/test_automation.sh env: - REDUCE_SDK: 'True' ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - job: RunAutomationReduced20190301Python3_8 From 72250a208ce335451db0f3090ee3145c8f35ea0c Mon Sep 17 00:00:00 2001 From: Jianhui Harold Date: Thu, 19 Mar 2020 14:38:10 +0800 Subject: [PATCH 6/9] remove Reduce on name --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3d2b4d56c88..e226f8b2a92 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -399,7 +399,7 @@ jobs: displayName: 'Run Integration Test against Profiles' - job: RunAutomationPython3_6 - displayName: Run Automation Reduced Python 3.6 + displayName: Run Automation Python 3.6 dependsOn: BuildPythonWheel timeoutInMinutes: 90 @@ -481,7 +481,7 @@ jobs: ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch) - job: RunAutomationPython3_8 - displayName: Run Automation Reduced Python 3.8 + displayName: Run Automation Python 3.8 dependsOn: BuildPythonWheel timeoutInMinutes: 90 From 82f2e9ad2f46a752427b8c9de5a263605e2f07b7 Mon Sep 17 00:00:00 2001 From: Jianhui Harold Date: Thu, 19 Mar 2020 17:38:27 +0800 Subject: [PATCH 7/9] deal with CI stage --- tools/automation/tests/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/automation/tests/__init__.py b/tools/automation/tests/__init__.py index cd4a03d2071..835a5de197f 100644 --- a/tools/automation/tests/__init__.py +++ b/tools/automation/tests/__init__.py @@ -36,12 +36,14 @@ def _extract_module_name(path): raise ValueError('unexpected error: unable to extract name from path: {}'.format(path)) -def _extract_modified_files(target_branch=None): - if target_branch is None: - ado_pr_target_branch = os.environ.get('ADO_PULL_REQUEST_TARGET_BRANCH') - qualified_target_branch = 'origin/{}'.format(ado_pr_target_branch) +def _extract_modified_files(target_branch=os.environ.get('ADO_PULL_REQUEST_TARGET_BRANCH')): + ado_raw_env_replacement = '$(System.PullRequest.TargetBranch)' + + if target_branch == ado_raw_env_replacement: + # in ADO env but not in PR stage + return ['core'] else: - qualified_target_branch = target_branch + qualified_target_branch = 'origin/{}'.format(target_branch) cmd_tpl = 'git --no-pager diff --name-only --diff-filter=ACMRT {} -- src/' cmd = cmd_tpl.format(qualified_target_branch) @@ -105,7 +107,7 @@ def execute(args): modified_modules = _extract_top_level_modified_modules() if any(base_mod in modified_modules for base_mod in ['core', 'testsdk', 'telemetry']): - pass + pass # if modified modules contains those 3 modules, run all tests else: test_paths = [] for mod in modified_modules: From c1d633fe96ca119a27a224307db579b04fa39faf Mon Sep 17 00:00:00 2001 From: Jianhui Harold Date: Thu, 19 Mar 2020 17:54:23 +0800 Subject: [PATCH 8/9] fix bug --- tools/automation/tests/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/automation/tests/__init__.py b/tools/automation/tests/__init__.py index 835a5de197f..54d418cebf9 100644 --- a/tools/automation/tests/__init__.py +++ b/tools/automation/tests/__init__.py @@ -41,7 +41,9 @@ def _extract_modified_files(target_branch=os.environ.get('ADO_PULL_REQUEST_TARGE if target_branch == ado_raw_env_replacement: # in ADO env but not in PR stage - return ['core'] + + # dummy file name src/azure-cli-core/azure/cli/core/__init__.py + return [os.path.join('src', 'azure-cli-core', 'azure', 'cli', 'core', '__init__.py')] else: qualified_target_branch = 'origin/{}'.format(target_branch) From a590a3059cf10d4746f2540a4c5da1b87cc5cfda Mon Sep 17 00:00:00 2001 From: Jianhui Harold Date: Thu, 19 Mar 2020 17:58:56 +0800 Subject: [PATCH 9/9] dedup test_paths --- tools/automation/tests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/automation/tests/__init__.py b/tools/automation/tests/__init__.py index 54d418cebf9..09016bfcc0a 100644 --- a/tools/automation/tests/__init__.py +++ b/tools/automation/tests/__init__.py @@ -111,10 +111,10 @@ def execute(args): if any(base_mod in modified_modules for base_mod in ['core', 'testsdk', 'telemetry']): pass # if modified modules contains those 3 modules, run all tests else: - test_paths = [] + test_paths = set() for mod in modified_modules: try: - test_paths.append(os.path.normpath(test_index[mod])) + test_paths.add(os.path.normpath(test_index[mod])) except KeyError: display("no tests found in module: {}".format(mod)) args.tests = test_paths