From 7f415adfb572e46c99eddc379ae733e801ce8b64 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Thu, 22 Apr 2021 11:34:04 +0800 Subject: [PATCH 01/42] add live test pipeline for aks --- .../azcli-aks-live-test/.gitignore | 3 + .../azcli-aks-live-test/Dockerfile | 8 ++ .../azcli-aks-live-test/build_image.sh | 7 ++ .../azcli-aks-live-test/clone_repo.sh | 34 +++++++++ .../azcli-aks-live-test/setup_venv.sh | 28 +++++++ .../azcli-aks-live-test/start_container.sh | 10 +++ .../azcli-aks-live-test/test_cli.sh | 28 +++++++ .../azcli-aks-live-test/test_ext.sh | 50 +++++++++++++ .../azcli-aks-live-test/transcribe_env.sh | 18 +++++ .../vsts-azcli-aks-live-test.yaml | 74 +++++++++++++++++++ 10 files changed, 260 insertions(+) create mode 100644 src/aks-preview/azcli-aks-live-test/.gitignore create mode 100644 src/aks-preview/azcli-aks-live-test/Dockerfile create mode 100755 src/aks-preview/azcli-aks-live-test/build_image.sh create mode 100755 src/aks-preview/azcli-aks-live-test/clone_repo.sh create mode 100755 src/aks-preview/azcli-aks-live-test/setup_venv.sh create mode 100755 src/aks-preview/azcli-aks-live-test/start_container.sh create mode 100755 src/aks-preview/azcli-aks-live-test/test_cli.sh create mode 100755 src/aks-preview/azcli-aks-live-test/test_ext.sh create mode 100755 src/aks-preview/azcli-aks-live-test/transcribe_env.sh create mode 100644 src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml diff --git a/src/aks-preview/azcli-aks-live-test/.gitignore b/src/aks-preview/azcli-aks-live-test/.gitignore new file mode 100644 index 00000000000..a53b92a3593 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/.gitignore @@ -0,0 +1,3 @@ +env.list +*.json +*.xml diff --git a/src/aks-preview/azcli-aks-live-test/Dockerfile b/src/aks-preview/azcli-aks-live-test/Dockerfile new file mode 100644 index 00000000000..07af67fd197 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:bionic-20210325 +RUN apt update && \ + apt install -y software-properties-common && \ + add-apt-repository -y ppa:deadsnakes/ppa && \ + apt update && \ + apt install -y python3.8 python3.8-venv python3.8-dev python3-pip gcc +WORKDIR /opt/ +CMD ["/bin/bash"] diff --git a/src/aks-preview/azcli-aks-live-test/build_image.sh b/src/aks-preview/azcli-aks-live-test/build_image.sh new file mode 100755 index 00000000000..7da9284dc37 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/build_image.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -eux +pwd + +# build docker image +docker build -t azcli-aks-live-test-image:latest -f ./Dockerfile . diff --git a/src/aks-preview/azcli-aks-live-test/clone_repo.sh b/src/aks-preview/azcli-aks-live-test/clone_repo.sh new file mode 100755 index 00000000000..70731b0027c --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/clone_repo.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -eux +pwd + +# clone azure-cli (default is the official repo) +# git clone https://github.com/Azure/azure-cli.git +git clone $CLI_REPO + +# ckeckout to a specific azure-cli branch (default is the dev branch) +cd azure-cli/ +git branch -a +git checkout $CLI_BRANCH +cd - + +# clone azure-cli-extensions when manually specify the extension repo +if [[ $MANUAL_EXT == true && -n $EXT_REPO && -n $EXT_BRANCH ]]; then + echo "Manually specify the extension repo, delete the current 'azure-cli-extensions' directory!" + rm -rf azure-cli-extensions/ + git clone $EXT_REPO + cd azure-cli-extensions/ + git checkout $EXT_BRANCH + cd - +fi + +# check current branch & commit logs in azure-cli-extensions +cd azure-cli-extensions/ +git branch -a +git log -10 +cd - + +# move live test related files to the same level as the checkout directory ($(Agent.BuildDirectory)/s) +mv azure-cli-extensions/src/aks-preview/azcli-aks-live-test/* ./ +ls -alh diff --git a/src/aks-preview/azcli-aks-live-test/setup_venv.sh b/src/aks-preview/azcli-aks-live-test/setup_venv.sh new file mode 100755 index 00000000000..aba4c4c9250 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/setup_venv.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eux +pwd +ls -alh + +# delete existing venv +rm -rf azEnv || true + +# install python packages +python$PYTHON_VERSION -m venv azEnv +source azEnv/bin/activate +python -m pip install -U pip +pip install azdev +pip install pytest-json-report pytest-rerunfailures --upgrade +# pip install pytest-html --upgrade + +# check existing az +which az || az version || az extension list || true + +# install latest az +azdev setup -c azure-cli -r azure-cli-extensions +deactivate +source azEnv/bin/activate + +# check installation result +which az +az version diff --git a/src/aks-preview/azcli-aks-live-test/start_container.sh b/src/aks-preview/azcli-aks-live-test/start_container.sh new file mode 100755 index 00000000000..bd083352dc1 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/start_container.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -eux +pwd + +# transcribe environment variables +./transcribe_env.sh + +# start container +docker run -t -d --env-file ./env.list -v $PWD:/opt --name "azcli-aks-live-test-container" azcli-aks-live-test-image:latest diff --git a/src/aks-preview/azcli-aks-live-test/test_cli.sh b/src/aks-preview/azcli-aks-live-test/test_cli.sh new file mode 100755 index 00000000000..b03f40127ca --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/test_cli.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eux +pwd + +# activate virtualenv +source azEnv/bin/activate + +# remove extension +echo "Remove existing aks-preview extension (if any)" +if az extension remove --name aks-preview || azdev extension remove aks-preview; then + deactivate + source azEnv/bin/activate +fi + +# test cli +if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then + echo "Test in record mode!" + azdev test acs --no-exitfirst --xml-path cli_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_report.json --reruns 3 --capture=sys" +fi + +if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then + echo "Test in live mode!" + az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID + az account set -s $AZCLI_ALT_SUBSCRIPTION_ID + az account show + azdev test acs --live --no-exitfirst --xml-path cli_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_live_report.json --reruns 3 --capture=sys" +fi diff --git a/src/aks-preview/azcli-aks-live-test/test_ext.sh b/src/aks-preview/azcli-aks-live-test/test_ext.sh new file mode 100755 index 00000000000..61aa350e975 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/test_ext.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -eux +pwd + +# activate virtualenv +source azEnv/bin/activate + +# remove extension +echo "Remove existing aks-preview extension (if any)" +if az extension remove --name aks-preview || azdev extension remove aks-preview; then + deactivate + source azEnv/bin/activate +fi + +# install latest extension +echo "Install the latest aks-preview extension and re-activate the virtualenv" +azdev extension add aks-preview +az extension list +azdev extension list | grep "aks-preview" -C 5 +deactivate +source azEnv/bin/activate + +# Ensure that the command index is updated by calling a specific command in aks-preview, so that all the commands defined in aks-preview are loaded correctly +# Otherwise, cold boot execution of azdev test may use the api version adopted by the acs command group in azure-cli (which may diverge from the api version used in current aks-preview) +retry_count=0 +while ! az aks command invoke --help --debug && [[ $retry_count < 3 ]] +do + retry_count=`expr $retry_count + 1` + echo $retry_count"th retry to install aks-preview..." + azdev extension add aks-preview --debug + az extension list --debug + azdev extension list --debug | grep "aks-preview" -C 5 + deactivate + source azEnv/bin/activate +done + +# test ext +if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then + echo "Test in record mode!" + azdev test aks-preview --no-exitfirst --xml-path ext_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_report.json --reruns 3 --capture=sys" +fi + +if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then + echo "Test in live mode!" + az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID + az account set -s $AZCLI_ALT_SUBSCRIPTION_ID + az account show + azdev test aks-preview --live --no-exitfirst --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" +fi diff --git a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh new file mode 100755 index 00000000000..18ff6bc60fa --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +cat /dev/null > env.list +echo "TENANT_ID=$TENANT_ID" >> env.list +echo "AZCLI_ALT_SUBSCRIPTION_ID=$AZCLI_ALT_SUBSCRIPTION_ID" >> env.list +echo "AZCLI_ALT_CLIENT_ID=$AZCLI_ALT_CLIENT_ID" >> env.list +echo "AZCLI_ALT_CLIENT_SECRET=$MAPPED_AZCLI_ALT_CLIENT_SECRET" >> env.list +echo "TEST_SECRET=$MAPPED_TEST_SECRET" >> env.list + +echo "PYTHON_VERSION=$PYTHON_VERSION" >> env.list +echo "COVERAGE=$COVERAGE" >> env.list +echo "TEST_MODE=$TEST_MODE" >> env.list +echo "PARALLELISM=$PARALLELISM" >> env.list +echo "CLI_REPO=$CLI_REPO" >> env.list +echo "CLI_BRANCH=$CLI_BRANCH" >> env.list +echo "MANUAL_EXT=$MANUAL_EXT" >> env.list +echo "EXT_REPO=$EXT_REPO" >> env.list +echo "EXT_BRANCH=$EXT_BRANCH" >> env.list diff --git a/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml new file mode 100644 index 00000000000..2c7fc4c4b69 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml @@ -0,0 +1,74 @@ +name: $(Date:yyyyMMdd)$(Rev:.r)_Python$(PYTHON_VERSION)_Coverage-$(COVERAGE)_Mode-$(TEST_MODE)_Branch-$(Build.SourceBranchName) + +trigger: none + +jobs: +- job: LiveTest + pool: + vmImage: 'ubuntu-16.04' + timeoutInMinutes: 360 + displayName: "Live Test with Python" + steps: + - bash: | + pwd + ls -alh + mkdir azure-cli-extensions + shopt -s extglob dotglob + mv !(azure-cli-extensions) azure-cli-extensions + shopt -u extglob dotglob + ls -alh + displayName: "Move All Checkout Files to the Newly Created 'azure-cli-extensions' Directory" + - bash: | + ./azure-cli-extensions/src/aks-preview/azcli-aks-live-test/clone_repo.sh + condition: succeeded() + displayName: "Clone GitHub Repo and Move Live Test Related Files" + - bash: | + ./build_image.sh + condition: succeeded() + displayName: "Build Live Test Image" + - bash: | + ./start_container.sh + env: + MAPPED_AZCLI_ALT_CLIENT_SECRET: $(AZCLI_ALT_CLIENT_SECRET) + condition: succeeded() + displayName: "Start Container" + - bash: | + docker exec "azcli-aks-live-test-container" /opt/setup_venv.sh + condition: succeeded() + displayName: "Set up Virtual Environment" + - bash: | + docker exec "azcli-aks-live-test-container" /opt/test_cli.sh + condition: and(succeeded(), in(variables['COVERAGE'], 'cli', 'all')) + displayName: Perform Test for CLI + - bash: | + docker exec "azcli-aks-live-test-container" /opt/test_ext.sh + condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all')) + displayName: Perform Test for EXT + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: 'cli_report.json' + ArtifactName: 'cli_report.json' + publishLocation: 'Container' + condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'cli', 'all'), in(variables['TEST_MODE'], 'record', 'all')) + displayName: Publish CLI Record Test Report + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: 'cli_live_report.json' + ArtifactName: 'cli_live_report.json' + publishLocation: 'Container' + condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'cli', 'all'), in(variables['TEST_MODE'], 'live', 'all')) + displayName: Publish CLI Live Test Report + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: 'ext_report.json' + ArtifactName: 'ext_report.json' + publishLocation: 'Container' + condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all'), in(variables['TEST_MODE'], 'record', 'all')) + displayName: Publish EXT Record Test Report + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: 'ext_live_report.json' + ArtifactName: 'ext_live_report.json' + publishLocation: 'Container' + condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'cli', 'all'), in(variables['TEST_MODE'], 'live', 'all')) + displayName: Publish EXT Live Test Report From 51cd3ea638660c68a6e68a23af96dfd03e7ec4b5 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Fri, 23 Apr 2021 16:38:50 +0800 Subject: [PATCH 02/42] * add readme and history * update Dockerfile, add build option * change cd - to pushd/popd --- src/aks-preview/azcli-aks-live-test/Dockerfile | 3 +-- src/aks-preview/azcli-aks-live-test/HISTORY.rst | 9 +++++++++ src/aks-preview/azcli-aks-live-test/README.md | 11 +++++++++++ .../azcli-aks-live-test/build_image.sh | 7 ------- .../azcli-aks-live-test/clone_repo.sh | 12 ++++++------ .../azcli-aks-live-test/prepare_image.sh | 16 ++++++++++++++++ .../azcli-aks-live-test/start_container.sh | 6 ++++-- .../azcli-aks-live-test/transcribe_env.sh | 5 ++++- .../vsts-azcli-aks-live-test.yaml | 4 ++-- 9 files changed, 53 insertions(+), 20 deletions(-) create mode 100644 src/aks-preview/azcli-aks-live-test/HISTORY.rst create mode 100644 src/aks-preview/azcli-aks-live-test/README.md delete mode 100755 src/aks-preview/azcli-aks-live-test/build_image.sh create mode 100755 src/aks-preview/azcli-aks-live-test/prepare_image.sh diff --git a/src/aks-preview/azcli-aks-live-test/Dockerfile b/src/aks-preview/azcli-aks-live-test/Dockerfile index 07af67fd197..dcc76f411c9 100644 --- a/src/aks-preview/azcli-aks-live-test/Dockerfile +++ b/src/aks-preview/azcli-aks-live-test/Dockerfile @@ -1,8 +1,7 @@ -FROM ubuntu:bionic-20210325 +FROM ubuntu:18.04 RUN apt update && \ apt install -y software-properties-common && \ add-apt-repository -y ppa:deadsnakes/ppa && \ apt update && \ apt install -y python3.8 python3.8-venv python3.8-dev python3-pip gcc -WORKDIR /opt/ CMD ["/bin/bash"] diff --git a/src/aks-preview/azcli-aks-live-test/HISTORY.rst b/src/aks-preview/azcli-aks-live-test/HISTORY.rst new file mode 100644 index 00000000000..93372ba3c64 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/HISTORY.rst @@ -0,0 +1,9 @@ +.. :changelog: + +Release History +=============== + +0.1.0 (4/23/2021) +++++++ + +* Add live test pipeline for aks commands diff --git a/src/aks-preview/azcli-aks-live-test/README.md b/src/aks-preview/azcli-aks-live-test/README.md new file mode 100644 index 00000000000..649b2031e8e --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/README.md @@ -0,0 +1,11 @@ +# Azure CLI AKS Live Test Pipeline + +This is a pipeline to test newly added aks commands in aks-preview/acs. + +## How to use + +**By default**, this pipeline will be **triggered** when submitting a **PR** to the master branch of the official repo which involves modifying the file under src/aks-preview and test the acs command group in azure-cli and the aks-preview command group in azure-cli-extensions. If the variables of the pipeline are not modified, the test will be executed based on the latest commit of the dev branch in the offical repo of azure-cli. The test will be performed in record mode first, and then in live mode. After the test, you can get the test results from the pipeline artifact (for different modes (record/live) and different modules (cli/ext)). + +You can also trigger this pipeline **manually**. In this way, you **must** set the variable *MANUAL_EXT* to true before running the pipeline, and provide the your *EXT_REPO/CLI_REPO* url and *EXT_BRANCH/CLI_BRANCH* name at the same time. + +For more details, you may refer to this [wiki](https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/Azure-CLI-AKS-Live-Test-Pipeline). diff --git a/src/aks-preview/azcli-aks-live-test/build_image.sh b/src/aks-preview/azcli-aks-live-test/build_image.sh deleted file mode 100755 index 7da9284dc37..00000000000 --- a/src/aks-preview/azcli-aks-live-test/build_image.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -eux -pwd - -# build docker image -docker build -t azcli-aks-live-test-image:latest -f ./Dockerfile . diff --git a/src/aks-preview/azcli-aks-live-test/clone_repo.sh b/src/aks-preview/azcli-aks-live-test/clone_repo.sh index 70731b0027c..575fc4f5f54 100755 --- a/src/aks-preview/azcli-aks-live-test/clone_repo.sh +++ b/src/aks-preview/azcli-aks-live-test/clone_repo.sh @@ -8,26 +8,26 @@ pwd git clone $CLI_REPO # ckeckout to a specific azure-cli branch (default is the dev branch) -cd azure-cli/ +pushd azure-cli/ git branch -a git checkout $CLI_BRANCH -cd - +popd # clone azure-cli-extensions when manually specify the extension repo if [[ $MANUAL_EXT == true && -n $EXT_REPO && -n $EXT_BRANCH ]]; then echo "Manually specify the extension repo, delete the current 'azure-cli-extensions' directory!" rm -rf azure-cli-extensions/ git clone $EXT_REPO - cd azure-cli-extensions/ + pushd azure-cli-extensions/ git checkout $EXT_BRANCH - cd - + popd fi # check current branch & commit logs in azure-cli-extensions -cd azure-cli-extensions/ +pushd azure-cli-extensions/ git branch -a git log -10 -cd - +popd # move live test related files to the same level as the checkout directory ($(Agent.BuildDirectory)/s) mv azure-cli-extensions/src/aks-preview/azcli-aks-live-test/* ./ diff --git a/src/aks-preview/azcli-aks-live-test/prepare_image.sh b/src/aks-preview/azcli-aks-live-test/prepare_image.sh new file mode 100755 index 00000000000..6b706932b62 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/prepare_image.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -eux +pwd + +# prepare docker image +if [[ $BUILD_IMAGE == true ]]; then + echo "Building test image '$IMAGE_NAME:$IMAGE_TAG'..." + docker build -t $IMAGE_NAME:$IMAGE_TAG -f ./Dockerfile . +else + echo "Pulling test image from '$IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG'..." + if [[ ! docker pull $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG ]]; then + echo "Failed to pull image, start local build..." + docker build -t $IMAGE_NAME:$IMAGE_TAG -f ./Dockerfile . + fi +fi diff --git a/src/aks-preview/azcli-aks-live-test/start_container.sh b/src/aks-preview/azcli-aks-live-test/start_container.sh index bd083352dc1..f53a2810b6f 100755 --- a/src/aks-preview/azcli-aks-live-test/start_container.sh +++ b/src/aks-preview/azcli-aks-live-test/start_container.sh @@ -6,5 +6,7 @@ pwd # transcribe environment variables ./transcribe_env.sh -# start container -docker run -t -d --env-file ./env.list -v $PWD:/opt --name "azcli-aks-live-test-container" azcli-aks-live-test-image:latest +# start container in backgroud with env.list as environment varialbes +# mount current directory ($(Agent.BuildDirectory)/s) to /opt in container +# set working directory as /opt in container +docker run -t -d --env-file ./env.list -v $PWD:/opt -w /opt --name "azcli-aks-live-test-container" $IMAGE_NAME:$IMAGE_TAG diff --git a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh index 18ff6bc60fa..7d6abd4a8d7 100755 --- a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh +++ b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh @@ -5,7 +5,6 @@ echo "TENANT_ID=$TENANT_ID" >> env.list echo "AZCLI_ALT_SUBSCRIPTION_ID=$AZCLI_ALT_SUBSCRIPTION_ID" >> env.list echo "AZCLI_ALT_CLIENT_ID=$AZCLI_ALT_CLIENT_ID" >> env.list echo "AZCLI_ALT_CLIENT_SECRET=$MAPPED_AZCLI_ALT_CLIENT_SECRET" >> env.list -echo "TEST_SECRET=$MAPPED_TEST_SECRET" >> env.list echo "PYTHON_VERSION=$PYTHON_VERSION" >> env.list echo "COVERAGE=$COVERAGE" >> env.list @@ -16,3 +15,7 @@ echo "CLI_BRANCH=$CLI_BRANCH" >> env.list echo "MANUAL_EXT=$MANUAL_EXT" >> env.list echo "EXT_REPO=$EXT_REPO" >> env.list echo "EXT_BRANCH=$EXT_BRANCH" >> env.list +echo "BUILD_IMAGE=$BUILD_IMAGE" >> env.list +echo "IMAGE_PREFIX=$IMAGE_PREFIX" >> env.list +echo "IMAGE_NAME=$IMAGE_NAME" >> env.list +echo "IMAGE_TAG=$IMAGE_TAG" >> env.list diff --git a/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml index 2c7fc4c4b69..a699e568ff8 100644 --- a/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml +++ b/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml @@ -23,9 +23,9 @@ jobs: condition: succeeded() displayName: "Clone GitHub Repo and Move Live Test Related Files" - bash: | - ./build_image.sh + ./prepare_image.sh condition: succeeded() - displayName: "Build Live Test Image" + displayName: "Prepare Live Test Image" - bash: | ./start_container.sh env: From c90d0f4a64d480fec931e27b6b5b32321f284de6 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Fri, 23 Apr 2021 16:55:20 +0800 Subject: [PATCH 03/42] fix condition bug in prepare_image.sh --- src/aks-preview/azcli-aks-live-test/prepare_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-preview/azcli-aks-live-test/prepare_image.sh b/src/aks-preview/azcli-aks-live-test/prepare_image.sh index 6b706932b62..7fd47d571e0 100755 --- a/src/aks-preview/azcli-aks-live-test/prepare_image.sh +++ b/src/aks-preview/azcli-aks-live-test/prepare_image.sh @@ -9,7 +9,7 @@ if [[ $BUILD_IMAGE == true ]]; then docker build -t $IMAGE_NAME:$IMAGE_TAG -f ./Dockerfile . else echo "Pulling test image from '$IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG'..." - if [[ ! docker pull $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG ]]; then + if ! docker pull $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG; then echo "Failed to pull image, start local build..." docker build -t $IMAGE_NAME:$IMAGE_TAG -f ./Dockerfile . fi From 94e93e9e7cf1ba9cb5be673505f0ce7af5064d9e Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sun, 25 Apr 2021 11:07:01 +0800 Subject: [PATCH 04/42] add ci mode (only test diff) --- src/aks-preview/azcli-aks-live-test/test_cli.sh | 6 +++++- src/aks-preview/azcli-aks-live-test/test_ext.sh | 6 +++++- src/aks-preview/azcli-aks-live-test/transcribe_env.sh | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/aks-preview/azcli-aks-live-test/test_cli.sh b/src/aks-preview/azcli-aks-live-test/test_cli.sh index b03f40127ca..84a3749f86d 100755 --- a/src/aks-preview/azcli-aks-live-test/test_cli.sh +++ b/src/aks-preview/azcli-aks-live-test/test_cli.sh @@ -24,5 +24,9 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID az account set -s $AZCLI_ALT_SUBSCRIPTION_ID az account show - azdev test acs --live --no-exitfirst --xml-path cli_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_live_report.json --reruns 3 --capture=sys" + if [[ $TEST_DIFF == true && "$(Build.Reason)" == "PullRequest" ]]; then + azdev test aks-preview --live --no-exitfirst --repo=azure-cli/ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) --cli-ci --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" + else + azdev test acs --live --no-exitfirst --xml-path cli_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_live_report.json --reruns 3 --capture=sys" + fi fi diff --git a/src/aks-preview/azcli-aks-live-test/test_ext.sh b/src/aks-preview/azcli-aks-live-test/test_ext.sh index 61aa350e975..4519992be2c 100755 --- a/src/aks-preview/azcli-aks-live-test/test_ext.sh +++ b/src/aks-preview/azcli-aks-live-test/test_ext.sh @@ -46,5 +46,9 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID az account set -s $AZCLI_ALT_SUBSCRIPTION_ID az account show - azdev test aks-preview --live --no-exitfirst --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" + if [[ $TEST_DIFF == true && "$(Build.Reason)" == "PullRequest" ]]; then + azdev test aks-preview --live --no-exitfirst --repo=azure-cli-extensions/ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) --cli-ci --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" + else + azdev test aks-preview --live --no-exitfirst --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" + fi fi diff --git a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh index 7d6abd4a8d7..8479fd3a083 100755 --- a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh +++ b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh @@ -19,3 +19,4 @@ echo "BUILD_IMAGE=$BUILD_IMAGE" >> env.list echo "IMAGE_PREFIX=$IMAGE_PREFIX" >> env.list echo "IMAGE_NAME=$IMAGE_NAME" >> env.list echo "IMAGE_TAG=$IMAGE_TAG" >> env.list +echo "TEST_DIFF=$TEST_DIFF" >> env.list From 3390af48e9445470f6365fb6ec52a54d097a3d2c Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sun, 25 Apr 2021 11:20:08 +0800 Subject: [PATCH 05/42] update variable names in bash scripts --- src/aks-preview/azcli-aks-live-test/test_cli.sh | 4 ++-- src/aks-preview/azcli-aks-live-test/test_ext.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/aks-preview/azcli-aks-live-test/test_cli.sh b/src/aks-preview/azcli-aks-live-test/test_cli.sh index 84a3749f86d..21df4bdd70e 100755 --- a/src/aks-preview/azcli-aks-live-test/test_cli.sh +++ b/src/aks-preview/azcli-aks-live-test/test_cli.sh @@ -24,8 +24,8 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID az account set -s $AZCLI_ALT_SUBSCRIPTION_ID az account show - if [[ $TEST_DIFF == true && "$(Build.Reason)" == "PullRequest" ]]; then - azdev test aks-preview --live --no-exitfirst --repo=azure-cli/ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) --cli-ci --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" + if [[ $TEST_DIFF == true && $BUILD_REASON == "PullRequest" ]]; then + azdev test aks-preview --live --no-exitfirst --repo=azure-cli/ --src=HEAD --tgt=origin/$SYSTEM_PULLREQUEST_TARGETBRANCH --cli-ci --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" else azdev test acs --live --no-exitfirst --xml-path cli_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_live_report.json --reruns 3 --capture=sys" fi diff --git a/src/aks-preview/azcli-aks-live-test/test_ext.sh b/src/aks-preview/azcli-aks-live-test/test_ext.sh index 4519992be2c..3671e90eccf 100755 --- a/src/aks-preview/azcli-aks-live-test/test_ext.sh +++ b/src/aks-preview/azcli-aks-live-test/test_ext.sh @@ -46,8 +46,8 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID az account set -s $AZCLI_ALT_SUBSCRIPTION_ID az account show - if [[ $TEST_DIFF == true && "$(Build.Reason)" == "PullRequest" ]]; then - azdev test aks-preview --live --no-exitfirst --repo=azure-cli-extensions/ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) --cli-ci --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" + if [[ $TEST_DIFF == true && $BUILD_REASON == "PullRequest" ]]; then + azdev test aks-preview --live --no-exitfirst --repo=azure-cli-extensions/ --src=HEAD --tgt=origin/$SYSTEM_PULLREQUEST_TARGETBRANCH --cli-ci --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" else azdev test aks-preview --live --no-exitfirst --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" fi From 57f08be4b1e30a0f7889f03ea7e56f93789608c5 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sun, 25 Apr 2021 13:53:57 +0800 Subject: [PATCH 06/42] source bash scripts & add clean up script --- src/aks-preview/azcli-aks-live-test/clean_up.sh | 12 ++++++++++++ .../azcli-aks-live-test/start_container.sh | 2 +- .../azcli-aks-live-test/transcribe_env.sh | 8 ++++++++ .../vsts-azcli-aks-live-test.yaml | 6 +++--- 4 files changed, 24 insertions(+), 4 deletions(-) create mode 100755 src/aks-preview/azcli-aks-live-test/clean_up.sh diff --git a/src/aks-preview/azcli-aks-live-test/clean_up.sh b/src/aks-preview/azcli-aks-live-test/clean_up.sh new file mode 100755 index 00000000000..7175d775979 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/clean_up.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# login +az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID +az account set -s $AZCLI_ALT_SUBSCRIPTION_ID +az account show + +# list the details of resource groups whose names start with "clitest" +az group list --query "([].name)[?starts_with(@, 'clitest')]" -o tsv | xargs -i az group show -n {} + +# delete all resource groups whose names start with "clitest" +# az group list --query "([].name)[?starts_with(@, 'clitest')]" -o tsv | xargs -i az group delete --no-wait -y -n {} diff --git a/src/aks-preview/azcli-aks-live-test/start_container.sh b/src/aks-preview/azcli-aks-live-test/start_container.sh index f53a2810b6f..313ef82951f 100755 --- a/src/aks-preview/azcli-aks-live-test/start_container.sh +++ b/src/aks-preview/azcli-aks-live-test/start_container.sh @@ -4,7 +4,7 @@ set -eux pwd # transcribe environment variables -./transcribe_env.sh +source ./transcribe_env.sh # start container in backgroud with env.list as environment varialbes # mount current directory ($(Agent.BuildDirectory)/s) to /opt in container diff --git a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh index 8479fd3a083..09ff2ba16ac 100755 --- a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh +++ b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh @@ -1,11 +1,19 @@ #!/bin/bash +# clear cat /dev/null > env.list + +# tenant, sub, client echo "TENANT_ID=$TENANT_ID" >> env.list echo "AZCLI_ALT_SUBSCRIPTION_ID=$AZCLI_ALT_SUBSCRIPTION_ID" >> env.list echo "AZCLI_ALT_CLIENT_ID=$AZCLI_ALT_CLIENT_ID" >> env.list echo "AZCLI_ALT_CLIENT_SECRET=$MAPPED_AZCLI_ALT_CLIENT_SECRET" >> env.list +# predefined variables +echo "BUILD_REASON=$(Build.Reason)" >> env.list +echo "SYSTEM_PULLREQUEST_TARGETBRANCH=$(System.PullRequest.TargetBranch)" >> env.list + +# variables echo "PYTHON_VERSION=$PYTHON_VERSION" >> env.list echo "COVERAGE=$COVERAGE" >> env.list echo "TEST_MODE=$TEST_MODE" >> env.list diff --git a/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml index a699e568ff8..3573e5ed1e7 100644 --- a/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml +++ b/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml @@ -19,15 +19,15 @@ jobs: ls -alh displayName: "Move All Checkout Files to the Newly Created 'azure-cli-extensions' Directory" - bash: | - ./azure-cli-extensions/src/aks-preview/azcli-aks-live-test/clone_repo.sh + source ./azure-cli-extensions/src/aks-preview/azcli-aks-live-test/clone_repo.sh condition: succeeded() displayName: "Clone GitHub Repo and Move Live Test Related Files" - bash: | - ./prepare_image.sh + source ./prepare_image.sh condition: succeeded() displayName: "Prepare Live Test Image" - bash: | - ./start_container.sh + source ./start_container.sh env: MAPPED_AZCLI_ALT_CLIENT_SECRET: $(AZCLI_ALT_CLIENT_SECRET) condition: succeeded() From 6db5ef625dea231a0ad6a0d6e751af32b248b53e Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sun, 25 Apr 2021 14:05:28 +0800 Subject: [PATCH 07/42] wrap predefined variables --- src/aks-preview/azcli-aks-live-test/transcribe_env.sh | 4 ++-- .../azcli-aks-live-test/vsts-azcli-aks-live-test.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh index 09ff2ba16ac..22cc51b1091 100755 --- a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh +++ b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh @@ -10,8 +10,8 @@ echo "AZCLI_ALT_CLIENT_ID=$AZCLI_ALT_CLIENT_ID" >> env.list echo "AZCLI_ALT_CLIENT_SECRET=$MAPPED_AZCLI_ALT_CLIENT_SECRET" >> env.list # predefined variables -echo "BUILD_REASON=$(Build.Reason)" >> env.list -echo "SYSTEM_PULLREQUEST_TARGETBRANCH=$(System.PullRequest.TargetBranch)" >> env.list +echo "BUILD_REASON=$BUILD_REASON" >> env.list +echo "SYSTEM_PULLREQUEST_TARGETBRANCH=$SYSTEM_PULLREQUEST_TARGETBRANCH" >> env.list # variables echo "PYTHON_VERSION=$PYTHON_VERSION" >> env.list diff --git a/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml index 3573e5ed1e7..31420f77fe1 100644 --- a/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml +++ b/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml @@ -30,6 +30,8 @@ jobs: source ./start_container.sh env: MAPPED_AZCLI_ALT_CLIENT_SECRET: $(AZCLI_ALT_CLIENT_SECRET) + BUILD_REASON: $(Build.Reason) + SYSTEM_PULLREQUEST_TARGETBRANCH: $(System.PullRequest.TargetBranch) condition: succeeded() displayName: "Start Container" - bash: | From 5df5e6f3a15aa6b7d4e36a234bef1a00a23ef131 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sun, 25 Apr 2021 14:18:33 +0800 Subject: [PATCH 08/42] install git in image --- src/aks-preview/azcli-aks-live-test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-preview/azcli-aks-live-test/Dockerfile b/src/aks-preview/azcli-aks-live-test/Dockerfile index dcc76f411c9..abf405bcd40 100644 --- a/src/aks-preview/azcli-aks-live-test/Dockerfile +++ b/src/aks-preview/azcli-aks-live-test/Dockerfile @@ -3,5 +3,5 @@ RUN apt update && \ apt install -y software-properties-common && \ add-apt-repository -y ppa:deadsnakes/ppa && \ apt update && \ - apt install -y python3.8 python3.8-venv python3.8-dev python3-pip gcc + apt install -y python3.8 python3.8-venv python3.8-dev python3-pip gcc git CMD ["/bin/bash"] From f7163fd31151e3cc9d96632a2d27d20eaf4aba3d Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sun, 25 Apr 2021 14:35:27 +0800 Subject: [PATCH 09/42] remove ci mode in cli test --- src/aks-preview/azcli-aks-live-test/test_cli.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/aks-preview/azcli-aks-live-test/test_cli.sh b/src/aks-preview/azcli-aks-live-test/test_cli.sh index 21df4bdd70e..b03f40127ca 100755 --- a/src/aks-preview/azcli-aks-live-test/test_cli.sh +++ b/src/aks-preview/azcli-aks-live-test/test_cli.sh @@ -24,9 +24,5 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID az account set -s $AZCLI_ALT_SUBSCRIPTION_ID az account show - if [[ $TEST_DIFF == true && $BUILD_REASON == "PullRequest" ]]; then - azdev test aks-preview --live --no-exitfirst --repo=azure-cli/ --src=HEAD --tgt=origin/$SYSTEM_PULLREQUEST_TARGETBRANCH --cli-ci --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" - else - azdev test acs --live --no-exitfirst --xml-path cli_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_live_report.json --reruns 3 --capture=sys" - fi + azdev test acs --live --no-exitfirst --xml-path cli_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_live_report.json --reruns 3 --capture=sys" fi From c446a0ed6cf7d29c8141f8fbdebded9785404f54 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Mon, 26 Apr 2021 19:17:48 +0800 Subject: [PATCH 10/42] add azdev test sp env vars --- src/aks-preview/azcli-aks-live-test/transcribe_env.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh index 22cc51b1091..771d2cde6df 100755 --- a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh +++ b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh @@ -9,6 +9,10 @@ echo "AZCLI_ALT_SUBSCRIPTION_ID=$AZCLI_ALT_SUBSCRIPTION_ID" >> env.list echo "AZCLI_ALT_CLIENT_ID=$AZCLI_ALT_CLIENT_ID" >> env.list echo "AZCLI_ALT_CLIENT_SECRET=$MAPPED_AZCLI_ALT_CLIENT_SECRET" >> env.list +# azdev env +echo "AZURE_CLI_TEST_DEV_SP_NAME=$AZCLI_ALT_CLIENT_ID" >> env.list +echo "AZURE_CLI_TEST_DEV_SP_PASSWORD=$MAPPED_AZCLI_ALT_CLIENT_SECRET" >> env.list + # predefined variables echo "BUILD_REASON=$BUILD_REASON" >> env.list echo "SYSTEM_PULLREQUEST_TARGETBRANCH=$SYSTEM_PULLREQUEST_TARGETBRANCH" >> env.list From 0c21d3b563cb2c55402d6e7dd5165d52bd2e80d1 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Tue, 27 Apr 2021 09:31:08 +0800 Subject: [PATCH 11/42] fix yaml bug --- .../azcli-aks-live-test/vsts-azcli-aks-live-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml index 31420f77fe1..baab8382d49 100644 --- a/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml +++ b/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml @@ -72,5 +72,5 @@ jobs: PathtoPublish: 'ext_live_report.json' ArtifactName: 'ext_live_report.json' publishLocation: 'Container' - condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'cli', 'all'), in(variables['TEST_MODE'], 'live', 'all')) + condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all'), in(variables['TEST_MODE'], 'live', 'all')) displayName: Publish EXT Live Test Report From 288debae71a099e94938f0c8d1622134f9d2eb54 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Wed, 28 Apr 2021 19:37:05 +0800 Subject: [PATCH 12/42] add init filter --- .../azcli-aks-live-test/.gitignore | 1 + .../ext_matrix_default.json | 41 ++++++ src/aks-preview/azcli-aks-live-test/main.py | 129 ++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 src/aks-preview/azcli-aks-live-test/ext_matrix_default.json create mode 100644 src/aks-preview/azcli-aks-live-test/main.py diff --git a/src/aks-preview/azcli-aks-live-test/.gitignore b/src/aks-preview/azcli-aks-live-test/.gitignore index a53b92a3593..88c05203107 100644 --- a/src/aks-preview/azcli-aks-live-test/.gitignore +++ b/src/aks-preview/azcli-aks-live-test/.gitignore @@ -1,3 +1,4 @@ env.list *.json *.xml +!ext_matrix_default.json diff --git a/src/aks-preview/azcli-aks-live-test/ext_matrix_default.json b/src/aks-preview/azcli-aks-live-test/ext_matrix_default.json new file mode 100644 index 00000000000..6e3c8a73401 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/ext_matrix_default.json @@ -0,0 +1,41 @@ +{ + "coverage": { + "test_aks_commands": [ + "AzureKubernetesServiceScenarioTest" + ] + }, + "exclude": { + "reason": { + "naf": [ + "test_aks_create_addon_with_azurekeyvaultsecretsprovider_with_secret_rotation", + "test_aks_create_with_auto_upgrade_channel", + "test_aks_create_with_pod_identity_enabled", + "test_aks_create_with_azurekeyvaultsecretsprovider_addon", + "test_aks_create_using_azurecni_with_pod_identity_enabled", + "test_aks_create_with_gitops_addon", + "test_aks_create_with_node_config", + "test_aks_custom_kubelet_identity", + "test_aks_disable_addon_gitops", + "test_aks_disable_addon_openservicemesh", + "test_aks_pod_identity_usage", + "test_aks_create_with_openservicemesh_addon", + "test_aks_update_azurekeyvaultsecretsprovider_with_secret_rotation", + "test_aks_enable_addon_with_azurekeyvaultsecretsprovider", + "test_aks_enable_addon_with_gitops", + "test_aks_create_with_fips" + ], + "unknown": [ + "test_aks_create_and_update_with_managed_aad_enable_azure_rbac", + "test_aks_create_with_virtual_node_addon" + ], + "code bug": [ + "test_aks_create_with_ingress_appgw_addon_with_deprecated_subet_prefix", + "test_aks_byo_subnet_with_ingress_appgw_addon", + "test_aks_nodepool_get_upgrades", + "test_aks_byo_appgw_with_ingress_appgw_addon", + "test_aks_enable_addon_with_openservicemesh", + "test_aks_create_with_ingress_appgw_addon" + ] + } + } +} \ No newline at end of file diff --git a/src/aks-preview/azcli-aks-live-test/main.py b/src/aks-preview/azcli-aks-live-test/main.py new file mode 100644 index 00000000000..7198e9311fb --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/main.py @@ -0,0 +1,129 @@ +import azdev +import os +import glob +import json +from azdev.utilities import EXTENSION_PREFIX, get_path_table, get_name_index +import azdev.operations.testtool as testtool + +# const +EXTENSION_NAME = "aks-preview" +AKS_PREVIEW_MOD_NAME = EXTENSION_PREFIX + "aks_preview" # azext_aks_preview + + +def init_argparse(): + parser = argparse.ArgumentParser() + parser.add_argument("tests", nargs="+", help="test name") + parser.add_argument("-s", "--series", action="store_true", + default=False, help="series test") + parser.add_argument("-l", "--live", action="store_true", + default=False, help="live test") + parser.add_argument("-d", "--discover", action="store_true", + default=False, help="discover test index") + parser.add_argument("--no-exitfirst", action="store_true", + default=False, help="no exit first") + parser.add_argument("--xml-path", type=str, + default="azcli_aks_runner.xml", help="junit log path") + parser.add_argument("-n", "--parallelism", type=str, + default="8", help="test parallelism") + parser.add_argument("-p", "--json-report-path", type=str, + required=True, help="json report path") + parser.add_argument("-f", "--json-report-file", type=str, + default="azcli_aks_runner_report.json", help="json report filename") + parser.add_argument("-r", "--reruns", type=str, + default="3", help="rerun times") + parser.add_argument("-c", "--capture", type=str, + default="sys", help="test capture") + # parser.add_argument("-a", "--pytest-args", + # nargs=argparse.REMAINDER, help="pytest args") + args = parser.parse_args() + return args + + +def get_ext_test_index(): + # path table & name index + path_table = get_path_table() + command_modules = path_table['mod'] + extensions = path_table["ext"] + inverse_name_table = get_name_index(invert=True) + + # import_name & mod_data + aks_preview_mod_path = extensions[AKS_PREVIEW_MOD_NAME] + glob_pattern = os.path.normcase( + os.path.join("{}*".format(EXTENSION_PREFIX))) + file_path = glob.glob(os.path.join(aks_preview_mod_path, glob_pattern))[0] + import_name = os.path.basename(file_path) + mod_data = { + "alt_name": inverse_name_table[AKS_PREVIEW_MOD_NAME], + "filepath": os.path.join(file_path, "tests", "latest"), + "base_path": "{}.tests.{}".format(import_name, "latest"), + "files": {} + } + + # azdev hook + ext_test = testtool._discover_module_tests(import_name, mod_data) + ext_test_index = ext_test["files"] + return ext_test_index + + +def get_ext_matrix(ext_matrix_file_path): + json_file = open(ext_matrix_file_path, 'r') + ext_matrix = json.load(json_file) + json_file.close() + + +def get_ext_filted_test_cases(ext_test_index, ext_matrix): + # ext test cases + ext_test_cases = [] + ext_coverage = ext_matrix["coverage"] + for fileName, className in ext_coverage.items(): + for c in className: + ext_test_cases.extend(ext_test_index[fileName][c]) + print(len(ext_test_cases)) + + # ext exclude cases + ext_exclude_test_cases = [] + ext_exclude = ext_matrix["exclude"] + for k, v in ext_exclude["reason"].items(): + ext_exclude_test_cases.extend(v) + print(len(ext_exclude_test_cases)) + + # ext filtered cases + ext_filtered_test_cases = [ + x for x in ext_test_cases if x not in ext_exclude_test_cases] + print(len(ext_filtered_test_cases)) + + +def decorate_qualified_prefix(test_cases, prefix): + decorated_test_cases = ["{}.{}".format(prefix, x) for x in test_cases] + return decorated_test_cases + + +def main(): + args = init_argparse() + report_file_full_path = os.path.realpath(os.path.join( + args.json_report_path, args.json_report_file)) + print("report file full path: {}".format(report_file_full_path)) + + ext_matrix_file_path = "/home/fumingzhang/azure-cli-extensions/src/aks-preview/azcli-aks-live-test/ext_matrix_default.json" + ext_test_index = get_ext_test_index() + ext_matrix = get_ext_matrix() + ext_filtered_test_cases = get_ext_filted_test_cases() + ext_qualified_test_cases = decorate_qualified_prefix( + ext_filtered_test_cases, AKS_PREVIEW_MOD_NAME) + + pytest_args = [] + if not args.series: + pytest_args.append("-n ".format(args.parallelism)) + pytest_args.append("--json-report") + pytest_args.append("--json-report-file {}".format(report_file_full_path)) + pytest_args.append("--reruns {}".format(args.reruns)) + pytest_args.append("--capture {}".format(args.capture)) + pytest_args = [" ".join(pytest_args)] + print("pytest_args: {}".format(pytest_args)) + + run_tests(ext_qualified_test_cases, xml_path=args.xml_path, discover=args.discover, in_series=args.series, + run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + + +if __name__ == "__main__": + main() From f897d859e6832ecaf21965bf864671f1bd8d5cc1 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Thu, 29 Apr 2021 14:25:47 +0800 Subject: [PATCH 13/42] * replace azdev with az-aks-tool * add custom preparer --- .../az-aks-tool/__init__.py | 0 .../az-aks-tool/custom_preparers.py | 54 ++++++++ .../{ => az-aks-tool}/main.py | 115 ++++++++++++------ .../azcli-aks-live-test/az-aks-tool/utils.py | 17 +++ .../ext_matrix_default.json | 62 +++++----- .../azcli-aks-live-test/test_ext.sh | 26 +++- 6 files changed, 198 insertions(+), 76 deletions(-) create mode 100644 src/aks-preview/azcli-aks-live-test/az-aks-tool/__init__.py create mode 100644 src/aks-preview/azcli-aks-live-test/az-aks-tool/custom_preparers.py rename src/aks-preview/azcli-aks-live-test/{ => az-aks-tool}/main.py (51%) create mode 100644 src/aks-preview/azcli-aks-live-test/az-aks-tool/utils.py diff --git a/src/aks-preview/azcli-aks-live-test/az-aks-tool/__init__.py b/src/aks-preview/azcli-aks-live-test/az-aks-tool/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/aks-preview/azcli-aks-live-test/az-aks-tool/custom_preparers.py b/src/aks-preview/azcli-aks-live-test/az-aks-tool/custom_preparers.py new file mode 100644 index 00000000000..85b72e12aa6 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/az-aks-tool/custom_preparers.py @@ -0,0 +1,54 @@ +import os +from datetime import datetime + +from azure.cli.testsdk import ResourceGroupPreparer, CliTestError, get_dummy_cli + + +class AKSCustomResourceGroupPreparer(ResourceGroupPreparer): + def __init__(self, name_prefix='clitest.rg', + parameter_name='resource_group', + parameter_name_for_location='resource_group_location', location='westus', + dev_setting_name='AZURE_CLI_TEST_DEV_RESOURCE_GROUP_NAME', + dev_setting_location='AZURE_CLI_TEST_DEV_RESOURCE_GROUP_LOCATION', + random_name_length=75, key='rg'): + if ' ' in name_prefix: + raise CliTestError( + 'Error: Space character in resource group name prefix \'%s\'' % name_prefix) + super(AKSCustomResourceGroupPreparer, self).__init__( + name_prefix, random_name_length) + self.cli_ctx = get_dummy_cli() + self.location = location + self.parameter_name = parameter_name + self.parameter_name_for_location = parameter_name_for_location + self.key = key + + self.dev_setting_name = os.environ.get(dev_setting_name, None) + # use environment variable to modify the default value of location + self.dev_setting_location = os.environ.get(dev_setting_location, None) + if self.dev_setting_location is not None: + self.location = self.dev_setting_location + + def create_resource(self, name, **kwargs): + if self.dev_setting_name: + self.test_class_instance.kwargs[self.key] = self.dev_setting_name + return {self.parameter_name: self.dev_setting_name, + self.parameter_name_for_location: self.dev_setting_location} + + tags = {'product': 'azurecli', 'cause': 'automation', + 'date': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')} + if 'ENV_JOB_NAME' in os.environ: + tags['job'] = os.environ['ENV_JOB_NAME'] + tags = ' '.join(['{}={}'.format(key, value) + for key, value in tags.items()]) + template = 'az group create --location {} --name {} --tag ' + tags + self.live_only_execute( + self.cli_ctx, template.format(self.location, name)) + + self.test_class_instance.kwargs[self.key] = name + return {self.parameter_name: name, self.parameter_name_for_location: self.location} + + def remove_resource(self, name, **kwargs): + # delete group if test is being recorded and if the group is not a dev rg + if not self.dev_setting_name: + self.live_only_execute( + self.cli_ctx, 'az group delete --name {} --yes --no-wait'.format(name)) diff --git a/src/aks-preview/azcli-aks-live-test/main.py b/src/aks-preview/azcli-aks-live-test/az-aks-tool/main.py similarity index 51% rename from src/aks-preview/azcli-aks-live-test/main.py rename to src/aks-preview/azcli-aks-live-test/az-aks-tool/main.py index 7198e9311fb..c738b1cce48 100644 --- a/src/aks-preview/azcli-aks-live-test/main.py +++ b/src/aks-preview/azcli-aks-live-test/az-aks-tool/main.py @@ -1,9 +1,10 @@ -import azdev -import os +import argparse import glob -import json +import os +from azdev.operations.testtool import _discover_module_tests, run_tests from azdev.utilities import EXTENSION_PREFIX, get_path_table, get_name_index -import azdev.operations.testtool as testtool + +from utils import get_test_matrix, decorate_qualified_prefix, check_file_existence # const EXTENSION_NAME = "aks-preview" @@ -12,7 +13,13 @@ def init_argparse(): parser = argparse.ArgumentParser() - parser.add_argument("tests", nargs="+", help="test name") + parser.add_argument("-t", "--tests", nargs='+', help="test case names") + parser.add_argument("-cm", "--cli-matrix", type=str, + help="cli test matrix") + parser.add_argument("-em", "--ext-matrix", type=str, + help="extension test matrix") + parser.add_argument("-emf", "--ext-matrix-filter", nargs="+", + help="extension test matrix") parser.add_argument("-s", "--series", action="store_true", default=False, help="series test") parser.add_argument("-l", "--live", action="store_true", @@ -60,69 +67,101 @@ def get_ext_test_index(): } # azdev hook - ext_test = testtool._discover_module_tests(import_name, mod_data) + ext_test = _discover_module_tests(import_name, mod_data) ext_test_index = ext_test["files"] return ext_test_index -def get_ext_matrix(ext_matrix_file_path): - json_file = open(ext_matrix_file_path, 'r') - ext_matrix = json.load(json_file) - json_file.close() - - -def get_ext_filted_test_cases(ext_test_index, ext_matrix): - # ext test cases +def get_ext_test_cases(ext_test_index, ext_matrix): ext_test_cases = [] ext_coverage = ext_matrix["coverage"] for fileName, className in ext_coverage.items(): for c in className: ext_test_cases.extend(ext_test_index[fileName][c]) - print(len(ext_test_cases)) + return ext_test_cases + - # ext exclude cases +def get_ext_exclude_test_cases(ext_matrix, ext_matrix_filter): ext_exclude_test_cases = [] ext_exclude = ext_matrix["exclude"] - for k, v in ext_exclude["reason"].items(): - ext_exclude_test_cases.extend(v) - print(len(ext_exclude_test_cases)) + if not ext_matrix_filter or "all" in ext_matrix_filter: + for k, v in ext_exclude.items(): + ext_exclude_test_cases.extend(v) + else: + for k, v in ext_exclude.items(): + if k in ext_matrix_filter: + ext_exclude_test_cases.extend(v) + return ext_exclude_test_cases + - # ext filtered cases +def get_ext_filted_test_cases(ext_test_cases, ext_exclude_test_cases): ext_filtered_test_cases = [ x for x in ext_test_cases if x not in ext_exclude_test_cases] - print(len(ext_filtered_test_cases)) - - -def decorate_qualified_prefix(test_cases, prefix): - decorated_test_cases = ["{}.{}".format(prefix, x) for x in test_cases] - return decorated_test_cases + return ext_filtered_test_cases def main(): args = init_argparse() + + import sys + print(sys.argv) + print(args) + print(os.getcwd()) + + # test cases + test_cases = args.tests + ext_matrix_file_path = args.ext_matrix + cli_matrix_file_path = args.cli_matrix + if not test_cases and not check_file_existence(ext_matrix_file_path) and not check_file_existence(cli_matrix_file_path): + print("At least one of 'tests', 'cli_matrix' and 'ext_matrix' must be provided!") + exit(-1) + + # report file report_file_full_path = os.path.realpath(os.path.join( args.json_report_path, args.json_report_file)) print("report file full path: {}".format(report_file_full_path)) - ext_matrix_file_path = "/home/fumingzhang/azure-cli-extensions/src/aks-preview/azcli-aks-live-test/ext_matrix_default.json" - ext_test_index = get_ext_test_index() - ext_matrix = get_ext_matrix() - ext_filtered_test_cases = get_ext_filted_test_cases() - ext_qualified_test_cases = decorate_qualified_prefix( - ext_filtered_test_cases, AKS_PREVIEW_MOD_NAME) - + # pytest args pytest_args = [] - if not args.series: - pytest_args.append("-n ".format(args.parallelism)) + if not args.series and args.parallelism: + pytest_args.append("-n {}".format(args.parallelism)) pytest_args.append("--json-report") pytest_args.append("--json-report-file {}".format(report_file_full_path)) pytest_args.append("--reruns {}".format(args.reruns)) pytest_args.append("--capture {}".format(args.capture)) pytest_args = [" ".join(pytest_args)] print("pytest_args: {}".format(pytest_args)) - - run_tests(ext_qualified_test_cases, xml_path=args.xml_path, discover=args.discover, in_series=args.series, - run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + print() + + # ext matrix + if check_file_existence(ext_matrix_file_path): + ext_test_index = get_ext_test_index() + ext_matrix = get_test_matrix(ext_matrix_file_path) + ext_test_cases = get_ext_test_cases(ext_test_index, ext_matrix) + ext_exclude_test_cases = get_ext_exclude_test_cases( + ext_matrix, args.ext_matrix_filter) + ext_filtered_test_cases = get_ext_filted_test_cases( + ext_test_cases, ext_exclude_test_cases) + # add prefix + ext_qualified_test_cases = decorate_qualified_prefix( + ext_filtered_test_cases, AKS_PREVIEW_MOD_NAME) + print("According to 'ext_matrix', we get {} cases, need to exclude {} cases, finally get {} cases".format( + len(ext_test_cases), len(ext_exclude_test_cases), len(ext_filtered_test_cases))) + print("Perform following tests: {}".format(ext_qualified_test_cases)) + run_tests(ext_qualified_test_cases, xml_path=args.xml_path, discover=args.discover, in_series=args.series, + run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + + # cli matrix + if check_file_existence(cli_matrix_file_path): + print("Currently not support!") + pass + + # tests + if test_cases: + print("Accroding to 'tests', we get {} cases".format(len(test_cases))) + print("Perform following tets: {}".format(test_cases)) + run_tests(test_cases, xml_path=args.xml_path, discover=args.discover, in_series=args.series, + run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) if __name__ == "__main__": diff --git a/src/aks-preview/azcli-aks-live-test/az-aks-tool/utils.py b/src/aks-preview/azcli-aks-live-test/az-aks-tool/utils.py new file mode 100644 index 00000000000..cacede57208 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/az-aks-tool/utils.py @@ -0,0 +1,17 @@ +import json +import os + +def get_test_matrix(matrix_file_path): + json_file = open(matrix_file_path, 'r') + test_matrix = json.load(json_file) + json_file.close() + return test_matrix + +def decorate_qualified_prefix(test_cases, prefix): + decorated_test_cases = ["{}.{}".format(prefix, x) for x in test_cases] + return decorated_test_cases + +def check_file_existence(file_path): + if file_path is not None and os.path.isfile(file_path): + return True + return False diff --git a/src/aks-preview/azcli-aks-live-test/ext_matrix_default.json b/src/aks-preview/azcli-aks-live-test/ext_matrix_default.json index 6e3c8a73401..641621960b5 100644 --- a/src/aks-preview/azcli-aks-live-test/ext_matrix_default.json +++ b/src/aks-preview/azcli-aks-live-test/ext_matrix_default.json @@ -5,37 +5,35 @@ ] }, "exclude": { - "reason": { - "naf": [ - "test_aks_create_addon_with_azurekeyvaultsecretsprovider_with_secret_rotation", - "test_aks_create_with_auto_upgrade_channel", - "test_aks_create_with_pod_identity_enabled", - "test_aks_create_with_azurekeyvaultsecretsprovider_addon", - "test_aks_create_using_azurecni_with_pod_identity_enabled", - "test_aks_create_with_gitops_addon", - "test_aks_create_with_node_config", - "test_aks_custom_kubelet_identity", - "test_aks_disable_addon_gitops", - "test_aks_disable_addon_openservicemesh", - "test_aks_pod_identity_usage", - "test_aks_create_with_openservicemesh_addon", - "test_aks_update_azurekeyvaultsecretsprovider_with_secret_rotation", - "test_aks_enable_addon_with_azurekeyvaultsecretsprovider", - "test_aks_enable_addon_with_gitops", - "test_aks_create_with_fips" - ], - "unknown": [ - "test_aks_create_and_update_with_managed_aad_enable_azure_rbac", - "test_aks_create_with_virtual_node_addon" - ], - "code bug": [ - "test_aks_create_with_ingress_appgw_addon_with_deprecated_subet_prefix", - "test_aks_byo_subnet_with_ingress_appgw_addon", - "test_aks_nodepool_get_upgrades", - "test_aks_byo_appgw_with_ingress_appgw_addon", - "test_aks_enable_addon_with_openservicemesh", - "test_aks_create_with_ingress_appgw_addon" - ] - } + "naf": [ + "test_aks_create_addon_with_azurekeyvaultsecretsprovider_with_secret_rotation", + "test_aks_create_with_auto_upgrade_channel", + "test_aks_create_with_pod_identity_enabled", + "test_aks_create_with_azurekeyvaultsecretsprovider_addon", + "test_aks_create_using_azurecni_with_pod_identity_enabled", + "test_aks_create_with_gitops_addon", + "test_aks_create_with_node_config", + "test_aks_custom_kubelet_identity", + "test_aks_disable_addon_gitops", + "test_aks_disable_addon_openservicemesh", + "test_aks_pod_identity_usage", + "test_aks_create_with_openservicemesh_addon", + "test_aks_update_azurekeyvaultsecretsprovider_with_secret_rotation", + "test_aks_enable_addon_with_azurekeyvaultsecretsprovider", + "test_aks_enable_addon_with_gitops", + "test_aks_create_with_fips" + ], + "unknown": [ + "test_aks_create_and_update_with_managed_aad_enable_azure_rbac", + "test_aks_create_with_virtual_node_addon" + ], + "code bug": [ + "test_aks_create_with_ingress_appgw_addon_with_deprecated_subet_prefix", + "test_aks_byo_subnet_with_ingress_appgw_addon", + "test_aks_nodepool_get_upgrades", + "test_aks_byo_appgw_with_ingress_appgw_addon", + "test_aks_enable_addon_with_openservicemesh", + "test_aks_create_with_ingress_appgw_addon" + ] } } \ No newline at end of file diff --git a/src/aks-preview/azcli-aks-live-test/test_ext.sh b/src/aks-preview/azcli-aks-live-test/test_ext.sh index 3671e90eccf..072f9248dc1 100755 --- a/src/aks-preview/azcli-aks-live-test/test_ext.sh +++ b/src/aks-preview/azcli-aks-live-test/test_ext.sh @@ -35,10 +35,21 @@ do source azEnv/bin/activate done +# prepare run flags +run_flags="-em ext_matrix_default.json --no-exitfirst --discover --json-report-path ./ --reruns 3 --capture=sys" +if [ $PARALLELISM -ge 2 ]; then + run_flags+=" -n $PARALLELISM" +else + run_flags+=" -s" +fi + # test ext if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then echo "Test in record mode!" - azdev test aks-preview --no-exitfirst --xml-path ext_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_report.json --reruns 3 --capture=sys" + run_flags+=" --json-report-file=ext_report.json" + echo "run flags: ${run_flags}" + echo ${run_flags} | xargs python -u az-aks-tool/main.py + # azdev test aks-preview --no-exitfirst --xml-path ext_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_report.json --reruns 3 --capture=sys" fi if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then @@ -46,9 +57,12 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID az account set -s $AZCLI_ALT_SUBSCRIPTION_ID az account show - if [[ $TEST_DIFF == true && $BUILD_REASON == "PullRequest" ]]; then - azdev test aks-preview --live --no-exitfirst --repo=azure-cli-extensions/ --src=HEAD --tgt=origin/$SYSTEM_PULLREQUEST_TARGETBRANCH --cli-ci --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" - else - azdev test aks-preview --live --no-exitfirst --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" - fi + run_flags+=" -l --json-report-file=ext_live_report.json" + echo "run flags: ${run_flags}" + echo ${run_flags} | xargs python -u az-aks-tool/main.py + # if [[ $TEST_DIFF == true && $BUILD_REASON == "PullRequest" ]]; then + # azdev test aks-preview --live --no-exitfirst --repo=azure-cli-extensions/ --src=HEAD --tgt=origin/$SYSTEM_PULLREQUEST_TARGETBRANCH --cli-ci --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" + # else + # azdev test aks-preview --live --no-exitfirst --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" + # fi fi From 325af8b46538a4f628815c01cbecfc0594c07521 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Thu, 29 Apr 2021 15:26:28 +0800 Subject: [PATCH 14/42] replace ResourceGroupPreparer with AKSCustomResourceGroupPreparer --- .../{az-aks-tool => az_aks_tool}/__init__.py | 0 .../{az-aks-tool => az_aks_tool}/main.py | 11 +-- .../{az-aks-tool => az_aks_tool}/utils.py | 0 .../azcli-aks-live-test/setup_venv.sh | 3 +- .../azcli-aks-live-test/test_ext.sh | 4 +- .../azcli-aks-live-test/transcribe_env.sh | 14 ++- .../tests/latest}/custom_preparers.py | 2 +- .../tests/latest/test_aks_commands.py | 90 +++++++++---------- 8 files changed, 65 insertions(+), 59 deletions(-) rename src/aks-preview/azcli-aks-live-test/{az-aks-tool => az_aks_tool}/__init__.py (100%) rename src/aks-preview/azcli-aks-live-test/{az-aks-tool => az_aks_tool}/main.py (97%) rename src/aks-preview/azcli-aks-live-test/{az-aks-tool => az_aks_tool}/utils.py (100%) rename src/aks-preview/{azcli-aks-live-test/az-aks-tool => azext_aks_preview/tests/latest}/custom_preparers.py (96%) diff --git a/src/aks-preview/azcli-aks-live-test/az-aks-tool/__init__.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/__init__.py similarity index 100% rename from src/aks-preview/azcli-aks-live-test/az-aks-tool/__init__.py rename to src/aks-preview/azcli-aks-live-test/az_aks_tool/__init__.py diff --git a/src/aks-preview/azcli-aks-live-test/az-aks-tool/main.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py similarity index 97% rename from src/aks-preview/azcli-aks-live-test/az-aks-tool/main.py rename to src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py index c738b1cce48..28b117bcd48 100644 --- a/src/aks-preview/azcli-aks-live-test/az-aks-tool/main.py +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py @@ -1,6 +1,7 @@ import argparse import glob import os +import sys from azdev.operations.testtool import _discover_module_tests, run_tests from azdev.utilities import EXTENSION_PREFIX, get_path_table, get_name_index @@ -101,20 +102,16 @@ def get_ext_filted_test_cases(ext_test_cases, ext_exclude_test_cases): def main(): + print("args: {}".format(sys.argv)) args = init_argparse() - import sys - print(sys.argv) - print(args) - print(os.getcwd()) - # test cases test_cases = args.tests ext_matrix_file_path = args.ext_matrix cli_matrix_file_path = args.cli_matrix if not test_cases and not check_file_existence(ext_matrix_file_path) and not check_file_existence(cli_matrix_file_path): - print("At least one of 'tests', 'cli_matrix' and 'ext_matrix' must be provided!") - exit(-1) + sys.exit( + "At least one of 'tests', 'cli_matrix' and 'ext_matrix' must be provided!") # report file report_file_full_path = os.path.realpath(os.path.join( diff --git a/src/aks-preview/azcli-aks-live-test/az-aks-tool/utils.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py similarity index 100% rename from src/aks-preview/azcli-aks-live-test/az-aks-tool/utils.py rename to src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py diff --git a/src/aks-preview/azcli-aks-live-test/setup_venv.sh b/src/aks-preview/azcli-aks-live-test/setup_venv.sh index aba4c4c9250..5f804ecdc0b 100755 --- a/src/aks-preview/azcli-aks-live-test/setup_venv.sh +++ b/src/aks-preview/azcli-aks-live-test/setup_venv.sh @@ -11,7 +11,8 @@ rm -rf azEnv || true python$PYTHON_VERSION -m venv azEnv source azEnv/bin/activate python -m pip install -U pip -pip install azdev +# fixed azdev version to avoid call failure in az_aks_tool +pip install azdev==0.1.32 pip install pytest-json-report pytest-rerunfailures --upgrade # pip install pytest-html --upgrade diff --git a/src/aks-preview/azcli-aks-live-test/test_ext.sh b/src/aks-preview/azcli-aks-live-test/test_ext.sh index 072f9248dc1..dbfb4f88289 100755 --- a/src/aks-preview/azcli-aks-live-test/test_ext.sh +++ b/src/aks-preview/azcli-aks-live-test/test_ext.sh @@ -48,7 +48,7 @@ if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then echo "Test in record mode!" run_flags+=" --json-report-file=ext_report.json" echo "run flags: ${run_flags}" - echo ${run_flags} | xargs python -u az-aks-tool/main.py + echo ${run_flags} | xargs python -u az_aks_tool/main.py # azdev test aks-preview --no-exitfirst --xml-path ext_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_report.json --reruns 3 --capture=sys" fi @@ -59,7 +59,7 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then az account show run_flags+=" -l --json-report-file=ext_live_report.json" echo "run flags: ${run_flags}" - echo ${run_flags} | xargs python -u az-aks-tool/main.py + echo ${run_flags} | xargs python -u az_aks_tool/main.py # if [[ $TEST_DIFF == true && $BUILD_REASON == "PullRequest" ]]; then # azdev test aks-preview --live --no-exitfirst --repo=azure-cli-extensions/ --src=HEAD --tgt=origin/$SYSTEM_PULLREQUEST_TARGETBRANCH --cli-ci --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" # else diff --git a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh index 771d2cde6df..3ad532602a8 100755 --- a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh +++ b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh @@ -12,23 +12,31 @@ echo "AZCLI_ALT_CLIENT_SECRET=$MAPPED_AZCLI_ALT_CLIENT_SECRET" >> env.list # azdev env echo "AZURE_CLI_TEST_DEV_SP_NAME=$AZCLI_ALT_CLIENT_ID" >> env.list echo "AZURE_CLI_TEST_DEV_SP_PASSWORD=$MAPPED_AZCLI_ALT_CLIENT_SECRET" >> env.list +echo "AZURE_CLI_TEST_DEV_RESOURCE_GROUP_LOCATION=$TEST_LOCATION" >> env.list # predefined variables echo "BUILD_REASON=$BUILD_REASON" >> env.list echo "SYSTEM_PULLREQUEST_TARGETBRANCH=$SYSTEM_PULLREQUEST_TARGETBRANCH" >> env.list -# variables -echo "PYTHON_VERSION=$PYTHON_VERSION" >> env.list +# test echo "COVERAGE=$COVERAGE" >> env.list echo "TEST_MODE=$TEST_MODE" >> env.list echo "PARALLELISM=$PARALLELISM" >> env.list +echo "TEST_CASES=$TEST_CASES" >> env.list + +# repo echo "CLI_REPO=$CLI_REPO" >> env.list echo "CLI_BRANCH=$CLI_BRANCH" >> env.list echo "MANUAL_EXT=$MANUAL_EXT" >> env.list echo "EXT_REPO=$EXT_REPO" >> env.list echo "EXT_BRANCH=$EXT_BRANCH" >> env.list + +# image echo "BUILD_IMAGE=$BUILD_IMAGE" >> env.list echo "IMAGE_PREFIX=$IMAGE_PREFIX" >> env.list echo "IMAGE_NAME=$IMAGE_NAME" >> env.list +echo "IMAGE_SUFFIX=$IMAGE_SUFFIX" >> env.list echo "IMAGE_TAG=$IMAGE_TAG" >> env.list -echo "TEST_DIFF=$TEST_DIFF" >> env.list + +# misc +echo "PYTHON_VERSION=$PYTHON_VERSION" >> env.list diff --git a/src/aks-preview/azcli-aks-live-test/az-aks-tool/custom_preparers.py b/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py similarity index 96% rename from src/aks-preview/azcli-aks-live-test/az-aks-tool/custom_preparers.py rename to src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py index 85b72e12aa6..eb3d5789535 100644 --- a/src/aks-preview/azcli-aks-live-test/az-aks-tool/custom_preparers.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py @@ -25,7 +25,7 @@ def __init__(self, name_prefix='clitest.rg', self.dev_setting_name = os.environ.get(dev_setting_name, None) # use environment variable to modify the default value of location self.dev_setting_location = os.environ.get(dev_setting_location, None) - if self.dev_setting_location is not None: + if self.dev_setting_location is not None and self.dev_setting_location != "": self.location = self.dev_setting_location def create_resource(self, name, **kwargs): diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index fba2c0da28b..c91b77ae0de 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -7,8 +7,8 @@ import os from .recording_processors import KeyReplacer -from azure.cli.testsdk import ( - ResourceGroupPreparer, RoleBasedServicePrincipalPreparer, ScenarioTest, live_only) +from .custom_preparers import AKSCustomResourceGroupPreparer +from azure.cli.testsdk import (RoleBasedServicePrincipalPreparer, ScenarioTest, live_only) from azure_devtools.scenario_tests import AllowLargeResponse @@ -45,7 +45,7 @@ def test_get_os_options(self): # without live only fails with needs .ssh fails (maybe generate-ssh-keys would fix) and maybe az login. @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_and_update_with_managed_aad(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -78,7 +78,7 @@ def test_aks_create_and_update_with_managed_aad(self, resource_group, resource_g # without live only fails with needs .ssh fails (maybe generate-ssh-keys would fix) and maybe az login. @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='canadacentral') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='canadacentral') def test_aks_create_aadv1_and_update_with_managed_aad(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -120,7 +120,7 @@ def test_aks_create_aadv1_and_update_with_managed_aad(self, resource_group, reso # without live only fails with needs .ssh fails (maybe generate-ssh-keys would fix) and maybe az login. @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='canadacentral') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='canadacentral') def test_aks_create_nonaad_and_update_with_managed_aad(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -152,7 +152,7 @@ def test_aks_create_nonaad_and_update_with_managed_aad(self, resource_group, res # without live only fails with needs .ssh fails (maybe generate-ssh-keys would fix) and maybe az login. @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_and_update_with_managed_aad_enable_azure_rbac(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -185,7 +185,7 @@ def test_aks_create_and_update_with_managed_aad_enable_azure_rbac(self, resource ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_ingress_appgw_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -203,7 +203,7 @@ def test_aks_create_with_ingress_appgw_addon(self, resource_group, resource_grou ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_ingress_appgw_addon_with_deprecated_subet_prefix(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -222,7 +222,7 @@ def test_aks_create_with_ingress_appgw_addon_with_deprecated_subet_prefix(self, @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_byo_subnet_with_ingress_appgw_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) vnet_name = self.create_random_name('cliakstest', 16) @@ -274,7 +274,7 @@ def test_aks_byo_subnet_with_ingress_appgw_addon(self, resource_group, resource_ @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_byo_appgw_with_ingress_appgw_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) vnet_name = self.create_random_name('cliakstest', 16) @@ -345,7 +345,7 @@ def test_aks_byo_appgw_with_ingress_appgw_addon(self, resource_group, resource_g }) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_openservicemesh_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -361,7 +361,7 @@ def test_aks_create_with_openservicemesh_addon(self, resource_group, resource_gr ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_enable_addon_with_openservicemesh(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -382,7 +382,7 @@ def test_aks_enable_addon_with_openservicemesh(self, resource_group, resource_gr ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_disable_addon_openservicemesh(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -405,7 +405,7 @@ def test_aks_disable_addon_openservicemesh(self, resource_group, resource_group_ ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_azurekeyvaultsecretsprovider_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -430,7 +430,7 @@ def test_aks_create_with_azurekeyvaultsecretsprovider_addon(self, resource_group ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_addon_with_azurekeyvaultsecretsprovider_with_secret_rotation(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -455,7 +455,7 @@ def test_aks_create_addon_with_azurekeyvaultsecretsprovider_with_secret_rotation ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_enable_addon_with_azurekeyvaultsecretsprovider(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -503,7 +503,7 @@ def test_aks_enable_addon_with_azurekeyvaultsecretsprovider(self, resource_group ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_update_azurekeyvaultsecretsprovider_with_secret_rotation(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -546,7 +546,7 @@ def test_aks_update_azurekeyvaultsecretsprovider_with_secret_rotation(self, reso ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_confcom_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -564,7 +564,7 @@ def test_aks_create_with_confcom_addon(self, resource_group, resource_group_loca ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_confcom_addon_helper_enabled(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -582,7 +582,7 @@ def test_aks_create_with_confcom_addon_helper_enabled(self, resource_group, reso ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_enable_addons_confcom_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -606,7 +606,7 @@ def test_aks_enable_addons_confcom_addon(self, resource_group, resource_group_lo ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_disable_addons_confcom_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -632,7 +632,7 @@ def test_aks_disable_addons_confcom_addon(self, resource_group, resource_group_l @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_virtual_node_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) vnet_name = self.create_random_name('cliakstest', 16) @@ -686,7 +686,7 @@ def test_aks_create_with_virtual_node_addon(self, resource_group, resource_group @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_stop_and_start(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -706,7 +706,7 @@ def test_aks_stop_and_start(self, resource_group, resource_group_location): self.cmd(start_cmd) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_managed_disk(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -723,7 +723,7 @@ def test_aks_create_with_managed_disk(self, resource_group, resource_group_locat ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_ephemeral_disk(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -741,7 +741,7 @@ def test_aks_create_with_ephemeral_disk(self, resource_group, resource_group_loc ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_nodepool_get_upgrades(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) node_pool_name = self.create_random_name('c', 6) @@ -776,7 +776,7 @@ def test_aks_nodepool_get_upgrades(self, resource_group, resource_group_location 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_upgrade_node_image_only_cluster(self, resource_group, resource_group_location): # kwargs for string formatting aks_name = self.create_random_name('cliakstest', 16) @@ -807,7 +807,7 @@ def test_aks_upgrade_node_image_only_cluster(self, resource_group, resource_grou ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_upgrade_node_image_only_nodepool(self, resource_group, resource_group_location): # kwargs for string formatting aks_name = self.create_random_name('cliakstest', 16) @@ -844,7 +844,7 @@ def test_aks_upgrade_node_image_only_nodepool(self, resource_group, resource_gro ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_windows(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -893,7 +893,7 @@ def test_aks_create_with_windows(self, resource_group, resource_group_location): 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus2euap') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus2euap') def test_aks_create_with_fips(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -927,7 +927,7 @@ def test_aks_create_with_fips(self, resource_group, resource_group_location): 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_ahub(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -978,7 +978,7 @@ def test_aks_create_with_ahub(self, resource_group, resource_group_location): @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westeurope') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westeurope') def test_aks_update_to_msi_cluster(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1003,7 +1003,7 @@ def test_aks_update_to_msi_cluster(self, resource_group, resource_group_location @live_only() # without live only fails with need az login @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') def test_aks_create_with_gitops_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1019,7 +1019,7 @@ def test_aks_create_with_gitops_addon(self, resource_group, resource_group_locat @live_only() # without live only fails with need az login @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') def test_aks_enable_addon_with_gitops(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1041,7 +1041,7 @@ def test_aks_enable_addon_with_gitops(self, resource_group, resource_group_locat @live_only() # without live only fails with need az login @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') def test_aks_disable_addon_gitops(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1064,7 +1064,7 @@ def test_aks_disable_addon_gitops(self, resource_group, resource_group_location) @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westeurope') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westeurope') def test_aks_update_to_msi_cluster_with_addons(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1088,7 +1088,7 @@ def test_aks_update_to_msi_cluster_with_addons(self, resource_group, resource_gr 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_auto_upgrade_channel(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1117,7 +1117,7 @@ def test_aks_create_with_auto_upgrade_channel(self, resource_group, resource_gro 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_node_config(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1150,7 +1150,7 @@ def test_aks_create_with_node_config(self, resource_group, resource_group_locati ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_none_private_dns_zone(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -1178,7 +1178,7 @@ def test_aks_create_none_private_dns_zone(self, resource_group, resource_group_l @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_fqdn_subdomain(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -1241,7 +1241,7 @@ def test_aks_create_fqdn_subdomain(self, resource_group, resource_group_location 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_pod_identity_enabled(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1322,7 +1322,7 @@ def test_aks_create_with_pod_identity_enabled(self, resource_group, resource_gro ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_using_azurecni_with_pod_identity_enabled(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1404,7 +1404,7 @@ def test_aks_create_using_azurecni_with_pod_identity_enabled(self, resource_grou # for this case we cannot use recording to capture the fixture, therefore we need to mark it as live_only @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_pod_identity_usage(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) identity_name = self.create_random_name('id', 6) @@ -1494,7 +1494,7 @@ def test_aks_pod_identity_usage(self, resource_group, resource_group_location): ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_update_with_windows_password(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -1540,7 +1540,7 @@ def test_aks_update_with_windows_password(self, resource_group, resource_group_l @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_custom_kubelet_identity(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) control_plane_identity_name = self.create_random_name('cliakstest', 16) From f7868166149c581d812ce28515c6870912fabb4a Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Thu, 29 Apr 2021 15:54:40 +0800 Subject: [PATCH 15/42] fix import bug --- .../azext_aks_preview/tests/latest/custom_preparers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py b/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py index eb3d5789535..ae602380a0c 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py @@ -1,7 +1,8 @@ import os from datetime import datetime -from azure.cli.testsdk import ResourceGroupPreparer, CliTestError, get_dummy_cli +from azure.cli.testsdk import ResourceGroupPreparer, CliTestError +from azure.cli.testsdk.reverse_dependency import get_dummy_cli class AKSCustomResourceGroupPreparer(ResourceGroupPreparer): From 9690beeea8072df310846ccdbe0ef0c19de14d71 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Thu, 29 Apr 2021 16:25:59 +0800 Subject: [PATCH 16/42] add license --- .../azcli-aks-live-test/az_aks_tool/main.py | 5 +++++ .../azcli-aks-live-test/az_aks_tool/utils.py | 5 +++++ src/aks-preview/azcli-aks-live-test/test_ext.sh | 8 +------- .../tests/latest/custom_preparers.py | 11 +++++++++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py index 28b117bcd48..78c0b9706a7 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + import argparse import glob import os diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py index cacede57208..46d717a9508 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + import json import os diff --git a/src/aks-preview/azcli-aks-live-test/test_ext.sh b/src/aks-preview/azcli-aks-live-test/test_ext.sh index dbfb4f88289..1d0ba54c871 100755 --- a/src/aks-preview/azcli-aks-live-test/test_ext.sh +++ b/src/aks-preview/azcli-aks-live-test/test_ext.sh @@ -24,7 +24,7 @@ source azEnv/bin/activate # Ensure that the command index is updated by calling a specific command in aks-preview, so that all the commands defined in aks-preview are loaded correctly # Otherwise, cold boot execution of azdev test may use the api version adopted by the acs command group in azure-cli (which may diverge from the api version used in current aks-preview) retry_count=0 -while ! az aks command invoke --help --debug && [[ $retry_count < 3 ]] +while ! az aks command invoke --help && [[ $retry_count < 3 ]] do retry_count=`expr $retry_count + 1` echo $retry_count"th retry to install aks-preview..." @@ -49,7 +49,6 @@ if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then run_flags+=" --json-report-file=ext_report.json" echo "run flags: ${run_flags}" echo ${run_flags} | xargs python -u az_aks_tool/main.py - # azdev test aks-preview --no-exitfirst --xml-path ext_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_report.json --reruns 3 --capture=sys" fi if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then @@ -60,9 +59,4 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then run_flags+=" -l --json-report-file=ext_live_report.json" echo "run flags: ${run_flags}" echo ${run_flags} | xargs python -u az_aks_tool/main.py - # if [[ $TEST_DIFF == true && $BUILD_REASON == "PullRequest" ]]; then - # azdev test aks-preview --live --no-exitfirst --repo=azure-cli-extensions/ --src=HEAD --tgt=origin/$SYSTEM_PULLREQUEST_TARGETBRANCH --cli-ci --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" - # else - # azdev test aks-preview --live --no-exitfirst --xml-path ext_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=ext_live_report.json --reruns 3 --capture=sys" - # fi fi diff --git a/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py b/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py index ae602380a0c..f0d066b7e9b 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py @@ -1,11 +1,18 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + import os from datetime import datetime -from azure.cli.testsdk import ResourceGroupPreparer, CliTestError +from azure.cli.testsdk import CliTestError +from azure.cli.testsdk.preparers import NoTrafficRecordingPreparer +from azure_devtools.scenario_tests import SingleValueReplacer from azure.cli.testsdk.reverse_dependency import get_dummy_cli -class AKSCustomResourceGroupPreparer(ResourceGroupPreparer): +class AKSCustomResourceGroupPreparer(NoTrafficRecordingPreparer, SingleValueReplacer): def __init__(self, name_prefix='clitest.rg', parameter_name='resource_group', parameter_name_for_location='resource_group_location', location='westus', From 11f2330ace0981c57144665f5b0b9918dbc584f8 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Thu, 6 May 2021 14:34:10 +0800 Subject: [PATCH 17/42] update filter --- .../azcli-aks-live-test/az_aks_tool/ext.py | 88 +++++++++++++++ .../azcli-aks-live-test/az_aks_tool/main.py | 100 +++++------------- .../azcli-aks-live-test/az_aks_tool/utils.py | 62 ++++++++++- .../azcli-aks-live-test/prepare_image.sh | 4 +- .../azcli-aks-live-test/transcribe_env.sh | 1 - 5 files changed, 175 insertions(+), 80 deletions(-) create mode 100644 src/aks-preview/azcli-aks-live-test/az_aks_tool/ext.py diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/ext.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/ext.py new file mode 100644 index 00000000000..dea9f9896f0 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/ext.py @@ -0,0 +1,88 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import glob +import os +from azdev.operations.testtool import _discover_module_tests +from azdev.utilities import EXTENSION_PREFIX, get_path_table, get_name_index + +import utils + + +def get_ext_test_index(mod_name): + # path table & name index + path_table = get_path_table() + command_modules = path_table['mod'] + extensions = path_table["ext"] + inverse_name_table = get_name_index(invert=True) + + # import_name & mod_data + aks_preview_mod_path = extensions[mod_name] + glob_pattern = os.path.normcase( + os.path.join("{}*".format(EXTENSION_PREFIX))) + file_path = glob.glob(os.path.join(aks_preview_mod_path, glob_pattern))[0] + import_name = os.path.basename(file_path) + mod_data = { + "alt_name": inverse_name_table[mod_name], + "filepath": os.path.join(file_path, "tests", "latest"), + "base_path": "{}.tests.{}".format(import_name, "latest"), + "files": {} + } + + # azdev hook + ext_test = _discover_module_tests(import_name, mod_data) + ext_test_index = ext_test["files"] + return ext_test_index + + +def get_ext_test_cases(ext_test_index, ext_matrix, ext_extra_coverage): + ext_test_cases = [] + ext_coverage = ext_matrix["coverage"] + for fileName, className in ext_coverage.items(): + for c in className: + ext_test_cases.extend(ext_test_index[fileName][c]) + # extra coverage + if ext_extra_coverage: + # method 1: fileName.className + file_class_pairs = utils.extract_file_class_pairs(ext_extra_coverage) + valid_file_class_pairs = utils.filter_valid_file_class_pairs( + file_class_pairs, ext_test_index) + for valid_pair in valid_file_class_pairs: + ext_test_cases.extend( + ext_test_index[valid_pair[0]][valid_pair[1]]) + # method 2: test cases + ext_test_cases.extend(utils.filter_valid_test_cases( + ext_extra_coverage, ext_test_index)) + return list(set(ext_test_cases)) + + +def get_ext_exclude_test_cases(ext_test_index, ext_matrix, ext_filter): + ext_exclude_test_cases = [] + ext_exclude = ext_matrix["exclude"] + if not ext_filter: + for k, v in ext_exclude.items(): + ext_exclude_test_cases.extend(v) + else: + # method 1: matrix exclude key + for k, v in ext_exclude.items(): + if k in ext_filter or "default" in ext_filter: + ext_exclude_test_cases.extend(v) + # method 2: fileName.className + file_class_pairs = utils.extract_file_class_pairs(ext_filter) + valid_file_class_pairs = utils.filter_valid_file_class_pairs( + file_class_pairs, ext_test_index) + for valid_pair in valid_file_class_pairs: + ext_exclude_test_cases.extend( + ext_test_index[valid_pair[0]][valid_pair[1]]) + # method 3: test cases + ext_exclude_test_cases.extend( + utils.filter_valid_test_cases(ext_filter, ext_test_index)) + return list(set(ext_exclude_test_cases)) + + +def get_ext_filted_test_cases(ext_test_cases, ext_exclude_test_cases): + ext_filtered_test_cases = [ + x for x in ext_test_cases if x not in ext_exclude_test_cases] + return ext_filtered_test_cases diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py index 78c0b9706a7..2bfebae47a2 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py @@ -4,16 +4,15 @@ # -------------------------------------------------------------------------------------------- import argparse -import glob import os import sys -from azdev.operations.testtool import _discover_module_tests, run_tests -from azdev.utilities import EXTENSION_PREFIX, get_path_table, get_name_index +from azdev.operations.testtool import run_tests +from azdev.utilities import EXTENSION_PREFIX -from utils import get_test_matrix, decorate_qualified_prefix, check_file_existence +import ext +import utils # const -EXTENSION_NAME = "aks-preview" AKS_PREVIEW_MOD_NAME = EXTENSION_PREFIX + "aks_preview" # azext_aks_preview @@ -21,11 +20,17 @@ def init_argparse(): parser = argparse.ArgumentParser() parser.add_argument("-t", "--tests", nargs='+', help="test case names") parser.add_argument("-cm", "--cli-matrix", type=str, - help="cli test matrix") + help="full path to cli test matrix") + parser.add_argument("-cc", "--cli-coverage", nargs="+", + help="cli test extra coverage") + parser.add_argument("-cf", "--cli-filter", nargs="+", + help="cli test filter") parser.add_argument("-em", "--ext-matrix", type=str, - help="extension test matrix") - parser.add_argument("-emf", "--ext-matrix-filter", nargs="+", - help="extension test matrix") + help="full path to extension test matrix") + parser.add_argument("-ec", "--ext-coverage", nargs="+", + help="extension test extra coverage") + parser.add_argument("-ef", "--ext-filter", nargs="+", + help="extension test filter") parser.add_argument("-s", "--series", action="store_true", default=False, help="series test") parser.add_argument("-l", "--live", action="store_true", @@ -52,60 +57,6 @@ def init_argparse(): return args -def get_ext_test_index(): - # path table & name index - path_table = get_path_table() - command_modules = path_table['mod'] - extensions = path_table["ext"] - inverse_name_table = get_name_index(invert=True) - - # import_name & mod_data - aks_preview_mod_path = extensions[AKS_PREVIEW_MOD_NAME] - glob_pattern = os.path.normcase( - os.path.join("{}*".format(EXTENSION_PREFIX))) - file_path = glob.glob(os.path.join(aks_preview_mod_path, glob_pattern))[0] - import_name = os.path.basename(file_path) - mod_data = { - "alt_name": inverse_name_table[AKS_PREVIEW_MOD_NAME], - "filepath": os.path.join(file_path, "tests", "latest"), - "base_path": "{}.tests.{}".format(import_name, "latest"), - "files": {} - } - - # azdev hook - ext_test = _discover_module_tests(import_name, mod_data) - ext_test_index = ext_test["files"] - return ext_test_index - - -def get_ext_test_cases(ext_test_index, ext_matrix): - ext_test_cases = [] - ext_coverage = ext_matrix["coverage"] - for fileName, className in ext_coverage.items(): - for c in className: - ext_test_cases.extend(ext_test_index[fileName][c]) - return ext_test_cases - - -def get_ext_exclude_test_cases(ext_matrix, ext_matrix_filter): - ext_exclude_test_cases = [] - ext_exclude = ext_matrix["exclude"] - if not ext_matrix_filter or "all" in ext_matrix_filter: - for k, v in ext_exclude.items(): - ext_exclude_test_cases.extend(v) - else: - for k, v in ext_exclude.items(): - if k in ext_matrix_filter: - ext_exclude_test_cases.extend(v) - return ext_exclude_test_cases - - -def get_ext_filted_test_cases(ext_test_cases, ext_exclude_test_cases): - ext_filtered_test_cases = [ - x for x in ext_test_cases if x not in ext_exclude_test_cases] - return ext_filtered_test_cases - - def main(): print("args: {}".format(sys.argv)) args = init_argparse() @@ -114,7 +65,7 @@ def main(): test_cases = args.tests ext_matrix_file_path = args.ext_matrix cli_matrix_file_path = args.cli_matrix - if not test_cases and not check_file_existence(ext_matrix_file_path) and not check_file_existence(cli_matrix_file_path): + if not test_cases and not utils.check_file_existence(ext_matrix_file_path) and not utils.check_file_existence(cli_matrix_file_path): sys.exit( "At least one of 'tests', 'cli_matrix' and 'ext_matrix' must be provided!") @@ -136,25 +87,26 @@ def main(): print() # ext matrix - if check_file_existence(ext_matrix_file_path): - ext_test_index = get_ext_test_index() - ext_matrix = get_test_matrix(ext_matrix_file_path) - ext_test_cases = get_ext_test_cases(ext_test_index, ext_matrix) - ext_exclude_test_cases = get_ext_exclude_test_cases( - ext_matrix, args.ext_matrix_filter) - ext_filtered_test_cases = get_ext_filted_test_cases( + if utils.check_file_existence(ext_matrix_file_path): + ext_test_index = ext.get_ext_test_index(AKS_PREVIEW_MOD_NAME) + ext_matrix = utils.get_test_matrix(ext_matrix_file_path) + ext_test_cases = ext.get_ext_test_cases( + ext_test_index, ext_matrix, args.ext_coverage) + ext_exclude_test_cases = ext.get_ext_exclude_test_cases(ext_test_index, + ext_matrix, args.ext_filter) + ext_filtered_test_cases = ext.get_ext_filted_test_cases( ext_test_cases, ext_exclude_test_cases) # add prefix - ext_qualified_test_cases = decorate_qualified_prefix( + ext_qualified_test_cases = utils.decorate_qualified_prefix( ext_filtered_test_cases, AKS_PREVIEW_MOD_NAME) - print("According to 'ext_matrix', we get {} cases, need to exclude {} cases, finally get {} cases".format( + print("According to 'ext_matrix' and filters, we get {} cases, need to exclude {} cases, finally get {} cases!".format( len(ext_test_cases), len(ext_exclude_test_cases), len(ext_filtered_test_cases))) print("Perform following tests: {}".format(ext_qualified_test_cases)) run_tests(ext_qualified_test_cases, xml_path=args.xml_path, discover=args.discover, in_series=args.series, run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) # cli matrix - if check_file_existence(cli_matrix_file_path): + if utils.check_file_existence(cli_matrix_file_path): print("Currently not support!") pass diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py index 46d717a9508..9054d15504e 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py @@ -3,20 +3,74 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +from collections import Iterable import json import os + +def check_file_existence(file_path): + if file_path is not None and os.path.isfile(file_path): + return True + return False + + def get_test_matrix(matrix_file_path): json_file = open(matrix_file_path, 'r') test_matrix = json.load(json_file) json_file.close() return test_matrix + def decorate_qualified_prefix(test_cases, prefix): decorated_test_cases = ["{}.{}".format(prefix, x) for x in test_cases] return decorated_test_cases -def check_file_existence(file_path): - if file_path is not None and os.path.isfile(file_path): - return True - return False + +def extract_file_class_pairs(tag_list): + pairs = [] + for k in tag_list: + tags = k.split(".") + if len(tags) == 2: + pairs.append((tags[0], tags[1])) + return pairs + + +def filter_valid_file_class_pairs(pairs, test_index): + valid_pairs = [] + for pair in pairs: + if pair[0] in test_index and pair[1] in test_index[pair[0]]: + valid_pairs.append(pair) + print("Find valid file & class pair: '{}'".format(pair)) + else: + print("Invalid file & class pair: '{}'".format(pair)) + return valid_pairs + + +def get_all_values_from_nested_dict(d): + for v in d.values(): + if isinstance(v, dict): + yield from get_all_values_from_nested_dict(v) + else: + yield v + + +def flatten_nested_list(lis): + for item in lis: + if isinstance(item, Iterable) and not isinstance(item, str): + for x in flatten_nested_list(item): + yield x + else: + yield item + + +def filter_valid_test_cases(test_cases, test_index): + valid_test_cases = [] + nested_test_cases = list(get_all_values_from_nested_dict(test_index)) + falttened_test_cases = list(flatten_nested_list(nested_test_cases)) + for test_case in test_cases: + if test_case in falttened_test_cases: + valid_test_cases.append(test_case) + print("Find valid test case: '{}'".format(test_case)) + else: + print("Invalid test case: '{}'".format(test_case)) + return valid_test_cases diff --git a/src/aks-preview/azcli-aks-live-test/prepare_image.sh b/src/aks-preview/azcli-aks-live-test/prepare_image.sh index 7fd47d571e0..93f17467e31 100755 --- a/src/aks-preview/azcli-aks-live-test/prepare_image.sh +++ b/src/aks-preview/azcli-aks-live-test/prepare_image.sh @@ -9,7 +9,9 @@ if [[ $BUILD_IMAGE == true ]]; then docker build -t $IMAGE_NAME:$IMAGE_TAG -f ./Dockerfile . else echo "Pulling test image from '$IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG'..." - if ! docker pull $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG; then + if docker pull $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG; then + docker tag $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:$IMAGE_TAG + else echo "Failed to pull image, start local build..." docker build -t $IMAGE_NAME:$IMAGE_TAG -f ./Dockerfile . fi diff --git a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh index 3ad532602a8..837ddc6cb1a 100755 --- a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh +++ b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh @@ -35,7 +35,6 @@ echo "EXT_BRANCH=$EXT_BRANCH" >> env.list echo "BUILD_IMAGE=$BUILD_IMAGE" >> env.list echo "IMAGE_PREFIX=$IMAGE_PREFIX" >> env.list echo "IMAGE_NAME=$IMAGE_NAME" >> env.list -echo "IMAGE_SUFFIX=$IMAGE_SUFFIX" >> env.list echo "IMAGE_TAG=$IMAGE_TAG" >> env.list # misc From 20fe1a7e273e4d4c701dfaa6955dc72320ac7efb Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Thu, 6 May 2021 14:43:47 +0800 Subject: [PATCH 18/42] update script --- src/aks-preview/azcli-aks-live-test/test_ext.sh | 9 +++++++++ src/aks-preview/azcli-aks-live-test/transcribe_env.sh | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/aks-preview/azcli-aks-live-test/test_ext.sh b/src/aks-preview/azcli-aks-live-test/test_ext.sh index 1d0ba54c871..3334ee9b67d 100755 --- a/src/aks-preview/azcli-aks-live-test/test_ext.sh +++ b/src/aks-preview/azcli-aks-live-test/test_ext.sh @@ -37,11 +37,20 @@ done # prepare run flags run_flags="-em ext_matrix_default.json --no-exitfirst --discover --json-report-path ./ --reruns 3 --capture=sys" +# parallel if [ $PARALLELISM -ge 2 ]; then run_flags+=" -n $PARALLELISM" else run_flags+=" -s" fi +# ext filter +if [[ -n $EXT_TEST_FILTER ]]; then + run_flags+=" -ef $EXT_TEST_FILTER" +fi +# ext extra coverage +if [[ $EXT_TEST_COVERAGE ]]; then + run_flags+=" -ec $EXT_TEST_COVERAGE" +fi # test ext if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then diff --git a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh index 837ddc6cb1a..7715da60337 100755 --- a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh +++ b/src/aks-preview/azcli-aks-live-test/transcribe_env.sh @@ -23,6 +23,8 @@ echo "COVERAGE=$COVERAGE" >> env.list echo "TEST_MODE=$TEST_MODE" >> env.list echo "PARALLELISM=$PARALLELISM" >> env.list echo "TEST_CASES=$TEST_CASES" >> env.list +echo "EXT_TEST_FILTER=$EXT_TEST_FILTER" >> env.list +echo "EXT_TEST_COVERAGE=$EXT_TEST_COVERAGE" >> env.list # repo echo "CLI_REPO=$CLI_REPO" >> env.list From c17d64c6c549832dc66413d2308567de66a40751 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Thu, 6 May 2021 18:17:09 +0800 Subject: [PATCH 19/42] add unit tests --- .../azcli-aks-live-test/.gitignore | 1 + .../azcli-aks-live-test/az_aks_tool/ext.py | 13 +- .../azcli-aks-live-test/az_aks_tool/main.py | 2 +- .../az_aks_tool/test/__init__.py | 0 .../az_aks_tool/test/test.py | 141 ++++++++++++++++++ .../az_aks_tool/test/testdata.json | 22 +++ .../azcli-aks-live-test/az_aks_tool/utils.py | 6 + 7 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 src/aks-preview/azcli-aks-live-test/az_aks_tool/test/__init__.py create mode 100644 src/aks-preview/azcli-aks-live-test/az_aks_tool/test/test.py create mode 100644 src/aks-preview/azcli-aks-live-test/az_aks_tool/test/testdata.json diff --git a/src/aks-preview/azcli-aks-live-test/.gitignore b/src/aks-preview/azcli-aks-live-test/.gitignore index 88c05203107..819270b0650 100644 --- a/src/aks-preview/azcli-aks-live-test/.gitignore +++ b/src/aks-preview/azcli-aks-live-test/.gitignore @@ -2,3 +2,4 @@ env.list *.json *.xml !ext_matrix_default.json +!testdata.json diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/ext.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/ext.py index dea9f9896f0..6ee04394fa4 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/ext.py +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/ext.py @@ -62,8 +62,19 @@ def get_ext_exclude_test_cases(ext_test_index, ext_matrix, ext_filter): ext_exclude_test_cases = [] ext_exclude = ext_matrix["exclude"] if not ext_filter: + matrix_test_cases = [] + matrix_file_class_pairs = [] for k, v in ext_exclude.items(): - ext_exclude_test_cases.extend(v) + # method 1: reason -> test cases + matrix_test_cases.extend(v) + # method 2: fileName -> className + matrix_file_class_pairs.extend((k, x) for x in v) + # method 1: reason -> test cases + ext_exclude_test_cases.extend(utils.filter_valid_test_cases(matrix_test_cases, ext_test_index)) + # method 2: fileName -> className + valid_matrix_file_class_pairs = utils.filter_valid_file_class_pairs(matrix_file_class_pairs, ext_test_index) + for valid_matrix_pair in valid_matrix_file_class_pairs: + ext_exclude_test_cases.extend(ext_test_index[valid_matrix_pair[0]][valid_matrix_pair[1]]) else: # method 1: matrix exclude key for k, v in ext_exclude.items(): diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py index 2bfebae47a2..2943988aae8 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py @@ -94,7 +94,7 @@ def main(): ext_test_index, ext_matrix, args.ext_coverage) ext_exclude_test_cases = ext.get_ext_exclude_test_cases(ext_test_index, ext_matrix, args.ext_filter) - ext_filtered_test_cases = ext.get_ext_filted_test_cases( + ext_filtered_test_cases = utils.get_filted_test_cases( ext_test_cases, ext_exclude_test_cases) # add prefix ext_qualified_test_cases = utils.decorate_qualified_prefix( diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/__init__.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/test.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/test.py new file mode 100644 index 00000000000..9923eec4237 --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/test.py @@ -0,0 +1,141 @@ +import os +import sys +import unittest + + +class UtilsTestCase(unittest.TestCase): + + def test_utils_check_file_existence(self): + s1 = check_file_existence(None) + s2 = check_file_existence(THIS_DIR) + s3 = check_file_existence(THIS_FILE) + s4 = check_file_existence(os.path.join(THIS_DIR, "testdata.json")) + self.assertEqual(s1, False) + self.assertEqual(s2, False) + self.assertEqual(s3, True) + self.assertEqual(s4, True) + + def test_utils_get_test_matrix(self): + s = get_test_matrix(os.path.join(THIS_DIR, "testdata.json")) + self.assertEqual(len(s), 2) # number of keys + + def test_utils_decorate_qualified_prefix(self): + test_cases = ["a", "b", "c"] + s = decorate_qualified_prefix(test_cases, "p") + self.assertEqual(s, ["p.a", "p.b", "p.c"]) + + def test_utils_extract_file_class_pairs(self): + test_cases = ["abc", "a.b", "a.b.c"] + s = extract_file_class_pairs(test_cases) + self.assertEqual(s, [("a", "b")]) + + def test_utils_filter_valid_file_class_pairs(self): + pairs = [("a", "b"), ("c", "d"), ("e", "f")] + test_index = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": { + "y": ["xxx", "yyy"] + } + } + s = filter_valid_file_class_pairs(pairs, test_index) + self.assertEqual(s, [("a", "b")]) + + def test_utils_get_all_values_from_nested_dict(self): + d = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": ["xxx"] + } + s = list(get_all_values_from_nested_dict(d)) + self.assertEqual(s, [["aaa", "bbb"], ["xxx"]]) + + def test_utils_flatten_nested_list(self): + lis = [["aaa", "bbb"], ["xxx"]] + s = list(flatten_nested_list(lis)) + self.assertEqual(s, ["aaa", "bbb", "xxx"]) + + def test_utils_filter_valid_test_cases(self): + test_cases = ["abc", "a.b", "a.b.c", "aaa", "yyy"] + test_index = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": { + "y": ["xxx", "yyy"] + } + } + s = filter_valid_test_cases(test_cases, test_index) + self.assertEqual(s, ["aaa", "yyy"]) + + def test_get_filted_test_cases(self): + test_cases = ["a", "b", "c", "d"] + s1 = get_filted_test_cases(test_cases, []) + t1 = ["a", "b", "c", "d"] + s2 = get_filted_test_cases(test_cases, ["a", "x"]) + t2 = ["b", "c", "d"] + self.assertEqual(s1, t1) + self.assertEqual(s2, t2) + + +class ExtTestCase(unittest.TestCase): + + def test_ext_get_ext_test_index(self): + pass + + def test_ext_get_ext_test_cases(self): + ext_test_index = { + "test_validators": { + "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], + "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], + "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] + } + } + ext_matrix = get_test_matrix(os.path.join(THIS_DIR, "testdata.json")) + base = ext_test_index["test_validators"] + s1 = get_ext_test_cases(ext_test_index, ext_matrix, []).sort() + t1 = (base["TestValidateIPRanges"] + + base["TestClusterAutoscalerParamsValidators"]).sort() + s2 = get_ext_test_cases(ext_test_index, ext_matrix, [ + "test_valid_vnet_subnet_id", "abc"]).sort() + t2 = (base["TestValidateIPRanges"] + base["TestClusterAutoscalerParamsValidators"] + + ["test_valid_vnet_subnet_id"]).sort() + s3 = get_ext_test_cases(ext_test_index, ext_matrix, [ + "test_validators.TestSubnetId"]).sort() + t3 = (base["TestValidateIPRanges"] + + base["TestClusterAutoscalerParamsValidators"] + base["TestSubnetId"]).sort() + self.assertEqual(s1, t1) + self.assertEqual(s2, t2) + self.assertEqual(s3, t3) + + def test_ext_get_ext_exclude_test_cases(self): + ext_test_index = { + "test_validators": { + "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], + "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], + "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] + } + } + ext_matrix = get_test_matrix(os.path.join(THIS_DIR, "testdata.json")) + base = ext_test_index["test_validators"] + s1 = get_ext_exclude_test_cases(ext_test_index, ext_matrix, []).sort() + t1 = ["test_local_ip_address", "test_invalid_ip", "test_IPv6"].sort() + s2 = get_ext_exclude_test_cases(ext_test_index, ext_matrix, [ + "test_valid_vnet_subnet_id", "abc", "test_validators.TestClusterAutoscalerParamsValidators"]).sort() + t2 = (["test_local_ip_address", "test_invalid_ip", "test_IPv6"] + + base["TestClusterAutoscalerParamsValidators"] + ["test_valid_vnet_subnet_id"]).sort() + self.assertEqual(s1, t1) + self.assertEqual(s2, t2) + + +if __name__ == '__main__': + THIS_FILE = os.path.abspath(__file__) + THIS_DIR = os.path.dirname(THIS_FILE) + PARENT_DIR = os.path.dirname(THIS_DIR) + sys.path.append(PARENT_DIR) + from utils import * + from ext import * + unittest.main() + sys.path.remove(PARENT_DIR) diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/testdata.json b/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/testdata.json new file mode 100644 index 00000000000..25d4d385b4a --- /dev/null +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/testdata.json @@ -0,0 +1,22 @@ +{ + "coverage": { + "test_validators": [ + "TestValidateIPRanges", + "TestClusterAutoscalerParamsValidators" + ] + }, + "exclude": { + "iprange": [ + "test_simultaneous_allow_and_disallow_with_spaces", + "test_simultaneous_enable_and_disable_with_spaces", + "test_disable_authorized_ip_ranges" + ], + "test_local_ip_address": [ + "test_aks_create_and_update_with_managed_aad_enable_azure_rbac", + "test_aks_create_with_virtual_node_addon" + ], + "test_validators": [ + "TestClusterAutoscalerParamsValidators" + ] + } +} \ No newline at end of file diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py index 9054d15504e..f8662123522 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py @@ -74,3 +74,9 @@ def filter_valid_test_cases(test_cases, test_index): else: print("Invalid test case: '{}'".format(test_case)) return valid_test_cases + + +def get_filted_test_cases(test_cases, exclude_test_cases): + filtered_test_cases = [ + x for x in test_cases if x not in exclude_test_cases] + return filtered_test_cases From edafbe9a7e88ce027d24dc69b1c6a01a27287c0d Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Fri, 7 May 2021 09:58:38 +0800 Subject: [PATCH 20/42] add license --- .../az_aks_tool/{test => tests}/__init__.py | 0 .../azcli-aks-live-test/az_aks_tool/{test => tests}/test.py | 5 +++++ .../az_aks_tool/{test => tests}/testdata.json | 0 3 files changed, 5 insertions(+) rename src/aks-preview/azcli-aks-live-test/az_aks_tool/{test => tests}/__init__.py (100%) rename src/aks-preview/azcli-aks-live-test/az_aks_tool/{test => tests}/test.py (94%) rename src/aks-preview/azcli-aks-live-test/az_aks_tool/{test => tests}/testdata.json (100%) diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/__init__.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/__init__.py similarity index 100% rename from src/aks-preview/azcli-aks-live-test/az_aks_tool/test/__init__.py rename to src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/__init__.py diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/test.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/test.py similarity index 94% rename from src/aks-preview/azcli-aks-live-test/az_aks_tool/test/test.py rename to src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/test.py index 9923eec4237..6a4f1e036fd 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/test.py +++ b/src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/test.py @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + import os import sys import unittest diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/test/testdata.json b/src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/testdata.json similarity index 100% rename from src/aks-preview/azcli-aks-live-test/az_aks_tool/test/testdata.json rename to src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/testdata.json From e217974b9a0e1c7610e6e1c816382bda9c7332cb Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 13:29:52 +0800 Subject: [PATCH 21/42] * refine unit test & add coverage report for ext * fix import path --- .../az_aks_tool/tests/test.py | 146 ------------------ .../.gitignore | 0 .../Dockerfile | 0 .../HISTORY.rst | 0 .../README.md | 0 .../__init__.py | 0 .../az_aks_tool}/__init__.py | 0 .../az_aks_tool/ext.py | 37 ++--- .../azcli_aks_live_test/az_aks_tool/log.py | 26 ++++ .../az_aks_tool/main.py | 34 ++-- .../az_aks_tool/tests/__init__.py | 0 .../az_aks_tool/tests/test_ext.py | 77 +++++++++ .../az_aks_tool/tests/test_log.py | 28 ++++ .../az_aks_tool/tests/test_main.py | 21 +++ .../az_aks_tool/tests/test_utils.py | 91 +++++++++++ .../az_aks_tool/tests/testdata.json | 4 - .../az_aks_tool/utils.py | 10 +- .../clean_up.sh | 0 .../clone_repo.sh | 0 .../ext_matrix_default.json | 0 .../prepare_image.sh | 0 .../setup_venv.sh | 3 + .../start_container.sh | 0 .../test_cli.sh | 0 .../test_ext.sh | 36 ++++- .../transcribe_env.sh | 0 .../vsts-azcli-aks-live-test.yaml | 0 27 files changed, 326 insertions(+), 187 deletions(-) delete mode 100644 src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/test.py rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/.gitignore (100%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/Dockerfile (100%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/HISTORY.rst (100%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/README.md (100%) rename src/aks-preview/{azcli-aks-live-test/az_aks_tool => azcli_aks_live_test}/__init__.py (100%) rename src/aks-preview/{azcli-aks-live-test/az_aks_tool/tests => azcli_aks_live_test/az_aks_tool}/__init__.py (100%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/az_aks_tool/ext.py (80%) create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/az_aks_tool/main.py (83%) create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/__init__.py create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_ext.py create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_log.py create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_main.py create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_utils.py rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/az_aks_tool/tests/testdata.json (72%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/az_aks_tool/utils.py (87%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/clean_up.sh (100%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/clone_repo.sh (100%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/ext_matrix_default.json (100%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/prepare_image.sh (100%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/setup_venv.sh (88%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/start_container.sh (100%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/test_cli.sh (100%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/test_ext.sh (62%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/transcribe_env.sh (100%) rename src/aks-preview/{azcli-aks-live-test => azcli_aks_live_test}/vsts-azcli-aks-live-test.yaml (100%) diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/test.py b/src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/test.py deleted file mode 100644 index 6a4f1e036fd..00000000000 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/test.py +++ /dev/null @@ -1,146 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -import os -import sys -import unittest - - -class UtilsTestCase(unittest.TestCase): - - def test_utils_check_file_existence(self): - s1 = check_file_existence(None) - s2 = check_file_existence(THIS_DIR) - s3 = check_file_existence(THIS_FILE) - s4 = check_file_existence(os.path.join(THIS_DIR, "testdata.json")) - self.assertEqual(s1, False) - self.assertEqual(s2, False) - self.assertEqual(s3, True) - self.assertEqual(s4, True) - - def test_utils_get_test_matrix(self): - s = get_test_matrix(os.path.join(THIS_DIR, "testdata.json")) - self.assertEqual(len(s), 2) # number of keys - - def test_utils_decorate_qualified_prefix(self): - test_cases = ["a", "b", "c"] - s = decorate_qualified_prefix(test_cases, "p") - self.assertEqual(s, ["p.a", "p.b", "p.c"]) - - def test_utils_extract_file_class_pairs(self): - test_cases = ["abc", "a.b", "a.b.c"] - s = extract_file_class_pairs(test_cases) - self.assertEqual(s, [("a", "b")]) - - def test_utils_filter_valid_file_class_pairs(self): - pairs = [("a", "b"), ("c", "d"), ("e", "f")] - test_index = { - "a": { - "b": ["aaa", "bbb"] - }, - "x": { - "y": ["xxx", "yyy"] - } - } - s = filter_valid_file_class_pairs(pairs, test_index) - self.assertEqual(s, [("a", "b")]) - - def test_utils_get_all_values_from_nested_dict(self): - d = { - "a": { - "b": ["aaa", "bbb"] - }, - "x": ["xxx"] - } - s = list(get_all_values_from_nested_dict(d)) - self.assertEqual(s, [["aaa", "bbb"], ["xxx"]]) - - def test_utils_flatten_nested_list(self): - lis = [["aaa", "bbb"], ["xxx"]] - s = list(flatten_nested_list(lis)) - self.assertEqual(s, ["aaa", "bbb", "xxx"]) - - def test_utils_filter_valid_test_cases(self): - test_cases = ["abc", "a.b", "a.b.c", "aaa", "yyy"] - test_index = { - "a": { - "b": ["aaa", "bbb"] - }, - "x": { - "y": ["xxx", "yyy"] - } - } - s = filter_valid_test_cases(test_cases, test_index) - self.assertEqual(s, ["aaa", "yyy"]) - - def test_get_filted_test_cases(self): - test_cases = ["a", "b", "c", "d"] - s1 = get_filted_test_cases(test_cases, []) - t1 = ["a", "b", "c", "d"] - s2 = get_filted_test_cases(test_cases, ["a", "x"]) - t2 = ["b", "c", "d"] - self.assertEqual(s1, t1) - self.assertEqual(s2, t2) - - -class ExtTestCase(unittest.TestCase): - - def test_ext_get_ext_test_index(self): - pass - - def test_ext_get_ext_test_cases(self): - ext_test_index = { - "test_validators": { - "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], - "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], - "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] - } - } - ext_matrix = get_test_matrix(os.path.join(THIS_DIR, "testdata.json")) - base = ext_test_index["test_validators"] - s1 = get_ext_test_cases(ext_test_index, ext_matrix, []).sort() - t1 = (base["TestValidateIPRanges"] + - base["TestClusterAutoscalerParamsValidators"]).sort() - s2 = get_ext_test_cases(ext_test_index, ext_matrix, [ - "test_valid_vnet_subnet_id", "abc"]).sort() - t2 = (base["TestValidateIPRanges"] + base["TestClusterAutoscalerParamsValidators"] + - ["test_valid_vnet_subnet_id"]).sort() - s3 = get_ext_test_cases(ext_test_index, ext_matrix, [ - "test_validators.TestSubnetId"]).sort() - t3 = (base["TestValidateIPRanges"] + - base["TestClusterAutoscalerParamsValidators"] + base["TestSubnetId"]).sort() - self.assertEqual(s1, t1) - self.assertEqual(s2, t2) - self.assertEqual(s3, t3) - - def test_ext_get_ext_exclude_test_cases(self): - ext_test_index = { - "test_validators": { - "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], - "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], - "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] - } - } - ext_matrix = get_test_matrix(os.path.join(THIS_DIR, "testdata.json")) - base = ext_test_index["test_validators"] - s1 = get_ext_exclude_test_cases(ext_test_index, ext_matrix, []).sort() - t1 = ["test_local_ip_address", "test_invalid_ip", "test_IPv6"].sort() - s2 = get_ext_exclude_test_cases(ext_test_index, ext_matrix, [ - "test_valid_vnet_subnet_id", "abc", "test_validators.TestClusterAutoscalerParamsValidators"]).sort() - t2 = (["test_local_ip_address", "test_invalid_ip", "test_IPv6"] + - base["TestClusterAutoscalerParamsValidators"] + ["test_valid_vnet_subnet_id"]).sort() - self.assertEqual(s1, t1) - self.assertEqual(s2, t2) - - -if __name__ == '__main__': - THIS_FILE = os.path.abspath(__file__) - THIS_DIR = os.path.dirname(THIS_FILE) - PARENT_DIR = os.path.dirname(THIS_DIR) - sys.path.append(PARENT_DIR) - from utils import * - from ext import * - unittest.main() - sys.path.remove(PARENT_DIR) diff --git a/src/aks-preview/azcli-aks-live-test/.gitignore b/src/aks-preview/azcli_aks_live_test/.gitignore similarity index 100% rename from src/aks-preview/azcli-aks-live-test/.gitignore rename to src/aks-preview/azcli_aks_live_test/.gitignore diff --git a/src/aks-preview/azcli-aks-live-test/Dockerfile b/src/aks-preview/azcli_aks_live_test/Dockerfile similarity index 100% rename from src/aks-preview/azcli-aks-live-test/Dockerfile rename to src/aks-preview/azcli_aks_live_test/Dockerfile diff --git a/src/aks-preview/azcli-aks-live-test/HISTORY.rst b/src/aks-preview/azcli_aks_live_test/HISTORY.rst similarity index 100% rename from src/aks-preview/azcli-aks-live-test/HISTORY.rst rename to src/aks-preview/azcli_aks_live_test/HISTORY.rst diff --git a/src/aks-preview/azcli-aks-live-test/README.md b/src/aks-preview/azcli_aks_live_test/README.md similarity index 100% rename from src/aks-preview/azcli-aks-live-test/README.md rename to src/aks-preview/azcli_aks_live_test/README.md diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/__init__.py b/src/aks-preview/azcli_aks_live_test/__init__.py similarity index 100% rename from src/aks-preview/azcli-aks-live-test/az_aks_tool/__init__.py rename to src/aks-preview/azcli_aks_live_test/__init__.py diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/__init__.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/__init__.py similarity index 100% rename from src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/__init__.py rename to src/aks-preview/azcli_aks_live_test/az_aks_tool/__init__.py diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/ext.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/ext.py similarity index 80% rename from src/aks-preview/azcli-aks-live-test/az_aks_tool/ext.py rename to src/aks-preview/azcli_aks_live_test/az_aks_tool/ext.py index 6ee04394fa4..886b8bdb461 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/ext.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/ext.py @@ -5,20 +5,21 @@ import glob import os +import logging from azdev.operations.testtool import _discover_module_tests from azdev.utilities import EXTENSION_PREFIX, get_path_table, get_name_index -import utils +import azcli_aks_live_test.az_aks_tool.utils as utils +logger = logging.getLogger(__name__) def get_ext_test_index(mod_name): - # path table & name index + # key value pairs of all modules(in azcli & extention) and its absolute path, used later to find test indexes path_table = get_path_table() - command_modules = path_table['mod'] extensions = path_table["ext"] inverse_name_table = get_name_index(invert=True) - # import_name & mod_data + # construct 'import_name' & mod_data', used later to find test indexes aks_preview_mod_path = extensions[mod_name] glob_pattern = os.path.normcase( os.path.join("{}*".format(EXTENSION_PREFIX))) @@ -31,7 +32,7 @@ def get_ext_test_index(mod_name): "files": {} } - # azdev hook + # use azdev internal func '_discover_module_tests' to find test index ext_test = _discover_module_tests(import_name, mod_data) ext_test_index = ext_test["files"] return ext_test_index @@ -40,10 +41,11 @@ def get_ext_test_index(mod_name): def get_ext_test_cases(ext_test_index, ext_matrix, ext_extra_coverage): ext_test_cases = [] ext_coverage = ext_matrix["coverage"] + # default coverage for fileName, className in ext_coverage.items(): for c in className: ext_test_cases.extend(ext_test_index[fileName][c]) - # extra coverage + # custom extra coverage if ext_extra_coverage: # method 1: fileName.className file_class_pairs = utils.extract_file_class_pairs(ext_extra_coverage) @@ -61,7 +63,8 @@ def get_ext_test_cases(ext_test_index, ext_matrix, ext_extra_coverage): def get_ext_exclude_test_cases(ext_test_index, ext_matrix, ext_filter): ext_exclude_test_cases = [] ext_exclude = ext_matrix["exclude"] - if not ext_filter: + # default exclude + if not ext_filter or "default" in ext_filter: matrix_test_cases = [] matrix_file_class_pairs = [] for k, v in ext_exclude.items(): @@ -70,15 +73,19 @@ def get_ext_exclude_test_cases(ext_test_index, ext_matrix, ext_filter): # method 2: fileName -> className matrix_file_class_pairs.extend((k, x) for x in v) # method 1: reason -> test cases - ext_exclude_test_cases.extend(utils.filter_valid_test_cases(matrix_test_cases, ext_test_index)) + ext_exclude_test_cases.extend( + utils.filter_valid_test_cases(matrix_test_cases, ext_test_index)) # method 2: fileName -> className - valid_matrix_file_class_pairs = utils.filter_valid_file_class_pairs(matrix_file_class_pairs, ext_test_index) + valid_matrix_file_class_pairs = utils.filter_valid_file_class_pairs( + matrix_file_class_pairs, ext_test_index) for valid_matrix_pair in valid_matrix_file_class_pairs: - ext_exclude_test_cases.extend(ext_test_index[valid_matrix_pair[0]][valid_matrix_pair[1]]) - else: + ext_exclude_test_cases.extend( + ext_test_index[valid_matrix_pair[0]][valid_matrix_pair[1]]) + # custom filter + if ext_filter: # method 1: matrix exclude key for k, v in ext_exclude.items(): - if k in ext_filter or "default" in ext_filter: + if k in ext_filter: ext_exclude_test_cases.extend(v) # method 2: fileName.className file_class_pairs = utils.extract_file_class_pairs(ext_filter) @@ -91,9 +98,3 @@ def get_ext_exclude_test_cases(ext_test_index, ext_matrix, ext_filter): ext_exclude_test_cases.extend( utils.filter_valid_test_cases(ext_filter, ext_test_index)) return list(set(ext_exclude_test_cases)) - - -def get_ext_filted_test_cases(ext_test_cases, ext_exclude_test_cases): - ext_filtered_test_cases = [ - x for x in ext_test_cases if x not in ext_exclude_test_cases] - return ext_filtered_test_cases diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py new file mode 100644 index 00000000000..12fdfd2490e --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py @@ -0,0 +1,26 @@ +import logging +import sys + +def setup_logging(root_logger_name="", log_path="az_aks_tool.log"): + if root_logger_name == "": + try: + root_logger_name = __name__.split(".")[0] + except Exception as e: + print("Failed to parse module name from '{}', error: {}".format(__name__, e)) + logger = logging.getLogger(root_logger_name) + logger.setLevel(level=logging.DEBUG) + + # Formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # FileHandler + file_handler = logging.FileHandler(filename=log_path, mode="w") + file_handler.setFormatter(formatter) + file_handler.setLevel(level=logging.DEBUG) + logger.addHandler(file_handler) + + # StreamHandler + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setFormatter(formatter) + stream_handler.setLevel(level=logging.INFO) + logger.addHandler(stream_handler) diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py similarity index 83% rename from src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py rename to src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py index 2943988aae8..bce844a2f5a 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/main.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py @@ -6,17 +6,19 @@ import argparse import os import sys +import logging from azdev.operations.testtool import run_tests from azdev.utilities import EXTENSION_PREFIX -import ext -import utils +import azcli_aks_live_test.az_aks_tool.utils as utils +import azcli_aks_live_test.az_aks_tool.ext as ext +from azcli_aks_live_test.az_aks_tool.log import setup_logging # const AKS_PREVIEW_MOD_NAME = EXTENSION_PREFIX + "aks_preview" # azext_aks_preview -def init_argparse(): +def init_argparse(args): parser = argparse.ArgumentParser() parser.add_argument("-t", "--tests", nargs='+', help="test case names") parser.add_argument("-cm", "--cli-matrix", type=str, @@ -53,14 +55,19 @@ def init_argparse(): default="sys", help="test capture") # parser.add_argument("-a", "--pytest-args", # nargs=argparse.REMAINDER, help="pytest args") - args = parser.parse_args() + args = parser.parse_args(args) return args def main(): - print("args: {}".format(sys.argv)) - args = init_argparse() + # setup logger + setup_logging() + logger = logging.getLogger(__name__) + # parse args + logger.info("raw args: {}".format(sys.argv)) + args = init_argparse(sys.argv[1:]) + # test cases test_cases = args.tests ext_matrix_file_path = args.ext_matrix @@ -72,7 +79,7 @@ def main(): # report file report_file_full_path = os.path.realpath(os.path.join( args.json_report_path, args.json_report_file)) - print("report file full path: {}".format(report_file_full_path)) + logger.info("report file full path: {}".format(report_file_full_path)) # pytest args pytest_args = [] @@ -83,8 +90,7 @@ def main(): pytest_args.append("--reruns {}".format(args.reruns)) pytest_args.append("--capture {}".format(args.capture)) pytest_args = [" ".join(pytest_args)] - print("pytest_args: {}".format(pytest_args)) - print() + logger.info("pytest_args: {}".format(pytest_args)) # ext matrix if utils.check_file_existence(ext_matrix_file_path): @@ -99,21 +105,21 @@ def main(): # add prefix ext_qualified_test_cases = utils.decorate_qualified_prefix( ext_filtered_test_cases, AKS_PREVIEW_MOD_NAME) - print("According to 'ext_matrix' and filters, we get {} cases, need to exclude {} cases, finally get {} cases!".format( + logger.info("According to 'ext_matrix' and filters, we get {} cases, need to exclude {} cases, finally get {} cases!".format( len(ext_test_cases), len(ext_exclude_test_cases), len(ext_filtered_test_cases))) - print("Perform following tests: {}".format(ext_qualified_test_cases)) + logger.info("Perform following tests: {}".format(ext_qualified_test_cases)) run_tests(ext_qualified_test_cases, xml_path=args.xml_path, discover=args.discover, in_series=args.series, run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) # cli matrix if utils.check_file_existence(cli_matrix_file_path): - print("Currently not support!") + logger.warning("Currently not support!") pass # tests if test_cases: - print("Accroding to 'tests', we get {} cases".format(len(test_cases))) - print("Perform following tets: {}".format(test_cases)) + logger.info("Accroding to 'tests', we get {} cases".format(len(test_cases))) + logger.info("Perform following tets: {}".format(test_cases)) run_tests(test_cases, xml_path=args.xml_path, discover=args.discover, in_series=args.series, run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/__init__.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_ext.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_ext.py new file mode 100644 index 00000000000..7adcbffe3d2 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_ext.py @@ -0,0 +1,77 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import azcli_aks_live_test.az_aks_tool.utils as utils +import azcli_aks_live_test.az_aks_tool.ext as ext + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + +class ExtTestCase(unittest.TestCase): + + def test_ext_get_ext_test_index(self): + pass + + def test_ext_get_ext_test_cases(self): + ext_test_index = { + "test_validators": { + "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], + "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], + "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] + } + } + ext_matrix = utils.get_test_matrix( + os.path.join(THIS_DIR, "testdata.json")) + base = ext_test_index["test_validators"] + s1 = sorted(ext.get_ext_test_cases(ext_test_index, ext_matrix, [])) + t1 = sorted((base["TestValidateIPRanges"] + + base["TestClusterAutoscalerParamsValidators"])) + s2 = sorted(ext.get_ext_test_cases(ext_test_index, ext_matrix, [ + "test_valid_vnet_subnet_id", "abc"])) + t2 = sorted((base["TestValidateIPRanges"] + base["TestClusterAutoscalerParamsValidators"] + + ["test_valid_vnet_subnet_id"])) + s3 = sorted(ext.get_ext_test_cases(ext_test_index, ext_matrix, [ + "test_validators.TestSubnetId"])) + t3 = sorted((base["TestValidateIPRanges"] + + base["TestClusterAutoscalerParamsValidators"] + base["TestSubnetId"])) + self.assertEqual(s1, t1) + self.assertEqual(s2, t2) + self.assertEqual(s3, t3) + + def test_ext_get_ext_exclude_test_cases(self): + ext_test_index = { + "test_validators": { + "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], + "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], + "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] + } + } + ext_matrix = utils.get_test_matrix( + os.path.join(THIS_DIR, "testdata.json")) + base = ext_test_index["test_validators"] + s1 = sorted(ext.get_ext_exclude_test_cases( + ext_test_index, ext_matrix, [])) + t1 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", + "test_disable_authorized_ip_ranges"] + base["TestClusterAutoscalerParamsValidators"]) + s2 = sorted(ext.get_ext_exclude_test_cases(ext_test_index, + ext_matrix, ["iprange"])) + t2 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", + "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges"]) + s3 = sorted(ext.get_ext_exclude_test_cases(ext_test_index, + ext_matrix, ["default", "test_invalid_subnet_id"])) + t3 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", + "test_disable_authorized_ip_ranges"] + base["TestClusterAutoscalerParamsValidators"] + ["test_invalid_subnet_id"]) + s4 = sorted(ext.get_ext_exclude_test_cases(ext_test_index, ext_matrix, [ + "test_valid_vnet_subnet_id", "abc", "test_validators.TestClusterAutoscalerParamsValidators"])) + t4 = sorted( + (base["TestClusterAutoscalerParamsValidators"] + ["test_valid_vnet_subnet_id"])) + self.assertEqual(s1, t1) + self.assertEqual(s2, t2) + self.assertEqual(s3, t3) + self.assertEqual(s4, t4) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_log.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_log.py new file mode 100644 index 00000000000..70706b0e854 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_log.py @@ -0,0 +1,28 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import logging +import os +import unittest + +import azcli_aks_live_test.az_aks_tool.log as log + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + + +class LogTestCase(unittest.TestCase): + + def test_setup_logging(self): + log.setup_logging("unittest", "unittest_log.log") + logger = logging.getLogger("unittest.test_setup_logging") + logger.debug("test setup logging") + logger.info("test setup logging") + logger.warning("test setup logging") + f = open("unittest_log.log", "r") + raw_logs = f.readlines() + f.close() + self.assertEqual(len(raw_logs), 3) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_main.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_main.py new file mode 100644 index 00000000000..5df317f1277 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_main.py @@ -0,0 +1,21 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import logging +import os +import unittest + +import azcli_aks_live_test.az_aks_tool.main as main + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + + +class MainTestCase(unittest.TestCase): + + def test_init_argparse(self): + args = main.init_argparse(["-p", "./"]) + self.assertEqual(args.json_report_path, "./") diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_utils.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_utils.py new file mode 100644 index 00000000000..e96be8a650e --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_utils.py @@ -0,0 +1,91 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import azcli_aks_live_test.az_aks_tool.utils as utils + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + + +class UtilsTestCase(unittest.TestCase): + + def test_utils_check_file_existence(self): + s1 = utils.check_file_existence(None) + s2 = utils.check_file_existence(THIS_DIR) + s3 = utils.check_file_existence(THIS_FILE) + s4 = utils.check_file_existence( + os.path.join(THIS_DIR, "testdata.json")) + self.assertEqual(s1, False) + self.assertEqual(s2, False) + self.assertEqual(s3, True) + self.assertEqual(s4, True) + + def test_utils_get_test_matrix(self): + s = utils.get_test_matrix(os.path.join(THIS_DIR, "testdata.json")) + self.assertEqual(len(s), 2) # number of keys + + def test_utils_decorate_qualified_prefix(self): + test_cases = ["a", "b", "c"] + s = utils.decorate_qualified_prefix(test_cases, "p") + self.assertEqual(s, ["p.a", "p.b", "p.c"]) + + def test_utils_extract_file_class_pairs(self): + test_cases = ["abc", "a.b", "a.b.c"] + s = utils.extract_file_class_pairs(test_cases) + self.assertEqual(s, [("a", "b")]) + + def test_utils_filter_valid_file_class_pairs(self): + pairs = [("a", "b"), ("c", "d"), ("e", "f")] + test_index = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": { + "y": ["xxx", "yyy"] + } + } + s = utils.filter_valid_file_class_pairs(pairs, test_index) + self.assertEqual(s, [("a", "b")]) + + def test_utils_get_all_values_from_nested_dict(self): + d = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": ["xxx"] + } + s = list(utils.get_all_values_from_nested_dict(d)) + self.assertEqual(s, [["aaa", "bbb"], ["xxx"]]) + + def test_utils_flatten_nested_list(self): + lis = [["aaa", "bbb"], ["xxx"]] + s = list(utils.flatten_nested_list(lis)) + self.assertEqual(s, ["aaa", "bbb", "xxx"]) + + def test_utils_filter_valid_test_cases(self): + test_cases = ["abc", "a.b", "a.b.c", "aaa", "yyy"] + test_index = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": { + "y": ["xxx", "yyy"] + } + } + s = utils.filter_valid_test_cases(test_cases, test_index) + self.assertEqual(s, ["aaa", "yyy"]) + + def test_get_filted_test_cases(self): + test_cases = ["a", "b", "c", "d"] + s1 = utils.get_filted_test_cases(test_cases, []) + t1 = ["a", "b", "c", "d"] + s2 = utils.get_filted_test_cases(test_cases, ["a", "x"]) + t2 = ["b", "c", "d"] + self.assertEqual(s1, t1) + self.assertEqual(s2, t2) diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/testdata.json b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/testdata.json similarity index 72% rename from src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/testdata.json rename to src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/testdata.json index 25d4d385b4a..221e6291e3e 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/tests/testdata.json +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/testdata.json @@ -11,10 +11,6 @@ "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges" ], - "test_local_ip_address": [ - "test_aks_create_and_update_with_managed_aad_enable_azure_rbac", - "test_aks_create_with_virtual_node_addon" - ], "test_validators": [ "TestClusterAutoscalerParamsValidators" ] diff --git a/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py similarity index 87% rename from src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py rename to src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py index f8662123522..a7bee249e21 100644 --- a/src/aks-preview/azcli-aks-live-test/az_aks_tool/utils.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py @@ -6,6 +6,8 @@ from collections import Iterable import json import os +import logging +logger = logging.getLogger(__name__) def check_file_existence(file_path): @@ -40,9 +42,9 @@ def filter_valid_file_class_pairs(pairs, test_index): for pair in pairs: if pair[0] in test_index and pair[1] in test_index[pair[0]]: valid_pairs.append(pair) - print("Find valid file & class pair: '{}'".format(pair)) + logger.info("Valid file & class pair: '{}'".format(pair)) else: - print("Invalid file & class pair: '{}'".format(pair)) + logger.info("Invalid file & class pair: '{}'".format(pair)) return valid_pairs @@ -70,9 +72,9 @@ def filter_valid_test_cases(test_cases, test_index): for test_case in test_cases: if test_case in falttened_test_cases: valid_test_cases.append(test_case) - print("Find valid test case: '{}'".format(test_case)) + logger.info("Valid test case: '{}'".format(test_case)) else: - print("Invalid test case: '{}'".format(test_case)) + logger.info("Invalid test case: '{}'".format(test_case)) return valid_test_cases diff --git a/src/aks-preview/azcli-aks-live-test/clean_up.sh b/src/aks-preview/azcli_aks_live_test/clean_up.sh similarity index 100% rename from src/aks-preview/azcli-aks-live-test/clean_up.sh rename to src/aks-preview/azcli_aks_live_test/clean_up.sh diff --git a/src/aks-preview/azcli-aks-live-test/clone_repo.sh b/src/aks-preview/azcli_aks_live_test/clone_repo.sh similarity index 100% rename from src/aks-preview/azcli-aks-live-test/clone_repo.sh rename to src/aks-preview/azcli_aks_live_test/clone_repo.sh diff --git a/src/aks-preview/azcli-aks-live-test/ext_matrix_default.json b/src/aks-preview/azcli_aks_live_test/ext_matrix_default.json similarity index 100% rename from src/aks-preview/azcli-aks-live-test/ext_matrix_default.json rename to src/aks-preview/azcli_aks_live_test/ext_matrix_default.json diff --git a/src/aks-preview/azcli-aks-live-test/prepare_image.sh b/src/aks-preview/azcli_aks_live_test/prepare_image.sh similarity index 100% rename from src/aks-preview/azcli-aks-live-test/prepare_image.sh rename to src/aks-preview/azcli_aks_live_test/prepare_image.sh diff --git a/src/aks-preview/azcli-aks-live-test/setup_venv.sh b/src/aks-preview/azcli_aks_live_test/setup_venv.sh similarity index 88% rename from src/aks-preview/azcli-aks-live-test/setup_venv.sh rename to src/aks-preview/azcli_aks_live_test/setup_venv.sh index 5f804ecdc0b..5b5d3c5f3e1 100755 --- a/src/aks-preview/azcli-aks-live-test/setup_venv.sh +++ b/src/aks-preview/azcli_aks_live_test/setup_venv.sh @@ -13,8 +13,11 @@ source azEnv/bin/activate python -m pip install -U pip # fixed azdev version to avoid call failure in az_aks_tool pip install azdev==0.1.32 +# install pytest plugins pip install pytest-json-report pytest-rerunfailures --upgrade # pip install pytest-html --upgrade +# module for measuring code coverage +pip install coverage # check existing az which az || az version || az extension list || true diff --git a/src/aks-preview/azcli-aks-live-test/start_container.sh b/src/aks-preview/azcli_aks_live_test/start_container.sh similarity index 100% rename from src/aks-preview/azcli-aks-live-test/start_container.sh rename to src/aks-preview/azcli_aks_live_test/start_container.sh diff --git a/src/aks-preview/azcli-aks-live-test/test_cli.sh b/src/aks-preview/azcli_aks_live_test/test_cli.sh similarity index 100% rename from src/aks-preview/azcli-aks-live-test/test_cli.sh rename to src/aks-preview/azcli_aks_live_test/test_cli.sh diff --git a/src/aks-preview/azcli-aks-live-test/test_ext.sh b/src/aks-preview/azcli_aks_live_test/test_ext.sh similarity index 62% rename from src/aks-preview/azcli-aks-live-test/test_ext.sh rename to src/aks-preview/azcli_aks_live_test/test_ext.sh index 3334ee9b67d..9d159c3d63d 100755 --- a/src/aks-preview/azcli-aks-live-test/test_ext.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext.sh @@ -35,6 +35,39 @@ do source azEnv/bin/activate done +# unit test & coverage report +# azcli_aks_live_test +azcli_aks_live_test_unit_test_result="" +pushd azure-cli-extensions/src/aks-preview/azcli_aks_live_test/ +# clean existing coverage report +(coverage combine || true) && (coverage erase || true) +if ! coverage run --source=. --omit=*/tests/* -p -m unittest ; then + azcli_aks_live_test_unit_test_result="error" +fi +# currently no test written in pytest format under 'azcli_aks_live_test/' +# coverage run --source=. --omit=*/tests/* -p -m pytest +coverage combine && coverage json -o coverage_azcli_aks_live_test.json +coverage report -m +popd + +# azext_aks_preview +azext_aks_preview_unit_test_result="" +pushd azure-cli-extensions/src/aks-preview/azext_aks_preview +# clean existing coverage report +(coverage combine || true) && (coverage erase || true) +# currently test using module 'unittest' is the same as module 'pytest', and test using 'pytest' is just recording test +if ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m unittest || coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m pytest; then + azext_aks_preview_unit_test_result="error" +fi +coverage combine && coverage json -o coverage_azext_aks_preview.json +coverage report -m +popd + +if [[ $azcli_aks_live_test_unit_test_result == "error" || $azext_aks_preview_unit_test_result == "error" ]]; then + echo "Unit test failed!" + exit 1 +fi + # prepare run flags run_flags="-em ext_matrix_default.json --no-exitfirst --discover --json-report-path ./ --reruns 3 --capture=sys" # parallel @@ -52,7 +85,7 @@ if [[ $EXT_TEST_COVERAGE ]]; then run_flags+=" -ec $EXT_TEST_COVERAGE" fi -# test ext +# recording test if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then echo "Test in record mode!" run_flags+=" --json-report-file=ext_report.json" @@ -60,6 +93,7 @@ if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then echo ${run_flags} | xargs python -u az_aks_tool/main.py fi +# live test if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then echo "Test in live mode!" az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID diff --git a/src/aks-preview/azcli-aks-live-test/transcribe_env.sh b/src/aks-preview/azcli_aks_live_test/transcribe_env.sh similarity index 100% rename from src/aks-preview/azcli-aks-live-test/transcribe_env.sh rename to src/aks-preview/azcli_aks_live_test/transcribe_env.sh diff --git a/src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml similarity index 100% rename from src/aks-preview/azcli-aks-live-test/vsts-azcli-aks-live-test.yaml rename to src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml From a23b397801532248e06890ad8a6d872b528622cb Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 13:57:45 +0800 Subject: [PATCH 22/42] refine report upload --- .../azcli_aks_live_test/az_aks_tool/main.py | 22 +++++++------ .../az_aks_tool/tests/test_main.py | 2 +- .../azcli_aks_live_test/setup_venv.sh | 11 ++++--- .../azcli_aks_live_test/test_cli.sh | 6 ++-- .../azcli_aks_live_test/test_ext.sh | 10 ++++-- .../vsts-azcli-aks-live-test.yaml | 32 ++++--------------- 6 files changed, 38 insertions(+), 45 deletions(-) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py index bce844a2f5a..94d98f2e0c2 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py @@ -41,12 +41,12 @@ def init_argparse(args): default=False, help="discover test index") parser.add_argument("--no-exitfirst", action="store_true", default=False, help="no exit first") - parser.add_argument("--xml-path", type=str, - default="azcli_aks_runner.xml", help="junit log path") + parser.add_argument("--xml-file", type=str, + default="azcli_aks_runner.xml", help="junit/xml report filename") parser.add_argument("-n", "--parallelism", type=str, default="8", help="test parallelism") - parser.add_argument("-p", "--json-report-path", type=str, - required=True, help="json report path") + parser.add_argument("-p", "--report-path", type=str, + required=True, help="report path") parser.add_argument("-f", "--json-report-file", type=str, default="azcli_aks_runner_report.json", help="json report filename") parser.add_argument("-r", "--reruns", type=str, @@ -77,16 +77,18 @@ def main(): "At least one of 'tests', 'cli_matrix' and 'ext_matrix' must be provided!") # report file - report_file_full_path = os.path.realpath(os.path.join( - args.json_report_path, args.json_report_file)) - logger.info("report file full path: {}".format(report_file_full_path)) + json_report_file_full_path = os.path.realpath(os.path.join( + args.report_path, args.json_report_file)) + logger.info("json report file full path: {}".format(json_report_file_full_path)) + xml_path = os.path.realpath(os.path.join(args.report_path, args.xml_file)) + logger.info("junit/xml report file full path: {}".format(xml_path)) # pytest args pytest_args = [] if not args.series and args.parallelism: pytest_args.append("-n {}".format(args.parallelism)) pytest_args.append("--json-report") - pytest_args.append("--json-report-file {}".format(report_file_full_path)) + pytest_args.append("--json-report-file {}".format(json_report_file_full_path)) pytest_args.append("--reruns {}".format(args.reruns)) pytest_args.append("--capture {}".format(args.capture)) pytest_args = [" ".join(pytest_args)] @@ -108,7 +110,7 @@ def main(): logger.info("According to 'ext_matrix' and filters, we get {} cases, need to exclude {} cases, finally get {} cases!".format( len(ext_test_cases), len(ext_exclude_test_cases), len(ext_filtered_test_cases))) logger.info("Perform following tests: {}".format(ext_qualified_test_cases)) - run_tests(ext_qualified_test_cases, xml_path=args.xml_path, discover=args.discover, in_series=args.series, + run_tests(ext_qualified_test_cases, xml_path=xml_path, discover=args.discover, in_series=args.series, run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) # cli matrix @@ -120,7 +122,7 @@ def main(): if test_cases: logger.info("Accroding to 'tests', we get {} cases".format(len(test_cases))) logger.info("Perform following tets: {}".format(test_cases)) - run_tests(test_cases, xml_path=args.xml_path, discover=args.discover, in_series=args.series, + run_tests(test_cases, xml_path=xml_path, discover=args.discover, in_series=args.series, run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_main.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_main.py index 5df317f1277..50ba6386643 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_main.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_main.py @@ -18,4 +18,4 @@ class MainTestCase(unittest.TestCase): def test_init_argparse(self): args = main.init_argparse(["-p", "./"]) - self.assertEqual(args.json_report_path, "./") + self.assertEqual(args.report_path, "./") diff --git a/src/aks-preview/azcli_aks_live_test/setup_venv.sh b/src/aks-preview/azcli_aks_live_test/setup_venv.sh index 5b5d3c5f3e1..d84d1c88be9 100755 --- a/src/aks-preview/azcli_aks_live_test/setup_venv.sh +++ b/src/aks-preview/azcli_aks_live_test/setup_venv.sh @@ -19,14 +19,17 @@ pip install pytest-json-report pytest-rerunfailures --upgrade # module for measuring code coverage pip install coverage -# check existing az +# pre-install: check existing az which az || az version || az extension list || true -# install latest az -azdev setup -c azure-cli -r azure-cli-extensions +# install az from cloned repos with azdev +azdev setup -c azure-cli/ -r azure-cli-extensions/ deactivate source azEnv/bin/activate -# check installation result +# post-install: check installation result which az az version + +# mkdir to store reports +mkdir -p reports/ diff --git a/src/aks-preview/azcli_aks_live_test/test_cli.sh b/src/aks-preview/azcli_aks_live_test/test_cli.sh index b03f40127ca..7fedade8d1e 100755 --- a/src/aks-preview/azcli_aks_live_test/test_cli.sh +++ b/src/aks-preview/azcli_aks_live_test/test_cli.sh @@ -16,7 +16,8 @@ fi # test cli if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then echo "Test in record mode!" - azdev test acs --no-exitfirst --xml-path cli_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_report.json --reruns 3 --capture=sys" + azdev test acs --no-exitfirst --xml-path cli_result.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_report.json --reruns 3 --capture=sys" + cp cli_report.json cli_result.xml reports/ fi if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then @@ -24,5 +25,6 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID az account set -s $AZCLI_ALT_SUBSCRIPTION_ID az account show - azdev test acs --live --no-exitfirst --xml-path cli_live_test.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_live_report.json --reruns 3 --capture=sys" + azdev test acs --live --no-exitfirst --xml-path cli_live_result.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_live_report.json --reruns 3 --capture=sys" + cp cli_live_report.json cli_live_result.xml reports/ fi diff --git a/src/aks-preview/azcli_aks_live_test/test_ext.sh b/src/aks-preview/azcli_aks_live_test/test_ext.sh index 9d159c3d63d..60058d01eec 100755 --- a/src/aks-preview/azcli_aks_live_test/test_ext.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext.sh @@ -49,6 +49,7 @@ fi coverage combine && coverage json -o coverage_azcli_aks_live_test.json coverage report -m popd +cp azure-cli-extensions/src/aks-preview/azcli_aks_live_test/coverage_azcli_aks_live_test.json reports/ # azext_aks_preview azext_aks_preview_unit_test_result="" @@ -62,6 +63,7 @@ fi coverage combine && coverage json -o coverage_azext_aks_preview.json coverage report -m popd +cp azure-cli-extensions/src/aks-preview/azext_aks_preview/coverage_azext_aks_preview.json reports/ if [[ $azcli_aks_live_test_unit_test_result == "error" || $azext_aks_preview_unit_test_result == "error" ]]; then echo "Unit test failed!" @@ -69,7 +71,7 @@ if [[ $azcli_aks_live_test_unit_test_result == "error" || $azext_aks_preview_uni fi # prepare run flags -run_flags="-em ext_matrix_default.json --no-exitfirst --discover --json-report-path ./ --reruns 3 --capture=sys" +run_flags="-em ext_matrix_default.json --no-exitfirst --discover --report-path ./ --reruns 3 --capture=sys" # parallel if [ $PARALLELISM -ge 2 ]; then run_flags+=" -n $PARALLELISM" @@ -89,8 +91,10 @@ fi if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then echo "Test in record mode!" run_flags+=" --json-report-file=ext_report.json" + run_flags+=" --xml-file=ext_result.xml" echo "run flags: ${run_flags}" - echo ${run_flags} | xargs python -u az_aks_tool/main.py + echo ${run_flags} | xargs python -u az_aks_tool/main.py + cp ext_report.json ext_result.xml reports/ fi # live test @@ -100,6 +104,8 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then az account set -s $AZCLI_ALT_SUBSCRIPTION_ID az account show run_flags+=" -l --json-report-file=ext_live_report.json" + run_flags+=" --xml-file=ext_live_result.xml" echo "run flags: ${run_flags}" echo ${run_flags} | xargs python -u az_aks_tool/main.py + cp ext_live_report.json ext_live_result.xml reports/ fi diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml index baab8382d49..63b5b7b1500 100644 --- a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml +++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml @@ -46,31 +46,11 @@ jobs: docker exec "azcli-aks-live-test-container" /opt/test_ext.sh condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all')) displayName: Perform Test for EXT - - task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: 'cli_report.json' - ArtifactName: 'cli_report.json' - publishLocation: 'Container' - condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'cli', 'all'), in(variables['TEST_MODE'], 'record', 'all')) - displayName: Publish CLI Record Test Report - - task: PublishBuildArtifacts@1 + - task: CopyFiles@2 inputs: - PathtoPublish: 'cli_live_report.json' - ArtifactName: 'cli_live_report.json' - publishLocation: 'Container' - condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'cli', 'all'), in(variables['TEST_MODE'], 'live', 'all')) - displayName: Publish CLI Live Test Report + contents: 'reports/**' + targetFolder: $(Build.ArtifactStagingDirectory) - task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: 'ext_report.json' - ArtifactName: 'ext_report.json' - publishLocation: 'Container' - condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all'), in(variables['TEST_MODE'], 'record', 'all')) - displayName: Publish EXT Record Test Report - - task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: 'ext_live_report.json' - ArtifactName: 'ext_live_report.json' - publishLocation: 'Container' - condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all'), in(variables['TEST_MODE'], 'live', 'all')) - displayName: Publish EXT Live Test Report + inputs: + pathToPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'reports' From 781ebf5f9d6f83f6e133be7aff9dacebfaba401d Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 14:02:45 +0800 Subject: [PATCH 23/42] fix yaml --- .../azcli_aks_live_test/vsts-azcli-aks-live-test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml index 63b5b7b1500..827507b1886 100644 --- a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml +++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml @@ -51,6 +51,6 @@ jobs: contents: 'reports/**' targetFolder: $(Build.ArtifactStagingDirectory) - task: PublishBuildArtifacts@1 - inputs: - pathToPublish: $(Build.ArtifactStagingDirectory) - artifactName: 'reports' + inputs: + pathToPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'reports' From 413166fa7f7553211cedcf0c424a50873bd11aa7 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 14:07:22 +0800 Subject: [PATCH 24/42] fix path --- src/aks-preview/azcli_aks_live_test/clone_repo.sh | 2 +- .../azcli_aks_live_test/vsts-azcli-aks-live-test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aks-preview/azcli_aks_live_test/clone_repo.sh b/src/aks-preview/azcli_aks_live_test/clone_repo.sh index 575fc4f5f54..8d10a23483a 100755 --- a/src/aks-preview/azcli_aks_live_test/clone_repo.sh +++ b/src/aks-preview/azcli_aks_live_test/clone_repo.sh @@ -30,5 +30,5 @@ git log -10 popd # move live test related files to the same level as the checkout directory ($(Agent.BuildDirectory)/s) -mv azure-cli-extensions/src/aks-preview/azcli-aks-live-test/* ./ +mv azure-cli-extensions/src/aks-preview/azcli_aks_live_test/* ./ ls -alh diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml index 827507b1886..d85fbf15647 100644 --- a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml +++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml @@ -19,7 +19,7 @@ jobs: ls -alh displayName: "Move All Checkout Files to the Newly Created 'azure-cli-extensions' Directory" - bash: | - source ./azure-cli-extensions/src/aks-preview/azcli-aks-live-test/clone_repo.sh + source ./azure-cli-extensions/src/aks-preview/azcli_aks_live_test/clone_repo.sh condition: succeeded() displayName: "Clone GitHub Repo and Move Live Test Related Files" - bash: | From 0ea8d41db99a1339b4eb53a3aae3a16eaec93fea Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 14:15:35 +0800 Subject: [PATCH 25/42] remove dockerfile and local build option --- src/aks-preview/azcli_aks_live_test/Dockerfile | 7 ------- .../azcli_aks_live_test/ext_matrix_default.json | 2 +- .../azcli_aks_live_test/prepare_image.sh | 15 +++------------ .../azcli_aks_live_test/transcribe_env.sh | 1 - 4 files changed, 4 insertions(+), 21 deletions(-) delete mode 100644 src/aks-preview/azcli_aks_live_test/Dockerfile diff --git a/src/aks-preview/azcli_aks_live_test/Dockerfile b/src/aks-preview/azcli_aks_live_test/Dockerfile deleted file mode 100644 index abf405bcd40..00000000000 --- a/src/aks-preview/azcli_aks_live_test/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ubuntu:18.04 -RUN apt update && \ - apt install -y software-properties-common && \ - add-apt-repository -y ppa:deadsnakes/ppa && \ - apt update && \ - apt install -y python3.8 python3.8-venv python3.8-dev python3-pip gcc git -CMD ["/bin/bash"] diff --git a/src/aks-preview/azcli_aks_live_test/ext_matrix_default.json b/src/aks-preview/azcli_aks_live_test/ext_matrix_default.json index 641621960b5..13d3d7295af 100644 --- a/src/aks-preview/azcli_aks_live_test/ext_matrix_default.json +++ b/src/aks-preview/azcli_aks_live_test/ext_matrix_default.json @@ -5,7 +5,7 @@ ] }, "exclude": { - "naf": [ + "need additional feature": [ "test_aks_create_addon_with_azurekeyvaultsecretsprovider_with_secret_rotation", "test_aks_create_with_auto_upgrade_channel", "test_aks_create_with_pod_identity_enabled", diff --git a/src/aks-preview/azcli_aks_live_test/prepare_image.sh b/src/aks-preview/azcli_aks_live_test/prepare_image.sh index 93f17467e31..b4341d37272 100755 --- a/src/aks-preview/azcli_aks_live_test/prepare_image.sh +++ b/src/aks-preview/azcli_aks_live_test/prepare_image.sh @@ -4,15 +4,6 @@ set -eux pwd # prepare docker image -if [[ $BUILD_IMAGE == true ]]; then - echo "Building test image '$IMAGE_NAME:$IMAGE_TAG'..." - docker build -t $IMAGE_NAME:$IMAGE_TAG -f ./Dockerfile . -else - echo "Pulling test image from '$IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG'..." - if docker pull $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG; then - docker tag $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:$IMAGE_TAG - else - echo "Failed to pull image, start local build..." - docker build -t $IMAGE_NAME:$IMAGE_TAG -f ./Dockerfile . - fi -fi +echo "Pulling test image from '$IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG'..." +docker pull $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG +docker tag $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:$IMAGE_TAG diff --git a/src/aks-preview/azcli_aks_live_test/transcribe_env.sh b/src/aks-preview/azcli_aks_live_test/transcribe_env.sh index 7715da60337..bf5cc7970b3 100755 --- a/src/aks-preview/azcli_aks_live_test/transcribe_env.sh +++ b/src/aks-preview/azcli_aks_live_test/transcribe_env.sh @@ -34,7 +34,6 @@ echo "EXT_REPO=$EXT_REPO" >> env.list echo "EXT_BRANCH=$EXT_BRANCH" >> env.list # image -echo "BUILD_IMAGE=$BUILD_IMAGE" >> env.list echo "IMAGE_PREFIX=$IMAGE_PREFIX" >> env.list echo "IMAGE_NAME=$IMAGE_NAME" >> env.list echo "IMAGE_TAG=$IMAGE_TAG" >> env.list From c2d0456befb39593bd605e96f088ef9c30fa7990 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 14:21:57 +0800 Subject: [PATCH 26/42] fix unittest discover --- src/aks-preview/azcli_aks_live_test/test_ext.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aks-preview/azcli_aks_live_test/test_ext.sh b/src/aks-preview/azcli_aks_live_test/test_ext.sh index 60058d01eec..3ce13f3206f 100755 --- a/src/aks-preview/azcli_aks_live_test/test_ext.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext.sh @@ -41,7 +41,7 @@ azcli_aks_live_test_unit_test_result="" pushd azure-cli-extensions/src/aks-preview/azcli_aks_live_test/ # clean existing coverage report (coverage combine || true) && (coverage erase || true) -if ! coverage run --source=. --omit=*/tests/* -p -m unittest ; then +if ! coverage run --source=. --omit=*/tests/* -p -m unittest discover; then azcli_aks_live_test_unit_test_result="error" fi # currently no test written in pytest format under 'azcli_aks_live_test/' @@ -57,7 +57,7 @@ pushd azure-cli-extensions/src/aks-preview/azext_aks_preview # clean existing coverage report (coverage combine || true) && (coverage erase || true) # currently test using module 'unittest' is the same as module 'pytest', and test using 'pytest' is just recording test -if ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m unittest || coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m pytest; then +if ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m unittest discover || coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m pytest; then azext_aks_preview_unit_test_result="error" fi coverage combine && coverage json -o coverage_azext_aks_preview.json From 73a5a5baa10d97539c35b6301caff298549be2a5 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 15:03:05 +0800 Subject: [PATCH 27/42] fix file missing & replace with /usr/bin/env --- src/aks-preview/azcli_aks_live_test/clean_up.sh | 2 +- src/aks-preview/azcli_aks_live_test/clone_repo.sh | 6 +++--- src/aks-preview/azcli_aks_live_test/prepare_image.sh | 2 +- src/aks-preview/azcli_aks_live_test/setup_venv.sh | 2 +- src/aks-preview/azcli_aks_live_test/start_container.sh | 7 +++++-- src/aks-preview/azcli_aks_live_test/test_cli.sh | 2 +- src/aks-preview/azcli_aks_live_test/test_ext.sh | 2 +- src/aks-preview/azcli_aks_live_test/transcribe_env.sh | 2 +- 8 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/aks-preview/azcli_aks_live_test/clean_up.sh b/src/aks-preview/azcli_aks_live_test/clean_up.sh index 7175d775979..4bbeb2f3520 100755 --- a/src/aks-preview/azcli_aks_live_test/clean_up.sh +++ b/src/aks-preview/azcli_aks_live_test/clean_up.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # login az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID diff --git a/src/aks-preview/azcli_aks_live_test/clone_repo.sh b/src/aks-preview/azcli_aks_live_test/clone_repo.sh index 8d10a23483a..c6a21ed82d1 100755 --- a/src/aks-preview/azcli_aks_live_test/clone_repo.sh +++ b/src/aks-preview/azcli_aks_live_test/clone_repo.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eux pwd @@ -29,6 +29,6 @@ git branch -a git log -10 popd -# move live test related files to the same level as the checkout directory ($(Agent.BuildDirectory)/s) -mv azure-cli-extensions/src/aks-preview/azcli_aks_live_test/* ./ +# copy live test related files to the same level as the checkout directory ($(Agent.BuildDirectory)/s) +cp -r azure-cli-extensions/src/aks-preview/azcli_aks_live_test/* ./ ls -alh diff --git a/src/aks-preview/azcli_aks_live_test/prepare_image.sh b/src/aks-preview/azcli_aks_live_test/prepare_image.sh index b4341d37272..32793cd90ee 100755 --- a/src/aks-preview/azcli_aks_live_test/prepare_image.sh +++ b/src/aks-preview/azcli_aks_live_test/prepare_image.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eux pwd diff --git a/src/aks-preview/azcli_aks_live_test/setup_venv.sh b/src/aks-preview/azcli_aks_live_test/setup_venv.sh index d84d1c88be9..935ce03507f 100755 --- a/src/aks-preview/azcli_aks_live_test/setup_venv.sh +++ b/src/aks-preview/azcli_aks_live_test/setup_venv.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eux pwd diff --git a/src/aks-preview/azcli_aks_live_test/start_container.sh b/src/aks-preview/azcli_aks_live_test/start_container.sh index 313ef82951f..5d89bf829fb 100755 --- a/src/aks-preview/azcli_aks_live_test/start_container.sh +++ b/src/aks-preview/azcli_aks_live_test/start_container.sh @@ -1,12 +1,15 @@ -#!/bin/bash +#!/usr/bin/env bash set -eux pwd -# transcribe environment variables +# transcribe environment variables into file 'env.list' source ./transcribe_env.sh # start container in backgroud with env.list as environment varialbes # mount current directory ($(Agent.BuildDirectory)/s) to /opt in container # set working directory as /opt in container docker run -t -d --env-file ./env.list -v $PWD:/opt -w /opt --name "azcli-aks-live-test-container" $IMAGE_NAME:$IMAGE_TAG + +# remove env.list +rm ./env.list diff --git a/src/aks-preview/azcli_aks_live_test/test_cli.sh b/src/aks-preview/azcli_aks_live_test/test_cli.sh index 7fedade8d1e..b9c05233760 100755 --- a/src/aks-preview/azcli_aks_live_test/test_cli.sh +++ b/src/aks-preview/azcli_aks_live_test/test_cli.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eux pwd diff --git a/src/aks-preview/azcli_aks_live_test/test_ext.sh b/src/aks-preview/azcli_aks_live_test/test_ext.sh index 3ce13f3206f..b6a108b9bb0 100755 --- a/src/aks-preview/azcli_aks_live_test/test_ext.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eux pwd diff --git a/src/aks-preview/azcli_aks_live_test/transcribe_env.sh b/src/aks-preview/azcli_aks_live_test/transcribe_env.sh index bf5cc7970b3..f80ec8d6465 100755 --- a/src/aks-preview/azcli_aks_live_test/transcribe_env.sh +++ b/src/aks-preview/azcli_aks_live_test/transcribe_env.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # clear cat /dev/null > env.list From eb7e19d2b53a0e38d451676f19cfa7b79deefd6e Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 15:39:42 +0800 Subject: [PATCH 28/42] fix unittest --- src/aks-preview/azcli_aks_live_test/test_ext.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-preview/azcli_aks_live_test/test_ext.sh b/src/aks-preview/azcli_aks_live_test/test_ext.sh index b6a108b9bb0..d245af36909 100755 --- a/src/aks-preview/azcli_aks_live_test/test_ext.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext.sh @@ -57,7 +57,7 @@ pushd azure-cli-extensions/src/aks-preview/azext_aks_preview # clean existing coverage report (coverage combine || true) && (coverage erase || true) # currently test using module 'unittest' is the same as module 'pytest', and test using 'pytest' is just recording test -if ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m unittest discover || coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m pytest; then +if ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m unittest discover || ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m pytest; then azext_aks_preview_unit_test_result="error" fi coverage combine && coverage json -o coverage_azext_aks_preview.json From ccff955fad88af3adaab9ec67e1419a0a258fd41 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 17:00:58 +0800 Subject: [PATCH 29/42] split out a single unit test pipeline & refine log --- .../azcli_aks_live_test/az_aks_tool/log.py | 19 +++-- .../azcli_aks_live_test/az_aks_tool/main.py | 17 +++-- .../az_aks_tool/tests/test_log.py | 8 ++ .../{test_cli.sh => test_cli_live.sh} | 0 .../azcli_aks_live_test/test_cli_unit.sh | 16 ++++ .../azcli_aks_live_test/test_ext_live.sh | 76 +++++++++++++++++++ .../{test_ext.sh => test_ext_unit.sh} | 40 ---------- .../vsts-azcli-aks-live-test.yaml | 10 +-- .../vsts-azcli-aks-unit-test.yaml | 56 ++++++++++++++ 9 files changed, 186 insertions(+), 56 deletions(-) rename src/aks-preview/azcli_aks_live_test/{test_cli.sh => test_cli_live.sh} (100%) create mode 100644 src/aks-preview/azcli_aks_live_test/test_cli_unit.sh create mode 100755 src/aks-preview/azcli_aks_live_test/test_ext_live.sh rename src/aks-preview/azcli_aks_live_test/{test_ext.sh => test_ext_unit.sh} (68%) mode change 100755 => 100644 create mode 100644 src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py index 12fdfd2490e..ff1af6744c2 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py @@ -1,17 +1,26 @@ import logging import sys + +def parse_module_name(levels=1): + module_name = None + module_levels = __name__.split(".") + if len(module_levels) < levels: + print("Failed to parse {}-level module name from '{}'".format(levels, __name__)) + else: + module_name = ".".join(module_levels[:levels]) + return module_name + + def setup_logging(root_logger_name="", log_path="az_aks_tool.log"): if root_logger_name == "": - try: - root_logger_name = __name__.split(".")[0] - except Exception as e: - print("Failed to parse module name from '{}', error: {}".format(__name__, e)) + root_logger_name = parse_module_name() logger = logging.getLogger(root_logger_name) logger.setLevel(level=logging.DEBUG) # Formatter - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') # FileHandler file_handler = logging.FileHandler(filename=log_path, mode="w") diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py index 94d98f2e0c2..c49c88c0f46 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py @@ -12,7 +12,7 @@ import azcli_aks_live_test.az_aks_tool.utils as utils import azcli_aks_live_test.az_aks_tool.ext as ext -from azcli_aks_live_test.az_aks_tool.log import setup_logging +import azcli_aks_live_test.az_aks_tool.log as log # const AKS_PREVIEW_MOD_NAME = EXTENSION_PREFIX + "aks_preview" # azext_aks_preview @@ -61,8 +61,10 @@ def init_argparse(args): def main(): # setup logger - setup_logging() - logger = logging.getLogger(__name__) + root_module_name = log.parse_module_name(levels=1) + log.setup_logging(root_module_name) + current_module_name = log.parse_module_name(levels=2) + logger = logging.getLogger("{}.{}".format(current_module_name, __name__)) # parse args logger.info("raw args: {}".format(sys.argv)) @@ -109,9 +111,12 @@ def main(): ext_filtered_test_cases, AKS_PREVIEW_MOD_NAME) logger.info("According to 'ext_matrix' and filters, we get {} cases, need to exclude {} cases, finally get {} cases!".format( len(ext_test_cases), len(ext_exclude_test_cases), len(ext_filtered_test_cases))) - logger.info("Perform following tests: {}".format(ext_qualified_test_cases)) - run_tests(ext_qualified_test_cases, xml_path=xml_path, discover=args.discover, in_series=args.series, - run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + if len(ext_qualified_test_cases) == 0: + logger.warning("No test case! Skipping!") + else: + logger.info("Perform following tests: {}".format(ext_qualified_test_cases)) + run_tests(ext_qualified_test_cases, xml_path=xml_path, discover=args.discover, in_series=args.series, + run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) # cli matrix if utils.check_file_existence(cli_matrix_file_path): diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_log.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_log.py index 70706b0e854..9c6888c75a5 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_log.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_log.py @@ -16,6 +16,14 @@ class LogTestCase(unittest.TestCase): + def test_parse_module_name(self): + root_module_name = log.parse_module_name(levels=1) + current_module_name = log.parse_module_name(levels=2) + error_module_name = log.parse_module_name(levels=5) + self.assertEqual(root_module_name, "azcli_aks_live_test") + self.assertEqual(current_module_name, "azcli_aks_live_test.az_aks_tool") + self.assertEqual(error_module_name, None) + def test_setup_logging(self): log.setup_logging("unittest", "unittest_log.log") logger = logging.getLogger("unittest.test_setup_logging") diff --git a/src/aks-preview/azcli_aks_live_test/test_cli.sh b/src/aks-preview/azcli_aks_live_test/test_cli_live.sh similarity index 100% rename from src/aks-preview/azcli_aks_live_test/test_cli.sh rename to src/aks-preview/azcli_aks_live_test/test_cli_live.sh diff --git a/src/aks-preview/azcli_aks_live_test/test_cli_unit.sh b/src/aks-preview/azcli_aks_live_test/test_cli_unit.sh new file mode 100644 index 00000000000..8ef79d3e09c --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/test_cli_unit.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eux +pwd + +# activate virtualenv +source azEnv/bin/activate + +# remove extension +echo "Remove existing aks-preview extension (if any)" +if az extension remove --name aks-preview || azdev extension remove aks-preview; then + deactivate + source azEnv/bin/activate +fi + +echo "Implementing, pass for now!" diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh new file mode 100755 index 00000000000..5c2e0f2be5c --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +set -eux +pwd + +# activate virtualenv +source azEnv/bin/activate + +# remove extension +echo "Remove existing aks-preview extension (if any)" +if az extension remove --name aks-preview || azdev extension remove aks-preview; then + deactivate + source azEnv/bin/activate +fi + +# install latest extension +echo "Install the latest aks-preview extension and re-activate the virtualenv" +azdev extension add aks-preview +az extension list +azdev extension list | grep "aks-preview" -C 5 +deactivate +source azEnv/bin/activate + +# Ensure that the command index is updated by calling a specific command in aks-preview, so that all the commands defined in aks-preview are loaded correctly +# Otherwise, cold boot execution of azdev test may use the api version adopted by the acs command group in azure-cli (which may diverge from the api version used in current aks-preview) +retry_count=0 +while ! az aks command invoke --help && [[ $retry_count < 3 ]] +do + retry_count=`expr $retry_count + 1` + echo $retry_count"th retry to install aks-preview..." + azdev extension add aks-preview --debug + az extension list --debug + azdev extension list --debug | grep "aks-preview" -C 5 + deactivate + source azEnv/bin/activate +done + +# prepare run flags +run_flags="-em ext_matrix_default.json --no-exitfirst --discover --report-path ./ --reruns 3 --capture=sys" +# parallel +if [ $PARALLELISM -ge 2 ]; then + run_flags+=" -n $PARALLELISM" +else + run_flags+=" -s" +fi +# ext filter +if [[ -n $EXT_TEST_FILTER ]]; then + run_flags+=" -ef $EXT_TEST_FILTER" +fi +# ext extra coverage +if [[ $EXT_TEST_COVERAGE ]]; then + run_flags+=" -ec $EXT_TEST_COVERAGE" +fi + +# recording test +if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then + echo "Test in record mode!" + run_flags+=" --json-report-file=ext_report.json" + run_flags+=" --xml-file=ext_result.xml" + echo "run flags: ${run_flags}" + echo ${run_flags} | xargs python -u az_aks_tool/main.py + cp ext_report.json ext_result.xml reports/ +fi + +# live test +if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then + echo "Test in live mode!" + az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID + az account set -s $AZCLI_ALT_SUBSCRIPTION_ID + az account show + run_flags+=" -l --json-report-file=ext_live_report.json" + run_flags+=" --xml-file=ext_live_result.xml" + echo "run flags: ${run_flags}" + echo ${run_flags} | xargs python -u az_aks_tool/main.py + cp ext_live_report.json ext_live_result.xml reports/ +fi diff --git a/src/aks-preview/azcli_aks_live_test/test_ext.sh b/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh old mode 100755 new mode 100644 similarity index 68% rename from src/aks-preview/azcli_aks_live_test/test_ext.sh rename to src/aks-preview/azcli_aks_live_test/test_ext_unit.sh index d245af36909..fab2a7b09a8 --- a/src/aks-preview/azcli_aks_live_test/test_ext.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh @@ -69,43 +69,3 @@ if [[ $azcli_aks_live_test_unit_test_result == "error" || $azext_aks_preview_uni echo "Unit test failed!" exit 1 fi - -# prepare run flags -run_flags="-em ext_matrix_default.json --no-exitfirst --discover --report-path ./ --reruns 3 --capture=sys" -# parallel -if [ $PARALLELISM -ge 2 ]; then - run_flags+=" -n $PARALLELISM" -else - run_flags+=" -s" -fi -# ext filter -if [[ -n $EXT_TEST_FILTER ]]; then - run_flags+=" -ef $EXT_TEST_FILTER" -fi -# ext extra coverage -if [[ $EXT_TEST_COVERAGE ]]; then - run_flags+=" -ec $EXT_TEST_COVERAGE" -fi - -# recording test -if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then - echo "Test in record mode!" - run_flags+=" --json-report-file=ext_report.json" - run_flags+=" --xml-file=ext_result.xml" - echo "run flags: ${run_flags}" - echo ${run_flags} | xargs python -u az_aks_tool/main.py - cp ext_report.json ext_result.xml reports/ -fi - -# live test -if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then - echo "Test in live mode!" - az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID - az account set -s $AZCLI_ALT_SUBSCRIPTION_ID - az account show - run_flags+=" -l --json-report-file=ext_live_report.json" - run_flags+=" --xml-file=ext_live_result.xml" - echo "run flags: ${run_flags}" - echo ${run_flags} | xargs python -u az_aks_tool/main.py - cp ext_live_report.json ext_live_result.xml reports/ -fi diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml index d85fbf15647..22b34e0fe63 100644 --- a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml +++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml @@ -39,13 +39,13 @@ jobs: condition: succeeded() displayName: "Set up Virtual Environment" - bash: | - docker exec "azcli-aks-live-test-container" /opt/test_cli.sh + docker exec "azcli-aks-live-test-container" /opt/test_cli_live.sh condition: and(succeeded(), in(variables['COVERAGE'], 'cli', 'all')) - displayName: Perform Test for CLI + displayName: Perform Live Test for CLI - bash: | - docker exec "azcli-aks-live-test-container" /opt/test_ext.sh + docker exec "azcli-aks-live-test-container" /opt/test_ext_live.sh condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all')) - displayName: Perform Test for EXT + displayName: Perform Live Test for EXT - task: CopyFiles@2 inputs: contents: 'reports/**' @@ -53,4 +53,4 @@ jobs: - task: PublishBuildArtifacts@1 inputs: pathToPublish: $(Build.ArtifactStagingDirectory) - artifactName: 'reports' + artifactName: 'live test reports' diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml new file mode 100644 index 00000000000..e2d9e09c194 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml @@ -0,0 +1,56 @@ +name: $(Date:yyyyMMdd)$(Rev:.r)_Python$(PYTHON_VERSION)_Coverage-$(COVERAGE)_Mode-$(TEST_MODE)_Branch-$(Build.SourceBranchName) + +trigger: none + +jobs: +- job: UnitTest + pool: + vmImage: 'ubuntu-16.04' + timeoutInMinutes: 360 + displayName: "Unit Test with Python" + steps: + - bash: | + pwd + ls -alh + mkdir azure-cli-extensions + shopt -s extglob dotglob + mv !(azure-cli-extensions) azure-cli-extensions + shopt -u extglob dotglob + ls -alh + displayName: "Move All Checkout Files to the Newly Created 'azure-cli-extensions' Directory" + - bash: | + source ./azure-cli-extensions/src/aks-preview/azcli_aks_live_test/clone_repo.sh + condition: succeeded() + displayName: "Clone GitHub Repo and Move Test Related Files" + - bash: | + source ./prepare_image.sh + condition: succeeded() + displayName: "Prepare Test Image" + - bash: | + source ./start_container.sh + env: + MAPPED_AZCLI_ALT_CLIENT_SECRET: $(AZCLI_ALT_CLIENT_SECRET) + BUILD_REASON: $(Build.Reason) + SYSTEM_PULLREQUEST_TARGETBRANCH: $(System.PullRequest.TargetBranch) + condition: succeeded() + displayName: "Start Container" + - bash: | + docker exec "azcli-aks-live-test-container" /opt/setup_venv.sh + condition: succeeded() + displayName: "Set up Virtual Environment" + - bash: | + docker exec "azcli-aks-live-test-container" /opt/test_cli_unit.sh + condition: and(succeeded(), in(variables['COVERAGE'], 'cli', 'all')) + displayName: Perform Unit Test for CLI + - bash: | + docker exec "azcli-aks-live-test-container" /opt/test_ext_unit.sh + condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all')) + displayName: Perform Unit Test for EXT + - task: CopyFiles@2 + inputs: + contents: 'reports/**' + targetFolder: $(Build.ArtifactStagingDirectory) + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'unit test reports' From c13ccc478ada6cfd90e28f4a43fde030c659e6c6 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 17:14:44 +0800 Subject: [PATCH 30/42] update env passing --- .../azcli_aks_live_test/start_container.sh | 13 +++++++++++-- .../azcli_aks_live_test/test_cli_unit.sh | 0 .../azcli_aks_live_test/test_ext_unit.sh | 0 .../azcli_aks_live_test/transcribe_env.sh | 2 -- .../vsts-azcli-aks-unit-test.yaml | 8 ++++---- 5 files changed, 15 insertions(+), 8 deletions(-) mode change 100644 => 100755 src/aks-preview/azcli_aks_live_test/test_cli_unit.sh mode change 100644 => 100755 src/aks-preview/azcli_aks_live_test/test_ext_unit.sh diff --git a/src/aks-preview/azcli_aks_live_test/start_container.sh b/src/aks-preview/azcli_aks_live_test/start_container.sh index 5d89bf829fb..dee2c5278b3 100755 --- a/src/aks-preview/azcli_aks_live_test/start_container.sh +++ b/src/aks-preview/azcli_aks_live_test/start_container.sh @@ -6,10 +6,19 @@ pwd # transcribe environment variables into file 'env.list' source ./transcribe_env.sh -# start container in backgroud with env.list as environment varialbes +# take the first arg as container name +container_name=${1:-"azcli-aks-live-test-container"} + +# start container in backgroud with tty # mount current directory ($(Agent.BuildDirectory)/s) to /opt in container # set working directory as /opt in container -docker run -t -d --env-file ./env.list -v $PWD:/opt -w /opt --name "azcli-aks-live-test-container" $IMAGE_NAME:$IMAGE_TAG +# pass secrects as environment variables directly (instead of storing in env.list) +# pass other environment variables with env.list +docker run -t -d -v $PWD:/opt -w /opt \ +-e AZCLI_ALT_CLIENT_SECRET=$MAPPED_AZCLI_ALT_CLIENT_SECRET \ +-e AZURE_CLI_TEST_DEV_SP_PASSWORD=$MAPPED_AZCLI_ALT_CLIENT_SECRET \ +--env-file ./env.list \ +--name $container_name $IMAGE_NAME:$IMAGE_TAG # remove env.list rm ./env.list diff --git a/src/aks-preview/azcli_aks_live_test/test_cli_unit.sh b/src/aks-preview/azcli_aks_live_test/test_cli_unit.sh old mode 100644 new mode 100755 diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh b/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh old mode 100644 new mode 100755 diff --git a/src/aks-preview/azcli_aks_live_test/transcribe_env.sh b/src/aks-preview/azcli_aks_live_test/transcribe_env.sh index f80ec8d6465..228698838ee 100755 --- a/src/aks-preview/azcli_aks_live_test/transcribe_env.sh +++ b/src/aks-preview/azcli_aks_live_test/transcribe_env.sh @@ -7,11 +7,9 @@ cat /dev/null > env.list echo "TENANT_ID=$TENANT_ID" >> env.list echo "AZCLI_ALT_SUBSCRIPTION_ID=$AZCLI_ALT_SUBSCRIPTION_ID" >> env.list echo "AZCLI_ALT_CLIENT_ID=$AZCLI_ALT_CLIENT_ID" >> env.list -echo "AZCLI_ALT_CLIENT_SECRET=$MAPPED_AZCLI_ALT_CLIENT_SECRET" >> env.list # azdev env echo "AZURE_CLI_TEST_DEV_SP_NAME=$AZCLI_ALT_CLIENT_ID" >> env.list -echo "AZURE_CLI_TEST_DEV_SP_PASSWORD=$MAPPED_AZCLI_ALT_CLIENT_SECRET" >> env.list echo "AZURE_CLI_TEST_DEV_RESOURCE_GROUP_LOCATION=$TEST_LOCATION" >> env.list # predefined variables diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml index e2d9e09c194..ce792eea0dd 100644 --- a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml +++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml @@ -27,7 +27,7 @@ jobs: condition: succeeded() displayName: "Prepare Test Image" - bash: | - source ./start_container.sh + source ./start_container.sh "azcli-aks-unit-test-container" env: MAPPED_AZCLI_ALT_CLIENT_SECRET: $(AZCLI_ALT_CLIENT_SECRET) BUILD_REASON: $(Build.Reason) @@ -35,15 +35,15 @@ jobs: condition: succeeded() displayName: "Start Container" - bash: | - docker exec "azcli-aks-live-test-container" /opt/setup_venv.sh + docker exec "azcli-aks-unit-test-container" /opt/setup_venv.sh condition: succeeded() displayName: "Set up Virtual Environment" - bash: | - docker exec "azcli-aks-live-test-container" /opt/test_cli_unit.sh + docker exec "azcli-aks-unit-test-container" /opt/test_cli_unit.sh condition: and(succeeded(), in(variables['COVERAGE'], 'cli', 'all')) displayName: Perform Unit Test for CLI - bash: | - docker exec "azcli-aks-live-test-container" /opt/test_ext_unit.sh + docker exec "azcli-aks-unit-test-container" /opt/test_ext_unit.sh condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all')) displayName: Perform Unit Test for EXT - task: CopyFiles@2 From 4a001a84ba6e86b64d2d63e042a689dea64e36d8 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 18:39:19 +0800 Subject: [PATCH 31/42] add missing license header --- src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py index ff1af6744c2..304a05fdeec 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + import logging import sys From 624e42caf93596948ff786747930951e9a608f69 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Sat, 8 May 2021 19:01:25 +0800 Subject: [PATCH 32/42] replacing azdev --- .../azcli_aks_live_test/az_aks_tool/index.py | 4 + .../azcli_aks_live_test/az_aks_tool/run.py | 201 ++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/index.py create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/index.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/index.py new file mode 100644 index 00000000000..34913fb394d --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/index.py @@ -0,0 +1,4 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py new file mode 100644 index 00000000000..a502495f751 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py @@ -0,0 +1,201 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import subprocess +import logging +import traceback +from knack.util import CommandResultItem + +logger = logging.getLogger(__name__) + +def display(txt): + """ Output to stderr """ + print(txt, file=sys.stderr) + +class ProfileContext: + def __init__(self, profile_name=None): + self.target_profile = profile_name + + self.origin_profile = current_profile() + + def __enter__(self): + if self.target_profile is None or self.target_profile == self.origin_profile: + display('The tests are set to run against current profile "{}"'.format(self.origin_profile)) + else: + result = cmd('az cloud update --profile {}'.format(self.target_profile), + 'Switching to target profile "{}"...'.format(self.target_profile)) + if result.exit_code != 0: + raise Exception(result.error.output.decode('utf-8')) + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.target_profile is not None and self.target_profile != self.origin_profile: + display('Switching back to origin profile "{}"...'.format(self.origin_profile)) + call('az cloud update --profile {}'.format(self.origin_profile)) + + if exc_tb: + display('') + traceback.print_exception(exc_type, exc_val, exc_tb) + + +def current_profile(): + return cmd('az cloud show --query profile -otsv', show_stderr=False).result + + +def call(command, **kwargs): + """ Run an arbitrary command but don't buffer the output. + + :param command: The entire command line to run. + :param kwargs: Any kwargs supported by subprocess.Popen + :returns: (int) process exit code. + """ + return subprocess.call( + command, + shell=True, + **kwargs) + +def cmd(command, message=False, show_stderr=True, raise_error=False, **kwargs): + """ Run an arbitrary command. + + :param command: The entire command line to run. + :param message: A custom message to display, or True (bool) to use a default. + :param show_stderr: On error, display the contents of STDERR. + :param raise_error: On error, raise CommandError. + :param kwargs: Any kwargs supported by subprocess.Popen + :returns: CommandResultItem object. + """ + from azdev.utilities import IS_WINDOWS, display + + # use default message if custom not provided + if message is True: + message = 'Running: {}\n'.format(command) + + if message: + display(message) + + logger.info("Running: %s", command) + try: + output = subprocess.check_output( + command.split(), + stderr=subprocess.STDOUT if show_stderr else None, + shell=IS_WINDOWS, + **kwargs).decode('utf-8').strip() + logger.debug(output) + return CommandResultItem(output, exit_code=0, error=None) + except subprocess.CalledProcessError as err: + if raise_error: + raise CommandError(err.output.decode(), err.returncode, command) + return CommandResultItem(err.output, exit_code=err.returncode, error=err) + + +def get_test_runner(parallel, log_path, last_failed, no_exit_first, mark): + """Create a pytest execution method""" + def _run(test_paths, pytest_args): + + if os.name == 'posix': + arguments = ['-x', '-v', '--boxed', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path] + else: + arguments = ['-x', '-v', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path] + + if no_exit_first: + arguments.remove('-x') + + if mark: + arguments.append('-m "{}"'.format(mark)) + + arguments.extend(test_paths) + if parallel: + arguments += ['-n', 'auto'] + if last_failed: + arguments.append('--lf') + if pytest_args: + arguments += pytest_args + cmd = 'python -m pytest {}'.format(' '.join(arguments)) + logger.info('Running: %s', cmd) + return call(cmd) + + return _run + + +# pylint: disable=too-many-statements,too-many-locals +# def run_tests(tests, xml_path=None, discover=False, in_series=False, +# run_live=False, profile=None, last_failed=False, pytest_args=None, +# no_exit_first=False, mark=None, +# git_source=None, git_target=None, git_repo=None, +# cli_ci=False): + + + + +# path_table = get_path_table() + +# test_index = _get_test_index(profile or current_profile(), discover) + +# if not tests: +# tests = list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys()) +# if tests == ['CLI']: +# tests = list(path_table['mod'].keys()) + list(path_table['core'].keys()) +# elif tests == ['EXT']: +# tests = list(path_table['ext'].keys()) + +# # filter out tests whose modules haven't changed +# modified_mods = _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo) +# if modified_mods: +# display('\nTest on modules: {}\n'.format(', '.join(modified_mods))) + +# if cli_ci is True: +# ctx = CLIAzureDevOpsContext(git_repo, git_source, git_target) +# modified_mods = ctx.filter(test_index) + + +# # process environment variables +# if run_live: +# logger.warning('RUNNING TESTS LIVE') +# os.environ[ENV_VAR_TEST_LIVE] = 'True' + +# def _find_test(index, name): +# name_comps = name.split('.') +# num_comps = len(name_comps) +# key_error = KeyError() + +# for i in range(num_comps): +# check_name = '.'.join(name_comps[(-1 - i):]) +# try: +# match = index[check_name] +# if check_name != name: +# logger.info("Test found using just '%s'. The rest of the name was ignored.\n", check_name) +# return match +# except KeyError as ex: +# key_error = ex +# continue +# raise key_error + +# # lookup test paths from index +# test_paths = [] +# for t in modified_mods: +# try: +# test_path = os.path.normpath(_find_test(test_index, t)) +# test_paths.append(test_path) +# except KeyError: +# logger.warning("'%s' not found. If newly added, re-run with --discover", t) +# continue + +# exit_code = 0 + +# # Tests have been collected. Now run them. +# if not test_paths: +# logger.warning('No tests selected to run.') +# sys.exit(exit_code) + +# exit_code = 0 +# with ProfileContext(profile): +# runner = get_test_runner(parallel=not in_series, +# log_path=xml_path, +# last_failed=last_failed, +# no_exit_first=no_exit_first, +# mark=mark) +# exit_code = runner(test_paths=test_paths, pytest_args=pytest_args) + +# sys.exit(0 if not exit_code else 1) From 31c9b1886e1ef5094b68119d0ec8ab6076613f20 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Mon, 10 May 2021 15:58:19 +0800 Subject: [PATCH 33/42] remove azdev dependency --- .../azcli_aks_live_test/az_aks_tool/cli.py | 40 +++ .../azcli_aks_live_test/az_aks_tool/const.py | 19 ++ .../azcli_aks_live_test/az_aks_tool/ext.py | 90 ++---- .../azcli_aks_live_test/az_aks_tool/filter.py | 120 ++++++++ .../azcli_aks_live_test/az_aks_tool/index.py | 270 ++++++++++++++++++ .../azcli_aks_live_test/az_aks_tool/main.py | 124 ++++---- .../azcli_aks_live_test/az_aks_tool/run.py | 165 +++++------ .../azcli_aks_live_test/az_aks_tool/utils.py | 113 ++++---- 8 files changed, 663 insertions(+), 278 deletions(-) create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/cli.py create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/const.py create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/filter.py diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/cli.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/cli.py new file mode 100644 index 00000000000..ad608a2698d --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/cli.py @@ -0,0 +1,40 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import glob +import os +import logging + +import azcli_aks_live_test.az_aks_tool.const as const +import azcli_aks_live_test.az_aks_tool.index as index +logger = logging.getLogger(__name__) + + +def get_cli_module_data(mod_name=const.ACS_MOD_NAME, profile="latest"): + profile_split = profile.split('-') + profile_namespace = '_'.join([profile_split[-1]] + profile_split[:-1]) + + # key value pairs of all modules(in azcli & extention) and its absolute path, used later to find test indexes + path_table = index.get_path_table() + command_modules = path_table["mod"] + inverse_name_table = index.get_name_index(invert=True) + + # construct 'import_name' & mod_data', used later to find test indexes + acs_mod_path = command_modules[mod_name] + mod_data = { + "alt_name": "{}{}".format(const.COMMAND_MODULE_PREFIX, mod_name), + "filepath": os.path.join(acs_mod_path, "tests", profile_namespace), + "base_path": "azure.cli.command_modules.{}.tests.{}".format(mod_name, profile_namespace), + "files": {} + } + + cli_test = index.discover_module_tests(mod_name, mod_data) + return cli_test + + +def get_cli_test_index(module_data=None, mod_name=const.ACS_MOD_NAME, profile="latest"): + if not module_data: + module_data = get_cli_module_data(mod_name=mod_name, profile=profile) + return module_data["files"] diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/const.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/const.py new file mode 100644 index 00000000000..21aa560bfae --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/const.py @@ -0,0 +1,19 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import sys + +IS_WINDOWS = sys.platform.lower() in ['windows', 'win32'] + +CLI_REPO_NAME = "azure-cli" +EXT_REPO_NAME = 'azure-cli-extensions' +COMMAND_MODULE_PREFIX = 'azure-cli-' +EXTENSION_PREFIX = 'azext_' +ACS_MOD_NAME = "acs" +AKS_PREVIEW_MOD_NAME = EXTENSION_PREFIX + "aks_preview" # azext_aks_preview + +ENV_VAR_TEST_MODULES = 'AZDEV_TEST_TESTS' # comma-separated list of modules to test +ENV_VAR_VIRTUAL_ENV = ['VIRTUAL_ENV', 'CONDA_PREFIX'] # used by system to identify virtual environment +ENV_VAR_TEST_LIVE = 'AZURE_TEST_RUN_LIVE' # denotes that tests should be run live instead of played back diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/ext.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/ext.py index 886b8bdb461..608b545b9c9 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/ext.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/ext.py @@ -6,95 +6,39 @@ import glob import os import logging -from azdev.operations.testtool import _discover_module_tests -from azdev.utilities import EXTENSION_PREFIX, get_path_table, get_name_index -import azcli_aks_live_test.az_aks_tool.utils as utils +import azcli_aks_live_test.az_aks_tool.const as const +import azcli_aks_live_test.az_aks_tool.index as index logger = logging.getLogger(__name__) -def get_ext_test_index(mod_name): +def get_ext_module_data(mod_name=const.AKS_PREVIEW_MOD_NAME, profile="latest"): + profile_split = profile.split('-') + profile_namespace = '_'.join([profile_split[-1]] + profile_split[:-1]) + # key value pairs of all modules(in azcli & extention) and its absolute path, used later to find test indexes - path_table = get_path_table() + path_table = index.get_path_table() extensions = path_table["ext"] - inverse_name_table = get_name_index(invert=True) + inverse_name_table = index.get_name_index(invert=True) # construct 'import_name' & mod_data', used later to find test indexes aks_preview_mod_path = extensions[mod_name] glob_pattern = os.path.normcase( - os.path.join("{}*".format(EXTENSION_PREFIX))) + os.path.join("{}*".format(const.EXTENSION_PREFIX))) file_path = glob.glob(os.path.join(aks_preview_mod_path, glob_pattern))[0] import_name = os.path.basename(file_path) mod_data = { "alt_name": inverse_name_table[mod_name], - "filepath": os.path.join(file_path, "tests", "latest"), - "base_path": "{}.tests.{}".format(import_name, "latest"), + "filepath": os.path.join(file_path, "tests", profile_namespace), + "base_path": "{}.tests.{}".format(import_name, profile_namespace), "files": {} } - # use azdev internal func '_discover_module_tests' to find test index - ext_test = _discover_module_tests(import_name, mod_data) - ext_test_index = ext_test["files"] - return ext_test_index - - -def get_ext_test_cases(ext_test_index, ext_matrix, ext_extra_coverage): - ext_test_cases = [] - ext_coverage = ext_matrix["coverage"] - # default coverage - for fileName, className in ext_coverage.items(): - for c in className: - ext_test_cases.extend(ext_test_index[fileName][c]) - # custom extra coverage - if ext_extra_coverage: - # method 1: fileName.className - file_class_pairs = utils.extract_file_class_pairs(ext_extra_coverage) - valid_file_class_pairs = utils.filter_valid_file_class_pairs( - file_class_pairs, ext_test_index) - for valid_pair in valid_file_class_pairs: - ext_test_cases.extend( - ext_test_index[valid_pair[0]][valid_pair[1]]) - # method 2: test cases - ext_test_cases.extend(utils.filter_valid_test_cases( - ext_extra_coverage, ext_test_index)) - return list(set(ext_test_cases)) + ext_test = index.discover_module_tests(import_name, mod_data) + return ext_test -def get_ext_exclude_test_cases(ext_test_index, ext_matrix, ext_filter): - ext_exclude_test_cases = [] - ext_exclude = ext_matrix["exclude"] - # default exclude - if not ext_filter or "default" in ext_filter: - matrix_test_cases = [] - matrix_file_class_pairs = [] - for k, v in ext_exclude.items(): - # method 1: reason -> test cases - matrix_test_cases.extend(v) - # method 2: fileName -> className - matrix_file_class_pairs.extend((k, x) for x in v) - # method 1: reason -> test cases - ext_exclude_test_cases.extend( - utils.filter_valid_test_cases(matrix_test_cases, ext_test_index)) - # method 2: fileName -> className - valid_matrix_file_class_pairs = utils.filter_valid_file_class_pairs( - matrix_file_class_pairs, ext_test_index) - for valid_matrix_pair in valid_matrix_file_class_pairs: - ext_exclude_test_cases.extend( - ext_test_index[valid_matrix_pair[0]][valid_matrix_pair[1]]) - # custom filter - if ext_filter: - # method 1: matrix exclude key - for k, v in ext_exclude.items(): - if k in ext_filter: - ext_exclude_test_cases.extend(v) - # method 2: fileName.className - file_class_pairs = utils.extract_file_class_pairs(ext_filter) - valid_file_class_pairs = utils.filter_valid_file_class_pairs( - file_class_pairs, ext_test_index) - for valid_pair in valid_file_class_pairs: - ext_exclude_test_cases.extend( - ext_test_index[valid_pair[0]][valid_pair[1]]) - # method 3: test cases - ext_exclude_test_cases.extend( - utils.filter_valid_test_cases(ext_filter, ext_test_index)) - return list(set(ext_exclude_test_cases)) +def get_ext_test_index(module_data=None, mod_name=const.AKS_PREVIEW_MOD_NAME, profile="latest"): + if not module_data: + module_data = get_ext_module_data(mod_name=mod_name, profile=profile) + return module_data["files"] diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/filter.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/filter.py new file mode 100644 index 00000000000..fbb30569b8e --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/filter.py @@ -0,0 +1,120 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import logging +from collections import Iterable +logger = logging.getLogger(__name__) + + +def extract_file_class_pairs(tag_list): + pairs = [] + for k in tag_list: + tags = k.split(".") + if len(tags) == 2: + pairs.append((tags[0], tags[1])) + return pairs + + +def filter_valid_file_class_pairs(pairs, test_index): + valid_pairs = [] + for pair in pairs: + if pair[0] in test_index and pair[1] in test_index[pair[0]]: + valid_pairs.append(pair) + logger.debug("Valid file & class pair: '{}'".format(pair)) + else: + logger.debug("Invalid file & class pair: '{}'".format(pair)) + return valid_pairs + + +def get_all_values_from_nested_dict(d): + for v in d.values(): + if isinstance(v, dict): + yield from get_all_values_from_nested_dict(v) + else: + yield v + + +def flatten_nested_list(lis): + for item in lis: + if isinstance(item, Iterable) and not isinstance(item, str): + for x in flatten_nested_list(item): + yield x + else: + yield item + + +def filter_valid_test_cases(test_cases, test_index): + valid_test_cases = [] + nested_test_cases = list(get_all_values_from_nested_dict(test_index)) + falttened_test_cases = list(flatten_nested_list(nested_test_cases)) + for test_case in test_cases: + if test_case in falttened_test_cases: + valid_test_cases.append(test_case) + logger.debug("Valid test case: '{}'".format(test_case)) + else: + logger.debug("Invalid test case: '{}'".format(test_case)) + return valid_test_cases + + +def get_test_cases(test_index, matrix, extra_coverage=None): + test_cases = [] + coverage = matrix["coverage"] + # default coverage + for fileName, className in coverage.items(): + for c in className: + test_cases.extend(test_index[fileName][c]) + # custom extra coverage + if extra_coverage: + # method 1: fileName.className + file_class_pairs = extract_file_class_pairs(extra_coverage) + valid_file_class_pairs = filter_valid_file_class_pairs( + file_class_pairs, test_index) + for valid_pair in valid_file_class_pairs: + test_cases.extend( + test_index[valid_pair[0]][valid_pair[1]]) + # method 2: test cases + test_cases.extend(filter_valid_test_cases( + extra_coverage, test_index)) + return list(set(test_cases)) + + +def get_exclude_test_cases(test_index, matrix, extra_filter=None): + exclude_test_cases = [] + exclude = matrix["exclude"] + # default exclude + if not extra_filter or "default" in extra_filter: + matrix_test_cases = [] + matrix_file_class_pairs = [] + for k, v in exclude.items(): + # method 1: reason -> test cases + matrix_test_cases.extend(v) + # method 2: fileName -> className + matrix_file_class_pairs.extend((k, x) for x in v) + # method 1: reason -> test cases + exclude_test_cases.extend( + filter_valid_test_cases(matrix_test_cases, test_index)) + # method 2: fileName -> className + valid_matrix_file_class_pairs = filter_valid_file_class_pairs( + matrix_file_class_pairs, test_index) + for valid_matrix_pair in valid_matrix_file_class_pairs: + exclude_test_cases.extend( + test_index[valid_matrix_pair[0]][valid_matrix_pair[1]]) + # custom extra_filter + if extra_filter: + # method 1: matrix exclude key + for k, v in exclude.items(): + if k in extra_filter: + exclude_test_cases.extend(v) + # method 2: fileName.className + file_class_pairs = extract_file_class_pairs(extra_filter) + valid_file_class_pairs = filter_valid_file_class_pairs( + file_class_pairs, test_index) + for valid_pair in valid_file_class_pairs: + exclude_test_cases.extend( + test_index[valid_pair[0]][valid_pair[1]]) + # method 3: test cases + exclude_test_cases.extend( + filter_valid_test_cases(extra_filter, test_index)) + return list(set(exclude_test_cases)) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/index.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/index.py index 34913fb394d..abb8336f787 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/index.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/index.py @@ -2,3 +2,273 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- + +import glob +import logging +import os +from importlib import import_module + +import azcli_aks_live_test.az_aks_tool.utils as utils +import azcli_aks_live_test.az_aks_tool.const as const +logger = logging.getLogger(__name__) + +def get_repo_path(repo_name, root_path=None): + valid_repo_paths = [] + if root_path is None or not os.path.isdir(root_path): + logger.warning("Invalid root path '{}', setting root path as current work dir '{}'".format(root_path, os.getcwd())) + root_path = os.getcwd() + for path, _, _ in os.walk(root_path): + pattern = os.path.join(path, repo_name) + valid_repo_paths.extend(glob.glob(pattern)) + if len(valid_repo_paths) >= 1: + repo_path = valid_repo_paths[0] + if len(valid_repo_paths) >= 2: + logger.warning("Find {} '{}' repo paths: {}".format(len(valid_repo_paths), repo_name, valid_repo_paths)) + logger.info("Get '{}' repo path as '{}'".format(repo_name, repo_path)) + return repo_path + else: + raise Exception("Could not find valid path to repo '{}' from '{}'".format(repo_name, root_path)) + +def find_files(root_paths, file_pattern): + """ Returns the paths to all files that match a given pattern. + + :returns: Paths ([str]) to files matching the given pattern. + """ + if isinstance(root_paths, str): + root_paths = [root_paths] + paths = [] + for root_path in root_paths: + for path, _, _ in os.walk(root_path): + pattern = os.path.join(path, file_pattern) + paths.extend(glob.glob(pattern)) + return paths + +def get_name_index(invert=False, include_whl_extensions=False): + """ Returns a dictionary containing the long and short names of modules and extensions is {SHORT:LONG} format or + {LONG:SHORT} format when invert=True. """ + from azure.cli.core.extension import EXTENSIONS_DIR # pylint: disable=import-error + + table = {} + cli_repo_path = get_repo_path(const.CLI_REPO_NAME) + ext_repo_paths = get_repo_path(const.EXT_REPO_NAME) + + # unified azure-cli package (2.0.68 and later) + paths = os.path.normcase( + os.path.join( + cli_repo_path, 'src', 'azure-cli', 'azure', 'cli', 'command_modules', '*', '__init__.py' + ) + ) + modules_paths = glob.glob(paths) + core_paths = glob.glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py'))) + ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x] + whl_ext_paths = [] + if include_whl_extensions: + whl_ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x] + + def _update_table(paths, key): + folder = None + long_name = None + short_name = None + for path in paths: + folder = os.path.dirname(path) + base_name = os.path.basename(folder) + # determine long-names + if key == 'ext': + short_name = base_name + for item in os.listdir(folder): + if item.startswith(const.EXTENSION_PREFIX): + long_name = item + break + elif base_name.startswith(const.COMMAND_MODULE_PREFIX): + long_name = base_name + short_name = base_name.replace(const.COMMAND_MODULE_PREFIX, '') or '__main__' + else: + short_name = base_name + long_name = '{}{}'.format(const.COMMAND_MODULE_PREFIX, base_name) + if not invert: + table[short_name] = long_name + else: + table[long_name] = short_name + + _update_table(modules_paths, 'mod') + _update_table(core_paths, 'core') + _update_table(ext_paths, 'ext') + _update_table(whl_ext_paths, 'ext') + + return table + +# pylint: disable=too-many-statements +def get_path_table(include_only=None, include_whl_extensions=False): + """ Returns a table containing the long and short names of different modules and extensions and the path to them. + The structure looks like: + { + 'core': { + NAME: PATH, + ... + }, + 'mod': { + NAME: PATH, + ... + }, + 'ext': { + NAME: PATH, + ... + } + } + """ + from azure.cli.core.extension import EXTENSIONS_DIR # pylint: disable=import-error + + # determine whether the call will filter or return all + if isinstance(include_only, str): + include_only = [include_only] + get_all = not include_only + + table = {} + cli_repo_path = get_repo_path(const.CLI_REPO_NAME) + ext_repo_paths = get_repo_path(const.EXT_REPO_NAME) + + paths = os.path.normcase( + os.path.join( + cli_repo_path, 'src', 'azure-cli', 'azure', 'cli', 'command_modules', '*', '__init__.py' + ) + ) + modules_paths = glob.glob(paths) + core_paths = glob.glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py'))) + ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x] + whl_ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x] + + def _update_table(package_paths, key): + if key not in table: + table[key] = {} + + for path in package_paths: + folder = os.path.dirname(path) + base_name = os.path.basename(folder) + + if key == 'ext': + short_name = base_name + long_name = next((item for item in os.listdir(folder) if item.startswith(const.EXTENSION_PREFIX)), None) + else: + short_name = base_name + long_name = '{}{}'.format(const.COMMAND_MODULE_PREFIX, base_name) + + if get_all: + table[key][long_name if key == 'ext' else short_name] = folder + elif not include_only: + return # nothing left to filter + else: + # check and update filter + if short_name in include_only: + include_only.remove(short_name) + table[key][short_name] = folder + if long_name in include_only: + # long name takes precedence to ensure path doesn't appear twice + include_only.remove(long_name) + table[key].pop(short_name, None) + table[key][long_name] = folder + + _update_table(modules_paths, 'mod') + _update_table(core_paths, 'core') + _update_table(ext_paths, 'ext') + if include_whl_extensions: + _update_table(whl_ext_paths, 'ext') + + if include_only: + whl_extensions = [mod for whl_ext_path in whl_ext_paths for mod in include_only if mod in whl_ext_path] + if whl_extensions: + err = 'extension(s): [ {} ] installed from a wheel may need --include-whl-extensions option'.format( + ', '.join(whl_extensions)) + raise Exception(err) + + raise Exception('unrecognized modules: [ {} ]'.format(', '.join(include_only))) + + return table + +def discover_module_tests(mod_name, mod_data): + + # get the list of test files in each module + total_tests = 0 + total_files = 0 + logger.info('Mod: %s', mod_name) + try: + contents = os.listdir(mod_data['filepath']) + test_files = { + x[:-len('.py')]: {} for x in contents if x.startswith('test_') and x.endswith('.py') + } + total_files = len(test_files) + except FileNotFoundError: + logger.info(' No test files found.') + return None + + for file_name in test_files: + mod_data['files'][file_name] = {} + test_file_path = mod_data['base_path'] + '.' + file_name + try: + module = import_module(test_file_path) + except ImportError as ex: + logger.info(' %s', ex) + continue + module_dict = module.__dict__ + possible_test_classes = {x: y for x, y in module_dict.items() if not x.startswith('_')} + for class_name, class_def in possible_test_classes.items(): + try: + class_dict = class_def.__dict__ + except AttributeError: + # skip non-class symbols in files like constants, imported methods, etc. + continue + if class_dict.get('__module__') == test_file_path: + tests = [x for x in class_def.__dict__ if x.startswith('test_')] + if tests: + mod_data['files'][file_name][class_name] = tests + total_tests += len(tests) + logger.info(' %s tests found in %s files.', total_tests, total_files) + return mod_data + + +def build_test_index(module_data): + test_index = {} + conflicted_keys = [] + + def add_to_index(key, path): + key = key or mod_name + if key in test_index: + if key not in conflicted_keys: + conflicted_keys.append(key) + mod1 = utils.extract_module_name(path) + mod2 = utils.extract_module_name(test_index[key]) + if mod1 != mod2: + # resolve conflicted keys by prefixing with the module name and a dot (.) + logger.warning("'%s' exists in both '%s' and '%s'. Resolve using `%s.%s` or `%s.%s`", + key, mod1, mod2, mod1, key, mod2, key) + test_index['{}.{}'.format(mod1, key)] = path + test_index['{}.{}'.format(mod2, key)] = test_index[key] + else: + logger.error("'%s' exists twice in the '%s' module. " + "Please rename one or both and re-run --discover.", key, mod1) + else: + test_index[key] = path + + # build the index + for mod_name, mod_data in module_data.items(): + # don't add empty mods to the index + if not mod_data: + continue + + mod_path = mod_data['filepath'] + for file_name, file_data in mod_data['files'].items(): + file_path = os.path.join(mod_path, file_name) + '.py' + for class_name, test_list in file_data.items(): + for test_name in test_list: + test_path = '{}::{}::{}'.format(file_path, class_name, test_name) + add_to_index(test_name, test_path) + class_path = '{}::{}'.format(file_path, class_name) + add_to_index(class_name, class_path) + add_to_index(file_name, file_path) + add_to_index(mod_name, mod_path) + add_to_index(mod_data['alt_name'], mod_path) + + # remove the conflicted keys since they would arbitrarily point to a random implementation + for key in conflicted_keys: + del test_index[key] + + return test_index diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py index c49c88c0f46..e3e40e35236 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py @@ -7,20 +7,21 @@ import os import sys import logging -from azdev.operations.testtool import run_tests -from azdev.utilities import EXTENSION_PREFIX +import azcli_aks_live_test.az_aks_tool.const as const +import azcli_aks_live_test.az_aks_tool.log as log import azcli_aks_live_test.az_aks_tool.utils as utils +import azcli_aks_live_test.az_aks_tool.cli as cli import azcli_aks_live_test.az_aks_tool.ext as ext -import azcli_aks_live_test.az_aks_tool.log as log - -# const -AKS_PREVIEW_MOD_NAME = EXTENSION_PREFIX + "aks_preview" # azext_aks_preview +import azcli_aks_live_test.az_aks_tool.index as index +import azcli_aks_live_test.az_aks_tool.run as run def init_argparse(args): parser = argparse.ArgumentParser() parser.add_argument("-t", "--tests", nargs='+', help="test case names") + parser.add_argument("-m", "--mode", type=str, default="all", + help="test mode ('cli', 'ext', 'all')") parser.add_argument("-cm", "--cli-matrix", type=str, help="full path to cli test matrix") parser.add_argument("-cc", "--cli-coverage", nargs="+", @@ -37,8 +38,6 @@ def init_argparse(args): default=False, help="series test") parser.add_argument("-l", "--live", action="store_true", default=False, help="live test") - parser.add_argument("-d", "--discover", action="store_true", - default=False, help="discover test index") parser.add_argument("--no-exitfirst", action="store_true", default=False, help="no exit first") parser.add_argument("--xml-file", type=str, @@ -53,24 +52,25 @@ def init_argparse(args): default="3", help="rerun times") parser.add_argument("-c", "--capture", type=str, default="sys", help="test capture") - # parser.add_argument("-a", "--pytest-args", - # nargs=argparse.REMAINDER, help="pytest args") + parser.add_argument("--log-file", type=str, + default="az_aks_tool.log", help="log filename") args = parser.parse_args(args) return args def main(): + # parse args + print("raw args: {}".format(sys.argv)) + args = init_argparse(sys.argv[1:]) + # setup logger root_module_name = log.parse_module_name(levels=1) - log.setup_logging(root_module_name) + log.setup_logging(root_module_name, os.path.join( + args.report_path, args.log_file)) current_module_name = log.parse_module_name(levels=2) logger = logging.getLogger("{}.{}".format(current_module_name, __name__)) - # parse args - logger.info("raw args: {}".format(sys.argv)) - args = init_argparse(sys.argv[1:]) - - # test cases + # check test cases test_cases = args.tests ext_matrix_file_path = args.ext_matrix cli_matrix_file_path = args.cli_matrix @@ -78,57 +78,65 @@ def main(): sys.exit( "At least one of 'tests', 'cli_matrix' and 'ext_matrix' must be provided!") - # report file - json_report_file_full_path = os.path.realpath(os.path.join( - args.report_path, args.json_report_file)) - logger.info("json report file full path: {}".format(json_report_file_full_path)) - xml_path = os.path.realpath(os.path.join(args.report_path, args.xml_file)) - logger.info("junit/xml report file full path: {}".format(xml_path)) - - # pytest args + # prepare pytest args pytest_args = [] if not args.series and args.parallelism: pytest_args.append("-n {}".format(args.parallelism)) pytest_args.append("--json-report") - pytest_args.append("--json-report-file {}".format(json_report_file_full_path)) pytest_args.append("--reruns {}".format(args.reruns)) pytest_args.append("--capture {}".format(args.capture)) pytest_args = [" ".join(pytest_args)] logger.info("pytest_args: {}".format(pytest_args)) - # ext matrix - if utils.check_file_existence(ext_matrix_file_path): - ext_test_index = ext.get_ext_test_index(AKS_PREVIEW_MOD_NAME) - ext_matrix = utils.get_test_matrix(ext_matrix_file_path) - ext_test_cases = ext.get_ext_test_cases( - ext_test_index, ext_matrix, args.ext_coverage) - ext_exclude_test_cases = ext.get_ext_exclude_test_cases(ext_test_index, - ext_matrix, args.ext_filter) - ext_filtered_test_cases = utils.get_filted_test_cases( - ext_test_cases, ext_exclude_test_cases) - # add prefix - ext_qualified_test_cases = utils.decorate_qualified_prefix( - ext_filtered_test_cases, AKS_PREVIEW_MOD_NAME) - logger.info("According to 'ext_matrix' and filters, we get {} cases, need to exclude {} cases, finally get {} cases!".format( - len(ext_test_cases), len(ext_exclude_test_cases), len(ext_filtered_test_cases))) - if len(ext_qualified_test_cases) == 0: - logger.warning("No test case! Skipping!") - else: - logger.info("Perform following tests: {}".format(ext_qualified_test_cases)) - run_tests(ext_qualified_test_cases, xml_path=xml_path, discover=args.discover, in_series=args.series, - run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) - - # cli matrix - if utils.check_file_existence(cli_matrix_file_path): - logger.warning("Currently not support!") - pass - - # tests - if test_cases: - logger.info("Accroding to 'tests', we get {} cases".format(len(test_cases))) - logger.info("Perform following tets: {}".format(test_cases)) - run_tests(test_cases, xml_path=xml_path, discover=args.discover, in_series=args.series, - run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + # check mode & collect module data + enable_cli = False + enable_ext = False + module_data = {} + if args.mode == "cli" or args.mode == "all": + enable_cli = True + module_data[const.ACS_MOD_NAME] = cli.get_cli_module_data() + cli_test_index = cli.get_cli_test_index( + module_data[const.ACS_MOD_NAME]) + + if args.mode == "ext" or args.mode == "all": + enable_ext = True + module_data[const.AKS_PREVIEW_MOD_NAME] = ext.get_ext_module_data() + ext_test_index = ext.get_ext_test_index( + module_data[const.AKS_PREVIEW_MOD_NAME]) + + # build test index + logger.info("Building test index...") + test_index = index.build_test_index(module_data) + + # cli matrix test + if enable_cli: + cli_qualified_test_cases = utils.get_fully_qualified_test_cases( + cli_test_index, cli_matrix_file_path, const.ACS_MOD_NAME, args.cli_coverage, args.cli_filter) + logger.info("Perform following cli tests: {}".format( + cli_qualified_test_cases)) + exit_code = run.run_tests(cli_qualified_test_cases, test_index, mode="cli", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series, + run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + if exit_code != 0: + sys.exit("CLI test failed!") + + # ext matrix test + if enable_ext: + ext_qualified_test_cases = utils.get_fully_qualified_test_cases( + ext_test_index, ext_matrix_file_path, const.AKS_PREVIEW_MOD_NAME, args.ext_coverage, args.ext_filter) + logger.info("Perform following ext tests: {}".format( + ext_qualified_test_cases)) + exit_code = run.run_tests(ext_qualified_test_cases, test_index, mode="ext", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series, + run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + if exit_code != 0: + sys.exit("EXT test failed!") + + # raw tests + if test_cases and len(test_cases) > 0: + logger.info("Perform following raw tets: {}".format(test_cases)) + exit_code = run_tests(test_cases, test_index, mode="raw", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series, + run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + if exit_code != 0: + sys.exit("Raw test failed!") if __name__ == "__main__": diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py index a502495f751..d31caddd6fb 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py @@ -3,17 +3,17 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import logging import os import subprocess -import logging +import sys import traceback from knack.util import CommandResultItem +from azcli_aks_live_test.az_aks_tool.const import IS_WINDOWS +from azcli_aks_live_test.az_aks_tool.utils import heading, display logger = logging.getLogger(__name__) -def display(txt): - """ Output to stderr """ - print(txt, file=sys.stderr) class ProfileContext: def __init__(self, profile_name=None): @@ -23,7 +23,8 @@ def __init__(self, profile_name=None): def __enter__(self): if self.target_profile is None or self.target_profile == self.origin_profile: - display('The tests are set to run against current profile "{}"'.format(self.origin_profile)) + logger.info('The tests are set to run against current profile "{}"'.format( + self.origin_profile)) else: result = cmd('az cloud update --profile {}'.format(self.target_profile), 'Switching to target profile "{}"...'.format(self.target_profile)) @@ -32,11 +33,11 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): if self.target_profile is not None and self.target_profile != self.origin_profile: - display('Switching back to origin profile "{}"...'.format(self.origin_profile)) + logger.info('Switching back to origin profile "{}"...'.format( + self.origin_profile)) call('az cloud update --profile {}'.format(self.origin_profile)) if exc_tb: - display('') traceback.print_exception(exc_type, exc_val, exc_tb) @@ -56,6 +57,7 @@ def call(command, **kwargs): shell=True, **kwargs) + def cmd(command, message=False, show_stderr=True, raise_error=False, **kwargs): """ Run an arbitrary command. @@ -66,14 +68,13 @@ def cmd(command, message=False, show_stderr=True, raise_error=False, **kwargs): :param kwargs: Any kwargs supported by subprocess.Popen :returns: CommandResultItem object. """ - from azdev.utilities import IS_WINDOWS, display # use default message if custom not provided if message is True: message = 'Running: {}\n'.format(command) if message: - display(message) + logger.info(message) logger.info("Running: %s", command) try: @@ -95,9 +96,11 @@ def get_test_runner(parallel, log_path, last_failed, no_exit_first, mark): def _run(test_paths, pytest_args): if os.name == 'posix': - arguments = ['-x', '-v', '--boxed', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path] + arguments = ['-x', '-v', '--boxed', '-p no:warnings', + '--log-level=WARN', '--junit-xml', log_path] else: - arguments = ['-x', '-v', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path] + arguments = ['-x', '-v', '-p no:warnings', + '--log-level=WARN', '--junit-xml', log_path] if no_exit_first: arguments.remove('-x') @@ -119,83 +122,63 @@ def _run(test_paths, pytest_args): return _run -# pylint: disable=too-many-statements,too-many-locals -# def run_tests(tests, xml_path=None, discover=False, in_series=False, -# run_live=False, profile=None, last_failed=False, pytest_args=None, -# no_exit_first=False, mark=None, -# git_source=None, git_target=None, git_repo=None, -# cli_ci=False): - - - - -# path_table = get_path_table() - -# test_index = _get_test_index(profile or current_profile(), discover) - -# if not tests: -# tests = list(path_table['mod'].keys()) + list(path_table['core'].keys()) + list(path_table['ext'].keys()) -# if tests == ['CLI']: -# tests = list(path_table['mod'].keys()) + list(path_table['core'].keys()) -# elif tests == ['EXT']: -# tests = list(path_table['ext'].keys()) - -# # filter out tests whose modules haven't changed -# modified_mods = _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo) -# if modified_mods: -# display('\nTest on modules: {}\n'.format(', '.join(modified_mods))) - -# if cli_ci is True: -# ctx = CLIAzureDevOpsContext(git_repo, git_source, git_target) -# modified_mods = ctx.filter(test_index) - - -# # process environment variables -# if run_live: -# logger.warning('RUNNING TESTS LIVE') -# os.environ[ENV_VAR_TEST_LIVE] = 'True' - -# def _find_test(index, name): -# name_comps = name.split('.') -# num_comps = len(name_comps) -# key_error = KeyError() - -# for i in range(num_comps): -# check_name = '.'.join(name_comps[(-1 - i):]) -# try: -# match = index[check_name] -# if check_name != name: -# logger.info("Test found using just '%s'. The rest of the name was ignored.\n", check_name) -# return match -# except KeyError as ex: -# key_error = ex -# continue -# raise key_error - -# # lookup test paths from index -# test_paths = [] -# for t in modified_mods: -# try: -# test_path = os.path.normpath(_find_test(test_index, t)) -# test_paths.append(test_path) -# except KeyError: -# logger.warning("'%s' not found. If newly added, re-run with --discover", t) -# continue - -# exit_code = 0 - -# # Tests have been collected. Now run them. -# if not test_paths: -# logger.warning('No tests selected to run.') -# sys.exit(exit_code) - -# exit_code = 0 -# with ProfileContext(profile): -# runner = get_test_runner(parallel=not in_series, -# log_path=xml_path, -# last_failed=last_failed, -# no_exit_first=no_exit_first, -# mark=mark) -# exit_code = runner(test_paths=test_paths, pytest_args=pytest_args) - -# sys.exit(0 if not exit_code else 1) +def run_tests(tests, test_index, mode, base_path, xml_file, json_file, in_series=False, + run_live=False, profile=None, last_failed=False, no_exit_first=False, mark=None, pytest_args=None): + + heading('Run Tests') + + # process file path + xml_path = os.path.realpath(os.path.join(base_path, mode + "_" + xml_file)) + json_path = os.path.realpath(os.path.join(base_path, mode + "_" + json_file)) + pytest_args.append("--json-report-file {}".format(json_path)) + logger.info("junit/xml report file full path: {}".format(xml_path)) + logger.info("json report file full path: {}".format(json_path)) + + # process environment variables + if run_live: + logger.warning('RUNNING TESTS LIVE') + os.environ[ENV_VAR_TEST_LIVE] = 'True' + + def _find_test(index, name): + name_comps = name.split('.') + num_comps = len(name_comps) + key_error = KeyError() + + for i in range(num_comps): + check_name = '.'.join(name_comps[(-1 - i):]) + try: + match = index[check_name] + if check_name != name: + logger.info( + "Test found using just '%s'. The rest of the name was ignored.", check_name) + return match + except KeyError as ex: + key_error = ex + continue + raise key_error + + # lookup test paths from index + test_paths = [] + for t in tests: + try: + test_path = os.path.normpath(_find_test(test_index, t)) + test_paths.append(test_path) + except KeyError: + logger.warning("'%s' not found.", t) + continue + + # Tests have been collected. Now run them. + exit_code = 0 + if not test_paths: + logger.warning('No tests selected to run.') + return exit_code + + with ProfileContext(profile): + runner = get_test_runner(parallel=not in_series, + log_path=xml_path, + last_failed=last_failed, + no_exit_first=no_exit_first, + mark=mark) + exit_code = runner(test_paths=test_paths, pytest_args=pytest_args) + + return 0 if not exit_code else 1 diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py index a7bee249e21..20e93701787 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py @@ -3,10 +3,14 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from collections import Iterable import json import os +import sys +import re import logging + +import azcli_aks_live_test.az_aks_tool.const as const +import azcli_aks_live_test.az_aks_tool.filter as custom_filter logger = logging.getLogger(__name__) @@ -23,62 +27,59 @@ def get_test_matrix(matrix_file_path): return test_matrix -def decorate_qualified_prefix(test_cases, prefix): - decorated_test_cases = ["{}.{}".format(prefix, x) for x in test_cases] - return decorated_test_cases - - -def extract_file_class_pairs(tag_list): - pairs = [] - for k in tag_list: - tags = k.split(".") - if len(tags) == 2: - pairs.append((tags[0], tags[1])) - return pairs - - -def filter_valid_file_class_pairs(pairs, test_index): - valid_pairs = [] - for pair in pairs: - if pair[0] in test_index and pair[1] in test_index[pair[0]]: - valid_pairs.append(pair) - logger.info("Valid file & class pair: '{}'".format(pair)) - else: - logger.info("Invalid file & class pair: '{}'".format(pair)) - return valid_pairs - - -def get_all_values_from_nested_dict(d): - for v in d.values(): - if isinstance(v, dict): - yield from get_all_values_from_nested_dict(v) - else: - yield v - - -def flatten_nested_list(lis): - for item in lis: - if isinstance(item, Iterable) and not isinstance(item, str): - for x in flatten_nested_list(item): - yield x - else: - yield item - - -def filter_valid_test_cases(test_cases, test_index): - valid_test_cases = [] - nested_test_cases = list(get_all_values_from_nested_dict(test_index)) - falttened_test_cases = list(flatten_nested_list(nested_test_cases)) - for test_case in test_cases: - if test_case in falttened_test_cases: - valid_test_cases.append(test_case) - logger.info("Valid test case: '{}'".format(test_case)) - else: - logger.info("Invalid test case: '{}'".format(test_case)) - return valid_test_cases - - def get_filted_test_cases(test_cases, exclude_test_cases): filtered_test_cases = [ x for x in test_cases if x not in exclude_test_cases] + logger.info("Find {} cases, exclude {} cases, finally get {} cases!".format( + len(test_cases), len(exclude_test_cases), len(filtered_test_cases))) return filtered_test_cases + + +def add_qualified_prefix(test_cases, prefix): + decorated_test_cases = ["{}.{}".format(prefix, x) for x in test_cases] + return decorated_test_cases + + +def get_fully_qualified_test_cases(test_index, matrix_file_path, mod_name, extra_coverage=None, extra_filter=None): + qualified_test_cases = [] + if check_file_existence(matrix_file_path): + matrix = get_test_matrix(matrix_file_path) + test_cases = custom_filter.get_test_cases( + test_index, matrix, extra_coverage) + exclude_test_cases = custom_filter.get_exclude_test_cases(test_index, + matrix, extra_filter) + filtered_test_cases = get_filted_test_cases( + test_cases, exclude_test_cases) + # add prefix + qualified_test_cases = add_qualified_prefix( + filtered_test_cases, mod_name) + return qualified_test_cases + + +def display(txt): + """ Output to stderr """ + print(txt, file=sys.stderr) + + +def heading(txt): + """ Create standard heading to stderr """ + line_len = len(txt) + 4 + display('\n' + '=' * line_len) + display('| {} |'.format(txt)) + display('=' * line_len + '\n') + + +def extract_module_name(path): + _CORE_NAME_REGEX = re.compile( + r'azure-cli-(?P[^/\\]+)[/\\]azure[/\\]cli') + _MOD_NAME_REGEX = re.compile( + r'azure-cli[/\\]azure[/\\]cli[/\\]command_modules[/\\](?P[^/\\]+)') + _EXT_NAME_REGEX = re.compile(r'.*(?Pazext_[^/\\]+).*') + + for expression in [_MOD_NAME_REGEX, _CORE_NAME_REGEX, _EXT_NAME_REGEX]: + match = re.search(expression, path) + if not match: + continue + return match.groupdict().get('name') + raise Exception( + 'unexpected error: unable to extract name from path: {}'.format(path)) From 12f008feee5de4b4de608689d47a614d1f19a4ae Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Mon, 10 May 2021 16:29:07 +0800 Subject: [PATCH 34/42] update unittest --- .../azcli_aks_live_test/az_aks_tool/const.py | 2 - .../azcli_aks_live_test/az_aks_tool/run.py | 2 +- .../az_aks_tool/tests/test_cli.py | 22 ++++ .../az_aks_tool/tests/test_ext.py | 61 +-------- .../az_aks_tool/tests/test_filter.py | 121 ++++++++++++++++++ .../az_aks_tool/tests/test_index.py | 34 +++++ .../az_aks_tool/tests/test_run.py | 33 +++++ .../az_aks_tool/tests/test_utils.py | 69 +++------- .../azcli_aks_live_test/az_aks_tool/utils.py | 11 +- .../azcli_aks_live_test/test_ext_live.sh | 2 +- 10 files changed, 234 insertions(+), 123 deletions(-) create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_cli.py create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_filter.py create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_index.py create mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_run.py diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/const.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/const.py index 21aa560bfae..8285bbb80c3 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/const.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/const.py @@ -14,6 +14,4 @@ ACS_MOD_NAME = "acs" AKS_PREVIEW_MOD_NAME = EXTENSION_PREFIX + "aks_preview" # azext_aks_preview -ENV_VAR_TEST_MODULES = 'AZDEV_TEST_TESTS' # comma-separated list of modules to test -ENV_VAR_VIRTUAL_ENV = ['VIRTUAL_ENV', 'CONDA_PREFIX'] # used by system to identify virtual environment ENV_VAR_TEST_LIVE = 'AZURE_TEST_RUN_LIVE' # denotes that tests should be run live instead of played back diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py index d31caddd6fb..8cb8cea790a 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py @@ -11,7 +11,7 @@ from knack.util import CommandResultItem from azcli_aks_live_test.az_aks_tool.const import IS_WINDOWS -from azcli_aks_live_test.az_aks_tool.utils import heading, display +from azcli_aks_live_test.az_aks_tool.utils import heading logger = logging.getLogger(__name__) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_cli.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_cli.py new file mode 100644 index 00000000000..013ba1ecee3 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_cli.py @@ -0,0 +1,22 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import azcli_aks_live_test.az_aks_tool.utils as utils +import azcli_aks_live_test.az_aks_tool.cli as cli + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + +class CliTestCase(unittest.TestCase): + + def test_get_cli_module_data(self): + pass + + def test_get_cli_test_index(self): + pass diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_ext.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_ext.py index 7adcbffe3d2..88abfcbf1cc 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_ext.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_ext.py @@ -15,63 +15,8 @@ class ExtTestCase(unittest.TestCase): - def test_ext_get_ext_test_index(self): + def test_get_ext_module_data(self): pass - def test_ext_get_ext_test_cases(self): - ext_test_index = { - "test_validators": { - "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], - "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], - "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] - } - } - ext_matrix = utils.get_test_matrix( - os.path.join(THIS_DIR, "testdata.json")) - base = ext_test_index["test_validators"] - s1 = sorted(ext.get_ext_test_cases(ext_test_index, ext_matrix, [])) - t1 = sorted((base["TestValidateIPRanges"] + - base["TestClusterAutoscalerParamsValidators"])) - s2 = sorted(ext.get_ext_test_cases(ext_test_index, ext_matrix, [ - "test_valid_vnet_subnet_id", "abc"])) - t2 = sorted((base["TestValidateIPRanges"] + base["TestClusterAutoscalerParamsValidators"] + - ["test_valid_vnet_subnet_id"])) - s3 = sorted(ext.get_ext_test_cases(ext_test_index, ext_matrix, [ - "test_validators.TestSubnetId"])) - t3 = sorted((base["TestValidateIPRanges"] + - base["TestClusterAutoscalerParamsValidators"] + base["TestSubnetId"])) - self.assertEqual(s1, t1) - self.assertEqual(s2, t2) - self.assertEqual(s3, t3) - - def test_ext_get_ext_exclude_test_cases(self): - ext_test_index = { - "test_validators": { - "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], - "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], - "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] - } - } - ext_matrix = utils.get_test_matrix( - os.path.join(THIS_DIR, "testdata.json")) - base = ext_test_index["test_validators"] - s1 = sorted(ext.get_ext_exclude_test_cases( - ext_test_index, ext_matrix, [])) - t1 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", - "test_disable_authorized_ip_ranges"] + base["TestClusterAutoscalerParamsValidators"]) - s2 = sorted(ext.get_ext_exclude_test_cases(ext_test_index, - ext_matrix, ["iprange"])) - t2 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", - "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges"]) - s3 = sorted(ext.get_ext_exclude_test_cases(ext_test_index, - ext_matrix, ["default", "test_invalid_subnet_id"])) - t3 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", - "test_disable_authorized_ip_ranges"] + base["TestClusterAutoscalerParamsValidators"] + ["test_invalid_subnet_id"]) - s4 = sorted(ext.get_ext_exclude_test_cases(ext_test_index, ext_matrix, [ - "test_valid_vnet_subnet_id", "abc", "test_validators.TestClusterAutoscalerParamsValidators"])) - t4 = sorted( - (base["TestClusterAutoscalerParamsValidators"] + ["test_valid_vnet_subnet_id"])) - self.assertEqual(s1, t1) - self.assertEqual(s2, t2) - self.assertEqual(s3, t3) - self.assertEqual(s4, t4) + def test_get_ext_test_index(self): + pass diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_filter.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_filter.py new file mode 100644 index 00000000000..5b419ee873d --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_filter.py @@ -0,0 +1,121 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import azcli_aks_live_test.az_aks_tool.utils as utils +import azcli_aks_live_test.az_aks_tool.filter as custom_filter + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + + +class FilterTestCase(unittest.TestCase): + + def test_extract_file_class_pairs(self): + test_cases = ["abc", "a.b", "a.b.c"] + s = custom_filter.extract_file_class_pairs(test_cases) + self.assertEqual(s, [("a", "b")]) + + def test_filter_valid_file_class_pairs(self): + pairs = [("a", "b"), ("c", "d"), ("e", "f")] + test_index = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": { + "y": ["xxx", "yyy"] + } + } + s = custom_filter.filter_valid_file_class_pairs(pairs, test_index) + self.assertEqual(s, [("a", "b")]) + + def test_get_all_values_from_nested_dict(self): + d = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": ["xxx"] + } + s = list(custom_filter.get_all_values_from_nested_dict(d)) + self.assertEqual(s, [["aaa", "bbb"], ["xxx"]]) + + def test_flatten_nested_list(self): + lis = [["aaa", "bbb"], ["xxx"]] + s = list(custom_filter.flatten_nested_list(lis)) + self.assertEqual(s, ["aaa", "bbb", "xxx"]) + + def test_filter_valid_test_cases(self): + test_cases = ["abc", "a.b", "a.b.c", "aaa", "yyy"] + test_index = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": { + "y": ["xxx", "yyy"] + } + } + s = custom_filter.filter_valid_test_cases(test_cases, test_index) + self.assertEqual(s, ["aaa", "yyy"]) + + def test_get_test_cases(self): + test_index = { + "test_validators": { + "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], + "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], + "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] + } + } + matrix = utils.get_test_matrix( + os.path.join(THIS_DIR, "testdata.json")) + base = test_index["test_validators"] + s1 = sorted(custom_filter.get_test_cases(test_index, matrix, [])) + t1 = sorted((base["TestValidateIPRanges"] + + base["TestClusterAutoscalerParamsValidators"])) + s2 = sorted(custom_filter.get_test_cases(test_index, matrix, [ + "test_valid_vnet_subnet_id", "abc"])) + t2 = sorted((base["TestValidateIPRanges"] + base["TestClusterAutoscalerParamsValidators"] + + ["test_valid_vnet_subnet_id"])) + s3 = sorted(custom_filter.get_test_cases(test_index, matrix, [ + "test_validators.TestSubnetId"])) + t3 = sorted((base["TestValidateIPRanges"] + + base["TestClusterAutoscalerParamsValidators"] + base["TestSubnetId"])) + self.assertEqual(s1, t1) + self.assertEqual(s2, t2) + self.assertEqual(s3, t3) + + def test_get_exclude_test_cases(self): + test_index = { + "test_validators": { + "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], + "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], + "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] + } + } + matrix = utils.get_test_matrix( + os.path.join(THIS_DIR, "testdata.json")) + base = test_index["test_validators"] + s1 = sorted(custom_filter.get_exclude_test_cases( + test_index, matrix, [])) + t1 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", + "test_disable_authorized_ip_ranges"] + base["TestClusterAutoscalerParamsValidators"]) + s2 = sorted(custom_filter.get_exclude_test_cases(test_index, + matrix, ["iprange"])) + t2 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", + "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges"]) + s3 = sorted(custom_filter.get_exclude_test_cases(test_index, + matrix, ["default", "test_invalid_subnet_id"])) + t3 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", + "test_disable_authorized_ip_ranges"] + base["TestClusterAutoscalerParamsValidators"] + ["test_invalid_subnet_id"]) + s4 = sorted(custom_filter.get_exclude_test_cases(test_index, matrix, [ + "test_valid_vnet_subnet_id", "abc", "test_validators.TestClusterAutoscalerParamsValidators"])) + t4 = sorted( + (base["TestClusterAutoscalerParamsValidators"] + ["test_valid_vnet_subnet_id"])) + self.assertEqual(s1, t1) + self.assertEqual(s2, t2) + self.assertEqual(s3, t3) + self.assertEqual(s4, t4) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_index.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_index.py new file mode 100644 index 00000000000..98af3d30bb9 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_index.py @@ -0,0 +1,34 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import azcli_aks_live_test.az_aks_tool.utils as utils +import azcli_aks_live_test.az_aks_tool.index as index + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + +class IndexTestCase(unittest.TestCase): + + def test_get_repo_path(self): + pass + + def test_find_files(self): + pass + + def test_get_name_index(self): + pass + + def test_get_path_table(self): + pass + + def test_discover_module_tests(self): + pass + + def build_test_index(self): + pass diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_run.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_run.py new file mode 100644 index 00000000000..1b7664e099c --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_run.py @@ -0,0 +1,33 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import azcli_aks_live_test.az_aks_tool.utils as utils +import azcli_aks_live_test.az_aks_tool.cli as cli + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + +class RunTestCase(unittest.TestCase): + + def test_current_profile(self): + pass + + def test_call(self): + pass + + def test_cmd(self): + pass + + def test_get_test_runner(self): + pass + + def run_tests(self): + pass + + diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_utils.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_utils.py index e96be8a650e..c9791e75868 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_utils.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_utils.py @@ -15,7 +15,7 @@ class UtilsTestCase(unittest.TestCase): - def test_utils_check_file_existence(self): + def test_check_file_existence(self): s1 = utils.check_file_existence(None) s2 = utils.check_file_existence(THIS_DIR) s3 = utils.check_file_existence(THIS_FILE) @@ -26,61 +26,10 @@ def test_utils_check_file_existence(self): self.assertEqual(s3, True) self.assertEqual(s4, True) - def test_utils_get_test_matrix(self): + def test_get_test_matrix(self): s = utils.get_test_matrix(os.path.join(THIS_DIR, "testdata.json")) self.assertEqual(len(s), 2) # number of keys - def test_utils_decorate_qualified_prefix(self): - test_cases = ["a", "b", "c"] - s = utils.decorate_qualified_prefix(test_cases, "p") - self.assertEqual(s, ["p.a", "p.b", "p.c"]) - - def test_utils_extract_file_class_pairs(self): - test_cases = ["abc", "a.b", "a.b.c"] - s = utils.extract_file_class_pairs(test_cases) - self.assertEqual(s, [("a", "b")]) - - def test_utils_filter_valid_file_class_pairs(self): - pairs = [("a", "b"), ("c", "d"), ("e", "f")] - test_index = { - "a": { - "b": ["aaa", "bbb"] - }, - "x": { - "y": ["xxx", "yyy"] - } - } - s = utils.filter_valid_file_class_pairs(pairs, test_index) - self.assertEqual(s, [("a", "b")]) - - def test_utils_get_all_values_from_nested_dict(self): - d = { - "a": { - "b": ["aaa", "bbb"] - }, - "x": ["xxx"] - } - s = list(utils.get_all_values_from_nested_dict(d)) - self.assertEqual(s, [["aaa", "bbb"], ["xxx"]]) - - def test_utils_flatten_nested_list(self): - lis = [["aaa", "bbb"], ["xxx"]] - s = list(utils.flatten_nested_list(lis)) - self.assertEqual(s, ["aaa", "bbb", "xxx"]) - - def test_utils_filter_valid_test_cases(self): - test_cases = ["abc", "a.b", "a.b.c", "aaa", "yyy"] - test_index = { - "a": { - "b": ["aaa", "bbb"] - }, - "x": { - "y": ["xxx", "yyy"] - } - } - s = utils.filter_valid_test_cases(test_cases, test_index) - self.assertEqual(s, ["aaa", "yyy"]) - def test_get_filted_test_cases(self): test_cases = ["a", "b", "c", "d"] s1 = utils.get_filted_test_cases(test_cases, []) @@ -89,3 +38,17 @@ def test_get_filted_test_cases(self): t2 = ["b", "c", "d"] self.assertEqual(s1, t1) self.assertEqual(s2, t2) + + def test_add_qualified_prefix(self): + test_cases = ["a", "b", "c"] + s = utils.add_qualified_prefix(test_cases, "p") + self.assertEqual(s, ["p.a", "p.b", "p.c"]) + + def test_get_fully_qualified_test_cases(self): + pass + + def test_heading(self): + pass + + def test_extract_module_name(self): + pass diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py index 20e93701787..3bdb9ce5bb8 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py @@ -56,17 +56,12 @@ def get_fully_qualified_test_cases(test_index, matrix_file_path, mod_name, extra return qualified_test_cases -def display(txt): - """ Output to stderr """ - print(txt, file=sys.stderr) - - def heading(txt): """ Create standard heading to stderr """ line_len = len(txt) + 4 - display('\n' + '=' * line_len) - display('| {} |'.format(txt)) - display('=' * line_len + '\n') + print('\n' + '=' * line_len, file=sys.stderr) + print('| {} |'.format(txt), file=sys.stderr) + print('=' * line_len + '\n', file=sys.stderr) def extract_module_name(path): diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh index 5c2e0f2be5c..4dc8506188d 100755 --- a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh @@ -36,7 +36,7 @@ do done # prepare run flags -run_flags="-em ext_matrix_default.json --no-exitfirst --discover --report-path ./ --reruns 3 --capture=sys" +run_flags="-m ext -em ext_matrix_default.json --no-exitfirst --report-path ./ --reruns 3 --capture=sys" # parallel if [ $PARALLELISM -ge 2 ]; then run_flags+=" -n $PARALLELISM" From 8496d62bddfc8420a47a574269184f0115756d86 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Mon, 10 May 2021 16:41:44 +0800 Subject: [PATCH 35/42] fix naming conflict --- src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py | 8 ++++++-- src/aks-preview/azcli_aks_live_test/test_cli_live.sh | 4 ++-- src/aks-preview/azcli_aks_live_test/test_ext_live.sh | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py index 8cb8cea790a..a639f0736c7 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py @@ -128,8 +128,12 @@ def run_tests(tests, test_index, mode, base_path, xml_file, json_file, in_series heading('Run Tests') # process file path - xml_path = os.path.realpath(os.path.join(base_path, mode + "_" + xml_file)) - json_path = os.path.realpath(os.path.join(base_path, mode + "_" + json_file)) + if not xml_file.startswith(mode): + xml_file = "{}_{}".format(mode, xml_file) + if not json_file.startswith(mode): + json_file = "{}_{}".format(mode, json_file) + xml_path = os.path.realpath(os.path.join(base_path, xml_file)) + json_path = os.path.realpath(os.path.join(base_path, json_file)) pytest_args.append("--json-report-file {}".format(json_path)) logger.info("junit/xml report file full path: {}".format(xml_path)) logger.info("json report file full path: {}".format(json_path)) diff --git a/src/aks-preview/azcli_aks_live_test/test_cli_live.sh b/src/aks-preview/azcli_aks_live_test/test_cli_live.sh index b9c05233760..56e7a7c085b 100755 --- a/src/aks-preview/azcli_aks_live_test/test_cli_live.sh +++ b/src/aks-preview/azcli_aks_live_test/test_cli_live.sh @@ -17,7 +17,7 @@ fi if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then echo "Test in record mode!" azdev test acs --no-exitfirst --xml-path cli_result.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_report.json --reruns 3 --capture=sys" - cp cli_report.json cli_result.xml reports/ + cp *cli_report.json *cli_result.xml reports/ fi if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then @@ -26,5 +26,5 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then az account set -s $AZCLI_ALT_SUBSCRIPTION_ID az account show azdev test acs --live --no-exitfirst --xml-path cli_live_result.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_live_report.json --reruns 3 --capture=sys" - cp cli_live_report.json cli_live_result.xml reports/ + cp *cli_live_report.json *cli_live_result.xml reports/ fi diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh index 4dc8506188d..becc829fd04 100755 --- a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh @@ -59,7 +59,7 @@ if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then run_flags+=" --xml-file=ext_result.xml" echo "run flags: ${run_flags}" echo ${run_flags} | xargs python -u az_aks_tool/main.py - cp ext_report.json ext_result.xml reports/ + cp *ext_report.json *ext_result.xml reports/ fi # live test @@ -72,5 +72,5 @@ if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then run_flags+=" --xml-file=ext_live_result.xml" echo "run flags: ${run_flags}" echo ${run_flags} | xargs python -u az_aks_tool/main.py - cp ext_live_report.json ext_live_result.xml reports/ + cp *ext_live_report.json *ext_live_result.xml reports/ fi From 6fccd93921c474f2d423bec7d78c02e92cc58c51 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Mon, 10 May 2021 17:11:05 +0800 Subject: [PATCH 36/42] update readme & fix live run bug --- src/aks-preview/azcli_aks_live_test/README.md | 10 ++++------ src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py | 2 +- src/aks-preview/azcli_aks_live_test/test_ext_live.sh | 8 ++++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/aks-preview/azcli_aks_live_test/README.md b/src/aks-preview/azcli_aks_live_test/README.md index 649b2031e8e..206a5eae203 100644 --- a/src/aks-preview/azcli_aks_live_test/README.md +++ b/src/aks-preview/azcli_aks_live_test/README.md @@ -1,11 +1,9 @@ -# Azure CLI AKS Live Test Pipeline +# Azure CLI AKS Live Test Pipeline & Azure CLI AKS Unit Test Pipeline -This is a pipeline to test newly added aks commands in aks-preview/acs. +These pipelines are used to test newly added aks commands in module aks-preview (azure-cli-extensions) / acs (azure-cli, not covered by default). ## How to use -**By default**, this pipeline will be **triggered** when submitting a **PR** to the master branch of the official repo which involves modifying the file under src/aks-preview and test the acs command group in azure-cli and the aks-preview command group in azure-cli-extensions. If the variables of the pipeline are not modified, the test will be executed based on the latest commit of the dev branch in the offical repo of azure-cli. The test will be performed in record mode first, and then in live mode. After the test, you can get the test results from the pipeline artifact (for different modes (record/live) and different modules (cli/ext)). +**By default**, these pipelines will be **triggered** when submitting a **PR** to the master branch of the official repo which involves modifying the files under src/aks-preview. Then they will test the aks-preview command group in azure-cli-extensions. For live test pipeline, the test will be performed in record mode first, and then in live mode. Due to some specific [reasons](https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/157433/Live-Test-Failures-in-aks-preview-(with-bare-sub)), some test cases would fail. These test cases have been filtered out by file 'ext_matrix_default.json'. For unit test pipeline, the test will be perfomed with 'unittest' and 'pytest' modules. A code coverage report will be generated after the unit tests. You can find test reports and coverage report from pipeline artifacts. -You can also trigger this pipeline **manually**. In this way, you **must** set the variable *MANUAL_EXT* to true before running the pipeline, and provide the your *EXT_REPO/CLI_REPO* url and *EXT_BRANCH/CLI_BRANCH* name at the same time. - -For more details, you may refer to this [wiki](https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/Azure-CLI-AKS-Live-Test-Pipeline). +You can also trigger this pipeline **manually**. For more details, you may refer to this [wiki](https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/Azure-CLI-AKS-Live-Test-Pipeline). diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py index a639f0736c7..195f3d3bc12 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py +++ b/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py @@ -10,7 +10,7 @@ import traceback from knack.util import CommandResultItem -from azcli_aks_live_test.az_aks_tool.const import IS_WINDOWS +from azcli_aks_live_test.az_aks_tool.const import IS_WINDOWS, ENV_VAR_TEST_LIVE from azcli_aks_live_test.az_aks_tool.utils import heading logger = logging.getLogger(__name__) diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh index becc829fd04..36dc1b60b4d 100755 --- a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh @@ -43,12 +43,16 @@ if [ $PARALLELISM -ge 2 ]; then else run_flags+=" -s" fi -# ext filter +# test cases +if [[ -n $TEST_CASES ]]; then + run_flags+=" -t $TEST_CASES" +fi +# ext extra filter if [[ -n $EXT_TEST_FILTER ]]; then run_flags+=" -ef $EXT_TEST_FILTER" fi # ext extra coverage -if [[ $EXT_TEST_COVERAGE ]]; then +if [[ -n $EXT_TEST_COVERAGE ]]; then run_flags+=" -ec $EXT_TEST_COVERAGE" fi From 81f7430d7f3617a24ae98357120d2e7954baec4e Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Fri, 14 May 2021 10:39:33 +0800 Subject: [PATCH 37/42] update folder structure --- src/aks-preview/az_aks_tool/.gitignore | 1 + .../__init__.py | 0 .../az_aks_tool/cli.py | 14 +++++---- .../az_aks_tool/const.py | 0 .../az_aks_tool/ext.py | 14 +++++---- .../az_aks_tool/filter.py | 0 .../az_aks_tool/index.py | 15 +++++++-- .../az_aks_tool/log.py | 0 .../az_aks_tool/main.py | 31 +++++++++---------- .../az_aks_tool/run.py | 4 +-- .../tests}/__init__.py | 0 .../az_aks_tool/tests/test_cli.py | 6 ++-- .../az_aks_tool/tests/test_ext.py | 6 ++-- .../az_aks_tool/tests/test_filter.py | 4 +-- .../az_aks_tool/tests/test_index.py | 4 +-- .../az_aks_tool/tests/test_log.py | 6 ++-- .../az_aks_tool/tests/test_main.py | 2 +- .../az_aks_tool/tests/test_run.py | 4 +-- .../az_aks_tool/tests/test_utils.py | 2 +- .../az_aks_tool/tests/testdata.json | 0 .../az_aks_tool/utils.py | 4 +-- .../az_aks_tool/tests/__init__.py | 0 .../azcli_aks_live_test/clone_repo.sh | 3 +- .../azcli_aks_live_test/setup_venv.sh | 3 ++ .../azcli_aks_live_test/test_ext_unit.sh | 16 +++++----- 25 files changed, 76 insertions(+), 63 deletions(-) create mode 100644 src/aks-preview/az_aks_tool/.gitignore rename src/aks-preview/{azcli_aks_live_test => az_aks_tool}/__init__.py (100%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/cli.py (80%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/const.py (100%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/ext.py (82%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/filter.py (100%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/index.py (95%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/log.py (100%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/main.py (87%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/run.py (97%) rename src/aks-preview/{azcli_aks_live_test/az_aks_tool => az_aks_tool/tests}/__init__.py (100%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/tests/test_cli.py (80%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/tests/test_ext.py (80%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/tests/test_filter.py (98%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/tests/test_index.py (88%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/tests/test_log.py (81%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/tests/test_main.py (92%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/tests/test_run.py (87%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/tests/test_utils.py (97%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/tests/testdata.json (100%) rename src/aks-preview/{azcli_aks_live_test => }/az_aks_tool/utils.py (96%) delete mode 100644 src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/__init__.py diff --git a/src/aks-preview/az_aks_tool/.gitignore b/src/aks-preview/az_aks_tool/.gitignore new file mode 100644 index 00000000000..bee8a64b79a --- /dev/null +++ b/src/aks-preview/az_aks_tool/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/src/aks-preview/azcli_aks_live_test/__init__.py b/src/aks-preview/az_aks_tool/__init__.py similarity index 100% rename from src/aks-preview/azcli_aks_live_test/__init__.py rename to src/aks-preview/az_aks_tool/__init__.py diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/cli.py b/src/aks-preview/az_aks_tool/cli.py similarity index 80% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/cli.py rename to src/aks-preview/az_aks_tool/cli.py index ad608a2698d..ed21d2ce0a2 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/cli.py +++ b/src/aks-preview/az_aks_tool/cli.py @@ -7,12 +7,12 @@ import os import logging -import azcli_aks_live_test.az_aks_tool.const as const -import azcli_aks_live_test.az_aks_tool.index as index +import az_aks_tool.const as const +import az_aks_tool.index as index logger = logging.getLogger(__name__) -def get_cli_module_data(mod_name=const.ACS_MOD_NAME, profile="latest"): +def get_cli_mod_data(mod_name=const.ACS_MOD_NAME, profile="latest"): profile_split = profile.split('-') profile_namespace = '_'.join([profile_split[-1]] + profile_split[:-1]) @@ -35,6 +35,8 @@ def get_cli_module_data(mod_name=const.ACS_MOD_NAME, profile="latest"): def get_cli_test_index(module_data=None, mod_name=const.ACS_MOD_NAME, profile="latest"): - if not module_data: - module_data = get_cli_module_data(mod_name=mod_name, profile=profile) - return module_data["files"] + if mod_name in module_data: + mod_data = module_data[mod_name] + else: + mod_data = get_cli_mod_data(mod_name=mod_name, profile=profile) + return mod_data["files"] diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/const.py b/src/aks-preview/az_aks_tool/const.py similarity index 100% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/const.py rename to src/aks-preview/az_aks_tool/const.py diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/ext.py b/src/aks-preview/az_aks_tool/ext.py similarity index 82% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/ext.py rename to src/aks-preview/az_aks_tool/ext.py index 608b545b9c9..8fa39c1e242 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/ext.py +++ b/src/aks-preview/az_aks_tool/ext.py @@ -7,12 +7,12 @@ import os import logging -import azcli_aks_live_test.az_aks_tool.const as const -import azcli_aks_live_test.az_aks_tool.index as index +import az_aks_tool.const as const +import az_aks_tool.index as index logger = logging.getLogger(__name__) -def get_ext_module_data(mod_name=const.AKS_PREVIEW_MOD_NAME, profile="latest"): +def get_ext_mod_data(mod_name=const.AKS_PREVIEW_MOD_NAME, profile="latest"): profile_split = profile.split('-') profile_namespace = '_'.join([profile_split[-1]] + profile_split[:-1]) @@ -39,6 +39,8 @@ def get_ext_module_data(mod_name=const.AKS_PREVIEW_MOD_NAME, profile="latest"): def get_ext_test_index(module_data=None, mod_name=const.AKS_PREVIEW_MOD_NAME, profile="latest"): - if not module_data: - module_data = get_ext_module_data(mod_name=mod_name, profile=profile) - return module_data["files"] + if mod_name in module_data: + mod_data = module_data[mod_name] + else: + mod_data = get_ext_mod_data(mod_name=mod_name, profile=profile) + return mod_data["files"] diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/filter.py b/src/aks-preview/az_aks_tool/filter.py similarity index 100% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/filter.py rename to src/aks-preview/az_aks_tool/filter.py diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/index.py b/src/aks-preview/az_aks_tool/index.py similarity index 95% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/index.py rename to src/aks-preview/az_aks_tool/index.py index abb8336f787..cf3ec208b11 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/index.py +++ b/src/aks-preview/az_aks_tool/index.py @@ -8,11 +8,18 @@ import os from importlib import import_module -import azcli_aks_live_test.az_aks_tool.utils as utils -import azcli_aks_live_test.az_aks_tool.const as const +import az_aks_tool.utils as utils +import az_aks_tool.const as const logger = logging.getLogger(__name__) + def get_repo_path(repo_name, root_path=None): + # find cache from environment variable + repo_path = os.environ.get("{}_PATH".format(repo_name), "") + if os.path.isdir(repo_path): + logger.info("Find cached '{}' repo path: '{}'".format(repo_name, repo_path)) + return repo_path + # search from root_path valid_repo_paths = [] if root_path is None or not os.path.isdir(root_path): logger.warning("Invalid root path '{}', setting root path as current work dir '{}'".format(root_path, os.getcwd())) @@ -25,9 +32,11 @@ def get_repo_path(repo_name, root_path=None): if len(valid_repo_paths) >= 2: logger.warning("Find {} '{}' repo paths: {}".format(len(valid_repo_paths), repo_name, valid_repo_paths)) logger.info("Get '{}' repo path as '{}'".format(repo_name, repo_path)) + os.environ["{}_PATH".format(repo_name)] = str(repo_path) return repo_path else: - raise Exception("Could not find valid path to repo '{}' from '{}'".format(repo_name, root_path)) + logger.warning("Could not find valid path to repo '{}' from '{}'".format(repo_name, root_path)) + return "" def find_files(root_paths, file_pattern): """ Returns the paths to all files that match a given pattern. diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py b/src/aks-preview/az_aks_tool/log.py similarity index 100% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/log.py rename to src/aks-preview/az_aks_tool/log.py diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py b/src/aks-preview/az_aks_tool/main.py similarity index 87% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py rename to src/aks-preview/az_aks_tool/main.py index e3e40e35236..f8b3788c983 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/main.py +++ b/src/aks-preview/az_aks_tool/main.py @@ -8,20 +8,20 @@ import sys import logging -import azcli_aks_live_test.az_aks_tool.const as const -import azcli_aks_live_test.az_aks_tool.log as log -import azcli_aks_live_test.az_aks_tool.utils as utils -import azcli_aks_live_test.az_aks_tool.cli as cli -import azcli_aks_live_test.az_aks_tool.ext as ext -import azcli_aks_live_test.az_aks_tool.index as index -import azcli_aks_live_test.az_aks_tool.run as run +import az_aks_tool.const as const +import az_aks_tool.log as log +import az_aks_tool.utils as utils +import az_aks_tool.cli as cli +import az_aks_tool.ext as ext +import az_aks_tool.index as index +import az_aks_tool.run as run def init_argparse(args): parser = argparse.ArgumentParser() - parser.add_argument("-t", "--tests", nargs='+', help="test case names") parser.add_argument("-m", "--mode", type=str, default="all", help="test mode ('cli', 'ext', 'all')") + parser.add_argument("-t", "--tests", nargs='+', help="test case names") parser.add_argument("-cm", "--cli-matrix", type=str, help="full path to cli test matrix") parser.add_argument("-cc", "--cli-coverage", nargs="+", @@ -38,7 +38,7 @@ def init_argparse(args): default=False, help="series test") parser.add_argument("-l", "--live", action="store_true", default=False, help="live test") - parser.add_argument("--no-exitfirst", action="store_true", + parser.add_argument("-ne", "--no-exitfirst", action="store_true", default=False, help="no exit first") parser.add_argument("--xml-file", type=str, default="azcli_aks_runner.xml", help="junit/xml report filename") @@ -67,8 +67,7 @@ def main(): root_module_name = log.parse_module_name(levels=1) log.setup_logging(root_module_name, os.path.join( args.report_path, args.log_file)) - current_module_name = log.parse_module_name(levels=2) - logger = logging.getLogger("{}.{}".format(current_module_name, __name__)) + logger = logging.getLogger("{}.{}".format(root_module_name, __name__)) # check test cases test_cases = args.tests @@ -94,15 +93,13 @@ def main(): module_data = {} if args.mode == "cli" or args.mode == "all": enable_cli = True - module_data[const.ACS_MOD_NAME] = cli.get_cli_module_data() - cli_test_index = cli.get_cli_test_index( - module_data[const.ACS_MOD_NAME]) + module_data[const.ACS_MOD_NAME] = cli.get_cli_mod_data() + cli_test_index = cli.get_cli_test_index(module_data) if args.mode == "ext" or args.mode == "all": enable_ext = True - module_data[const.AKS_PREVIEW_MOD_NAME] = ext.get_ext_module_data() - ext_test_index = ext.get_ext_test_index( - module_data[const.AKS_PREVIEW_MOD_NAME]) + module_data[const.AKS_PREVIEW_MOD_NAME] = ext.get_ext_mod_data() + ext_test_index = ext.get_ext_test_index(module_data) # build test index logger.info("Building test index...") diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py b/src/aks-preview/az_aks_tool/run.py similarity index 97% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py rename to src/aks-preview/az_aks_tool/run.py index 195f3d3bc12..05300f218d3 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/run.py +++ b/src/aks-preview/az_aks_tool/run.py @@ -10,8 +10,8 @@ import traceback from knack.util import CommandResultItem -from azcli_aks_live_test.az_aks_tool.const import IS_WINDOWS, ENV_VAR_TEST_LIVE -from azcli_aks_live_test.az_aks_tool.utils import heading +from az_aks_tool.const import IS_WINDOWS, ENV_VAR_TEST_LIVE +from az_aks_tool.utils import heading logger = logging.getLogger(__name__) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/__init__.py b/src/aks-preview/az_aks_tool/tests/__init__.py similarity index 100% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/__init__.py rename to src/aks-preview/az_aks_tool/tests/__init__.py diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_cli.py b/src/aks-preview/az_aks_tool/tests/test_cli.py similarity index 80% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_cli.py rename to src/aks-preview/az_aks_tool/tests/test_cli.py index 013ba1ecee3..5b9000b7af6 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_cli.py +++ b/src/aks-preview/az_aks_tool/tests/test_cli.py @@ -6,8 +6,8 @@ import os import unittest -import azcli_aks_live_test.az_aks_tool.utils as utils -import azcli_aks_live_test.az_aks_tool.cli as cli +import az_aks_tool.utils as utils +import az_aks_tool.cli as cli THIS_FILE = os.path.abspath(__file__) THIS_DIR = os.path.dirname(THIS_FILE) @@ -15,7 +15,7 @@ class CliTestCase(unittest.TestCase): - def test_get_cli_module_data(self): + def test_get_cli_mod_data(self): pass def test_get_cli_test_index(self): diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_ext.py b/src/aks-preview/az_aks_tool/tests/test_ext.py similarity index 80% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_ext.py rename to src/aks-preview/az_aks_tool/tests/test_ext.py index 88abfcbf1cc..ec194f16641 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_ext.py +++ b/src/aks-preview/az_aks_tool/tests/test_ext.py @@ -6,8 +6,8 @@ import os import unittest -import azcli_aks_live_test.az_aks_tool.utils as utils -import azcli_aks_live_test.az_aks_tool.ext as ext +import az_aks_tool.utils as utils +import az_aks_tool.ext as ext THIS_FILE = os.path.abspath(__file__) THIS_DIR = os.path.dirname(THIS_FILE) @@ -15,7 +15,7 @@ class ExtTestCase(unittest.TestCase): - def test_get_ext_module_data(self): + def test_get_ext_mod_data(self): pass def test_get_ext_test_index(self): diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_filter.py b/src/aks-preview/az_aks_tool/tests/test_filter.py similarity index 98% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_filter.py rename to src/aks-preview/az_aks_tool/tests/test_filter.py index 5b419ee873d..d9da9e6a624 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_filter.py +++ b/src/aks-preview/az_aks_tool/tests/test_filter.py @@ -6,8 +6,8 @@ import os import unittest -import azcli_aks_live_test.az_aks_tool.utils as utils -import azcli_aks_live_test.az_aks_tool.filter as custom_filter +import az_aks_tool.utils as utils +import az_aks_tool.filter as custom_filter THIS_FILE = os.path.abspath(__file__) THIS_DIR = os.path.dirname(THIS_FILE) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_index.py b/src/aks-preview/az_aks_tool/tests/test_index.py similarity index 88% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_index.py rename to src/aks-preview/az_aks_tool/tests/test_index.py index 98af3d30bb9..68b5816db2f 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_index.py +++ b/src/aks-preview/az_aks_tool/tests/test_index.py @@ -6,8 +6,8 @@ import os import unittest -import azcli_aks_live_test.az_aks_tool.utils as utils -import azcli_aks_live_test.az_aks_tool.index as index +import az_aks_tool.utils as utils +import az_aks_tool.index as index THIS_FILE = os.path.abspath(__file__) THIS_DIR = os.path.dirname(THIS_FILE) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_log.py b/src/aks-preview/az_aks_tool/tests/test_log.py similarity index 81% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_log.py rename to src/aks-preview/az_aks_tool/tests/test_log.py index 9c6888c75a5..4caafc341ba 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_log.py +++ b/src/aks-preview/az_aks_tool/tests/test_log.py @@ -7,7 +7,7 @@ import os import unittest -import azcli_aks_live_test.az_aks_tool.log as log +import az_aks_tool.log as log THIS_FILE = os.path.abspath(__file__) THIS_DIR = os.path.dirname(THIS_FILE) @@ -18,10 +18,8 @@ class LogTestCase(unittest.TestCase): def test_parse_module_name(self): root_module_name = log.parse_module_name(levels=1) - current_module_name = log.parse_module_name(levels=2) error_module_name = log.parse_module_name(levels=5) - self.assertEqual(root_module_name, "azcli_aks_live_test") - self.assertEqual(current_module_name, "azcli_aks_live_test.az_aks_tool") + self.assertEqual(root_module_name, "az_aks_tool") self.assertEqual(error_module_name, None) def test_setup_logging(self): diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_main.py b/src/aks-preview/az_aks_tool/tests/test_main.py similarity index 92% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_main.py rename to src/aks-preview/az_aks_tool/tests/test_main.py index 50ba6386643..36f4becd0f4 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_main.py +++ b/src/aks-preview/az_aks_tool/tests/test_main.py @@ -7,7 +7,7 @@ import os import unittest -import azcli_aks_live_test.az_aks_tool.main as main +import az_aks_tool.main as main THIS_FILE = os.path.abspath(__file__) THIS_DIR = os.path.dirname(THIS_FILE) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_run.py b/src/aks-preview/az_aks_tool/tests/test_run.py similarity index 87% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_run.py rename to src/aks-preview/az_aks_tool/tests/test_run.py index 1b7664e099c..d69375d491d 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_run.py +++ b/src/aks-preview/az_aks_tool/tests/test_run.py @@ -6,8 +6,8 @@ import os import unittest -import azcli_aks_live_test.az_aks_tool.utils as utils -import azcli_aks_live_test.az_aks_tool.cli as cli +import az_aks_tool.utils as utils +import az_aks_tool.cli as cli THIS_FILE = os.path.abspath(__file__) THIS_DIR = os.path.dirname(THIS_FILE) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_utils.py b/src/aks-preview/az_aks_tool/tests/test_utils.py similarity index 97% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_utils.py rename to src/aks-preview/az_aks_tool/tests/test_utils.py index c9791e75868..6bef0b49a0f 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/test_utils.py +++ b/src/aks-preview/az_aks_tool/tests/test_utils.py @@ -6,7 +6,7 @@ import os import unittest -import azcli_aks_live_test.az_aks_tool.utils as utils +import az_aks_tool.utils as utils THIS_FILE = os.path.abspath(__file__) THIS_DIR = os.path.dirname(THIS_FILE) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/testdata.json b/src/aks-preview/az_aks_tool/tests/testdata.json similarity index 100% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/testdata.json rename to src/aks-preview/az_aks_tool/tests/testdata.json diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py b/src/aks-preview/az_aks_tool/utils.py similarity index 96% rename from src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py rename to src/aks-preview/az_aks_tool/utils.py index 3bdb9ce5bb8..3a07634bd39 100644 --- a/src/aks-preview/azcli_aks_live_test/az_aks_tool/utils.py +++ b/src/aks-preview/az_aks_tool/utils.py @@ -9,8 +9,8 @@ import re import logging -import azcli_aks_live_test.az_aks_tool.const as const -import azcli_aks_live_test.az_aks_tool.filter as custom_filter +import az_aks_tool.const as const +import az_aks_tool.filter as custom_filter logger = logging.getLogger(__name__) diff --git a/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/__init__.py b/src/aks-preview/azcli_aks_live_test/az_aks_tool/tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/aks-preview/azcli_aks_live_test/clone_repo.sh b/src/aks-preview/azcli_aks_live_test/clone_repo.sh index c6a21ed82d1..4dccf3260d4 100755 --- a/src/aks-preview/azcli_aks_live_test/clone_repo.sh +++ b/src/aks-preview/azcli_aks_live_test/clone_repo.sh @@ -30,5 +30,6 @@ git log -10 popd # copy live test related files to the same level as the checkout directory ($(Agent.BuildDirectory)/s) -cp -r azure-cli-extensions/src/aks-preview/azcli_aks_live_test/* ./ +cp -rT azure-cli-extensions/src/aks-preview/azcli_aks_live_test/ ./ +cp -r azure-cli-extensions/src/aks-preview/az_aks_tool/ ./ ls -alh diff --git a/src/aks-preview/azcli_aks_live_test/setup_venv.sh b/src/aks-preview/azcli_aks_live_test/setup_venv.sh index 935ce03507f..642d9a7c436 100755 --- a/src/aks-preview/azcli_aks_live_test/setup_venv.sh +++ b/src/aks-preview/azcli_aks_live_test/setup_venv.sh @@ -33,3 +33,6 @@ az version # mkdir to store reports mkdir -p reports/ + +# export PYTHONPATH +export PYTHONPATH=$(pwd) diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh b/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh index fab2a7b09a8..a83fa7568ea 100755 --- a/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh @@ -36,20 +36,20 @@ do done # unit test & coverage report -# azcli_aks_live_test -azcli_aks_live_test_unit_test_result="" -pushd azure-cli-extensions/src/aks-preview/azcli_aks_live_test/ +# az_aks_tool +az_aks_tool_unit_test_result="" +pushd azure-cli-extensions/src/aks-preview/az_aks_tool/ # clean existing coverage report (coverage combine || true) && (coverage erase || true) if ! coverage run --source=. --omit=*/tests/* -p -m unittest discover; then - azcli_aks_live_test_unit_test_result="error" + az_aks_tool_unit_test_result="error" fi -# currently no test written in pytest format under 'azcli_aks_live_test/' +# currently no test written in pytest format under 'az_aks_tool/' # coverage run --source=. --omit=*/tests/* -p -m pytest -coverage combine && coverage json -o coverage_azcli_aks_live_test.json +coverage combine && coverage json -o coverage_az_aks_tool.json coverage report -m popd -cp azure-cli-extensions/src/aks-preview/azcli_aks_live_test/coverage_azcli_aks_live_test.json reports/ +cp azure-cli-extensions/src/aks-preview/az_aks_tool/coverage_az_aks_tool.json reports/ # azext_aks_preview azext_aks_preview_unit_test_result="" @@ -65,7 +65,7 @@ coverage report -m popd cp azure-cli-extensions/src/aks-preview/azext_aks_preview/coverage_azext_aks_preview.json reports/ -if [[ $azcli_aks_live_test_unit_test_result == "error" || $azext_aks_preview_unit_test_result == "error" ]]; then +if [[ $az_aks_tool_unit_test_result == "error" || $azext_aks_preview_unit_test_result == "error" ]]; then echo "Unit test failed!" exit 1 fi From 5e87e34ce0a236b26541ca28fda47198a33a5c60 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Fri, 14 May 2021 11:59:48 +0800 Subject: [PATCH 38/42] update test --- .../azext_aks_preview/tests/latest/test_aks_commands.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index c91b77ae0de..a1eceef5f18 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -5,12 +5,14 @@ import unittest import os -from .recording_processors import KeyReplacer -from .custom_preparers import AKSCustomResourceGroupPreparer -from azure.cli.testsdk import (RoleBasedServicePrincipalPreparer, ScenarioTest, live_only) +from azure.cli.testsdk import ( + ResourceGroupPreparer, RoleBasedServicePrincipalPreparer, ScenarioTest, live_only) from azure_devtools.scenario_tests import AllowLargeResponse +from .recording_processors import KeyReplacer +from .custom_preparers import AKSCustomResourceGroupPreparer + def _get_test_data_file(filename): curr_dir = os.path.dirname(os.path.realpath(__file__)) From 5aefd43798c247a4405a5480862f00aacd12d695 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Fri, 14 May 2021 14:59:01 +0800 Subject: [PATCH 39/42] update pipeline --- src/aks-preview/az_aks_tool/index.py | 36 ++++++++++------- src/aks-preview/az_aks_tool/log.py | 4 +- src/aks-preview/az_aks_tool/main.py | 39 +++++++++++-------- .../azcli_aks_live_test/test_ext_live.sh | 4 +- 4 files changed, 48 insertions(+), 35 deletions(-) diff --git a/src/aks-preview/az_aks_tool/index.py b/src/aks-preview/az_aks_tool/index.py index cf3ec208b11..77333655492 100644 --- a/src/aks-preview/az_aks_tool/index.py +++ b/src/aks-preview/az_aks_tool/index.py @@ -19,24 +19,32 @@ def get_repo_path(repo_name, root_path=None): if os.path.isdir(repo_path): logger.info("Find cached '{}' repo path: '{}'".format(repo_name, repo_path)) return repo_path + # search from root_path + candidate_root_paths = [os.getcwd(), os.path.expanduser("~")] valid_repo_paths = [] + repo_path = "" if root_path is None or not os.path.isdir(root_path): - logger.warning("Invalid root path '{}', setting root path as current work dir '{}'".format(root_path, os.getcwd())) - root_path = os.getcwd() - for path, _, _ in os.walk(root_path): - pattern = os.path.join(path, repo_name) - valid_repo_paths.extend(glob.glob(pattern)) - if len(valid_repo_paths) >= 1: - repo_path = valid_repo_paths[0] - if len(valid_repo_paths) >= 2: - logger.warning("Find {} '{}' repo paths: {}".format(len(valid_repo_paths), repo_name, valid_repo_paths)) - logger.info("Get '{}' repo path as '{}'".format(repo_name, repo_path)) - os.environ["{}_PATH".format(repo_name)] = str(repo_path) - return repo_path + logger.warning("Invalid root path '{}'!".format(root_path)) else: - logger.warning("Could not find valid path to repo '{}' from '{}'".format(repo_name, root_path)) - return "" + candidate_root_paths = [root_path] + candidate_root_paths + logger.info("Setting root path from '{}'".format(candidate_root_paths)) + for candidate_root_path in candidate_root_paths: + root_path = candidate_root_path + logger.info("Searching from root path: '{}'".format(root_path)) + for path, _, _ in os.walk(root_path): + pattern = os.path.join(path, repo_name) + valid_repo_paths.extend(glob.glob(pattern)) + if len(valid_repo_paths) >= 1: + repo_path = valid_repo_paths[0] + if len(valid_repo_paths) >= 2: + logger.warning("Find {} '{}' repo paths: {}".format(len(valid_repo_paths), repo_name, valid_repo_paths)) + logger.info("Set '{}' repo path as '{}'".format(repo_name, repo_path)) + os.environ["{}_PATH".format(repo_name)] = str(repo_path) + return repo_path + else: + logger.warning("Could not find valid path to repo '{}' from '{}'".format(repo_name, root_path)) + return repo_path def find_files(root_paths, file_pattern): """ Returns the paths to all files that match a given pattern. diff --git a/src/aks-preview/az_aks_tool/log.py b/src/aks-preview/az_aks_tool/log.py index 304a05fdeec..a5d27925983 100644 --- a/src/aks-preview/az_aks_tool/log.py +++ b/src/aks-preview/az_aks_tool/log.py @@ -17,8 +17,8 @@ def parse_module_name(levels=1): return module_name -def setup_logging(root_logger_name="", log_path="az_aks_tool.log"): - if root_logger_name == "": +def setup_logging(root_logger_name=None, log_path="az_aks_tool.log"): + if root_logger_name == "" or root_logger_name.isspace(): root_logger_name = parse_module_name() logger = logging.getLogger(root_logger_name) logger.setLevel(level=logging.DEBUG) diff --git a/src/aks-preview/az_aks_tool/main.py b/src/aks-preview/az_aks_tool/main.py index f8b3788c983..5cc5b94691c 100644 --- a/src/aks-preview/az_aks_tool/main.py +++ b/src/aks-preview/az_aks_tool/main.py @@ -19,8 +19,12 @@ def init_argparse(args): parser = argparse.ArgumentParser() - parser.add_argument("-m", "--mode", type=str, default="all", - help="test mode ('cli', 'ext', 'all')") + parser.add_argument("-c", "--cli", action="store_true", default=False, + help="enbale cli test") + parser.add_argument("-e", "--ext", action="store_true", default=False, + help="enbale ext test") + parser.add_argument("-a", "--all", action="store_true", default=False, + help="enbale all tests (cli & ext)") parser.add_argument("-t", "--tests", nargs='+', help="test case names") parser.add_argument("-cm", "--cli-matrix", type=str, help="full path to cli test matrix") @@ -40,18 +44,18 @@ def init_argparse(args): default=False, help="live test") parser.add_argument("-ne", "--no-exitfirst", action="store_true", default=False, help="no exit first") - parser.add_argument("--xml-file", type=str, - default="azcli_aks_runner.xml", help="junit/xml report filename") - parser.add_argument("-n", "--parallelism", type=str, + parser.add_argument("-j", "--parallelism", type=str, default="8", help="test parallelism") + parser.add_argument("--reruns", type=str, + default="3", help="rerun times") + parser.add_argument("--capture", type=str, + default="sys", help="test capture") parser.add_argument("-p", "--report-path", type=str, required=True, help="report path") parser.add_argument("-f", "--json-report-file", type=str, default="azcli_aks_runner_report.json", help="json report filename") - parser.add_argument("-r", "--reruns", type=str, - default="3", help="rerun times") - parser.add_argument("-c", "--capture", type=str, - default="sys", help="test capture") + parser.add_argument("--xml-file", type=str, + default="azcli_aks_runner.xml", help="junit/xml report filename") parser.add_argument("--log-file", type=str, default="az_aks_tool.log", help="log filename") args = parser.parse_args(args) @@ -91,12 +95,12 @@ def main(): enable_cli = False enable_ext = False module_data = {} - if args.mode == "cli" or args.mode == "all": + if args.cli or args.all: enable_cli = True module_data[const.ACS_MOD_NAME] = cli.get_cli_mod_data() cli_test_index = cli.get_cli_test_index(module_data) - if args.mode == "ext" or args.mode == "all": + if args.ext or args.all: enable_ext = True module_data[const.AKS_PREVIEW_MOD_NAME] = ext.get_ext_mod_data() ext_test_index = ext.get_ext_test_index(module_data) @@ -114,7 +118,7 @@ def main(): exit_code = run.run_tests(cli_qualified_test_cases, test_index, mode="cli", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series, run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) if exit_code != 0: - sys.exit("CLI test failed!") + sys.exit("CLI test failed with exit code: {}".format(exit_code)) # ext matrix test if enable_ext: @@ -125,15 +129,16 @@ def main(): exit_code = run.run_tests(ext_qualified_test_cases, test_index, mode="ext", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series, run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) if exit_code != 0: - sys.exit("EXT test failed!") + sys.exit("EXT test failed with exit code: {}".format(exit_code)) # raw tests - if test_cases and len(test_cases) > 0: + if test_cases: + logger.info("Get {} cases!".format(len(test_cases))) logger.info("Perform following raw tets: {}".format(test_cases)) - exit_code = run_tests(test_cases, test_index, mode="raw", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series, - run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + exit_code = run.run_tests(test_cases, test_index, mode="raw", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series, + run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) if exit_code != 0: - sys.exit("Raw test failed!") + sys.exit("Raw test failed with exit code: {}".format(exit_code)) if __name__ == "__main__": diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh index 36dc1b60b4d..b344a6bec39 100755 --- a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh @@ -36,10 +36,10 @@ do done # prepare run flags -run_flags="-m ext -em ext_matrix_default.json --no-exitfirst --report-path ./ --reruns 3 --capture=sys" +run_flags="-e -em ext_matrix_default.json --no-exitfirst --report-path ./ --reruns 3 --capture=sys" # parallel if [ $PARALLELISM -ge 2 ]; then - run_flags+=" -n $PARALLELISM" + run_flags+=" -j $PARALLELISM" else run_flags+=" -s" fi From 1c83831c2fec0fba291fa47991205604d07c667b Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Fri, 14 May 2021 17:41:22 +0800 Subject: [PATCH 40/42] update filter mechanism --- src/aks-preview/az_aks_tool/filter.py | 4 +-- src/aks-preview/az_aks_tool/index.py | 3 +- src/aks-preview/az_aks_tool/main.py | 8 ++--- src/aks-preview/az_aks_tool/run.py | 13 +++++++- src/aks-preview/az_aks_tool/utils.py | 31 ++++++++++--------- .../azcli_aks_live_test/test_ext_live.sh | 2 +- 6 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/aks-preview/az_aks_tool/filter.py b/src/aks-preview/az_aks_tool/filter.py index fbb30569b8e..718a34d4058 100644 --- a/src/aks-preview/az_aks_tool/filter.py +++ b/src/aks-preview/az_aks_tool/filter.py @@ -60,7 +60,7 @@ def filter_valid_test_cases(test_cases, test_index): def get_test_cases(test_index, matrix, extra_coverage=None): test_cases = [] - coverage = matrix["coverage"] + coverage = matrix.get("coverage", {}) # default coverage for fileName, className in coverage.items(): for c in className: @@ -82,7 +82,7 @@ def get_test_cases(test_index, matrix, extra_coverage=None): def get_exclude_test_cases(test_index, matrix, extra_filter=None): exclude_test_cases = [] - exclude = matrix["exclude"] + exclude = matrix.get("exclude", {}) # default exclude if not extra_filter or "default" in extra_filter: matrix_test_cases = [] diff --git a/src/aks-preview/az_aks_tool/index.py b/src/aks-preview/az_aks_tool/index.py index 77333655492..11d7e405179 100644 --- a/src/aks-preview/az_aks_tool/index.py +++ b/src/aks-preview/az_aks_tool/index.py @@ -260,8 +260,7 @@ def add_to_index(key, path): test_index['{}.{}'.format(mod1, key)] = path test_index['{}.{}'.format(mod2, key)] = test_index[key] else: - logger.error("'%s' exists twice in the '%s' module. " - "Please rename one or both and re-run --discover.", key, mod1) + logger.error("'%s' exists twice in the '%s' module", key, mod1) else: test_index[key] = path diff --git a/src/aks-preview/az_aks_tool/main.py b/src/aks-preview/az_aks_tool/main.py index 5cc5b94691c..4adff1651c5 100644 --- a/src/aks-preview/az_aks_tool/main.py +++ b/src/aks-preview/az_aks_tool/main.py @@ -73,13 +73,13 @@ def main(): args.report_path, args.log_file)) logger = logging.getLogger("{}.{}".format(root_module_name, __name__)) - # check test cases + # # check test cases test_cases = args.tests ext_matrix_file_path = args.ext_matrix cli_matrix_file_path = args.cli_matrix - if not test_cases and not utils.check_file_existence(ext_matrix_file_path) and not utils.check_file_existence(cli_matrix_file_path): - sys.exit( - "At least one of 'tests', 'cli_matrix' and 'ext_matrix' must be provided!") + # if not test_cases and not utils.check_file_existence(ext_matrix_file_path) and not utils.check_file_existence(cli_matrix_file_path): + # sys.exit( + # "At least one of 'tests', 'cli_matrix' and 'ext_matrix' must be provided!") # prepare pytest args pytest_args = [] diff --git a/src/aks-preview/az_aks_tool/run.py b/src/aks-preview/az_aks_tool/run.py index 05300f218d3..13c2f2ae426 100644 --- a/src/aks-preview/az_aks_tool/run.py +++ b/src/aks-preview/az_aks_tool/run.py @@ -45,6 +45,17 @@ def current_profile(): return cmd('az cloud show --query profile -otsv', show_stderr=False).result +class CommandError(Exception): + + def __init__(self, output, exit_code, command): + message = "Command `{}` failed with exit code {}:\n{}".format( + command, exit_code, output) + self.exit_code = exit_code + self.output = output + self.command = command + super().__init__(message) + + def call(command, **kwargs): """ Run an arbitrary command but don't buffer the output. @@ -169,7 +180,7 @@ def _find_test(index, name): test_paths.append(test_path) except KeyError: logger.warning("'%s' not found.", t) - continue + continue # Tests have been collected. Now run them. exit_code = 0 diff --git a/src/aks-preview/az_aks_tool/utils.py b/src/aks-preview/az_aks_tool/utils.py index 3a07634bd39..f92117bbf56 100644 --- a/src/aks-preview/az_aks_tool/utils.py +++ b/src/aks-preview/az_aks_tool/utils.py @@ -21,9 +21,13 @@ def check_file_existence(file_path): def get_test_matrix(matrix_file_path): - json_file = open(matrix_file_path, 'r') - test_matrix = json.load(json_file) - json_file.close() + test_matrix = {} + if check_file_existence(matrix_file_path): + json_file = open(matrix_file_path, 'r') + test_matrix = json.load(json_file) + json_file.close() + else: + logger.warning("Matrix file '{}' not exists!".format(matrix_file_path)) return test_matrix @@ -42,17 +46,16 @@ def add_qualified_prefix(test_cases, prefix): def get_fully_qualified_test_cases(test_index, matrix_file_path, mod_name, extra_coverage=None, extra_filter=None): qualified_test_cases = [] - if check_file_existence(matrix_file_path): - matrix = get_test_matrix(matrix_file_path) - test_cases = custom_filter.get_test_cases( - test_index, matrix, extra_coverage) - exclude_test_cases = custom_filter.get_exclude_test_cases(test_index, - matrix, extra_filter) - filtered_test_cases = get_filted_test_cases( - test_cases, exclude_test_cases) - # add prefix - qualified_test_cases = add_qualified_prefix( - filtered_test_cases, mod_name) + matrix = get_test_matrix(matrix_file_path) + test_cases = custom_filter.get_test_cases( + test_index, matrix, extra_coverage) + exclude_test_cases = custom_filter.get_exclude_test_cases(test_index, + matrix, extra_filter) + filtered_test_cases = get_filted_test_cases( + test_cases, exclude_test_cases) + # add prefix + qualified_test_cases = add_qualified_prefix( + filtered_test_cases, mod_name) return qualified_test_cases diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh index b344a6bec39..97fbbfcf254 100755 --- a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh +++ b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh @@ -24,7 +24,7 @@ source azEnv/bin/activate # Ensure that the command index is updated by calling a specific command in aks-preview, so that all the commands defined in aks-preview are loaded correctly # Otherwise, cold boot execution of azdev test may use the api version adopted by the acs command group in azure-cli (which may diverge from the api version used in current aks-preview) retry_count=0 -while ! az aks command invoke --help && [[ $retry_count < 3 ]] +while ! az aks maintenanceconfiguration show --help && [[ $retry_count < 3 ]] do retry_count=`expr $retry_count + 1` echo $retry_count"th retry to install aks-preview..." From 82a2e4f8665e303d0362231034c24cc817f332d3 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Mon, 17 May 2021 09:26:38 +0800 Subject: [PATCH 41/42] update comment --- src/aks-preview/az_aks_tool/main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/aks-preview/az_aks_tool/main.py b/src/aks-preview/az_aks_tool/main.py index 4adff1651c5..f991c289c68 100644 --- a/src/aks-preview/az_aks_tool/main.py +++ b/src/aks-preview/az_aks_tool/main.py @@ -77,9 +77,6 @@ def main(): test_cases = args.tests ext_matrix_file_path = args.ext_matrix cli_matrix_file_path = args.cli_matrix - # if not test_cases and not utils.check_file_existence(ext_matrix_file_path) and not utils.check_file_existence(cli_matrix_file_path): - # sys.exit( - # "At least one of 'tests', 'cli_matrix' and 'ext_matrix' must be provided!") # prepare pytest args pytest_args = [] From 9088573bef108dbe08b6a38cb7b42f19f7fc6a87 Mon Sep 17 00:00:00 2001 From: Fuming Zhang Date: Mon, 17 May 2021 11:57:57 +0800 Subject: [PATCH 42/42] remove init file --- src/aks-preview/az_aks_tool/__init__.py | 0 src/aks-preview/az_aks_tool/main.py | 10 ++++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) delete mode 100644 src/aks-preview/az_aks_tool/__init__.py diff --git a/src/aks-preview/az_aks_tool/__init__.py b/src/aks-preview/az_aks_tool/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/aks-preview/az_aks_tool/main.py b/src/aks-preview/az_aks_tool/main.py index f991c289c68..9864eeaaa72 100644 --- a/src/aks-preview/az_aks_tool/main.py +++ b/src/aks-preview/az_aks_tool/main.py @@ -103,8 +103,14 @@ def main(): ext_test_index = ext.get_ext_test_index(module_data) # build test index - logger.info("Building test index...") - test_index = index.build_test_index(module_data) + if enable_cli or enable_ext: + logger.info("Building test index...") + test_index = index.build_test_index(module_data) + else: + logger.error( + "Both modes 'cli' and 'ext' are not enabled! No test will be performed!") + logger.error( + "Please provide at least one of the following parameters (-a, -c, -e) to enable the test!") # cli matrix test if enable_cli: