From b359549230cedb9884b4eb52257eca47c8d8af4b Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Wed, 19 Aug 2020 14:06:26 +0800 Subject: [PATCH 1/4] add deb package test --- azure-pipelines.yml | 4 +-- scripts/release/debian/test_deb_in_docker.sh | 27 ++++++++++++++++++++ scripts/release/debian/test_deb_package.py | 27 ++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 scripts/release/debian/test_deb_in_docker.sh create mode 100644 scripts/release/debian/test_deb_package.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5ad693a2d66..b0e1c58ddee 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -788,7 +788,7 @@ jobs: docker pull ${BASE_IMAGES[$i]} - docker run --rm -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts ${BASE_IMAGES[$i]} /bin/bash -c "apt-get update && apt-get install -y apt-transport-https && dpkg -i /mnt/artifacts/azure-cli_$CLI_VERSION-1~${DISTROS[$i]}_all.deb && time az self-test && time az --version && sleep 5" + docker run --rm -e DISTRO=${DISTROS[$i]} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${BASE_IMAGES[$i]} /bin/bash -c "apt-get update && apt-get install -y apt-transport-https && dpkg -i /mnt/artifacts/azure-cli_$CLI_VERSION-1~${DISTROS[$i]}_all.deb && time az self-test && time az --version && sleep 5" done # Distros that do require libssl1.1 @@ -802,7 +802,7 @@ jobs: docker pull ${BASE_IMAGES[$i]} - docker run --rm -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts ${BASE_IMAGES[$i]} /bin/bash -c "apt-get update && apt-get install -y libssl1.1 apt-transport-https && dpkg -i /mnt/artifacts/azure-cli_$CLI_VERSION-1~${DISTROS[$i]}_all.deb && time az self-test && time az --version && sleep 5" + docker run --rm -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts ${BASE_IMAGES[$i]} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" done displayName: 'Bash Script' diff --git a/scripts/release/debian/test_deb_in_docker.sh b/scripts/release/debian/test_deb_in_docker.sh new file mode 100644 index 00000000000..769e90cf7f0 --- /dev/null +++ b/scripts/release/debian/test_deb_in_docker.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# This script should be run in a ubuntu/debian docker. +set -exv + +export USERNAME=azureuser + +apt-get update +apt-get install -y apt-transport-https python3-pip +dpkg -i /mnt/artifacts/azure-cli_$CLI_VERSION-1~${DISTRO}_all.deb + +ln -s /usr/bin/python3 /usr/bin/python +ln -s /usr/bin/pip3 /usr/bin/pip +time az self-test +time az --version + +cd /azure-cli/ +pip3 install wheel +./scripts/ci/build.sh + +/opt/az/bin/python3 -m pip install pytest +/opt/az/bin/python3 -m pip install pytest-xdist + +find /azure-cli/artifacts/build -name "azure_cli_testsdk*" | xargs /opt/az/bin/python3 -m pip install --upgrade --ignore-installed +find /azure-cli/artifacts/build -name "azure_cli_fulltest*" | xargs /opt/az/bin/python3 -m pip install --upgrade --ignore-installed --no-deps + +/opt/az/bin/python3 /azure-cli/scripts/release/rpm/test_deb_package.py diff --git a/scripts/release/debian/test_deb_package.py b/scripts/release/debian/test_deb_package.py new file mode 100644 index 00000000000..716c871ebbc --- /dev/null +++ b/scripts/release/debian/test_deb_package.py @@ -0,0 +1,27 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import sys +import subprocess + +root_dir = '/opt/az/lib/python3.6/site-packages/azure/cli/command_modules' +mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__'] + +pytest_base_cmd = '/opt/az/bin/python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN' +pytest_parallel_cmd = '{} -n auto'.format(pytest_base_cmd) + +for mod_name in mod_list: + if mod_name in ['botservice', 'network']: + exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_base_cmd, mod_name, mod_name)], shell=True) + else: + exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_parallel_cmd, mod_name, mod_name)], shell=True) + if exit_code == 5: + print('No tests found for {}'.format(mod_name)) + elif exit_code != 0: + sys.exit(exit_code) + +exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/azure-cli-core.xml --pyargs azure.cli.core'.format(pytest_parallel_cmd)], shell=True) +sys.exit(exit_code) From 727c988d92b57af4293fe3d871679519d1fb0383 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Wed, 19 Aug 2020 15:28:00 +0800 Subject: [PATCH 2/4] modify pipeline --- azure-pipelines.yml | 170 ++++++------------- scripts/release/debian/test_deb_in_docker.sh | 11 +- 2 files changed, 65 insertions(+), 116 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b0e1c58ddee..dd8825b7a24 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -629,8 +629,8 @@ jobs: displayName: 'Test Yum Package' -- job: BuildUbuntuPackages - displayName: Build Ubuntu Packages +- job: BuildDebPackages + displayName: Build Deb Packages dependsOn: BuildPythonWheel condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual')) @@ -639,52 +639,29 @@ jobs: strategy: matrix: Xenial: + deb_system: ubuntu distro: xenial Trusty: + deb_system: ubuntu distro: trusty Bionic: + deb_system: ubuntu distro: bionic Eoan: + deb_system: ubuntu distro: eoan Focal: + deb_system: ubuntu distro: focal - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' - artifactName: pypi - - - task: Bash@3 - displayName: 'Build Ubuntu $(distro) Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/debian/pipeline.sh - env: - DISTRO: $(distro) - DISTRO_BASE_IMAGE: ubuntu:$(distro) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: debian' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: ubuntu-$(distro) - -- job: BuildDebianPackages - displayName: Build Debian Packages - - dependsOn: BuildPythonWheel - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual')) - pool: - vmImage: 'ubuntu-16.04' - strategy: - matrix: Jessie: + deb_system: debian distro: jessie Stretch: + deb_system: debian distro: stretch Buster: - distro: buster + deb_system: debian + distro: buster steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build Artifacts' @@ -692,29 +669,53 @@ jobs: TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' artifactName: pypi - - task: Bash@3 - displayName: 'Build Debian $(distro) Package' + displayName: 'Build $(deb_system) $(distro) Package' inputs: targetType: 'filePath' filePath: scripts/release/debian/pipeline.sh env: DISTRO: $(distro) - DISTRO_BASE_IMAGE: debian:$(distro) + DISTRO_BASE_IMAGE: $(deb_system):$(distro) - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: debian' inputs: TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: debian-$(distro) + ArtifactName: $(deb_system)-$(distro) - job: TestDebPackages displayName: Test Deb Packages dependsOn: - - BuildUbuntuPackages - - BuildDebianPackages + - BuildDebPackages condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual')) + strategy: + matrix: + Xenial: + deb_system: ubuntu + distro: xenial + Trusty: + deb_system: ubuntu + distro: trusty + Bionic: + deb_system: ubuntu + distro: bionic + Eoan: + deb_system: ubuntu + distro: eoan + Focal: + deb_system: ubuntu + distro: focal + Jessie: + deb_system: debian + distro: jessie + Stretch: + deb_system: debian + distro: stretch + Buster: + deb_system: debian + distro: buster pool: vmImage: 'ubuntu-16.04' steps: @@ -725,86 +726,27 @@ jobs: artifactName: metadata - task: DownloadPipelineArtifact@1 - displayName: 'Download Debian:Stretch Build' + displayName: 'Download $(deb_system):$(distro) Build' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: debian-stretch + artifactName: $(deb_system)-$(distro) - - task: DownloadPipelineArtifact@1 - displayName: 'Download Debian:Buster Build' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: debian-buster - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Debian:Jessie Builds' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: debian-jessie - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Ubuntu:Bionic Builds' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: ubuntu-bionic - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Ubuntu:Xenial Builds' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: ubuntu-xenial - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Ubuntu:Trusty Builds' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: ubuntu-trusty - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Ubuntu:Eoan Builds' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: ubuntu-eoan - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Ubuntu:Focal Builds' + - task: Bash@3 + displayName: 'Test $(deb_system) $(distro) Package' + env: + DISTRO: $(distro) + DISTRO_BASE_IMAGE: $(deb_system):$(distro) inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: ubuntu-focal + targetType: inline +      script: | + set -exv - - bash: | - set -exv - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - DISTROS=(jessie stretch xenial trusty) - BASE_IMAGES=(debian:jessie debian:stretch ubuntu:xenial ubuntu:trusty) - - # Distros that don't require libssl1.1 - for i in ${!DISTROS[@]}; do - echo "== Test debian package on ${DISTROS[$i]} ==" - - DEB_FILE=$SYSTEM_ARTIFACTSDIRECTORY/debian/azure-cli_$CLI_VERSION-1~${DISTROS[$i]}_all.deb - - docker pull ${BASE_IMAGES[$i]} - - docker run --rm -e DISTRO=${DISTROS[$i]} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${BASE_IMAGES[$i]} /bin/bash -c "apt-get update && apt-get install -y apt-transport-https && dpkg -i /mnt/artifacts/azure-cli_$CLI_VERSION-1~${DISTROS[$i]}_all.deb && time az self-test && time az --version && sleep 5" - done - - # Distros that do require libssl1.1 - DISTROS=(bionic buster eoan focal) - BASE_IMAGES=(ubuntu:bionic debian:buster ubuntu:eoan ubuntu:focal) - - for i in ${!DISTROS[@]}; do - echo "== Test debian package on ${DISTROS[$i]} ==" - - DEB_FILE=$SYSTEM_ARTIFACTSDIRECTORY/debian/azure-cli_$CLI_VERSION-1~${DISTROS[$i]}_all.deb - - docker pull ${BASE_IMAGES[$i]} - - docker run --rm -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts ${BASE_IMAGES[$i]} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" - done - displayName: 'Bash Script' + echo "== Test debian package on ${DISTRO} ==" + docker pull ${DISTRO_BASE_IMAGE} + docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" + displayName: 'Test $(deb_system) $(distro) Package' - job: CheckStyle displayName: "Check CLI Style" diff --git a/scripts/release/debian/test_deb_in_docker.sh b/scripts/release/debian/test_deb_in_docker.sh index 769e90cf7f0..100800f60f7 100644 --- a/scripts/release/debian/test_deb_in_docker.sh +++ b/scripts/release/debian/test_deb_in_docker.sh @@ -5,8 +5,15 @@ set -exv export USERNAME=azureuser -apt-get update -apt-get install -y apt-transport-https python3-pip +apt update +apt install -y apt-transport-https python3-pip + +# The distros that need libssl1.1 +case ${DISTRO} in + bionic|buster|eoan|focal) apt install -y libssl1.1;; + *) :;; +esac + dpkg -i /mnt/artifacts/azure-cli_$CLI_VERSION-1~${DISTRO}_all.deb ln -s /usr/bin/python3 /usr/bin/python From c8ee8603e82df514d4c694f1b59c70c383b64648 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Wed, 19 Aug 2020 17:35:48 +0800 Subject: [PATCH 3/4] fix yaml --- azure-pipelines.yml | 7 ++----- scripts/release/debian/test_deb_in_docker.sh | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dd8825b7a24..d964b096fb0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -737,16 +737,13 @@ jobs: DISTRO: $(distro) DISTRO_BASE_IMAGE: $(deb_system):$(distro) inputs: - targetType: inline -      script: | + targetType: 'inline' + script: | set -exv - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - echo "== Test debian package on ${DISTRO} ==" docker pull ${DISTRO_BASE_IMAGE} docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" - displayName: 'Test $(deb_system) $(distro) Package' - job: CheckStyle displayName: "Check CLI Style" diff --git a/scripts/release/debian/test_deb_in_docker.sh b/scripts/release/debian/test_deb_in_docker.sh index 100800f60f7..75930ca55a4 100644 --- a/scripts/release/debian/test_deb_in_docker.sh +++ b/scripts/release/debian/test_deb_in_docker.sh @@ -6,7 +6,7 @@ set -exv export USERNAME=azureuser apt update -apt install -y apt-transport-https python3-pip +apt install -y apt-transport-https python3-pip git # The distros that need libssl1.1 case ${DISTRO} in @@ -31,4 +31,4 @@ pip3 install wheel find /azure-cli/artifacts/build -name "azure_cli_testsdk*" | xargs /opt/az/bin/python3 -m pip install --upgrade --ignore-installed find /azure-cli/artifacts/build -name "azure_cli_fulltest*" | xargs /opt/az/bin/python3 -m pip install --upgrade --ignore-installed --no-deps -/opt/az/bin/python3 /azure-cli/scripts/release/rpm/test_deb_package.py +/opt/az/bin/python3 /azure-cli/scripts/release/debian/test_deb_package.py From 2d9b655fd64504efdd91b6a962272c95a37d9e51 Mon Sep 17 00:00:00 2001 From: Feng Zhou Date: Thu, 20 Aug 2020 16:50:25 +0800 Subject: [PATCH 4/4] use bundled python to build packages --- scripts/release/debian/test_deb_in_docker.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/release/debian/test_deb_in_docker.sh b/scripts/release/debian/test_deb_in_docker.sh index 75930ca55a4..4e2d9868942 100644 --- a/scripts/release/debian/test_deb_in_docker.sh +++ b/scripts/release/debian/test_deb_in_docker.sh @@ -6,7 +6,7 @@ set -exv export USERNAME=azureuser apt update -apt install -y apt-transport-https python3-pip git +apt install -y apt-transport-https git # The distros that need libssl1.1 case ${DISTRO} in @@ -16,13 +16,12 @@ esac dpkg -i /mnt/artifacts/azure-cli_$CLI_VERSION-1~${DISTRO}_all.deb -ln -s /usr/bin/python3 /usr/bin/python -ln -s /usr/bin/pip3 /usr/bin/pip time az self-test time az --version cd /azure-cli/ -pip3 install wheel +/opt/az/bin/python3 -m pip install wheel +ln -sf /opt/az/bin/python3 /usr/bin/python ./scripts/ci/build.sh /opt/az/bin/python3 -m pip install pytest