From 334fc413b9e3690f1b414eb80d32cf25524836e9 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sun, 4 Oct 2020 16:05:51 -0700 Subject: [PATCH 1/6] properly update the dev versioning requirements for packages --- eng/tox/sanitize_setup.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/eng/tox/sanitize_setup.py b/eng/tox/sanitize_setup.py index ceeb1e85d77a..beab949a4d8b 100644 --- a/eng/tox/sanitize_setup.py +++ b/eng/tox/sanitize_setup.py @@ -13,8 +13,10 @@ import logging import glob from packaging.specifiers import SpecifierSet +from packaging.version import Version from pkg_resources import Requirement from pypi_tools.pypi import PyPIClient +import re root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) setup_parser_path = os.path.abspath(os.path.join(root_dir, "eng", "versioning")) @@ -22,7 +24,7 @@ sys.path.append(setup_parser_path) from setup_parser import get_install_requires, parse_setup -DEV_BUILD_IDENTIFIER = ".dev" +DEV_BUILD_IDENTIFIER = "a" def update_requires(setup_py_path, requires_dict): # This method changes package requirement by overriding the specifier @@ -62,12 +64,16 @@ def get_version(pkg_name): # When building package with dev build version, version for packages in same service is updated to dev build # and other packages will not have dev build number # strip dev build number so we can check if package exists in PyPI and replace - if DEV_BUILD_IDENTIFIER in version: - version = version[:version.find(DEV_BUILD_IDENTIFIER)] + + version_obj = Version(version) + if version_obj.pre[0] == DEV_BUILD_IDENTIFIER: + version = version_obj.base_version + return version else: logging.error("setyp.py is not found for package {} to identify current version".format(pkg_name)) exit(1) + def process_requires(setup_py_path): # This method process package requirement to verify if all required packages are available on PyPI @@ -82,11 +88,12 @@ def process_requires(setup_py_path): for req in requires: pkg_name = req.key spec = SpecifierSet(str(req).replace(pkg_name, "")) + if not is_required_version_on_pypi(pkg_name, spec): old_req = str(req) version = get_version(pkg_name) logging.info("Updating version {0} in requirement {1} to dev build version".format(version, old_req)) - new_req = old_req.replace(version, "{}.dev".format(version)) + new_req = re.sub('{}[a]*'.format(version), "{}{}".format(version, DEV_BUILD_IDENTIFIER), old_req) logging.info("New requirement for package {0}: {1}".format(pkg_name, new_req)) requirement_to_update[old_req] = new_req From 3a768f91b14ac3bf4ec043d171cbdddc95c006f6 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sun, 4 Oct 2020 16:15:27 -0700 Subject: [PATCH 2/6] got it --- eng/tox/sanitize_setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/tox/sanitize_setup.py b/eng/tox/sanitize_setup.py index beab949a4d8b..72fe0eb86822 100644 --- a/eng/tox/sanitize_setup.py +++ b/eng/tox/sanitize_setup.py @@ -16,7 +16,6 @@ from packaging.version import Version from pkg_resources import Requirement from pypi_tools.pypi import PyPIClient -import re root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) setup_parser_path = os.path.abspath(os.path.join(root_dir, "eng", "versioning")) @@ -93,7 +92,7 @@ def process_requires(setup_py_path): old_req = str(req) version = get_version(pkg_name) logging.info("Updating version {0} in requirement {1} to dev build version".format(version, old_req)) - new_req = re.sub('{}[a]*'.format(version), "{}{}".format(version, DEV_BUILD_IDENTIFIER), old_req) + new_req = old_req.replace(version, "{}{}".format(version, DEV_BUILD_IDENTIFIER)) logging.info("New requirement for package {0}: {1}".format(pkg_name, new_req)) requirement_to_update[old_req] = new_req From 8c64001a8dbbe3a95ca5b630215af5ed04ed76ff Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sun, 4 Oct 2020 17:00:54 -0700 Subject: [PATCH 3/6] add only the blob-changefeed override due to outdated req on azure-storage-blob --- scripts/devops_tasks/git_helper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/devops_tasks/git_helper.py b/scripts/devops_tasks/git_helper.py index f1aa54472a52..d6eaa0d57dbe 100644 --- a/scripts/devops_tasks/git_helper.py +++ b/scripts/devops_tasks/git_helper.py @@ -23,7 +23,8 @@ 'azure-cosmos': '3.2.0', 'azure-servicebus': '0.50.3', 'azure-eventgrid': '1.3.0', - 'azure-schemaregistry-avroserializer': '1.0.0b1' + 'azure-schemaregistry-avroserializer': '1.0.0b1', + 'azure-storage-blob-changefeed' : '12.0.0b2' } # This method identifies release tag for latest or oldest released version of a given package From aae55667b3f366a4bfd665aedb5b1995d3520665 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sun, 4 Oct 2020 17:12:30 -0700 Subject: [PATCH 4/6] update regression to update the local versions of the package to MATCH the wheel versions. nightly should be expecting azure-storage-blob==12.3.7a --- eng/pipelines/templates/steps/test_regression.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eng/pipelines/templates/steps/test_regression.yml b/eng/pipelines/templates/steps/test_regression.yml index bef11b0692af..ae67c3155a4b 100644 --- a/eng/pipelines/templates/steps/test_regression.yml +++ b/eng/pipelines/templates/steps/test_regression.yml @@ -2,6 +2,7 @@ parameters: BuildTargetingString: 'azure-*' ServiceDirectory: '' BuildStagingDirectory: $(Build.ArtifactStagingDirectory) + DevFeedName: 'public/azure-sdk-for-python' steps: - task: UsePythonVersion@0 @@ -14,6 +15,11 @@ steps: artifactName: 'artifacts' targetPath: $(Build.ArtifactStagingDirectory) + - ${{if eq(variables['System.TeamProject'], 'internal') }}: + - template: ../steps/auth-dev-feed.yml + parameters: + DevFeedName: ${{ parameters.DevFeedName }} + - script: | pip install -r eng/ci_tools.txt displayName: 'Prep Environment' From d7edadf4abe6c61941cf2f70693bb678d32cbfe4 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sun, 4 Oct 2020 17:28:23 -0700 Subject: [PATCH 5/6] resolve falty logic causing dev build to always be true. --- eng/tox/sanitize_setup.py | 5 +++-- scripts/devops_tasks/build_packages.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/tox/sanitize_setup.py b/eng/tox/sanitize_setup.py index 72fe0eb86822..2716e922e82c 100644 --- a/eng/tox/sanitize_setup.py +++ b/eng/tox/sanitize_setup.py @@ -65,8 +65,9 @@ def get_version(pkg_name): # strip dev build number so we can check if package exists in PyPI and replace version_obj = Version(version) - if version_obj.pre[0] == DEV_BUILD_IDENTIFIER: - version = version_obj.base_version + if version_obj.pre: + if version_obj.pre[0] == DEV_BUILD_IDENTIFIER: + version = version_obj.base_version return version else: diff --git a/scripts/devops_tasks/build_packages.py b/scripts/devops_tasks/build_packages.py index c7463e4b7eed..e8bec139ac05 100644 --- a/scripts/devops_tasks/build_packages.py +++ b/scripts/devops_tasks/build_packages.py @@ -105,4 +105,4 @@ def verify_update_package_requirement(pkg_root): target_dir = root_dir targeted_packages = process_glob_string(args.glob_string, target_dir, args.package_filter_string) - build_packages(targeted_packages, args.distribution_directory, args.is_dev_build) + build_packages(targeted_packages, args.distribution_directory, bool(args.is_dev_build)) From db62f308fe805bbae1214178cea73640ad7773b4 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Sun, 4 Oct 2020 18:56:41 -0700 Subject: [PATCH 6/6] set dev-build so that the files on disk match the expected version that will be installed from the wheel directory --- eng/pipelines/templates/steps/test_regression.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eng/pipelines/templates/steps/test_regression.yml b/eng/pipelines/templates/steps/test_regression.yml index ae67c3155a4b..1fba92c816f4 100644 --- a/eng/pipelines/templates/steps/test_regression.yml +++ b/eng/pipelines/templates/steps/test_regression.yml @@ -15,6 +15,11 @@ steps: artifactName: 'artifacts' targetPath: $(Build.ArtifactStagingDirectory) + - template: ../steps/set-dev-build.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} + - ${{if eq(variables['System.TeamProject'], 'internal') }}: - template: ../steps/auth-dev-feed.yml parameters: