From 87bc0b9da8998d40976c883c6e38216cf4d5c651 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 18 Sep 2024 17:40:30 -0700 Subject: [PATCH 01/18] working on a true solution to incompatible requirements on dev_req --- eng/tox/install_depend_packages.py | 121 ++++++++++++++++++++++++++--- tools/azure-sdk-tools/setup.py | 2 +- 2 files changed, 111 insertions(+), 12 deletions(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index 6334442ca384..8378fd2f9998 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -5,24 +5,23 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - import argparse import os import sys import logging import re +import pkginfo + from subprocess import check_call -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Callable, Optional from pkg_resources import parse_version, Requirement from pypi_tools.pypi import PyPIClient from packaging.specifiers import SpecifierSet -from packaging.version import Version, parse -import pdb +from packaging.version import Version from ci_tools.parsing import ParsedSetup, parse_require from ci_tools.functions import compare_python_version - from typing import List DEV_REQ_FILE = "dev_requirements.txt" @@ -89,6 +88,98 @@ } } +def resolve_compatible_package(target_package_name: str, target_package_version: str, target_package_reqs: List[Requirement], input_packages: List[Requirement]) -> Optional[str]: + """ + This function resolves a compatible version of target_package_name that is compatible with the input_packages. It is intended to be used + when a dev requirement is incompatible with the current set of packages being installed, so it only walks backwards from newest version of + target_package_name to oldest. + """ + + pypi = PyPIClient() + + pkgs = {req.key: req for req in input_packages} + + for requirement in input_packages: + version = next(iter(requirement.specifier)).version + # requirement_release = pypi.project_release(requirement.key, version) + + for target_dependency_req in target_package_reqs: + if target_dependency_req.key in pkgs: + if str(target_dependency_req.specifier): + # check compatibility here. if the dep is compatible with the current set of packages, we can use it. + # otherwise we need to keep walking backwards until we find a compatible version. + pass + else: + # no specifier, we can use this version. + return None + else: + continue + + + + + +def handle_incompatible_minimum_dev_reqs(setup_path: str, filtered_requirement_list: List[str], packages_for_install: List[Requirement]) -> List[str]: + """ + This function is used to handle the case where a dev requirement is incompatible with the current set of packages + being installed. This is used to update or remove dev_requirements that are incompatible with a targeted set of packages. + + :param str setup_path: The path to the setup.py file whos dev_requirements are being filtered. + + :param List[str] filtered_requirement_list: A list of dev requirements that have been filtered out of the original dev requirements file. This list + must be filtered for compatibility with packages_for_install. + + :param List[Requirement] packages_for_install: A list of packages that dev_requirements MUST be compatible with. + """ + + cleansed_reqs = [] + + for req in filtered_requirement_list: + cleansed_req = req.strip().replace("-e ", "").split("#")[0].split(";")[0] + + if cleansed_req: + # this is a replaced dev req that we can use pkginfo to resolve + if os.path.exists(cleansed_req): + try: + local_package_metadata = pkginfo.get_metadata(cleansed_req) + + if local_package_metadata: + local_reqs = [Requirement(r) for r in local_package_metadata.requires_dist] + new_req = resolve_compatible_package(local_package_metadata.name, local_package_metadata.version, local_reqs, packages_for_install) + if new_req: + cleansed_reqs.append(new_req) + else: + cleansed_reqs.append(cleansed_req) + else: + logging.error(f"Error while processing locally built requirement {cleansed_req}. Unable to resolve metadata.") + cleansed_reqs.append(cleansed_req) + except Exception as e: + logging.error(f"Error while processing locally built requirement {cleansed_req}: {e}") + cleansed_reqs.append(cleansed_req) + # relative requirement + elif cleansed_req.startswith("."): + try: + local_package = ParsedSetup.from_path(os.path.join(setup_path, cleansed_req)) + local_reqs = [Requirement(r) for r in local_package.requires] + new_req = resolve_compatible_package(local_package.name, local_package.version, local_reqs, packages_for_install) + + if new_req: + cleansed_reqs.append(new_req) + else: + cleansed_reqs.append(new_req) + + except Exception as e: + logging.error(f"Error while processing relative requirement {cleansed_req}: {e}") + cleansed_reqs.append(cleansed_req) + else: + # doing nothing here, as we don't understand how to resolve this yet + cleansed_reqs.append(cleansed_req) + logging.info(f"While filtering incompatible minimum dev requirements, found a requirement that I don't know how to deal with yet: \"{cleansed_req}\"") + continue + + + return cleansed_reqs + def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir): # This method identifies latest/ minimal version of dependent packages and installs them from pyPI @@ -104,7 +195,12 @@ def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir): logging.info("%s released packages: %s", dependency_type, released_packages) # filter released packages from dev_requirements and create a new file "new_dev_requirements.txt" - dev_req_file_path = filter_dev_requirements(setup_py_file_path, released_packages, temp_dir) + + additionalFilterFn = None + if dependency_type == "Minimum": + additionalFilterFn = handle_incompatible_minimum_dev_reqs + + dev_req_file_path = filter_dev_requirements(setup_py_file_path, released_packages, temp_dir, additionalFilterFn) if override_added_packages: logging.info(f"Expanding the requirement set by the packages {override_added_packages}.") @@ -251,7 +347,6 @@ def process_requirement(req, dependency_type, orig_pkg_name): # think of the various versions that come back from pypi as the top of a funnel # We apply generic overrides -> platform specific overrides -> package specific overrides - versions = process_bounded_versions(orig_pkg_name, pkg_name, versions) # Search from lowest to latest in case of finding minimum dependency @@ -304,7 +399,7 @@ def check_req_against_exclusion(req, req_to_exclude): return req_id == req_to_exclude -def filter_dev_requirements(setup_py_path, released_packages, temp_dir): +def filter_dev_requirements(setup_py_path, released_packages, temp_dir, additionalFilterFn: Optional[Callable[[str, List[str],List[Requirement]], List[str]]] = None): # This method returns list of requirements from dev_requirements by filtering out packages in given list dev_req_path = os.path.join(os.path.dirname(setup_py_path), DEV_REQ_FILE) requirements = [] @@ -313,12 +408,13 @@ def filter_dev_requirements(setup_py_path, released_packages, temp_dir): # filter out any package available on PyPI (released_packages) # include packages without relative reference and packages not available on PyPI - released_packages = [p.split("==")[0] for p in released_packages] + released_packages = [parse_require(p) for p in released_packages] + released_package_names = [p.key for p in released_packages] # find prebuilt whl paths in dev requiremente prebuilt_dev_reqs = [os.path.basename(req.replace("\n", "")) for req in requirements if os.path.sep in req] # filter any req if wheel is for a released package - req_to_exclude = [req for req in prebuilt_dev_reqs if req.split("-")[0].replace("_", "-") in released_packages] - req_to_exclude.extend(released_packages) + req_to_exclude = [req for req in prebuilt_dev_reqs if req.split("-")[0].replace("_", "-") in released_package_names] + req_to_exclude.extend(released_package_names) filtered_req = [ req @@ -327,6 +423,9 @@ def filter_dev_requirements(setup_py_path, released_packages, temp_dir): and not any([check_req_against_exclusion(req, i) for i in req_to_exclude]) ] + if additionalFilterFn: + filtered_req = additionalFilterFn(setup_py_path, filtered_req, released_packages) + logging.info("Filtered dev requirements: %s", filtered_req) new_dev_req_path = "" diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index 82ef03bd524f..c9d32ad17728 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -55,7 +55,7 @@ extras_require={ ":python_version>='3.5'": ["pytest-asyncio>=0.9.0"], ":python_version<'3.11'": ["tomli==2.0.1"], - "build": ["six", "setuptools", "pyparsing", "certifi", "cibuildwheel"], + "build": ["six", "setuptools", "pyparsing", "certifi", "cibuildwheel", "pkginfo"], "conda": ["beautifulsoup4"], "systemperf": ["aiohttp>=3.0", "requests>=2.0", "tornado==6.0.3", "httpx>=0.21", "azure-core"], "ghtools": ["GitPython", "PyGithub>=1.59.0", "requests>=2.0"], From 0338ac92f3bd94a8c6d4925e028fe491a898add5 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 18 Sep 2024 18:26:21 -0700 Subject: [PATCH 02/18] commit progress, I'm not super happy with the results so far its not very re-entrant --- eng/tox/install_depend_packages.py | 52 +++++++++++++++++++----------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index 8378fd2f9998..46f9689a7c9b 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -88,34 +88,49 @@ } } -def resolve_compatible_package(target_package_name: str, target_package_version: str, target_package_reqs: List[Requirement], input_packages: List[Requirement]) -> Optional[str]: +def resolve_compatible_package(dev_req_pkg_name: str, dev_req_pkg_version: str, dev_req_pkg_reqs: List[Requirement], immovable_requirements: List[Requirement]) -> Optional[str]: """ - This function resolves a compatible version of target_package_name that is compatible with the input_packages. It is intended to be used + This function resolves a compatible version of dev_req_pkg_name that is compatible with the input_packages. It is intended to be used when a dev requirement is incompatible with the current set of packages being installed, so it only walks backwards from newest version of - target_package_name to oldest. + dev_req_pkg_name to oldest. """ pypi = PyPIClient() + immovable_pkgs = {req.key: req for req in immovable_requirements} + + # Let's use a real use-case to walk through this function. We're going to use the azure-ai-language-conversations package as an example. + # immovable_pkgs = the selected mindependency for azure-ai-language-conversations + # -> "azure-core==1.28.0", + # -> "isodate==0.6.1", + # -> "typing-extensions==4.0.1", + for pkg in immovable_pkgs: + required_pkg_version = next(iter(immovable_pkgs[pkg].specifier)).version + # as we walk through each hard dependency, we need to check all of the dev_requirements for compatibility with these packages + for dev_req_dependency in dev_req_pkg_reqs: + # if the dev_req_dependency is present in the hard dependencies, we need to check compatibility + if dev_req_dependency.key in immovable_pkgs: + if str(dev_req_dependency.specifier): + # check compatibility here. if the dep is compatible with the current set of packages, we can use it. + # otherwise we need to keep walking backwards until we find a compatible version. + # from here on we have to check the same dependency for this package on the retrieved version as well + available_versions = pypi.get_ordered_versions(dev_req_dependency.key, True) # they come back smallest to largest by default + available_versions.reverse() - pkgs = {req.key: req for req in input_packages} + if not required_pkg_version in dev_req_dependency.specifier: + breakpoint() - for requirement in input_packages: - version = next(iter(requirement.specifier)).version - # requirement_release = pypi.project_release(requirement.key, version) + pypi.get_ordered_versions(dev_req_dependency.key, True) - for target_dependency_req in target_package_reqs: - if target_dependency_req.key in pkgs: - if str(target_dependency_req.specifier): - # check compatibility here. if the dep is compatible with the current set of packages, we can use it. - # otherwise we need to keep walking backwards until we find a compatible version. - pass - else: - # no specifier, we can use this version. - return None - else: - continue + # # we're attempting to do this entirely with metadata instead of downloading each version + # for version in available_versions: + # version_release = pypi.project_release(dev_req_dependency.key, version) + # breakpoint() + # breakpoint() + + # no changes necessary + return None @@ -177,7 +192,6 @@ def handle_incompatible_minimum_dev_reqs(setup_path: str, filtered_requirement_l logging.info(f"While filtering incompatible minimum dev requirements, found a requirement that I don't know how to deal with yet: \"{cleansed_req}\"") continue - return cleansed_reqs From 455d484a71de6575778b4a4175a1dd6c8e68604d Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 30 Sep 2024 14:22:41 -0700 Subject: [PATCH 03/18] commenting the new function --- eng/tox/install_depend_packages.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index c61c7537843d..1e4dff912039 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -117,8 +117,6 @@ def resolve_compatible_package(dev_req_pkg_name: str, dev_req_pkg_version: str, available_versions.reverse() if not required_pkg_version in dev_req_dependency.specifier: - breakpoint() - pypi.get_ordered_versions(dev_req_dependency.key, True) @@ -437,8 +435,8 @@ def filter_dev_requirements(setup_py_path, released_packages, temp_dir, addition and not any([check_req_against_exclusion(req, i) for i in req_to_exclude]) ] - if additionalFilterFn: - filtered_req = additionalFilterFn(setup_py_path, filtered_req, released_packages) + # if additionalFilterFn: + # filtered_req = additionalFilterFn(setup_py_path, filtered_req, released_packages) logging.info("Filtered dev requirements: %s", filtered_req) From b1a60b7dac16305dfa141b52e6b5ba3ea0a5e931 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 30 Sep 2024 18:31:51 -0700 Subject: [PATCH 04/18] we now are recognizing when we have a incompatible dev req, working on implementing fallback resolution --- eng/tox/install_depend_packages.py | 151 +++++++++++++++++++---------- 1 file changed, 99 insertions(+), 52 deletions(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index 1e4dff912039..54d163a0a0ca 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -88,15 +88,43 @@ } } -def resolve_compatible_package(dev_req_pkg_name: str, dev_req_pkg_version: str, dev_req_pkg_reqs: List[Requirement], immovable_requirements: List[Requirement]) -> Optional[str]: +def is_package_compatible(package_requirements: List[Requirement], immutable_requirements: List[Requirement]) -> bool: """ - This function resolves a compatible version of dev_req_pkg_name that is compatible with the input_packages. It is intended to be used - when a dev requirement is incompatible with the current set of packages being installed, so it only walks backwards from newest version of - dev_req_pkg_name to oldest. + This function accepts a set of requirements for a package, and ensures that the package is compatible with the immutable_requirements. + + It is factored this way because we retrieve requirements differently according to the source of the package. + If published, we can get requires() from PyPI + If locally built wheel, we can get requires() from the metadata of the package + If local relative requirement, we can get requires() from a ParsedSetup of the setup.py for th package + + :param List[Requirement] package_requirements: The dependencies of a dev_requirement file. This is the set of requirements that we are checking compatibility for. + :param List[Requirement] immutable_requirements: A list of requirements that the other packages must be compatible with. + """ + + for immutable_requirement in immutable_requirements: + for package_requirement in package_requirements: + if package_requirement.key == immutable_requirement.key: + # if the dev_req line has a requirement that conflicts with the immutable requirement, we need to resolve it + # we KNOW that the immutable requirement will be of form package==version, so we can reliably pull out the version + # and check it against the specifier of the dev_req line. + immutable_version = next(iter(immutable_requirement.specifier)).version + + if not package_requirement.specifier.contains(immutable_version): + logging.info(f"Dev req dependency {package_requirement} is not compatible with immutable requirement {immutable_requirement}.") + return False + + return True + +def resolve_compatible_package(package_name: str, package_version: str, immutable_requirements: List[Requirement]) -> Optional[str]: + """ + This function accepts a targeted package, a set of requirements for that package, and a set of immutable_requirements that the package must be compatible with. + + This function should only be utilized when a package is found to be incompatible with the immutable_requirements. It will attempt to resolve the incompatibility by + walking backwards through different versions of until a compatible version is found that works with the immutable_requirements. """ pypi = PyPIClient() - immovable_pkgs = {req.key: req for req in immovable_requirements} + immovable_pkgs = {req.key: req for req in immutable_requirements} # Let's use a real use-case to walk through this function. We're going to use the azure-ai-language-conversations package as an example. # immovable_pkgs = the selected mindependency for azure-ai-language-conversations @@ -106,18 +134,18 @@ def resolve_compatible_package(dev_req_pkg_name: str, dev_req_pkg_version: str, for pkg in immovable_pkgs: required_pkg_version = next(iter(immovable_pkgs[pkg].specifier)).version # as we walk through each hard dependency, we need to check all of the dev_requirements for compatibility with these packages - for dev_req_dependency in dev_req_pkg_reqs: - # if the dev_req_dependency is present in the hard dependencies, we need to check compatibility - if dev_req_dependency.key in immovable_pkgs: - if str(dev_req_dependency.specifier): - # check compatibility here. if the dep is compatible with the current set of packages, we can use it. - # otherwise we need to keep walking backwards until we find a compatible version. - # from here on we have to check the same dependency for this package on the retrieved version as well - available_versions = pypi.get_ordered_versions(dev_req_dependency.key, True) # they come back smallest to largest by default - available_versions.reverse() + # for dev_req_dependency in dev_req_pkg_reqs: + # # if the dev_req_dependency is present in the hard dependencies, we need to check compatibility + # if dev_req_dependency.key in immovable_pkgs: + # if str(dev_req_dependency.specifier): + # # check compatibility here. if the dep is compatible with the current set of packages, we can use it. + # # otherwise we need to keep walking backwards until we find a compatible version. + # # from here on we have to check the same dependency for this package on the retrieved version as well + # available_versions = pypi.get_ordered_versions(dev_req_dependency.key, True) # they come back smallest to largest by default + # available_versions.reverse() - if not required_pkg_version in dev_req_dependency.specifier: - pypi.get_ordered_versions(dev_req_dependency.key, True) + # if not required_pkg_version in dev_req_dependency.specifier: + # pypi.get_ordered_versions(dev_req_dependency.key, True) # # we're attempting to do this entirely with metadata instead of downloading each version @@ -139,55 +167,64 @@ def handle_incompatible_minimum_dev_reqs(setup_path: str, filtered_requirement_l :param str setup_path: The path to the setup.py file whos dev_requirements are being filtered. - :param List[str] filtered_requirement_list: A list of dev requirements that have been filtered out of the original dev requirements file. This list - must be filtered for compatibility with packages_for_install. + :param List[str] filtered_requirement_list: A filtered copy of the dev_requirements.txt for the targeted package. This list will be + modified in place to remove any requirements incompatible with the packages_for_install. :param List[Requirement] packages_for_install: A list of packages that dev_requirements MUST be compatible with. """ cleansed_reqs = [] - for req in filtered_requirement_list: - cleansed_req = req.strip().replace("-e ", "").split("#")[0].split(";")[0] + for dev_requirement_line in filtered_requirement_list: + cleansed_dev_requirement_line = dev_requirement_line.strip().replace("-e ", "").split("#")[0].split(";")[0] - if cleansed_req: - # this is a replaced dev req that we can use pkginfo to resolve - if os.path.exists(cleansed_req): + if cleansed_dev_requirement_line: + # this is probably a replaced built wheel + if os.path.exists(cleansed_dev_requirement_line) and os.path.isfile(cleansed_dev_requirement_line): + logging.info(f"We are processing a replaced relative requirement: {cleansed_dev_requirement_line}") try: - local_package_metadata = pkginfo.get_metadata(cleansed_req) - + local_package_metadata = pkginfo.get_metadata(cleansed_dev_requirement_line) if local_package_metadata: - local_reqs = [Requirement(r) for r in local_package_metadata.requires_dist] - new_req = resolve_compatible_package(local_package_metadata.name, local_package_metadata.version, local_reqs, packages_for_install) - if new_req: - cleansed_reqs.append(new_req) - else: - cleansed_reqs.append(cleansed_req) + requirements_for_dev_req = [Requirement(r) for r in local_package_metadata.requires_dist] + if not is_package_compatible(requirements_for_dev_req, packages_for_install): + breakpoint() + new_req = resolve_compatible_package(local_package_metadata.name, local_package_metadata.version, packages_for_install) + + if new_req: + cleansed_reqs.append(new_req) + else: + cleansed_reqs.append(cleansed_dev_requirement_line) else: - logging.error(f"Error while processing locally built requirement {cleansed_req}. Unable to resolve metadata.") - cleansed_reqs.append(cleansed_req) + logging.error(f"Error while processing locally built requirement {cleansed_dev_requirement_line}. Unable to resolve metadata.") + cleansed_reqs.append(cleansed_dev_requirement_line) except Exception as e: - logging.error(f"Error while processing locally built requirement {cleansed_req}: {e}") - cleansed_reqs.append(cleansed_req) - # relative requirement - elif cleansed_req.startswith("."): + logging.error(f"Error while processing locally built requirement {cleansed_dev_requirement_line}: {e}") + cleansed_reqs.append(cleansed_dev_requirement_line) + # this is a relative requirement + elif cleansed_dev_requirement_line.startswith("."): + logging.info(f"We are processing a relative requirement: {cleansed_dev_requirement_line}") try: - local_package = ParsedSetup.from_path(os.path.join(setup_path, cleansed_req)) - local_reqs = [Requirement(r) for r in local_package.requires] - new_req = resolve_compatible_package(local_package.name, local_package.version, local_reqs, packages_for_install) - - if new_req: - cleansed_reqs.append(new_req) - else: - cleansed_reqs.append(new_req) + local_package = ParsedSetup.from_path(os.path.join(setup_path, cleansed_dev_requirement_line)) + + if local_package: + requirements_for_dev_req = [Requirement(r) for r in local_package.requires] + if not is_package_compatible(requirements_for_dev_req, packages_for_install): + breakpoint() + new_req = resolve_compatible_package(local_package.name, local_package.version, packages_for_install) + if new_req: + cleansed_reqs.append(new_req) + else: + cleansed_reqs.append(cleansed_reqs) except Exception as e: - logging.error(f"Error while processing relative requirement {cleansed_req}: {e}") - cleansed_reqs.append(cleansed_req) + logging.error(f"Error while processing relative requirement {cleansed_dev_requirement_line}: {e}") + cleansed_reqs.append(cleansed_dev_requirement_line) else: + breakpoint() + # try to parse it as a standard specifier. If it is, we can probably add it to the list of requirements unless it conflicts with the current set of packages # doing nothing here, as we don't understand how to resolve this yet - cleansed_reqs.append(cleansed_req) - logging.info(f"While filtering incompatible minimum dev requirements, found a requirement that I don't know how to deal with yet: \"{cleansed_req}\"") + cleansed_reqs.append(cleansed_dev_requirement_line) + logging.info(f"While filtering incompatible minimum dev requirements, found a requirement that we don't know how to deal with yet: \"{cleansed_dev_requirement_line}\"") continue return cleansed_reqs @@ -206,12 +243,14 @@ def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir): override_added_packages.extend(check_pkg_against_overrides(pkg_spec)) logging.info("%s released packages: %s", dependency_type, released_packages) - # filter released packages from dev_requirements and create a new file "new_dev_requirements.txt" additionalFilterFn = None if dependency_type == "Minimum": additionalFilterFn = handle_incompatible_minimum_dev_reqs + # before september 2024, filter_dev_requirements only would remove any packages present in released_packages from the dev_requirements, + # then create a new file "new_dev_requirements.txt" without the problematic packages. + # after september 2024, filter_dev_requirements will also check for **compatibility** with the packages being installed when filtering the dev_requirements. dev_req_file_path = filter_dev_requirements(setup_py_file_path, released_packages, temp_dir, additionalFilterFn) if override_added_packages: @@ -412,6 +451,13 @@ def check_req_against_exclusion(req, req_to_exclude): def filter_dev_requirements(setup_py_path, released_packages, temp_dir, additionalFilterFn: Optional[Callable[[str, List[str],List[Requirement]], List[str]]] = None): + """ + This function takes an existing package path, a list of specific package specifiers that we have resolved, a temporary directory to write + the modified dev_requirements to, and an optional additionalFilterFn that can be used to further filter the dev_requirements file if necessary. + + The function will filter out any requirements present in the dev_requirements file that are present in the released_packages list (aka are required + by the package). + """ # This method returns list of requirements from dev_requirements by filtering out packages in given list dev_req_path = os.path.join(os.path.dirname(setup_py_path), DEV_REQ_FILE) requirements = [] @@ -435,8 +481,9 @@ def filter_dev_requirements(setup_py_path, released_packages, temp_dir, addition and not any([check_req_against_exclusion(req, i) for i in req_to_exclude]) ] - # if additionalFilterFn: - # filtered_req = additionalFilterFn(setup_py_path, filtered_req, released_packages) + if additionalFilterFn: + # this filter function handles the case where a dev requirement is incompatible with the current set of targeted packages + filtered_req = additionalFilterFn(setup_py_path, filtered_req, released_packages) logging.info("Filtered dev requirements: %s", filtered_req) From a860e46f3479d742149d09f0fa377d3d855c5e93 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Mon, 30 Sep 2024 19:13:37 -0700 Subject: [PATCH 05/18] we've very close now. --- eng/tox/install_depend_packages.py | 67 ++++++++++++++++-------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index 54d163a0a0ca..52e2f26ee039 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -88,7 +88,7 @@ } } -def is_package_compatible(package_requirements: List[Requirement], immutable_requirements: List[Requirement]) -> bool: +def is_package_compatible(package_name: str, package_requirements: List[Requirement], immutable_requirements: List[Requirement], should_log: bool = True) -> bool: """ This function accepts a set of requirements for a package, and ensures that the package is compatible with the immutable_requirements. @@ -110,50 +110,57 @@ def is_package_compatible(package_requirements: List[Requirement], immutable_req immutable_version = next(iter(immutable_requirement.specifier)).version if not package_requirement.specifier.contains(immutable_version): - logging.info(f"Dev req dependency {package_requirement} is not compatible with immutable requirement {immutable_requirement}.") + if should_log: + logging.info(f"Dev req dependency {package_name}'s requirement specifier of {package_requirement} is not compatible with immutable requirement {immutable_requirement}.") return False return True -def resolve_compatible_package(package_name: str, package_version: str, immutable_requirements: List[Requirement]) -> Optional[str]: +def resolve_compatible_package(package_name: str, immutable_requirements: List[Requirement]) -> Optional[str]: """ - This function accepts a targeted package, a set of requirements for that package, and a set of immutable_requirements that the package must be compatible with. + This function attempts to resolve a compatible package version for whatever set of immutable_requirements that the package must be compatible with. - This function should only be utilized when a package is found to be incompatible with the immutable_requirements. It will attempt to resolve the incompatibility by + It should only be utilized when a package is found to be incompatible with the immutable_requirements. It will attempt to resolve the incompatibility by walking backwards through different versions of until a compatible version is found that works with the immutable_requirements. """ pypi = PyPIClient() immovable_pkgs = {req.key: req for req in immutable_requirements} - # Let's use a real use-case to walk through this function. We're going to use the azure-ai-language-conversations package as an example. + # Let's use a real use-case to walk through this function. We're going to use the azure-ai-language-conversations package as an example.\ + # immovable_pkgs = the selected mindependency for azure-ai-language-conversations # -> "azure-core==1.28.0", # -> "isodate==0.6.1", # -> "typing-extensions==4.0.1", + # we have the following dev_reqs for azure-ai-language-conversations + # -> ../azure-sdk-tools + # -> ../azure-identity + # -> ../azure-core + + # as we walk each of the dev reqs, we check for compatibility with the immovable_packages. (this happens in is_package_compatible) + # if the dev req is incompatible, we need to resolve it. THIS function is what resolves it! + + # since we already know that package_name is incompatible with the immovable_pkgs, we need to walk backwards through the versions of package_name + # checking to ensure that each version is compatible with the immovable_pkgs. + # if we find one that is, we will return a new requirement string for that package which will replace this dev_req line. for pkg in immovable_pkgs: + required_package = immovable_pkgs[pkg].name required_pkg_version = next(iter(immovable_pkgs[pkg].specifier)).version - # as we walk through each hard dependency, we need to check all of the dev_requirements for compatibility with these packages - # for dev_req_dependency in dev_req_pkg_reqs: - # # if the dev_req_dependency is present in the hard dependencies, we need to check compatibility - # if dev_req_dependency.key in immovable_pkgs: - # if str(dev_req_dependency.specifier): - # # check compatibility here. if the dep is compatible with the current set of packages, we can use it. - # # otherwise we need to keep walking backwards until we find a compatible version. - # # from here on we have to check the same dependency for this package on the retrieved version as well - # available_versions = pypi.get_ordered_versions(dev_req_dependency.key, True) # they come back smallest to largest by default - # available_versions.reverse() - # if not required_pkg_version in dev_req_dependency.specifier: - # pypi.get_ordered_versions(dev_req_dependency.key, True) + versions = pypi.get_ordered_versions(package_name, True) + versions.reverse() + for version in versions: + version_release = pypi.project_release(package_name, version).get("info", {}).get("requires_dist", []) - # # we're attempting to do this entirely with metadata instead of downloading each version - # for version in available_versions: - # version_release = pypi.project_release(dev_req_dependency.key, version) - # breakpoint() + if version_release: + requirements_for_dev_req = [Requirement(r) for r in version_release] - # breakpoint() + compatible = is_package_compatible(required_package, requirements_for_dev_req, immutable_requirements, should_log=False) + if compatible: + # we have found a compatible version. We can return this as the new requirement line for the dev_req file. + return f"{package_name}=={version}" # no changes necessary return None @@ -186,9 +193,8 @@ def handle_incompatible_minimum_dev_reqs(setup_path: str, filtered_requirement_l local_package_metadata = pkginfo.get_metadata(cleansed_dev_requirement_line) if local_package_metadata: requirements_for_dev_req = [Requirement(r) for r in local_package_metadata.requires_dist] - if not is_package_compatible(requirements_for_dev_req, packages_for_install): - breakpoint() - new_req = resolve_compatible_package(local_package_metadata.name, local_package_metadata.version, packages_for_install) + if not is_package_compatible(local_package_metadata.name, requirements_for_dev_req, packages_for_install): + new_req = resolve_compatible_package(local_package_metadata.name, packages_for_install) if new_req: cleansed_reqs.append(new_req) @@ -208,13 +214,12 @@ def handle_incompatible_minimum_dev_reqs(setup_path: str, filtered_requirement_l if local_package: requirements_for_dev_req = [Requirement(r) for r in local_package.requires] - if not is_package_compatible(requirements_for_dev_req, packages_for_install): - breakpoint() - new_req = resolve_compatible_package(local_package.name, local_package.version, packages_for_install) + if not is_package_compatible(local_package.name, requirements_for_dev_req, packages_for_install): + new_req = resolve_compatible_package(local_package.name, packages_for_install) if new_req: cleansed_reqs.append(new_req) else: - cleansed_reqs.append(cleansed_reqs) + cleansed_reqs.append(cleansed_dev_requirement_line) except Exception as e: logging.error(f"Error while processing relative requirement {cleansed_dev_requirement_line}: {e}") @@ -224,7 +229,6 @@ def handle_incompatible_minimum_dev_reqs(setup_path: str, filtered_requirement_l # try to parse it as a standard specifier. If it is, we can probably add it to the list of requirements unless it conflicts with the current set of packages # doing nothing here, as we don't understand how to resolve this yet cleansed_reqs.append(cleansed_dev_requirement_line) - logging.info(f"While filtering incompatible minimum dev requirements, found a requirement that we don't know how to deal with yet: \"{cleansed_dev_requirement_line}\"") continue return cleansed_reqs @@ -253,6 +257,7 @@ def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir): # after september 2024, filter_dev_requirements will also check for **compatibility** with the packages being installed when filtering the dev_requirements. dev_req_file_path = filter_dev_requirements(setup_py_file_path, released_packages, temp_dir, additionalFilterFn) + breakpoint() if override_added_packages: logging.info(f"Expanding the requirement set by the packages {override_added_packages}.") From e26e83f69f01a54e352672bd9b18649df8750be1 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 1 Oct 2024 11:54:03 -0700 Subject: [PATCH 06/18] handle incompatible dev requirements --- eng/tox/install_depend_packages.py | 77 +++++++++++++------ scripts/devops_tasks/common_tasks.py | 2 +- .../azure-identity-broker/pyproject.toml | 3 +- tools/azure-sdk-tools/pypi_tools/pypi.py | 10 +-- 4 files changed, 59 insertions(+), 33 deletions(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index 52e2f26ee039..4e745232976a 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -116,7 +116,7 @@ def is_package_compatible(package_name: str, package_requirements: List[Requirem return True -def resolve_compatible_package(package_name: str, immutable_requirements: List[Requirement]) -> Optional[str]: +def resolve_compatible_package(package_name: str, package_version: Optional[str], immutable_requirements: List[Requirement]) -> Optional[str]: """ This function attempts to resolve a compatible package version for whatever set of immutable_requirements that the package must be compatible with. @@ -151,6 +151,11 @@ def resolve_compatible_package(package_name: str, immutable_requirements: List[R versions = pypi.get_ordered_versions(package_name, True) versions.reverse() + # only allow prerelease versions if the dev_req we're targeting is also prerelease + if required_pkg_version: + if not Version(required_pkg_version).is_prerelease: + versions = [v for v in versions if not v.is_prerelease] + for version in versions: version_release = pypi.project_release(package_name, version).get("info", {}).get("requires_dist", []) @@ -186,50 +191,73 @@ def handle_incompatible_minimum_dev_reqs(setup_path: str, filtered_requirement_l cleansed_dev_requirement_line = dev_requirement_line.strip().replace("-e ", "").split("#")[0].split(";")[0] if cleansed_dev_requirement_line: - # this is probably a replaced built wheel + dev_req_package = None + dev_req_version = None + requirements_for_dev_req = [] + + # this is a locally built wheel file, ise pkginfo to get the metadata if os.path.exists(cleansed_dev_requirement_line) and os.path.isfile(cleansed_dev_requirement_line): - logging.info(f"We are processing a replaced relative requirement: {cleansed_dev_requirement_line}") + logging.info(f"We are processing a replaced relative requirement built into a wheel: {cleansed_dev_requirement_line}") try: local_package_metadata = pkginfo.get_metadata(cleansed_dev_requirement_line) if local_package_metadata: + dev_req_package = local_package_metadata.name + dev_req_version = local_package_metadata.version requirements_for_dev_req = [Requirement(r) for r in local_package_metadata.requires_dist] - if not is_package_compatible(local_package_metadata.name, requirements_for_dev_req, packages_for_install): - new_req = resolve_compatible_package(local_package_metadata.name, packages_for_install) - - if new_req: - cleansed_reqs.append(new_req) - else: - cleansed_reqs.append(cleansed_dev_requirement_line) else: logging.error(f"Error while processing locally built requirement {cleansed_dev_requirement_line}. Unable to resolve metadata.") cleansed_reqs.append(cleansed_dev_requirement_line) except Exception as e: - logging.error(f"Error while processing locally built requirement {cleansed_dev_requirement_line}: {e}") + logging.error(f"Unable to determine metadata for locally built requirement {cleansed_dev_requirement_line}: {e}") cleansed_reqs.append(cleansed_dev_requirement_line) - # this is a relative requirement + continue + + # this is a relative requirement to a package path in the repo, use our ParsedSetup class to get data from setup.py or pyproject.toml elif cleansed_dev_requirement_line.startswith("."): logging.info(f"We are processing a relative requirement: {cleansed_dev_requirement_line}") try: local_package = ParsedSetup.from_path(os.path.join(setup_path, cleansed_dev_requirement_line)) if local_package: + dev_req_package = local_package.name + dev_req_version = local_package.version requirements_for_dev_req = [Requirement(r) for r in local_package.requires] - if not is_package_compatible(local_package.name, requirements_for_dev_req, packages_for_install): - new_req = resolve_compatible_package(local_package.name, packages_for_install) - if new_req: - cleansed_reqs.append(new_req) - else: - cleansed_reqs.append(cleansed_dev_requirement_line) + else: + logging.error(f"Error while processing relative requirement {cleansed_dev_requirement_line}. Unable to resolve metadata.") + cleansed_reqs.append(cleansed_dev_requirement_line) except Exception as e: - logging.error(f"Error while processing relative requirement {cleansed_dev_requirement_line}: {e}") + logging.error(f"Unable to determine metadata for relative requirement \"{cleansed_dev_requirement_line}\", not modifying: {e}") cleansed_reqs.append(cleansed_dev_requirement_line) + continue + # If we got here, this has to be a standard requirement, attempt to parse it as a specifier and if unable to do so, + # simply add it to the list as a last fallback. we will log so that we can implement a fix for the edge case later. else: - breakpoint() - # try to parse it as a standard specifier. If it is, we can probably add it to the list of requirements unless it conflicts with the current set of packages - # doing nothing here, as we don't understand how to resolve this yet + logging.info(f"We are processing a standard requirement: {cleansed_dev_requirement_line}") cleansed_reqs.append(cleansed_dev_requirement_line) - continue + + # todo, fix this + # try: + # dev_req_package = Requirement(cleansed_dev_requirement_line).name + # dev_req_version = Requirement(cleansed_dev_requirement_line).specifier + # requirements_for_dev_req = [Requirement(cleansed_dev_requirement_line)] + # except Exception as e: + # logging.error(f"Unable to parse standard requirement {cleansed_dev_requirement_line}: {e}") + # cleansed_reqs.append(cleansed_dev_requirement_line) + # continue + + # we understand how to parse it, so we should handle it + if dev_req_package: + if not is_package_compatible(dev_req_package, requirements_for_dev_req, packages_for_install): + new_req = resolve_compatible_package(dev_req_package, dev_req_version, packages_for_install) + + if new_req: + cleansed_reqs.append(new_req) + else: + logging.error(f"Found incompatible dev requirement {dev_req_package}, but unable to locate a compatible version. Not modifying the line: \"{dev_requirement_line}\".") + cleansed_reqs.append(cleansed_dev_requirement_line) + else: + cleansed_reqs.append(cleansed_dev_requirement_line) return cleansed_reqs @@ -257,7 +285,6 @@ def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir): # after september 2024, filter_dev_requirements will also check for **compatibility** with the packages being installed when filtering the dev_requirements. dev_req_file_path = filter_dev_requirements(setup_py_file_path, released_packages, temp_dir, additionalFilterFn) - breakpoint() if override_added_packages: logging.info(f"Expanding the requirement set by the packages {override_added_packages}.") @@ -497,7 +524,7 @@ def filter_dev_requirements(setup_py_path, released_packages, temp_dir, addition # create new dev requirements file with different name for filtered requirements new_dev_req_path = os.path.join(temp_dir, NEW_DEV_REQ_FILE) with open(new_dev_req_path, "w") as dev_req_file: - dev_req_file.writelines(filtered_req) + dev_req_file.writelines(line if line.endswith("\n") else line + "\n" for line in filtered_req) return new_dev_req_path diff --git a/scripts/devops_tasks/common_tasks.py b/scripts/devops_tasks/common_tasks.py index ab6737362c8b..864bcaccc39f 100644 --- a/scripts/devops_tasks/common_tasks.py +++ b/scripts/devops_tasks/common_tasks.py @@ -205,7 +205,7 @@ def is_required_version_on_pypi(package_name: str, spec: str) -> bool: versions = [str(v) for v in versions if v in specifier] except: logging.error("Package {} is not found on PyPI".format(package_name)) - return versions + return bool(versions) def find_packages_missing_on_pypi(path: str) -> Iterable[str]: diff --git a/sdk/identity/azure-identity-broker/pyproject.toml b/sdk/identity/azure-identity-broker/pyproject.toml index cc83baa914bb..ea31fd0986d0 100644 --- a/sdk/identity/azure-identity-broker/pyproject.toml +++ b/sdk/identity/azure-identity-broker/pyproject.toml @@ -1,4 +1,3 @@ [tool.azure-sdk-build] type_check_samples = false -pyright = false -mindependency = false \ No newline at end of file +pyright = false \ No newline at end of file diff --git a/tools/azure-sdk-tools/pypi_tools/pypi.py b/tools/azure-sdk-tools/pypi_tools/pypi.py index a4a4dcb18aef..b68e5daae8d1 100644 --- a/tools/azure-sdk-tools/pypi_tools/pypi.py +++ b/tools/azure-sdk-tools/pypi_tools/pypi.py @@ -1,5 +1,5 @@ import logging -from packaging.version import InvalidVersion, parse as Version +from packaging.version import InvalidVersion, Version, parse import sys import pdb from urllib3 import Retry, PoolManager @@ -41,13 +41,13 @@ def filter_packages_for_compatibility(self, package_name, version_set): # only need the packaging.specifiers import if we're actually executing this filter. from packaging.specifiers import InvalidSpecifier, SpecifierSet - results = [] + results: List[Version] = [] for version in version_set: requires_python = self.project_release(package_name, version)["info"]["requires_python"] if requires_python: try: - if Version(".".join(map(str, sys.version_info[:3]))) in SpecifierSet(requires_python): + if parse(".".join(map(str, sys.version_info[:3]))) in SpecifierSet(requires_python): results.append(version) except InvalidSpecifier: logging.warn(f"Invalid python_requires {requires_python!r} for package {package_name}=={version}") @@ -60,10 +60,10 @@ def filter_packages_for_compatibility(self, package_name, version_set): def get_ordered_versions(self, package_name, filter_by_compatibility=False) -> List[Version]: project = self.project(package_name) - versions = [] + versions: List[Version] = [] for package_version in project["releases"].keys(): try: - versions.append(Version(package_version)) + versions.append(parse(package_version)) except InvalidVersion as e: logging.warn(f"Invalid version {package_version} for package {package_name}") continue From c1164c9f802a0b950d070f5eae11f060a7d58b3c Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 1 Oct 2024 16:47:46 -0700 Subject: [PATCH 07/18] move new functions to some place that can be more easily tested --- eng/tox/install_depend_packages.py | 176 +-------------- tools/azure-sdk-tools/ci_tools/functions.py | 231 +++++++++++++++++++- tools/azure-sdk-tools/ci_tools/variables.py | 8 +- 3 files changed, 224 insertions(+), 191 deletions(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index 4e745232976a..19720638a11b 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -10,7 +10,6 @@ import sys import logging import re -import pkginfo from subprocess import check_call from typing import TYPE_CHECKING, Callable, Optional @@ -20,7 +19,7 @@ from packaging.version import Version from ci_tools.parsing import ParsedSetup, parse_require -from ci_tools.functions import compare_python_version +from ci_tools.functions import compare_python_version, handle_incompatible_minimum_dev_reqs from typing import List @@ -88,179 +87,6 @@ } } -def is_package_compatible(package_name: str, package_requirements: List[Requirement], immutable_requirements: List[Requirement], should_log: bool = True) -> bool: - """ - This function accepts a set of requirements for a package, and ensures that the package is compatible with the immutable_requirements. - - It is factored this way because we retrieve requirements differently according to the source of the package. - If published, we can get requires() from PyPI - If locally built wheel, we can get requires() from the metadata of the package - If local relative requirement, we can get requires() from a ParsedSetup of the setup.py for th package - - :param List[Requirement] package_requirements: The dependencies of a dev_requirement file. This is the set of requirements that we are checking compatibility for. - :param List[Requirement] immutable_requirements: A list of requirements that the other packages must be compatible with. - """ - - for immutable_requirement in immutable_requirements: - for package_requirement in package_requirements: - if package_requirement.key == immutable_requirement.key: - # if the dev_req line has a requirement that conflicts with the immutable requirement, we need to resolve it - # we KNOW that the immutable requirement will be of form package==version, so we can reliably pull out the version - # and check it against the specifier of the dev_req line. - immutable_version = next(iter(immutable_requirement.specifier)).version - - if not package_requirement.specifier.contains(immutable_version): - if should_log: - logging.info(f"Dev req dependency {package_name}'s requirement specifier of {package_requirement} is not compatible with immutable requirement {immutable_requirement}.") - return False - - return True - -def resolve_compatible_package(package_name: str, package_version: Optional[str], immutable_requirements: List[Requirement]) -> Optional[str]: - """ - This function attempts to resolve a compatible package version for whatever set of immutable_requirements that the package must be compatible with. - - It should only be utilized when a package is found to be incompatible with the immutable_requirements. It will attempt to resolve the incompatibility by - walking backwards through different versions of until a compatible version is found that works with the immutable_requirements. - """ - - pypi = PyPIClient() - immovable_pkgs = {req.key: req for req in immutable_requirements} - - # Let's use a real use-case to walk through this function. We're going to use the azure-ai-language-conversations package as an example.\ - - # immovable_pkgs = the selected mindependency for azure-ai-language-conversations - # -> "azure-core==1.28.0", - # -> "isodate==0.6.1", - # -> "typing-extensions==4.0.1", - # we have the following dev_reqs for azure-ai-language-conversations - # -> ../azure-sdk-tools - # -> ../azure-identity - # -> ../azure-core - - # as we walk each of the dev reqs, we check for compatibility with the immovable_packages. (this happens in is_package_compatible) - # if the dev req is incompatible, we need to resolve it. THIS function is what resolves it! - - # since we already know that package_name is incompatible with the immovable_pkgs, we need to walk backwards through the versions of package_name - # checking to ensure that each version is compatible with the immovable_pkgs. - # if we find one that is, we will return a new requirement string for that package which will replace this dev_req line. - for pkg in immovable_pkgs: - required_package = immovable_pkgs[pkg].name - required_pkg_version = next(iter(immovable_pkgs[pkg].specifier)).version - - versions = pypi.get_ordered_versions(package_name, True) - versions.reverse() - - # only allow prerelease versions if the dev_req we're targeting is also prerelease - if required_pkg_version: - if not Version(required_pkg_version).is_prerelease: - versions = [v for v in versions if not v.is_prerelease] - - for version in versions: - version_release = pypi.project_release(package_name, version).get("info", {}).get("requires_dist", []) - - if version_release: - requirements_for_dev_req = [Requirement(r) for r in version_release] - - compatible = is_package_compatible(required_package, requirements_for_dev_req, immutable_requirements, should_log=False) - if compatible: - # we have found a compatible version. We can return this as the new requirement line for the dev_req file. - return f"{package_name}=={version}" - - # no changes necessary - return None - - - -def handle_incompatible_minimum_dev_reqs(setup_path: str, filtered_requirement_list: List[str], packages_for_install: List[Requirement]) -> List[str]: - """ - This function is used to handle the case where a dev requirement is incompatible with the current set of packages - being installed. This is used to update or remove dev_requirements that are incompatible with a targeted set of packages. - - :param str setup_path: The path to the setup.py file whos dev_requirements are being filtered. - - :param List[str] filtered_requirement_list: A filtered copy of the dev_requirements.txt for the targeted package. This list will be - modified in place to remove any requirements incompatible with the packages_for_install. - - :param List[Requirement] packages_for_install: A list of packages that dev_requirements MUST be compatible with. - """ - - cleansed_reqs = [] - - for dev_requirement_line in filtered_requirement_list: - cleansed_dev_requirement_line = dev_requirement_line.strip().replace("-e ", "").split("#")[0].split(";")[0] - - if cleansed_dev_requirement_line: - dev_req_package = None - dev_req_version = None - requirements_for_dev_req = [] - - # this is a locally built wheel file, ise pkginfo to get the metadata - if os.path.exists(cleansed_dev_requirement_line) and os.path.isfile(cleansed_dev_requirement_line): - logging.info(f"We are processing a replaced relative requirement built into a wheel: {cleansed_dev_requirement_line}") - try: - local_package_metadata = pkginfo.get_metadata(cleansed_dev_requirement_line) - if local_package_metadata: - dev_req_package = local_package_metadata.name - dev_req_version = local_package_metadata.version - requirements_for_dev_req = [Requirement(r) for r in local_package_metadata.requires_dist] - else: - logging.error(f"Error while processing locally built requirement {cleansed_dev_requirement_line}. Unable to resolve metadata.") - cleansed_reqs.append(cleansed_dev_requirement_line) - except Exception as e: - logging.error(f"Unable to determine metadata for locally built requirement {cleansed_dev_requirement_line}: {e}") - cleansed_reqs.append(cleansed_dev_requirement_line) - continue - - # this is a relative requirement to a package path in the repo, use our ParsedSetup class to get data from setup.py or pyproject.toml - elif cleansed_dev_requirement_line.startswith("."): - logging.info(f"We are processing a relative requirement: {cleansed_dev_requirement_line}") - try: - local_package = ParsedSetup.from_path(os.path.join(setup_path, cleansed_dev_requirement_line)) - - if local_package: - dev_req_package = local_package.name - dev_req_version = local_package.version - requirements_for_dev_req = [Requirement(r) for r in local_package.requires] - else: - logging.error(f"Error while processing relative requirement {cleansed_dev_requirement_line}. Unable to resolve metadata.") - cleansed_reqs.append(cleansed_dev_requirement_line) - - except Exception as e: - logging.error(f"Unable to determine metadata for relative requirement \"{cleansed_dev_requirement_line}\", not modifying: {e}") - cleansed_reqs.append(cleansed_dev_requirement_line) - continue - # If we got here, this has to be a standard requirement, attempt to parse it as a specifier and if unable to do so, - # simply add it to the list as a last fallback. we will log so that we can implement a fix for the edge case later. - else: - logging.info(f"We are processing a standard requirement: {cleansed_dev_requirement_line}") - cleansed_reqs.append(cleansed_dev_requirement_line) - - # todo, fix this - # try: - # dev_req_package = Requirement(cleansed_dev_requirement_line).name - # dev_req_version = Requirement(cleansed_dev_requirement_line).specifier - # requirements_for_dev_req = [Requirement(cleansed_dev_requirement_line)] - # except Exception as e: - # logging.error(f"Unable to parse standard requirement {cleansed_dev_requirement_line}: {e}") - # cleansed_reqs.append(cleansed_dev_requirement_line) - # continue - - # we understand how to parse it, so we should handle it - if dev_req_package: - if not is_package_compatible(dev_req_package, requirements_for_dev_req, packages_for_install): - new_req = resolve_compatible_package(dev_req_package, dev_req_version, packages_for_install) - - if new_req: - cleansed_reqs.append(new_req) - else: - logging.error(f"Found incompatible dev requirement {dev_req_package}, but unable to locate a compatible version. Not modifying the line: \"{dev_requirement_line}\".") - cleansed_reqs.append(cleansed_dev_requirement_line) - else: - cleansed_reqs.append(cleansed_dev_requirement_line) - - return cleansed_reqs - def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir): # This method identifies latest/ minimal version of dependent packages and installs them from pyPI diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 882284e6d6a9..909fc764d7d3 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -14,7 +14,7 @@ from pypi_tools.pypi import PyPIClient import os, sys, platform, glob, re, logging -from typing import List, Any +from typing import List, Any, Optional INACTIVE_CLASSIFIER = "Development Status :: 7 - Inactive" @@ -269,7 +269,7 @@ def is_required_version_on_pypi(package_name, spec): return versions -def get_package_from_repo(pkg_name: str, repo_root: str = None) -> ParsedSetup: +def get_package_from_repo(pkg_name: str, repo_root: Optional[str] = None) -> Optional[ParsedSetup]: root_dir = discover_repo_root(repo_root) glob_path = os.path.join(root_dir, "sdk", "*", pkg_name, "setup.py") @@ -283,7 +283,7 @@ def get_package_from_repo(pkg_name: str, repo_root: str = None) -> ParsedSetup: return None -def get_package_from_repo_or_folder(req: str, prebuilt_wheel_dir: str = None) -> str: +def get_package_from_repo_or_folder(req: str, prebuilt_wheel_dir: Optional[str] = None) -> Optional[str]: """Takes a package name and a possible prebuilt wheel directory. Attempts to resolve a wheel that matches the package name, and if it can't, attempts to find the package within the repo to install directly from path on disk. @@ -293,7 +293,7 @@ def get_package_from_repo_or_folder(req: str, prebuilt_wheel_dir: str = None) -> local_package = get_package_from_repo(req) - if prebuilt_wheel_dir and os.path.exists(prebuilt_wheel_dir): + if prebuilt_wheel_dir and os.path.exists(prebuilt_wheel_dir) and local_package: prebuilt_package = discover_prebuilt_package(prebuilt_wheel_dir, local_package.setup_filename, "wheel") if prebuilt_package: # return the first package found, there should only be a single one matching given that our prebuilt wheel directory @@ -301,10 +301,13 @@ def get_package_from_repo_or_folder(req: str, prebuilt_wheel_dir: str = None) -> # ref tox_harness replace_dev_reqs() calls return os.path.join(prebuilt_wheel_dir, prebuilt_package[0]) - return local_package.folder + if local_package: + return local_package.folder + else: + return None -def get_version_from_repo(pkg_name: str, repo_root: str = None) -> str: +def get_version_from_repo(pkg_name: str, repo_root: Optional[str] = None) -> str: pkg_info = get_package_from_repo(pkg_name, repo_root) if pkg_info: # Remove dev build part if version for this package is already updated to dev build @@ -387,7 +390,7 @@ def process_requires(setup_py_path: str, is_dev_build: bool = False): logging.info("Package requirement is updated in setup.py") -def find_sdist(dist_dir: str, pkg_name: str, pkg_version: str) -> str: +def find_sdist(dist_dir: str, pkg_name: str, pkg_version: str) -> Optional[str]: """This function attempts to look within a directory (and all subdirs therein) and find a source distribution for the targeted package and version.""" # This function will find a sdist for given package name if not os.path.exists(dist_dir): @@ -416,7 +419,9 @@ def find_sdist(dist_dir: str, pkg_name: str, pkg_version: str) -> str: return packages[0] -def pip_install(requirements: List[str], include_dependencies: bool = True, python_executable: str = None) -> bool: +def pip_install( + requirements: List[str], include_dependencies: bool = True, python_executable: Optional[str] = None +) -> bool: """ Attempts to invoke an install operation using the invoking python's pip. Empty requirements are auto-success. """ @@ -457,11 +462,11 @@ def pip_uninstall(requirements: List[str], python_executable: str) -> bool: return False -def pip_install_requirements_file(requirements_file: str, python_executable: str = None) -> bool: +def pip_install_requirements_file(requirements_file: str, python_executable: Optional[str] = None) -> bool: return pip_install(["-r", requirements_file], True, python_executable) -def get_pip_list_output(python_executable: str = None): +def get_pip_list_output(python_executable: Optional[str] = None): """Uses the invoking python executable to get the output from pip list.""" exe = python_executable or sys.executable @@ -487,7 +492,7 @@ def get_pip_list_output(python_executable: str = None): return collected_output -def pytest(args: [], cwd: str = None, python_executable: str = None) -> bool: +def pytest(args: list, cwd: Optional[str] = None, python_executable: Optional[str] = None) -> bool: """ Invokes a set of tests, returns true if successful, false otherwise. """ @@ -526,6 +531,7 @@ def get_interpreter_compatible_tags() -> List[str]: tag_strings = output.split(os.linesep) + index = 0 for index, value in enumerate(tag_strings): if "Compatible tags" in value: break @@ -542,7 +548,7 @@ def check_whl_against_tags(whl_name: str, tags: List[str]) -> bool: return False -def find_whl(whl_dir: str, pkg_name: str, pkg_version: str) -> str: +def find_whl(whl_dir: str, pkg_name: str, pkg_version: str) -> Optional[str]: """This function attempts to look within a directory (and all subdirs therein) and find a wheel that matches our targeted name and version AND whose compilation is compatible with the invoking interpreter.""" if not os.path.exists(whl_dir): @@ -625,3 +631,204 @@ def discover_prebuilt_package(dist_directory: str, setup_path: str, package_type if prebuilt_package is not None: packages.append(prebuilt_package) return packages + + +def is_package_compatible( + package_name: str, + package_requirements: List[Requirement], + immutable_requirements: List[Requirement], + should_log: bool = True, +) -> bool: + """ + This function accepts a set of requirements for a package, and ensures that the package is compatible with the immutable_requirements. + + It is factored this way because we retrieve requirements differently according to the source of the package. + If published, we can get requires() from PyPI + If locally built wheel, we can get requires() from the metadata of the package + If local relative requirement, we can get requires() from a ParsedSetup of the setup.py for th package + + :param List[Requirement] package_requirements: The dependencies of a dev_requirement file. This is the set of requirements that we are checking compatibility for. + :param List[Requirement] immutable_requirements: A list of requirements that the other packages must be compatible with. + """ + + for immutable_requirement in immutable_requirements: + for package_requirement in package_requirements: + if package_requirement.key == immutable_requirement.key: + # if the dev_req line has a requirement that conflicts with the immutable requirement, we need to resolve it + # we KNOW that the immutable requirement will be of form package==version, so we can reliably pull out the version + # and check it against the specifier of the dev_req line. + immutable_version = next(iter(immutable_requirement.specifier)).version + + if not package_requirement.specifier.contains(immutable_version): + if should_log: + logging.info( + f"Dev req dependency {package_name}'s requirement specifier of {package_requirement} is not compatible with immutable requirement {immutable_requirement}." + ) + return False + + return True + + +def resolve_compatible_package( + package_name: str, package_version: Optional[str], immutable_requirements: List[Requirement] +) -> Optional[str]: + """ + This function attempts to resolve a compatible package version for whatever set of immutable_requirements that the package must be compatible with. + + It should only be utilized when a package is found to be incompatible with the immutable_requirements. It will attempt to resolve the incompatibility by + walking backwards through different versions of until a compatible version is found that works with the immutable_requirements. + """ + + pypi = PyPIClient() + immovable_pkgs = {req.key: req for req in immutable_requirements} + + # Let's use a real use-case to walk through this function. We're going to use the azure-ai-language-conversations package as an example.\ + + # immovable_pkgs = the selected mindependency for azure-ai-language-conversations + # -> "azure-core==1.28.0", + # -> "isodate==0.6.1", + # -> "typing-extensions==4.0.1", + # we have the following dev_reqs for azure-ai-language-conversations + # -> ../azure-sdk-tools + # -> ../azure-identity + # -> ../azure-core + + # as we walk each of the dev reqs, we check for compatibility with the immovable_packages. (this happens in is_package_compatible) + # if the dev req is incompatible, we need to resolve it. THIS function is what resolves it! + + # since we already know that package_name is incompatible with the immovable_pkgs, we need to walk backwards through the versions of package_name + # checking to ensure that each version is compatible with the immovable_pkgs. + # if we find one that is, we will return a new requirement string for that package which will replace this dev_req line. + for pkg in immovable_pkgs: + required_package = immovable_pkgs[pkg].name + required_pkg_version = next(iter(immovable_pkgs[pkg].specifier)).version + + versions = pypi.get_ordered_versions(package_name, True) + versions.reverse() + + # only allow prerelease versions if the dev_req we're targeting is also prerelease + if required_pkg_version: + if not Version(required_pkg_version).is_prerelease: + versions = [v for v in versions if not v.is_prerelease] + + for version in versions: + version_release = pypi.project_release(package_name, version).get("info", {}).get("requires_dist", []) + + if version_release: + requirements_for_dev_req = [Requirement(r) for r in version_release] + + compatible = is_package_compatible( + required_package, requirements_for_dev_req, immutable_requirements, should_log=False + ) + if compatible: + # we have found a compatible version. We can return this as the new requirement line for the dev_req file. + return f"{package_name}=={version}" + + # no changes necessary + return None + + +def handle_incompatible_minimum_dev_reqs( + setup_path: str, filtered_requirement_list: List[str], packages_for_install: List[Requirement] +) -> List[str]: + """ + This function is used to handle the case where a dev requirement is incompatible with the current set of packages + being installed. This is used to update or remove dev_requirements that are incompatible with a targeted set of packages. + + :param str setup_path: The path to the setup.py file whos dev_requirements are being filtered. + + :param List[str] filtered_requirement_list: A filtered copy of the dev_requirements.txt for the targeted package. This list will be + modified in place to remove any requirements incompatible with the packages_for_install. + + :param List[Requirement] packages_for_install: A list of packages that dev_requirements MUST be compatible with. + """ + + cleansed_reqs = [] + + for dev_requirement_line in filtered_requirement_list: + cleansed_dev_requirement_line = dev_requirement_line.strip().replace("-e ", "").split("#")[0].split(";")[0] + + if cleansed_dev_requirement_line: + dev_req_package = None + dev_req_version = None + requirements_for_dev_req = [] + + # this is a locally built wheel file, ise pkginfo to get the metadata + if os.path.exists(cleansed_dev_requirement_line) and os.path.isfile(cleansed_dev_requirement_line): + logging.info( + f"We are processing a replaced relative requirement built into a wheel: {cleansed_dev_requirement_line}" + ) + import pkginfo + + try: + local_package_metadata = pkginfo.get_metadata(cleansed_dev_requirement_line) + if local_package_metadata: + dev_req_package = local_package_metadata.name + dev_req_version = local_package_metadata.version + requirements_for_dev_req = [Requirement(r) for r in local_package_metadata.requires_dist] + else: + logging.error( + f"Error while processing locally built requirement {cleansed_dev_requirement_line}. Unable to resolve metadata." + ) + cleansed_reqs.append(cleansed_dev_requirement_line) + except Exception as e: + logging.error( + f"Unable to determine metadata for locally built requirement {cleansed_dev_requirement_line}: {e}" + ) + cleansed_reqs.append(cleansed_dev_requirement_line) + continue + + # this is a relative requirement to a package path in the repo, use our ParsedSetup class to get data from setup.py or pyproject.toml + elif cleansed_dev_requirement_line.startswith("."): + logging.info(f"We are processing a relative requirement: {cleansed_dev_requirement_line}") + try: + local_package = ParsedSetup.from_path(os.path.join(setup_path, cleansed_dev_requirement_line)) + + if local_package: + dev_req_package = local_package.name + dev_req_version = local_package.version + requirements_for_dev_req = [Requirement(r) for r in local_package.requires] + else: + logging.error( + f"Error while processing relative requirement {cleansed_dev_requirement_line}. Unable to resolve metadata." + ) + cleansed_reqs.append(cleansed_dev_requirement_line) + + except Exception as e: + logging.error( + f'Unable to determine metadata for relative requirement "{cleansed_dev_requirement_line}", not modifying: {e}' + ) + cleansed_reqs.append(cleansed_dev_requirement_line) + continue + # If we got here, this has to be a standard requirement, attempt to parse it as a specifier and if unable to do so, + # simply add it to the list as a last fallback. we will log so that we can implement a fix for the edge case later. + else: + logging.info(f"We are processing a standard requirement: {cleansed_dev_requirement_line}") + cleansed_reqs.append(cleansed_dev_requirement_line) + + # todo, fix this + # try: + # dev_req_package = Requirement(cleansed_dev_requirement_line).name + # dev_req_version = Requirement(cleansed_dev_requirement_line).specifier + # requirements_for_dev_req = [Requirement(cleansed_dev_requirement_line)] + # except Exception as e: + # logging.error(f"Unable to parse standard requirement {cleansed_dev_requirement_line}: {e}") + # cleansed_reqs.append(cleansed_dev_requirement_line) + # continue + + # we understand how to parse it, so we should handle it + if dev_req_package: + if not is_package_compatible(dev_req_package, requirements_for_dev_req, packages_for_install): + new_req = resolve_compatible_package(dev_req_package, dev_req_version, packages_for_install) + + if new_req: + cleansed_reqs.append(new_req) + else: + logging.error( + f'Found incompatible dev requirement {dev_req_package}, but unable to locate a compatible version. Not modifying the line: "{dev_requirement_line}".' + ) + cleansed_reqs.append(cleansed_dev_requirement_line) + else: + cleansed_reqs.append(cleansed_dev_requirement_line) + + return cleansed_reqs diff --git a/tools/azure-sdk-tools/ci_tools/variables.py b/tools/azure-sdk-tools/ci_tools/variables.py index d74417ad8f7d..0b84dae8b2b8 100644 --- a/tools/azure-sdk-tools/ci_tools/variables.py +++ b/tools/azure-sdk-tools/ci_tools/variables.py @@ -1,5 +1,5 @@ import os - +from typing import Optional def str_to_bool(input_string: str) -> bool: """ @@ -15,7 +15,7 @@ def str_to_bool(input_string: str) -> bool: return False -def discover_repo_root(input_repo: str = None): +def discover_repo_root(input_repo: Optional[str] = None): """ Resolves the root of the repository given a current working directory. This function should be used if a target repo argument is not provided. If the value of input_repo has value, that will supplant the path ascension logic. @@ -38,7 +38,7 @@ def discover_repo_root(input_repo: str = None): ) -def get_artifact_directory(input_directory: str = None) -> str: +def get_artifact_directory(input_directory: Optional[str] = None) -> str: """ Resolves the root of an artifact directory that the \"sdk_build\" action will output to! """ @@ -49,7 +49,7 @@ def get_artifact_directory(input_directory: str = None) -> str: return os.getenv("SDK_ARTIFACT_DIRECTORY", os.path.join(discover_repo_root(), ".artifacts")) -def get_log_directory(input_directory: str = None) -> str: +def get_log_directory(input_directory: Optional[str] = None) -> str: """ Resolves the location of the log directory. """ From 6c6cc93db05ce3f40d2116ec403537f0afbee0c2 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 1 Oct 2024 16:58:39 -0700 Subject: [PATCH 08/18] ensure we don't accidentally manipulate the standard requirements --- tools/azure-sdk-tools/ci_tools/functions.py | 12 +----------- .../tests/test_conflict_resolution.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 11 deletions(-) create mode 100644 tools/azure-sdk-tools/tests/test_conflict_resolution.py diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 909fc764d7d3..005edda414b5 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -804,17 +804,7 @@ def handle_incompatible_minimum_dev_reqs( # simply add it to the list as a last fallback. we will log so that we can implement a fix for the edge case later. else: logging.info(f"We are processing a standard requirement: {cleansed_dev_requirement_line}") - cleansed_reqs.append(cleansed_dev_requirement_line) - - # todo, fix this - # try: - # dev_req_package = Requirement(cleansed_dev_requirement_line).name - # dev_req_version = Requirement(cleansed_dev_requirement_line).specifier - # requirements_for_dev_req = [Requirement(cleansed_dev_requirement_line)] - # except Exception as e: - # logging.error(f"Unable to parse standard requirement {cleansed_dev_requirement_line}: {e}") - # cleansed_reqs.append(cleansed_dev_requirement_line) - # continue + cleansed_reqs.append(dev_requirement_line) # we understand how to parse it, so we should handle it if dev_req_package: diff --git a/tools/azure-sdk-tools/tests/test_conflict_resolution.py b/tools/azure-sdk-tools/tests/test_conflict_resolution.py new file mode 100644 index 000000000000..01c00149f85c --- /dev/null +++ b/tools/azure-sdk-tools/tests/test_conflict_resolution.py @@ -0,0 +1,11 @@ +import os + +from unittest.mock import patch + +from tempfile import TemporaryDirectory + +from ci_tools.functions import resolve_compatible_package, is_package_compatible + + + + From 150b282988eb4c54bacb40e17190d7ce715b986c Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 1 Oct 2024 17:45:57 -0700 Subject: [PATCH 09/18] add a couple tests --- tools/azure-sdk-tools/ci_tools/functions.py | 2 +- .../tests/test_conflict_resolution.py | 22 ++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 005edda414b5..135d8c24e010 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -735,7 +735,7 @@ def handle_incompatible_minimum_dev_reqs( This function is used to handle the case where a dev requirement is incompatible with the current set of packages being installed. This is used to update or remove dev_requirements that are incompatible with a targeted set of packages. - :param str setup_path: The path to the setup.py file whos dev_requirements are being filtered. + :param str setup_path: The path to the setup.py file whose dev_requirements are being filtered. :param List[str] filtered_requirement_list: A filtered copy of the dev_requirements.txt for the targeted package. This list will be modified in place to remove any requirements incompatible with the packages_for_install. diff --git a/tools/azure-sdk-tools/tests/test_conflict_resolution.py b/tools/azure-sdk-tools/tests/test_conflict_resolution.py index 01c00149f85c..ad769a2026cd 100644 --- a/tools/azure-sdk-tools/tests/test_conflict_resolution.py +++ b/tools/azure-sdk-tools/tests/test_conflict_resolution.py @@ -1,11 +1,27 @@ -import os +import pytest from unittest.mock import patch - from tempfile import TemporaryDirectory - from ci_tools.functions import resolve_compatible_package, is_package_compatible +from typing import Optional, List +from packaging.version import Version +from pkg_resources import Requirement +@pytest.mark.parametrize( + "input_requirements, immutable_requirements, expected_result", + [([Requirement("sphinx==1.0.0")], [Requirement("sphinx>=1.0.0")], True), + ([Requirement("sphinx==1.0.0")], [Requirement("sphinx>=1.1.0")], False)], +) +def test_incompatible_specifier(fake_package_input_requirements, immutable_requirements, expected_result): + result = is_package_compatible("fake-package", fake_package_input_requirements, immutable_requirements) + assert result == expected_result +def test_identity_resolution(): + result = resolve_compatible_package( + "azure-identity", + "1.18.1", + [Requirement("azure-core>=1.28.0"), Requirement("isodate>=0.6.1"), Requirement("typing-extensions>=4.0.1")], + ) + assert result == "azure-identity==1.16.0" From e5621faddc0e309e2c1194ffc8bc45771c9654cf Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:35:30 -0700 Subject: [PATCH 10/18] Update eng/tox/install_depend_packages.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com> --- eng/tox/install_depend_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index 19720638a11b..5e5a765e6c66 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -102,7 +102,7 @@ def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir): logging.info("%s released packages: %s", dependency_type, released_packages) - additionalFilterFn = None + additional_filter_fn = None if dependency_type == "Minimum": additionalFilterFn = handle_incompatible_minimum_dev_reqs From a6d75efccc9b414286ef11f77e6627de6f4a2036 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 2 Oct 2024 16:44:45 -0700 Subject: [PATCH 11/18] save reformatted functions.py --- tools/azure-sdk-tools/ci_tools/functions.py | 60 ++++++++++++--------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 135d8c24e010..5c8e9d7eb4e8 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -192,9 +192,13 @@ def discover_targeted_packages( :param str glob_string: The basic glob used to query packages within the repo. Defaults to "azure-*" :param str target_root_dir: The root directory in which globbing will begin. - :param str additional_contains_filter: Additional filter option. Used when needing to provide one-off filtration that doesn't merit an additional filter_type. Defaults to empty string. - :param str filter_type: One a string representing a filter function as a set of options. Options [ "Build", "Docs", "Regression", "Omit_management" ] Defaults to "Build". - :param bool compatibility_filter: Enables or disables compatibility filtering of found packages. If the invoking python executable does not match a found package's specifiers, the package will be omitted. Defaults to True. + :param str additional_contains_filter: Additional filter option. + Used when needing to provide one-off filtration that doesn't merit an additional filter_type. Defaults to empty string. + :param str filter_type: One a string representing a filter function as a set of options. + Options [ "Build", "Docs", "Regression", "Omit_management" ] Defaults to "Build". + :param bool compatibility_filter: Enables or disables compatibility filtering of found packages. + If the invoking python executable does not match a found package's specifiers, the package will be omitted. + Defaults to True. """ # glob the starting package set @@ -640,49 +644,54 @@ def is_package_compatible( should_log: bool = True, ) -> bool: """ - This function accepts a set of requirements for a package, and ensures that the package is compatible with the immutable_requirements. + This function accepts a set of requirements for a package, and ensures that the package is compatible with the + immutable_requirements. It is factored this way because we retrieve requirements differently according to the source of the package. If published, we can get requires() from PyPI If locally built wheel, we can get requires() from the metadata of the package If local relative requirement, we can get requires() from a ParsedSetup of the setup.py for th package - :param List[Requirement] package_requirements: The dependencies of a dev_requirement file. This is the set of requirements that we are checking compatibility for. - :param List[Requirement] immutable_requirements: A list of requirements that the other packages must be compatible with. + :param List[Requirement] package_requirements: The dependencies of a dev_requirement file. This is the set of + requirements that we are checking compatibility for. + :param List[Requirement] immutable_requirements: A list of requirements that the other packages must be compatible + with. """ for immutable_requirement in immutable_requirements: for package_requirement in package_requirements: if package_requirement.key == immutable_requirement.key: - # if the dev_req line has a requirement that conflicts with the immutable requirement, we need to resolve it - # we KNOW that the immutable requirement will be of form package==version, so we can reliably pull out the version - # and check it against the specifier of the dev_req line. + # if the dev_req line has a requirement that conflicts with the immutable requirement, + # we need to resolve it. We KNOW that the immutable requirement will be of form package==version, + # so we can reliably pull out the version and check it against the specifier of the dev_req line. immutable_version = next(iter(immutable_requirement.specifier)).version if not package_requirement.specifier.contains(immutable_version): if should_log: logging.info( - f"Dev req dependency {package_name}'s requirement specifier of {package_requirement} is not compatible with immutable requirement {immutable_requirement}." + f"Dev req dependency {package_name}'s requirement specifier of {package_requirement}" + f"is not compatible with immutable requirement {immutable_requirement}." ) return False return True -def resolve_compatible_package( - package_name: str, package_version: Optional[str], immutable_requirements: List[Requirement] -) -> Optional[str]: +def resolve_compatible_package(package_name: str, immutable_requirements: List[Requirement]) -> Optional[str]: """ - This function attempts to resolve a compatible package version for whatever set of immutable_requirements that the package must be compatible with. + This function attempts to resolve a compatible package version for whatever set of immutable_requirements that + the package must be compatible with. - It should only be utilized when a package is found to be incompatible with the immutable_requirements. It will attempt to resolve the incompatibility by - walking backwards through different versions of until a compatible version is found that works with the immutable_requirements. + It should only be utilized when a package is found to be incompatible with the immutable_requirements. + It will attempt to resolve the incompatibility by walking backwards through different versions of + until a compatible version is found that works with the immutable_requirements. """ pypi = PyPIClient() immovable_pkgs = {req.key: req for req in immutable_requirements} - # Let's use a real use-case to walk through this function. We're going to use the azure-ai-language-conversations package as an example.\ + # Let's use a real use-case to walk through this function. We're going to use the azure-ai-language-conversations + # package as an example. # immovable_pkgs = the selected mindependency for azure-ai-language-conversations # -> "azure-core==1.28.0", @@ -693,11 +702,12 @@ def resolve_compatible_package( # -> ../azure-identity # -> ../azure-core - # as we walk each of the dev reqs, we check for compatibility with the immovable_packages. (this happens in is_package_compatible) - # if the dev req is incompatible, we need to resolve it. THIS function is what resolves it! + # as we walk each of the dev reqs, we check for compatibility with the immovable_packages. + # (this happens in is_package_compatible) if the dev req is incompatible, we need to resolve it. + # THIS function is what resolves it! - # since we already know that package_name is incompatible with the immovable_pkgs, we need to walk backwards through the versions of package_name - # checking to ensure that each version is compatible with the immovable_pkgs. + # since we already know that package_name is incompatible with the immovable_pkgs, we need to walk backwards + # through the versions of package_name checking to ensure that each version is compatible with the immovable_pkgs. # if we find one that is, we will return a new requirement string for that package which will replace this dev_req line. for pkg in immovable_pkgs: required_package = immovable_pkgs[pkg].name @@ -733,11 +743,13 @@ def handle_incompatible_minimum_dev_reqs( ) -> List[str]: """ This function is used to handle the case where a dev requirement is incompatible with the current set of packages - being installed. This is used to update or remove dev_requirements that are incompatible with a targeted set of packages. + being installed. This is used to update or remove dev_requirements that are incompatible with a targeted set of + packages. :param str setup_path: The path to the setup.py file whose dev_requirements are being filtered. - :param List[str] filtered_requirement_list: A filtered copy of the dev_requirements.txt for the targeted package. This list will be + :param List[str] filtered_requirement_list: A filtered copy of the dev_requirements.txt for the targeted package. + This list will be modified in place to remove any requirements incompatible with the packages_for_install. :param List[Requirement] packages_for_install: A list of packages that dev_requirements MUST be compatible with. @@ -809,7 +821,7 @@ def handle_incompatible_minimum_dev_reqs( # we understand how to parse it, so we should handle it if dev_req_package: if not is_package_compatible(dev_req_package, requirements_for_dev_req, packages_for_install): - new_req = resolve_compatible_package(dev_req_package, dev_req_version, packages_for_install) + new_req = resolve_compatible_package(dev_req_package, packages_for_install) if new_req: cleansed_reqs.append(new_req) From c0406bdc3bf45435a452f20d8ae069696d43efde Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 2 Oct 2024 16:46:09 -0700 Subject: [PATCH 12/18] repair test in resolve_compatible_package --- tools/azure-sdk-tools/tests/test_conflict_resolution.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/azure-sdk-tools/tests/test_conflict_resolution.py b/tools/azure-sdk-tools/tests/test_conflict_resolution.py index ad769a2026cd..c2b2a70bfc42 100644 --- a/tools/azure-sdk-tools/tests/test_conflict_resolution.py +++ b/tools/azure-sdk-tools/tests/test_conflict_resolution.py @@ -21,7 +21,6 @@ def test_incompatible_specifier(fake_package_input_requirements, immutable_requi def test_identity_resolution(): result = resolve_compatible_package( "azure-identity", - "1.18.1", [Requirement("azure-core>=1.28.0"), Requirement("isodate>=0.6.1"), Requirement("typing-extensions>=4.0.1")], ) assert result == "azure-identity==1.16.0" From 80abb0176d42931517b28cf769a4e4e975d559ef Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:47:25 -0700 Subject: [PATCH 13/18] Update tools/azure-sdk-tools/ci_tools/functions.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com> --- tools/azure-sdk-tools/ci_tools/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 5c8e9d7eb4e8..02dee964e15c 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -650,7 +650,7 @@ def is_package_compatible( It is factored this way because we retrieve requirements differently according to the source of the package. If published, we can get requires() from PyPI If locally built wheel, we can get requires() from the metadata of the package - If local relative requirement, we can get requires() from a ParsedSetup of the setup.py for th package + If local relative requirement, we can get requires() from a ParsedSetup of the setup.py for the package :param List[Requirement] package_requirements: The dependencies of a dev_requirement file. This is the set of requirements that we are checking compatibility for. From 782311c867651e806504f54104de7f63f9eb09c9 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 2 Oct 2024 18:30:13 -0700 Subject: [PATCH 14/18] didn't rename a parametrized argument on a test to match with the definition of the parameterized inputs --- tools/azure-sdk-tools/tests/test_conflict_resolution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/tests/test_conflict_resolution.py b/tools/azure-sdk-tools/tests/test_conflict_resolution.py index c2b2a70bfc42..36c4ad3d4a6f 100644 --- a/tools/azure-sdk-tools/tests/test_conflict_resolution.py +++ b/tools/azure-sdk-tools/tests/test_conflict_resolution.py @@ -9,7 +9,7 @@ @pytest.mark.parametrize( - "input_requirements, immutable_requirements, expected_result", + "fake_package_input_requirements, immutable_requirements, expected_result", [([Requirement("sphinx==1.0.0")], [Requirement("sphinx>=1.0.0")], True), ([Requirement("sphinx==1.0.0")], [Requirement("sphinx>=1.1.0")], False)], ) From b33a965c34d4a07e265bcb19497037c04089292b Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 3 Oct 2024 10:32:59 -0700 Subject: [PATCH 15/18] fully rename function --- eng/tox/install_depend_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index 5e5a765e6c66..e7b8e174fd0b 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -104,12 +104,12 @@ def install_dependent_packages(setup_py_file_path, dependency_type, temp_dir): additional_filter_fn = None if dependency_type == "Minimum": - additionalFilterFn = handle_incompatible_minimum_dev_reqs + additional_filter_fn = handle_incompatible_minimum_dev_reqs # before september 2024, filter_dev_requirements only would remove any packages present in released_packages from the dev_requirements, # then create a new file "new_dev_requirements.txt" without the problematic packages. # after september 2024, filter_dev_requirements will also check for **compatibility** with the packages being installed when filtering the dev_requirements. - dev_req_file_path = filter_dev_requirements(setup_py_file_path, released_packages, temp_dir, additionalFilterFn) + dev_req_file_path = filter_dev_requirements(setup_py_file_path, released_packages, temp_dir, additional_filter_fn) if override_added_packages: logging.info(f"Expanding the requirement set by the packages {override_added_packages}.") From 746036d8e84a81031191dec7446da8cf3ff129a7 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 3 Oct 2024 10:37:43 -0700 Subject: [PATCH 16/18] handle when there is no specifier in the dev req --- tools/azure-sdk-tools/ci_tools/functions.py | 11 +++++++++-- .../azure-sdk-tools/tests/test_conflict_resolution.py | 8 ++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 02dee964e15c..c02929bfe743 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -664,7 +664,11 @@ def is_package_compatible( # if the dev_req line has a requirement that conflicts with the immutable requirement, # we need to resolve it. We KNOW that the immutable requirement will be of form package==version, # so we can reliably pull out the version and check it against the specifier of the dev_req line. - immutable_version = next(iter(immutable_requirement.specifier)).version + try: + immutable_version = next(iter(immutable_requirement.specifier)).version + # we have no specifier set, so we don't even need to check this + except StopIteration: + continue if not package_requirement.specifier.contains(immutable_version): if should_log: @@ -711,7 +715,10 @@ def resolve_compatible_package(package_name: str, immutable_requirements: List[R # if we find one that is, we will return a new requirement string for that package which will replace this dev_req line. for pkg in immovable_pkgs: required_package = immovable_pkgs[pkg].name - required_pkg_version = next(iter(immovable_pkgs[pkg].specifier)).version + try: + required_pkg_version = next(iter(immovable_pkgs[pkg].specifier)).version + except StopIteration: + required_pkg_version = None versions = pypi.get_ordered_versions(package_name, True) versions.reverse() diff --git a/tools/azure-sdk-tools/tests/test_conflict_resolution.py b/tools/azure-sdk-tools/tests/test_conflict_resolution.py index 36c4ad3d4a6f..9884f338d896 100644 --- a/tools/azure-sdk-tools/tests/test_conflict_resolution.py +++ b/tools/azure-sdk-tools/tests/test_conflict_resolution.py @@ -24,3 +24,11 @@ def test_identity_resolution(): [Requirement("azure-core>=1.28.0"), Requirement("isodate>=0.6.1"), Requirement("typing-extensions>=4.0.1")], ) assert result == "azure-identity==1.16.0" + + +def test_resolution_no_requirement(): + result = resolve_compatible_package( + "azure-identity", + [Requirement("azure-core")], + ) + assert result == "azure-identity==1.18.0" From 2e41797f84e7c9c9c190c0eda18e1ef69ad0cc52 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:27:41 -0700 Subject: [PATCH 17/18] Update eng/tox/install_depend_packages.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com> --- eng/tox/install_depend_packages.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index e7b8e174fd0b..ffb64279c2db 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -308,7 +308,12 @@ def check_req_against_exclusion(req, req_to_exclude): return req_id == req_to_exclude -def filter_dev_requirements(setup_py_path, released_packages, temp_dir, additionalFilterFn: Optional[Callable[[str, List[str],List[Requirement]], List[str]]] = None): +def filter_dev_requirements( + setup_py_path, + released_packages, + temp_dir, + additional_filter_fn: Optional[Callable[[str, List[str],List[Requirement]], List[str]]] = None, +): """ This function takes an existing package path, a list of specific package specifiers that we have resolved, a temporary directory to write the modified dev_requirements to, and an optional additionalFilterFn that can be used to further filter the dev_requirements file if necessary. From 3f52835e4742e82f1c867722e27ed40fd6523ce7 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Thu, 3 Oct 2024 15:28:53 -0700 Subject: [PATCH 18/18] additionalFilterFn -> additional_filter_fn --- eng/tox/install_depend_packages.py | 31 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index e7b8e174fd0b..8b5c946e3a8b 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -59,7 +59,7 @@ "azure-eventhub-checkpointstoretable": {"azure-core": "1.25.0", "azure-eventhub": "5.11.0"}, "azure-identity": {"msal": "1.23.0"}, "azure-core-tracing-opentelemetry": {"azure-core": "1.28.0"}, - "azure-storage-file-datalake": {"azure-storage-blob": "12.22.0"} + "azure-storage-file-datalake": {"azure-storage-blob": "12.22.0"}, } MAXIMUM_VERSION_SPECIFIC_OVERRIDES = {} @@ -67,12 +67,7 @@ # PLATFORM SPECIFIC OVERRIDES provide additional generic (EG not tied to the package whos dependencies are being processed) # filtering on a _per platform_ basis. Primarily used to limit certain packages due to platform compatbility PLATFORM_SPECIFIC_MINIMUM_OVERRIDES = { - ">=3.12.0": { - "azure-core": "1.23.1", - "aiohttp": "3.8.6", - "six": "1.16.0", - "requests": "2.30.0" - } + ">=3.12.0": {"azure-core": "1.23.1", "aiohttp": "3.8.6", "six": "1.16.0", "requests": "2.30.0"} } PLATFORM_SPECIFIC_MAXIMUM_OVERRIDES = {} @@ -164,6 +159,7 @@ def find_released_packages(setup_py_path, dependency_type): return avlble_packages + def process_bounded_versions(originating_pkg_name: str, pkg_name: str, versions: List[str]) -> List[str]: """ Processes a target package based on an originating package (target is a dep of originating) and the versions available from pypi for the target package. @@ -187,9 +183,7 @@ def process_bounded_versions(originating_pkg_name: str, pkg_name: str, versions: restrictions = PLATFORM_SPECIFIC_MINIMUM_OVERRIDES[platform_bound] if pkg_name in restrictions: - versions = [ - v for v in versions if parse_version(v) >= parse_version(restrictions[pkg_name]) - ] + versions = [v for v in versions if parse_version(v) >= parse_version(restrictions[pkg_name])] # lower bound package-specific if ( @@ -214,9 +208,7 @@ def process_bounded_versions(originating_pkg_name: str, pkg_name: str, versions: restrictions = PLATFORM_SPECIFIC_MAXIMUM_OVERRIDES[platform_bound] if pkg_name in restrictions: - versions = [ - v for v in versions if parse_version(v) <= parse_version(restrictions[pkg_name]) - ] + versions = [v for v in versions if parse_version(v) <= parse_version(restrictions[pkg_name])] # upper bound package-specific if ( @@ -308,10 +300,15 @@ def check_req_against_exclusion(req, req_to_exclude): return req_id == req_to_exclude -def filter_dev_requirements(setup_py_path, released_packages, temp_dir, additionalFilterFn: Optional[Callable[[str, List[str],List[Requirement]], List[str]]] = None): +def filter_dev_requirements( + setup_py_path, + released_packages, + temp_dir, + additional_filter_fn: Optional[Callable[[str, List[str], List[Requirement]], List[str]]] = None, +): """ This function takes an existing package path, a list of specific package specifiers that we have resolved, a temporary directory to write - the modified dev_requirements to, and an optional additionalFilterFn that can be used to further filter the dev_requirements file if necessary. + the modified dev_requirements to, and an optional additional_filter_fn that can be used to further filter the dev_requirements file if necessary. The function will filter out any requirements present in the dev_requirements file that are present in the released_packages list (aka are required by the package). @@ -339,9 +336,9 @@ def filter_dev_requirements(setup_py_path, released_packages, temp_dir, addition and not any([check_req_against_exclusion(req, i) for i in req_to_exclude]) ] - if additionalFilterFn: + if additional_filter_fn: # this filter function handles the case where a dev requirement is incompatible with the current set of targeted packages - filtered_req = additionalFilterFn(setup_py_path, filtered_req, released_packages) + filtered_req = additional_filter_fn(setup_py_path, filtered_req, released_packages) logging.info("Filtered dev requirements: %s", filtered_req)