diff --git a/conda/conda-recipes/azure-mgmt/meta.yaml b/conda/conda-recipes/azure-mgmt/meta.yaml index 6b6a9d67e940..191506a42734 100644 --- a/conda/conda-recipes/azure-mgmt/meta.yaml +++ b/conda/conda-recipes/azure-mgmt/meta.yaml @@ -73,7 +73,7 @@ test: - azure.mgmt.applicationinsights.v2022_06_15.aio.operations - azure.mgmt.applicationinsights.v2022_06_15.models - azure.mgmt.applicationinsights.v2022_06_15.operations - - azure-mgmt-arizeaiobservabilityeval + - azure.mgmt.arizeaiobservabilityeval - azure.mgmt.arizeaiobservabilityeval.aio - azure.mgmt.arizeaiobservabilityeval.aio.operations - azure.mgmt.arizeaiobservabilityeval.models @@ -150,7 +150,7 @@ test: - azure.mgmt.botservice.aio.operations - azure.mgmt.botservice.models - azure.mgmt.botservice.operations - - azure-mgmt-carbonoptimization + - azure.mgmt.carbonoptimization - azure.mgmt.carbonoptimization.aio - azure.mgmt.carbonoptimization.aio.operations - azure.mgmt.carbonoptimization.models @@ -260,11 +260,6 @@ test: - azure.mgmt.dashboard.operations - azure.mgmt.databox - azure.mgmt.databox.aio - - azure.mgmt.datab - - azure.mgmt.datab.aio - - azure.mgmt.datab.aio.operations - - azure.mgmt.datab.models - - azure.mgmt.datab.operations - azure.mgmt.databoxedge - azure.mgmt.databoxedge.aio - azure.mgmt.databoxedge.aio @@ -419,7 +414,7 @@ test: - azure.mgmt.hanaonazure.aio.operations - azure.mgmt.hanaonazure.models - azure.mgmt.hanaonazure.operations - - azure-mgmt-hardwaresecuritymodules + - azure.mgmt.hardwaresecuritymodules - azure.mgmt.hardwaresecuritymodules.aio - azure.mgmt.hardwaresecuritymodules.aio.operations - azure.mgmt.hardwaresecuritymodules.models @@ -515,7 +510,7 @@ test: - azure.mgmt.labservices.aio.operations - azure.mgmt.labservices.models - azure.mgmt.labservices.operations - - azure-mgmt-lambdatesthyperexecute + - azure.mgmt.lambdatesthyperexecute - azure.mgmt.lambdatesthyperexecute.aio - azure.mgmt.lambdatesthyperexecute.aio.operations - azure.mgmt.lambdatesthyperexecute.models @@ -610,7 +605,7 @@ test: - azure.mgmt.mongocluster.aio.operations - azure.mgmt.mongocluster.models - azure.mgmt.mongocluster.operations - - azure-mgmt-mongodbatlas + - azure.mgmt.mongodbatlas - azure.mgmt.mongodbatlas.aio - azure.mgmt.mongodbatlas.aio.operations - azure.mgmt.mongodbatlas.models @@ -730,7 +725,7 @@ test: - azure.mgmt.privatedns.aio.operations - azure.mgmt.privatedns.models - azure.mgmt.privatedns.operations - - azure-mgmt-purestorageblock + - azure.mgmt.purestorageblock - azure.mgmt.purestorageblock.aio - azure.mgmt.purestorageblock.aio.operations - azure.mgmt.purestorageblock.models @@ -791,7 +786,7 @@ test: - azure.mgmt.recoveryservicesbackup.passivestamp.aio.operations - azure.mgmt.recoveryservicesbackup.passivestamp.models - azure.mgmt.recoveryservicesbackup.passivestamp.operations - - azure-mgmt-recoveryservicesdatareplication + - azure.mgmt.recoveryservicesdatareplication - azure.mgmt.recoveryservicesdatareplication.aio - azure.mgmt.recoveryservicesdatareplication.aio.operations - azure.mgmt.recoveryservicesdatareplication.models @@ -977,7 +972,7 @@ test: - azure.mgmt.storage.aio.operations - azure.mgmt.storage.models - azure.mgmt.storage.operations - - azure-mgmt-storageactions + - azure.mgmt.storageactions - azure.mgmt.storageactions.aio - azure.mgmt.storageactions.aio.operations - azure.mgmt.storageactions.models diff --git a/conda/conda_helper_functions.py b/conda/conda_helper_functions.py new file mode 100644 index 000000000000..2b9e0b2d8e35 --- /dev/null +++ b/conda/conda_helper_functions.py @@ -0,0 +1,344 @@ +""" +Helper functions for updating conda files. +""" + +import os +import glob +from functools import lru_cache +from typing import Optional +import csv +from ci_tools.logging import logger +import urllib.request +from datetime import datetime +from ci_tools.parsing import ParsedSetup +from packaging.version import Version +from pypi_tools.pypi import PyPIClient, retrieve_versions_from_pypi + + +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +SDK_DIR = os.path.join(ROOT_DIR, "sdk") + +AZURE_SDK_CSV_URL = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/python-packages.csv" +PACKAGE_COL = "Package" +LATEST_GA_DATE_COL = "LatestGADate" +VERSION_GA_COL = "VersionGA" +FIRST_GA_DATE_COL = "FirstGADate" +DISPLAY_NAME_COL = "DisplayName" +SERVICE_NAME_COL = "ServiceName" +REPO_PATH_COL = "RepoPath" +TYPE_COL = "Type" +SUPPORT_COL = "Support" + +# ===================================== +# Helpers for handling bundled releases +# ===================================== + + +@lru_cache(maxsize=None) +def _build_package_path_index() -> dict[str, str]: + """ + Build a one-time index mapping package names to their filesystem paths. + + This scans the sdk/ directory once and caches the result for all subsequent lookups. + """ + all_paths = glob.glob(os.path.join(SDK_DIR, "*", "*")) + # Exclude temp directories like .tox, .venv, __pycache__, etc. + return { + os.path.basename(p): p + for p in all_paths + if os.path.isdir(p) and not os.path.basename(p).startswith((".", "__")) + } + + +def get_package_path(package_name: str) -> Optional[str]: + """Get the filesystem path of an SDK package given its name.""" + path_index = _build_package_path_index() + package_path = path_index.get(package_name) + if not package_path: + logger.warning(f"Package path not found for package: {package_name}") + return None + return package_path + + +def get_bundle_name(package_name: str) -> Optional[str]: + """ + Check bundled release config from package's pyproject.toml file given the package name. + + If bundled, return the bundle name; otherwise, return None. + """ + package_path = get_package_path(package_name) + if not package_path: + logger.warning(f"Cannot determine package path for {package_name}") + return None + parsed = ParsedSetup.from_path(package_path) + if not parsed: + # can't proceed, need to know if it's bundled or not + logger.error(f"Failed to parse setup for package {package_name}") + raise Exception(f"Failed to parse setup for package {package_name}") + + conda_config = parsed.get_conda_config() + + if not conda_config: + if is_stable_on_pypi(package_name): + raise Exception( + f"Stable release package {package_name} needs a conda config" + ) + + logger.warning( + f"No conda config found for package {package_name}, which may be a pre-release" + ) + return None + + if conda_config and "bundle_name" in conda_config: + return conda_config["bundle_name"] + + return None + + +def map_bundle_to_packages( + package_names: list[str], +) -> tuple[dict[str, list[str]], list[str]]: + """Create a mapping of bundle names to their constituent package names. + + :return: Tuple of (bundle_map, failed_packages) where failed_packages are packages that threw exceptions. + """ + logger.info("Mapping bundle names to packages...") + + bundle_map = {} + failed_packages = [] + for package_name in package_names: + logger.debug(f"Processing package for bundle mapping: {package_name}") + try: + bundle_name = get_bundle_name(package_name) + if bundle_name: + logger.debug(f"Bundle name for package {package_name}: {bundle_name}") + bundle_map.setdefault(bundle_name, []).append(package_name) + except Exception as e: + logger.error(f"Failed to get bundle name for {package_name}: {e}") + failed_packages.append(package_name) + continue + + return bundle_map, failed_packages + + +# ===================================== +# Utility functions for parsing data +# ===================================== + + +def parse_csv() -> list[dict[str, str]]: + """Download and parse the Azure SDK Python packages CSV file.""" + try: + logger.info(f"Downloading CSV from {AZURE_SDK_CSV_URL}") + + with urllib.request.urlopen(AZURE_SDK_CSV_URL, timeout=10) as response: + csv_content = response.read().decode("utf-8") + + # Parse the CSV content + csv_reader = csv.DictReader(csv_content.splitlines()) + packages = list(csv_reader) + + logger.info(f"Successfully parsed {len(packages)} packages from CSV") + + return packages + + except Exception as e: + logger.error(f"Failed to download or parse CSV: {e}") + return [] + + +def is_mgmt_package(pkg: dict[str, str]) -> bool: + pkg_name = pkg.get(PACKAGE_COL, "") + _type = pkg.get(TYPE_COL, "") + if _type == "mgmt": + return True + elif _type == "client": + return False + else: + return pkg_name != "azure-mgmt-core" and ( + "mgmt" in pkg_name or "cognitiveservices" in pkg_name + ) + + +def separate_packages_by_type( + packages: list[dict[str, str]], +) -> tuple[list[dict[str, str]], list[dict[str, str]]]: + """Separate packages into data plane and management plane libraries.""" + data_plane_packages = [] + mgmt_plane_packages = [] + + for pkg in packages: + if is_mgmt_package(pkg): + mgmt_plane_packages.append(pkg) + else: + data_plane_packages.append(pkg) + + logger.debug( + f"Separated {len(data_plane_packages)} data plane and {len(mgmt_plane_packages)} management plane packages" + ) + + return (data_plane_packages, mgmt_plane_packages) + + +def package_needs_update( + package_row: dict[str, str], prev_release_date: str, is_new=False +) -> bool: + """ + Check if the package is new or needs version update (i.e., FirstGADate or LatestGADate is after the last release). + + :param package_row: The parsed CSV row for the package. + :param prev_release_date: The date of the previous release in "mm/dd/yyyy" format. + :param is_new: Whether to check for new package (FirstGADate) or outdated package (LatestGADate). + :return: if the package is new or needs an update. + """ + compare_date = ( + package_row.get(FIRST_GA_DATE_COL) + if is_new + else package_row.get(LATEST_GA_DATE_COL) + ) + + logger.debug( + f"Checking {'new package' if is_new else 'outdated package'} for package {package_row.get(PACKAGE_COL)} with against date: {compare_date}" + ) + + if not compare_date: + logger.debug( + f"Package {package_row.get(PACKAGE_COL)} is skipped due to missing {FIRST_GA_DATE_COL if is_new else LATEST_GA_DATE_COL}." + ) + + return False + + try: + # Convert string dates to datetime objects for proper comparison + compare_date = datetime.strptime(compare_date, "%m/%d/%Y") + prev_date = datetime.strptime(prev_release_date, "%m/%d/%Y") + logger.debug( + f"Comparing {package_row.get(PACKAGE_COL)} CompareDate {compare_date} with previous release date {prev_date}" + ) + return compare_date > prev_date + except ValueError as e: + logger.error( + f"Date parsing error for package {package_row.get(PACKAGE_COL)}: {e}" + ) + return False + + +def is_stable_on_pypi(package_name: str) -> bool: + """ + Check if a package has any stable (GA) release on PyPI. + + :param package_name: The name of the package to check. + :return: True if any stable version exists on PyPI, False otherwise. + """ + try: + versions = retrieve_versions_from_pypi(package_name) + if not versions: + logger.warning(f"No versions found on PyPI for {package_name}") + return False + + # Check if any version is stable (not a prerelease) + for v in versions: + if not Version(v).is_prerelease: + logger.debug(f"Package {package_name} has stable version {v}") + return True + + logger.debug(f"Package {package_name} has no stable versions") + return False + + except Exception as e: + logger.warning(f"Failed to check PyPI for {package_name}: {e}") + return False + + +def get_package_data_from_pypi( + package_name: str, +) -> tuple[Optional[str], Optional[str]]: + """Fetch the latest version and download URI for a package from PyPI.""" + try: + client = PyPIClient() + data = client.project(package_name) + + # Get the latest version + latest_version = data["info"]["version"] + if latest_version in data["releases"] and data["releases"][latest_version]: + # Get the source distribution (sdist) if available + files = data["releases"][latest_version] + source_dist = next((f for f in files if f["packagetype"] == "sdist"), None) + if source_dist: + download_url = source_dist["url"] + logger.info( + f"Found download URL for {package_name}=={latest_version}: {download_url}" + ) + return latest_version, download_url + + except Exception as e: + logger.error(f"Failed to fetch download URI from PyPI for {package_name}: {e}") + return None, None + + +def build_package_index(conda_artifacts: list[dict]) -> dict[str, tuple[int, int]]: + """Build an index of package name -> (artifact_idx, checkout_idx) for fast lookups in conda-sdk-client.yml.""" + package_index = {} + + for artifact_idx, artifact in enumerate(conda_artifacts): + if "checkout" in artifact: + for checkout_idx, checkout_item in enumerate(artifact["checkout"]): + package_name = checkout_item.get("package") + if package_name: + package_index[package_name] = (artifact_idx, checkout_idx) + return package_index + + +def get_valid_package_imports(package_name: str) -> list[str]: + """ + Inspect the package's actual module structure and return only valid imports. + + :param package_name: The name of the package (e.g., "azure-mgmt-advisor" or "azure-eventgrid"). + :return: List of valid module names for import (e.g., ["azure.eventgrid", "azure.eventgrid.aio"]). + """ + package_path = get_package_path(package_name) + if not package_path: + logger.warning( + f"Could not find package path for {package_name} to determine imports, using fallback" + ) + return [package_name.replace("-", ".")] + else: + parsed = ParsedSetup.from_path(package_path) + if not parsed or not parsed.namespace: + logger.warning( + f"Could not parse namespace for {package_name}, using fallback" + ) + module_name = package_name.replace("-", ".") + else: + module_name = parsed.namespace + + imports = [module_name] + + # Construct the path to the actual module directory + module_parts = module_name.split(".") + module_dir = os.path.join(package_path, *module_parts) + + if not os.path.isdir(module_dir): + logger.warning( + f"Module directory not found for {package_name} at {module_dir}, using base import only" + ) + return imports + + # Check for common submodules and only add if they exist + submodules_to_check = ["aio", "models", "operations"] + + for submodule_name in submodules_to_check: + submodule_path = os.path.join(module_dir, submodule_name) + if os.path.isdir(submodule_path) and os.path.exists( + os.path.join(submodule_path, "__init__.py") + ): + imports.append(f"{module_name}.{submodule_name}") + + # Check for aio.operations (nested submodule) + aio_operations_path = os.path.join(module_dir, "aio", "operations") + if os.path.isdir(aio_operations_path) and os.path.exists( + os.path.join(aio_operations_path, "__init__.py") + ): + imports.append(f"{module_name}.aio.operations") + + return imports diff --git a/conda/update_conda_files.py b/conda/update_conda_files.py new file mode 100644 index 000000000000..417f45bcbf0b --- /dev/null +++ b/conda/update_conda_files.py @@ -0,0 +1,1134 @@ +"""Update package versions, yml files, release-logs, and changelogs for conda packages.""" + +import os +import argparse +import yaml +import re +import glob +from datetime import datetime +from dateutil.relativedelta import relativedelta +from ci_tools.logging import logger, configure_logging +from ci_tools.parsing import ParsedSetup +from typing import Optional +from conda_helper_functions import ( + parse_csv, + separate_packages_by_type, + package_needs_update, + get_package_data_from_pypi, + build_package_index, + get_package_path, + get_bundle_name, + map_bundle_to_packages, + get_valid_package_imports, + PACKAGE_COL, + VERSION_GA_COL, + LATEST_GA_DATE_COL, + REPO_PATH_COL, + SUPPORT_COL, +) + +# paths +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +SDK_DIR = os.path.join(ROOT_DIR, "sdk") +CONDA_DIR = os.path.join(ROOT_DIR, "conda") +CONDA_RECIPES_DIR = os.path.join(CONDA_DIR, "conda-recipes") +CONDA_RELEASE_LOGS_DIR = os.path.join(CONDA_DIR, "conda-releaselogs") +CONDA_ENV_PATH = os.path.join(CONDA_RECIPES_DIR, "conda_env.yml") +CONDA_CLIENT_YAML_PATH = os.path.join( + ROOT_DIR, "eng", "pipelines", "templates", "stages", "conda-sdk-client.yml" +) +CONDA_MGMT_META_YAML_PATH = os.path.join(CONDA_RECIPES_DIR, "azure-mgmt", "meta.yaml") + +# constants +RELEASE_PERIOD_MONTHS = 3 + +# packages that should be shipped but are known to be missing from the csv - store version here +PACKAGES_WITH_DOWNLOAD_URI = { + "msal": "", + "msal-extensions": "", +} + + +# ===================================== +# Helpers for updating conda_env.yml +# ===================================== + + +class quoted(str): + pass + + +def quoted_presenter(dumper, data): + """YAML presenter to force quotes around a string.""" + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="'") + + +def update_conda_version( + target_release_date: Optional[datetime] = None, +) -> tuple[datetime, str]: + """Update the AZURESDK_CONDA_VERSION in conda_env.yml and return the old and new versions. + + Args: + target_release_date: Optional specific release date to use. If None, calculates + the next release date by adding RELEASE_PERIOD_MONTHS to the old version. + """ + + with open(CONDA_ENV_PATH, "r") as file: + conda_env_data = yaml.safe_load(file) + + old_version = conda_env_data["variables"]["AZURESDK_CONDA_VERSION"] + old_date = datetime.strptime(old_version, "%Y.%m.%d") + + if target_release_date: + new_date = target_release_date + else: + new_date = old_date + relativedelta(months=RELEASE_PERIOD_MONTHS) + + # bump version + new_version = new_date.strftime("%Y.%m.%d") + conda_env_data["variables"]["AZURESDK_CONDA_VERSION"] = quoted(new_version) + + yaml.add_representer(quoted, quoted_presenter) + + with open(CONDA_ENV_PATH, "w") as file: + yaml.dump(conda_env_data, file, default_flow_style=False, sort_keys=False) + + logger.info(f"Updated AZURESDK_CONDA_VERSION from {old_version} to {new_version}") + + return old_date, new_version + + +# ===================================== +# Helpers for updating conda-sdk-client.yml +# ===================================== + + +class IndentDumper(yaml.SafeDumper): + """Used to preserve indentation levels in conda-sdk-client.yml.""" + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + + +def update_conda_sdk_client_yml( + package_dict: dict[str, dict[str, str]], + packages_to_update: list[str], + new_data_plane_packages: list[str], + new_mgmt_plane_packages: list[str], +) -> list[str]: + """ + Update outdated package versions and add new entries in conda-sdk-client.yml file + + :param package_dict: Dictionary mapping package names to their CSV row data. + :param packages_to_update: List of package names that need version updates. + :param new_data_plane_packages: List of new data plane package names. + :param new_mgmt_plane_packages: List of new management plane package names. + :return: List of package names that were not updated or added and may require manual action. + """ + updated_count = 0 + added_count = 0 + result = [] + + with open(CONDA_CLIENT_YAML_PATH, "r") as file: + conda_client_data = yaml.safe_load(file) + + conda_artifacts = conda_client_data["extends"]["parameters"]["stages"][0]["jobs"][ + 0 + ]["steps"][0]["parameters"]["CondaArtifacts"] + + # === Update outdated package versions === + + logger.info( + f"Detected {len(packages_to_update)} outdated package versions to update in conda-sdk-client.yml" + ) + package_index = build_package_index(conda_artifacts) + + for pkg_name in packages_to_update: + pkg = package_dict.get(pkg_name, {}) + new_version = pkg.get(VERSION_GA_COL) + if pkg_name in package_index: + artifact_idx, checkout_idx = package_index[pkg_name] + checkout_item = conda_artifacts[artifact_idx]["checkout"][checkout_idx] + + if "version" in checkout_item: + old_version = checkout_item.get("version", "") + checkout_item["version"] = new_version + logger.info(f"Updated {pkg_name}: {old_version} -> {new_version}") + updated_count += 1 + else: + logger.warning( + f"Package {pkg_name} has no 'version' field, skipping update" + ) + result.append(pkg_name) + else: + logger.warning( + f"Package {pkg_name} not found in conda-sdk-client.yml, skipping update" + ) + result.append(pkg_name) + + # handle download_uri for packages known to be missing from the csv + for pkg_name in PACKAGES_WITH_DOWNLOAD_URI: + if pkg_name in package_index: + artifact_idx, checkout_idx = package_index[pkg_name] + checkout_item = conda_artifacts[artifact_idx]["checkout"][checkout_idx] + + curr_download_uri = checkout_item.get("download_uri", "") + latest_version, download_uri = get_package_data_from_pypi(pkg_name) + + if not latest_version or not download_uri: + logger.warning( + f"Could not retrieve latest version or download URI for {pkg_name} from PyPI, skipping" + ) + result.append(pkg_name) + continue + + # store retrieved version for release log + PACKAGES_WITH_DOWNLOAD_URI[pkg_name] = latest_version + + if curr_download_uri != download_uri: + # version needs update + logger.info( + f"Package {pkg_name} download_uri mismatch with PyPi, updating {curr_download_uri} to {download_uri}" + ) + # checkout for these packages only has download_uri, no version field + checkout_item["download_uri"] = download_uri + logger.info( + f"Updated download_uri for {pkg_name} with version {latest_version}: {download_uri}" + ) + updated_count += 1 + else: + logger.warning( + f"Package {pkg_name} not found in conda-sdk-client.yml, skipping download_uri update" + ) + result.append(pkg_name) + + # === Add new data plane packages === + + logger.info( + f"Detected {len(new_data_plane_packages)} new data plane packages to add to conda-sdk-client.yml" + ) + + parameters = conda_client_data["parameters"] + + # quick look up for handling bundled package releases + existing_parameter_names = [p.get("name") for p in parameters] + existing_artifact_names = { + a.get("name"): idx for idx, a in enumerate(conda_artifacts) + } + + for package_name in new_data_plane_packages: + pkg = package_dict.get(package_name, {}) + + if package_name in package_index: + logger.warning( + f"New package {package_name} already exists in conda-sdk-client.yml, skipping addition" + ) + result.append(package_name) + continue + + # bundle info is based on pyproject.toml + bundle_name = get_bundle_name(package_name) + + if bundle_name: + # package is part of a bundle + logger.info( + f"Package {package_name} belongs to release bundle {bundle_name}" + ) + release_name = f"release_{bundle_name.replace('-', '_')}" + display_name = bundle_name + else: + # package is released individually + release_name = f"release_{package_name.replace('-', '_')}" + display_name = package_name + + # add new release parameter if not exists + if release_name not in existing_parameter_names: + logger.info(f"Adding new release parameter: {release_name}") + new_parameter = { + "name": release_name, + "displayName": display_name, + "type": "boolean", + "default": True, + } + parameters.append(new_parameter) + existing_parameter_names.append(release_name) + + # add to CondaArtifacts + curr_version = pkg.get(VERSION_GA_COL) + + if not curr_version: + logger.error( + f"Package {package_name} is missing version info, skipping addition" + ) + result.append(package_name) + continue + + checkout_package = {"package": package_name, "version": curr_version} + common_root, service_name = determine_service_info(pkg, bundle_name) + + if package_name in existing_artifact_names: + # individual released package already exists + logger.warning( + f"New package {package_name} already exists in conda-sdk-client.yml, skipping addition" + ) + result.append(package_name) + continue + + if bundle_name and bundle_name in existing_artifact_names: + # bundle already exists, will append packages to it + logger.info( + f"Release bundle {bundle_name} already exists in conda-sdk-client.yml, will append package {package_name} to it" + ) + conda_artifacts[existing_artifact_names[bundle_name]]["checkout"].append( + checkout_package + ) + else: + # no existing artifact, whether bundle or not -> create + new_artifact_entry = { + "name": bundle_name if bundle_name else package_name, + "common_root": common_root, + "service": service_name, + "in_batch": f"${{{{ parameters.{release_name} }}}}", + "checkout": [checkout_package], + } + # append before azure-mgmt entry + conda_artifacts.insert(len(conda_artifacts) - 1, new_artifact_entry) + + added_count += 1 + logger.info(f"Added new data plane package: {package_name}") + + existing_artifact_names[bundle_name if bundle_name else package_name] = ( + len(conda_artifacts) - 2 + ) # new index + + # === Add new mgmt plane packages === + + logger.info( + f"Detected {len(new_mgmt_plane_packages)} new management plane packages to add to conda-sdk-client.yml" + ) + + # assumes azure-mgmt will always be the last CondaArtifacts entry + azure_mgmt_artifact_checkout = conda_artifacts[-1]["checkout"] + + for package_name in new_mgmt_plane_packages: + pkg = package_dict.get(package_name, {}) + + if package_name in package_index: + logger.warning( + f"New package {package_name} already exists in conda-sdk-client.yml, skipping addition" + ) + result.append(package_name) + continue + + new_mgmt_entry = { + "package": package_name, + "version": pkg.get(VERSION_GA_COL), + } + + azure_mgmt_artifact_checkout.append(new_mgmt_entry) + + added_count += 1 + logger.info(f"Added new management plane package: {package_name}") + + # sort mgmt packages alphabetically + azure_mgmt_artifact_checkout.sort(key=lambda x: x["package"]) + + if updated_count > 0 or added_count > 0: + with open(CONDA_CLIENT_YAML_PATH, "w") as file: + yaml.dump( + conda_client_data, + file, + Dumper=IndentDumper, + default_flow_style=False, + sort_keys=False, + indent=2, + width=float("inf"), + ) + logger.info( + f"Successfully updated {updated_count} package versions in conda-sdk-client.yml" + ) + else: + logger.warning("No packages were found in the YAML file to update") + return result + + +# ===================================== +# Helpers for creating conda-recipes//meta.yaml files +# ===================================== + + +def determine_service_info( + pkg: dict[str, str], bundle_name: Optional[str] +) -> tuple[str, str]: + """ + Returns the common root and service name for the given package. + + :param package_name: The name of the package (e.g., "azure-ai-textanalytics"). + :param bundle_name: The name of the bundle/release group the package belongs to, if any. + """ + # defaults + package_name = pkg.get(PACKAGE_COL, "") + service_name = pkg.get(REPO_PATH_COL, "").lower() + common_root = "azure" + + package_path = get_package_path(package_name) + if not service_name and package_path: + service_name = os.path.basename(os.path.dirname(package_path)) + + if bundle_name and service_name: + common_root = f"azure/{service_name}" + + return common_root, service_name + + +def format_requirement(req: str) -> str: + """Format a requirement string for conda meta.yaml.""" + name_unpinned = re.split(r"[>=={{{{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }}}}" + + # translate compatible release (~=) to >= for yml + req = req.replace("~=", ">=") + return req + + +def get_package_requirements(parsed: ParsedSetup) -> tuple[list[str], list[str]]: + """Retrieve the host and run requirements for a data plane package meta.yaml.""" + host_requirements = set(["pip", "python"]) + run_requirements = set(["python"]) + + # reqs commonly seen in existing meta.yaml files that aren't always in setup.py or pyproject.toml + for essential_req in ["azure-core", "aiohttp"]: + req_name = format_requirement(essential_req) + host_requirements.add(req_name) + run_requirements.add(req_name) + + package_path = get_package_path(parsed.name) + if not package_path: + logger.error(f"Could not find package path for {parsed.name}") + return list(host_requirements), list(run_requirements) + + # get requirements from setup.py or pyproject.toml + install_reqs = parsed.requires + + for req in install_reqs: + req_name = format_requirement(req) + host_requirements.add(req_name) + run_requirements.add(req_name) + + return list(host_requirements), list(run_requirements) + + +def get_package_metadata( + package_name: str, package_path: str, is_bundle: bool = False +) -> tuple[str, str, str]: + """Extract package metadata for about section in meta.yaml. + + :param package_name: The name of the package or bundle. + :param package_path: The filesystem path to the package. + :param is_bundle: Whether this is a release bundle (affects URL structure). + """ + service_dir = os.path.basename(os.path.dirname(package_path)) + + # For bundles, URL points to service directory; for individual packages, include package name + if is_bundle: + home_url = ( + f"https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/{service_dir}" + ) + else: + home_url = f"https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/{service_dir}/{package_name}" + + summary = f"Microsoft Azure {service_dir.replace('-', ' ').title()} Client Library for Python" + + conda_url = f"https://aka.ms/azsdk/conda/releases/{service_dir}" + description = ( + f"This is the {summary}.\n Please see {conda_url} for version details." + ) + + return home_url, summary, description + + +def generate_data_plane_meta_yaml( + bundle_map: dict[str, list[str]], + package_name: str, + bundle_name: Optional[str], +) -> str: + """ + Generate the meta.yaml content for a data plane package or release group. + """ + # Use bundle_name if available for recipe name and env var derivation + # however, env var name is arbitrary and replaced in conda_functions.py + recipe_name = bundle_name if bundle_name else package_name + src_distr_name = recipe_name.split("-")[-1].upper() + src_distribution_env_var = f"{src_distr_name}_SOURCE_DISTRIBUTION" + + if bundle_name: + # handle grouped packages + logger.info( + f"Generating meta.yaml for release group {bundle_name} including packages: {bundle_map[bundle_name]}" + ) + host_reqs = set() + run_reqs = set() + pkg_imports = [] + + for pkg in bundle_map[bundle_name]: + package_path = get_package_path(pkg) + parsed_setup = ParsedSetup.from_path(package_path) + + pkg_host_reqs, pkg_run_reqs = get_package_requirements(parsed_setup) + host_reqs.update(pkg_host_reqs) + run_reqs.update(pkg_run_reqs) + + # Get valid imports for this package (including .aio if it exists) + pkg_imports.extend(get_valid_package_imports(pkg)) + host_reqs = list(host_reqs) + run_reqs = list(run_reqs) + pkg_imports = list(set(pkg_imports)) # deduplicate + + package_path = get_package_path(bundle_map[bundle_name][0]) + + if not package_path: + logger.error( + f"Could not find package path for {bundle_name} to extract metadata, skipping meta.yaml generation" + ) + return "" + + home_url, summary, description = get_package_metadata( + bundle_name, package_path, is_bundle=True + ) + else: + logger.info(f"Generating meta.yaml for package {package_name}") + package_path = get_package_path(package_name) + + if not package_path: + logger.error( + f"Could not find package path for {package_name} to extract metadata, skipping meta.yaml generation" + ) + return "" + parsed_setup = ParsedSetup.from_path(package_path) + + host_reqs, run_reqs = get_package_requirements(parsed_setup) + # Get valid imports for this package (including .aio if it exists) + pkg_imports = get_valid_package_imports(package_name) + + home_url, summary, description = get_package_metadata( + package_name, package_path + ) + + # Format requirements with proper YAML indentation + host_reqs_str = "\n - ".join(host_reqs) + run_reqs_str = "\n - ".join(run_reqs) + pkg_imports_str = "\n - ".join(pkg_imports) + meta_yaml_content = f"""{{% set name = "{recipe_name}" %}} + +package: + name: "{{{{ name|lower }}}}" + version: {{{{ environ.get('AZURESDK_CONDA_VERSION', '0.0.0') }}}} + +source: + url: {{{{ environ.get('{src_distribution_env_var}', '') }}}} + +build: + noarch: python + number: 0 + script: "{{{{ PYTHON }}}} -m pip install . -vv" + +requirements: + host: + - {host_reqs_str} + run: + - {run_reqs_str} + +test: + imports: + - {pkg_imports_str} + +about: + home: "{home_url}" + license: MIT + license_family: MIT + license_file: + summary: "{summary}" + description: | + {description} + doc_url: + dev_url: + +extra: + recipe-maintainers: + - xiangyan99 +""" + return meta_yaml_content + + +def add_new_data_plane_packages( + bundle_map: dict[str, list[str]], + new_data_plane_names: list[str], +) -> list[str]: + """Create meta.yaml files for new data plane packages and add import tests.""" + if len(new_data_plane_names) == 0: + return [] + + logger.info(f"Adding {len(new_data_plane_names)} new data plane packages") + result = [] + + # bundles are processed once when encountering the first package in that group + bundles_processed = set() + for package_name in new_data_plane_names: + logger.info(f"Adding new data plane meta.yaml for: {package_name}") + + folder_name = package_name + bundle_name = get_bundle_name(package_name) + if bundle_name: + if bundle_name in bundles_processed: + logger.info( + f"Meta.yaml for bundle {bundle_name} already created, skipping {package_name}" + ) + continue + folder_name = bundle_name + + pkg_yaml_path = os.path.join(CONDA_RECIPES_DIR, folder_name, "meta.yaml") + os.makedirs(os.path.dirname(pkg_yaml_path), exist_ok=True) + + try: + meta_yml = generate_data_plane_meta_yaml( + bundle_map, package_name, bundle_name + ) + if bundle_name: + bundles_processed.add(bundle_name) + except Exception as e: + logger.error( + f"Failed to generate meta.yaml content for {package_name} and skipping, error: {e}" + ) + result.append(package_name) + continue + + if not meta_yml: + logger.error( + f"Meta.yaml content for {package_name} is empty, skipping file creation" + ) + result.append(package_name) + continue + + try: + with open(pkg_yaml_path, "w") as f: + f.write(meta_yml) + logger.info(f"Created meta.yaml for {package_name} at {pkg_yaml_path}") + except Exception as e: + logger.error(f"Failed to create meta.yaml for {package_name}: {e}") + result.append(package_name) + + return result + + +# ===================================== +# Helpers for adding new mgmt plane packages to azure-mgmt/meta.yaml +# ===================================== + + +def add_new_mgmt_plane_packages(new_mgmt_plane_names: list[str]) -> list[str]: + """Update azure-mgmt/meta.yaml with new management libraries, and add import tests.""" + if len(new_mgmt_plane_names) == 0: + return [] + logger.info(f"Adding {len(new_mgmt_plane_names)} new management plane packages") + result = [] + + # can't use pyyaml due to jinja2 + with open(CONDA_MGMT_META_YAML_PATH, "r") as file: + content = file.read() + + test_match = re.search( + r"^test:\s*\n\s*imports:(.*?)^(?=\w)", content, re.MULTILINE | re.DOTALL + ) + if not test_match: + logger.error("Could not find 'test: imports:' section in meta.yaml") + result.extend(new_mgmt_plane_names) + return result + + existing_imports_text = test_match.group(1) + existing_imports = [ + line.strip() + for line in existing_imports_text.strip().split("\n") + if line.strip().startswith("-") + ] + + new_imports = [] + for package_name in new_mgmt_plane_names: + if not package_name: + logger.warning("Skipping package with missing name") + continue + + imports = get_valid_package_imports(package_name) + # Format imports for YAML with "- " prefix + formatted = [f"- {imp}" for imp in imports] + new_imports.extend(formatted) + logger.info( + f"Generated {len(imports)} import statements for {package_name}: {formatted}" + ) + + all_imports = list(set(existing_imports + new_imports)) + + # sort alphabetically + all_imports.sort() + + # format imports with proper indentation + formatted_imports = "\n".join(f" {imp}" for imp in all_imports) + + # replace the imports section + new_imports_section = f"test:\n imports:\n{formatted_imports}\n\n" + updated_content = re.sub( + r"^test:\s*\n\s*imports:.*?^(?=\w)", + new_imports_section, + content, + flags=re.MULTILINE | re.DOTALL, + ) + + try: + with open(CONDA_MGMT_META_YAML_PATH, "w") as file: + file.write(updated_content) + except Exception as e: + logger.error(f"Failed to update {CONDA_MGMT_META_YAML_PATH}: {e}") + result.extend(new_mgmt_plane_names) + + logger.info( + f"Added {len(new_mgmt_plane_names)} new management plane packages to meta.yaml" + ) + return result + + +# ===================================== +# Helpers for updating release logs +# ===================================== + + +def update_data_plane_release_logs( + package_dict: dict, + bundle_map: dict[str, list[str]], + new_data_plane_names: list[str], + release_date: str, +) -> list[str]: + """ + Add and update release logs for data plane conda packages. Release log includes versions of all packages for the release + """ + result = [] + + # Update all existing data plane release logs by file + + existing_release_logs = glob.glob(os.path.join(CONDA_RELEASE_LOGS_DIR, "*.md")) + for release_log_path in existing_release_logs: + curr_service_name = os.path.basename(release_log_path).replace(".md", "") + # skip azure-mgmt here + if curr_service_name == "azure-mgmt": + continue + if ( + curr_service_name not in package_dict + and curr_service_name not in bundle_map + and curr_service_name not in PACKAGES_WITH_DOWNLOAD_URI + ): + logger.warning( + f"Existing release log service {curr_service_name} was not found in CSV data, skipping update. It may be deprecated." + ) + result.append(curr_service_name) + continue + + pkg_updates = [] + if curr_service_name in bundle_map: + # handle grouped packages + pkg_names_in_bundle = bundle_map[curr_service_name] + for pkg_name in pkg_names_in_bundle: + pkg = package_dict.get(pkg_name, {}) + version = pkg.get(VERSION_GA_COL) + if version: + pkg_updates.append(f"- {pkg_name}-{version}") + else: + # shouldn't happen, but fallback + logger.error( + f"Package {pkg_name} in group {curr_service_name} is missing version info, it may be deprecated. Skipping in release log update" + ) + result.append(pkg_name) + else: + # handle exception for packages with download_uri + if curr_service_name in PACKAGES_WITH_DOWNLOAD_URI: + version = PACKAGES_WITH_DOWNLOAD_URI[curr_service_name] + if not version: + logger.error( + f"Package {curr_service_name} with download_uri is missing version info, it may be deprecated. Skipping in release log update" + ) + result.append(curr_service_name) + continue + else: + pkg = package_dict.get(curr_service_name, {}) + version = pkg.get(VERSION_GA_COL) + + if version: + pkg_updates.append(f"- {curr_service_name}-{version}") + else: + logger.error( + f"Package {curr_service_name} is missing version info, it may be deprecated. Skipping in release log update" + ) + result.append(curr_service_name) + try: + with open(release_log_path, "r") as f: + existing_content = f.read() + + lines = existing_content.split("\n") + + new_release = f"\n## {release_date}\n\n" + + # check if release is already logged + if new_release in existing_content: + logger.info( + f"Release log for {curr_service_name} already contains entry for {release_date}, overwriting" + ) + # remove existing release section to overwrite + release_idx = lines.index(new_release.strip()) + + ## find next release heading or end of file + next_release_idx = next( + ( + i + for i in range(release_idx + 1, len(lines)) + if lines[i].startswith("## ") + ), + len(lines), + ) + del lines[release_idx:next_release_idx] + + new_release += "### Packages included\n\n" + new_release += "\n".join(pkg_updates) + lines.insert(1, new_release) + + updated_content = "\n".join(lines) + + with open(release_log_path, "w") as f: + f.write(updated_content) + + logger.info(f"Updated release log for {os.path.basename(release_log_path)}") + except Exception as e: + logger.error( + f"Failed to update release log {os.path.basename(release_log_path)}: {e}" + ) + result.append(curr_service_name) + + # Handle brand new packages + for package_name in new_data_plane_names: + pkg = package_dict.get(package_name, {}) + version = pkg.get(VERSION_GA_COL) + + if not version: + logger.warning(f"Skipping {package_name} with missing version") + result.append(package_name) + continue + + bundle_name = get_bundle_name(package_name) + # check for bundle + if bundle_name: + display_name = bundle_name + else: + display_name = package_name + + release_log_path = os.path.join(CONDA_RELEASE_LOGS_DIR, f"{display_name}.md") + + if not os.path.exists(release_log_path): + # Add brand new release log file + logger.info(f"Creating new release log for: {display_name}") + + title_parts = display_name.replace("azure-", "").split("-") + title = " ".join(word.title() for word in title_parts) + + content = f"# Azure {title} client library for Python (conda)\n\n" + content += f"## {release_date}\n\n" + content += "### Packages included\n\n" + + pkg_updates = [] + if bundle_name: + pkg_names_in_log = bundle_map.get(bundle_name, []) + for pkg_name in pkg_names_in_log: + pkg = package_dict.get(pkg_name, {}) + version = pkg.get(VERSION_GA_COL) + pkg_updates.append(f"- {pkg_name}-{version}") + else: + pkg = package_dict.get(package_name, {}) + version = pkg.get(VERSION_GA_COL) + pkg_updates.append(f"- {package_name}-{version}") + content += "\n".join(pkg_updates) + + try: + with open(release_log_path, "w") as f: + f.write(content) + logger.info(f"Created new release log for {display_name}") + except Exception as e: + logger.error(f"Failed to create release log for {display_name}: {e}") + result.append(display_name) + + else: + logger.info( + f"Release log for {display_name} already exists, check that new package {package_name} is included" + ) + + return result + + +def update_mgmt_plane_release_log( + package_dict: dict, + all_mgmt_plane_names: list[str], + release_date: str, +) -> list[str]: + """ + Update azure-mgmt release log. + """ + result = [] + + mgmt_log_path = os.path.join(CONDA_RELEASE_LOGS_DIR, "azure-mgmt.md") + if not os.path.exists(mgmt_log_path): + logger.error("Management plane release log azure-mgmt.md does not exist.") + return all_mgmt_plane_names # all new packages need attention + + pkg_updates = [] + for package_name in all_mgmt_plane_names: + pkg = package_dict.get(package_name, {}) + version = pkg.get(VERSION_GA_COL) + + if not version: + logger.warning( + f"Skipping release log update of {package_name} with missing version" + ) + result.append(package_name) + continue + + pkg_updates.append(f"- {package_name}-{version}") + + try: + with open(mgmt_log_path, "r") as f: + existing_content = f.read() + + lines = existing_content.split("\n") + + new_release = f"\n## {release_date}\n\n" + + # check if release is already logged + if new_release in existing_content: + logger.info( + f"Release log for azure-mgmt already contains entry for {release_date}, overwriting" + ) + # remove existing release section to overwrite + release_idx = lines.index(new_release.strip()) + + ## find next release heading or end of file + next_release_idx = next( + ( + i + for i in range(release_idx + 1, len(lines)) + if lines[i].startswith("## ") + ), + len(lines), + ) + del lines[release_idx:next_release_idx] + + new_release += "### Packages included\n\n" + + new_release += "\n".join(pkg_updates) + lines.insert(1, new_release) + updated_content = "\n".join(lines) + + with open(mgmt_log_path, "w") as f: + f.write(updated_content) + except Exception as e: + logger.error(f"Failed to update azure-mgmt release log: {e}") + return all_mgmt_plane_names + + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Update conda package files and versions for release." + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Enable debug logging", + ) + parser.add_argument( + "--release-date", + type=str, + default=None, + help="Release date in 'MM.DD' format (e.g., '03.01', '12.01'). " + "Year is determined automatically. If not provided, the next release date is calculated.", + ) + + args = parser.parse_args() + configure_logging(args) + + # Handle release date + if args.release_date: + try: + current_year = datetime.now().year + target_release_date = datetime.strptime( + f"{current_year}.{args.release_date}", "%Y.%m.%d" + ) + logger.info( + f"Using provided release date: {target_release_date.strftime('%Y.%m.%d')}" + ) + except ValueError as e: + logger.error(f"Invalid release date format '{args.release_date}': {e}") + logger.error("Expected format: 'MM.DD' (e.g., '03.01', '12.01')") + exit(1) + else: + logger.info( + "No release date provided, auto bumping old date by {} months.".format( + RELEASE_PERIOD_MONTHS + ) + ) + target_release_date = None + + old_date, new_version = update_conda_version(target_release_date) + + # Output version as Azure DevOps pipeline variable to use in PR + print(f"##vso[task.setvariable variable=CondaReleaseVersion]{new_version}") + + # convert to mm/dd/yyyy format for comparison with CSV dates + old_version = old_date.strftime("%m/%d/%Y") + + packages = parse_csv() + if not packages: + logger.error("No packages found in CSV data.") + exit(1) + + # Only ship GA packages that are not deprecated + packages = [ + pkg + for pkg in packages + if ( + (pkg.get(VERSION_GA_COL) and pkg.get(LATEST_GA_DATE_COL)) + and not pkg.get(SUPPORT_COL) == "deprecated" + ) + ] + logger.info(f"Filtered to {len(packages)} GA packages") + + data_pkgs, mgmt_pkgs = separate_packages_by_type(packages) + + outdated_data_plane_names = [ + pkg.get(PACKAGE_COL, "") + for pkg in data_pkgs + if package_needs_update(pkg, old_version, is_new=False) + ] + new_data_plane_names = [ + pkg.get(PACKAGE_COL, "") + for pkg in data_pkgs + if package_needs_update(pkg, old_version, is_new=True) + ] + outdated_mgmt_plane_names = [ + pkg.get(PACKAGE_COL, "") + for pkg in mgmt_pkgs + if package_needs_update(pkg, old_version, is_new=False) + ] + new_mgmt_plane_names = [ + pkg.get(PACKAGE_COL, "") + for pkg in mgmt_pkgs + if package_needs_update(pkg, old_version, is_new=True) + ] + + # don't overlap new packages with outdated packages + outdated_data_plane_names = [ + name for name in outdated_data_plane_names if name not in new_data_plane_names + ] + outdated_mgmt_plane_names = [ + name for name in outdated_mgmt_plane_names if name not in new_mgmt_plane_names + ] + + # map package name to csv row for easy lookup + package_dict = {pkg.get(PACKAGE_COL, ""): pkg for pkg in packages} + + outdated_package_names = outdated_data_plane_names + outdated_mgmt_plane_names + + # update conda-sdk-client.yml + conda_sdk_client_pkgs_result = update_conda_sdk_client_yml( + package_dict, outdated_package_names, new_data_plane_names, new_mgmt_plane_names + ) + + # pre-process bundled data packages to minimize file writes for new data plane packages, + # and release logs (mgmt packages are always bundled together) + bundle_map, bundle_failed_pkgs = map_bundle_to_packages( + [pkg.get(PACKAGE_COL, "") for pkg in data_pkgs] + ) + logger.info( + f"Identified {len(bundle_map)} release bundles from package data: {bundle_map}" + ) + + # handle new data plane libraries + new_data_plane_results = add_new_data_plane_packages( + bundle_map, new_data_plane_names + ) + + # handle new mgmt plane libraries + new_mgmt_plane_results = add_new_mgmt_plane_packages(new_mgmt_plane_names) + + # add/update release logs + data_plane_release_log_results = update_data_plane_release_logs( + package_dict, bundle_map, new_data_plane_names, new_version + ) + + all_mgmt_plane_names = [pkg.get(PACKAGE_COL, "") for pkg in mgmt_pkgs] + + mgmt_plane_release_log_results = update_mgmt_plane_release_log( + package_dict, all_mgmt_plane_names, new_version + ) + + print("=== REPORT ===") + + if conda_sdk_client_pkgs_result: + print( + "The following packages may require manual adjustments in conda-sdk-client.yml:" + ) + for pkg_name in conda_sdk_client_pkgs_result: + print(f"- {pkg_name}") + + if new_data_plane_results: + print( + "\nThe following new data plane packages may require manual meta.yaml creation or adjustments:" + ) + for pkg_name in new_data_plane_results: + print(f"- {pkg_name}") + + if new_mgmt_plane_results: + print( + "\nThe following new management plane packages may require manual adjustments in azure-mgmt/meta.yaml:" + ) + for pkg_name in new_mgmt_plane_results: + print(f"- {pkg_name}") + + if data_plane_release_log_results: + print( + "\nThe following data plane packages may require manual adjustments in release logs:" + ) + for pkg_name in data_plane_release_log_results: + print(f"- {pkg_name}") + + if mgmt_plane_release_log_results: + print( + "\nThe following management plane packages may require manual adjustments in azure-mgmt release log:" + ) + for pkg_name in mgmt_plane_release_log_results: + print(f"- {pkg_name}") + + if bundle_failed_pkgs: + print( + "\nThe following packages errored when constructing bundle map, they may need a [tool.azure-sdk-conda] section in their pyproject.toml for proper release grouping, and may have been improperly individually processed." + ) + for pkg_name in bundle_failed_pkgs: + print(f"- {pkg_name}") + + if len(new_data_plane_names) > 0: + print("\n=== Manual Steps for New Data Plane Packages ===") + print( + "- A dummy placeholder library needs to be requested on Conda for new data plane packages." + ) + print("- A new AKA link needs to be created for each new release log.") + print( + "\nSee the generated PR description for further details. The new data plane package names are:" + ) + for pkg_name in new_data_plane_names: + print(f"{pkg_name}") diff --git a/eng/pipelines/conda-update-pipeline.yml b/eng/pipelines/conda-update-pipeline.yml new file mode 100644 index 000000000000..c7a4a6db1d66 --- /dev/null +++ b/eng/pipelines/conda-update-pipeline.yml @@ -0,0 +1,121 @@ +trigger: none +pr: none + +parameters: + - name: releaseDate + displayName: Release Date (MM.DD) + type: string + default: 'auto' + values: + - auto + - 03.01 + - 06.01 + - 09.01 + - 12.01 + - name: dryRun + displayName: Dry Run (skip PR submission) + type: boolean + default: false + +# Scheduled to run a week before each quarterly release +schedules: + - cron: "0 0 24 11 *" + displayName: Pre-December Quarterly Release + branches: + include: + - main + always: true + - cron: "0 0 22 2 *" + displayName: Pre-March Quarterly Release + branches: + include: + - main + always: true + - cron: "0 0 25 5 *" + displayName: Pre-June Quarterly Release + branches: + include: + - main + always: true + - cron: "0 0 25 8 *" + displayName: Pre-September Quarterly Release + branches: + include: + - main + always: true + +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + stages: + - stage: UpdateCondaFiles + displayName: Update Conda Files + + jobs: + - job: UpdateCondaFilesJob + timeoutInMinutes: 90 + displayName: Update Conda Files and Submit PR + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - name: ReleaseDate + ${{ if eq(parameters.releaseDate, 'auto') }}: + ${{ if contains(variables['Build.CronSchedule.DisplayName'], 'December') }}: + value: '12.01' + ${{ elseif contains(variables['Build.CronSchedule.DisplayName'], 'March') }}: + value: '03.01' + ${{ elseif contains(variables['Build.CronSchedule.DisplayName'], 'June') }}: + value: '06.01' + ${{ elseif contains(variables['Build.CronSchedule.DisplayName'], 'September') }}: + value: '09.01' + ${{ else }}: + value: 'Unknown' + ${{ else }}: + value: ${{ parameters.releaseDate }} + + pool: + name: azsdk-pool + image: ubuntu-24.04 + os: linux + + steps: + - checkout: self + persistCredentials: true + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11' + inputs: + versionSpec: '3.11' + + - script: | + python -m pip install --upgrade pip + python -m pip install "eng/tools/azure-sdk-tools[build]" + python -m pip install python-dateutil + displayName: 'Prep Environment' + + - script: | + if [ "$(ReleaseDate)" != "Unknown" ]; then + python conda/update_conda_files.py --release-date "$(ReleaseDate)" + else + python conda/update_conda_files.py + fi + displayName: 'Update Conda Files' + + - ${{ if eq(parameters.dryRun, false) }}: + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + PRBranchName: conda-update-$(Build.BuildId) + CommitMsg: 'Update conda files for $(CondaReleaseVersion) release' + PRTitle: 'Conda Release $(CondaReleaseVersion) generated by $(Build.BuildId)' + PRBody: | + This PR was automatically generated to update Conda files for a new release. + + - Updates outdated package versions in conda-sdk-client.yml + - Adds new packages to conda-sdk-client.yml + - Adds/updates yamls and changelogs + + ## Next Steps + - [ ] For new data plane packages, submit this form to create a private dummy library placeholder in Conda before uploading a release: https://forms.office.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR180k2XpSUFBtXHTh8-jMUlUNlA1MFpZOVhZME1aNU1EU1Y3SjZRU0JNRC4u + - [ ] After this PR is merged and succeeds the Conda build, approve the pipeline to upload the releases to Conda + - [ ] After upload, delete the dummy libraries and make the new packages publicly available in Conda. + - [ ] Create an AKA link for new release logs here: http://aka.ms/ + BaseBranchName: main diff --git a/eng/pipelines/templates/stages/conda-sdk-client.yml b/eng/pipelines/templates/stages/conda-sdk-client.yml index 98b6811d2600..68e89ba8f7f0 100644 --- a/eng/pipelines/templates/stages/conda-sdk-client.yml +++ b/eng/pipelines/templates/stages/conda-sdk-client.yml @@ -905,5 +905,3 @@ extends: version: 1.0.0 - package: azure-mgmt-workloadssapvirtualinstance version: 1.0.0 - - diff --git a/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index 0c5bb8b5ebab..5c8504ba4fbb 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -16,6 +16,7 @@ # this assumes the presence of "packaging" from packaging.requirements import Requirement +from packaging.version import Version, InvalidVersion from setuptools import Extension from ci_tools.variables import str_to_bool @@ -351,6 +352,9 @@ def from_path(cls, parse_directory_or_file: str): def get_build_config(self) -> Optional[Dict[str, Any]]: return get_build_config(self.folder) + def get_conda_config(self) -> Optional[Dict[str, Any]]: + return get_conda_config(self.folder) + def get_config_setting(self, setting: str, default: Any = True) -> Any: return get_config_setting(self.folder, setting, default) @@ -383,7 +387,7 @@ def __str__(self): def update_build_config(package_path: str, new_build_config: Dict[str, Any]) -> Dict[str, Any]: """ - Attempts to update a pyproject.toml's [tools.azure-sdk-tools] section with a new check configuration. + Attempts to update a pyproject.toml's [tool.azure-sdk-tools] section with a new check configuration. This function can only append or override existing check values. It cannot delete them. """ @@ -454,6 +458,27 @@ def get_build_config(package_path: str) -> Optional[Dict[str, Any]]: return {} +def get_conda_config(package_path: str) -> Optional[Dict[str, Any]]: + """ + Attempts to retrieve all values within [tool.azure-sdk-conda] section of a pyproject.toml. + """ + if os.path.isfile(package_path): + package_path = os.path.dirname(package_path) + + toml_file = os.path.join(package_path, "pyproject.toml") + + if os.path.exists(toml_file): + try: + with open(toml_file, "rb") as f: + toml_dict = toml.load(f) + if "tool" in toml_dict: + tool_configs = toml_dict["tool"] + if "azure-sdk-conda" in tool_configs: + return tool_configs["azure-sdk-conda"] + except: + return {} + + def get_ci_config(package_path: str) -> Optional[Dict[str, Any]]: """ Attempts to retrieve the parsed toml content of a CI.yml associated with this package. diff --git a/sdk/ai/azure-ai-voicelive/pyproject.toml b/sdk/ai/azure-ai-voicelive/pyproject.toml index 3e2540ab6842..02fb44dc7667 100644 --- a/sdk/ai/azure-ai-voicelive/pyproject.toml +++ b/sdk/ai/azure-ai-voicelive/pyproject.toml @@ -76,4 +76,7 @@ exclude = [ pytyped = ["py.typed"] [tool.pytest.ini_options] asyncio_default_fixture_loop_scope = "function" -asyncio_mode = "auto" \ No newline at end of file +asyncio_mode = "auto" + +[tool.azure-sdk-conda] +in_bundle = false \ No newline at end of file