diff --git a/src/k8s-extension/HISTORY.rst b/src/k8s-extension/HISTORY.rst index b973a4fe888..4f8d13c1f81 100644 --- a/src/k8s-extension/HISTORY.rst +++ b/src/k8s-extension/HISTORY.rst @@ -3,6 +3,14 @@ Release History =============== +1.4.3 +++++++++++++++++++ +* microsoft.azuremonitor.containers: Extend ContainerInsights Extension dataCollectionSettings with streams and containerlogv2 field. Also, add a kind tag in DCR creation for the ContainerInsights extension. +* microsoft.dapr: Use semver instead of packaging +* microsoft.azuremonitor.containers.metrics: update logic to sanitize cluster name for dc* objects +* microsoft.openservicemesh: Fix osm-arc version check for CI tags +* add support to skip provisioning of prerequisites for Azure Monitor K8s extensions + 1.4.2 ++++++++++++++++++ * microsoft.azuremonitor.containers: ContainerInsights Extension Managed Identity Auth Enabled by default diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMonitorMetrics.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMonitorMetrics.py index 9eb4a74dab0..dc89e2b6050 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMonitorMetrics.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMonitorMetrics.py @@ -9,7 +9,7 @@ import json import re -from ..utils import get_cluster_rp_api_version +from ..utils import get_cluster_rp_api_version, is_skip_prerequisites_specified from knack.log import get_logger @@ -48,16 +48,19 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t if release_train is None: release_train = 'stable' - cluster_subscription = get_subscription_id(cmd.cli_ctx) - ensure_azure_monitor_profile_prerequisites( - cmd, - cluster_rp, - cluster_subscription, - resource_group_name, - cluster_name, - configuration_settings, - cluster_type - ) + if not is_skip_prerequisites_specified(configuration_settings): + cluster_subscription = get_subscription_id(cmd.cli_ctx) + ensure_azure_monitor_profile_prerequisites( + cmd, + cluster_rp, + cluster_subscription, + resource_group_name, + cluster_name, + configuration_settings, + cluster_type + ) + else: + logger.info("Provisioning of prerequisites is skipped") create_identity = True extension = Extension( @@ -72,6 +75,16 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t return extension, name, create_identity def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp, yes): - # cluster_rp, _ = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_rp) + cluster_rp, _ = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_rp) + try: + extension = client.get(resource_group_name, cluster_rp, cluster_type, cluster_name, name) + except Exception: + pass # its OK to ignore the exception since MSI auth in preview + + if (extension is not None) and (extension.configuration_settings is not None): + if is_skip_prerequisites_specified(extension.configuration_settings): + logger.info("Deprovisioning of prerequisites is skipped") + return + cluster_subscription = get_subscription_id(cmd.cli_ctx) unlink_azure_monitor_profile_artifacts(cmd, cluster_subscription, resource_group_name, cluster_name) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/ContainerInsights.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/ContainerInsights.py index a943d88584c..9f8e5b067ae 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/ContainerInsights.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/ContainerInsights.py @@ -9,7 +9,7 @@ import json import re -from ..utils import get_cluster_rp_api_version +from ..utils import get_cluster_rp_api_version, is_skip_prerequisites_specified from .. import consts from knack.log import get_logger @@ -31,6 +31,7 @@ cf_resources, cf_resource_groups, cf_log_analytics) logger = get_logger(__name__) +DCR_API_VERSION = "2022-06-01" class ContainerInsights(DefaultExtension): @@ -59,8 +60,11 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t 'only supports cluster scope and single instance of this extension.', extension_type) logger.warning("Defaulting to extension name '%s' and release-namespace '%s'", name, release_namespace) - _get_container_insights_settings(cmd, resource_group_name, cluster_rp, cluster_type, cluster_name, configuration_settings, - configuration_protected_settings, is_ci_extension_type) + if not is_skip_prerequisites_specified(configuration_settings): + _get_container_insights_settings(cmd, resource_group_name, cluster_rp, cluster_type, cluster_name, configuration_settings, + configuration_protected_settings, is_ci_extension_type) + else: + logger.info("Provisioning of prerequisites is skipped") # NOTE-2: Return a valid Extension object, Instance name and flag for Identity create_identity = True @@ -85,6 +89,11 @@ def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_t except Exception: pass # its OK to ignore the exception since MSI auth in preview + if (extension is not None) and (extension.configuration_settings is not None): + if is_skip_prerequisites_specified(extension.configuration_settings): + logger.info("Deprovisioning of prerequisites is skipped") + return + subscription_id = get_subscription_id(cmd.cli_ctx) # handle cluster type here cluster_resource_id = '/subscriptions/{0}/resourceGroups/{1}/providers/{2}/{3}/{4}'.format(subscription_id, resource_group_name, cluster_rp, cluster_type, cluster_name) @@ -100,7 +109,7 @@ def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_t if (isinstance(useAADAuthSetting, str) and str(useAADAuthSetting).lower() == "true") or (isinstance(useAADAuthSetting, bool) and useAADAuthSetting): useAADAuth = True if useAADAuth: - association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2021-04-01" + association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version={DCR_API_VERSION}" for _ in range(3): try: send_raw_request(cmd.cli_ctx, "GET", association_url,) @@ -114,7 +123,7 @@ def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_t pass # its OK to ignore the exception since MSI auth in preview if isDCRAExists: - association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2021-04-01" + association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version={DCR_API_VERSION}" for _ in range(3): try: send_raw_request(cmd.cli_ctx, "DELETE", association_url,) @@ -455,6 +464,8 @@ def _get_container_insights_settings(cmd, cluster_resource_group_name, cluster_r subscription_id = get_subscription_id(cmd.cli_ctx) workspace_resource_id = '' useAADAuth = True + if 'amalogs.useAADAuth' not in configuration_settings: + configuration_settings['amalogs.useAADAuth'] = "true" extensionSettings = {} if configuration_settings is not None: @@ -471,11 +482,15 @@ def _get_container_insights_settings(cmd, cluster_resource_group_name, cluster_r logger.info("provided useAADAuth flag is : %s", useAADAuthSetting) if (isinstance(useAADAuthSetting, str) and str(useAADAuthSetting).lower() == "true") or (isinstance(useAADAuthSetting, bool) and useAADAuthSetting): useAADAuth = True + else: + useAADAuth = False elif 'amalogs.useAADAuth' in configuration_settings: useAADAuthSetting = configuration_settings['amalogs.useAADAuth'] logger.info("provided useAADAuth flag is : %s", useAADAuthSetting) if (isinstance(useAADAuthSetting, str) and str(useAADAuthSetting).lower() == "true") or (isinstance(useAADAuthSetting, bool) and useAADAuthSetting): useAADAuth = True + else: + useAADAuth = False if useAADAuth and ('dataCollectionSettings' in configuration_settings): dataCollectionSettingsString = configuration_settings["dataCollectionSettings"] logger.info("provided dataCollectionSettings is : %s", dataCollectionSettingsString) @@ -495,6 +510,14 @@ def _get_container_insights_settings(cmd, cluster_resource_group_name, cluster_r namspaces = dataCollectionSettings["namespaces"] if isinstance(namspaces, list) is False: raise InvalidArgumentValueError('namespaces must be an array type') + if 'enableContainerLogV2' in dataCollectionSettings.keys(): + enableContainerLogV2Value = dataCollectionSettings["enableContainerLogV2"] + if not isinstance(enableContainerLogV2Value, bool): + raise InvalidArgumentValueError('enableContainerLogV2Value value MUST be either true or false') + if 'streams' in dataCollectionSettings.keys(): + streams = dataCollectionSettings["streams"] + if isinstance(streams, list) is False: + raise InvalidArgumentValueError('streams must be an array type') extensionSettings["dataCollectionSettings"] = dataCollectionSettings workspace_resource_id = workspace_resource_id.strip() @@ -673,23 +696,27 @@ def _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_ if (cluster_region not in region_ids): raise ClientRequestError(f"Data Collection Rule Associations are not supported for cluster region {cluster_region}") - dcr_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{dcr_resource_id}?api-version=2021-04-01" + dcr_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{dcr_resource_id}?api-version={DCR_API_VERSION}" # get existing tags on the container insights extension DCR if the customer added any existing_tags = get_existing_container_insights_extension_dcr_tags(cmd, dcr_url) + streams = ["Microsoft-ContainerInsights-Group-Default"] + if extensionSettings is not None and 'dataCollectionSettings' in extensionSettings.keys(): + dataCollectionSettings = extensionSettings["dataCollectionSettings"] + if dataCollectionSettings is not None and 'streams' in dataCollectionSettings.keys(): + streams = dataCollectionSettings["streams"] # create the DCR dcr_creation_body = json.dumps( { "location": workspace_region, "tags": existing_tags, + "kind": "Linux", "properties": { "dataSources": { "extensions": [ { "name": "ContainerInsightsExtension", - "streams": [ - "Microsoft-ContainerInsights-Group-Default" - ], + "streams": streams, "extensionName": "ContainerInsights", "extensionSettings": extensionSettings } @@ -697,10 +724,7 @@ def _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_ }, "dataFlows": [ { - "streams": [ - "Microsoft-ContainerInsights-Group-Default" - - ], + "streams": streams, "destinations": ["la-workspace"], } ], @@ -735,7 +759,7 @@ def _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_ }, } ) - association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2021-04-01" + association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version={DCR_API_VERSION}" for _ in range(3): try: send_raw_request(cmd.cli_ctx, "PUT", association_url, body=association_body,) diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/Dapr.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/Dapr.py index 752e6c39587..46520c69a7a 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/Dapr.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/Dapr.py @@ -13,7 +13,7 @@ from copy import deepcopy from knack.log import get_logger from knack.prompting import prompt, prompt_y_n -from packaging import version as packaging_version +from semver import VersionInfo from ..vendored_sdks.models import Extension, PatchExtension, Scope, ScopeCluster from .DefaultExtension import DefaultExtension @@ -204,7 +204,7 @@ def _is_downgrade(v1: str, v2: str) -> bool: Returns True if version v1 is less than version v2. """ try: - return packaging_version.Version(v1) < packaging_version.Version(v2) - except packaging_version.InvalidVersion: + return VersionInfo.parse(v1) < VersionInfo.parse(v2) + except ValueError: logger.debug("Warning: Unable to compare versions %s and %s.", v1, v2) return True # This will cause the apply-CRDs hook to be disabled, which is safe. diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/OpenServiceMesh.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/OpenServiceMesh.py index ec7738cdb92..2853f2a42d5 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/OpenServiceMesh.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/OpenServiceMesh.py @@ -71,7 +71,6 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t def _validate_tested_distro(cmd, cluster_resource_group_name, cluster_name, extension_version, extension_release_train): - logger.warning("Running validated distros...") field_unavailable_error = '\"testedDistros\" field unavailable for version {0} of microsoft.openservicemesh, ' \ 'cannot determine if this Kubernetes distribution has been properly tested'.format(extension_version) @@ -104,7 +103,10 @@ def _validate_tested_distro(cmd, cluster_resource_group_name, cluster_name, exte extension_version = tags[len(tags) - 1] - if version.parse(str(extension_version)) <= version.parse("0.8.3"): + ext_str = str(extension_version) + + # Don't parse version for test and CI tags + if "pr" not in ext_str and "release" not in ext_str and version.parse(ext_str) <= version.parse("0.8.3"): logger.warning(field_unavailable_error) return diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/dc/defaults.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/dc/defaults.py index 8dd7260377e..9acd2ae8e23 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/dc/defaults.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/dc/defaults.py @@ -13,10 +13,10 @@ # All DC* object names should end only in alpha numeric (after `length` trim) # DCE remove underscore from cluster name def sanitize_name(name, objtype, length): + name = name[0:length] length = length - 1 if objtype == DC_TYPE.DCE: - name = name.replace("_", "") - name = name[0:length] + name = ''.join(char for char in name if char.isalnum() or char == '-') lastIndexAlphaNumeric = len(name) - 1 while ((name[lastIndexAlphaNumeric].isalnum() is False) and lastIndexAlphaNumeric > -1): lastIndexAlphaNumeric = lastIndexAlphaNumeric - 1 diff --git a/src/k8s-extension/azext_k8s_extension/utils.py b/src/k8s-extension/azext_k8s_extension/utils.py index 11112935c87..66740036efa 100644 --- a/src/k8s-extension/azext_k8s_extension/utils.py +++ b/src/k8s-extension/azext_k8s_extension/utils.py @@ -54,3 +54,17 @@ def is_dogfood_cluster(cmd): urlparse(cmd.cli_ctx.cloud.endpoints.resource_manager).hostname == consts.DF_RM_HOSTNAME ) + + +def is_skip_prerequisites_specified(configuration_settings): + # Determine if provisioning to prerequisites should be skipped by a configuration setting named skipPrerequisites. + SKIP_PREQUISITES = 'skipPrerequisites' + + has_skip_prerequisites_set = False + + if SKIP_PREQUISITES in configuration_settings: + skip_prerequisites_configuration_setting = configuration_settings[SKIP_PREQUISITES] + if (isinstance(skip_prerequisites_configuration_setting, str) and str(skip_prerequisites_configuration_setting).lower() == "true") or (isinstance(skip_prerequisites_configuration_setting, bool) and skip_prerequisites_configuration_setting): + has_skip_prerequisites_set = True + + return has_skip_prerequisites_set diff --git a/src/k8s-extension/setup.py b/src/k8s-extension/setup.py index beda2d80308..f2e6e28f7be 100644 --- a/src/k8s-extension/setup.py +++ b/src/k8s-extension/setup.py @@ -33,7 +33,7 @@ # TODO: Add any additional SDK dependencies here DEPENDENCIES = [] -VERSION = "1.4.2" +VERSION = "1.4.3" with open("README.rst", "r", encoding="utf-8") as f: README = f.read()