diff --git a/arm/Microsoft.ContainerService/managedClusters/.bicep/nested_rbac.bicep b/arm/Microsoft.ContainerService/managedClusters/.bicep/nested_rbac.bicep index 79db5f959b..7cb5a582b4 100644 --- a/arm/Microsoft.ContainerService/managedClusters/.bicep/nested_rbac.bicep +++ b/arm/Microsoft.ContainerService/managedClusters/.bicep/nested_rbac.bicep @@ -6,6 +6,7 @@ var builtInRoleNames = { 'Owner': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8e3af657-a8ff-443c-a75c-2fe8c4bcb635') 'Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') 'Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'acdd72a7-3385-48ef-bd42-f606fba81ae7') + 'AcrPull': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') 'Azure Kubernetes Service Cluster Admin Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '0ab0b1a8-8aac-4efd-b8c2-3ee1fb270be8') 'Azure Kubernetes Service Cluster User Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4abbcc35-e782-43d8-92c5-2d3f1bd2253f') 'Azure Kubernetes Service Contributor Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ed7f3fbd-7b88-4dd4-9017-9adb7ce333f8') @@ -19,6 +20,7 @@ var builtInRoleNames = { 'Managed Application Operator Role': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'c7393b34-138c-406f-901b-d8cf2b17e6ae') 'Managed Applications Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b9331d33-8a36-4f8c-b097-4f54124fdb44') 'Monitoring Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '749f88d5-cbae-40b8-bcfc-e573ddc772fa') + 'Managed Identity Operator': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f1a07417-d97a-45cb-824c-7a7467783830') 'Monitoring Metrics Publisher': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') 'Monitoring Reader': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '43d0d8ad-25c7-4714-9337-8ba259a9fe05') 'Resource Policy Contributor': subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '36243c78-bf99-498c-9df9-86d9f8d28608') diff --git a/arm/Microsoft.ContainerService/managedClusters/deploy.bicep b/arm/Microsoft.ContainerService/managedClusters/deploy.bicep index 60dab758d0..08c5242fb2 100644 --- a/arm/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/arm/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -98,11 +98,26 @@ param aadProfileManaged bool = true @description('Optional. Specifies whether to enable Azure RBAC for Kubernetes authorization.') param aadProfileEnableAzureRBAC bool = true +@description('Optional. If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled.') +param disableLocalAccounts bool = false + @description('Optional. Name of the resource group containing agent pool nodes.') param nodeResourceGroup string = '${resourceGroup().name}_aks_${name}_nodes' +@description('Optional. IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer.') +param authorizedIPRanges array = [] + +@description('Optional. Whether to disable run command for the cluster or not.') +param disableRunCommand bool = false + @description('Optional. Specifies whether to create the cluster as a private cluster or not.') -param aksClusterEnablePrivateCluster bool = false +param enablePrivateCluster bool = false + +@description('Optional. Whether to create additional public FQDN for private cluster or not.') +param enablePrivateClusterPublicFQDN bool = false + +@description('Optional. If AKS will create a Private DNS Zone in the Node Resource Group.') +param usePrivateDNSZone bool = false @description('Required. Properties of the primary agent pool.') param primaryAgentPoolProfile array @@ -125,6 +140,16 @@ param azurePolicyVersion string = 'v2' @description('Optional. Specifies whether the kubeDashboard add-on is enabled or not.') param kubeDashboardEnabled bool = false +@description('Optional. Specifies whether the KeyvaultSecretsProvider add-on is enabled or not.') +param enableKeyvaultSecretsProvider bool = false + +@allowed([ + 'false' + 'true' +]) +@description('Optional. Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation.') +param enableSecretRotation string = 'false' + @description('Optional. Specifies the scan interval of the auto-scaler of the AKS cluster.') param autoScalerProfileScanInterval string = '10s' @@ -149,6 +174,63 @@ param autoScalerProfileUtilizationThreshold string = '0.5' @description('Optional. Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster.') param autoScalerProfileMaxGracefulTerminationSec string = '600' +@allowed([ + 'false' + 'true' +]) +@description('Optional. Specifies the balance of similar node groups for the auto-scaler of the AKS cluster.') +param autoScalerProfileBalanceSimilarNodeGroups string = 'false' + +@allowed([ + 'least-waste' + 'most-pods' + 'priority' + 'random' +]) +@description('Optional. Specifies the expand strategy for the auto-scaler of the AKS cluster.') +param autoScalerProfileExpander string = 'random' + +@description('Optional. Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster.') +param autoScalerProfileMaxEmptyBulkDelete string = '10' + +@description('Optional. Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an "m". No unit of time other than minutes (m) is supported.') +param autoScalerProfileMaxNodeProvisionTime string = '15m' + +@description('Optional. Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0.') +param autoScalerProfileMaxTotalUnreadyPercentage string = '45' + +@description('Optional. For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit ("s" for seconds, "m" for minutes, "h" for hours, etc).') +param autoScalerProfileNewPodScaleUpDelay string = '0s' + +@description('Optional. Specifies the ok total unready count for the auto-scaler of the AKS cluster.') +param autoScalerProfileOkTotalUnreadyCount string = '3' + +@allowed([ + 'false' + 'true' +]) +@description('Optional. Specifies if nodes with local storage should be skipped for the auto-scaler of the AKS cluster.') +param autoScalerProfileSkipNodesWithLocalStorage string = 'true' + +@allowed([ + 'false' + 'true' +]) +@description('Optional. Specifies if nodes with system pods should be skipped for the auto-scaler of the AKS cluster.') +param autoScalerProfileSkipNodesWithSystemPods string = 'true' + +@description('Optional. Running in Kubenet is disabled by default due to the security related nature of AAD Pod Identity and the risks of IP spoofing.') +param podIdentityProfileAllowNetworkPluginKubenet bool = false + +@description('Optional. Whether the pod identity addon is enabled.') +param podIdentityProfileEnable bool = false + +@description('Optional. The pod identities to use in the cluster.') +param podIdentityProfileUserAssignedIdentities array = [] + +@description('Optional. The pod identity exceptions to allow.') +param podIdentityProfileUserAssignedIdentityExceptions array = [] + @description('Optional. Resource ID of the diagnostic storage account.') param diagnosticStorageAccountId string = '' @@ -299,8 +381,15 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2021-10-01' kubeDashboard: { enabled: kubeDashboardEnabled } + azureKeyvaultSecretsProvider: { + enabled: enableKeyvaultSecretsProvider + config: { + enableSecretRotation: enableSecretRotation + } + } } enableRBAC: aadProfileEnableAzureRBAC + disableLocalAccounts: disableLocalAccounts nodeResourceGroup: nodeResourceGroup networkProfile: { networkPlugin: !empty(aksClusterNetworkPlugin) ? any(aksClusterNetworkPlugin) : null @@ -323,17 +412,36 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2021-10-01' tenantID: aadProfileTenantId } autoScalerProfile: { - 'scan-interval': autoScalerProfileScanInterval + 'balance-similar-node-groups': autoScalerProfileBalanceSimilarNodeGroups + 'expander': autoScalerProfileExpander + 'max-empty-bulk-delete': autoScalerProfileMaxEmptyBulkDelete + 'max-graceful-termination-sec': autoScalerProfileMaxGracefulTerminationSec + 'max-node-provision-time': autoScalerProfileMaxNodeProvisionTime + 'max-total-unready-percentage': autoScalerProfileMaxTotalUnreadyPercentage + 'new-pod-scale-up-delay': autoScalerProfileNewPodScaleUpDelay + 'ok-total-unready-count': autoScalerProfileOkTotalUnreadyCount 'scale-down-delay-after-add': autoScalerProfileScaleDownDelayAfterAdd 'scale-down-delay-after-delete': autoScalerProfileScaleDownDelayAfterDelete 'scale-down-delay-after-failure': autoScalerProfileScaleDownDelayAfterFailure 'scale-down-unneeded-time': autoScalerProfileScaleDownUnneededTime 'scale-down-unready-time': autoScalerProfileScaleDownUnreadyTime 'scale-down-utilization-threshold': autoScalerProfileUtilizationThreshold - 'max-graceful-termination-sec': autoScalerProfileMaxGracefulTerminationSec + 'scan-interval': autoScalerProfileScanInterval + 'skip-nodes-with-local-storage': autoScalerProfileSkipNodesWithLocalStorage + 'skip-nodes-with-system-pods': autoScalerProfileSkipNodesWithSystemPods } apiServerAccessProfile: { - enablePrivateCluster: aksClusterEnablePrivateCluster + authorizedIPRanges: authorizedIPRanges + disableRunCommand: disableRunCommand + enablePrivateCluster: enablePrivateCluster + enablePrivateClusterPublicFQDN: enablePrivateClusterPublicFQDN + privateDNSZone: usePrivateDNSZone ? 'system' : '' + } + podIdentityProfile: { + allowNetworkPluginKubenet: podIdentityProfileAllowNetworkPluginKubenet + enabled: podIdentityProfileEnable + userAssignedIdentities: podIdentityProfileUserAssignedIdentities + userAssignedIdentityExceptions: podIdentityProfileUserAssignedIdentityExceptions } } } @@ -421,7 +529,13 @@ output resourceGroupName string = resourceGroup().name output name string = managedCluster.name @description('The control plane FQDN of the managed cluster') -output controlPlaneFQDN string = (aksClusterEnablePrivateCluster ? managedCluster.properties.privateFQDN : managedCluster.properties.fqdn) +output controlPlaneFQDN string = enablePrivateCluster ? managedCluster.properties.privateFQDN : managedCluster.properties.fqdn @description('The principal ID of the system assigned identity.') output systemAssignedPrincipalId string = systemAssignedIdentity && contains(managedCluster.identity, 'principalId') ? managedCluster.identity.principalId : '' + +@description('The Object ID of the AKS identity.') +output kubeletidentityObjectId string = contains(managedCluster.properties, 'identityProfile') ? contains(managedCluster.properties.identityProfile, 'kubeletidentity') ? managedCluster.properties.identityProfile.kubeletidentity.objectId : '' : '' + +@description('The Object ID of the OMS agent identity.') +output omsagentIdentityObjectId string = contains(managedCluster.properties, 'addonProfiles') ? contains(managedCluster.properties.addonProfiles, 'omsagent') ? contains(managedCluster.properties.addonProfiles.omsagent, 'identity') ? managedCluster.properties.addonProfiles.omsagent.identity.objectId : '' : '' : '' diff --git a/arm/Microsoft.ContainerService/managedClusters/readme.md b/arm/Microsoft.ContainerService/managedClusters/readme.md index e2755effe2..c2055cc157 100644 --- a/arm/Microsoft.ContainerService/managedClusters/readme.md +++ b/arm/Microsoft.ContainerService/managedClusters/readme.md @@ -23,13 +23,12 @@ This module deploys Azure Kubernetes Cluster (AKS). | `aadProfileServerAppID` | string | | | Optional. The server AAD application ID. | | `aadProfileServerAppSecret` | string | | | Optional. The server AAD application secret. | | `aadProfileTenantId` | string | `[subscription().tenantId]` | | Optional. Specifies the tenant ID of the Azure Active Directory used by the AKS cluster for authentication. | -| `aciConnectorLinuxEnabled` | bool | | | Optional. Specifies whether the aciConnectorLinux add-on is enabled or not. | +| `aciConnectorLinuxEnabled` | bool | `False` | | Optional. Specifies whether the aciConnectorLinux add-on is enabled or not. | | `agentPools` | _[agentPools](agentPools/readme.md)_ array | `[]` | | Optional. Define one or more secondary/additional agent pools | | `aksClusterAdminUsername` | string | `azureuser` | | Optional. Specifies the administrator username of Linux virtual machines. | | `aksClusterDnsPrefix` | string | `[parameters('name')]` | | Optional. Specifies the DNS prefix specified when creating the managed cluster. | | `aksClusterDnsServiceIP` | string | | | Optional. Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. | | `aksClusterDockerBridgeCidr` | string | | | Optional. Specifies the CIDR notation IP range assigned to the Docker bridge network. It must not overlap with any Subnet IP ranges or the Kubernetes service address range. | -| `aksClusterEnablePrivateCluster` | bool | | | Optional. Specifies whether to create the cluster as a private cluster or not. | | `aksClusterKubernetesVersion` | string | | | Optional. Version of Kubernetes specified when creating the managed cluster. | | `aksClusterLoadBalancerSku` | string | `standard` | `[basic, standard]` | Optional. Specifies the sku of the load balancer used by the virtual machine scale sets used by nodepools. | | `aksClusterNetworkPlugin` | string | | `[, azure, kubenet]` | Optional. Specifies the network plugin used for building Kubernetes network. - azure or kubenet. | @@ -40,13 +39,23 @@ This module deploys Azure Kubernetes Cluster (AKS). | `aksClusterSkuTier` | string | `Free` | `[Free, Paid]` | Optional. Tier of a managed cluster SKU. - Free or Paid | | `aksClusterSshPublicKey` | string | | | Optional. Specifies the SSH RSA public key string for the Linux nodes. | | `aksServicePrincipalProfile` | object | `{object}` | | Optional. Information about a service principal identity for the cluster to use for manipulating Azure APIs. | +| `authorizedIPRanges` | array | `[]` | | Optional. IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer. | +| `autoScalerProfileBalanceSimilarNodeGroups` | string | `false` | `[false, true]` | Optional. Specifies the balance of similar node groups for the auto-scaler of the AKS cluster. | +| `autoScalerProfileExpander` | string | `random` | `[least-waste, most-pods, priority, random]` | Optional. Specifies the expand strategy for the auto-scaler of the AKS cluster. | +| `autoScalerProfileMaxEmptyBulkDelete` | string | `10` | | Optional. Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster. | | `autoScalerProfileMaxGracefulTerminationSec` | string | `600` | | Optional. Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster. | +| `autoScalerProfileMaxNodeProvisionTime` | string | `15m` | | Optional. Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an "m". No unit of time other than minutes (m) is supported. | +| `autoScalerProfileMaxTotalUnreadyPercentage` | string | `45` | | Optional. Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0. | +| `autoScalerProfileNewPodScaleUpDelay` | string | `0s` | | Optional. For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit ("s" for seconds, "m" for minutes, "h" for hours, etc). | +| `autoScalerProfileOkTotalUnreadyCount` | string | `3` | | Optional. Specifies the ok total unready count for the auto-scaler of the AKS cluster. | | `autoScalerProfileScaleDownDelayAfterAdd` | string | `10m` | | Optional. Specifies the scale down delay after add of the auto-scaler of the AKS cluster. | | `autoScalerProfileScaleDownDelayAfterDelete` | string | `20s` | | Optional. Specifies the scale down delay after delete of the auto-scaler of the AKS cluster. | | `autoScalerProfileScaleDownDelayAfterFailure` | string | `3m` | | Optional. Specifies scale down delay after failure of the auto-scaler of the AKS cluster. | | `autoScalerProfileScaleDownUnneededTime` | string | `10m` | | Optional. Specifies the scale down unneeded time of the auto-scaler of the AKS cluster. | | `autoScalerProfileScaleDownUnreadyTime` | string | `20m` | | Optional. Specifies the scale down unready time of the auto-scaler of the AKS cluster. | | `autoScalerProfileScanInterval` | string | `10s` | | Optional. Specifies the scan interval of the auto-scaler of the AKS cluster. | +| `autoScalerProfileSkipNodesWithLocalStorage` | string | `true` | `[false, true]` | Optional. Specifies if nodes with local storage should be skipped for the auto-scaler of the AKS cluster. | +| `autoScalerProfileSkipNodesWithSystemPods` | string | `true` | `[false, true]` | Optional. Specifies if nodes with system pods should be skipped for the auto-scaler of the AKS cluster. | | `autoScalerProfileUtilizationThreshold` | string | `0.5` | | Optional. Specifies the utilization threshold of the auto-scaler of the AKS cluster. | | `azurePolicyEnabled` | bool | `True` | | Optional. Specifies whether the azurepolicy add-on is enabled or not. | | `azurePolicyVersion` | string | `v2` | | Optional. Specifies the azure policy version to use. | @@ -56,21 +65,32 @@ This module deploys Azure Kubernetes Cluster (AKS). | `diagnosticLogsRetentionInDays` | int | `365` | | Optional. Specifies the number of days that logs will be kept for; a value of 0 will retain data indefinitely. | | `diagnosticStorageAccountId` | string | | | Optional. Resource ID of the diagnostic storage account. | | `diagnosticWorkspaceId` | string | | | Optional. Resource ID of the diagnostic log analytics workspace. | -| `httpApplicationRoutingEnabled` | bool | | | Optional. Specifies whether the httpApplicationRouting add-on is enabled or not. | -| `kubeDashboardEnabled` | bool | | | Optional. Specifies whether the kubeDashboard add-on is enabled or not. | +| `disableLocalAccounts` | bool | `False` | | Optional. If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled. | +| `disableRunCommand` | bool | `False` | | Optional. Whether to disable run command for the cluster or not. | +| `enableKeyvaultSecretsProvider` | bool | `False` | | Optional. Specifies whether the KeyvaultSecretsProvider add-on is enabled or not. | +| `enablePrivateCluster` | bool | `False` | | Optional. Specifies whether to create the cluster as a private cluster or not. | +| `enablePrivateClusterPublicFQDN` | bool | `False` | | Optional. Whether to create additional public FQDN for private cluster or not. | +| `enableSecretRotation` | string | `false` | `[false, true]` | Optional. Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation. | +| `httpApplicationRoutingEnabled` | bool | `False` | | Optional. Specifies whether the httpApplicationRouting add-on is enabled or not. | +| `kubeDashboardEnabled` | bool | `False` | | Optional. Specifies whether the kubeDashboard add-on is enabled or not. | | `location` | string | `[resourceGroup().location]` | | Optional. Specifies the location of AKS cluster. It picks up Resource Group's location by default. | | `lock` | string | `NotSpecified` | `[CanNotDelete, NotSpecified, ReadOnly]` | Optional. Specify the type of lock. | | `logsToEnable` | array | `[kube-apiserver, kube-audit, kube-controller-manager, kube-scheduler, cluster-autoscaler]` | `[kube-apiserver, kube-audit, kube-controller-manager, kube-scheduler, cluster-autoscaler]` | Optional. The name of logs that will be streamed. | -| `managedOutboundIPCount` | int | | | Optional. Outbound IP Count for the Load balancer. | +| `managedOutboundIPCount` | int | `0` | | Optional. Outbound IP Count for the Load balancer. | | `metricsToEnable` | array | `[AllMetrics]` | `[AllMetrics]` | Optional. The name of metrics that will be streamed. | | `monitoringWorkspaceId` | string | | | Optional. Resource ID of the monitoring log analytics workspace. | | `name` | string | | | Required. Specifies the name of the AKS cluster. | | `nodeResourceGroup` | string | `[format('{0}_aks_{1}_nodes', resourceGroup().name, parameters('name'))]` | | Optional. Name of the resource group containing agent pool nodes. | | `omsAgentEnabled` | bool | `True` | | Optional. Specifies whether the OMS agent is enabled. | +| `podIdentityProfileAllowNetworkPluginKubenet` | bool | `False` | | Optional. Running in Kubenet is disabled by default due to the security related nature of AAD Pod Identity and the risks of IP spoofing. | +| `podIdentityProfileEnable` | bool | `False` | | Optional. Whether the pod identity addon is enabled. | +| `podIdentityProfileUserAssignedIdentities` | array | `[]` | | Optional. The pod identities to use in the cluster. | +| `podIdentityProfileUserAssignedIdentityExceptions` | array | `[]` | | Optional. The pod identity exceptions to allow. | | `primaryAgentPoolProfile` | array | | | Required. Properties of the primary agent pool. | | `roleAssignments` | array | `[]` | | Optional. Array of role assignment objects that contain the 'roleDefinitionIdOrName' and 'principalId' to define RBAC role assignments on this resource. In the roleDefinitionIdOrName attribute, you can provide either the display name of the role definition, or its fully qualified ID in the following format: '/providers/Microsoft.Authorization/roleDefinitions/c2f4ef07-c644-48eb-af81-4b1b4947fb11' | -| `systemAssignedIdentity` | bool | | | Optional. Enables system assigned managed identity on the resource. | +| `systemAssignedIdentity` | bool | `False` | | Optional. Enables system assigned managed identity on the resource. | | `tags` | object | `{object}` | | Optional. Tags of the resource. | +| `usePrivateDNSZone` | bool | `False` | | Optional. If AKS will create a Private DNS Zone in the Node Resource Group. | | `userAssignedIdentities` | object | `{object}` | | Optional. The ID(s) to assign to the resource. | ### Parameter Usage: `roleAssignments` @@ -200,7 +220,9 @@ You can specify multiple user assigned identities to a resource by providing add | Output Name | Type | Description | | :-- | :-- | :-- | | `controlPlaneFQDN` | string | The control plane FQDN of the managed cluster | +| `kubeletidentityObjectId` | string | The Object ID of the AKS identity. | | `name` | string | The name of the managed cluster | +| `omsagentIdentityObjectId` | string | The Object ID of the OMS agent identity. | | `resourceGroupName` | string | The resource group the managed cluster was deployed into | | `resourceId` | string | The resource ID of the managed cluster | | `systemAssignedPrincipalId` | string | The principal ID of the system assigned identity. |