From f7ef080b7a30e5fcb76a9dbd05c82f349027856d Mon Sep 17 00:00:00 2001 From: Asad Arif Date: Thu, 16 Nov 2023 15:33:29 +0000 Subject: [PATCH 1/8] Initial commit --- .../.test/azure/dependencies.bicep | 20 ++++++++++++ .../managed-cluster/main.bicep | 12 +++++++ .../managed-cluster/main.json | 31 ++++++++++++++----- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/modules/container-service/managed-cluster/.test/azure/dependencies.bicep b/modules/container-service/managed-cluster/.test/azure/dependencies.bicep index 1cdf9b765a..f1ca2663b9 100644 --- a/modules/container-service/managed-cluster/.test/azure/dependencies.bicep +++ b/modules/container-service/managed-cluster/.test/azure/dependencies.bicep @@ -79,6 +79,13 @@ resource keyVault 'Microsoft.KeyVault/vaults@2022-11-01' = { kty: 'RSA' } } + + resource kmskey 'keys@2022-07-01' = { + name: 'kmsEncryptionKey' + properties: { + kty: 'RSA' + } + } } resource diskEncryptionSet 'Microsoft.Compute/diskEncryptionSets@2022-07-02' = { @@ -98,6 +105,16 @@ resource diskEncryptionSet 'Microsoft.Compute/diskEncryptionSets@2022-07-02' = { } } +resource keyPermissionsKeyVaultCryptoUser 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-KeyVault-Crypto-User-RoleAssignment') + scope: keyVault + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') // KeyVault-Crypto-User + principalType: 'ServicePrincipal' + } +} + resource keyPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = { name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-KeyVault-Key-Read-RoleAssignment') scope: keyVault @@ -160,3 +177,6 @@ output dnsZoneResourceId string = dnsZone.id @description('The resource ID of the created Log Analytics Workspace.') output logAnalyticsWorkspaceResourceId string = logAnalyticsWorkspace.id + +@description('The uri including version of the KMS Key.') +output keyUriWithVersion string = keyVault::kmskey.properties.keyUriWithVersion diff --git a/modules/container-service/managed-cluster/main.bicep b/modules/container-service/managed-cluster/main.bicep index fc2de0e96b..d9a4a713f6 100644 --- a/modules/container-service/managed-cluster/main.bicep +++ b/modules/container-service/managed-cluster/main.bicep @@ -387,6 +387,12 @@ param diagnosticMetricsToEnable array = [ @description('Optional. The name of the diagnostic setting, if deployed. If left empty, it defaults to "-diagnosticSettings".') param diagnosticSettingsName string = '' +@description('Optional. Whether to enable Key Management Service.') +param enableAzureKeyVaultKms bool = false + +@description('Conditional. Object that contains the \'keyId\', \'keyVaultNetworkAccess\' and \'keyVaultResourceId\' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true.') +param keyVaultKms object = {} + var diagnosticsLogsSpecified = [for category in filter(diagnosticLogCategoriesToEnable, item => item != 'allLogs' && item != ''): { category: category enabled: true @@ -596,6 +602,12 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2023-07-02-p userAssignedIdentityExceptions: podIdentityProfileUserAssignedIdentityExceptions } securityProfile: { + azureKeyVaultKms: enableAzureKeyVaultKms ? { + enabled: enableAzureKeyVaultKms + keyId: keyVaultKms.keyId + keyVaultNetworkAccess: keyVaultKms.keyVaultNetworkAccess + keyVaultResourceId: keyVaultKms.keyVaultNetworkAccess == 'Private' ? keyVaultKms.keyVaultResourceId : null + } : null defender: enableAzureDefender ? { securityMonitoring: { enabled: enableAzureDefender diff --git a/modules/container-service/managed-cluster/main.json b/modules/container-service/managed-cluster/main.json index 9923e70e43..0b9e41eb24 100644 --- a/modules/container-service/managed-cluster/main.json +++ b/modules/container-service/managed-cluster/main.json @@ -5,8 +5,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.22.6.54827", - "templateHash": "9286702996832369711" + "version": "0.21.1.54444", + "templateHash": "5758086513280284891" }, "name": "Azure Kubernetes Service (AKS) Managed Clusters", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.", @@ -878,6 +878,20 @@ "metadata": { "description": "Optional. The name of the diagnostic setting, if deployed. If left empty, it defaults to \"-diagnosticSettings\"." } + }, + "enableAzureKeyVaultKms": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable Key Management Service." + } + }, + "keyVaultKms": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Conditional. Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true." + } } }, "variables": { @@ -1079,6 +1093,7 @@ "userAssignedIdentityExceptions": "[parameters('podIdentityProfileUserAssignedIdentityExceptions')]" }, "securityProfile": { + "azureKeyVaultKms": "[if(parameters('enableAzureKeyVaultKms'), createObject('enabled', parameters('enableAzureKeyVaultKms'), 'keyId', parameters('keyVaultKms').keyId, 'keyVaultNetworkAccess', parameters('keyVaultKms').keyVaultNetworkAccess, 'keyVaultResourceId', if(equals(parameters('keyVaultKms').keyVaultNetworkAccess, 'Private'), parameters('keyVaultKms').keyVaultResourceId, null())), null())]", "defender": "[if(parameters('enableAzureDefender'), createObject('securityMonitoring', createObject('enabled', parameters('enableAzureDefender')), 'logAnalyticsWorkspaceResourceId', if(not(empty(parameters('monitoringWorkspaceId'))), parameters('monitoringWorkspaceId'), null())), null())]", "workloadIdentity": "[if(parameters('enableWorkloadIdentity'), createObject('enabled', parameters('enableWorkloadIdentity')), null())]" }, @@ -1240,8 +1255,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.22.6.54827", - "templateHash": "14295298572292657386" + "version": "0.21.1.54444", + "templateHash": "4770361784914879415" }, "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", @@ -1683,8 +1698,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.22.6.54827", - "templateHash": "5002606439705018990" + "version": "0.21.1.54444", + "templateHash": "14913275975998013893" }, "name": "Kubernetes Configuration Extensions", "description": "This module deploys a Kubernetes Configuration Extension.", @@ -1846,8 +1861,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.22.6.54827", - "templateHash": "6686104224333946371" + "version": "0.21.1.54444", + "templateHash": "11648869363176032755" }, "name": "Kubernetes Configuration Flux Configurations", "description": "This module deploys a Kubernetes Configuration Flux Configuration.", From c98e55e20aec6a32447b7b09e05065c8a635c978 Mon Sep 17 00:00:00 2001 From: aadev1 Date: Thu, 16 Nov 2023 16:33:36 +0000 Subject: [PATCH 2/8] Update readme --- .../container-service/managed-cluster/README.md | 16 ++++++++++++++++ .../container-service/managed-cluster/main.json | 16 ++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/modules/container-service/managed-cluster/README.md b/modules/container-service/managed-cluster/README.md index 81b0ac0576..cd273e0f9b 100644 --- a/modules/container-service/managed-cluster/README.md +++ b/modules/container-service/managed-cluster/README.md @@ -1073,6 +1073,7 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' | :-- | :-- | :-- | | [`aksServicePrincipalProfile`](#parameter-aksserviceprincipalprofile) | object | Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster. | | [`appGatewayResourceId`](#parameter-appgatewayresourceid) | string | Specifies the resource ID of connected application gateway. Required if `ingressApplicationGatewayEnabled` is set to `true`. | +| [`keyVaultKms`](#parameter-keyvaultkms) | object | Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true. | **Optional parameters** @@ -1123,6 +1124,7 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' | [`dnsServiceIP`](#parameter-dnsserviceip) | string | Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. | | [`dnsZoneResourceId`](#parameter-dnszoneresourceid) | string | Specifies the resource ID of connected DNS zone. It will be ignored if `webApplicationRoutingEnabled` is set to `false`. | | [`enableAzureDefender`](#parameter-enableazuredefender) | bool | Whether to enable Azure Defender. | +| [`enableAzureKeyVaultKms`](#parameter-enableazurekeyvaultkms) | bool | Whether to enable Key Management Service. | | [`enableDefaultTelemetry`](#parameter-enabledefaulttelemetry) | bool | Enable telemetry via a Globally Unique Identifier (GUID). | | [`enableDnsZoneContributorRoleAssignment`](#parameter-enablednszonecontributorroleassignment) | bool | Specifies whether assing the DNS zone contributor role to the cluster service principal. It will be ignored if `webApplicationRoutingEnabled` is set to `false` or `dnsZoneResourceId` not provided. | | [`enableKeyvaultSecretsProvider`](#parameter-enablekeyvaultsecretsprovider) | bool | Specifies whether the KeyvaultSecretsProvider add-on is enabled or not. | @@ -1510,6 +1512,13 @@ Whether to enable Azure Defender. - Type: bool - Default: `False` +### Parameter: `enableAzureKeyVaultKms` + +Whether to enable Key Management Service. +- Required: No +- Type: bool +- Default: `False` + ### Parameter: `enableDefaultTelemetry` Enable telemetry via a Globally Unique Identifier (GUID). @@ -1651,6 +1660,13 @@ Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. - Type: bool - Default: `False` +### Parameter: `keyVaultKms` + +Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true. +- Required: No +- Type: object +- Default: `{object}` + ### Parameter: `kubeDashboardEnabled` Specifies whether the kubeDashboard add-on is enabled or not. diff --git a/modules/container-service/managed-cluster/main.json b/modules/container-service/managed-cluster/main.json index 0b9e41eb24..f97abd76a0 100644 --- a/modules/container-service/managed-cluster/main.json +++ b/modules/container-service/managed-cluster/main.json @@ -5,8 +5,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.21.1.54444", - "templateHash": "5758086513280284891" + "version": "0.23.1.45101", + "templateHash": "9620688851153413997" }, "name": "Azure Kubernetes Service (AKS) Managed Clusters", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.", @@ -1255,8 +1255,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.21.1.54444", - "templateHash": "4770361784914879415" + "version": "0.23.1.45101", + "templateHash": "5488981271275128668" }, "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", @@ -1698,8 +1698,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.21.1.54444", - "templateHash": "14913275975998013893" + "version": "0.23.1.45101", + "templateHash": "18265527122738367400" }, "name": "Kubernetes Configuration Extensions", "description": "This module deploys a Kubernetes Configuration Extension.", @@ -1861,8 +1861,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.21.1.54444", - "templateHash": "11648869363176032755" + "version": "0.23.1.45101", + "templateHash": "8985718648814286209" }, "name": "Kubernetes Configuration Flux Configurations", "description": "This module deploys a Kubernetes Configuration Flux Configuration.", From a4b393cf99351740e4a759012f84c54448a41d03 Mon Sep 17 00:00:00 2001 From: Asad Arif Date: Thu, 16 Nov 2023 19:56:22 +0000 Subject: [PATCH 3/8] add Enable KMS in Azure test --- .../.test/azure/main.test.bicep | 517 +++++++++--------- 1 file changed, 261 insertions(+), 256 deletions(-) diff --git a/modules/container-service/managed-cluster/.test/azure/main.test.bicep b/modules/container-service/managed-cluster/.test/azure/main.test.bicep index 35a7bc0355..da74e93b7b 100644 --- a/modules/container-service/managed-cluster/.test/azure/main.test.bicep +++ b/modules/container-service/managed-cluster/.test/azure/main.test.bicep @@ -1,256 +1,261 @@ -targetScope = 'subscription' - -// ========== // -// Parameters // -// ========== // - -@description('Optional. The name of the resource group to deploy for testing purposes.') -@maxLength(90) -param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceShort}-rg' - -@description('Optional. The location to deploy resources to.') -param location string = deployment().location - -@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') -param serviceShort string = 'csmaz' - -@description('Generated. Used as a basis for unique resource names.') -param baseTime string = utcNow('u') - -@description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') -param enableDefaultTelemetry bool = true - -@description('Optional. A token to inject into the name of each resource.') -param namePrefix string = '[[namePrefix]]' - -// ============ // -// Dependencies // -// ============ // - -// General resources -// ================= -resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { - name: resourceGroupName - location: location -} - -module nestedDependencies 'dependencies.bicep' = { - scope: resourceGroup - name: '${uniqueString(deployment().name, location)}-nestedDependencies' - params: { - virtualNetworkName: 'dep-${namePrefix}-vnet-${serviceShort}' - managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}' - managedIdentityKubeletIdentityName: 'dep-${namePrefix}-msiki-${serviceShort}' - diskEncryptionSetName: 'dep-${namePrefix}-des-${serviceShort}' - proximityPlacementGroupName: 'dep-${namePrefix}-ppg-${serviceShort}' - // Adding base time to make the name unique as purge protection must be enabled (but may not be longer than 24 characters total) - keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-${substring(uniqueString(baseTime), 0, 3)}' - dnsZoneName: 'dep-${namePrefix}-dns-${serviceShort}.com' - logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' - } -} - -// Diagnostics -// =========== -module diagnosticDependencies '../../../../.shared/.templates/diagnostic.dependencies.bicep' = { - scope: resourceGroup - name: '${uniqueString(deployment().name, location)}-diagnosticDependencies' - params: { - storageAccountName: 'dep${namePrefix}diasa${serviceShort}01' - logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' - eventHubNamespaceEventHubName: 'dep-${namePrefix}-evh-${serviceShort}' - eventHubNamespaceName: 'dep-${namePrefix}-evhns-${serviceShort}' - location: location - } -} - -// ============== // -// Test Execution // -// ============== // - -module testDeployment '../../main.bicep' = { - scope: resourceGroup - name: '${uniqueString(deployment().name, location)}-test-${serviceShort}' - params: { - enableDefaultTelemetry: enableDefaultTelemetry - name: '${namePrefix}${serviceShort}001' - primaryAgentPoolProfile: [ - { - availabilityZones: [ - '3' - ] - count: 1 - enableAutoScaling: true - maxCount: 3 - maxPods: 30 - minCount: 1 - mode: 'System' - name: 'systempool' - osDiskSizeGB: 0 - osType: 'Linux' - serviceCidr: '' - storageProfile: 'ManagedDisks' - type: 'VirtualMachineScaleSets' - vmSize: 'Standard_DS2_v2' - vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[0] - } - ] - agentPools: [ - { - availabilityZones: [ - '3' - ] - count: 2 - enableAutoScaling: true - maxCount: 3 - maxPods: 30 - minCount: 1 - minPods: 2 - mode: 'User' - name: 'userpool1' - nodeLabels: {} - nodeTaints: [ - 'CriticalAddonsOnly=true:NoSchedule' - ] - osDiskSizeGB: 128 - osType: 'Linux' - scaleSetEvictionPolicy: 'Delete' - scaleSetPriority: 'Regular' - storageProfile: 'ManagedDisks' - type: 'VirtualMachineScaleSets' - vmSize: 'Standard_DS2_v2' - vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[1] - proximityPlacementGroupResourceId: nestedDependencies.outputs.proximityPlacementGroupResourceId - } - { - availabilityZones: [ - '3' - ] - count: 2 - enableAutoScaling: true - maxCount: 3 - maxPods: 30 - minCount: 1 - minPods: 2 - mode: 'User' - name: 'userpool2' - nodeLabels: {} - nodeTaints: [ - 'CriticalAddonsOnly=true:NoSchedule' - ] - osDiskSizeGB: 128 - osType: 'Linux' - scaleSetEvictionPolicy: 'Delete' - scaleSetPriority: 'Regular' - storageProfile: 'ManagedDisks' - type: 'VirtualMachineScaleSets' - vmSize: 'Standard_DS2_v2' - vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[2] - } - ] - autoUpgradeProfileUpgradeChannel: 'stable' - enableWorkloadIdentity: true - enableOidcIssuerProfile: true - networkPlugin: 'azure' - networkDataplane: 'azure' - networkPluginMode: 'overlay' - diagnosticStorageAccountId: diagnosticDependencies.outputs.storageAccountResourceId - diagnosticWorkspaceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId - diagnosticEventHubAuthorizationRuleId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId - diagnosticEventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName - diskEncryptionSetID: nestedDependencies.outputs.diskEncryptionSetResourceId - openServiceMeshEnabled: true - enableStorageProfileBlobCSIDriver: true - enableStorageProfileDiskCSIDriver: true - enableStorageProfileFileCSIDriver: true - enableStorageProfileSnapshotController: true - userAssignedIdentities: { - '${nestedDependencies.outputs.managedIdentityResourceId}': {} - } - identityProfile: { - kubeletidentity: { - resourceId: nestedDependencies.outputs.managedIdentityKubeletIdentityResourceId - } - } - omsAgentEnabled: true - monitoringWorkspaceId: nestedDependencies.outputs.logAnalyticsWorkspaceResourceId - enableAzureDefender: true - enableKeyvaultSecretsProvider: true - enablePodSecurityPolicy: false - lock: { - kind: 'CanNotDelete' - name: 'myCustomLockName' - } - roleAssignments: [ - { - roleDefinitionIdOrName: 'Reader' - principalId: nestedDependencies.outputs.managedIdentityPrincipalId - principalType: 'ServicePrincipal' - } - ] - tags: { - 'hidden-title': 'This is visible in the resource name' - Environment: 'Non-Prod' - Role: 'DeploymentValidation' - } - fluxExtension: { - configurationSettings: { - 'helm-controller.enabled': 'true' - 'source-controller.enabled': 'true' - 'kustomize-controller.enabled': 'true' - 'notification-controller.enabled': 'true' - 'image-automation-controller.enabled': 'false' - 'image-reflector-controller.enabled': 'false' - } - configurations: [ - { - namespace: 'flux-system' - scope: 'cluster' - gitRepository: { - repositoryRef: { - branch: 'main' - } - sshKnownHosts: '' - syncIntervalInSeconds: 300 - timeoutInSeconds: 180 - url: 'https://github.com/mspnp/aks-baseline' - } - } - { - namespace: 'flux-system-helm' - scope: 'cluster' - gitRepository: { - repositoryRef: { - branch: 'main' - } - sshKnownHosts: '' - syncIntervalInSeconds: 300 - timeoutInSeconds: 180 - url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' - } - kustomizations: { - infra: { - path: './infrastructure' - dependsOn: [] - timeoutInSeconds: 600 - syncIntervalInSeconds: 600 - validation: 'none' - prune: true - } - apps: { - path: './apps/staging' - dependsOn: [ - 'infra' - ] - timeoutInSeconds: 600 - syncIntervalInSeconds: 600 - retryIntervalInSeconds: 120 - prune: true - } - } - } - ] - } - } -} - +targetScope = 'subscription' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmaz' + +@description('Generated. Used as a basis for unique resource names.') +param baseTime string = utcNow('u') + +@description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') +param enableDefaultTelemetry bool = true + +@description('Optional. A token to inject into the name of each resource.') +param namePrefix string = '[[namePrefix]]' + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: resourceGroupName + location: location +} + +module nestedDependencies 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-nestedDependencies' + params: { + virtualNetworkName: 'dep-${namePrefix}-vnet-${serviceShort}' + managedIdentityName: 'dep-${namePrefix}-msi-${serviceShort}' + managedIdentityKubeletIdentityName: 'dep-${namePrefix}-msiki-${serviceShort}' + diskEncryptionSetName: 'dep-${namePrefix}-des-${serviceShort}' + proximityPlacementGroupName: 'dep-${namePrefix}-ppg-${serviceShort}' + // Adding base time to make the name unique as purge protection must be enabled (but may not be longer than 24 characters total) + keyVaultName: 'dep-${namePrefix}-kv-${serviceShort}-${substring(uniqueString(baseTime), 0, 3)}' + dnsZoneName: 'dep-${namePrefix}-dns-${serviceShort}.com' + logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' + } +} + +// Diagnostics +// =========== +module diagnosticDependencies '../../../../.shared/.templates/diagnostic.dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-diagnosticDependencies' + params: { + storageAccountName: 'dep${namePrefix}diasa${serviceShort}01' + logAnalyticsWorkspaceName: 'dep-${namePrefix}-law-${serviceShort}' + eventHubNamespaceEventHubName: 'dep-${namePrefix}-evh-${serviceShort}' + eventHubNamespaceName: 'dep-${namePrefix}-evhns-${serviceShort}' + location: location + } +} + +// ============== // +// Test Execution // +// ============== // + +module testDeployment '../../main.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-test-${serviceShort}' + params: { + enableDefaultTelemetry: enableDefaultTelemetry + name: '${namePrefix}${serviceShort}001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '3' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + osDiskSizeGB: 0 + osType: 'Linux' + serviceCidr: '' + storageProfile: 'ManagedDisks' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[0] + } + ] + agentPools: [ + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + storageProfile: 'ManagedDisks' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[1] + proximityPlacementGroupResourceId: nestedDependencies.outputs.proximityPlacementGroupResourceId + } + { + availabilityZones: [ + '3' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + storageProfile: 'ManagedDisks' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[2] + } + ] + autoUpgradeProfileUpgradeChannel: 'stable' + enableWorkloadIdentity: true + enableOidcIssuerProfile: true + networkPlugin: 'azure' + networkDataplane: 'azure' + networkPluginMode: 'overlay' + diagnosticStorageAccountId: diagnosticDependencies.outputs.storageAccountResourceId + diagnosticWorkspaceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + diagnosticEventHubAuthorizationRuleId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId + diagnosticEventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName + diskEncryptionSetID: nestedDependencies.outputs.diskEncryptionSetResourceId + openServiceMeshEnabled: true + enableStorageProfileBlobCSIDriver: true + enableStorageProfileDiskCSIDriver: true + enableStorageProfileFileCSIDriver: true + enableStorageProfileSnapshotController: true + userAssignedIdentities: { + '${nestedDependencies.outputs.managedIdentityResourceId}': {} + } + identityProfile: { + kubeletidentity: { + resourceId: nestedDependencies.outputs.managedIdentityKubeletIdentityResourceId + } + } + omsAgentEnabled: true + monitoringWorkspaceId: nestedDependencies.outputs.logAnalyticsWorkspaceResourceId + enableAzureDefender: true + enableKeyvaultSecretsProvider: true + enablePodSecurityPolicy: false + enableAzureKeyVaultKms: true + keyVaultKms: { + keyId: nestedDependencies.outputs.keyUriWithVersion + keyVaultNetworkAccess: 'Public' + } + lock: { + kind: 'CanNotDelete' + name: 'myCustomLockName' + } + roleAssignments: [ + { + roleDefinitionIdOrName: 'Reader' + principalId: nestedDependencies.outputs.managedIdentityPrincipalId + + principalType: 'ServicePrincipal' + } + ] + tags: { + 'hidden-title': 'This is visible in the resource name' + Environment: 'Non-Prod' + Role: 'DeploymentValidation' + } + fluxExtension: { + configurationSettings: { + 'helm-controller.enabled': 'true' + 'source-controller.enabled': 'true' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'true' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + } + configurations: [ + { + namespace: 'flux-system' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + } + { + namespace: 'flux-system-helm' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' + } + kustomizations: { + infra: { + path: './infrastructure' + dependsOn: [] + timeoutInSeconds: 600 + syncIntervalInSeconds: 600 + validation: 'none' + prune: true + } + apps: { + path: './apps/staging' + dependsOn: [ + 'infra' + ] + timeoutInSeconds: 600 + syncIntervalInSeconds: 600 + retryIntervalInSeconds: 120 + prune: true + } + } + } + ] + } + } +} From 2870e26087a6cf27c48271379522922df1c88d0a Mon Sep 17 00:00:00 2001 From: Asad Arif Date: Thu, 16 Nov 2023 20:00:06 +0000 Subject: [PATCH 4/8] Remove accidently added blank line --- .../managed-cluster/.test/azure/main.test.bicep | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/container-service/managed-cluster/.test/azure/main.test.bicep b/modules/container-service/managed-cluster/.test/azure/main.test.bicep index da74e93b7b..1546af70c7 100644 --- a/modules/container-service/managed-cluster/.test/azure/main.test.bicep +++ b/modules/container-service/managed-cluster/.test/azure/main.test.bicep @@ -190,7 +190,6 @@ module testDeployment '../../main.bicep' = { { roleDefinitionIdOrName: 'Reader' principalId: nestedDependencies.outputs.managedIdentityPrincipalId - principalType: 'ServicePrincipal' } ] From ff7b5905bc7a88f11a2e944b0c47113826d1c280 Mon Sep 17 00:00:00 2001 From: aadev1 Date: Thu, 16 Nov 2023 20:08:13 +0000 Subject: [PATCH 5/8] Update readme --- .../container-service/managed-cluster/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/modules/container-service/managed-cluster/README.md b/modules/container-service/managed-cluster/README.md index cd273e0f9b..1e4a6cf103 100644 --- a/modules/container-service/managed-cluster/README.md +++ b/modules/container-service/managed-cluster/README.md @@ -130,6 +130,7 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' diagnosticWorkspaceId: '' diskEncryptionSetID: '' enableAzureDefender: true + enableAzureKeyVaultKms: true enableDefaultTelemetry: '' enableKeyvaultSecretsProvider: true enableOidcIssuerProfile: true @@ -200,6 +201,10 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' resourceId: '' } } + keyVaultKms: { + keyId: '' + keyVaultNetworkAccess: 'Public' + } lock: { kind: 'CanNotDelete' name: 'myCustomLockName' @@ -345,6 +350,9 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' "enableAzureDefender": { "value": true }, + "enableAzureKeyVaultKms": { + "value": true + }, "enableDefaultTelemetry": { "value": "" }, @@ -437,6 +445,12 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' } } }, + "keyVaultKms": { + "value": { + "keyId": "", + "keyVaultNetworkAccess": "Public" + } + }, "lock": { "value": { "kind": "CanNotDelete", From 21929155d106518980640a480f06844ab06919d0 Mon Sep 17 00:00:00 2001 From: Asad Arif Date: Fri, 17 Nov 2023 15:50:37 +0000 Subject: [PATCH 6/8] Rebuild main.json --- .../managed-cluster/main.bicep | 6 ++ .../managed-cluster/main.json | 92 +++++-------------- 2 files changed, 29 insertions(+), 69 deletions(-) diff --git a/modules/container-service/managed-cluster/main.bicep b/modules/container-service/managed-cluster/main.bicep index 11c52e362c..cc9fd3c16b 100644 --- a/modules/container-service/managed-cluster/main.bicep +++ b/modules/container-service/managed-cluster/main.bicep @@ -314,6 +314,12 @@ param supportPlan string = 'KubernetesOfficial' @description('Optional. The diagnostic settings of the service.') param diagnosticSettings diagnosticSettingType +@description('Optional. Whether to enable Key Management Service.') +param enableAzureKeyVaultKms bool = false + +@description('Conditional. Object that contains the \'keyId\', \'keyVaultNetworkAccess\' and \'keyVaultResourceId\' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true.') +param keyVaultKms object = {} + @description('Optional. Specifies whether the OMS agent is enabled.') param omsAgentEnabled bool = true diff --git a/modules/container-service/managed-cluster/main.json b/modules/container-service/managed-cluster/main.json index bad4e70f8c..2aad7247ee 100644 --- a/modules/container-service/managed-cluster/main.json +++ b/modules/container-service/managed-cluster/main.json @@ -5,8 +5,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.23.1.45101", - "templateHash": "10758692765653328788" + "version": "0.21.1.54444", + "templateHash": "4058831928600365880" }, "name": "Azure Kubernetes Service (AKS) Managed Clusters", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.", @@ -864,6 +864,20 @@ "description": "Optional. The diagnostic settings of the service." } }, + "enableAzureKeyVaultKms": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable Key Management Service." + } + }, + "keyVaultKms": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Conditional. Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true." + } + }, "omsAgentEnabled": { "type": "bool", "defaultValue": true, @@ -938,62 +952,6 @@ "metadata": { "description": "Optional. Identities associated with the cluster." } - <<<<<<< HEAD - }, - "diagnosticLogCategoriesToEnable": { - "type": "array", - "defaultValue": [ - "allLogs" - ], - "allowedValues": [ - "", - "allLogs", - "kube-apiserver", - "kube-audit", - "kube-controller-manager", - "kube-scheduler", - "cluster-autoscaler", - "kube-audit-admin", - "guard" - ], - "metadata": { - "description": "Optional. The name of logs that will be streamed. \"allLogs\" includes all possible logs for the resource. Set to '' to disable log collection." - } - }, - "diagnosticMetricsToEnable": { - "type": "array", - "defaultValue": [ - "AllMetrics" - ], - "allowedValues": [ - "AllMetrics" - ], - "metadata": { - "description": "Optional. The name of metrics that will be streamed." - } - }, - "diagnosticSettingsName": { - "type": "string", - "defaultValue": "", - "metadata": { - "description": "Optional. The name of the diagnostic setting, if deployed. If left empty, it defaults to \"-diagnosticSettings\"." - } - }, - "enableAzureKeyVaultKms": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Optional. Whether to enable Key Management Service." - } - }, - "keyVaultKms": { - "type": "object", - "defaultValue": {}, - "metadata": { - "description": "Conditional. Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true." - } - ======= >>>>>>> - main } }, "variables": { @@ -1342,12 +1300,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.23.1.45101", - <<<<<<< HEAD - "templateHash": "5488981271275128668" - ======= - "templateHash": "13811832596066396545" - >>>>>>> main + "version": "0.21.1.54444", + "templateHash": "16583400731032415311" }, "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", @@ -1798,8 +1752,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.23.1.45101", - "templateHash": "18265527122738367400" + "version": "0.21.1.54444", + "templateHash": "14913275975998013893" }, "name": "Kubernetes Configuration Extensions", "description": "This module deploys a Kubernetes Configuration Extension.", @@ -1961,8 +1915,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.23.1.45101", - "templateHash": "8985718648814286209" + "version": "0.21.1.54444", + "templateHash": "11648869363176032755" }, "name": "Kubernetes Configuration Flux Configurations", "description": "This module deploys a Kubernetes Configuration Flux Configuration.", @@ -2235,4 +2189,4 @@ "value": "[if(contains(reference('managedCluster'), 'addonProfiles'), reference('managedCluster').addonProfiles, createObject())]" } } -} +} \ No newline at end of file From f45e34a40094d460ad8f4a2fd9d376842048e0f3 Mon Sep 17 00:00:00 2001 From: Asad Arif Date: Fri, 17 Nov 2023 15:58:52 +0000 Subject: [PATCH 7/8] Add KMS test back in --- .../tests/e2e/azure/main.test.bicep | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep b/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep index 6a080cf29b..2d835afd94 100644 --- a/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep +++ b/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep @@ -6,7 +6,7 @@ targetScope = 'subscription' @description('Optional. The name of the resource group to deploy for testing purposes.') @maxLength(90) -param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceShort}-rg' +param resourceGroupName string = 'dep-${namePrefix}-containerservice.managedclusters-${serviceShort}-rg' @description('Optional. The location to deploy resources to.') param location string = deployment().location @@ -52,7 +52,7 @@ module nestedDependencies 'dependencies.bicep' = { // Diagnostics // =========== -module diagnosticDependencies '../../../../.shared/.templates/diagnostic.dependencies.bicep' = { +module diagnosticDependencies '../../../../../.shared/.templates/diagnostic.dependencies.bicep' = { scope: resourceGroup name: '${uniqueString(deployment().name, location)}-diagnosticDependencies' params: { @@ -68,7 +68,7 @@ module diagnosticDependencies '../../../../.shared/.templates/diagnostic.depende // Test Execution // // ============== // -module testDeployment '../../main.bicep' = { +module testDeployment '../../../main.bicep' = { scope: resourceGroup name: '${uniqueString(deployment().name, location)}-test-${serviceShort}' params: { @@ -154,18 +154,30 @@ module testDeployment '../../main.bicep' = { networkPlugin: 'azure' networkDataplane: 'azure' networkPluginMode: 'overlay' - diagnosticStorageAccountId: diagnosticDependencies.outputs.storageAccountResourceId - diagnosticWorkspaceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId - diagnosticEventHubAuthorizationRuleId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId - diagnosticEventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName + diagnosticSettings: [ + { + name: 'customSetting' + metricCategories: [ + { + category: 'AllMetrics' + } + ] + eventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName + eventHubAuthorizationRuleResourceId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId + storageAccountResourceId: diagnosticDependencies.outputs.storageAccountResourceId + workspaceResourceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + } + ] diskEncryptionSetID: nestedDependencies.outputs.diskEncryptionSetResourceId openServiceMeshEnabled: true enableStorageProfileBlobCSIDriver: true enableStorageProfileDiskCSIDriver: true enableStorageProfileFileCSIDriver: true enableStorageProfileSnapshotController: true - userAssignedIdentities: { - '${nestedDependencies.outputs.managedIdentityResourceId}': {} + managedIdentities: { + userAssignedResourceIds: [ + nestedDependencies.outputs.managedIdentityResourceId + ] } identityProfile: { kubeletidentity: { @@ -177,6 +189,11 @@ module testDeployment '../../main.bicep' = { enableAzureDefender: true enableKeyvaultSecretsProvider: true enablePodSecurityPolicy: false + enableAzureKeyVaultKms: true + keyVaultKms: { + keyId: nestedDependencies.outputs.keyUriWithVersion + keyVaultNetworkAccess: 'Public' + } lock: { kind: 'CanNotDelete' name: 'myCustomLockName' @@ -185,7 +202,6 @@ module testDeployment '../../main.bicep' = { { roleDefinitionIdOrName: 'Reader' principalId: nestedDependencies.outputs.managedIdentityPrincipalId - principalType: 'ServicePrincipal' } ] From d3767efd8291b8aa0694fee0e0a1c28b69e56430 Mon Sep 17 00:00:00 2001 From: aadev1 Date: Fri, 17 Nov 2023 16:06:29 +0000 Subject: [PATCH 8/8] Update readme and generate main.json --- .../container-service/managed-cluster/README.md | 2 +- .../container-service/managed-cluster/main.json | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/container-service/managed-cluster/README.md b/modules/container-service/managed-cluster/README.md index 7972815907..f82561d20a 100644 --- a/modules/container-service/managed-cluster/README.md +++ b/modules/container-service/managed-cluster/README.md @@ -1836,7 +1836,7 @@ Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true. - Required: No - Type: object -- Default: `{object}` +- Default: `{}` ### Parameter: `kubeDashboardEnabled` diff --git a/modules/container-service/managed-cluster/main.json b/modules/container-service/managed-cluster/main.json index 2aad7247ee..7190d6b56c 100644 --- a/modules/container-service/managed-cluster/main.json +++ b/modules/container-service/managed-cluster/main.json @@ -5,8 +5,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.21.1.54444", - "templateHash": "4058831928600365880" + "version": "0.23.1.45101", + "templateHash": "1906456864170625087" }, "name": "Azure Kubernetes Service (AKS) Managed Clusters", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.", @@ -1300,8 +1300,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.21.1.54444", - "templateHash": "16583400731032415311" + "version": "0.23.1.45101", + "templateHash": "13811832596066396545" }, "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", @@ -1752,8 +1752,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.21.1.54444", - "templateHash": "14913275975998013893" + "version": "0.23.1.45101", + "templateHash": "18265527122738367400" }, "name": "Kubernetes Configuration Extensions", "description": "This module deploys a Kubernetes Configuration Extension.", @@ -1915,8 +1915,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.21.1.54444", - "templateHash": "11648869363176032755" + "version": "0.23.1.45101", + "templateHash": "8985718648814286209" }, "name": "Kubernetes Configuration Flux Configurations", "description": "This module deploys a Kubernetes Configuration Flux Configuration.",