From ab10a5168594fd108b957e2158486179ec707207 Mon Sep 17 00:00:00 2001 From: aadev1 <39670555+aadev1@users.noreply.github.com> Date: Fri, 17 Nov 2023 19:12:45 +0000 Subject: [PATCH 1/4] [Modules] Added Azure Key Vault key management service settings to Security profile (#4251) * Initial commit * Update readme * add Enable KMS in Azure test * Remove accidently added blank line * Update readme * Rebuild main.json * Add KMS test back in * Update readme and generate main.json --------- Co-authored-by: Asad Arif --- .../managed-cluster/README.md | 30 +++++++++++++++++++ .../managed-cluster/main.bicep | 12 ++++++++ .../managed-cluster/main.json | 17 ++++++++++- .../tests/e2e/azure/dependencies.bicep | 20 +++++++++++++ .../tests/e2e/azure/main.test.bicep | 5 ++++ 5 files changed, 83 insertions(+), 1 deletion(-) diff --git a/modules/container-service/managed-cluster/README.md b/modules/container-service/managed-cluster/README.md index aaf0d56ddb..f82561d20a 100644 --- a/modules/container-service/managed-cluster/README.md +++ b/modules/container-service/managed-cluster/README.md @@ -140,6 +140,7 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' ] diskEncryptionSetID: '' enableAzureDefender: true + enableAzureKeyVaultKms: true enableDefaultTelemetry: '' enableKeyvaultSecretsProvider: true enableOidcIssuerProfile: true @@ -210,6 +211,10 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' resourceId: '' } } + keyVaultKms: { + keyId: '' + keyVaultNetworkAccess: 'Public' + } lock: { kind: 'CanNotDelete' name: 'myCustomLockName' @@ -361,6 +366,9 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' "enableAzureDefender": { "value": true }, + "enableAzureKeyVaultKms": { + "value": true + }, "enableDefaultTelemetry": { "value": "" }, @@ -453,6 +461,12 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' } } }, + "keyVaultKms": { + "value": { + "keyId": "", + "keyVaultNetworkAccess": "Public" + } + }, "lock": { "value": { "kind": "CanNotDelete", @@ -1131,6 +1145,7 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' | :-- | :-- | :-- | | [`aksServicePrincipalProfile`](#parameter-aksserviceprincipalprofile) | object | Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster. | | [`appGatewayResourceId`](#parameter-appgatewayresourceid) | string | Specifies the resource ID of connected application gateway. Required if `ingressApplicationGatewayEnabled` is set to `true`. | +| [`keyVaultKms`](#parameter-keyvaultkms) | object | Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true. | **Optional parameters** @@ -1175,6 +1190,7 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' | [`dnsServiceIP`](#parameter-dnsserviceip) | string | Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. | | [`dnsZoneResourceId`](#parameter-dnszoneresourceid) | string | Specifies the resource ID of connected DNS zone. It will be ignored if `webApplicationRoutingEnabled` is set to `false`. | | [`enableAzureDefender`](#parameter-enableazuredefender) | bool | Whether to enable Azure Defender. | +| [`enableAzureKeyVaultKms`](#parameter-enableazurekeyvaultkms) | bool | Whether to enable Key Management Service. | | [`enableDefaultTelemetry`](#parameter-enabledefaulttelemetry) | bool | Enable telemetry via a Globally Unique Identifier (GUID). | | [`enableDnsZoneContributorRoleAssignment`](#parameter-enablednszonecontributorroleassignment) | bool | Specifies whether assing the DNS zone contributor role to the cluster service principal. It will be ignored if `webApplicationRoutingEnabled` is set to `false` or `dnsZoneResourceId` not provided. | | [`enableKeyvaultSecretsProvider`](#parameter-enablekeyvaultsecretsprovider) | bool | Specifies whether the KeyvaultSecretsProvider add-on is enabled or not. | @@ -1661,6 +1677,13 @@ Whether to enable Azure Defender. - Type: bool - Default: `False` +### Parameter: `enableAzureKeyVaultKms` + +Whether to enable Key Management Service. +- Required: No +- Type: bool +- Default: `False` + ### Parameter: `enableDefaultTelemetry` Enable telemetry via a Globally Unique Identifier (GUID). @@ -1808,6 +1831,13 @@ Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. - Type: bool - Default: `False` +### Parameter: `keyVaultKms` + +Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true. +- Required: No +- Type: object +- Default: `{}` + ### Parameter: `kubeDashboardEnabled` Specifies whether the kubeDashboard add-on is enabled or not. diff --git a/modules/container-service/managed-cluster/main.bicep b/modules/container-service/managed-cluster/main.bicep index efb5974f2d..cc9fd3c16b 100644 --- a/modules/container-service/managed-cluster/main.bicep +++ b/modules/container-service/managed-cluster/main.bicep @@ -314,6 +314,12 @@ param supportPlan string = 'KubernetesOfficial' @description('Optional. The diagnostic settings of the service.') param diagnosticSettings diagnosticSettingType +@description('Optional. Whether to enable Key Management Service.') +param enableAzureKeyVaultKms bool = false + +@description('Conditional. Object that contains the \'keyId\', \'keyVaultNetworkAccess\' and \'keyVaultResourceId\' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true.') +param keyVaultKms object = {} + @description('Optional. Specifies whether the OMS agent is enabled.') param omsAgentEnabled bool = true @@ -539,6 +545,12 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2023-07-02-p userAssignedIdentityExceptions: podIdentityProfileUserAssignedIdentityExceptions } securityProfile: { + azureKeyVaultKms: enableAzureKeyVaultKms ? { + enabled: enableAzureKeyVaultKms + keyId: keyVaultKms.keyId + keyVaultNetworkAccess: keyVaultKms.keyVaultNetworkAccess + keyVaultResourceId: keyVaultKms.keyVaultNetworkAccess == 'Private' ? keyVaultKms.keyVaultResourceId : null + } : null defender: enableAzureDefender ? { securityMonitoring: { enabled: enableAzureDefender diff --git a/modules/container-service/managed-cluster/main.json b/modules/container-service/managed-cluster/main.json index 55eb6b6a7c..7190d6b56c 100644 --- a/modules/container-service/managed-cluster/main.json +++ b/modules/container-service/managed-cluster/main.json @@ -6,7 +6,7 @@ "_generator": { "name": "bicep", "version": "0.23.1.45101", - "templateHash": "10758692765653328788" + "templateHash": "1906456864170625087" }, "name": "Azure Kubernetes Service (AKS) Managed Clusters", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.", @@ -864,6 +864,20 @@ "description": "Optional. The diagnostic settings of the service." } }, + "enableAzureKeyVaultKms": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Optional. Whether to enable Key Management Service." + } + }, + "keyVaultKms": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Conditional. Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true." + } + }, "omsAgentEnabled": { "type": "bool", "defaultValue": true, @@ -1116,6 +1130,7 @@ "userAssignedIdentityExceptions": "[parameters('podIdentityProfileUserAssignedIdentityExceptions')]" }, "securityProfile": { + "azureKeyVaultKms": "[if(parameters('enableAzureKeyVaultKms'), createObject('enabled', parameters('enableAzureKeyVaultKms'), 'keyId', parameters('keyVaultKms').keyId, 'keyVaultNetworkAccess', parameters('keyVaultKms').keyVaultNetworkAccess, 'keyVaultResourceId', if(equals(parameters('keyVaultKms').keyVaultNetworkAccess, 'Private'), parameters('keyVaultKms').keyVaultResourceId, null())), null())]", "defender": "[if(parameters('enableAzureDefender'), createObject('securityMonitoring', createObject('enabled', parameters('enableAzureDefender')), 'logAnalyticsWorkspaceResourceId', if(not(empty(parameters('monitoringWorkspaceId'))), parameters('monitoringWorkspaceId'), null())), null())]", "workloadIdentity": "[if(parameters('enableWorkloadIdentity'), createObject('enabled', parameters('enableWorkloadIdentity')), null())]" }, diff --git a/modules/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep b/modules/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep index 1cdf9b765a..f1ca2663b9 100644 --- a/modules/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep +++ b/modules/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep @@ -79,6 +79,13 @@ resource keyVault 'Microsoft.KeyVault/vaults@2022-11-01' = { kty: 'RSA' } } + + resource kmskey 'keys@2022-07-01' = { + name: 'kmsEncryptionKey' + properties: { + kty: 'RSA' + } + } } resource diskEncryptionSet 'Microsoft.Compute/diskEncryptionSets@2022-07-02' = { @@ -98,6 +105,16 @@ resource diskEncryptionSet 'Microsoft.Compute/diskEncryptionSets@2022-07-02' = { } } +resource keyPermissionsKeyVaultCryptoUser 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-KeyVault-Crypto-User-RoleAssignment') + scope: keyVault + properties: { + principalId: managedIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') // KeyVault-Crypto-User + principalType: 'ServicePrincipal' + } +} + resource keyPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = { name: guid('msi-${keyVault.id}-${location}-${managedIdentity.id}-KeyVault-Key-Read-RoleAssignment') scope: keyVault @@ -160,3 +177,6 @@ output dnsZoneResourceId string = dnsZone.id @description('The resource ID of the created Log Analytics Workspace.') output logAnalyticsWorkspaceResourceId string = logAnalyticsWorkspace.id + +@description('The uri including version of the KMS Key.') +output keyUriWithVersion string = keyVault::kmskey.properties.keyUriWithVersion diff --git a/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep b/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep index 7776f4752f..2d835afd94 100644 --- a/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep +++ b/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep @@ -189,6 +189,11 @@ module testDeployment '../../../main.bicep' = { enableAzureDefender: true enableKeyvaultSecretsProvider: true enablePodSecurityPolicy: false + enableAzureKeyVaultKms: true + keyVaultKms: { + keyId: nestedDependencies.outputs.keyUriWithVersion + keyVaultNetworkAccess: 'Public' + } lock: { kind: 'CanNotDelete' name: 'myCustomLockName' From 94e34469a223877ee86c453b6701de5637677e70 Mon Sep 17 00:00:00 2001 From: AlexanderSehr Date: Fri, 17 Nov 2023 20:37:56 +0100 Subject: [PATCH 2/4] Updated format to common cmk interface --- .../managed-cluster/README.md | 86 ++++++++++++------- .../managed-cluster/agent-pool/main.json | 4 +- .../managed-cluster/main.bicep | 42 ++++++--- .../managed-cluster/main.json | 85 ++++++++++++++---- .../tests/e2e/azure/dependencies.bicep | 23 +++-- .../tests/e2e/azure/main.test.bicep | 12 +-- .../tests/e2e/priv/dependencies.bicep | 16 ++-- .../tests/e2e/priv/main.test.bicep | 4 +- 8 files changed, 189 insertions(+), 83 deletions(-) diff --git a/modules/container-service/managed-cluster/README.md b/modules/container-service/managed-cluster/README.md index f82561d20a..fe444ca1da 100644 --- a/modules/container-service/managed-cluster/README.md +++ b/modules/container-service/managed-cluster/README.md @@ -124,6 +124,11 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' } ] autoUpgradeProfileUpgradeChannel: 'stable' + customerManagedKey: { + keyName: '' + keyVaultNetworkAccess: 'Public' + keyVaultResourceId: '' + } diagnosticSettings: [ { eventHubAuthorizationRuleResourceId: '' @@ -140,7 +145,6 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' ] diskEncryptionSetID: '' enableAzureDefender: true - enableAzureKeyVaultKms: true enableDefaultTelemetry: '' enableKeyvaultSecretsProvider: true enableOidcIssuerProfile: true @@ -211,10 +215,6 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' resourceId: '' } } - keyVaultKms: { - keyId: '' - keyVaultNetworkAccess: 'Public' - } lock: { kind: 'CanNotDelete' name: 'myCustomLockName' @@ -344,6 +344,13 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' "autoUpgradeProfileUpgradeChannel": { "value": "stable" }, + "customerManagedKey": { + "value": { + "keyName": "", + "keyVaultNetworkAccess": "Public", + "keyVaultResourceId": "" + } + }, "diagnosticSettings": { "value": [ { @@ -366,9 +373,6 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' "enableAzureDefender": { "value": true }, - "enableAzureKeyVaultKms": { - "value": true - }, "enableDefaultTelemetry": { "value": "" }, @@ -461,12 +465,6 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' } } }, - "keyVaultKms": { - "value": { - "keyId": "", - "keyVaultNetworkAccess": "Public" - } - }, "lock": { "value": { "kind": "CanNotDelete", @@ -1145,7 +1143,6 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' | :-- | :-- | :-- | | [`aksServicePrincipalProfile`](#parameter-aksserviceprincipalprofile) | object | Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster. | | [`appGatewayResourceId`](#parameter-appgatewayresourceid) | string | Specifies the resource ID of connected application gateway. Required if `ingressApplicationGatewayEnabled` is set to `true`. | -| [`keyVaultKms`](#parameter-keyvaultkms) | object | Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true. | **Optional parameters** @@ -1182,6 +1179,7 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' | [`autoUpgradeProfileUpgradeChannel`](#parameter-autoupgradeprofileupgradechannel) | string | Auto-upgrade channel on the AKS cluster. | | [`azurePolicyEnabled`](#parameter-azurepolicyenabled) | bool | Specifies whether the azurepolicy add-on is enabled or not. For security reasons, this setting should be enabled. | | [`azurePolicyVersion`](#parameter-azurepolicyversion) | string | Specifies the azure policy version to use. | +| [`customerManagedKey`](#parameter-customermanagedkey) | object | The customer managed key definition. | | [`diagnosticSettings`](#parameter-diagnosticsettings) | array | The diagnostic settings of the service. | | [`disableLocalAccounts`](#parameter-disablelocalaccounts) | bool | If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled. | | [`disableRunCommand`](#parameter-disableruncommand) | bool | Whether to disable run command for the cluster or not. | @@ -1190,7 +1188,6 @@ module managedCluster 'br:bicep/modules/container-service.managed-cluster:1.0.0' | [`dnsServiceIP`](#parameter-dnsserviceip) | string | Specifies the IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. | | [`dnsZoneResourceId`](#parameter-dnszoneresourceid) | string | Specifies the resource ID of connected DNS zone. It will be ignored if `webApplicationRoutingEnabled` is set to `false`. | | [`enableAzureDefender`](#parameter-enableazuredefender) | bool | Whether to enable Azure Defender. | -| [`enableAzureKeyVaultKms`](#parameter-enableazurekeyvaultkms) | bool | Whether to enable Key Management Service. | | [`enableDefaultTelemetry`](#parameter-enabledefaulttelemetry) | bool | Enable telemetry via a Globally Unique Identifier (GUID). | | [`enableDnsZoneContributorRoleAssignment`](#parameter-enablednszonecontributorroleassignment) | bool | Specifies whether assing the DNS zone contributor role to the cluster service principal. It will be ignored if `webApplicationRoutingEnabled` is set to `false` or `dnsZoneResourceId` not provided. | | [`enableKeyvaultSecretsProvider`](#parameter-enablekeyvaultsecretsprovider) | bool | Specifies whether the KeyvaultSecretsProvider add-on is enabled or not. | @@ -1513,6 +1510,49 @@ Specifies the azure policy version to use. - Type: string - Default: `'v2'` +### Parameter: `customerManagedKey` + +The customer managed key definition. +- Required: No +- Type: object + + +| Name | Required | Type | Description | +| :-- | :-- | :--| :-- | +| [`keyName`](#parameter-customermanagedkeykeyname) | Yes | string | Required. The name of the customer managed key to use for encryption. | +| [`keyVaultNetworkAccess`](#parameter-customermanagedkeykeyvaultnetworkaccess) | Yes | string | Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public. | +| [`keyVaultResourceId`](#parameter-customermanagedkeykeyvaultresourceid) | Yes | string | Required. The resource ID of a key vault to reference a customer managed key for encryption from. | +| [`keyVersion`](#parameter-customermanagedkeykeyversion) | No | string | Optional. The version of the customer managed key to reference for encryption. If not provided, using 'latest'. | + +### Parameter: `customerManagedKey.keyName` + +Required. The name of the customer managed key to use for encryption. + +- Required: Yes +- Type: string + +### Parameter: `customerManagedKey.keyVaultNetworkAccess` + +Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public. + +- Required: Yes +- Type: string +- Allowed: `[Private, Public]` + +### Parameter: `customerManagedKey.keyVaultResourceId` + +Required. The resource ID of a key vault to reference a customer managed key for encryption from. + +- Required: Yes +- Type: string + +### Parameter: `customerManagedKey.keyVersion` + +Optional. The version of the customer managed key to reference for encryption. If not provided, using 'latest'. + +- Required: No +- Type: string + ### Parameter: `diagnosticSettings` The diagnostic settings of the service. @@ -1677,13 +1717,6 @@ Whether to enable Azure Defender. - Type: bool - Default: `False` -### Parameter: `enableAzureKeyVaultKms` - -Whether to enable Key Management Service. -- Required: No -- Type: bool -- Default: `False` - ### Parameter: `enableDefaultTelemetry` Enable telemetry via a Globally Unique Identifier (GUID). @@ -1831,13 +1864,6 @@ Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. - Type: bool - Default: `False` -### Parameter: `keyVaultKms` - -Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true. -- Required: No -- Type: object -- Default: `{}` - ### Parameter: `kubeDashboardEnabled` Specifies whether the kubeDashboard add-on is enabled or not. diff --git a/modules/container-service/managed-cluster/agent-pool/main.json b/modules/container-service/managed-cluster/agent-pool/main.json index 878796aeb1..cf0f53629b 100644 --- a/modules/container-service/managed-cluster/agent-pool/main.json +++ b/modules/container-service/managed-cluster/agent-pool/main.json @@ -5,8 +5,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.22.6.54827", - "templateHash": "15823498371287518640" + "version": "0.23.1.45101", + "templateHash": "13811832596066396545" }, "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", diff --git a/modules/container-service/managed-cluster/main.bicep b/modules/container-service/managed-cluster/main.bicep index cc9fd3c16b..304a5c48e6 100644 --- a/modules/container-service/managed-cluster/main.bicep +++ b/modules/container-service/managed-cluster/main.bicep @@ -314,12 +314,6 @@ param supportPlan string = 'KubernetesOfficial' @description('Optional. The diagnostic settings of the service.') param diagnosticSettings diagnosticSettingType -@description('Optional. Whether to enable Key Management Service.') -param enableAzureKeyVaultKms bool = false - -@description('Conditional. Object that contains the \'keyId\', \'keyVaultNetworkAccess\' and \'keyVaultResourceId\' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true.') -param keyVaultKms object = {} - @description('Optional. Specifies whether the OMS agent is enabled.') param omsAgentEnabled bool = true @@ -354,6 +348,18 @@ param httpProxyConfig object = {} @description('Optional. Identities associated with the cluster.') param identityProfile object = {} +@description('Optional. The customer managed key definition.') +param customerManagedKey customerManagedKeyType + +resource cMKKeyVault 'Microsoft.KeyVault/vaults@2023-02-01' existing = if (!empty(customerManagedKey.?keyVaultResourceId)) { + name: last(split((customerManagedKey.?keyVaultResourceId ?? 'dummyVault'), '/')) + scope: resourceGroup(split((customerManagedKey.?keyVaultResourceId ?? '//'), '/')[2], split((customerManagedKey.?keyVaultResourceId ?? '////'), '/')[4]) + + resource cMKKey 'keys@2023-02-01' existing = if (!empty(customerManagedKey.?keyVaultResourceId) && !empty(customerManagedKey.?keyName)) { + name: customerManagedKey.?keyName ?? 'dummyKey' + } +} + var formattedUserAssignedIdentities = reduce(map((managedIdentities.?userAssignedResourceIds ?? []), (id) => { '${id}': {} }), {}, (cur, next) => union(cur, next)) // Converts the flat array to an object like { '${id1}': {}, '${id2}': {} } var identity = !empty(managedIdentities) ? { @@ -545,11 +551,11 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2023-07-02-p userAssignedIdentityExceptions: podIdentityProfileUserAssignedIdentityExceptions } securityProfile: { - azureKeyVaultKms: enableAzureKeyVaultKms ? { - enabled: enableAzureKeyVaultKms - keyId: keyVaultKms.keyId - keyVaultNetworkAccess: keyVaultKms.keyVaultNetworkAccess - keyVaultResourceId: keyVaultKms.keyVaultNetworkAccess == 'Private' ? keyVaultKms.keyVaultResourceId : null + azureKeyVaultKms: !empty(customerManagedKey) ? { + enabled: true + keyId: !empty(customerManagedKey.?keyVersion ?? '') ? '${cMKKeyVault::cMKKey.properties.keyUri}/${customerManagedKey!.keyVersion}' : cMKKeyVault::cMKKey.properties.keyUriWithVersion + keyVaultNetworkAccess: customerManagedKey!.keyVaultNetworkAccess + keyVaultResourceId: customerManagedKey!.keyVaultNetworkAccess == 'Private' ? cMKKeyVault.id : null } : null defender: enableAzureDefender ? { securityMonitoring: { @@ -818,3 +824,17 @@ type diagnosticSettingType = { @description('Optional. The full ARM resource ID of the Marketplace resource to which you would like to send Diagnostic Logs.') marketplacePartnerResourceId: string? }[]? + +type customerManagedKeyType = { + @description('Required. The resource ID of a key vault to reference a customer managed key for encryption from.') + keyVaultResourceId: string + + @description('Required. The name of the customer managed key to use for encryption.') + keyName: string + + @description('Optional. The version of the customer managed key to reference for encryption. If not provided, using \'latest\'.') + keyVersion: string? + + @description('Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public.') + keyVaultNetworkAccess: ('Private' | 'Public') +}? diff --git a/modules/container-service/managed-cluster/main.json b/modules/container-service/managed-cluster/main.json index 7190d6b56c..e6da45a8e2 100644 --- a/modules/container-service/managed-cluster/main.json +++ b/modules/container-service/managed-cluster/main.json @@ -6,7 +6,7 @@ "_generator": { "name": "bicep", "version": "0.23.1.45101", - "templateHash": "1906456864170625087" + "templateHash": "4013697482173328246" }, "name": "Azure Kubernetes Service (AKS) Managed Clusters", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.", @@ -232,6 +232,41 @@ } }, "nullable": true + }, + "customerManagedKeyType": { + "type": "object", + "properties": { + "keyVaultResourceId": { + "type": "string", + "metadata": { + "description": "Required. The resource ID of a key vault to reference a customer managed key for encryption from." + } + }, + "keyName": { + "type": "string", + "metadata": { + "description": "Required. The name of the customer managed key to use for encryption." + } + }, + "keyVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The version of the customer managed key to reference for encryption. If not provided, using 'latest'." + } + }, + "keyVaultNetworkAccess": { + "type": "string", + "allowedValues": [ + "Private", + "Public" + ], + "metadata": { + "description": "Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public." + } + } + }, + "nullable": true } }, "parameters": { @@ -864,20 +899,6 @@ "description": "Optional. The diagnostic settings of the service." } }, - "enableAzureKeyVaultKms": { - "type": "bool", - "defaultValue": false, - "metadata": { - "description": "Optional. Whether to enable Key Management Service." - } - }, - "keyVaultKms": { - "type": "object", - "defaultValue": {}, - "metadata": { - "description": "Conditional. Object that contains the 'keyId', 'keyVaultNetworkAccess' and 'keyVaultResourceId' to enable Key Management Service. Required if enableAzureKeyVaultKms is set to true." - } - }, "omsAgentEnabled": { "type": "bool", "defaultValue": true, @@ -952,6 +973,12 @@ "metadata": { "description": "Optional. Identities associated with the cluster." } + }, + "customerManagedKey": { + "$ref": "#/definitions/customerManagedKeyType", + "metadata": { + "description": "Optional. The customer managed key definition." + } } }, "variables": { @@ -997,6 +1024,27 @@ } }, "resources": { + "cMKKeyVault::cMKKey": { + "condition": "[and(not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'))), and(not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'))), not(empty(tryGet(parameters('customerManagedKey'), 'keyName')))))]", + "existing": true, + "type": "Microsoft.KeyVault/vaults/keys", + "apiVersion": "2023-02-01", + "subscriptionId": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '//'), '/')[2]]", + "resourceGroup": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '////'), '/')[4]]", + "name": "[format('{0}/{1}', last(split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), 'dummyVault'), '/')), coalesce(tryGet(parameters('customerManagedKey'), 'keyName'), 'dummyKey'))]", + "dependsOn": [ + "cMKKeyVault" + ] + }, + "cMKKeyVault": { + "condition": "[not(empty(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId')))]", + "existing": true, + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2023-02-01", + "subscriptionId": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '//'), '/')[2]]", + "resourceGroup": "[split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '////'), '/')[4]]", + "name": "[last(split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), 'dummyVault'), '/'))]" + }, "defaultTelemetry": { "condition": "[parameters('enableDefaultTelemetry')]", "type": "Microsoft.Resources/deployments", @@ -1130,7 +1178,7 @@ "userAssignedIdentityExceptions": "[parameters('podIdentityProfileUserAssignedIdentityExceptions')]" }, "securityProfile": { - "azureKeyVaultKms": "[if(parameters('enableAzureKeyVaultKms'), createObject('enabled', parameters('enableAzureKeyVaultKms'), 'keyId', parameters('keyVaultKms').keyId, 'keyVaultNetworkAccess', parameters('keyVaultKms').keyVaultNetworkAccess, 'keyVaultResourceId', if(equals(parameters('keyVaultKms').keyVaultNetworkAccess, 'Private'), parameters('keyVaultKms').keyVaultResourceId, null())), null())]", + "azureKeyVaultKms": "[if(not(empty(parameters('customerManagedKey'))), createObject('enabled', true(), 'keyId', if(not(empty(coalesce(tryGet(parameters('customerManagedKey'), 'keyVersion'), ''))), format('{0}/{1}', reference('cMKKeyVault::cMKKey').keyUri, parameters('customerManagedKey').keyVersion), reference('cMKKeyVault::cMKKey').keyUriWithVersion), 'keyVaultNetworkAccess', parameters('customerManagedKey').keyVaultNetworkAccess, 'keyVaultResourceId', if(equals(parameters('customerManagedKey').keyVaultNetworkAccess, 'Private'), extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '//'), '/')[2], split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), '////'), '/')[4]), 'Microsoft.KeyVault/vaults', last(split(coalesce(tryGet(parameters('customerManagedKey'), 'keyVaultResourceId'), 'dummyVault'), '/'))), null())), null())]", "defender": "[if(parameters('enableAzureDefender'), createObject('securityMonitoring', createObject('enabled', parameters('enableAzureDefender')), 'logAnalyticsWorkspaceResourceId', if(not(empty(parameters('monitoringWorkspaceId'))), parameters('monitoringWorkspaceId'), null())), null())]", "workloadIdentity": "[if(parameters('enableWorkloadIdentity'), createObject('enabled', parameters('enableWorkloadIdentity')), null())]" }, @@ -1149,7 +1197,10 @@ } }, "supportPlan": "[parameters('supportPlan')]" - } + }, + "dependsOn": [ + "cMKKeyVault" + ] }, "managedCluster_lock": { "condition": "[and(not(empty(coalesce(parameters('lock'), createObject()))), not(equals(tryGet(parameters('lock'), 'kind'), 'None')))]", diff --git a/modules/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep b/modules/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep index f1ca2663b9..40834512ba 100644 --- a/modules/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep +++ b/modules/container-service/managed-cluster/tests/e2e/azure/dependencies.bicep @@ -130,13 +130,6 @@ resource proximityPlacementGroup 'Microsoft.Compute/proximityPlacementGroups@202 location: location } -@description('The resource ID of the created Virtual Network Subnet.') -output subnetResourceIds array = [ - virtualNetwork.properties.subnets[0].id - virtualNetwork.properties.subnets[1].id - virtualNetwork.properties.subnets[2].id -] - resource dnsZone 'Microsoft.Network/dnsZones@2018-05-01' = { name: dnsZoneName location: 'global' @@ -178,5 +171,17 @@ output dnsZoneResourceId string = dnsZone.id @description('The resource ID of the created Log Analytics Workspace.') output logAnalyticsWorkspaceResourceId string = logAnalyticsWorkspace.id -@description('The uri including version of the KMS Key.') -output keyUriWithVersion string = keyVault::kmskey.properties.keyUriWithVersion +@description('The resource ID of the created Key Vault.') +output keyVaultResourceId string = keyVault.id + +@description('The name of the Key Vault Encryption Key.') +output keyVaultEncryptionKeyName string = keyVault::key.name + +@description('The resource ID of the created Virtual Network System Agent Pool Subnet.') +output systemPoolSubnetResourceId string = virtualNetwork.properties.subnets[0].id + +@description('The resource ID of the created Virtual Network Agent Pool 1 Subnet.') +output agentPool1SubnetResourceId string = virtualNetwork.properties.subnets[1].id + +@description('The resource ID of the created Virtual Network Agent Pool 2 Subnet.') +output agentPool2SubnetResourceId string = virtualNetwork.properties.subnets[2].id diff --git a/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep b/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep index 2d835afd94..32f8c42ed3 100644 --- a/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep +++ b/modules/container-service/managed-cluster/tests/e2e/azure/main.test.bicep @@ -92,7 +92,7 @@ module testDeployment '../../../main.bicep' = { storageProfile: 'ManagedDisks' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[0] + vnetSubnetID: nestedDependencies.outputs.systemPoolSubnetResourceId } ] agentPools: [ @@ -119,7 +119,7 @@ module testDeployment '../../../main.bicep' = { storageProfile: 'ManagedDisks' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[1] + vnetSubnetID: nestedDependencies.outputs.agentPool1SubnetResourceId proximityPlacementGroupResourceId: nestedDependencies.outputs.proximityPlacementGroupResourceId } { @@ -145,7 +145,7 @@ module testDeployment '../../../main.bicep' = { storageProfile: 'ManagedDisks' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[2] + vnetSubnetID: nestedDependencies.outputs.agentPool2SubnetResourceId } ] autoUpgradeProfileUpgradeChannel: 'stable' @@ -189,10 +189,10 @@ module testDeployment '../../../main.bicep' = { enableAzureDefender: true enableKeyvaultSecretsProvider: true enablePodSecurityPolicy: false - enableAzureKeyVaultKms: true - keyVaultKms: { - keyId: nestedDependencies.outputs.keyUriWithVersion + customerManagedKey: { + keyName: nestedDependencies.outputs.keyVaultEncryptionKeyName keyVaultNetworkAccess: 'Public' + keyVaultResourceId: nestedDependencies.outputs.keyVaultResourceId } lock: { kind: 'CanNotDelete' diff --git a/modules/container-service/managed-cluster/tests/e2e/priv/dependencies.bicep b/modules/container-service/managed-cluster/tests/e2e/priv/dependencies.bicep index b74bb113ac..3a7d3e9d62 100644 --- a/modules/container-service/managed-cluster/tests/e2e/priv/dependencies.bicep +++ b/modules/container-service/managed-cluster/tests/e2e/priv/dependencies.bicep @@ -31,14 +31,12 @@ resource virtualNetwork 'Microsoft.Network/virtualNetworks@2023-04-01' = { addressPrefix ] } - subnets: [ - { - name: 'defaultSubnet' + subnets: map(range(0, 2), i => { + name: 'subnet-${i}' properties: { - addressPrefix: cidrSubnet(addressPrefix, 16, 0) + addressPrefix: cidrSubnet(addressPrefix, 24, i) } - } - ] + }) } } @@ -85,3 +83,9 @@ output privateDnsZoneResourceId string = privateDnsZone.id @description('The resource ID of the VirtualNetwork created.') output vNetResourceId string = virtualNetwork.id + +@description('The resource ID of the created Virtual Network System Agent Pool Subnet.') +output systemPoolSubnetResourceId string = virtualNetwork.properties.subnets[0].id + +@description('The resource ID of the created Virtual Network Agent Pool 1 Subnet.') +output agentPoolSubnetResourceId string = virtualNetwork.properties.subnets[1].id diff --git a/modules/container-service/managed-cluster/tests/e2e/priv/main.test.bicep b/modules/container-service/managed-cluster/tests/e2e/priv/main.test.bicep index 8d911c5cc9..078372cab4 100644 --- a/modules/container-service/managed-cluster/tests/e2e/priv/main.test.bicep +++ b/modules/container-service/managed-cluster/tests/e2e/priv/main.test.bicep @@ -84,7 +84,7 @@ module testDeployment '../../../main.bicep' = { storageProfile: 'ManagedDisks' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + vnetSubnetID: nestedDependencies.outputs.systemPoolSubnetResourceId } ] agentPools: [ @@ -111,7 +111,7 @@ module testDeployment '../../../main.bicep' = { storageProfile: 'ManagedDisks' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + vnetSubnetID: nestedDependencies.outputs.agentPoolSubnetResourceId } { availabilityZones: [ From acfb6774b719103bba21cb0a24d3888911b17710 Mon Sep 17 00:00:00 2001 From: AlexanderSehr Date: Fri, 17 Nov 2023 20:46:38 +0100 Subject: [PATCH 3/4] Updaed api tests --- .../staticValidation/module.tests.ps1 | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/utilities/pipelines/staticValidation/module.tests.ps1 b/utilities/pipelines/staticValidation/module.tests.ps1 index 3e8ff1fe2e..71d90340cd 100644 --- a/utilities/pipelines/staticValidation/module.tests.ps1 +++ b/utilities/pipelines/staticValidation/module.tests.ps1 @@ -1200,7 +1200,7 @@ Describe 'API version tests' -Tag 'ApiCheck' { { $PSItem -like '*diagnosticsettings*' } { $testCases += @{ moduleName = $moduleFolderName - resourceType = 'diagnosticsettings' + resourceType = 'diagnosticSettings' ProviderNamespace = 'Microsoft.Insights' TargetApi = $resource.ApiVersion AvailableApiVersions = $ApiVersions @@ -1222,7 +1222,7 @@ Describe 'API version tests' -Tag 'ApiCheck' { { $PSItem -like '*roleAssignments' } { $testCases += @{ moduleName = $moduleFolderName - resourceType = 'roleassignments' + resourceType = 'roleAssignments' ProviderNamespace = 'Microsoft.Authorization' TargetApi = $resource.ApiVersion AvailableApiVersions = $ApiVersions @@ -1264,16 +1264,16 @@ Describe 'API version tests' -Tag 'ApiCheck' { [string] $ResourceType, [string] $TargetApi, [string] $ProviderNamespace, - [PSCustomObject] $AvailableApiVersions, + [hashtable] $AvailableApiVersions, [bool] $AllowPreviewVersionsInAPITests ) - if (-not (($AvailableApiVersions | Get-Member -Type NoteProperty).Name -contains $ProviderNamespace)) { + if ($AvailableApiVersions.Keys -notcontains $ProviderNamespace) { Write-Warning "[API Test] The Provider Namespace [$ProviderNamespace] is missing in your Azure API versions file. Please consider updating it and if it is still missing to open an issue in the 'AzureAPICrawler' PowerShell module's GitHub repository." Set-ItResult -Skipped -Because "The Azure API version file is missing the Provider Namespace [$ProviderNamespace]." return } - if (-not (($AvailableApiVersions.$ProviderNamespace | Get-Member -Type NoteProperty).Name -contains $ResourceType)) { + if ($AvailableApiVersions.$ProviderNamespace.Keys -notcontains $ResourceType) { Write-Warning "[API Test] The Provider Namespace [$ProviderNamespace] is missing the Resource Type [$ResourceType] in your API versions file. Please consider updating it and if it is still missing to open an issue in the 'AzureAPICrawler' PowerShell module's GitHub repository." Set-ItResult -Skipped -Because "The Azure API version file is missing the Resource Type [$ResourceType] for Provider Namespace [$ProviderNamespace]." return @@ -1297,10 +1297,15 @@ Describe 'API version tests' -Tag 'ApiCheck' { } $approvedApiVersions = $approvedApiVersions | Sort-Object -Unique -Descending - $approvedApiVersions | Should -Contain $TargetApi - # Provide a warning if an API version is second to next to expire. - if ($approvedApiVersions -contains $TargetApi) { + if ($approvedApiVersions -notcontains $TargetApi) { + # Using a warning now instead of an error, as we don't want to block PRs for this. + Write-Warning ("The used API version [$TargetApi] is not one of the most recent 5 versions. Please consider upgrading to one of the following: {0}" -f $approvedApiVersions -join ', ') + + # The original failed test was + # $approvedApiVersions | Should -Contain $TargetApi + } else { + # Provide a warning if an API version is second to next to expire. $indexOfVersion = $approvedApiVersions.IndexOf($TargetApi) # Example From 70e1ce5f32dbdaa3045fa21f7a0a7918a3764dc7 Mon Sep 17 00:00:00 2001 From: AlexanderSehr Date: Fri, 17 Nov 2023 20:59:07 +0100 Subject: [PATCH 4/4] Update to latest --- utilities/pipelines/staticValidation/module.tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/pipelines/staticValidation/module.tests.ps1 b/utilities/pipelines/staticValidation/module.tests.ps1 index 71d90340cd..f608f5f24b 100644 --- a/utilities/pipelines/staticValidation/module.tests.ps1 +++ b/utilities/pipelines/staticValidation/module.tests.ps1 @@ -1156,7 +1156,7 @@ Describe 'API version tests' -Tag 'ApiCheck' { return } - $ApiVersions = Get-Content -Path $apiSpecsFilePath -Raw | ConvertFrom-Json + $ApiVersions = Get-Content -Path $apiSpecsFilePath -Raw | ConvertFrom-Json -AsHashtable foreach ($moduleFolderPath in $moduleFolderPaths) { $moduleFolderName = $moduleFolderPath.Replace('\', '/').Split('/modules/')[1]