diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/flux/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/flux/deploy.test.bicep new file mode 100644 index 0000000000..97082bb9ef --- /dev/null +++ b/modules/Microsoft.ContainerService/managedClusters/.test/flux/deploy.test.bicep @@ -0,0 +1,108 @@ +targetScope = 'subscription' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmmf2' + +@description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') +param enableDefaultTelemetry bool = true + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: location +} + +module testDeployment '../../deploy.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-test-${serviceShort}' + params: { + name: '${serviceShort}001' + enableDefaultTelemetry: enableDefaultTelemetry + systemAssignedIdentity: true + primaryAgentPoolProfile: [ + { + name: 'systempool' + count: 1 + vmSize: 'Standard_DS2_v2' + mode: 'System' + } + ] + fluxReleaseTrain: 'Stable' + fluxVersion: '' + fluxConfigurationProtectedSettings: {} + fluxConfigurationSettings: { + 'helm-controller.enabled': 'true' + 'source-controller.enabled': 'true' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'true' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + } + fluxConfigurations: [ + { + namespace: 'flux-system' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + } + { + namespace: 'flux-system-helm' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' + } + kustomizations: { + infra: { + path: './infrastructure' + dependsOn: [] + timeoutInSeconds: 600 + syncIntervalInSeconds: 600 + validation: 'none' + prune: true + } + apps: { + path: './apps/staging' + dependsOn: [ + { + kustomizationName: 'infra' + } + ] + timeoutInSeconds: 600 + syncIntervalInSeconds: 600 + retryIntervalInSeconds: 600 + prune: true + } + } + } + ] + } +} diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index 53ac8e1856..82e23ff6bd 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -295,6 +295,22 @@ param tags object = {} @description('Optional. The resource ID of the disc encryption set to apply to the cluster. For security reasons, this value should be provided.') param diskEncryptionSetID string = '' +@description('Optional. ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is "true".') +param fluxReleaseTrain string = 'Stable' + +@description('Optional. Version of the extension for this extension, if it is "pinned" to a specific version.') +param fluxVersion string = '' + +@description('Optional. Configuration settings that are sensitive, as name-value pairs for configuring this extension.') +@secure() +param fluxConfigurationProtectedSettings object = {} + +@description('Optional. Configuration settings, as name-value pairs for configuring this extension.') +param fluxConfigurationSettings object = {} + +@description('Optional. A list of flux configuraitons.') +param fluxConfigurations array = [] + @description('Optional. The name of logs that will be streamed. "allLogs" includes all possible logs for the resource.') @allowed([ 'allLogs' @@ -553,6 +569,42 @@ module managedCluster_agentPools 'agentPools/deploy.bicep' = [for (agentPool, in } }] +module managedCluster_extension '../../Microsoft.KubernetesConfiguration/extensions/deploy.bicep' = if (!empty(fluxConfigurations)) { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxExtension' + params: { + clusterName: managedCluster.name + configurationProtectedSettings: !empty(fluxConfigurationProtectedSettings) ? fluxConfigurationProtectedSettings : {} + configurationSettings: !empty(fluxConfigurationSettings) ? fluxConfigurationSettings : {} + enableDefaultTelemetry: enableReferencedModulesTelemetry + extensionType: 'microsoft.flux' + location: location + name: 'flux' + releaseNamespace: 'flux-system' + releaseTrain: !empty(fluxReleaseTrain) ? fluxReleaseTrain : 'Stable' + version: !empty(fluxVersion) ? fluxVersion : '' + } +} + +module managedCluster_fluxConfiguration '../../Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep' = [for (fluxConfiguration, index) in fluxConfigurations: { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxConfiguration${index}' + params: { + bucket: contains(fluxConfiguration, 'bucket') ? fluxConfiguration.bucket : {} + clusterName: managedCluster.name + configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? fluxConfiguration.configurationProtectedSettings : {} + enableDefaultTelemetry: enableDefaultTelemetry + gitRepository: contains(fluxConfiguration, 'gitRepository') ? fluxConfiguration.gitRepository : {} + kustomizations: contains(fluxConfiguration, 'kustomizations') ? fluxConfiguration.kustomizations : {} + name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : toLower('${managedCluster.name}-fluxconfiguration${index}') + namespace: fluxConfiguration.namespace + scope: fluxConfiguration.scope + sourceKind: contains(fluxConfiguration, 'gitRepository') ? 'GitRepository' : 'Bucket' + suspend: contains(fluxConfiguration, 'suspend') ? fluxConfiguration.suspend : false + } + dependsOn: [ + managedCluster_extension + ] +}] + resource managedCluster_lock 'Microsoft.Authorization/locks@2020-05-01' = if (!empty(lock)) { name: '${managedCluster.name}-${lock}-lock' properties: { diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index 9765daf02e..e66f6a83f1 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -19,6 +19,8 @@ This module deploys Azure Kubernetes Cluster (AKS). | `Microsoft.ContainerService/managedClusters` | [2022-09-01](https://docs.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2022-09-01/managedClusters) | | `Microsoft.ContainerService/managedClusters/agentPools` | [2022-09-01](https://docs.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2022-09-01/managedClusters/agentPools) | | `Microsoft.Insights/diagnosticSettings` | [2021-05-01-preview](https://docs.microsoft.com/en-us/azure/templates/Microsoft.Insights/2021-05-01-preview/diagnosticSettings) | +| `Microsoft.KubernetesConfiguration/extensions` | [2022-03-01](https://docs.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/2022-03-01/extensions) | +| `Microsoft.KubernetesConfiguration/fluxConfigurations` | [2022-03-01](https://docs.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/2022-03-01/fluxConfigurations) | ## Parameters @@ -102,6 +104,11 @@ This module deploys Azure Kubernetes Cluster (AKS). | `enablePrivateClusterPublicFQDN` | bool | `False` | | Whether to create additional public FQDN for private cluster or not. | | `enableRBAC` | bool | `True` | | Whether to enable Kubernetes Role-Based Access Control. | | `enableSecretRotation` | string | `'false'` | `[false, true]` | Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation. | +| `fluxConfigurationProtectedSettings` | secureObject | `{object}` | | Configuration settings that are sensitive, as name-value pairs for configuring this extension. | +| `fluxConfigurations` | array | `[]` | | A list of flux configuraitons. | +| `fluxConfigurationSettings` | object | `{object}` | | Configuration settings, as name-value pairs for configuring this extension. | +| `fluxReleaseTrain` | string | `'Stable'` | | ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is "true". | +| `fluxVersion` | string | `''` | | Version of the extension for this extension, if it is "pinned" to a specific version. | | `httpApplicationRoutingEnabled` | bool | `False` | | Specifies whether the httpApplicationRouting add-on is enabled or not. | | `ingressApplicationGatewayEnabled` | bool | `False` | | Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. | | `kubeDashboardEnabled` | bool | `False` | | Specifies whether the kubeDashboard add-on is enabled or not. | @@ -365,7 +372,12 @@ userAssignedIdentities: { ## Cross-referenced modules -_None_ +This section gives you an overview of all local-referenced module files (i.e., other CARML modules that are referenced in this module) and all remote-referenced files (i.e., Bicep modules that are referenced from a Bicep Registry or Template Specs). + +| Reference | Type | +| :-- | :-- | +| `Microsoft.KubernetesConfiguration/extensions` | Local reference | +| `Microsoft.KubernetesConfiguration/fluxConfigurations` | Local reference | ## Deployment examples @@ -625,7 +637,185 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice

-

Example 2: Kubenet

+

Example 2: Flux

+ +
+ +via Bicep module + +```bicep +module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { + name: '${uniqueString(deployment().name, location)}-test-csmmf2' + params: { + // Required parameters + name: 'csmmf2001' + primaryAgentPoolProfile: [ + { + count: 1 + mode: 'System' + name: 'systempool' + vmSize: 'Standard_DS2_v2' + } + ] + // Non-required parameters + enableDefaultTelemetry: '' + fluxConfigurations: [ + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + namespace: 'flux-system' + } + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' + } + kustomizations: { + apps: { + path: './apps/staging' + prune: true + retryIntervalInSeconds: 600 + syncIntervalInSeconds: 600 + timeoutInSeconds: 600 + } + infra: { + dependsOn: [] + path: './infrastructure' + prune: true + syncIntervalInSeconds: 600 + timeoutInSeconds: 600 + validation: 'none' + } + } + namespace: 'flux-system-helm' + } + ] + fluxConfigurationSettings: { + 'helm-controller.enabled': 'true' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'true' + 'source-controller.enabled': 'true' + } + systemAssignedIdentity: true + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmmf2001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "count": 1, + "mode": "System", + "name": "systempool", + "vmSize": "Standard_DS2_v2" + } + ] + }, + // Non-required parameters + "enableDefaultTelemetry": { + "value": "" + }, + "fluxConfigurations": { + "value": [ + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" + }, + "namespace": "flux-system" + }, + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/Azure/gitops-flux2-kustomize-helm-mt" + }, + "kustomizations": { + "apps": { + "dependsOn": [ + { + "kustomizationName": "infra" + } + ], + "path": "./apps/staging", + "prune": true, + "retryIntervalInSeconds": 600, + "syncIntervalInSeconds": 600, + "timeoutInSeconds": 600 + }, + "infra": { + "dependsOn": [], + "path": "./infrastructure", + "prune": true, + "syncIntervalInSeconds": 600, + "timeoutInSeconds": 600, + "validation": "none" + } + }, + "namespace": "flux-system-helm" + } + ] + }, + "fluxConfigurationSettings": { + "value": { + "helm-controller.enabled": "true", + "image-automation-controller.enabled": "false", + "image-reflector-controller.enabled": "false", + "kustomize-controller.enabled": "true", + "notification-controller.enabled": "true", + "source-controller.enabled": "true" + } + }, + "systemAssignedIdentity": { + "value": true + } + } +} +``` + +
+

+ +

Example 3: Kubenet

@@ -866,7 +1056,7 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice

-

Example 3: Min

+

Example 4: Min

diff --git a/modules/Microsoft.KubernetesConfiguration/extensions/deploy.bicep b/modules/Microsoft.KubernetesConfiguration/extensions/deploy.bicep index 3ec32c5c9b..5630bacccd 100644 --- a/modules/Microsoft.KubernetesConfiguration/extensions/deploy.bicep +++ b/modules/Microsoft.KubernetesConfiguration/extensions/deploy.bicep @@ -11,6 +11,7 @@ param clusterName string param location string = resourceGroup().location @description('Optional. Configuration settings that are sensitive, as name-value pairs for configuring this extension.') +@secure() param configurationProtectedSettings object = {} @description('Optional. Configuration settings, as name-value pairs for configuring this extension.') diff --git a/modules/Microsoft.KubernetesConfiguration/extensions/readme.md b/modules/Microsoft.KubernetesConfiguration/extensions/readme.md index 08d5c117f1..2dcc1d827b 100644 --- a/modules/Microsoft.KubernetesConfiguration/extensions/readme.md +++ b/modules/Microsoft.KubernetesConfiguration/extensions/readme.md @@ -49,7 +49,7 @@ For Details see [Prerequisites](https://docs.microsoft.com/en-us/azure/azure-arc | Parameter Name | Type | Default Value | Description | | :-- | :-- | :-- | :-- | -| `configurationProtectedSettings` | object | `{object}` | Configuration settings that are sensitive, as name-value pairs for configuring this extension. | +| `configurationProtectedSettings` | secureObject | `{object}` | Configuration settings that are sensitive, as name-value pairs for configuring this extension. | | `configurationSettings` | object | `{object}` | Configuration settings, as name-value pairs for configuring this extension. | | `enableDefaultTelemetry` | bool | `True` | Enable telemetry via a Globally Unique Identifier (GUID). | | `location` | string | `[resourceGroup().location]` | Location for all resources. | diff --git a/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep b/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep index 8c39b77609..c9feeebf74 100644 --- a/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep +++ b/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep @@ -14,6 +14,7 @@ param location string = resourceGroup().location param bucket object = {} @description('Optional. Key-value pairs of protected configuration settings for the configuration.') +@secure() param configurationProtectedSettings object = {} @description('Optional. Parameters to reconcile to the GitRepository source kind type.') diff --git a/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/readme.md b/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/readme.md index 4d281df1d9..53b3512676 100644 --- a/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/readme.md +++ b/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/readme.md @@ -52,7 +52,7 @@ For Details see [Prerequisites](https://docs.microsoft.com/en-us/azure/azure-arc | Parameter Name | Type | Default Value | Description | | :-- | :-- | :-- | :-- | | `bucket` | object | `{object}` | Parameters to reconcile to the GitRepository source kind type. | -| `configurationProtectedSettings` | object | `{object}` | Key-value pairs of protected configuration settings for the configuration. | +| `configurationProtectedSettings` | secureObject | `{object}` | Key-value pairs of protected configuration settings for the configuration. | | `enableDefaultTelemetry` | bool | `True` | Enable telemetry via a Globally Unique Identifier (GUID). | | `gitRepository` | object | `{object}` | Parameters to reconcile to the GitRepository source kind type. | | `kustomizations` | object | `{object}` | Array of kustomizations used to reconcile the artifact pulled by the source type on the cluster. |