From bc4d6722867b6654379b80f8a2a2c3a350d954b5 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 10:19:30 +0100 Subject: [PATCH 01/29] Adding flux extension --- .../.test/min/deploy.test.bicep | 44 +++++++++++++ .../managedClusters/deploy.bicep | 20 ++++++ .../managedClusters/readme.md | 65 ++++++++++++++++++- 3 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep new file mode 100644 index 0000000000..de85dc3402 --- /dev/null +++ b/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep @@ -0,0 +1,44 @@ +targetScope = 'subscription' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmmin' + +@description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') +param enableDefaultTelemetry bool = true + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: location +} + + +module testDeployment '../../deploy.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-test-${serviceShort}' + params: { + name: '<>${serviceShort}001' + enableDefaultTelemetry: enableDefaultTelemetry + primaryAgentPoolProfile: [ + { + name: 'systempool' + } + ] + } +} diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index 04b4a0532f..ade05bd24f 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -295,6 +295,9 @@ param tags object = {} @description('Optional. The resource ID of the disc encryption set to apply to the cluster. For security reasons, this value should be provided.') param diskEncryptionSetID string = '' +@description('Optional. A flux configuraiton.') +param fluxConfiguration object = {} + @description('Optional. The name of logs that will be streamed. "allLogs" includes all possible logs for the resource.') @allowed([ 'allLogs' @@ -553,6 +556,23 @@ module managedCluster_agentPools 'agentPools/deploy.bicep' = [for (agentPool, in } }] +module fluxExtension '../../Microsoft.KubernetesConfiguration/extensions/deploy.bicep' = if (!empty(fluxConfiguration)) { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxExtension' + params: { + clusterName: managedCluster.name + name: '${managedCluster.name}-fluxExtension' + extensionType: 'microsoft.flux' + configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? fluxConfiguration.configurationProtectedSettings : {} + configurationSettings: contains(fluxConfiguration, 'configurationSettings') ? fluxConfiguration.configurationSettings : {} + enableDefaultTelemetry: enableReferencedModulesTelemetry + location: location + releaseNamespace: 'flux-system' + releaseTrain: contains(fluxConfiguration, 'releaseTrain') ? fluxConfiguration.releaseTrain : 'Stable' + targetNamespace: contains(fluxConfiguration, 'targetNamespace') ? fluxConfiguration.targetNamespace : '' + version: contains(fluxConfiguration, 'version') ? fluxConfiguration.version : '' + } +} + resource managedCluster_lock 'Microsoft.Authorization/locks@2020-05-01' = if (!empty(lock)) { name: '${managedCluster.name}-${lock}-lock' properties: { diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index 510d76a2a0..cb1b90b80c 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -19,6 +19,7 @@ This module deploys Azure Kubernetes Cluster (AKS). | `Microsoft.ContainerService/managedClusters` | [2022-09-01](https://docs.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2022-09-01/managedClusters) | | `Microsoft.ContainerService/managedClusters/agentPools` | [2022-09-01](https://docs.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2022-09-01/managedClusters/agentPools) | | `Microsoft.Insights/diagnosticSettings` | [2021-05-01-preview](https://docs.microsoft.com/en-us/azure/templates/Microsoft.Insights/2021-05-01-preview/diagnosticSettings) | +| `Microsoft.KubernetesConfiguration/extensions` | [2022-03-01](https://docs.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/2022-03-01/extensions) | ## Parameters @@ -102,6 +103,7 @@ This module deploys Azure Kubernetes Cluster (AKS). | `enablePrivateClusterPublicFQDN` | bool | `False` | | Whether to create additional public FQDN for private cluster or not. | | `enableRBAC` | bool | `True` | | Whether to enable Kubernetes Role-Based Access Control. | | `enableSecretRotation` | string | `'false'` | `[false, true]` | Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation. | +| `fluxConfiguration` | object | `{object}` | | A flux configuraiton. | | `httpApplicationRoutingEnabled` | bool | `False` | | Specifies whether the httpApplicationRouting add-on is enabled or not. | | `ingressApplicationGatewayEnabled` | bool | `False` | | Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. | | `kubeDashboardEnabled` | bool | `False` | | Specifies whether the kubeDashboard add-on is enabled or not. | @@ -364,7 +366,11 @@ userAssignedIdentities: { ## Cross-referenced modules -_None_ +This section gives you an overview of all local-referenced module files (i.e., other CARML modules that are referenced in this module) and all remote-referenced files (i.e., Bicep modules that are referenced from a Bicep Registry or Template Specs). + +| Reference | Type | +| :-- | :-- | +| `Microsoft.KubernetesConfiguration/extensions` | Local reference | ## Deployment examples @@ -864,3 +870,60 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice

+ +

Example 3: Min

+ +
+ +via Bicep module + +```bicep +module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { + name: '${uniqueString(deployment().name, location)}-test-csmmin' + params: { + // Required parameters + name: '<>csmmin001' + primaryAgentPoolProfile: [ + { + name: 'systempool' + } + ] + // Non-required parameters + enableDefaultTelemetry: '' + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "<>csmmin001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "name": "systempool" + } + ] + }, + // Non-required parameters + "enableDefaultTelemetry": { + "value": "" + } + } +} +``` + +
+

From e843b2e13e6b4183eb5d9deda2da87c6cea560ff Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 10:21:36 +0100 Subject: [PATCH 02/29] move other tests for speed --- .../{.test => .otherTests}/azure/dependencies.bicep | 0 .../{.test => .otherTests}/azure/deploy.test.bicep | 0 .../{.test => .otherTests}/kubenet/dependencies.bicep | 0 .../{.test => .otherTests}/kubenet/deploy.test.bicep | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename modules/Microsoft.ContainerService/managedClusters/{.test => .otherTests}/azure/dependencies.bicep (100%) rename modules/Microsoft.ContainerService/managedClusters/{.test => .otherTests}/azure/deploy.test.bicep (100%) rename modules/Microsoft.ContainerService/managedClusters/{.test => .otherTests}/kubenet/dependencies.bicep (100%) rename modules/Microsoft.ContainerService/managedClusters/{.test => .otherTests}/kubenet/deploy.test.bicep (100%) diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/azure/dependencies.bicep b/modules/Microsoft.ContainerService/managedClusters/.otherTests/azure/dependencies.bicep similarity index 100% rename from modules/Microsoft.ContainerService/managedClusters/.test/azure/dependencies.bicep rename to modules/Microsoft.ContainerService/managedClusters/.otherTests/azure/dependencies.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/azure/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.otherTests/azure/deploy.test.bicep similarity index 100% rename from modules/Microsoft.ContainerService/managedClusters/.test/azure/deploy.test.bicep rename to modules/Microsoft.ContainerService/managedClusters/.otherTests/azure/deploy.test.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/kubenet/dependencies.bicep b/modules/Microsoft.ContainerService/managedClusters/.otherTests/kubenet/dependencies.bicep similarity index 100% rename from modules/Microsoft.ContainerService/managedClusters/.test/kubenet/dependencies.bicep rename to modules/Microsoft.ContainerService/managedClusters/.otherTests/kubenet/dependencies.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/kubenet/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.otherTests/kubenet/deploy.test.bicep similarity index 100% rename from modules/Microsoft.ContainerService/managedClusters/.test/kubenet/deploy.test.bicep rename to modules/Microsoft.ContainerService/managedClusters/.otherTests/kubenet/deploy.test.bicep From 696d1b38b61519b416f376de55ec599d2db63c23 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 10:26:07 +0100 Subject: [PATCH 03/29] removed reade refs to other tests --- .../.test/min/deploy.test.bicep | 1 - .../managedClusters/readme.md | 494 +----------------- 2 files changed, 1 insertion(+), 494 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep index de85dc3402..fc8ded33b5 100644 --- a/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep @@ -28,7 +28,6 @@ resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { location: location } - module testDeployment '../../deploy.bicep' = { scope: resourceGroup name: '${uniqueString(deployment().name, location)}-test-${serviceShort}' diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index cb1b90b80c..6073eb9b5f 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -379,499 +379,7 @@ The following module usage examples are retrieved from the content of the files >**Note**: Each example lists all the required parameters first, followed by the rest - each in alphabetical order. -

Example 1: Azure

- -
- -via Bicep module - -```bicep -module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { - name: '${uniqueString(deployment().name, location)}-test-csmaz' - params: { - // Required parameters - name: '<>csmaz001' - primaryAgentPoolProfile: [ - { - availabilityZones: [ - '1' - ] - count: 1 - enableAutoScaling: true - maxCount: 3 - maxPods: 30 - minCount: 1 - mode: 'System' - name: 'systempool' - osDiskSizeGB: 0 - osType: 'Linux' - serviceCidr: '' - storageProfile: 'ManagedDisks' - type: 'VirtualMachineScaleSets' - vmSize: 'Standard_DS2_v2' - vnetSubnetID: '' - } - ] - // Non-required parameters - agentPools: [ - { - availabilityZones: [ - '1' - ] - count: 2 - enableAutoScaling: true - maxCount: 3 - maxPods: 30 - minCount: 1 - minPods: 2 - mode: 'User' - name: 'userpool1' - nodeLabels: {} - nodeTaints: [ - 'CriticalAddonsOnly=true:NoSchedule' - ] - osDiskSizeGB: 128 - osType: 'Linux' - scaleSetEvictionPolicy: 'Delete' - scaleSetPriority: 'Regular' - storageProfile: 'ManagedDisks' - type: 'VirtualMachineScaleSets' - vmSize: 'Standard_DS2_v2' - vnetSubnetID: '' - } - { - availabilityZones: [ - '1' - ] - count: 2 - enableAutoScaling: true - maxCount: 3 - maxPods: 30 - minCount: 1 - minPods: 2 - mode: 'User' - name: 'userpool2' - nodeLabels: {} - nodeTaints: [ - 'CriticalAddonsOnly=true:NoSchedule' - ] - osDiskSizeGB: 128 - osType: 'Linux' - scaleSetEvictionPolicy: 'Delete' - scaleSetPriority: 'Regular' - storageProfile: 'ManagedDisks' - type: 'VirtualMachineScaleSets' - vmSize: 'Standard_DS2_v2' - vnetSubnetID: '' - } - ] - aksClusterNetworkPlugin: 'azure' - diagnosticEventHubAuthorizationRuleId: '' - diagnosticEventHubName: '' - diagnosticLogsRetentionInDays: 7 - diagnosticStorageAccountId: '' - diagnosticWorkspaceId: '' - diskEncryptionSetID: '' - enableDefaultTelemetry: '' - lock: 'CanNotDelete' - roleAssignments: [ - { - principalIds: [ - '' - ] - principalType: 'ServicePrincipal' - roleDefinitionIdOrName: 'Reader' - } - ] - systemAssignedIdentity: true - } -} -``` - -
-

- -

- -via JSON Parameter file - -```json -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - // Required parameters - "name": { - "value": "<>csmaz001" - }, - "primaryAgentPoolProfile": { - "value": [ - { - "availabilityZones": [ - "1" - ], - "count": 1, - "enableAutoScaling": true, - "maxCount": 3, - "maxPods": 30, - "minCount": 1, - "mode": "System", - "name": "systempool", - "osDiskSizeGB": 0, - "osType": "Linux", - "serviceCidr": "", - "storageProfile": "ManagedDisks", - "type": "VirtualMachineScaleSets", - "vmSize": "Standard_DS2_v2", - "vnetSubnetID": "" - } - ] - }, - // Non-required parameters - "agentPools": { - "value": [ - { - "availabilityZones": [ - "1" - ], - "count": 2, - "enableAutoScaling": true, - "maxCount": 3, - "maxPods": 30, - "minCount": 1, - "minPods": 2, - "mode": "User", - "name": "userpool1", - "nodeLabels": {}, - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ], - "osDiskSizeGB": 128, - "osType": "Linux", - "scaleSetEvictionPolicy": "Delete", - "scaleSetPriority": "Regular", - "storageProfile": "ManagedDisks", - "type": "VirtualMachineScaleSets", - "vmSize": "Standard_DS2_v2", - "vnetSubnetID": "" - }, - { - "availabilityZones": [ - "1" - ], - "count": 2, - "enableAutoScaling": true, - "maxCount": 3, - "maxPods": 30, - "minCount": 1, - "minPods": 2, - "mode": "User", - "name": "userpool2", - "nodeLabels": {}, - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ], - "osDiskSizeGB": 128, - "osType": "Linux", - "scaleSetEvictionPolicy": "Delete", - "scaleSetPriority": "Regular", - "storageProfile": "ManagedDisks", - "type": "VirtualMachineScaleSets", - "vmSize": "Standard_DS2_v2", - "vnetSubnetID": "" - } - ] - }, - "aksClusterNetworkPlugin": { - "value": "azure" - }, - "diagnosticEventHubAuthorizationRuleId": { - "value": "" - }, - "diagnosticEventHubName": { - "value": "" - }, - "diagnosticLogsRetentionInDays": { - "value": 7 - }, - "diagnosticStorageAccountId": { - "value": "" - }, - "diagnosticWorkspaceId": { - "value": "" - }, - "diskEncryptionSetID": { - "value": "" - }, - "enableDefaultTelemetry": { - "value": "" - }, - "lock": { - "value": "CanNotDelete" - }, - "roleAssignments": { - "value": [ - { - "principalIds": [ - "" - ], - "principalType": "ServicePrincipal", - "roleDefinitionIdOrName": "Reader" - } - ] - }, - "systemAssignedIdentity": { - "value": true - } - } -} -``` - -
-

- -

Example 2: Kubenet

- -
- -via Bicep module - -```bicep -module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { - name: '${uniqueString(deployment().name, location)}-test-csmkube' - params: { - // Required parameters - name: '<>csmkube001' - primaryAgentPoolProfile: [ - { - availabilityZones: [ - '1' - ] - count: 1 - enableAutoScaling: true - maxCount: 3 - maxPods: 30 - minCount: 1 - mode: 'System' - name: 'systempool' - osDiskSizeGB: 0 - osType: 'Linux' - serviceCidr: '' - storageProfile: 'ManagedDisks' - type: 'VirtualMachineScaleSets' - vmSize: 'Standard_DS2_v2' - } - ] - // Non-required parameters - agentPools: [ - { - availabilityZones: [ - '1' - ] - count: 2 - enableAutoScaling: true - maxCount: 3 - maxPods: 30 - minCount: 1 - minPods: 2 - mode: 'User' - name: 'userpool1' - nodeLabels: {} - nodeTaints: [ - 'CriticalAddonsOnly=true:NoSchedule' - ] - osDiskSizeGB: 128 - osType: 'Linux' - scaleSetEvictionPolicy: 'Delete' - scaleSetPriority: 'Regular' - storageProfile: 'ManagedDisks' - type: 'VirtualMachineScaleSets' - vmSize: 'Standard_DS2_v2' - } - { - availabilityZones: [ - '1' - ] - count: 2 - enableAutoScaling: true - maxCount: 3 - maxPods: 30 - minCount: 1 - minPods: 2 - mode: 'User' - name: 'userpool2' - nodeLabels: {} - nodeTaints: [ - 'CriticalAddonsOnly=true:NoSchedule' - ] - osDiskSizeGB: 128 - osType: 'Linux' - scaleSetEvictionPolicy: 'Delete' - scaleSetPriority: 'Regular' - storageProfile: 'ManagedDisks' - type: 'VirtualMachineScaleSets' - vmSize: 'Standard_DS2_v2' - } - ] - aksClusterNetworkPlugin: 'kubenet' - diagnosticEventHubAuthorizationRuleId: '' - diagnosticEventHubName: '' - diagnosticLogsRetentionInDays: 7 - diagnosticStorageAccountId: '' - diagnosticWorkspaceId: '' - enableDefaultTelemetry: '' - roleAssignments: [ - { - principalIds: [ - '' - ] - principalType: 'ServicePrincipal' - roleDefinitionIdOrName: 'Reader' - } - ] - userAssignedIdentities: { - '': {} - } - } -} -``` - -
-

- -

- -via JSON Parameter file - -```json -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - // Required parameters - "name": { - "value": "<>csmkube001" - }, - "primaryAgentPoolProfile": { - "value": [ - { - "availabilityZones": [ - "1" - ], - "count": 1, - "enableAutoScaling": true, - "maxCount": 3, - "maxPods": 30, - "minCount": 1, - "mode": "System", - "name": "systempool", - "osDiskSizeGB": 0, - "osType": "Linux", - "serviceCidr": "", - "storageProfile": "ManagedDisks", - "type": "VirtualMachineScaleSets", - "vmSize": "Standard_DS2_v2" - } - ] - }, - // Non-required parameters - "agentPools": { - "value": [ - { - "availabilityZones": [ - "1" - ], - "count": 2, - "enableAutoScaling": true, - "maxCount": 3, - "maxPods": 30, - "minCount": 1, - "minPods": 2, - "mode": "User", - "name": "userpool1", - "nodeLabels": {}, - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ], - "osDiskSizeGB": 128, - "osType": "Linux", - "scaleSetEvictionPolicy": "Delete", - "scaleSetPriority": "Regular", - "storageProfile": "ManagedDisks", - "type": "VirtualMachineScaleSets", - "vmSize": "Standard_DS2_v2" - }, - { - "availabilityZones": [ - "1" - ], - "count": 2, - "enableAutoScaling": true, - "maxCount": 3, - "maxPods": 30, - "minCount": 1, - "minPods": 2, - "mode": "User", - "name": "userpool2", - "nodeLabels": {}, - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ], - "osDiskSizeGB": 128, - "osType": "Linux", - "scaleSetEvictionPolicy": "Delete", - "scaleSetPriority": "Regular", - "storageProfile": "ManagedDisks", - "type": "VirtualMachineScaleSets", - "vmSize": "Standard_DS2_v2" - } - ] - }, - "aksClusterNetworkPlugin": { - "value": "kubenet" - }, - "diagnosticEventHubAuthorizationRuleId": { - "value": "" - }, - "diagnosticEventHubName": { - "value": "" - }, - "diagnosticLogsRetentionInDays": { - "value": 7 - }, - "diagnosticStorageAccountId": { - "value": "" - }, - "diagnosticWorkspaceId": { - "value": "" - }, - "enableDefaultTelemetry": { - "value": "" - }, - "roleAssignments": { - "value": [ - { - "principalIds": [ - "" - ], - "principalType": "ServicePrincipal", - "roleDefinitionIdOrName": "Reader" - } - ] - }, - "userAssignedIdentities": { - "value": { - "": {} - } - } - } -} -``` - -
-

- -

Example 3: Min

+

Example 1: Min

From 7cd43d3df8f7399c37c50ba03d9c6ed75d9c7333 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 11:07:18 +0100 Subject: [PATCH 04/29] min test --- .../managedClusters/.test/min/deploy.test.bicep | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep index fc8ded33b5..b752b96308 100644 --- a/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep @@ -32,11 +32,15 @@ module testDeployment '../../deploy.bicep' = { scope: resourceGroup name: '${uniqueString(deployment().name, location)}-test-${serviceShort}' params: { - name: '<>${serviceShort}001' + name: '${serviceShort}001' enableDefaultTelemetry: enableDefaultTelemetry + systemAssignedIdentity: true primaryAgentPoolProfile: [ { name: 'systempool' + count: 1 + vmSize: 'Standard_DS2_v2' + mode: 'System' } ] } From 34b69cb194d449541e31f94e14ea8158413dd191 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 11:15:14 +0100 Subject: [PATCH 05/29] update readme --- .../managedClusters/readme.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index 6073eb9b5f..d174227baa 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -390,14 +390,18 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice name: '${uniqueString(deployment().name, location)}-test-csmmin' params: { // Required parameters - name: '<>csmmin001' + name: 'csmmin001' primaryAgentPoolProfile: [ { + count: 1 + mode: 'System' name: 'systempool' + vmSize: 'Standard_DS2_v2' } ] // Non-required parameters enableDefaultTelemetry: '' + systemAssignedIdentity: true } } ``` @@ -416,18 +420,24 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice "parameters": { // Required parameters "name": { - "value": "<>csmmin001" + "value": "csmmin001" }, "primaryAgentPoolProfile": { "value": [ { - "name": "systempool" + "count": 1, + "mode": "System", + "name": "systempool", + "vmSize": "Standard_DS2_v2" } ] }, // Non-required parameters "enableDefaultTelemetry": { "value": "" + }, + "systemAssignedIdentity": { + "value": true } } } From e5c8a0f22791e3d09f7c8c5b61d0968f42649937 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 11:54:41 +0100 Subject: [PATCH 06/29] test minimal fluxconfig --- .../min/deploy.test.bicep | 0 .../.test/minFlux/deploy.test.bicep | 50 +++++++++++++++++++ .../managedClusters/deploy.bicep | 2 +- .../managedClusters/readme.md | 10 +++- 4 files changed, 60 insertions(+), 2 deletions(-) rename modules/Microsoft.ContainerService/managedClusters/{.test => .otherTests}/min/deploy.test.bicep (100%) create mode 100644 modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.otherTests/min/deploy.test.bicep similarity index 100% rename from modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep rename to modules/Microsoft.ContainerService/managedClusters/.otherTests/min/deploy.test.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep new file mode 100644 index 0000000000..7321c61b31 --- /dev/null +++ b/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep @@ -0,0 +1,50 @@ +targetScope = 'subscription' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmmin' + +@description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') +param enableDefaultTelemetry bool = true + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: location +} + +module testDeployment '../../deploy.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-test-${serviceShort}' + params: { + name: '${serviceShort}001' + enableDefaultTelemetry: enableDefaultTelemetry + systemAssignedIdentity: true + primaryAgentPoolProfile: [ + { + name: 'systempool' + count: 1 + vmSize: 'Standard_DS2_v2' + mode: 'System' + } + ] + fluxConfiguration: { + enable: true + } + } +} diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index ade05bd24f..5cf35ac461 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -560,7 +560,7 @@ module fluxExtension '../../Microsoft.KubernetesConfiguration/extensions/deploy. name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxExtension' params: { clusterName: managedCluster.name - name: '${managedCluster.name}-fluxExtension' + name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : '${managedCluster.name}-fluxExtension' extensionType: 'microsoft.flux' configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? fluxConfiguration.configurationProtectedSettings : {} configurationSettings: contains(fluxConfiguration, 'configurationSettings') ? fluxConfiguration.configurationSettings : {} diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index d174227baa..abdb7a277f 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -379,7 +379,7 @@ The following module usage examples are retrieved from the content of the files >**Note**: Each example lists all the required parameters first, followed by the rest - each in alphabetical order. -

Example 1: Min

+

Example 1: Minflux

@@ -401,6 +401,9 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice ] // Non-required parameters enableDefaultTelemetry: '' + fluxConfiguration: { + enable: true + } systemAssignedIdentity: true } } @@ -436,6 +439,11 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice "enableDefaultTelemetry": { "value": "" }, + "fluxConfiguration": { + "value": { + "enable": true + } + }, "systemAssignedIdentity": { "value": true } From 4cb6d80758f997829196181d95444c997038aad4 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 12:18:44 +0100 Subject: [PATCH 07/29] enable commit/branch based CI --- .../modulePipelines/ms.containerservice.managedclusters.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azuredevops/modulePipelines/ms.containerservice.managedclusters.yml b/.azuredevops/modulePipelines/ms.containerservice.managedclusters.yml index 19a5f032d9..c43c0a47ee 100644 --- a/.azuredevops/modulePipelines/ms.containerservice.managedclusters.yml +++ b/.azuredevops/modulePipelines/ms.containerservice.managedclusters.yml @@ -12,7 +12,7 @@ parameters: - name: removeDeployment displayName: Remove deployed module type: boolean - default: true + default: false - name: prerelease displayName: Publish prerelease module type: boolean @@ -25,6 +25,7 @@ trigger: branches: include: - main + - 'users/mast/k8sExtensions' paths: include: - '/.azuredevops/modulePipelines/ms.containerservice.managedclusters.yml' From fa146368c426e933f378336c4efcde1e26c13162 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 12:52:24 +0100 Subject: [PATCH 08/29] *Test flux ext + config --- .../min/deploy.test.bicep | 0 .../.test/minFlux/deploy.test.bicep | 14 ++- .../managedClusters/deploy.bicep | 22 ++++- .../managedClusters/readme.md | 97 ++++++++++++++++++- 4 files changed, 128 insertions(+), 5 deletions(-) rename modules/Microsoft.ContainerService/managedClusters/{.otherTests => .test}/min/deploy.test.bicep (100%) diff --git a/modules/Microsoft.ContainerService/managedClusters/.otherTests/min/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep similarity index 100% rename from modules/Microsoft.ContainerService/managedClusters/.otherTests/min/deploy.test.bicep rename to modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep index 7321c61b31..b7e5236620 100644 --- a/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep @@ -12,7 +12,7 @@ param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceS param location string = deployment().location @description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') -param serviceShort string = 'csmmin' +param serviceShort string = 'csmminflux' @description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') param enableDefaultTelemetry bool = true @@ -45,6 +45,18 @@ module testDeployment '../../deploy.bicep' = { ] fluxConfiguration: { enable: true + namespace: 'flux-system' + scope: 'cluster' + sourceKind: 'GitRepository' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } } } } diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index 5cf35ac461..e91942fbe0 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -556,7 +556,7 @@ module managedCluster_agentPools 'agentPools/deploy.bicep' = [for (agentPool, in } }] -module fluxExtension '../../Microsoft.KubernetesConfiguration/extensions/deploy.bicep' = if (!empty(fluxConfiguration)) { +module managedCluster_fluxExtension '../../Microsoft.KubernetesConfiguration/extensions/deploy.bicep' = if (!empty(fluxConfiguration)) { name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxExtension' params: { clusterName: managedCluster.name @@ -573,6 +573,26 @@ module fluxExtension '../../Microsoft.KubernetesConfiguration/extensions/deploy. } } +module managedCluster_fluxConfiguration '../../Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep' = if (!empty(fluxConfiguration)) { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxConfiguration' + params: { + enableDefaultTelemetry: enableDefaultTelemetry + name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : '${managedCluster.name}-fluxConfiguration' + clusterName: managedCluster.name + bucket: contains(fluxConfiguration, 'bucket') ? (!empty(fluxConfiguration.bucket) ? fluxConfiguration.bucket : null) : null + configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? (!empty(fluxConfiguration.configurationProtectedSettings) ? fluxConfiguration.configurationProtectedSettings : {}) : {} + gitRepository: contains(fluxConfiguration, 'gitRepository') ? (!empty(fluxConfiguration.gitRepository) ? fluxConfiguration.gitRepository : null) : null + kustomizations: contains(fluxConfiguration, 'kustomizations') ? (!empty(fluxConfiguration.kustomizations) ? fluxConfiguration.kustomizations : {}) : {} + namespace: fluxConfiguration.namespace + scope: fluxConfiguration.scope + sourceKind: fluxConfiguration.sourceKind + suspend: contains(fluxConfiguration, 'suspend') ? (!empty(fluxConfiguration.suspend) ? fluxConfiguration.suspend : false) : false + } + dependsOn: [ + managedCluster_fluxExtension + ] +} + resource managedCluster_lock 'Microsoft.Authorization/locks@2020-05-01' = if (!empty(lock)) { name: '${managedCluster.name}-${lock}-lock' properties: { diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index abdb7a277f..a39a00361a 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -20,6 +20,7 @@ This module deploys Azure Kubernetes Cluster (AKS). | `Microsoft.ContainerService/managedClusters/agentPools` | [2022-09-01](https://docs.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2022-09-01/managedClusters/agentPools) | | `Microsoft.Insights/diagnosticSettings` | [2021-05-01-preview](https://docs.microsoft.com/en-us/azure/templates/Microsoft.Insights/2021-05-01-preview/diagnosticSettings) | | `Microsoft.KubernetesConfiguration/extensions` | [2022-03-01](https://docs.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/2022-03-01/extensions) | +| `Microsoft.KubernetesConfiguration/fluxConfigurations` | [2022-03-01](https://docs.microsoft.com/en-us/azure/templates/Microsoft.KubernetesConfiguration/2022-03-01/fluxConfigurations) | ## Parameters @@ -371,6 +372,7 @@ This section gives you an overview of all local-referenced module files (i.e., o | Reference | Type | | :-- | :-- | | `Microsoft.KubernetesConfiguration/extensions` | Local reference | +| `Microsoft.KubernetesConfiguration/fluxConfigurations` | Local reference | ## Deployment examples @@ -379,7 +381,7 @@ The following module usage examples are retrieved from the content of the files >**Note**: Each example lists all the required parameters first, followed by the rest - each in alphabetical order. -

Example 1: Minflux

+

Example 1: Min

@@ -401,8 +403,86 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice ] // Non-required parameters enableDefaultTelemetry: '' + systemAssignedIdentity: true + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmmin001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "count": 1, + "mode": "System", + "name": "systempool", + "vmSize": "Standard_DS2_v2" + } + ] + }, + // Non-required parameters + "enableDefaultTelemetry": { + "value": "" + }, + "systemAssignedIdentity": { + "value": true + } + } +} +``` + +
+

+ +

Example 2: Minflux

+ +
+ +via Bicep module + +```bicep +module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { + name: '${uniqueString(deployment().name, location)}-test-csmminflux' + params: { + // Required parameters + name: 'csmminflux001' + primaryAgentPoolProfile: [ + { + count: 1 + mode: 'System' + name: 'systempool' + vmSize: 'Standard_DS2_v2' + } + ] + // Non-required parameters + enableDefaultTelemetry: '' fluxConfiguration: { enable: true + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + namespace: 'flux-system' + sourceKind: 'GitRepository' } systemAssignedIdentity: true } @@ -423,7 +503,7 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice "parameters": { // Required parameters "name": { - "value": "csmmin001" + "value": "csmminflux001" }, "primaryAgentPoolProfile": { "value": [ @@ -441,7 +521,18 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice }, "fluxConfiguration": { "value": { - "enable": true + "enable": true, + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" + }, + "namespace": "flux-system", + "sourceKind": "GitRepository" } }, "systemAssignedIdentity": { From e942898a23b25f5153d0587d397b2663d4fb24ea Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 12:53:33 +0100 Subject: [PATCH 09/29] restore workflow file --- .../modulePipelines/ms.containerservice.managedclusters.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.azuredevops/modulePipelines/ms.containerservice.managedclusters.yml b/.azuredevops/modulePipelines/ms.containerservice.managedclusters.yml index c43c0a47ee..19a5f032d9 100644 --- a/.azuredevops/modulePipelines/ms.containerservice.managedclusters.yml +++ b/.azuredevops/modulePipelines/ms.containerservice.managedclusters.yml @@ -12,7 +12,7 @@ parameters: - name: removeDeployment displayName: Remove deployed module type: boolean - default: false + default: true - name: prerelease displayName: Publish prerelease module type: boolean @@ -25,7 +25,6 @@ trigger: branches: include: - main - - 'users/mast/k8sExtensions' paths: include: - '/.azuredevops/modulePipelines/ms.containerservice.managedclusters.yml' From 0a81b7829c7a3b533b594e46aefe4b6a5504dac3 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 13:05:51 +0100 Subject: [PATCH 10/29] fix for flux config --- .../managedClusters/deploy.bicep | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index e91942fbe0..ef7f354997 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -577,16 +577,16 @@ module managedCluster_fluxConfiguration '../../Microsoft.KubernetesConfiguration name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxConfiguration' params: { enableDefaultTelemetry: enableDefaultTelemetry - name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : '${managedCluster.name}-fluxConfiguration' clusterName: managedCluster.name - bucket: contains(fluxConfiguration, 'bucket') ? (!empty(fluxConfiguration.bucket) ? fluxConfiguration.bucket : null) : null - configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? (!empty(fluxConfiguration.configurationProtectedSettings) ? fluxConfiguration.configurationProtectedSettings : {}) : {} - gitRepository: contains(fluxConfiguration, 'gitRepository') ? (!empty(fluxConfiguration.gitRepository) ? fluxConfiguration.gitRepository : null) : null - kustomizations: contains(fluxConfiguration, 'kustomizations') ? (!empty(fluxConfiguration.kustomizations) ? fluxConfiguration.kustomizations : {}) : {} - namespace: fluxConfiguration.namespace scope: fluxConfiguration.scope + namespace: fluxConfiguration.namespace sourceKind: fluxConfiguration.sourceKind - suspend: contains(fluxConfiguration, 'suspend') ? (!empty(fluxConfiguration.suspend) ? fluxConfiguration.suspend : false) : false + name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : '${managedCluster.name}-fluxConfiguration' + bucket: contains(fluxConfiguration, 'bucket') ? fluxConfiguration.bucket : null + configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? fluxConfiguration.configurationProtectedSettings : {} + gitRepository: contains(fluxConfiguration, 'gitRepository') ? fluxConfiguration.gitRepository : null + kustomizations: contains(fluxConfiguration, 'kustomizations') ? fluxConfiguration.kustomizations : {} + suspend: contains(fluxConfiguration, 'suspend') ? fluxConfiguration.suspend : false } dependsOn: [ managedCluster_fluxExtension From f02301d7544d7386265ebb8687d8a80be0087a00 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 13:18:33 +0100 Subject: [PATCH 11/29] fix for defaults --- .../Microsoft.ContainerService/managedClusters/deploy.bicep | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index ef7f354997..12e714576f 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -582,9 +582,9 @@ module managedCluster_fluxConfiguration '../../Microsoft.KubernetesConfiguration namespace: fluxConfiguration.namespace sourceKind: fluxConfiguration.sourceKind name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : '${managedCluster.name}-fluxConfiguration' - bucket: contains(fluxConfiguration, 'bucket') ? fluxConfiguration.bucket : null + bucket: contains(fluxConfiguration, 'bucket') ? fluxConfiguration.bucket : {} configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? fluxConfiguration.configurationProtectedSettings : {} - gitRepository: contains(fluxConfiguration, 'gitRepository') ? fluxConfiguration.gitRepository : null + gitRepository: contains(fluxConfiguration, 'gitRepository') ? fluxConfiguration.gitRepository : {} kustomizations: contains(fluxConfiguration, 'kustomizations') ? fluxConfiguration.kustomizations : {} suspend: contains(fluxConfiguration, 'suspend') ? fluxConfiguration.suspend : false } From 8794a3de5fd63ec3a18b33bb4b42f01c9eba00a2 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 13:41:13 +0100 Subject: [PATCH 12/29] fix flux config name --- modules/Microsoft.ContainerService/managedClusters/deploy.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index 12e714576f..8ca7e6a5d0 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -581,7 +581,7 @@ module managedCluster_fluxConfiguration '../../Microsoft.KubernetesConfiguration scope: fluxConfiguration.scope namespace: fluxConfiguration.namespace sourceKind: fluxConfiguration.sourceKind - name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : '${managedCluster.name}-fluxConfiguration' + name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : '${managedCluster.name}-fluxconfiguration' bucket: contains(fluxConfiguration, 'bucket') ? fluxConfiguration.bucket : {} configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? fluxConfiguration.configurationProtectedSettings : {} gitRepository: contains(fluxConfiguration, 'gitRepository') ? fluxConfiguration.gitRepository : {} From f991f199e25afddb18379a0e2351f3d998f65f6d Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 14:48:01 +0100 Subject: [PATCH 13/29] added flux settings and configurations --- .../.test/minFlux/deploy.test.bicep | 26 +-- .../.test/minFluxDouble/deploy.test.bicep | 75 ++++++++ .../managedClusters/deploy.bicep | 31 ++-- .../managedClusters/readme.md | 173 +++++++++++++++--- 4 files changed, 253 insertions(+), 52 deletions(-) create mode 100644 modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep index b7e5236620..9678de6266 100644 --- a/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep @@ -43,20 +43,20 @@ module testDeployment '../../deploy.bicep' = { mode: 'System' } ] - fluxConfiguration: { - enable: true - namespace: 'flux-system' - scope: 'cluster' - sourceKind: 'GitRepository' - gitRepository: { - repositoryRef: { - branch: 'main' + fluxConfigurations: [ + { + namespace: 'flux-system' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' } - sshKnownHosts: '' - syncIntervalInSeconds: 300 - timeoutInSeconds: 180 - url: 'https://github.com/mspnp/aks-baseline' } - } + ] } } diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep new file mode 100644 index 0000000000..ab8433006f --- /dev/null +++ b/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep @@ -0,0 +1,75 @@ +targetScope = 'subscription' + +// ========== // +// Parameters // +// ========== // + +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'csmminflux' + +@description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') +param enableDefaultTelemetry bool = true + +// ============ // +// Dependencies // +// ============ // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: location +} + +module testDeployment '../../deploy.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-test-${serviceShort}' + params: { + name: '${serviceShort}001' + enableDefaultTelemetry: enableDefaultTelemetry + systemAssignedIdentity: true + primaryAgentPoolProfile: [ + { + name: 'systempool' + count: 1 + vmSize: 'Standard_DS2_v2' + mode: 'System' + } + ] + fluxConfigurations: [ + { + namespace: 'flux-system' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + } + { + namespace: 'flux-system-helm' + scope: 'cluster' + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' + } + } + ] + } +} diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index 8ca7e6a5d0..a9e989a08f 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -295,8 +295,14 @@ param tags object = {} @description('Optional. The resource ID of the disc encryption set to apply to the cluster. For security reasons, this value should be provided.') param diskEncryptionSetID string = '' -@description('Optional. A flux configuraiton.') -param fluxConfiguration object = {} +@description('Optional. ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is "true".') +param fluxReleaseTrain string = 'Stable' + +@description('Optional. Version of the extension for this extension, if it is "pinned" to a specific version.') +param fluxVersion string = '' + +@description('Optional. A list of flux configuraitons.') +param fluxConfigurations array = [] @description('Optional. The name of logs that will be streamed. "allLogs" includes all possible logs for the resource.') @allowed([ @@ -556,32 +562,29 @@ module managedCluster_agentPools 'agentPools/deploy.bicep' = [for (agentPool, in } }] -module managedCluster_fluxExtension '../../Microsoft.KubernetesConfiguration/extensions/deploy.bicep' = if (!empty(fluxConfiguration)) { +module managedCluster_fluxExtension '../../Microsoft.KubernetesConfiguration/extensions/deploy.bicep' = if (!empty(fluxConfigurations)) { name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxExtension' params: { clusterName: managedCluster.name - name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : '${managedCluster.name}-fluxExtension' + name: 'flux' extensionType: 'microsoft.flux' - configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? fluxConfiguration.configurationProtectedSettings : {} - configurationSettings: contains(fluxConfiguration, 'configurationSettings') ? fluxConfiguration.configurationSettings : {} enableDefaultTelemetry: enableReferencedModulesTelemetry location: location releaseNamespace: 'flux-system' - releaseTrain: contains(fluxConfiguration, 'releaseTrain') ? fluxConfiguration.releaseTrain : 'Stable' - targetNamespace: contains(fluxConfiguration, 'targetNamespace') ? fluxConfiguration.targetNamespace : '' - version: contains(fluxConfiguration, 'version') ? fluxConfiguration.version : '' + releaseTrain: !empty(fluxReleaseTrain) ? fluxReleaseTrain : 'Stable' + version: !empty(fluxVersion) ? fluxVersion : '' } } -module managedCluster_fluxConfiguration '../../Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep' = if (!empty(fluxConfiguration)) { - name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxConfiguration' +module managedCluster_fluxConfiguration '../../Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep' = [for (fluxConfiguration, index) in fluxConfigurations: { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxConfiguration${index}' params: { enableDefaultTelemetry: enableDefaultTelemetry clusterName: managedCluster.name scope: fluxConfiguration.scope namespace: fluxConfiguration.namespace - sourceKind: fluxConfiguration.sourceKind - name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : '${managedCluster.name}-fluxconfiguration' + sourceKind: contains(fluxConfiguration, 'gitRepository') ? 'GitRepository' : 'Bucket' + name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : '${managedCluster.name}-fluxconfiguration${index}' bucket: contains(fluxConfiguration, 'bucket') ? fluxConfiguration.bucket : {} configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? fluxConfiguration.configurationProtectedSettings : {} gitRepository: contains(fluxConfiguration, 'gitRepository') ? fluxConfiguration.gitRepository : {} @@ -591,7 +594,7 @@ module managedCluster_fluxConfiguration '../../Microsoft.KubernetesConfiguration dependsOn: [ managedCluster_fluxExtension ] -} +}] resource managedCluster_lock 'Microsoft.Authorization/locks@2020-05-01' = if (!empty(lock)) { name: '${managedCluster.name}-${lock}-lock' diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index a39a00361a..a23f1fbb97 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -104,7 +104,9 @@ This module deploys Azure Kubernetes Cluster (AKS). | `enablePrivateClusterPublicFQDN` | bool | `False` | | Whether to create additional public FQDN for private cluster or not. | | `enableRBAC` | bool | `True` | | Whether to enable Kubernetes Role-Based Access Control. | | `enableSecretRotation` | string | `'false'` | `[false, true]` | Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation. | -| `fluxConfiguration` | object | `{object}` | | A flux configuraiton. | +| `fluxConfigurations` | array | `[]` | | A list of flux configuraitons. | +| `fluxReleaseTrain` | string | `'Stable'` | | ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is "true". | +| `fluxVersion` | string | `''` | | Version of the extension for this extension, if it is "pinned" to a specific version. | | `httpApplicationRoutingEnabled` | bool | `False` | | Specifies whether the httpApplicationRouting add-on is enabled or not. | | `ingressApplicationGatewayEnabled` | bool | `False` | | Specifies whether the ingressApplicationGateway (AGIC) add-on is enabled or not. | | `kubeDashboardEnabled` | bool | `False` | | Specifies whether the kubeDashboard add-on is enabled or not. | @@ -470,20 +472,129 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice ] // Non-required parameters enableDefaultTelemetry: '' - fluxConfiguration: { - enable: true - gitRepository: { - repositoryRef: { - branch: 'main' + fluxConfigurations: [ + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' } - sshKnownHosts: '' - syncIntervalInSeconds: 300 - timeoutInSeconds: 180 - url: 'https://github.com/mspnp/aks-baseline' + namespace: 'flux-system' } - namespace: 'flux-system' - sourceKind: 'GitRepository' + ] + systemAssignedIdentity: true + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmminflux001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "count": 1, + "mode": "System", + "name": "systempool", + "vmSize": "Standard_DS2_v2" + } + ] + }, + // Non-required parameters + "enableDefaultTelemetry": { + "value": "" + }, + "fluxConfigurations": { + "value": [ + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" + }, + "namespace": "flux-system" + } + ] + }, + "systemAssignedIdentity": { + "value": true } + } +} +``` + +
+

+ +

Example 3: Minfluxdouble

+ +
+ +via Bicep module + +```bicep +module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { + name: '${uniqueString(deployment().name, location)}-test-csmminflux' + params: { + // Required parameters + name: 'csmminflux001' + primaryAgentPoolProfile: [ + { + count: 1 + mode: 'System' + name: 'systempool' + vmSize: 'Standard_DS2_v2' + } + ] + // Non-required parameters + enableDefaultTelemetry: '' + fluxConfigurations: [ + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + namespace: 'flux-system' + } + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' + } + namespace: 'flux-system-helm' + } + ] systemAssignedIdentity: true } } @@ -519,21 +630,33 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice "enableDefaultTelemetry": { "value": "" }, - "fluxConfiguration": { - "value": { - "enable": true, - "gitRepository": { - "repositoryRef": { - "branch": "main" + "fluxConfigurations": { + "value": [ + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" }, - "sshKnownHosts": "", - "syncIntervalInSeconds": 300, - "timeoutInSeconds": 180, - "url": "https://github.com/mspnp/aks-baseline" + "namespace": "flux-system" }, - "namespace": "flux-system", - "sourceKind": "GitRepository" - } + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/Azure/gitops-flux2-kustomize-helm-mt" + }, + "namespace": "flux-system-helm" + } + ] }, "systemAssignedIdentity": { "value": true From 5bd3564f51d7c685c96fd97c2c058e71f5d95612 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 15:16:30 +0100 Subject: [PATCH 14/29] fix config name --- .../managedClusters/.test/minFluxDouble/deploy.test.bicep | 2 +- modules/Microsoft.ContainerService/managedClusters/deploy.bicep | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep index ab8433006f..97ba9e6d4a 100644 --- a/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep @@ -12,7 +12,7 @@ param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceS param location string = deployment().location @description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') -param serviceShort string = 'csmminflux' +param serviceShort string = 'csmminfluxdouble' @description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') param enableDefaultTelemetry bool = true diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index a9e989a08f..5ed7c7450d 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -584,7 +584,7 @@ module managedCluster_fluxConfiguration '../../Microsoft.KubernetesConfiguration scope: fluxConfiguration.scope namespace: fluxConfiguration.namespace sourceKind: contains(fluxConfiguration, 'gitRepository') ? 'GitRepository' : 'Bucket' - name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : '${managedCluster.name}-fluxconfiguration${index}' + name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : toLower('${managedCluster.name}-fluxconfiguration${index}') bucket: contains(fluxConfiguration, 'bucket') ? fluxConfiguration.bucket : {} configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? fluxConfiguration.configurationProtectedSettings : {} gitRepository: contains(fluxConfiguration, 'gitRepository') ? fluxConfiguration.gitRepository : {} From 8b664ac3ec0f2b02ae48d4a323159b93b9c2878c Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 15:21:14 +0100 Subject: [PATCH 15/29] refresh readme --- .../Microsoft.ContainerService/managedClusters/readme.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index a23f1fbb97..98eff77bba 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -555,10 +555,10 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice ```bicep module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { - name: '${uniqueString(deployment().name, location)}-test-csmminflux' + name: '${uniqueString(deployment().name, location)}-test-csmminfluxdouble' params: { // Required parameters - name: 'csmminflux001' + name: 'csmminfluxdouble001' primaryAgentPoolProfile: [ { count: 1 @@ -614,7 +614,7 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice "parameters": { // Required parameters "name": { - "value": "csmminflux001" + "value": "csmminfluxdouble001" }, "primaryAgentPoolProfile": { "value": [ From 629db005d03bb260ed01f639dce75439fabf7d47 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 15:58:36 +0100 Subject: [PATCH 16/29] shorter name --- .../managedClusters/.test/minFlux/deploy.test.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep index 9678de6266..58fc624382 100644 --- a/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep @@ -12,7 +12,7 @@ param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceS param location string = deployment().location @description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') -param serviceShort string = 'csmminflux' +param serviceShort string = 'csmmf2' @description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') param enableDefaultTelemetry bool = true From e830603f0615b2a3642e326839eb1667a496b6a3 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 16:22:57 +0100 Subject: [PATCH 17/29] fix readme --- .../Microsoft.ContainerService/managedClusters/readme.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index 9e0e6e53ca..908e658774 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -459,10 +459,10 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice ```bicep module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { - name: '${uniqueString(deployment().name, location)}-test-csmminflux' + name: '${uniqueString(deployment().name, location)}-test-csmmf2' params: { // Required parameters - name: 'csmminflux001' + name: 'csmmf2001' primaryAgentPoolProfile: [ { count: 1 @@ -506,7 +506,7 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice "parameters": { // Required parameters "name": { - "value": "csmminflux001" + "value": "csmmf2001" }, "primaryAgentPoolProfile": { "value": [ From 7229cefb735254d29bd6fa0b9fc11972f7e6ba10 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 16:34:33 +0100 Subject: [PATCH 18/29] fixing deployment names/lengths --- .../managedClusters/.test/minFlux/deploy.test.bicep | 2 +- .../.test/minFluxDouble/deploy.test.bicep | 2 +- .../managedClusters/readme.md | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep index 58fc624382..852b04aa3a 100644 --- a/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep @@ -12,7 +12,7 @@ param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceS param location string = deployment().location @description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') -param serviceShort string = 'csmmf2' +param serviceShort string = 'csmmf' @description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') param enableDefaultTelemetry bool = true diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep index 97ba9e6d4a..f86bae8b4f 100644 --- a/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep @@ -12,7 +12,7 @@ param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceS param location string = deployment().location @description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') -param serviceShort string = 'csmminfluxdouble' +param serviceShort string = 'csmmf2' @description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') param enableDefaultTelemetry bool = true diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index 908e658774..9da31192ed 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -459,10 +459,10 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice ```bicep module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { - name: '${uniqueString(deployment().name, location)}-test-csmmf2' + name: '${uniqueString(deployment().name, location)}-test-csmmf' params: { // Required parameters - name: 'csmmf2001' + name: 'csmmf001' primaryAgentPoolProfile: [ { count: 1 @@ -506,7 +506,7 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice "parameters": { // Required parameters "name": { - "value": "csmmf2001" + "value": "csmmf001" }, "primaryAgentPoolProfile": { "value": [ @@ -556,10 +556,10 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice ```bicep module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { - name: '${uniqueString(deployment().name, location)}-test-csmminfluxdouble' + name: '${uniqueString(deployment().name, location)}-test-csmmf2' params: { // Required parameters - name: 'csmminfluxdouble001' + name: 'csmmf2001' primaryAgentPoolProfile: [ { count: 1 @@ -615,7 +615,7 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice "parameters": { // Required parameters "name": { - "value": "csmminfluxdouble001" + "value": "csmmf2001" }, "primaryAgentPoolProfile": { "value": [ From fe00284d742c86bcd6bf64b40bce323a4e19f0c7 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 17:51:18 +0100 Subject: [PATCH 19/29] moving tests back --- .../{.otherTests => .test}/azure/dependencies.bicep | 0 .../{.otherTests => .test}/azure/deploy.test.bicep | 0 .../{.otherTests => .test}/kubenet/dependencies.bicep | 0 .../{.otherTests => .test}/kubenet/deploy.test.bicep | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename modules/Microsoft.ContainerService/managedClusters/{.otherTests => .test}/azure/dependencies.bicep (100%) rename modules/Microsoft.ContainerService/managedClusters/{.otherTests => .test}/azure/deploy.test.bicep (100%) rename modules/Microsoft.ContainerService/managedClusters/{.otherTests => .test}/kubenet/dependencies.bicep (100%) rename modules/Microsoft.ContainerService/managedClusters/{.otherTests => .test}/kubenet/deploy.test.bicep (100%) diff --git a/modules/Microsoft.ContainerService/managedClusters/.otherTests/azure/dependencies.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/azure/dependencies.bicep similarity index 100% rename from modules/Microsoft.ContainerService/managedClusters/.otherTests/azure/dependencies.bicep rename to modules/Microsoft.ContainerService/managedClusters/.test/azure/dependencies.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.otherTests/azure/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/azure/deploy.test.bicep similarity index 100% rename from modules/Microsoft.ContainerService/managedClusters/.otherTests/azure/deploy.test.bicep rename to modules/Microsoft.ContainerService/managedClusters/.test/azure/deploy.test.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.otherTests/kubenet/dependencies.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/kubenet/dependencies.bicep similarity index 100% rename from modules/Microsoft.ContainerService/managedClusters/.otherTests/kubenet/dependencies.bicep rename to modules/Microsoft.ContainerService/managedClusters/.test/kubenet/dependencies.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.otherTests/kubenet/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/kubenet/deploy.test.bicep similarity index 100% rename from modules/Microsoft.ContainerService/managedClusters/.otherTests/kubenet/deploy.test.bicep rename to modules/Microsoft.ContainerService/managedClusters/.test/kubenet/deploy.test.bicep From 620ec809c0a82228564808f690cceec09e0eeb4f Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Thu, 9 Feb 2023 17:51:25 +0100 Subject: [PATCH 20/29] refreshing readme --- .../managedClusters/readme.md | 498 +++++++++++++++++- 1 file changed, 495 insertions(+), 3 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index 9da31192ed..8aaa7340de 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -384,7 +384,499 @@ The following module usage examples are retrieved from the content of the files >**Note**: Each example lists all the required parameters first, followed by the rest - each in alphabetical order. -

Example 1: Min

+

Example 1: Azure

+ +
+ +via Bicep module + +```bicep +module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { + name: '${uniqueString(deployment().name, location)}-test-csmaz' + params: { + // Required parameters + name: '<>csmaz001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '1' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + osDiskSizeGB: 0 + osType: 'Linux' + serviceCidr: '' + storageProfile: 'ManagedDisks' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + ] + // Non-required parameters + agentPools: [ + { + availabilityZones: [ + '1' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + storageProfile: 'ManagedDisks' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + { + availabilityZones: [ + '1' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + storageProfile: 'ManagedDisks' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + vnetSubnetID: '' + } + ] + aksClusterNetworkPlugin: 'azure' + diagnosticEventHubAuthorizationRuleId: '' + diagnosticEventHubName: '' + diagnosticLogsRetentionInDays: 7 + diagnosticStorageAccountId: '' + diagnosticWorkspaceId: '' + diskEncryptionSetID: '' + enableDefaultTelemetry: '' + lock: 'CanNotDelete' + roleAssignments: [ + { + principalIds: [ + '' + ] + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'Reader' + } + ] + systemAssignedIdentity: true + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "<>csmaz001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "availabilityZones": [ + "1" + ], + "count": 1, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "mode": "System", + "name": "systempool", + "osDiskSizeGB": 0, + "osType": "Linux", + "serviceCidr": "", + "storageProfile": "ManagedDisks", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + } + ] + }, + // Non-required parameters + "agentPools": { + "value": [ + { + "availabilityZones": [ + "1" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool1", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "storageProfile": "ManagedDisks", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + }, + { + "availabilityZones": [ + "1" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool2", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "storageProfile": "ManagedDisks", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2", + "vnetSubnetID": "" + } + ] + }, + "aksClusterNetworkPlugin": { + "value": "azure" + }, + "diagnosticEventHubAuthorizationRuleId": { + "value": "" + }, + "diagnosticEventHubName": { + "value": "" + }, + "diagnosticLogsRetentionInDays": { + "value": 7 + }, + "diagnosticStorageAccountId": { + "value": "" + }, + "diagnosticWorkspaceId": { + "value": "" + }, + "diskEncryptionSetID": { + "value": "" + }, + "enableDefaultTelemetry": { + "value": "" + }, + "lock": { + "value": "CanNotDelete" + }, + "roleAssignments": { + "value": [ + { + "principalIds": [ + "" + ], + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "Reader" + } + ] + }, + "systemAssignedIdentity": { + "value": true + } + } +} +``` + +
+

+ +

Example 2: Kubenet

+ +
+ +via Bicep module + +```bicep +module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { + name: '${uniqueString(deployment().name, location)}-test-csmkube' + params: { + // Required parameters + name: '<>csmkube001' + primaryAgentPoolProfile: [ + { + availabilityZones: [ + '1' + ] + count: 1 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + mode: 'System' + name: 'systempool' + osDiskSizeGB: 0 + osType: 'Linux' + serviceCidr: '' + storageProfile: 'ManagedDisks' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + // Non-required parameters + agentPools: [ + { + availabilityZones: [ + '1' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool1' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + storageProfile: 'ManagedDisks' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + { + availabilityZones: [ + '1' + ] + count: 2 + enableAutoScaling: true + maxCount: 3 + maxPods: 30 + minCount: 1 + minPods: 2 + mode: 'User' + name: 'userpool2' + nodeLabels: {} + nodeTaints: [ + 'CriticalAddonsOnly=true:NoSchedule' + ] + osDiskSizeGB: 128 + osType: 'Linux' + scaleSetEvictionPolicy: 'Delete' + scaleSetPriority: 'Regular' + storageProfile: 'ManagedDisks' + type: 'VirtualMachineScaleSets' + vmSize: 'Standard_DS2_v2' + } + ] + aksClusterNetworkPlugin: 'kubenet' + diagnosticEventHubAuthorizationRuleId: '' + diagnosticEventHubName: '' + diagnosticLogsRetentionInDays: 7 + diagnosticStorageAccountId: '' + diagnosticWorkspaceId: '' + enableDefaultTelemetry: '' + roleAssignments: [ + { + principalIds: [ + '' + ] + principalType: 'ServicePrincipal' + roleDefinitionIdOrName: 'Reader' + } + ] + userAssignedIdentities: { + '': {} + } + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "<>csmkube001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "availabilityZones": [ + "1" + ], + "count": 1, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "mode": "System", + "name": "systempool", + "osDiskSizeGB": 0, + "osType": "Linux", + "serviceCidr": "", + "storageProfile": "ManagedDisks", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + } + ] + }, + // Non-required parameters + "agentPools": { + "value": [ + { + "availabilityZones": [ + "1" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool1", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "storageProfile": "ManagedDisks", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + }, + { + "availabilityZones": [ + "1" + ], + "count": 2, + "enableAutoScaling": true, + "maxCount": 3, + "maxPods": 30, + "minCount": 1, + "minPods": 2, + "mode": "User", + "name": "userpool2", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "osDiskSizeGB": 128, + "osType": "Linux", + "scaleSetEvictionPolicy": "Delete", + "scaleSetPriority": "Regular", + "storageProfile": "ManagedDisks", + "type": "VirtualMachineScaleSets", + "vmSize": "Standard_DS2_v2" + } + ] + }, + "aksClusterNetworkPlugin": { + "value": "kubenet" + }, + "diagnosticEventHubAuthorizationRuleId": { + "value": "" + }, + "diagnosticEventHubName": { + "value": "" + }, + "diagnosticLogsRetentionInDays": { + "value": 7 + }, + "diagnosticStorageAccountId": { + "value": "" + }, + "diagnosticWorkspaceId": { + "value": "" + }, + "enableDefaultTelemetry": { + "value": "" + }, + "roleAssignments": { + "value": [ + { + "principalIds": [ + "" + ], + "principalType": "ServicePrincipal", + "roleDefinitionIdOrName": "Reader" + } + ] + }, + "userAssignedIdentities": { + "value": { + "": {} + } + } + } +} +``` + +
+

+ +

Example 3: Min

@@ -451,7 +943,7 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice

-

Example 2: Minflux

+

Example 4: Minflux

@@ -548,7 +1040,7 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice

-

Example 3: Minfluxdouble

+

Example 5: Minfluxdouble

From 156ebb501a5aef86a3102d4faa243d062cddd827 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 10 Feb 2023 10:19:44 +0100 Subject: [PATCH 21/29] Update modules/Microsoft.ContainerService/managedClusters/deploy.bicep Co-authored-by: Alexander Sehr --- modules/Microsoft.ContainerService/managedClusters/deploy.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index 228a7fafc6..296a95306c 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -562,7 +562,7 @@ module managedCluster_agentPools 'agentPools/deploy.bicep' = [for (agentPool, in } }] -module managedCluster_fluxExtension '../../Microsoft.KubernetesConfiguration/extensions/deploy.bicep' = if (!empty(fluxConfigurations)) { +module managedCluster_extension '../../Microsoft.KubernetesConfiguration/extensions/deploy.bicep' = if (!empty(fluxConfigurations)) { name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxExtension' params: { clusterName: managedCluster.name From ac802b3eacf2e15cff05b09fb99ddd5b1154130c Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 10 Feb 2023 11:48:26 +0100 Subject: [PATCH 22/29] Added flux configuration settings and fixed dependsOn --- .../managedClusters/deploy.bicep | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index 296a95306c..94e1eef2b2 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -301,6 +301,12 @@ param fluxReleaseTrain string = 'Stable' @description('Optional. Version of the extension for this extension, if it is "pinned" to a specific version.') param fluxVersion string = '' +@description('Optional. Configuration settings that are sensitive, as name-value pairs for configuring this extension.') +param fluxConfigurationProtectedSettings object = {} + +@description('Optional. Configuration settings, as name-value pairs for configuring this extension.') +param fluxConfigurationSettings object = {} + @description('Optional. A list of flux configuraitons.') param fluxConfigurations array = [] @@ -566,10 +572,12 @@ module managedCluster_extension '../../Microsoft.KubernetesConfiguration/extensi name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxExtension' params: { clusterName: managedCluster.name - name: 'flux' - extensionType: 'microsoft.flux' + configurationProtectedSettings: !empty(fluxConfigurationProtectedSettings) ? fluxConfigurationProtectedSettings : {} + configurationSettings: !empty(fluxConfigurationSettings) ? fluxConfigurationSettings : {} enableDefaultTelemetry: enableReferencedModulesTelemetry + extensionType: 'microsoft.flux' location: location + name: 'flux' releaseNamespace: 'flux-system' releaseTrain: !empty(fluxReleaseTrain) ? fluxReleaseTrain : 'Stable' version: !empty(fluxVersion) ? fluxVersion : '' @@ -579,20 +587,20 @@ module managedCluster_extension '../../Microsoft.KubernetesConfiguration/extensi module managedCluster_fluxConfiguration '../../Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep' = [for (fluxConfiguration, index) in fluxConfigurations: { name: '${uniqueString(deployment().name, location)}-ManagedCluster-FluxConfiguration${index}' params: { - enableDefaultTelemetry: enableDefaultTelemetry - clusterName: managedCluster.name - scope: fluxConfiguration.scope - namespace: fluxConfiguration.namespace - sourceKind: contains(fluxConfiguration, 'gitRepository') ? 'GitRepository' : 'Bucket' - name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : toLower('${managedCluster.name}-fluxconfiguration${index}') bucket: contains(fluxConfiguration, 'bucket') ? fluxConfiguration.bucket : {} + clusterName: managedCluster.name configurationProtectedSettings: contains(fluxConfiguration, 'configurationProtectedSettings') ? fluxConfiguration.configurationProtectedSettings : {} + enableDefaultTelemetry: enableDefaultTelemetry gitRepository: contains(fluxConfiguration, 'gitRepository') ? fluxConfiguration.gitRepository : {} kustomizations: contains(fluxConfiguration, 'kustomizations') ? fluxConfiguration.kustomizations : {} + name: contains(fluxConfiguration, 'name') ? fluxConfiguration.name : toLower('${managedCluster.name}-fluxconfiguration${index}') + namespace: fluxConfiguration.namespace + scope: fluxConfiguration.scope + sourceKind: contains(fluxConfiguration, 'gitRepository') ? 'GitRepository' : 'Bucket' suspend: contains(fluxConfiguration, 'suspend') ? fluxConfiguration.suspend : false } dependsOn: [ - managedCluster_fluxExtension + managedCluster_extension ] }] From 41ee43a76a418ec45d5f64ed9176a3f9eca087ca Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 10 Feb 2023 11:49:02 +0100 Subject: [PATCH 23/29] refreshed docs --- modules/Microsoft.ContainerService/managedClusters/readme.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index 8aaa7340de..b22196b1a2 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -104,7 +104,9 @@ This module deploys Azure Kubernetes Cluster (AKS). | `enablePrivateClusterPublicFQDN` | bool | `False` | | Whether to create additional public FQDN for private cluster or not. | | `enableRBAC` | bool | `True` | | Whether to enable Kubernetes Role-Based Access Control. | | `enableSecretRotation` | string | `'false'` | `[false, true]` | Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation. | +| `fluxConfigurationProtectedSettings` | object | `{object}` | | Configuration settings that are sensitive, as name-value pairs for configuring this extension. | | `fluxConfigurations` | array | `[]` | | A list of flux configuraitons. | +| `fluxConfigurationSettings` | object | `{object}` | | Configuration settings, as name-value pairs for configuring this extension. | | `fluxReleaseTrain` | string | `'Stable'` | | ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is "true". | | `fluxVersion` | string | `''` | | Version of the extension for this extension, if it is "pinned" to a specific version. | | `httpApplicationRoutingEnabled` | bool | `False` | | Specifies whether the httpApplicationRouting add-on is enabled or not. | From 49d0a6901465641dcc91011057a133ddbe109956 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 10 Feb 2023 12:00:01 +0100 Subject: [PATCH 24/29] renamed test minfluxdouble to flux with more features --- .../{minFluxDouble => flux}/deploy.test.bicep | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) rename modules/Microsoft.ContainerService/managedClusters/.test/{minFluxDouble => flux}/deploy.test.bicep (69%) diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/flux/deploy.test.bicep similarity index 69% rename from modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep rename to modules/Microsoft.ContainerService/managedClusters/.test/flux/deploy.test.bicep index f86bae8b4f..a35c447784 100644 --- a/modules/Microsoft.ContainerService/managedClusters/.test/minFluxDouble/deploy.test.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/.test/flux/deploy.test.bicep @@ -43,6 +43,14 @@ module testDeployment '../../deploy.bicep' = { mode: 'System' } ] + fluxConfigurationSettings: { + 'helm-controller.enabled': 'true' + 'source-controller.enabled': 'true' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'true' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + } fluxConfigurations: [ { namespace: 'flux-system' @@ -69,6 +77,28 @@ module testDeployment '../../deploy.bicep' = { timeoutInSeconds: 180 url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' } + kustomizations: { + infra: { + path: './infrastructure' + dependsOn: [] + timeoutInSeconds: 600 + syncIntervalInSeconds: 600 + validation: 'none' + prune: true + } + apps: { + path: './apps/staging' + dependsOn: [ + { + kustomizationName: 'infra' + } + ] + timeoutInSeconds: 600 + syncIntervalInSeconds: 600 + retryIntervalInSeconds: 600 + prune: true + } + } } ] } From 60d1e89b02a9fa08e0d5e6b5bee52e2e74bd79e7 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Fri, 10 Feb 2023 12:38:01 +0100 Subject: [PATCH 25/29] refresh --- .../.test/min/deploy.test.bicep | 47 --- .../managedClusters/readme.md | 368 +++++++++--------- 2 files changed, 179 insertions(+), 236 deletions(-) delete mode 100644 modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep deleted file mode 100644 index b752b96308..0000000000 --- a/modules/Microsoft.ContainerService/managedClusters/.test/min/deploy.test.bicep +++ /dev/null @@ -1,47 +0,0 @@ -targetScope = 'subscription' - -// ========== // -// Parameters // -// ========== // - -@description('Optional. The name of the resource group to deploy for testing purposes.') -@maxLength(90) -param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceShort}-rg' - -@description('Optional. The location to deploy resources to.') -param location string = deployment().location - -@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') -param serviceShort string = 'csmmin' - -@description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') -param enableDefaultTelemetry bool = true - -// ============ // -// Dependencies // -// ============ // - -// General resources -// ================= -resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { - name: resourceGroupName - location: location -} - -module testDeployment '../../deploy.bicep' = { - scope: resourceGroup - name: '${uniqueString(deployment().name, location)}-test-${serviceShort}' - params: { - name: '${serviceShort}001' - enableDefaultTelemetry: enableDefaultTelemetry - systemAssignedIdentity: true - primaryAgentPoolProfile: [ - { - name: 'systempool' - count: 1 - vmSize: 'Standard_DS2_v2' - mode: 'System' - } - ] - } -} diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index b22196b1a2..d2768dca89 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -637,7 +637,185 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice

-

Example 2: Kubenet

+

Example 2: Flux

+ +
+ +via Bicep module + +```bicep +module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { + name: '${uniqueString(deployment().name, location)}-test-csmmf2' + params: { + // Required parameters + name: 'csmmf2001' + primaryAgentPoolProfile: [ + { + count: 1 + mode: 'System' + name: 'systempool' + vmSize: 'Standard_DS2_v2' + } + ] + // Non-required parameters + enableDefaultTelemetry: '' + fluxConfigurations: [ + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/mspnp/aks-baseline' + } + namespace: 'flux-system' + } + { + gitRepository: { + repositoryRef: { + branch: 'main' + } + sshKnownHosts: '' + syncIntervalInSeconds: 300 + timeoutInSeconds: 180 + url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' + } + kustomizations: { + apps: { + path: './apps/staging' + prune: true + retryIntervalInSeconds: 600 + syncIntervalInSeconds: 600 + timeoutInSeconds: 600 + } + infra: { + dependsOn: [] + path: './infrastructure' + prune: true + syncIntervalInSeconds: 600 + timeoutInSeconds: 600 + validation: 'none' + } + } + namespace: 'flux-system-helm' + } + ] + fluxConfigurationSettings: { + 'helm-controller.enabled': 'true' + 'image-automation-controller.enabled': 'false' + 'image-reflector-controller.enabled': 'false' + 'kustomize-controller.enabled': 'true' + 'notification-controller.enabled': 'true' + 'source-controller.enabled': 'true' + } + systemAssignedIdentity: true + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + // Required parameters + "name": { + "value": "csmmf2001" + }, + "primaryAgentPoolProfile": { + "value": [ + { + "count": 1, + "mode": "System", + "name": "systempool", + "vmSize": "Standard_DS2_v2" + } + ] + }, + // Non-required parameters + "enableDefaultTelemetry": { + "value": "" + }, + "fluxConfigurations": { + "value": [ + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/mspnp/aks-baseline" + }, + "namespace": "flux-system" + }, + { + "gitRepository": { + "repositoryRef": { + "branch": "main" + }, + "sshKnownHosts": "", + "syncIntervalInSeconds": 300, + "timeoutInSeconds": 180, + "url": "https://github.com/Azure/gitops-flux2-kustomize-helm-mt" + }, + "kustomizations": { + "apps": { + "dependsOn": [ + { + "kustomizationName": "infra" + } + ], + "path": "./apps/staging", + "prune": true, + "retryIntervalInSeconds": 600, + "syncIntervalInSeconds": 600, + "timeoutInSeconds": 600 + }, + "infra": { + "dependsOn": [], + "path": "./infrastructure", + "prune": true, + "syncIntervalInSeconds": 600, + "timeoutInSeconds": 600, + "validation": "none" + } + }, + "namespace": "flux-system-helm" + } + ] + }, + "fluxConfigurationSettings": { + "value": { + "helm-controller.enabled": "true", + "image-automation-controller.enabled": "false", + "image-reflector-controller.enabled": "false", + "kustomize-controller.enabled": "true", + "notification-controller.enabled": "true", + "source-controller.enabled": "true" + } + }, + "systemAssignedIdentity": { + "value": true + } + } +} +``` + +
+

+ +

Example 3: Kubenet

@@ -878,73 +1056,6 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice

-

Example 3: Min

- -
- -via Bicep module - -```bicep -module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { - name: '${uniqueString(deployment().name, location)}-test-csmmin' - params: { - // Required parameters - name: 'csmmin001' - primaryAgentPoolProfile: [ - { - count: 1 - mode: 'System' - name: 'systempool' - vmSize: 'Standard_DS2_v2' - } - ] - // Non-required parameters - enableDefaultTelemetry: '' - systemAssignedIdentity: true - } -} -``` - -
-

- -

- -via JSON Parameter file - -```json -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - // Required parameters - "name": { - "value": "csmmin001" - }, - "primaryAgentPoolProfile": { - "value": [ - { - "count": 1, - "mode": "System", - "name": "systempool", - "vmSize": "Standard_DS2_v2" - } - ] - }, - // Non-required parameters - "enableDefaultTelemetry": { - "value": "" - }, - "systemAssignedIdentity": { - "value": true - } - } -} -``` - -
-

-

Example 4: Minflux

@@ -1041,124 +1152,3 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice

- -

Example 5: Minfluxdouble

- -
- -via Bicep module - -```bicep -module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { - name: '${uniqueString(deployment().name, location)}-test-csmmf2' - params: { - // Required parameters - name: 'csmmf2001' - primaryAgentPoolProfile: [ - { - count: 1 - mode: 'System' - name: 'systempool' - vmSize: 'Standard_DS2_v2' - } - ] - // Non-required parameters - enableDefaultTelemetry: '' - fluxConfigurations: [ - { - gitRepository: { - repositoryRef: { - branch: 'main' - } - sshKnownHosts: '' - syncIntervalInSeconds: 300 - timeoutInSeconds: 180 - url: 'https://github.com/mspnp/aks-baseline' - } - namespace: 'flux-system' - } - { - gitRepository: { - repositoryRef: { - branch: 'main' - } - sshKnownHosts: '' - syncIntervalInSeconds: 300 - timeoutInSeconds: 180 - url: 'https://github.com/Azure/gitops-flux2-kustomize-helm-mt' - } - namespace: 'flux-system-helm' - } - ] - systemAssignedIdentity: true - } -} -``` - -
-

- -

- -via JSON Parameter file - -```json -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - // Required parameters - "name": { - "value": "csmmf2001" - }, - "primaryAgentPoolProfile": { - "value": [ - { - "count": 1, - "mode": "System", - "name": "systempool", - "vmSize": "Standard_DS2_v2" - } - ] - }, - // Non-required parameters - "enableDefaultTelemetry": { - "value": "" - }, - "fluxConfigurations": { - "value": [ - { - "gitRepository": { - "repositoryRef": { - "branch": "main" - }, - "sshKnownHosts": "", - "syncIntervalInSeconds": 300, - "timeoutInSeconds": 180, - "url": "https://github.com/mspnp/aks-baseline" - }, - "namespace": "flux-system" - }, - { - "gitRepository": { - "repositoryRef": { - "branch": "main" - }, - "sshKnownHosts": "", - "syncIntervalInSeconds": 300, - "timeoutInSeconds": 180, - "url": "https://github.com/Azure/gitops-flux2-kustomize-helm-mt" - }, - "namespace": "flux-system-helm" - } - ] - }, - "systemAssignedIdentity": { - "value": true - } - } -} -``` - -
-

From a4057f3714e06a8a61444274a05d0bd4165ba638 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sat, 11 Feb 2023 11:18:59 +0100 Subject: [PATCH 26/29] added secure to fluxConfigurationProtectedSettings --- modules/Microsoft.ContainerService/managedClusters/deploy.bicep | 1 + modules/Microsoft.ContainerService/managedClusters/readme.md | 2 +- .../Microsoft.KubernetesConfiguration/extensions/deploy.bicep | 1 + modules/Microsoft.KubernetesConfiguration/extensions/readme.md | 2 +- .../fluxConfigurations/deploy.bicep | 1 + .../fluxConfigurations/readme.md | 2 +- 6 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep index 94e1eef2b2..82e23ff6bd 100644 --- a/modules/Microsoft.ContainerService/managedClusters/deploy.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/deploy.bicep @@ -302,6 +302,7 @@ param fluxReleaseTrain string = 'Stable' param fluxVersion string = '' @description('Optional. Configuration settings that are sensitive, as name-value pairs for configuring this extension.') +@secure() param fluxConfigurationProtectedSettings object = {} @description('Optional. Configuration settings, as name-value pairs for configuring this extension.') diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index 4b70e8425c..bc6ae5538a 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -104,7 +104,7 @@ This module deploys Azure Kubernetes Cluster (AKS). | `enablePrivateClusterPublicFQDN` | bool | `False` | | Whether to create additional public FQDN for private cluster or not. | | `enableRBAC` | bool | `True` | | Whether to enable Kubernetes Role-Based Access Control. | | `enableSecretRotation` | string | `'false'` | `[false, true]` | Specifies whether the KeyvaultSecretsProvider add-on uses secret rotation. | -| `fluxConfigurationProtectedSettings` | object | `{object}` | | Configuration settings that are sensitive, as name-value pairs for configuring this extension. | +| `fluxConfigurationProtectedSettings` | secureObject | `{object}` | | Configuration settings that are sensitive, as name-value pairs for configuring this extension. | | `fluxConfigurations` | array | `[]` | | A list of flux configuraitons. | | `fluxConfigurationSettings` | object | `{object}` | | Configuration settings, as name-value pairs for configuring this extension. | | `fluxReleaseTrain` | string | `'Stable'` | | ReleaseTrain this extension participates in for auto-upgrade (e.g. Stable, Preview, etc.) - only if autoUpgradeMinorVersion is "true". | diff --git a/modules/Microsoft.KubernetesConfiguration/extensions/deploy.bicep b/modules/Microsoft.KubernetesConfiguration/extensions/deploy.bicep index 3ec32c5c9b..5630bacccd 100644 --- a/modules/Microsoft.KubernetesConfiguration/extensions/deploy.bicep +++ b/modules/Microsoft.KubernetesConfiguration/extensions/deploy.bicep @@ -11,6 +11,7 @@ param clusterName string param location string = resourceGroup().location @description('Optional. Configuration settings that are sensitive, as name-value pairs for configuring this extension.') +@secure() param configurationProtectedSettings object = {} @description('Optional. Configuration settings, as name-value pairs for configuring this extension.') diff --git a/modules/Microsoft.KubernetesConfiguration/extensions/readme.md b/modules/Microsoft.KubernetesConfiguration/extensions/readme.md index 08d5c117f1..2dcc1d827b 100644 --- a/modules/Microsoft.KubernetesConfiguration/extensions/readme.md +++ b/modules/Microsoft.KubernetesConfiguration/extensions/readme.md @@ -49,7 +49,7 @@ For Details see [Prerequisites](https://docs.microsoft.com/en-us/azure/azure-arc | Parameter Name | Type | Default Value | Description | | :-- | :-- | :-- | :-- | -| `configurationProtectedSettings` | object | `{object}` | Configuration settings that are sensitive, as name-value pairs for configuring this extension. | +| `configurationProtectedSettings` | secureObject | `{object}` | Configuration settings that are sensitive, as name-value pairs for configuring this extension. | | `configurationSettings` | object | `{object}` | Configuration settings, as name-value pairs for configuring this extension. | | `enableDefaultTelemetry` | bool | `True` | Enable telemetry via a Globally Unique Identifier (GUID). | | `location` | string | `[resourceGroup().location]` | Location for all resources. | diff --git a/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep b/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep index 8c39b77609..c9feeebf74 100644 --- a/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep +++ b/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/deploy.bicep @@ -14,6 +14,7 @@ param location string = resourceGroup().location param bucket object = {} @description('Optional. Key-value pairs of protected configuration settings for the configuration.') +@secure() param configurationProtectedSettings object = {} @description('Optional. Parameters to reconcile to the GitRepository source kind type.') diff --git a/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/readme.md b/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/readme.md index 4d281df1d9..53b3512676 100644 --- a/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/readme.md +++ b/modules/Microsoft.KubernetesConfiguration/fluxConfigurations/readme.md @@ -52,7 +52,7 @@ For Details see [Prerequisites](https://docs.microsoft.com/en-us/azure/azure-arc | Parameter Name | Type | Default Value | Description | | :-- | :-- | :-- | :-- | | `bucket` | object | `{object}` | Parameters to reconcile to the GitRepository source kind type. | -| `configurationProtectedSettings` | object | `{object}` | Key-value pairs of protected configuration settings for the configuration. | +| `configurationProtectedSettings` | secureObject | `{object}` | Key-value pairs of protected configuration settings for the configuration. | | `enableDefaultTelemetry` | bool | `True` | Enable telemetry via a Globally Unique Identifier (GUID). | | `gitRepository` | object | `{object}` | Parameters to reconcile to the GitRepository source kind type. | | `kustomizations` | object | `{object}` | Array of kustomizations used to reconcile the artifact pulled by the source type on the cluster. | From 29f2435028ac494d29c089cb22825b33ec140d82 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sat, 11 Feb 2023 13:38:05 +0100 Subject: [PATCH 27/29] remove the minflux test --- .../.test/minFlux/deploy.test.bicep | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep deleted file mode 100644 index 852b04aa3a..0000000000 --- a/modules/Microsoft.ContainerService/managedClusters/.test/minFlux/deploy.test.bicep +++ /dev/null @@ -1,62 +0,0 @@ -targetScope = 'subscription' - -// ========== // -// Parameters // -// ========== // - -@description('Optional. The name of the resource group to deploy for testing purposes.') -@maxLength(90) -param resourceGroupName string = 'ms.containerservice.managedclusters-${serviceShort}-rg' - -@description('Optional. The location to deploy resources to.') -param location string = deployment().location - -@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') -param serviceShort string = 'csmmf' - -@description('Optional. Enable telemetry via a Globally Unique Identifier (GUID).') -param enableDefaultTelemetry bool = true - -// ============ // -// Dependencies // -// ============ // - -// General resources -// ================= -resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { - name: resourceGroupName - location: location -} - -module testDeployment '../../deploy.bicep' = { - scope: resourceGroup - name: '${uniqueString(deployment().name, location)}-test-${serviceShort}' - params: { - name: '${serviceShort}001' - enableDefaultTelemetry: enableDefaultTelemetry - systemAssignedIdentity: true - primaryAgentPoolProfile: [ - { - name: 'systempool' - count: 1 - vmSize: 'Standard_DS2_v2' - mode: 'System' - } - ] - fluxConfigurations: [ - { - namespace: 'flux-system' - scope: 'cluster' - gitRepository: { - repositoryRef: { - branch: 'main' - } - sshKnownHosts: '' - syncIntervalInSeconds: 300 - timeoutInSeconds: 180 - url: 'https://github.com/mspnp/aks-baseline' - } - } - ] - } -} From 157fde5d9dd92a1a26555c903e3f0e802d6949ee Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Sat, 11 Feb 2023 13:40:46 +0100 Subject: [PATCH 28/29] update readme --- .../managedClusters/readme.md | 97 ------------------- 1 file changed, 97 deletions(-) diff --git a/modules/Microsoft.ContainerService/managedClusters/readme.md b/modules/Microsoft.ContainerService/managedClusters/readme.md index bc6ae5538a..e66f6a83f1 100644 --- a/modules/Microsoft.ContainerService/managedClusters/readme.md +++ b/modules/Microsoft.ContainerService/managedClusters/readme.md @@ -1122,100 +1122,3 @@ module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bice

- -

Example 5: Minflux

- -
- -via Bicep module - -```bicep -module managedClusters './Microsoft.ContainerService/managedClusters/deploy.bicep' = { - name: '${uniqueString(deployment().name, location)}-test-csmmf' - params: { - // Required parameters - name: 'csmmf001' - primaryAgentPoolProfile: [ - { - count: 1 - mode: 'System' - name: 'systempool' - vmSize: 'Standard_DS2_v2' - } - ] - // Non-required parameters - enableDefaultTelemetry: '' - fluxConfigurations: [ - { - gitRepository: { - repositoryRef: { - branch: 'main' - } - sshKnownHosts: '' - syncIntervalInSeconds: 300 - timeoutInSeconds: 180 - url: 'https://github.com/mspnp/aks-baseline' - } - namespace: 'flux-system' - } - ] - systemAssignedIdentity: true - } -} -``` - -
-

- -

- -via JSON Parameter file - -```json -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - // Required parameters - "name": { - "value": "csmmf001" - }, - "primaryAgentPoolProfile": { - "value": [ - { - "count": 1, - "mode": "System", - "name": "systempool", - "vmSize": "Standard_DS2_v2" - } - ] - }, - // Non-required parameters - "enableDefaultTelemetry": { - "value": "" - }, - "fluxConfigurations": { - "value": [ - { - "gitRepository": { - "repositoryRef": { - "branch": "main" - }, - "sshKnownHosts": "", - "syncIntervalInSeconds": 300, - "timeoutInSeconds": 180, - "url": "https://github.com/mspnp/aks-baseline" - }, - "namespace": "flux-system" - } - ] - }, - "systemAssignedIdentity": { - "value": true - } - } -} -``` - -
-

From dfab33448b3277ad5b836d4f9ac7e7d0925bcc87 Mon Sep 17 00:00:00 2001 From: Marius Storhaug Date: Mon, 13 Feb 2023 08:55:36 +0100 Subject: [PATCH 29/29] added some params --- .../managedClusters/.test/flux/deploy.test.bicep | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/Microsoft.ContainerService/managedClusters/.test/flux/deploy.test.bicep b/modules/Microsoft.ContainerService/managedClusters/.test/flux/deploy.test.bicep index a35c447784..97082bb9ef 100644 --- a/modules/Microsoft.ContainerService/managedClusters/.test/flux/deploy.test.bicep +++ b/modules/Microsoft.ContainerService/managedClusters/.test/flux/deploy.test.bicep @@ -43,6 +43,9 @@ module testDeployment '../../deploy.bicep' = { mode: 'System' } ] + fluxReleaseTrain: 'Stable' + fluxVersion: '' + fluxConfigurationProtectedSettings: {} fluxConfigurationSettings: { 'helm-controller.enabled': 'true' 'source-controller.enabled': 'true'