diff --git a/arm/Microsoft.ContainerService/managedClusters/.parameters/azure.parameters.json b/arm/Microsoft.ContainerService/managedClusters/.parameters/azure.parameters.json index b87fbde799..82b26db678 100644 --- a/arm/Microsoft.ContainerService/managedClusters/.parameters/azure.parameters.json +++ b/arm/Microsoft.ContainerService/managedClusters/.parameters/azure.parameters.json @@ -35,57 +35,53 @@ "value": [ { "name": "userpool1", - "properties": { - "vmSize": "Standard_DS2_v2", - "osDiskSizeGB": 128, - "count": 2, - "osType": "Linux", - "maxCount": 3, - "minCount": 1, - "enableAutoScaling": true, - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "nodeLabels": {}, - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ], - "type": "VirtualMachineScaleSets", - "availabilityZones": [ - "1" - ], - "minPods": 2, - "maxPods": 30, - "storageProfile": "ManagedDisks", - "mode": "User", - "vnetSubnetID": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Network/virtualNetworks/adp-sxx-az-vnet-x-aks/subnets/Secondary" - } + "vmSize": "Standard_DS2_v2", + "osDiskSizeGB": 128, + "count": 2, + "osType": "Linux", + "maxCount": 3, + "minCount": 1, + "enableAutoScaling": true, + "scaleSetPriority": "Regular", + "scaleSetEvictionPolicy": "Delete", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "type": "VirtualMachineScaleSets", + "availabilityZones": [ + "1" + ], + "minPods": 2, + "maxPods": 30, + "storageProfile": "ManagedDisks", + "mode": "User", + "vnetSubnetID": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Network/virtualNetworks/adp-sxx-az-vnet-x-aks/subnets/Secondary" }, { "name": "userpool2", - "properties": { - "vmSize": "Standard_DS2_v2", - "osDiskSizeGB": 128, - "count": 2, - "osType": "Linux", - "maxCount": 3, - "minCount": 1, - "enableAutoScaling": true, - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "nodeLabels": {}, - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ], - "type": "VirtualMachineScaleSets", - "availabilityZones": [ - "1" - ], - "minPods": 2, - "maxPods": 30, - "storageProfile": "ManagedDisks", - "mode": "User", - "vnetSubnetID": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Network/virtualNetworks/adp-sxx-az-vnet-x-aks/subnets/Tertiary" - } + "vmSize": "Standard_DS2_v2", + "osDiskSizeGB": 128, + "count": 2, + "osType": "Linux", + "maxCount": 3, + "minCount": 1, + "enableAutoScaling": true, + "scaleSetPriority": "Regular", + "scaleSetEvictionPolicy": "Delete", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "type": "VirtualMachineScaleSets", + "availabilityZones": [ + "1" + ], + "minPods": 2, + "maxPods": 30, + "storageProfile": "ManagedDisks", + "mode": "User", + "vnetSubnetID": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Network/virtualNetworks/adp-sxx-az-vnet-x-aks/subnets/Tertiary" } ] }, diff --git a/arm/Microsoft.ContainerService/managedClusters/.parameters/kubenet.parameters.json b/arm/Microsoft.ContainerService/managedClusters/.parameters/kubenet.parameters.json index 2cb5c36d52..b85a89237f 100644 --- a/arm/Microsoft.ContainerService/managedClusters/.parameters/kubenet.parameters.json +++ b/arm/Microsoft.ContainerService/managedClusters/.parameters/kubenet.parameters.json @@ -34,55 +34,51 @@ "value": [ { "name": "userpool1", - "properties": { - "vmSize": "Standard_DS2_v2", - "osDiskSizeGB": 128, - "count": 2, - "osType": "Linux", - "maxCount": 3, - "minCount": 1, - "enableAutoScaling": true, - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "nodeLabels": {}, - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ], - "type": "VirtualMachineScaleSets", - "availabilityZones": [ - "1" - ], - "minPods": 2, - "maxPods": 30, - "storageProfile": "ManagedDisks", - "mode": "User" - } + "vmSize": "Standard_DS2_v2", + "osDiskSizeGB": 128, + "count": 2, + "osType": "Linux", + "maxCount": 3, + "minCount": 1, + "enableAutoScaling": true, + "scaleSetPriority": "Regular", + "scaleSetEvictionPolicy": "Delete", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "type": "VirtualMachineScaleSets", + "availabilityZones": [ + "1" + ], + "minPods": 2, + "maxPods": 30, + "storageProfile": "ManagedDisks", + "mode": "User" }, { "name": "userpool2", - "properties": { - "vmSize": "Standard_DS2_v2", - "osDiskSizeGB": 128, - "count": 2, - "osType": "Linux", - "maxCount": 3, - "minCount": 1, - "enableAutoScaling": true, - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "nodeLabels": {}, - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ], - "type": "VirtualMachineScaleSets", - "availabilityZones": [ - "1" - ], - "minPods": 2, - "maxPods": 30, - "storageProfile": "ManagedDisks", - "mode": "User" - } + "vmSize": "Standard_DS2_v2", + "osDiskSizeGB": 128, + "count": 2, + "osType": "Linux", + "maxCount": 3, + "minCount": 1, + "enableAutoScaling": true, + "scaleSetPriority": "Regular", + "scaleSetEvictionPolicy": "Delete", + "nodeLabels": {}, + "nodeTaints": [ + "CriticalAddonsOnly=true:NoSchedule" + ], + "type": "VirtualMachineScaleSets", + "availabilityZones": [ + "1" + ], + "minPods": 2, + "maxPods": 30, + "storageProfile": "ManagedDisks", + "mode": "User" } ] }, diff --git a/arm/Microsoft.ContainerService/managedClusters/agentPools/deploy.bicep b/arm/Microsoft.ContainerService/managedClusters/agentPools/deploy.bicep index 1bded3b3f8..214314448e 100644 --- a/arm/Microsoft.ContainerService/managedClusters/agentPools/deploy.bicep +++ b/arm/Microsoft.ContainerService/managedClusters/agentPools/deploy.bicep @@ -5,17 +5,503 @@ param managedClusterName string @description('Required. Name of the agent pool') param name string -@description('Required. Properties for the container service agent pool profile.') -param agentPoolProperties object +@description('Optional. The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is "VirtualMachineScaleSets". ') +param availabilityZones array = [] + +@description('Optional. Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1.') +@minValue(0) +@maxValue(1000) +param count int = 1 + +@description('Optional. This is the ARM ID of the source object to be used to create the target object.') +param sourceResourceId string = '' + +@description('Optional. Whether to enable auto-scaler') +@allowed([ + true + false +]) +param enableAutoScaling bool = false + +@description('Optional. This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption ') +@allowed([ + true + false +]) +param enableEncryptionAtHost bool = false + +@description('Optional. See Add a FIPS-enabled node pool (https://docs.microsoft.com/en-us/azure/aks/use-multiple-node-pools#add-a-fips-enabled-node-pool-preview) for more details.') +@allowed([ + true + false +]) +param enableFIPS bool = false + +@description('Optional. Some scenarios may require nodes in a node pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node (https://docs.microsoft.com/en-us/azure/aks/use-multiple-node-pools#assign-a-public-ip-per-node-for-your-node-pools). The default is false.') +@allowed([ + true + false +]) +param enableNodePublicIP bool = false + +@description('Optional. Whether to enable UltraSSD') +@allowed([ + true + false +]) +param enableUltraSSD bool = false + +@description('Optional. GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU.') +@allowed([ + 'MIG1g' + 'MIG2g' + 'MIG3g' + 'MIG4g' + 'MIG7g' + '' +]) +param gpuInstanceProfile string = '' + +@description('Optional. Allowed list of unsafe sysctls or unsafe sysctl patterns (ending in *).') +@allowed([ + 'kernel.shm*' + 'kernel.msg*' + 'kernel.sem*' + 'fs.mqueue.*' + 'net.*' +]) +param allowedUnsafeSysctls array = [ + 'kernel.shm*' + 'kernel.msg*' + 'kernel.sem*' + 'fs.mqueue.*' + 'net.*' +] + +@description('Optional. The maximum number of container log files that can be present for a container. The number must be >= 2.') +@minValue(2) +param containerLogMaxFiles int = 5 + +@description('Optional. The maximum size (e.g. 10 MB) of container log file before it is rotated.') +param containerLogMaxSizeMB int = 10 + +@description('Optional. Enable/Disable CPU CFS quota enforcement for containers that specify CPU limits.') +@allowed([ + true + false +]) +param cpuCfsQuota bool = true + +@description('Optional. Sets CPU CFS quota period value. Valid values are a sequence of decimal numbers with an optional fraction and a unit suffix. For example: "300ms", "2h45m". Supported units are "ns", "us", "ms", "s", "m", and "h" .') +param cpuCfsQuotaPeriod string = '100ms' + +@description('Optional. The static policy allows containers in Guaranteed pods with integer CPU requests access to exclusive CPUs on the node. See Kubernetes CPU management policies for more info (https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#cpu-management-policies)') +@allowed([ + 'none' + 'static' +]) +param cpuManagerPolicy string = 'none' + +@description('Optional. If set to true it will make the Kubelet fail to start if swap is enabled on the node.') +@allowed([ + true + false +]) +param failSwapOn bool = false + +@description('Optional. The percent of disk usage after which image garbage collection is always run. Minimum disk usage that will trigger garbage collection. To disable image garbage collection, set to 100.') +@minValue(0) +@maxValue(100) +param imageGcHighThreshold int = 85 + +@description('Optional. The percent of disk usage before which image garbage collection is never run. Minimum disk usage that can trigger garbage collection.') +@minValue(0) +@maxValue(100) +param imageGcLowThreshold int = 80 + +@description('Optional. The maximum number of processes per pod.') +@minValue(-1) +param podMaxPids int = -1 + +@description('Optional. Optimize NUMA node alignment. For more information see Kubernetes Topology Manager (https://kubernetes.io/docs/tasks/administer-cluster/topology-manager)') +@allowed([ + 'none' + 'best-effort' + 'restricted' + 'single-numa-node' +]) +param topologyManagerPolicy string = 'none' + +@description('Optional. Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage.') +param kubeletDiskType string = '' + +@description('Optional. The size in MB of a swap file that will be created on each node.') +@minValue(1) +@maxValue(2147483647) +param swapFileSizeMB int = 1500 + +@description('Optional. Sysctl setting fs.aio-max-nr.') +@minValue(65536) +@maxValue(6553500) +param fsAioMaxNr int = 65536 + +@description('Optional. Sysctl setting fs.file-max.') +@minValue(8192) +@maxValue(12000500) +param fsFileMax int = 8192 + +@description('Optional. Sysctl setting fs.inotify.max_user_watches.') +@minValue(781250) +@maxValue(2097152) +param fsInotifyMaxUserWatches int = 781250 + +@description('Optional. Sysctl setting fs.nr_open.') +@minValue(8192) +@maxValue(20000500) +param fsNrOpen int = 8192 + +@description('Optional. Sysctl setting kernel.threads-max.') +@minValue(20) +@maxValue(513785) +param kernelThreadsMax int = 20 + +@description('Optional. Sysctl setting net.core.netdev_max_backlog.') +@minValue(1000) +@maxValue(3240000) +param netCoreNetdevMaxBacklog int = 1000 + +@description('Optional. Sysctl setting net.core.optmem_max.') +@minValue(20480) +@maxValue(4194304) +param netCoreOptmemMax int = 20480 + +@description('Optional. Sysctl setting net.core.rmem_default.') +@minValue(212992) +@maxValue(134217728) +param netCoreRmemDefault int = 212992 + +@description('Optional. Sysctl setting net.core.rmem_max.') +@minValue(212992) +@maxValue(134217728) +param netCoreRmemMax int = 212992 + +@description('Optional. Sysctl setting net.core.somaxconn.') +@minValue(4096) +@maxValue(3240000) +param netCoreSomaxconn int = 4096 + +@description('Optional. Sysctl setting net.core.wmem_default.') +@minValue(212992) +@maxValue(134217728) +param netCoreWmemDefault int = 212992 + +@description('Optional. Sysctl setting net.core.wmem_max.') +@minValue(212992) +@maxValue(134217728) +param netCoreWmemMax int = 212992 + +@description('Optional. Sysctl setting net.ipv4.ip_local_port_range.') +param netIpv4IpLocalPortRange string = '' + +@description('Optional. Sysctl setting net.ipv4.neigh.default.gc_thresh1.') +@minValue(128) +@maxValue(80000) +param netIpv4NeighDefaultGcThresh1 int = 128 + +@description('Optional. Sysctl setting net.ipv4.neigh.default.gc_thresh2.') +@minValue(512) +@maxValue(90000) +param netIpv4NeighDefaultGcThresh2 int = 512 + +@description('Optional. Sysctl setting net.ipv4.neigh.default.gc_thresh3.') +@minValue(1024) +@maxValue(100000) +param netIpv4NeighDefaultGcThresh3 int = 1024 + +@description('Optional. Sysctl setting net.ipv4.tcp_fin_timeout.') +@minValue(5) +@maxValue(120) +param netIpv4TcpFinTimeout int = 5 + +@description('Optional. Sysctl setting net.ipv4.tcp_keepalive_intvl.') +@minValue(10) +@maxValue(75) +param netIpv4TcpkeepaliveIntvl int = 10 + +@description('Optional. Sysctl setting net.ipv4.tcp_keepalive_probes.') +@minValue(1) +@maxValue(15) +param netIpv4TcpKeepaliveProbes int = 1 + +@description('Optional. Sysctl setting net.ipv4.tcp_keepalive_time.') +@minValue(30) +@maxValue(432000) +param netIpv4TcpKeepaliveTime int = 30 + +@description('Optional. Sysctl setting net.ipv4.tcp_max_syn_backlog') +@minValue(128) +@maxValue(3240000) +param netIpv4TcpMaxSynBacklog int = 128 + +@description('Optional. Sysctl setting net.ipv4.tcp_max_tw_buckets.') +@minValue(8000) +@maxValue(1440000) +param netIpv4TcpMaxTwBuckets int = 8000 + +@description('Optional. Sysctl setting net.ipv4.tcp_tw_reuse.') +@allowed([ + true + false +]) +param netIpv4TcpTwReuse bool = false + +@description('Optional. Sysctl setting net.netfilter.nf_conntrack_buckets.') +@minValue(65536) +@maxValue(147456) +param netNetfilterNfConntrackBuckets int = 65536 + +@description('Optional. Sysctl setting net.netfilter.nf_conntrack_max.') +@minValue(131072) +@maxValue(589824) +param netNetfilterNfConntrackMax int = 131072 + +@description('Optional. Sysctl setting vm.max_map_count.') +@minValue(65530) +@maxValue(262144) +param vmMaxMapCount int = 65530 + +@description('Optional. Sysctl setting vm.swappiness.') +@minValue(0) +@maxValue(100) +param vmSwappiness int = 0 + +@description('Optional. Sysctl setting vm.vfs_cache_pressure.') +@minValue(0) +@maxValue(100) +param vmVfsCachePressure int = 0 + +@description('Optional. See Transparent Hugepages (https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge)') +@allowed([ + 'always' + 'defer' + 'defer+madvise' + 'madvise' + 'never' +]) +param transparentHugePageDefrag string = 'madvise' + +@description('Optional. See Transparent Hugepages (https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge)') +@allowed([ + 'always' + 'madvise' + 'never' +]) +param transparentHugePageEnabled string = 'always' + +@description('Optional. The maximum number of nodes for auto-scaling') +param maxCount int = -1 + +@description('Optional. The maximum number of pods that can run on a node.') +param maxPods int = -1 + +@description('Optional. The minimum number of nodes for auto-scaling') +param minCount int = -1 + +@description('Optional. A cluster must have at least one "System" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools') +param mode string = '' + +@description('Optional. The node labels to be persisted across all nodes in agent pool.') +param nodeLabels object = {} + +@description('Optional. ResourceId of the node PublicIPPrefix') +param nodePublicIpPrefixId string = '' + +@description('Optional. The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule. ') +param nodeTaints array = [] + +@description('Optional. As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://docs.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool).') +param orchestratorVersion string = '' + +@description('Optional. OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified.') +param osDiskSizeGB int = 0 + +@description('Optional. The default is "Ephemeral" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to "Managed". May not be changed after creation. For more information see Ephemeral OS (https://docs.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os).') +@allowed([ + 'Ephemeral' + 'Managed' + '' +]) +param osDiskType string = '' + +@description('Optional. Specifies an OS SKU. This value must not be specified if OSType is Windows.') +@allowed([ + 'CBLMariner' + 'Ubuntu' + '' +]) +param osSku string = '' + +@description('Optional. The operating system type. The default is Linux.') +@allowed([ + 'Linux' + 'Windows' +]) +param osType string = 'Linux' + +@description('Optional. Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName} ') +param podSubnetId string = '' + +@description('Optional. The ID for the Proximity Placement Group.') +param proximityPlacementGroupId string = '' + +@description('Optional. Describes how VMs are added to or removed from Agent Pools. See billing states (https://docs.microsoft.com/en-us/azure/virtual-machines/states-billing).') +@allowed([ + 'Deallocate' + 'Delete' +]) +param scaleDownMode string = 'Delete' + +@description('Optional. The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs ') +@allowed([ + 'Deallocate' + 'Delete' +]) +param scaleSetEvictionPolicy string = 'Delete' + +@description('Optional. The Virtual Machine Scale Set priority.') +@allowed([ + 'Regular' + 'Spot' + '' +]) +param scaleSetPriority string = '' + +@description('Optional. Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://docs.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing)') +param spotMaxPrice int = -1 + +@description('Optional. Tags of the resource.') +param tags object = {} + +@description('Optional. The type of Agent Pool.') +param type string = '' + +@description('Optional. This can either be set to an integer (e.g. "5") or a percentage (e.g. "50%"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade') +param maxSurge string = '' + +@description('Optional. VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions') +param vmSize string = 'Standard_D2s_v3' + +@description('Optional. Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName} ') +param vnetSubnetId string = '' + +@description('Optional. Determines the type of workload a node can run.') +param workloadRuntime string = '' + +var creationData = { + sourceResourceId: !empty(sourceResourceId) ? sourceResourceId : null +} + +var kubeletConfig = { + allowedUnsafeSysctls: allowedUnsafeSysctls + containerLogMaxFiles: containerLogMaxFiles + containerLogMaxSizeMB: !(containerLogMaxSizeMB == -1) ? containerLogMaxSizeMB : null + cpuCfsQuota: cpuCfsQuota + cpuCfsQuotaPeriod: cpuCfsQuotaPeriod + cpuManagerPolicy: cpuManagerPolicy + failSwapOn: failSwapOn + imageGcHighThreshold: imageGcHighThreshold + imageGcLowThreshold: imageGcLowThreshold + podMaxPids: podMaxPids + topologyManagerPolicy: topologyManagerPolicy +} + +var sysctls = { + fsAioMaxNr: fsAioMaxNr + fsFileMax: fsFileMax + fsInotifyMaxUserWatches: fsInotifyMaxUserWatches + fsNrOpen: fsNrOpen + kernelThreadsMax: kernelThreadsMax + netCoreNetdevMaxBacklog: netCoreNetdevMaxBacklog + netCoreOptmemMax: netCoreOptmemMax + netCoreRmemDefault: netCoreRmemDefault + netCoreRmemMax: netCoreRmemMax + netCoreSomaxconn: netCoreSomaxconn + netCoreWmemDefault: netCoreWmemDefault + netCoreWmemMax: netCoreWmemMax + netIpv4IpLocalPortRange: netIpv4IpLocalPortRange + netIpv4NeighDefaultGcThresh1: netIpv4NeighDefaultGcThresh1 + netIpv4NeighDefaultGcThresh2: netIpv4NeighDefaultGcThresh2 + netIpv4NeighDefaultGcThresh3: netIpv4NeighDefaultGcThresh3 + netIpv4TcpFinTimeout: netIpv4TcpFinTimeout + netIpv4TcpkeepaliveIntvl: netIpv4TcpkeepaliveIntvl + netIpv4TcpKeepaliveProbes: netIpv4TcpKeepaliveProbes + netIpv4TcpKeepaliveTime: netIpv4TcpKeepaliveTime + netIpv4TcpMaxSynBacklog: netIpv4TcpMaxSynBacklog + netIpv4TcpMaxTwBuckets: netIpv4TcpMaxTwBuckets + netIpv4TcpTwReuse: netIpv4TcpTwReuse + netNetfilterNfConntrackBuckets: netNetfilterNfConntrackBuckets + netNetfilterNfConntrackMax: netNetfilterNfConntrackMax + vmMaxMapCount: vmMaxMapCount + vmSwappiness: vmSwappiness + vmVfsCachePressure: vmVfsCachePressure +} + +var linuxOSConfig = { + swapFileSizeMB: swapFileSizeMB + sysctls: sysctls + transparentHugePageDefrag: transparentHugePageDefrag + transparentHugePageEnabled: transparentHugePageEnabled +} + +var upgradeSettings = { + maxSurge: maxSurge +} resource managedCluster 'Microsoft.ContainerService/managedClusters@2021-08-01' existing = { name: managedClusterName } -resource agentPool 'Microsoft.ContainerService/managedClusters/agentPools@2021-05-01' = { +resource agentPool 'Microsoft.ContainerService/managedClusters/agentPools@2021-08-01' = { name: name parent: managedCluster - properties: agentPoolProperties + properties: { + availabilityZones: availabilityZones + count: count + creationData: !empty(sourceResourceId) ? creationData : null + enableAutoScaling: enableAutoScaling + enableEncryptionAtHost: enableEncryptionAtHost + enableFIPS: enableFIPS + enableNodePublicIP: enableNodePublicIP + enableUltraSSD: enableUltraSSD + gpuInstanceProfile: !empty(gpuInstanceProfile) ? gpuInstanceProfile : null + kubeletConfig: kubeletConfig + kubeletDiskType: kubeletDiskType + linuxOSConfig: linuxOSConfig + maxCount: maxCount != -1 ? maxCount : null + maxPods: maxPods != -1 ? maxPods : null + minCount: minCount != -1 ? minCount : null + mode: !empty(mode) ? mode : null + nodeLabels: nodeLabels + nodePublicIPPrefixID: !empty(nodePublicIpPrefixId) ? nodePublicIpPrefixId : null + nodeTaints: nodeTaints + orchestratorVersion: orchestratorVersion + osDiskSizeGB: osDiskSizeGB != -1 ? osDiskSizeGB : null + osDiskType: !empty(osDiskType) ? osDiskType : null + osSKU: !empty(osSku) ? osSku : null + osType: osType + podSubnetID: !empty(podSubnetId) ? podSubnetId : null + proximityPlacementGroupID: !empty(proximityPlacementGroupId) ? proximityPlacementGroupId : null + scaleDownMode: scaleDownMode + scaleSetEvictionPolicy: scaleSetEvictionPolicy + scaleSetPriority: !empty(scaleSetPriority) ? scaleSetPriority : null + spotMaxPrice: spotMaxPrice + tags: tags + type: type + upgradeSettings: upgradeSettings + vmSize: vmSize + vnetSubnetID: vnetSubnetId + workloadRuntime: workloadRuntime + } } @description('The name of the agent pool') diff --git a/arm/Microsoft.ContainerService/managedClusters/agentPools/readme.md b/arm/Microsoft.ContainerService/managedClusters/agentPools/readme.md index fa853264e1..b52ec97630 100644 --- a/arm/Microsoft.ContainerService/managedClusters/agentPools/readme.md +++ b/arm/Microsoft.ContainerService/managedClusters/agentPools/readme.md @@ -6,55 +6,106 @@ This module deploys an Agent Pool for a Container Service Managed Cluster | Resource Type | API Version | | :-- | :-- | -| `Microsoft.ContainerService/managedClusters/agentPools` | 2021-05-01 | +| `Microsoft.ContainerService/managedClusters/agentPools` | 2021-08-01 | ## Parameters | Parameter Name | Type | Default Value | Possible Values | Description | | :-- | :-- | :-- | :-- | :-- | -| `agentPoolProperties` | object | | | Required. Properties for the container service agent pool profile. | +| `allowedUnsafeSysctls` | array | `[kernel.shm*, kernel.msg*, kernel.sem*, fs.mqueue.*, net.*]` | `[kernel.shm*, kernel.msg*, kernel.sem*, fs.mqueue.*, net.*]` | Optional. Allowed list of unsafe sysctls or unsafe sysctl patterns (ending in *). | +| `availabilityZones` | array | `[]` | | Optional. The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is "VirtualMachineScaleSets". | +| `containerLogMaxFiles` | int | `5` | | Optional. The maximum number of container log files that can be present for a container. The number must be >= 2. | +| `containerLogMaxSizeMB` | int | `10` | | Optional. The maximum size (e.g. 10 MB) of container log file before it is rotated. | +| `count` | int | `1` | | Optional. Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1. | +| `cpuCfsQuota` | bool | `True` | `[True, False]` | Optional. Enable/Disable CPU CFS quota enforcement for containers that specify CPU limits. | +| `cpuCfsQuotaPeriod` | string | `100ms` | | Optional. Sets CPU CFS quota period value. Valid values are a sequence of decimal numbers with an optional fraction and a unit suffix. For example: "300ms", "2h45m". Supported units are "ns", "us", "ms", "s", "m", and "h" . | +| `cpuManagerPolicy` | string | `none` | `[none, static]` | Optional. The static policy allows containers in Guaranteed pods with integer CPU requests access to exclusive CPUs on the node(s). See Kubernetes CPU management policies for more info () | +| `enableAutoScaling` | bool | | `[True, False]` | Optional. Whether to enable auto-scaler | +| `enableEncryptionAtHost` | bool | | `[True, False]` | Optional. This is only supported on certain VM sizes and in certain Azure regions. For more information, see: /azure/aks/enable-host-encryption | +| `enableFIPS` | bool | | `[True, False]` | Optional. See Add a FIPS-enabled node-pool () for more details. | +| `enableNodePublicIP` | bool | | `[True, False]` | Optional. Some scenarios may require nodes in a node-pool to receive their own dedicated public IP addresses. A common scenario is for gaming workloads, where a console needs to make a direct connection to a cloud virtual machine to minimize hops. For more information see assigning a public IP per node(s). (). The default is false. | +| `enableUltraSSD` | bool | | `[True, False]` | Optional. Whether to enable UltraSSD | +| `failSwapOn` | bool | | `[True, False]` | Optional. If set to true it will make the Kubelet fail to start if swap is enabled on the node(s). | +| `fsAioMaxNr` | int | `65536` | | Optional. Sysctl setting fs.aio-max-nr. | +| `fsFileMax` | int | `8192` | | Optional. Sysctl setting fs.file-max. | +| `fsInotifyMaxUserWatches` | int | `781250` | | Optional. Sysctl setting fs.inotify.max_user_watches. | +| `fsNrOpen` | int | `8192` | | Optional. Sysctl setting fs.nr_open. | +| `gpuInstanceProfile` | string | | `[MIG1g, MIG2g, MIG3g, MIG4g, MIG7g, ]` | Optional. GPUInstanceProfile to be used to specify GPU MIG instance profile for supported GPU VM SKU. | +| `imageGcHighThreshold` | int | `85` | | Optional. The percent of disk usage after which image garbage collection is always run. Minimum disk usage that will trigger garbage collection. To disable image garbage collection, set to 100. | +| `imageGcLowThreshold` | int | `80` | | Optional. The percent of disk usage before which image garbage collection is never run. Minimum disk usage that can trigger garbage collection. | +| `kernelThreadsMax` | int | `20` | | Optional. Sysctl setting kernel.threads-max. | +| `kubeletDiskType` | string | | | Optional. Determines the placement of emptyDir volumes, container runtime data root, and Kubelet ephemeral storage. | | `managedClusterName` | string | | | Required. Name of the managed cluster | +| `maxCount` | int | `-1` | | Optional. The maximum number of nodes for auto-scaling | +| `maxPods` | int | `-1` | | Optional. The maximum number of pods that can run on node(s). | +| `maxSurge` | string | | | Optional. This can either be set to an integer (e.g. "5") or a percentage (e.g. "50%"). If a percentage is specified, it is the percentage of the total agent pool size at the time of the upgrade. For percentages, fractional nodes are rounded up. If not specified, the default is 1. For more information, including best practices, see: /azure/aks/upgrade-cluster#customize-node-surge-upgrade | +| `minCount` | int | `-1` | | Optional. The minimum number of nodes for auto-scaling | +| `mode` | string | | | Optional. A cluster must have at least one "System" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools | | `name` | string | | | Required. Name of the agent pool | +| `netCoreNetdevMaxBacklog` | int | `1000` | | Optional. Sysctl setting net.core.netdev_max_backlog. | +| `netCoreOptmemMax` | int | `20480` | | Optional. Sysctl setting net.core.optmem_max. | +| `netCoreRmemDefault` | int | `212992` | | Optional. Sysctl setting net.core.rmem_default. | +| `netCoreRmemMax` | int | `212992` | | Optional. Sysctl setting net.core.rmem_max. | +| `netCoreSomaxconn` | int | `4096` | | Optional. Sysctl setting net.core.somaxconn. | +| `netCoreWmemDefault` | int | `212992` | | Optional. Sysctl setting net.core.wmem_default. | +| `netCoreWmemMax` | int | `212992` | | Optional. Sysctl setting net.core.wmem_max. | +| `netIpv4IpLocalPortRange` | string | | | Optional. Sysctl setting net.ipv4.ip_local_port_range. | +| `netIpv4NeighDefaultGcThresh1` | int | `128` | | Optional. Sysctl setting net.ipv4.neigh.default.gc_thresh1. | +| `netIpv4NeighDefaultGcThresh2` | int | `512` | | Optional. Sysctl setting net.ipv4.neigh.default.gc_thresh2. | +| `netIpv4NeighDefaultGcThresh3` | int | `1024` | | Optional. Sysctl setting net.ipv4.neigh.default.gc_thresh3. | +| `netIpv4TcpFinTimeout` | int | `5` | | Optional. Sysctl setting net.ipv4.tcp_fin_timeout. | +| `netIpv4TcpkeepaliveIntvl` | int | `10` | | Optional. Sysctl setting net.ipv4.tcp_keepalive_intvl. | +| `netIpv4TcpKeepaliveProbes` | int | `1` | | Optional. Sysctl setting net.ipv4.tcp_keepalive_probes. | +| `netIpv4TcpKeepaliveTime` | int | `30` | | Optional. Sysctl setting net.ipv4.tcp_keepalive_time. | +| `netIpv4TcpMaxSynBacklog` | int | `128` | | Optional. Sysctl setting net.ipv4.tcp_max_syn_backlog | +| `netIpv4TcpMaxTwBuckets` | int | `8000` | | Optional. Sysctl setting net.ipv4.tcp_max_tw_buckets. | +| `netIpv4TcpTwReuse` | bool | | `[True, False]` | Optional. Sysctl setting net.ipv4.tcp_tw_reuse. | +| `netNetfilterNfConntrackBuckets` | int | `65536` | | Optional. Sysctl setting net.netfilter.nf_conntrack_buckets. | +| `netNetfilterNfConntrackMax` | int | `131072` | | Optional. Sysctl setting net.netfilter.nf_conntrack_max. | +| `nodeLabels` | object | `{object}` | | Optional. The node-labels to be persisted across all nodes in agent pool. | +| `nodePublicIpPrefixId` | string | | | Optional. ResourceId of the node-PublicIPPrefix | +| `nodeTaints` | array | `[]` | | Optional. The taints added to new nodes during node-pool create and scale. For example, key=value:NoSchedule. | +| `orchestratorVersion` | string | | | Optional. As a best practice, you should upgrade all node-pools in an AKS cluster to the same Kubernetes version. The node-pool version must have the same major version as the control plane. The node-pool minor version must be within two minor versions of the control plane version. The node-pool version cannot be greater than the control plane version. For more information see upgrading a node-pool (https://docs.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool). | +| `osDiskSizeGB` | int | | | Optional. OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified. | +| `osDiskType` | string | | `[Ephemeral, Managed, ]` | Optional. The default is "Ephemeral" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to "Managed". May not be changed after creation. For more information see Ephemeral OS (https://docs.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os). | +| `osSku` | string | | `[CBLMariner, Ubuntu, ]` | Optional. Specifies an OS SKU. This value must not be specified if OSType is Windows. | +| `osType` | string | `Linux` | `[Linux, Windows]` | Optional. The operating system type. The default is Linux. | +| `podMaxPids` | int | `-1` | | Optional. The maximum number of processes per pod. | +| `podSubnetId` | string | | | Optional. Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node-subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName} | +| `proximityPlacementGroupId` | string | | | Optional. The ID for the Proximity Placement Group. | +| `scaleDownMode` | string | `Delete` | `[Deallocate, Delete]` | Optional. Describes how VMs are added to or removed from Agent Pools. See billing states (https://docs.microsoft.com/en-us/azure/virtual-machines/states-billing). | +| `scaleSetEvictionPolicy` | string | `Delete` | `[Deallocate, Delete]` | Optional. The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs | +| `scaleSetPriority` | string | | `[Regular, Spot, ]` | Optional. The Virtual Machine Scale Set priority. | +| `sourceResourceId` | string | | | Optional. This is the ARM ID of the source object to be used to create the target object. | +| `spotMaxPrice` | int | `-1` | | Optional. Possible values are any decimal value greater than zero or -1 which indicates the willingness to pay any on-demand price. For more details on spot pricing, see spot VMs pricing (https://docs.microsoft.com/en-us/azure/virtual-machines/spot-vms#pricing) | +| `swapFileSizeMB` | int | `1500` | | Optional. The size in MB of a swap file that will be created on the node(s). | +| `tags` | object | `{object}` | | Optional. Tags of the resource. | +| `topologyManagerPolicy` | string | `none` | `[none, best-effort, restricted, single-numa-node]` | Optional. Optimize NUMA node-alignment. For more information see Kubernetes Topology Manager (https://kubernetes.io/docs/tasks/administer-cluster/topology-manager) | +| `transparentHugePageDefrag` | string | `madvise` | `[always, defer, defer+madvise, madvise, never]` | Optional. See Transparent Hugepages (https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge) | +| `transparentHugePageEnabled` | string | `always` | `[always, madvise, never]` | Optional. See Transparent Hugepages (https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge) | +| `type` | string | | | Optional. The type of Agent Pool. | +| `vmMaxMapCount` | int | `65530` | | Optional. Sysctl setting vm.max_map_count. | +| `vmSize` | string | `Standard_D2s_v3` | | Optional. VM size. VM size availability varies by region. If nodes contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions | +| `vmSwappiness` | int | | | Optional. Sysctl setting vm.swappiness. | +| `vmVfsCachePressure` | int | | | Optional. Sysctl setting vm.vfs_cache_pressure. | +| `vnetSubnetId` | string | | | Optional. Node-Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName} | +| `workloadRuntime` | string | | | Optional. Determines the type of workload nodes can run. | -### Parameter Usage: `agentPoolProperties` +### Parameter Usage: `tags` -This object contains the configuration for the agent pool profile. The following example shows an agent pool profile configuration. -For available properties check +Tag names and tag values can be provided as needed. A tag can be left without a value. ```json -"agentPoolProperties": { - "value": { - "vmSize": "Standard_DS3_v2", - "osDiskSizeGB": 128, - "count": 2, - "osType": "Linux", - "maxCount": 5, - "minCount": 1, - "enableAutoScaling": true, - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "nodeLabels": {}, - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ], - "type": "VirtualMachineScaleSets", - "availabilityZones": [ - "1", - "2", - "3" - ], - "maxPods": 30, - "storageProfile": "ManagedDisks", - "mode": "System", - "vnetSubnetID": "/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/resourceGroups/myRg/providers/Microsoft.Network/virtualNetworks/myVnet/subnets/mySubnet", - "tags": { - "Owner": "test.user@testcompany.com", - "BusinessUnit": "IaCs", - "Environment": "PROD", - "Region": "USEast" - } - } - } +"tags": { + "value": { + "Environment": "Non-Prod", + "Contact": "test.user@testcompany.com", + "PurchaseOrder": "1234", + "CostCenter": "7890", + "ServiceName": "DeploymentValidation", + "Role": "DeploymentValidation" + } +} ``` ## Outputs @@ -67,4 +118,4 @@ For available properties check - -```json -"agentPools": { - "value": [ - { - "name": "pool1", - "properties": { - "vmSize": "Standard_DS3_v2", - "osDiskSizeGB": 128, - "count": 2, - "osType": "Linux", - "maxCount": 5, - "minCount": 1, - "enableAutoScaling": true, - "scaleSetPriority": "Regular", - "scaleSetEvictionPolicy": "Delete", - "nodeLabels": {}, - "nodeTaints": [ - "CriticalAddonsOnly=true:NoSchedule" - ], - "type": "VirtualMachineScaleSets", - "availabilityZones": [ - "1", - "2", - "3" - ], - "maxPods": 30, - "storageProfile": "ManagedDisks", - "mode": "System", - "vnetSubnetID": "/subscriptions/XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX/resourceGroups/myRg/providers/Microsoft.Network/virtualNetworks/myVnet/subnets/mySubnet", - "tags": { - "Owner": "test.user@testcompany.com", - "BusinessUnit": "IaCs", - "Environment": "PROD", - "Region": "USEast" - } - } - }, - { - "name": "pool2", - "properties": { - "..." - } - } - ] - } -``` - ### Parameter Usage: `userAssignedIdentities` You can specify multiple user assigned identities to a resource by providing additional resource IDs using the following format: @@ -260,5 +209,5 @@ You can specify multiple user assigned identities to a resource by providing add - [Locks](https://docs.microsoft.com/en-us/azure/templates/Microsoft.Authorization/2016-09-01/locks) - [Roleassignments](https://docs.microsoft.com/en-us/azure/templates/Microsoft.Authorization/2020-04-01-preview/roleAssignments) - [Managedclusters](https://docs.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2021-07-01/managedClusters) -- [Managedclusters/Agentpools](https://docs.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2021-05-01/managedClusters/agentPools) +- [Managedclusters/Agentpools](https://docs.microsoft.com/en-us/azure/templates/Microsoft.ContainerService/2021-08-01/managedClusters/agentPools) - [Diagnosticsettings](https://docs.microsoft.com/en-us/azure/templates/Microsoft.Insights/2021-05-01-preview/diagnosticSettings)