From 56a0ae6fd9a8ebc954342c37ec05d39ac8164345 Mon Sep 17 00:00:00 2001 From: Ryan VanGundy Date: Wed, 21 May 2025 00:00:49 -0400 Subject: [PATCH 1/4] Use system assigned user ID --- terraform/cluster/azure-aks/main.tf | 85 ++++++++++++----------------- 1 file changed, 36 insertions(+), 49 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 1866192b..8dd837bd 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -206,15 +206,6 @@ resource "azurerm_log_analytics_workspace" "aks_logs" { # AKS Cluster #----------------------------------------------------------------------------------------------------------------------- -resource "azurerm_user_assigned_identity" "cluster" { - name = "${var.name}-${var.context_id}" - location = var.region - resource_group_name = azurerm_resource_group.aks.name - tags = merge({ - Name = "${var.name}-${var.context_id}" - }, local.tags) -} - resource "azurerm_kubernetes_cluster" "main" { name = local.cluster_name location = azurerm_resource_group.aks.location @@ -246,12 +237,12 @@ resource "azurerm_kubernetes_cluster" "main" { vnet_subnet_id = coalesce(var.vnet_subnet_id, try(data.azurerm_subnet.private[0].id, null)) orchestrator_version = var.kubernetes_version only_critical_addons_enabled = var.default_node_pool.only_critical_addons_enabled - # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs + # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs os_disk_type = var.default_node_pool.os_disk_type host_encryption_enabled = var.default_node_pool.host_encryption_enabled - # checkov:skip=CKV_AZURE_168: This is set in the variable by default to 50 + # checkov:skip=CKV_AZURE_168: This is set in the variable by default to 50 max_pods = var.default_node_pool.max_pods temporary_name_for_rotation = "rotate" } @@ -285,17 +276,13 @@ resource "azurerm_kubernetes_cluster" "main" { } identity { - type = "UserAssigned" - identity_ids = concat( - [azurerm_user_assigned_identity.cluster.id], - var.additional_cluster_identity_ids - ) + type = length(var.additional_cluster_identity_ids) > 0 ? "SystemAssigned, UserAssigned" : "SystemAssigned" + identity_ids = var.additional_cluster_identity_ids } kubelet_identity { - client_id = azurerm_user_assigned_identity.cluster.client_id - object_id = azurerm_user_assigned_identity.cluster.principal_id - user_assigned_identity_id = azurerm_user_assigned_identity.cluster.id + client_id = length(var.additional_cluster_identity_ids) > 0 ? var.additional_cluster_identity_ids[0] : null + user_assigned_identity_id = length(var.additional_cluster_identity_ids) > 0 ? var.additional_cluster_identity_ids[0] : null } tags = merge({ @@ -303,6 +290,36 @@ resource "azurerm_kubernetes_cluster" "main" { }, local.tags) } +resource "azurerm_role_assignment" "aks_vmss_contributor" { + scope = azurerm_resource_group.aks.id + role_definition_name = "Virtual Machine Contributor" + principal_id = azurerm_kubernetes_cluster.main.identity[0].principal_id +} + +resource "azurerm_role_assignment" "azurerm_disk_encryption_set_key_vault_access" { + scope = azurerm_key_vault.key_vault.id + role_definition_name = "Key Vault Crypto Service Encryption User" + principal_id = azurerm_kubernetes_cluster.main.identity[0].principal_id +} + +resource "azurerm_role_assignment" "aks_network_contributor" { + scope = azurerm_resource_group.aks.id + role_definition_name = "Network Contributor" + principal_id = azurerm_kubernetes_cluster.main.identity[0].principal_id +} + +resource "azurerm_role_assignment" "des_reader" { + scope = azurerm_disk_encryption_set.main.id + role_definition_name = "Reader" + principal_id = azurerm_kubernetes_cluster.main.identity[0].principal_id +} + +resource "azurerm_role_assignment" "control_plane_managed_identity_operator_on_kubelet" { + scope = azurerm_kubernetes_cluster.main.id + role_definition_name = "Managed Identity Operator" + principal_id = azurerm_kubernetes_cluster.main.identity[0].principal_id +} + resource "azurerm_kubernetes_cluster_node_pool" "autoscaled" { count = var.autoscaled_node_pool.enabled ? 1 : 0 name = var.autoscaled_node_pool.name @@ -330,33 +347,3 @@ resource "local_file" "kube_config" { content = azurerm_kubernetes_cluster.main.kube_config_raw filename = local.kubeconfig_path } - -resource "azurerm_role_assignment" "aks_vmss_contributor" { - scope = azurerm_resource_group.aks.id - role_definition_name = "Virtual Machine Contributor" - principal_id = azurerm_user_assigned_identity.cluster.principal_id -} - -resource "azurerm_role_assignment" "azurerm_disk_encryption_set_key_vault_access" { - scope = azurerm_key_vault.key_vault.id - role_definition_name = "Key Vault Crypto Service Encryption User" - principal_id = azurerm_user_assigned_identity.cluster.principal_id -} - -resource "azurerm_role_assignment" "aks_network_contributor" { - scope = azurerm_resource_group.aks.id - role_definition_name = "Network Contributor" - principal_id = azurerm_user_assigned_identity.cluster.principal_id -} - -resource "azurerm_role_assignment" "des_reader" { - scope = azurerm_disk_encryption_set.main.id - role_definition_name = "Reader" - principal_id = azurerm_user_assigned_identity.cluster.principal_id -} - -resource "azurerm_role_assignment" "control_plane_managed_identity_operator_on_kubelet" { - scope = azurerm_user_assigned_identity.cluster.id - role_definition_name = "Managed Identity Operator" - principal_id = azurerm_user_assigned_identity.cluster.principal_id -} From 0a913c8993729d11ed77ce360534750fcb43da3b Mon Sep 17 00:00:00 2001 From: Ryan VanGundy Date: Wed, 21 May 2025 00:01:03 -0400 Subject: [PATCH 2/4] Use system assigned user ID --- docs/terraform/cluster/azure-aks.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/terraform/cluster/azure-aks.md b/docs/terraform/cluster/azure-aks.md index c7f949ab..7422dc41 100644 --- a/docs/terraform/cluster/azure-aks.md +++ b/docs/terraform/cluster/azure-aks.md @@ -57,7 +57,6 @@ No modules. | [azurerm_role_assignment.azurerm_disk_encryption_set_key_vault_access](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.control_plane_managed_identity_operator_on_kubelet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.des_reader](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | -| [azurerm_user_assigned_identity.cluster](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) | resource | | [local_file.kube_config](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [random_string.key](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [time_static.expiry](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/static) | resource | From 37786d8922e2577297e2ceacaab7f41ef85c50f8 Mon Sep 17 00:00:00 2001 From: Ryan VanGundy Date: Wed, 21 May 2025 08:32:21 -0400 Subject: [PATCH 3/4] Use default SystemAssigned IDs --- terraform/cluster/azure-aks/main.tf | 44 +++++---------------- terraform/cluster/azure-aks/test.tftest.hcl | 37 +++++++++++++++++ terraform/cluster/azure-aks/variables.tf | 22 ++++++++++- 3 files changed, 66 insertions(+), 37 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 8dd837bd..e9891848 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -276,13 +276,17 @@ resource "azurerm_kubernetes_cluster" "main" { } identity { - type = length(var.additional_cluster_identity_ids) > 0 ? "SystemAssigned, UserAssigned" : "SystemAssigned" - identity_ids = var.additional_cluster_identity_ids + type = length(var.user_assigned_identity_ids) > 0 ? "UserAssigned" : "SystemAssigned" + identity_ids = var.user_assigned_identity_ids } - kubelet_identity { - client_id = length(var.additional_cluster_identity_ids) > 0 ? var.additional_cluster_identity_ids[0] : null - user_assigned_identity_id = length(var.additional_cluster_identity_ids) > 0 ? var.additional_cluster_identity_ids[0] : null + dynamic "kubelet_identity" { + for_each = var.kubelet_user_assigned_identity_id != null ? [1] : [] + content { + client_id = var.kubelet_client_id + object_id = var.kubelet_object_id + user_assigned_identity_id = var.kubelet_user_assigned_identity_id + } } tags = merge({ @@ -290,36 +294,6 @@ resource "azurerm_kubernetes_cluster" "main" { }, local.tags) } -resource "azurerm_role_assignment" "aks_vmss_contributor" { - scope = azurerm_resource_group.aks.id - role_definition_name = "Virtual Machine Contributor" - principal_id = azurerm_kubernetes_cluster.main.identity[0].principal_id -} - -resource "azurerm_role_assignment" "azurerm_disk_encryption_set_key_vault_access" { - scope = azurerm_key_vault.key_vault.id - role_definition_name = "Key Vault Crypto Service Encryption User" - principal_id = azurerm_kubernetes_cluster.main.identity[0].principal_id -} - -resource "azurerm_role_assignment" "aks_network_contributor" { - scope = azurerm_resource_group.aks.id - role_definition_name = "Network Contributor" - principal_id = azurerm_kubernetes_cluster.main.identity[0].principal_id -} - -resource "azurerm_role_assignment" "des_reader" { - scope = azurerm_disk_encryption_set.main.id - role_definition_name = "Reader" - principal_id = azurerm_kubernetes_cluster.main.identity[0].principal_id -} - -resource "azurerm_role_assignment" "control_plane_managed_identity_operator_on_kubelet" { - scope = azurerm_kubernetes_cluster.main.id - role_definition_name = "Managed Identity Operator" - principal_id = azurerm_kubernetes_cluster.main.identity[0].principal_id -} - resource "azurerm_kubernetes_cluster_node_pool" "autoscaled" { count = var.autoscaled_node_pool.enabled ? 1 : 0 name = var.autoscaled_node_pool.name diff --git a/terraform/cluster/azure-aks/test.tftest.hcl b/terraform/cluster/azure-aks/test.tftest.hcl index 475bb70d..b0fc91bd 100644 --- a/terraform/cluster/azure-aks/test.tftest.hcl +++ b/terraform/cluster/azure-aks/test.tftest.hcl @@ -66,6 +66,11 @@ run "minimal_configuration" { condition = azurerm_kubernetes_cluster.main.local_account_disabled == false error_message = "Local accounts should be enabled by default" } + + assert { + condition = azurerm_kubernetes_cluster.main.identity[0].type == "SystemAssigned" + error_message = "Cluster should use system-assigned identity by default" + } } # Tests a full configuration with all optional variables explicitly set, @@ -79,6 +84,13 @@ run "full_configuration" { cluster_name = "test-cluster" resource_group_name = "test-rg" kubernetes_version = "1.32" + user_assigned_identity_ids = [ + "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity-1", + "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity-2" + ] + kubelet_client_id = "test-client-id" + kubelet_object_id = "test-object-id" + kubelet_user_assigned_identity_id = "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity-1" default_node_pool = { name = "system" vm_size = "Standard_D2s_v3" @@ -181,6 +193,31 @@ run "full_configuration" { condition = azurerm_kubernetes_cluster.main.local_account_disabled == false error_message = "Local accounts should be enabled" } + + assert { + condition = azurerm_kubernetes_cluster.main.identity[0].type == "UserAssigned" + error_message = "Cluster should use user-assigned identity when IDs are provided" + } + + assert { + condition = length(azurerm_kubernetes_cluster.main.identity[0].identity_ids) == 2 + error_message = "Cluster should have 2 user-assigned identity IDs" + } + + assert { + condition = azurerm_kubernetes_cluster.main.kubelet_identity[0].client_id == "test-client-id" + error_message = "Kubelet client ID should match input" + } + + assert { + condition = azurerm_kubernetes_cluster.main.kubelet_identity[0].object_id == "test-object-id" + error_message = "Kubelet object ID should match input" + } + + assert { + condition = azurerm_kubernetes_cluster.main.kubelet_identity[0].user_assigned_identity_id == "/subscriptions/12345678-1234-9876-4563-123456789012/resourceGroups/example-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-identity-1" + error_message = "Kubelet user-assigned identity ID should match input" + } } # Tests the private cluster configuration, ensuring that enabling the private_cluster_enabled diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index dde63e97..1033f120 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -199,9 +199,9 @@ variable "expiration_date" { default = null } -variable "additional_cluster_identity_ids" { +variable "user_assigned_identity_ids" { type = list(string) - description = "Additional user assigned identity IDs for the AKS cluster" + description = "User assigned identity IDs for the AKS cluster. If provided, the cluster will use only user-assigned identities." default = [] } @@ -234,3 +234,21 @@ variable "endpoint_private_access" { type = bool default = false } + +variable "kubelet_client_id" { + description = "Client ID of the user-assigned identity to use for the kubelet. If not provided, the cluster will use the system-assigned identity." + type = string + default = null +} + +variable "kubelet_object_id" { + description = "Object ID of the user-assigned identity to use for the kubelet. If not provided, the cluster will use the system-assigned identity." + type = string + default = null +} + +variable "kubelet_user_assigned_identity_id" { + description = "Resource ID of the user-assigned identity to use for the kubelet. If not provided, the cluster will use the system-assigned identity." + type = string + default = null +} From 47e0407462cc0db9c6c12431ffa1e202665a6bb0 Mon Sep 17 00:00:00 2001 From: Ryan VanGundy Date: Wed, 21 May 2025 08:32:31 -0400 Subject: [PATCH 4/4] docs --- docs/terraform/cluster/azure-aks.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/terraform/cluster/azure-aks.md b/docs/terraform/cluster/azure-aks.md index 7422dc41..06e23dc3 100644 --- a/docs/terraform/cluster/azure-aks.md +++ b/docs/terraform/cluster/azure-aks.md @@ -52,11 +52,6 @@ No modules. | [azurerm_kubernetes_cluster_node_pool.autoscaled](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource | | [azurerm_log_analytics_workspace.aks_logs](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/log_analytics_workspace) | resource | | [azurerm_resource_group.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | -| [azurerm_role_assignment.aks_network_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | -| [azurerm_role_assignment.aks_vmss_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | -| [azurerm_role_assignment.azurerm_disk_encryption_set_key_vault_access](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | -| [azurerm_role_assignment.control_plane_managed_identity_operator_on_kubelet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | -| [azurerm_role_assignment.des_reader](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [local_file.kube_config](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource | | [random_string.key](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [time_static.expiry](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/static) | resource | @@ -67,7 +62,6 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [additional\_cluster\_identity\_ids](#input\_additional\_cluster\_identity\_ids) | Additional user assigned identity IDs for the AKS cluster | `list(string)` | `[]` | no | | [auto\_scaler\_profile](#input\_auto\_scaler\_profile) | Configuration for the AKS cluster's auto-scaler | 
object({
    balance_similar_node_groups      = bool
    max_graceful_termination_sec     = number
    scale_down_delay_after_add       = string
    scale_down_delay_after_delete    = string
    scale_down_delay_after_failure   = string
    scan_interval                    = string
    scale_down_unneeded              = string
    scale_down_unready               = string
    scale_down_utilization_threshold = string
  })
| 
{
  "balance_similar_node_groups": true,
  "max_graceful_termination_sec": 600,
  "scale_down_delay_after_add": "10m",
  "scale_down_delay_after_delete": "10s",
  "scale_down_delay_after_failure": "3m",
  "scale_down_unneeded": "10m",
  "scale_down_unready": "20m",
  "scale_down_utilization_threshold": "0.5",
  "scan_interval": "10s"
}
| no | | [automatic\_upgrade\_channel](#input\_automatic\_upgrade\_channel) | The automatic upgrade channel for the AKS cluster | `string` | `"stable"` | no | | [autoscaled\_node\_pool](#input\_autoscaled\_node\_pool) | Configuration for the autoscaled node pool | 
object({
    enabled                 = bool
    name                    = string
    vm_size                 = string
    mode                    = string
    os_disk_type            = string
    max_pods                = number
    host_encryption_enabled = bool
    min_count               = number
    max_count               = number
  })
| 
{
  "enabled": true,
  "host_encryption_enabled": true,
  "max_count": 3,
  "max_pods": 110,
  "min_count": 1,
  "mode": "User",
  "name": "autoscaled",
  "os_disk_type": "Managed",
  "vm_size": "Standard_D2s_v3"
}
| no | @@ -79,6 +73,9 @@ No modules. | [dns\_service\_ip](#input\_dns\_service\_ip) | IP address for Kubernetes DNS service | `string` | `"10.96.0.10"` | no | | [endpoint\_private\_access](#input\_endpoint\_private\_access) | Whether to enable private access to the Kubernetes API server | `bool` | `false` | no | | [expiration\_date](#input\_expiration\_date) | The expiration date for the AKS cluster's key vault | `string` | `null` | no | +| [kubelet\_client\_id](#input\_kubelet\_client\_id) | Client ID of the user-assigned identity to use for the kubelet. If not provided, the cluster will use the system-assigned identity. | `string` | `null` | no | +| [kubelet\_object\_id](#input\_kubelet\_object\_id) | Object ID of the user-assigned identity to use for the kubelet. If not provided, the cluster will use the system-assigned identity. | `string` | `null` | no | +| [kubelet\_user\_assigned\_identity\_id](#input\_kubelet\_user\_assigned\_identity\_id) | Resource ID of the user-assigned identity to use for the kubelet. If not provided, the cluster will use the system-assigned identity. | `string` | `null` | no | | [kubernetes\_version](#input\_kubernetes\_version) | Version of Kubernetes to use | `string` | `"1.32"` | no | | [local\_account\_disabled](#input\_local\_account\_disabled) | Whether to disable local accounts for the AKS cluster | `bool` | `false` | no | | [name](#input\_name) | Name of the resource | `string` | `"cluster"` | no | @@ -92,6 +89,7 @@ No modules. | [sku\_tier](#input\_sku\_tier) | The SKU tier for the AKS cluster | `string` | `"Standard"` | no | | [soft\_delete\_retention\_days](#input\_soft\_delete\_retention\_days) | The number of days to retain the AKS cluster's key vault | `number` | `7` | no | | [tags](#input\_tags) | Tags to apply to the resources | `map(string)` | `{}` | no | +| [user\_assigned\_identity\_ids](#input\_user\_assigned\_identity\_ids) | User assigned identity IDs for the AKS cluster. If provided, the cluster will use only user-assigned identities. | `list(string)` | `[]` | no | | [vnet\_module\_name](#input\_vnet\_module\_name) | Name on the VNET module | `string` | `"network"` | no | | [vnet\_subnet\_id](#input\_vnet\_subnet\_id) | ID of the subnet | `string` | `null` | no | | [workload\_autoscaler\_profile](#input\_workload\_autoscaler\_profile) | Configuration for the AKS cluster's workload autoscaler | 
object({
    keda_enabled                    = bool
    vertical_pod_autoscaler_enabled = bool
  })
| 
{
  "keda_enabled": false,
  "vertical_pod_autoscaler_enabled": false
}
| no |