From 5133e7b600320f101b89e6eb8c15fe59ef0bc83b Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Fri, 2 May 2025 15:13:32 +0200 Subject: [PATCH 01/18] Adds AKS support --- .../cluster/azure-aks/.terraform.lock.hcl | 41 ++++++ terraform/cluster/azure-aks/main.tf | 85 +++++++++++++ terraform/cluster/azure-aks/outputs.tf | 18 +++ terraform/cluster/azure-aks/variables.tf | 54 ++++++++ .../network/azure-vpc/.terraform.lock.hcl | 21 +++ terraform/network/azure-vpc/main.tf | 120 ++++++++++++++++++ terraform/network/azure-vpc/outputs.tf | 18 +++ terraform/network/azure-vpc/variables.tf | 55 ++++++++ 8 files changed, 412 insertions(+) create mode 100644 terraform/cluster/azure-aks/.terraform.lock.hcl create mode 100644 terraform/cluster/azure-aks/main.tf create mode 100644 terraform/cluster/azure-aks/outputs.tf create mode 100644 terraform/cluster/azure-aks/variables.tf create mode 100644 terraform/network/azure-vpc/.terraform.lock.hcl create mode 100644 terraform/network/azure-vpc/main.tf create mode 100644 terraform/network/azure-vpc/outputs.tf create mode 100644 terraform/network/azure-vpc/variables.tf diff --git a/terraform/cluster/azure-aks/.terraform.lock.hcl b/terraform/cluster/azure-aks/.terraform.lock.hcl new file mode 100644 index 00000000..3a5884b7 --- /dev/null +++ b/terraform/cluster/azure-aks/.terraform.lock.hcl @@ -0,0 +1,41 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.27.0" + constraints = "~> 4.27.0" + hashes = [ + "h1:hmAzHk4XVbrGQ5iJJj1QdFx0aWNW9Hjh+GIE6S8G5I8=", + "zh:0c69edea1995bd3bd9e61980757169c35bf22281b660b5c755b6cb13d08d29d2", + "zh:25b86bf7b9678371d8573983954c571696f3e64a3967133be3b835da36307106", + "zh:49921cff4f26a49bafada60cd07dabb52c5eb35231059ed928a4f4722e269c82", + "zh:4b986166531f9fd1289f01d8220519443e74888a21da512c1b841b006dad6215", + "zh:53fb65b2ca4df637f03e4748a100a7d7fc77249e307c03e294d6259cec0310f6", + "zh:5c0d021a387ca4e2a5a01da009746a08c45f08e971c10d9bda54539d7264d671", + "zh:600043f2b20dc5a45275e43f175c19fe8b6e8e9557a0c884aef018f1f63de90e", + "zh:a0284f6f38912f67bb4cb7829fda3fa75be81fea6a9b21119965c2a839430092", + "zh:a7ac0576e2069ef77557042c6b5157ded364fbd355b2f9bf7f5441622424086e", + "zh:c5db0bcafe986868e28cc6225b68b2d1cf4bf631939d260ca845f17a9aa1677d", + "zh:ce620c0eb71b1fdd925828b30cf232a869abccf1c459180f2f991c4166315251", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/local" { + version = "2.5.2" + hashes = [ + "h1:IyFbOIO6mhikFNL/2h1iZJ6kyN3U00jgkpCLUCThAfE=", + "zh:136299545178ce281c56f36965bf91c35407c11897f7082b3b983d86cb79b511", + "zh:3b4486858aa9cb8163378722b642c57c529b6c64bfbfc9461d940a84cd66ebea", + "zh:4855ee628ead847741aa4f4fc9bed50cfdbf197f2912775dd9fe7bc43fa077c0", + "zh:4b8cd2583d1edcac4011caafe8afb7a95e8110a607a1d5fb87d921178074a69b", + "zh:52084ddaff8c8cd3f9e7bcb7ce4dc1eab00602912c96da43c29b4762dc376038", + "zh:71562d330d3f92d79b2952ffdda0dad167e952e46200c767dd30c6af8d7c0ed3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:805f81ade06ff68fa8b908d31892eaed5c180ae031c77ad35f82cb7a74b97cf4", + "zh:8b6b3ebeaaa8e38dd04e56996abe80db9be6f4c1df75ac3cccc77642899bd464", + "zh:ad07750576b99248037b897de71113cc19b1a8d0bc235eb99173cc83d0de3b1b", + "zh:b9f1c3bfadb74068f5c205292badb0661e17ac05eb23bfe8bd809691e4583d0e", + "zh:cc4cbcd67414fefb111c1bf7ab0bc4beb8c0b553d01719ad17de9a047adff4d1", + ] +} diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf new file mode 100644 index 00000000..acfb1514 --- /dev/null +++ b/terraform/cluster/azure-aks/main.tf @@ -0,0 +1,85 @@ +#--------------------------------------------------------------------------------------------------- +# Versions +#--------------------------------------------------------------------------------------------------- + +terraform { + required_version = ">=1.8" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.27.0" + } + } +} + +#----------------------------------------------------------------------------------------------------------------------- +# Azure Provider configuration +#----------------------------------------------------------------------------------------------------------------------- + +provider "azurerm" { + use_oidc = var.azure_use_oidc + client_id = var.azure_client_id + tenant_id = var.azure_tenant_id + subscription_id = var.azure_subscription_id + features {} +} + +#----------------------------------------------------------------------------------------------------------------------- +# Locals +#----------------------------------------------------------------------------------------------------------------------- + +locals { + kubeconfig_path = "${var.context_path}/.kube/config" +} + +#----------------------------------------------------------------------------------------------------------------------- +# AKS Cluster +#----------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_resource_group" "aks" { + name = "${var.prefix}-aks-rg" + location = "eastus" +} + +data "azurerm_subnet" "private" { + name = "${var.prefix}-priv-subnet-1" + resource_group_name = "${var.prefix}-vpc-rg" + virtual_network_name = "${var.prefix}-vpc" +} + +resource "azurerm_kubernetes_cluster" "main" { + name = "${var.prefix}-${var.cluster_name}" + location = azurerm_resource_group.aks.location + resource_group_name = azurerm_resource_group.aks.name + dns_prefix = "${var.prefix}-${var.cluster_name}" + kubernetes_version = var.kubernetes_version + role_based_access_control_enabled = var.role_based_access_control_enabled + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_D2_v2" + vnet_subnet_id = data.azurerm_subnet.private.id + + upgrade_settings { + drain_timeout_in_minutes = 0 + max_surge = "10%" + node_soak_duration_in_minutes = 0 + } + } + + identity { + type = "SystemAssigned" + } + + lifecycle { + ignore_changes = [ + default_node_pool[0].node_count + ] + } +} + +resource "local_file" "kube_config" { + content = azurerm_kubernetes_cluster.main.kube_config_raw + filename = "${local.kubeconfig_path}" +} diff --git a/terraform/cluster/azure-aks/outputs.tf b/terraform/cluster/azure-aks/outputs.tf new file mode 100644 index 00000000..e42d9292 --- /dev/null +++ b/terraform/cluster/azure-aks/outputs.tf @@ -0,0 +1,18 @@ +#----------------------------------------------------------------------------------------------------------------------- +# Outputs +#----------------------------------------------------------------------------------------------------------------------- + +output "cluster_name" { + description = "Name of the AKS cluster" + value = azurerm_kubernetes_cluster.main.name +} + +output "resource_group_name" { + description = "Name of the resource group containing the AKS cluster" + value = azurerm_resource_group.aks.name +} + +output "cluster_identity" { + description = "System assigned identity of the AKS cluster" + value = azurerm_kubernetes_cluster.main.identity[0].principal_id +} diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf new file mode 100644 index 00000000..b906dba4 --- /dev/null +++ b/terraform/cluster/azure-aks/variables.tf @@ -0,0 +1,54 @@ +#----------------------------------------------------------------------------------------------------------------------- +# Variables +#----------------------------------------------------------------------------------------------------------------------- + +variable "prefix" { + description = "Prefix for the resources" + type = string + default = "windsor" +} + +variable "cluster_name" { + description = "Name of the AKS cluster" + type = string + default = "aks-cluster" +} + +variable "kubernetes_version" { + description = "Version of Kubernetes to use" + type = string + default = "1.32" +} + +variable "context_path" { + type = string + description = "The path to the context folder, where kubeconfig is stored" + default = "" +} + +variable "azure_use_oidc" { + type = bool + description = "Whether to use OIDC for the AKS cluster" + default = false +} + +variable "azure_client_id" { + type = string + description = "Client ID for the AKS cluster" +} + +variable "azure_tenant_id" { + type = string + description = "Tenant ID for the AKS cluster" +} + +variable "azure_subscription_id" { + type = string + description = "Subscription ID for the AKS cluster" +} + +variable "role_based_access_control_enabled" { + type = bool + description = "Whether to enable role-based access control for the AKS cluster" + default = true +} diff --git a/terraform/network/azure-vpc/.terraform.lock.hcl b/terraform/network/azure-vpc/.terraform.lock.hcl new file mode 100644 index 00000000..771c21bc --- /dev/null +++ b/terraform/network/azure-vpc/.terraform.lock.hcl @@ -0,0 +1,21 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.27.0" + hashes = [ + "h1:hmAzHk4XVbrGQ5iJJj1QdFx0aWNW9Hjh+GIE6S8G5I8=", + "zh:0c69edea1995bd3bd9e61980757169c35bf22281b660b5c755b6cb13d08d29d2", + "zh:25b86bf7b9678371d8573983954c571696f3e64a3967133be3b835da36307106", + "zh:49921cff4f26a49bafada60cd07dabb52c5eb35231059ed928a4f4722e269c82", + "zh:4b986166531f9fd1289f01d8220519443e74888a21da512c1b841b006dad6215", + "zh:53fb65b2ca4df637f03e4748a100a7d7fc77249e307c03e294d6259cec0310f6", + "zh:5c0d021a387ca4e2a5a01da009746a08c45f08e971c10d9bda54539d7264d671", + "zh:600043f2b20dc5a45275e43f175c19fe8b6e8e9557a0c884aef018f1f63de90e", + "zh:a0284f6f38912f67bb4cb7829fda3fa75be81fea6a9b21119965c2a839430092", + "zh:a7ac0576e2069ef77557042c6b5157ded364fbd355b2f9bf7f5441622424086e", + "zh:c5db0bcafe986868e28cc6225b68b2d1cf4bf631939d260ca845f17a9aa1677d", + "zh:ce620c0eb71b1fdd925828b30cf232a869abccf1c459180f2f991c4166315251", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/network/azure-vpc/main.tf b/terraform/network/azure-vpc/main.tf new file mode 100644 index 00000000..8177b81c --- /dev/null +++ b/terraform/network/azure-vpc/main.tf @@ -0,0 +1,120 @@ +#--------------------------------------------------------------------------------------------------- +# Versions +#--------------------------------------------------------------------------------------------------- + +terraform { + required_version = ">=1.8" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.27.0" + } + } +} + +#----------------------------------------------------------------------------------------------------------------------- +# Azure Provider configuration +#----------------------------------------------------------------------------------------------------------------------- + +provider "azurerm" { + use_oidc = var.azure_use_oidc + client_id = var.azure_client_id + tenant_id = var.azure_tenant_id + subscription_id = var.azure_subscription_id + features {} +} + +#----------------------------------------------------------------------------------------------------------------------- +# Resource Group +#----------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_resource_group" "main" { + name = "${var.prefix}-vpc-rg" + location = "eastus" +} + +#----------------------------------------------------------------------------------------------------------------------- +# Virtual Network +#----------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_virtual_network" "main" { + name = "${var.prefix}-vpc" + address_space = [var.vpc_cidr] + location = azurerm_resource_group.main.location + resource_group_name = azurerm_resource_group.main.name +} + +#----------------------------------------------------------------------------------------------------------------------- +# Subnets +#----------------------------------------------------------------------------------------------------------------------- + +# Public subnets +resource "azurerm_subnet" "public" { + count = length(var.vpc_subnets["public"]) > 0 ? length(var.vpc_subnets["public"]) : var.zones + name = "${var.prefix}-pub-subnet-${count.index + 1}" + resource_group_name = azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = length(var.vpc_subnets["public"]) > 0 ? [var.vpc_subnets["public"][count.index]] : ["${join(".", slice(split(".", var.vpc_cidr), 0, 2))}.${count.index + 1}.0/24"] +} + +# Private subnets +resource "azurerm_subnet" "private" { + count = length(var.vpc_subnets["private"]) > 0 ? length(var.vpc_subnets["private"]) : var.zones + name = "${var.prefix}-priv-subnet-${count.index + 1}" + resource_group_name = azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = length(var.vpc_subnets["private"]) > 0 ? [var.vpc_subnets["private"][count.index]] : ["${join(".", slice(split(".", var.vpc_cidr), 0, 2))}.1${count.index + 1}.0/24"] +} + +# Data subnets +resource "azurerm_subnet" "data" { + count = length(var.vpc_subnets["data"]) > 0 ? length(var.vpc_subnets["data"]) : var.zones + name = "${var.prefix}-data-subnet-${count.index + 1}" + resource_group_name = azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = length(var.vpc_subnets["data"]) > 0 ? [var.vpc_subnets["data"][count.index]] : ["${join(".", slice(split(".", var.vpc_cidr), 0, 2))}.2${count.index + 1}.0/24"] +} + +#----------------------------------------------------------------------------------------------------------------------- +# NAT Gateway +#----------------------------------------------------------------------------------------------------------------------- + +# Public IP for NAT Gateway +resource "azurerm_public_ip" "nat" { + count = var.zones + name = "${var.prefix}-nat-gw-ip-${count.index + 1}" + location = azurerm_resource_group.main.location + resource_group_name = azurerm_resource_group.main.name + allocation_method = "Static" + sku = "Standard" +} + +# NAT Gateway +resource "azurerm_nat_gateway" "main" { + count = var.zones + name = "${var.prefix}-nat-gw-${count.index + 1}" + location = azurerm_resource_group.main.location + resource_group_name = azurerm_resource_group.main.name + sku_name = "Standard" +} + +# Associate public IP with NAT Gateway +resource "azurerm_nat_gateway_public_ip_association" "main" { + count = var.zones + nat_gateway_id = azurerm_nat_gateway.main[count.index].id + public_ip_address_id = azurerm_public_ip.nat[count.index].id +} + +# Associate NAT Gateway with private subnet +resource "azurerm_subnet_nat_gateway_association" "private" { + count = var.zones + subnet_id = azurerm_subnet.private[count.index].id + nat_gateway_id = azurerm_nat_gateway.main[count.index].id +} + +# Associate NAT Gateway with data subnet +resource "azurerm_subnet_nat_gateway_association" "data" { + count = var.zones + subnet_id = azurerm_subnet.data[count.index].id + nat_gateway_id = azurerm_nat_gateway.main[count.index].id +} diff --git a/terraform/network/azure-vpc/outputs.tf b/terraform/network/azure-vpc/outputs.tf new file mode 100644 index 00000000..3f9c4560 --- /dev/null +++ b/terraform/network/azure-vpc/outputs.tf @@ -0,0 +1,18 @@ +#----------------------------------------------------------------------------------------------------------------------- +# Outputs +#----------------------------------------------------------------------------------------------------------------------- + +output "public_subnet_ids" { + description = "IDs of created public subnets" + value = azurerm_subnet.public[*].id +} + +output "private_subnet_ids" { + description = "IDs of created private subnets" + value = azurerm_subnet.private[*].id +} + +output "data_subnet_ids" { + description = "IDs of created data subnets" + value = azurerm_subnet.data[*].id +} diff --git a/terraform/network/azure-vpc/variables.tf b/terraform/network/azure-vpc/variables.tf new file mode 100644 index 00000000..369e7400 --- /dev/null +++ b/terraform/network/azure-vpc/variables.tf @@ -0,0 +1,55 @@ + +# Variables +variable "prefix" { + description = "Prefix for the resources" + type = string + default = "windsor" +} + +variable "zones" { + description = "Number of availability zones to create" + type = number + default = 1 +} + +variable "vpc_cidr" { + description = "CIDR block for VPC" + type = string + default = "10.20.0.0/16" +} + +variable "vpc_subnets" { + description = "Subnets to create in the VPC" + type = map(list(string)) + # example: { + # public = ["10.20.1.0/24", "10.20.2.0/24", "10.20.3.0/24"] + # private = ["10.20.11.0/24", "10.20.12.0/24", "10.20.13.0/24"] + # data = ["10.20.21.0/24", "10.20.22.0/24", "10.20.23.0/24"] + # } + default = { + public = [] + private = [] + data = [] + } +} + +variable "azure_use_oidc" { + type = bool + description = "Whether to use OIDC for the AKS cluster" + default = false +} + +variable "azure_client_id" { + type = string + description = "Client ID for the AKS cluster" +} + +variable "azure_tenant_id" { + type = string + description = "Tenant ID for the AKS cluster" +} + +variable "azure_subscription_id" { + type = string + description = "Subscription ID for the AKS cluster" +} From a90c2ab9088c476ae9aaad94dc57d2ff864ddbbb Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Fri, 2 May 2025 15:15:30 +0200 Subject: [PATCH 02/18] Fixes formatting --- terraform/cluster/azure-aks/main.tf | 14 +++++++------- terraform/network/azure-vpc/outputs.tf | 2 +- terraform/network/azure-vpc/variables.tf | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index acfb1514..37a3ec93 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -29,7 +29,7 @@ provider "azurerm" { #----------------------------------------------------------------------------------------------------------------------- locals { - kubeconfig_path = "${var.context_path}/.kube/config" + kubeconfig_path = "${var.context_path}/.kube/config" } #----------------------------------------------------------------------------------------------------------------------- @@ -48,11 +48,11 @@ data "azurerm_subnet" "private" { } resource "azurerm_kubernetes_cluster" "main" { - name = "${var.prefix}-${var.cluster_name}" - location = azurerm_resource_group.aks.location - resource_group_name = azurerm_resource_group.aks.name - dns_prefix = "${var.prefix}-${var.cluster_name}" - kubernetes_version = var.kubernetes_version + name = "${var.prefix}-${var.cluster_name}" + location = azurerm_resource_group.aks.location + resource_group_name = azurerm_resource_group.aks.name + dns_prefix = "${var.prefix}-${var.cluster_name}" + kubernetes_version = var.kubernetes_version role_based_access_control_enabled = var.role_based_access_control_enabled default_node_pool { @@ -81,5 +81,5 @@ resource "azurerm_kubernetes_cluster" "main" { resource "local_file" "kube_config" { content = azurerm_kubernetes_cluster.main.kube_config_raw - filename = "${local.kubeconfig_path}" + filename = local.kubeconfig_path } diff --git a/terraform/network/azure-vpc/outputs.tf b/terraform/network/azure-vpc/outputs.tf index 3f9c4560..64eeba11 100644 --- a/terraform/network/azure-vpc/outputs.tf +++ b/terraform/network/azure-vpc/outputs.tf @@ -8,7 +8,7 @@ output "public_subnet_ids" { } output "private_subnet_ids" { - description = "IDs of created private subnets" + description = "IDs of created private subnets" value = azurerm_subnet.private[*].id } diff --git a/terraform/network/azure-vpc/variables.tf b/terraform/network/azure-vpc/variables.tf index 369e7400..288737fa 100644 --- a/terraform/network/azure-vpc/variables.tf +++ b/terraform/network/azure-vpc/variables.tf @@ -20,7 +20,7 @@ variable "vpc_cidr" { variable "vpc_subnets" { description = "Subnets to create in the VPC" - type = map(list(string)) + type = map(list(string)) # example: { # public = ["10.20.1.0/24", "10.20.2.0/24", "10.20.3.0/24"] # private = ["10.20.11.0/24", "10.20.12.0/24", "10.20.13.0/24"] From 3834d1032dd3faf7903e253500d5fa20a49ff8bc Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Fri, 2 May 2025 17:30:52 +0200 Subject: [PATCH 03/18] Enables autoscaling --- terraform/cluster/azure-aks/main.tf | 42 ++++++++++++++++++++---- terraform/cluster/azure-aks/variables.tf | 25 ++++++++++++++ 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 37a3ec93..08981ed0 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -56,16 +56,29 @@ resource "azurerm_kubernetes_cluster" "main" { role_based_access_control_enabled = var.role_based_access_control_enabled default_node_pool { - name = "default" + name = "system" node_count = 1 - vm_size = "Standard_D2_v2" + vm_size = "Standard_D2s_v3" vnet_subnet_id = data.azurerm_subnet.private.id + orchestrator_version = var.kubernetes_version + only_critical_addons_enabled = true + } - upgrade_settings { - drain_timeout_in_minutes = 0 - max_surge = "10%" - node_soak_duration_in_minutes = 0 - } + auto_scaler_profile { + balance_similar_node_groups = var.auto_scaler_profile["balance_similar_node_groups"] + max_graceful_termination_sec = var.auto_scaler_profile["max_graceful_termination_sec"] + scale_down_delay_after_add = var.auto_scaler_profile["scale_down_delay_after_add"] + scale_down_delay_after_delete = var.auto_scaler_profile["scale_down_delay_after_delete"] + scale_down_delay_after_failure = var.auto_scaler_profile["scale_down_delay_after_failure"] + scan_interval = var.auto_scaler_profile["scan_interval"] + scale_down_unneeded = var.auto_scaler_profile["scale_down_unneeded"] + scale_down_unready = var.auto_scaler_profile["scale_down_unready"] + scale_down_utilization_threshold = var.auto_scaler_profile["scale_down_utilization_threshold"] + } + + workload_autoscaler_profile { + keda_enabled = var.workload_autoscaler_profile["keda_enabled"] + vertical_pod_autoscaler_enabled = var.workload_autoscaler_profile["vertical_pod_autoscaler_enabled"] } identity { @@ -79,6 +92,21 @@ resource "azurerm_kubernetes_cluster" "main" { } } +resource "azurerm_kubernetes_cluster_node_pool" "autoscaled" { + name = "autoscaled" + kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id + vm_size = "Standard_D2s_v3" + mode = "User" + auto_scaling_enabled = true + min_count = 1 + max_count = 3 + vnet_subnet_id = data.azurerm_subnet.private.id + orchestrator_version = var.kubernetes_version + node_labels = { + role = "app" + } +} + resource "local_file" "kube_config" { content = azurerm_kubernetes_cluster.main.kube_config_raw filename = local.kubeconfig_path diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index b906dba4..318ac7e0 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -52,3 +52,28 @@ variable "role_based_access_control_enabled" { description = "Whether to enable role-based access control for the AKS cluster" default = true } + +variable "auto_scaler_profile" { + type = map(string) + description = "Configuration for the AKS cluster's auto-scaler" + default = { + balance_similar_node_groups = true + max_graceful_termination_sec = 600 + scale_down_delay_after_add = "10m" + scale_down_delay_after_delete = "10s" + scale_down_delay_after_failure = "3m" + scan_interval = "10s" + scale_down_unneeded = "10m" + scale_down_unready = "20m" + scale_down_utilization_threshold = "0.5" + } +} + +variable "workload_autoscaler_profile" { + type = map(string) + description = "Configuration for the AKS cluster's workload autoscaler" + default = { + keda_enabled = false + vertical_pod_autoscaler_enabled = false + } +} From 0ccdc6bb0ba7f8007b485499f76c3f5f12c2c14b Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Fri, 2 May 2025 17:41:24 +0200 Subject: [PATCH 04/18] Fix formatting --- terraform/cluster/azure-aks/main.tf | 30 ++++++++++++------------ terraform/cluster/azure-aks/variables.tf | 28 +++++++++++++--------- terraform/network/azure-vpc/main.tf | 2 +- terraform/network/azure-vpc/variables.tf | 6 +++++ 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 08981ed0..71015a78 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -38,7 +38,7 @@ locals { resource "azurerm_resource_group" "aks" { name = "${var.prefix}-aks-rg" - location = "eastus" + location = var.region } data "azurerm_subnet" "private" { @@ -56,28 +56,28 @@ resource "azurerm_kubernetes_cluster" "main" { role_based_access_control_enabled = var.role_based_access_control_enabled default_node_pool { - name = "system" - node_count = 1 - vm_size = "Standard_D2s_v3" - vnet_subnet_id = data.azurerm_subnet.private.id - orchestrator_version = var.kubernetes_version + name = "system" + node_count = 1 + vm_size = "Standard_D2s_v3" + vnet_subnet_id = data.azurerm_subnet.private.id + orchestrator_version = var.kubernetes_version only_critical_addons_enabled = true } auto_scaler_profile { - balance_similar_node_groups = var.auto_scaler_profile["balance_similar_node_groups"] - max_graceful_termination_sec = var.auto_scaler_profile["max_graceful_termination_sec"] - scale_down_delay_after_add = var.auto_scaler_profile["scale_down_delay_after_add"] - scale_down_delay_after_delete = var.auto_scaler_profile["scale_down_delay_after_delete"] - scale_down_delay_after_failure = var.auto_scaler_profile["scale_down_delay_after_failure"] - scan_interval = var.auto_scaler_profile["scan_interval"] - scale_down_unneeded = var.auto_scaler_profile["scale_down_unneeded"] - scale_down_unready = var.auto_scaler_profile["scale_down_unready"] + balance_similar_node_groups = var.auto_scaler_profile["balance_similar_node_groups"] + max_graceful_termination_sec = var.auto_scaler_profile["max_graceful_termination_sec"] + scale_down_delay_after_add = var.auto_scaler_profile["scale_down_delay_after_add"] + scale_down_delay_after_delete = var.auto_scaler_profile["scale_down_delay_after_delete"] + scale_down_delay_after_failure = var.auto_scaler_profile["scale_down_delay_after_failure"] + scan_interval = var.auto_scaler_profile["scan_interval"] + scale_down_unneeded = var.auto_scaler_profile["scale_down_unneeded"] + scale_down_unready = var.auto_scaler_profile["scale_down_unready"] scale_down_utilization_threshold = var.auto_scaler_profile["scale_down_utilization_threshold"] } workload_autoscaler_profile { - keda_enabled = var.workload_autoscaler_profile["keda_enabled"] + keda_enabled = var.workload_autoscaler_profile["keda_enabled"] vertical_pod_autoscaler_enabled = var.workload_autoscaler_profile["vertical_pod_autoscaler_enabled"] } diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index 318ac7e0..a5750d62 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -8,6 +8,12 @@ variable "prefix" { default = "windsor" } +variable "region" { + description = "Region for the resources" + type = string + default = "eastus" +} + variable "cluster_name" { description = "Name of the AKS cluster" type = string @@ -56,15 +62,15 @@ variable "role_based_access_control_enabled" { variable "auto_scaler_profile" { type = map(string) description = "Configuration for the AKS cluster's auto-scaler" - default = { - balance_similar_node_groups = true - max_graceful_termination_sec = 600 - scale_down_delay_after_add = "10m" - scale_down_delay_after_delete = "10s" - scale_down_delay_after_failure = "3m" - scan_interval = "10s" - scale_down_unneeded = "10m" - scale_down_unready = "20m" + default = { + balance_similar_node_groups = true + max_graceful_termination_sec = 600 + scale_down_delay_after_add = "10m" + scale_down_delay_after_delete = "10s" + scale_down_delay_after_failure = "3m" + scan_interval = "10s" + scale_down_unneeded = "10m" + scale_down_unready = "20m" scale_down_utilization_threshold = "0.5" } } @@ -72,8 +78,8 @@ variable "auto_scaler_profile" { variable "workload_autoscaler_profile" { type = map(string) description = "Configuration for the AKS cluster's workload autoscaler" - default = { - keda_enabled = false + default = { + keda_enabled = false vertical_pod_autoscaler_enabled = false } } diff --git a/terraform/network/azure-vpc/main.tf b/terraform/network/azure-vpc/main.tf index 8177b81c..a3bafc52 100644 --- a/terraform/network/azure-vpc/main.tf +++ b/terraform/network/azure-vpc/main.tf @@ -30,7 +30,7 @@ provider "azurerm" { resource "azurerm_resource_group" "main" { name = "${var.prefix}-vpc-rg" - location = "eastus" + location = var.region } #----------------------------------------------------------------------------------------------------------------------- diff --git a/terraform/network/azure-vpc/variables.tf b/terraform/network/azure-vpc/variables.tf index 288737fa..8bc6cacb 100644 --- a/terraform/network/azure-vpc/variables.tf +++ b/terraform/network/azure-vpc/variables.tf @@ -6,6 +6,12 @@ variable "prefix" { default = "windsor" } +variable "region" { + description = "Region for the resources" + type = string + default = "eastus" +} + variable "zones" { description = "Number of availability zones to create" type = number From b384f5ab50c4eb48301917d20394b0f907c4e2a2 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Mon, 5 May 2025 15:03:48 +0200 Subject: [PATCH 05/18] Fixes checkov findings --- .../cluster/azure-aks/.terraform.lock.hcl | 19 ++ terraform/cluster/azure-aks/main.tf | 186 ++++++++++++++++-- terraform/cluster/azure-aks/variables.tf | 66 +++++++ 3 files changed, 254 insertions(+), 17 deletions(-) diff --git a/terraform/cluster/azure-aks/.terraform.lock.hcl b/terraform/cluster/azure-aks/.terraform.lock.hcl index 3a5884b7..7af8464f 100644 --- a/terraform/cluster/azure-aks/.terraform.lock.hcl +++ b/terraform/cluster/azure-aks/.terraform.lock.hcl @@ -39,3 +39,22 @@ provider "registry.terraform.io/hashicorp/local" { "zh:cc4cbcd67414fefb111c1bf7ab0bc4beb8c0b553d01719ad17de9a047adff4d1", ] } + +provider "registry.terraform.io/hashicorp/random" { + version = "3.7.2" + hashes = [ + "h1:KG4NuIBl1mRWU0KD/BGfCi1YN/j3F7H4YgeeM7iSdNs=", + "zh:14829603a32e4bc4d05062f059e545a91e27ff033756b48afbae6b3c835f508f", + "zh:1527fb07d9fea400d70e9e6eb4a2b918d5060d604749b6f1c361518e7da546dc", + "zh:1e86bcd7ebec85ba336b423ba1db046aeaa3c0e5f921039b3f1a6fc2f978feab", + "zh:24536dec8bde66753f4b4030b8f3ef43c196d69cccbea1c382d01b222478c7a3", + "zh:29f1786486759fad9b0ce4fdfbbfece9343ad47cd50119045075e05afe49d212", + "zh:4d701e978c2dd8604ba1ce962b047607701e65c078cb22e97171513e9e57491f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b8434212eef0f8c83f5a90c6d76feaf850f6502b61b53c329e85b3b281cba34", + "zh:ac8a23c212258b7976e1621275e3af7099e7e4a3d4478cf8d5d2a27f3bc3e967", + "zh:b516ca74431f3df4c6cf90ddcdb4042c626e026317a33c53f0b445a3d93b720d", + "zh:dc76e4326aec2490c1600d6871a95e78f9050f9ce427c71707ea412a2f2f1a62", + "zh:eac7b63e86c749c7d48f527671c7aee5b4e26c10be6ad7232d6860167f99dbb0", + ] +} diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 71015a78..92ffb2ca 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -21,7 +21,26 @@ provider "azurerm" { client_id = var.azure_client_id tenant_id = var.azure_tenant_id subscription_id = var.azure_subscription_id - features {} + features { + resource_group { + prevent_deletion_if_contains_resources = false + } + key_vault { + purge_soft_deleted_keys_on_destroy = true + recover_soft_deleted_keys = true + } + } +} + +data "azurerm_client_config" "current" {} + +#----------------------------------------------------------------------------------------------------------------------- +# Resource Groups +#----------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_resource_group" "aks" { + name = "${var.prefix}-aks-rg" + location = var.region } #----------------------------------------------------------------------------------------------------------------------- @@ -33,14 +52,120 @@ locals { } #----------------------------------------------------------------------------------------------------------------------- -# AKS Cluster +# Key Vault #----------------------------------------------------------------------------------------------------------------------- -resource "azurerm_resource_group" "aks" { - name = "${var.prefix}-aks-rg" - location = var.region +resource "azurerm_key_vault" "key_vault" { + name = "${var.prefix}-keyvault" + location = azurerm_resource_group.aks.location + resource_group_name = azurerm_resource_group.aks.name + tenant_id = data.azurerm_client_config.current.tenant_id + sku_name = "premium" + enabled_for_disk_encryption = true + purge_protection_enabled = true + soft_delete_retention_days = 7 + + access_policy { + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azurerm_client_config.current.object_id + + key_permissions = [ + "Create", + "Delete", + "Get", + "Purge", + "Recover", + "Update", + "GetRotationPolicy", + "SetRotationPolicy" + ] + + secret_permissions = [ + "Set", + ] + } +} + +resource "azurerm_key_vault_access_policy" "key_vault_access_policy_disk" { + key_vault_id = azurerm_key_vault.key_vault.id + + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = azurerm_disk_encryption_set.main.identity.0.principal_id + + key_permissions = [ + "Get", + "Decrypt", + "Encrypt", + "Sign", + "UnwrapKey", + "Verify", + "WrapKey", + ] + + depends_on = [ + azurerm_disk_encryption_set.main + ] +} + +resource "random_string" "key_vault_key_name" { + length = 6 + special = false + upper = false + numeric = false } +resource "azurerm_key_vault_key" "key_vault_key" { + name = "${var.prefix}-key-${random_string.key_vault_key_name.result}" + key_vault_id = azurerm_key_vault.key_vault.id + key_type = "RSA" + key_size = 2048 + + key_opts = [ + "decrypt", + "encrypt", + "sign", + "unwrapKey", + "verify", + "wrapKey", + ] + + rotation_policy { + automatic { + time_before_expiry = "P30D" + } + + expire_after = "P90D" + notify_before_expiry = "P29D" + } +} + +resource "azurerm_disk_encryption_set" "main" { + name = "${var.prefix}-des-${random_string.key_vault_key_name.result}" + resource_group_name = azurerm_resource_group.aks.name + location = azurerm_resource_group.aks.location + key_vault_key_id = azurerm_key_vault_key.key_vault_key.id + + identity { + type = "SystemAssigned" + } +} + +#----------------------------------------------------------------------------------------------------------------------- +# Log Analytics Workspace +#----------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_log_analytics_workspace" "aks_logs" { + name = "${var.prefix}-aks-logs" + location = azurerm_resource_group.aks.location + resource_group_name = azurerm_resource_group.aks.name + sku = "PerGB2018" + retention_in_days = 30 +} + +#----------------------------------------------------------------------------------------------------------------------- +# AKS Cluster +#----------------------------------------------------------------------------------------------------------------------- + data "azurerm_subnet" "private" { name = "${var.prefix}-priv-subnet-1" resource_group_name = "${var.prefix}-vpc-rg" @@ -54,6 +179,19 @@ resource "azurerm_kubernetes_cluster" "main" { dns_prefix = "${var.prefix}-${var.cluster_name}" kubernetes_version = var.kubernetes_version role_based_access_control_enabled = var.role_based_access_control_enabled + automatic_upgrade_channel = var.automatic_upgrade_channel + sku_tier = var.sku_tier + # checkov:skip=CKV_AZURE_6: this feature is in preview + # api_server_authorized_ip_ranges = var.api_server_authorized_ip_ranges + private_cluster_enabled = var.private_cluster_enabled + disk_encryption_set_id = azurerm_disk_encryption_set.main.id + # checkov:skip=CKV_AZURE_116: this replaces the addon_profile + azure_policy_enabled = var.azure_policy_enabled + local_account_disabled = var.local_account_disabled + + key_vault_secrets_provider { + secret_rotation_enabled = true + } default_node_pool { name = "system" @@ -62,6 +200,10 @@ resource "azurerm_kubernetes_cluster" "main" { vnet_subnet_id = data.azurerm_subnet.private.id orchestrator_version = var.kubernetes_version only_critical_addons_enabled = true + # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs + os_disk_type = var.os_disk_type + host_encryption_enabled = var.host_encryption_enabled + max_pods = var.max_pods } auto_scaler_profile { @@ -81,6 +223,15 @@ resource "azurerm_kubernetes_cluster" "main" { vertical_pod_autoscaler_enabled = var.workload_autoscaler_profile["vertical_pod_autoscaler_enabled"] } + network_profile { + network_policy = "azure" + network_plugin = "azure" + } + + oms_agent { + log_analytics_workspace_id = azurerm_log_analytics_workspace.aks_logs.id + } + identity { type = "SystemAssigned" } @@ -93,18 +244,19 @@ resource "azurerm_kubernetes_cluster" "main" { } resource "azurerm_kubernetes_cluster_node_pool" "autoscaled" { - name = "autoscaled" - kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id - vm_size = "Standard_D2s_v3" - mode = "User" - auto_scaling_enabled = true - min_count = 1 - max_count = 3 - vnet_subnet_id = data.azurerm_subnet.private.id - orchestrator_version = var.kubernetes_version - node_labels = { - role = "app" - } + name = "autoscaled" + kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id + vm_size = "Standard_D2s_v3" + mode = "User" + auto_scaling_enabled = true + min_count = var.min_count + max_count = var.max_count + vnet_subnet_id = data.azurerm_subnet.private.id + orchestrator_version = var.kubernetes_version + # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs + os_disk_type = var.os_disk_type + max_pods = var.max_pods + host_encryption_enabled = var.host_encryption_enabled } resource "local_file" "kube_config" { diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index a5750d62..32e50d4e 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -83,3 +83,69 @@ variable "workload_autoscaler_profile" { vertical_pod_autoscaler_enabled = false } } + +variable "automatic_upgrade_channel" { + type = string + description = "The automatic upgrade channel for the AKS cluster" + default = "stable" +} + +variable "sku_tier" { + type = string + description = "The SKU tier for the AKS cluster" + default = "Standard" +} + +variable "api_server_authorized_ip_ranges" { + type = list(string) + description = "The API server authorized IP ranges for the AKS cluster" + default = ["0.0.0.0/0"] +} + +variable "private_cluster_enabled" { + type = bool + description = "Whether to enable private cluster for the AKS cluster" + default = false +} + +variable "azure_policy_enabled" { + type = bool + description = "Whether to enable Azure Policy for the AKS cluster" + default = true +} + +variable "local_account_disabled" { + type = bool + description = "Whether to disable local accounts for the AKS cluster" + default = false +} + +variable "os_disk_type" { + type = string + description = "The type of OS disk for the AKS cluster" + default = "Managed" +} + +variable "host_encryption_enabled" { + type = bool + description = "Whether to enable host encryption for the AKS cluster" + default = true +} + +variable "max_pods" { + type = number + description = "The maximum number of pods for the AKS cluster" + default = 50 +} + +variable "min_count" { + type = number + description = "The minimum number of nodes for the AKS cluster" + default = 1 +} + +variable "max_count" { + type = number + description = "The maximum number of nodes for the AKS cluster" + default = 3 +} From d37509ed1958a96b9026e53c5818e41830cffa2e Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Mon, 5 May 2025 15:05:02 +0200 Subject: [PATCH 06/18] Fix formatting --- terraform/cluster/azure-aks/main.tf | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 92ffb2ca..31809dae 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -201,9 +201,9 @@ resource "azurerm_kubernetes_cluster" "main" { orchestrator_version = var.kubernetes_version only_critical_addons_enabled = true # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs - os_disk_type = var.os_disk_type - host_encryption_enabled = var.host_encryption_enabled - max_pods = var.max_pods + os_disk_type = var.os_disk_type + host_encryption_enabled = var.host_encryption_enabled + max_pods = var.max_pods } auto_scaler_profile { @@ -244,15 +244,15 @@ resource "azurerm_kubernetes_cluster" "main" { } resource "azurerm_kubernetes_cluster_node_pool" "autoscaled" { - name = "autoscaled" - kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id - vm_size = "Standard_D2s_v3" - mode = "User" - auto_scaling_enabled = true - min_count = var.min_count - max_count = var.max_count - vnet_subnet_id = data.azurerm_subnet.private.id - orchestrator_version = var.kubernetes_version + name = "autoscaled" + kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id + vm_size = "Standard_D2s_v3" + mode = "User" + auto_scaling_enabled = true + min_count = var.min_count + max_count = var.max_count + vnet_subnet_id = data.azurerm_subnet.private.id + orchestrator_version = var.kubernetes_version # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs os_disk_type = var.os_disk_type max_pods = var.max_pods From 89cce760c99f276344632a2772a431ffd8f7a3a8 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Mon, 5 May 2025 15:21:16 +0200 Subject: [PATCH 07/18] Fix checkov findings --- terraform/cluster/azure-aks/main.tf | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 31809dae..24272a0d 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -56,6 +56,8 @@ locals { #----------------------------------------------------------------------------------------------------------------------- resource "azurerm_key_vault" "key_vault" { + # checkov:skip=CKV2_AZURE_32: We are using a public cluster for testing + # private clusters are encouraged for production name = "${var.prefix}-keyvault" location = azurerm_resource_group.aks.location resource_group_name = azurerm_resource_group.aks.name @@ -64,6 +66,12 @@ resource "azurerm_key_vault" "key_vault" { enabled_for_disk_encryption = true purge_protection_enabled = true soft_delete_retention_days = 7 + public_network_access_enabled = false + + network_acls { + default_action = "Deny" + bypass = "AzureServices" + } access_policy { tenant_id = data.azurerm_client_config.current.tenant_id @@ -117,8 +125,9 @@ resource "random_string" "key_vault_key_name" { resource "azurerm_key_vault_key" "key_vault_key" { name = "${var.prefix}-key-${random_string.key_vault_key_name.result}" key_vault_id = azurerm_key_vault.key_vault.id - key_type = "RSA" + key_type = "RSA-HSM" key_size = 2048 + expiration_date = timeadd(timestamp(), "8760h") key_opts = [ "decrypt", @@ -181,12 +190,15 @@ resource "azurerm_kubernetes_cluster" "main" { role_based_access_control_enabled = var.role_based_access_control_enabled automatic_upgrade_channel = var.automatic_upgrade_channel sku_tier = var.sku_tier - # checkov:skip=CKV_AZURE_6: this feature is in preview + # checkov:skip=CKV_AZURE_6: This feature is in preview # api_server_authorized_ip_ranges = var.api_server_authorized_ip_ranges + # checkov:skip=CKV_AZURE_115: We are using a public cluster for testing + # private clusters are encouraged for production private_cluster_enabled = var.private_cluster_enabled disk_encryption_set_id = azurerm_disk_encryption_set.main.id - # checkov:skip=CKV_AZURE_116: this replaces the addon_profile + # checkov:skip=CKV_AZURE_116: This replaces the addon_profile azure_policy_enabled = var.azure_policy_enabled + # checkov:skip=CKV_AZURE_141: We are setting this to false to avoid the creation of an AD local_account_disabled = var.local_account_disabled key_vault_secrets_provider { From 7e012c549a86c3640cffbaf1a87b3c726a60b7fb Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Mon, 5 May 2025 15:21:37 +0200 Subject: [PATCH 08/18] Fix formatting --- terraform/cluster/azure-aks/main.tf | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 24272a0d..086926bb 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -58,14 +58,14 @@ locals { resource "azurerm_key_vault" "key_vault" { # checkov:skip=CKV2_AZURE_32: We are using a public cluster for testing # private clusters are encouraged for production - name = "${var.prefix}-keyvault" - location = azurerm_resource_group.aks.location - resource_group_name = azurerm_resource_group.aks.name - tenant_id = data.azurerm_client_config.current.tenant_id - sku_name = "premium" - enabled_for_disk_encryption = true - purge_protection_enabled = true - soft_delete_retention_days = 7 + name = "${var.prefix}-keyvault" + location = azurerm_resource_group.aks.location + resource_group_name = azurerm_resource_group.aks.name + tenant_id = data.azurerm_client_config.current.tenant_id + sku_name = "premium" + enabled_for_disk_encryption = true + purge_protection_enabled = true + soft_delete_retention_days = 7 public_network_access_enabled = false network_acls { @@ -123,10 +123,10 @@ resource "random_string" "key_vault_key_name" { } resource "azurerm_key_vault_key" "key_vault_key" { - name = "${var.prefix}-key-${random_string.key_vault_key_name.result}" - key_vault_id = azurerm_key_vault.key_vault.id - key_type = "RSA-HSM" - key_size = 2048 + name = "${var.prefix}-key-${random_string.key_vault_key_name.result}" + key_vault_id = azurerm_key_vault.key_vault.id + key_type = "RSA-HSM" + key_size = 2048 expiration_date = timeadd(timestamp(), "8760h") key_opts = [ @@ -197,7 +197,7 @@ resource "azurerm_kubernetes_cluster" "main" { private_cluster_enabled = var.private_cluster_enabled disk_encryption_set_id = azurerm_disk_encryption_set.main.id # checkov:skip=CKV_AZURE_116: This replaces the addon_profile - azure_policy_enabled = var.azure_policy_enabled + azure_policy_enabled = var.azure_policy_enabled # checkov:skip=CKV_AZURE_141: We are setting this to false to avoid the creation of an AD local_account_disabled = var.local_account_disabled From 45969cb6ecc3311b9db1c4e6d55b50057e1b3bec Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Mon, 5 May 2025 16:44:16 +0200 Subject: [PATCH 09/18] Fixes --- terraform/cluster/azure-aks/main.tf | 24 +++++++++++++----------- terraform/cluster/azure-aks/variables.tf | 12 ++++++++++++ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 086926bb..618e65cf 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -56,20 +56,22 @@ locals { #----------------------------------------------------------------------------------------------------------------------- resource "azurerm_key_vault" "key_vault" { + name = "${var.prefix}-keyvault" + location = azurerm_resource_group.aks.location + resource_group_name = azurerm_resource_group.aks.name + tenant_id = data.azurerm_client_config.current.tenant_id + sku_name = "premium" + enabled_for_disk_encryption = true + purge_protection_enabled = true + soft_delete_retention_days = 7 # checkov:skip=CKV2_AZURE_32: We are using a public cluster for testing - # private clusters are encouraged for production - name = "${var.prefix}-keyvault" - location = azurerm_resource_group.aks.location - resource_group_name = azurerm_resource_group.aks.name - tenant_id = data.azurerm_client_config.current.tenant_id - sku_name = "premium" - enabled_for_disk_encryption = true - purge_protection_enabled = true - soft_delete_retention_days = 7 - public_network_access_enabled = false + # private services are encouraged for production + public_network_access_enabled = var.public_network_access_enabled + # checkov:skip=CKV_AZURE_109: We are using a public cluster for testing + # private services are encouraged for production. Change to "Deny" for production. network_acls { - default_action = "Deny" + default_action = var.network_acls_default_action bypass = "AzureServices" } diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index 32e50d4e..11a6127d 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -149,3 +149,15 @@ variable "max_count" { description = "The maximum number of nodes for the AKS cluster" default = 3 } + +variable "public_network_access_enabled" { + type = bool + description = "Whether to enable public network access for the AKS cluster" + default = true +} + +variable "network_acls_default_action" { + type = string + description = "The default action for the AKS cluster's network ACLs" + default = "Allow" +} From c7234de74b0d8a3ccabf8fb89d2253cce6ec1135 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Mon, 5 May 2025 16:46:04 +0200 Subject: [PATCH 10/18] Fix checkov finding --- terraform/cluster/azure-aks/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 618e65cf..1121e181 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -64,7 +64,7 @@ resource "azurerm_key_vault" "key_vault" { enabled_for_disk_encryption = true purge_protection_enabled = true soft_delete_retention_days = 7 - # checkov:skip=CKV2_AZURE_32: We are using a public cluster for testing + # checkov:skip=CKV_AZURE_189: We are using a public cluster for testing # private services are encouraged for production public_network_access_enabled = var.public_network_access_enabled From 67a9beb1ad10f57401f227eb6034bb98706210fe Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Mon, 5 May 2025 16:48:07 +0200 Subject: [PATCH 11/18] Fix checkov finding --- terraform/cluster/azure-aks/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 1121e181..7dc9c30b 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -56,6 +56,7 @@ locals { #----------------------------------------------------------------------------------------------------------------------- resource "azurerm_key_vault" "key_vault" { + # checkov:skip=CKV2_AZURE_32: We are using a public cluster for testing, there is no need for private endpoints. name = "${var.prefix}-keyvault" location = azurerm_resource_group.aks.location resource_group_name = azurerm_resource_group.aks.name From bddfd4547e5dbd78f7cdceaa0858e41e18c6bea5 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Mon, 5 May 2025 18:32:18 +0200 Subject: [PATCH 12/18] Fixes key expiricy issue --- .../cluster/azure-aks/.terraform.lock.hcl | 19 +++++++++++++++++++ terraform/cluster/azure-aks/main.tf | 4 +++- terraform/cluster/azure-aks/variables.tf | 6 ++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/terraform/cluster/azure-aks/.terraform.lock.hcl b/terraform/cluster/azure-aks/.terraform.lock.hcl index 7af8464f..d6600340 100644 --- a/terraform/cluster/azure-aks/.terraform.lock.hcl +++ b/terraform/cluster/azure-aks/.terraform.lock.hcl @@ -58,3 +58,22 @@ provider "registry.terraform.io/hashicorp/random" { "zh:eac7b63e86c749c7d48f527671c7aee5b4e26c10be6ad7232d6860167f99dbb0", ] } + +provider "registry.terraform.io/hashicorp/time" { + version = "0.13.1" + hashes = [ + "h1:ZT5ppCNIModqk3iOkVt5my8b8yBHmDpl663JtXAIRqM=", + "zh:02cb9aab1002f0f2a94a4f85acec8893297dc75915f7404c165983f720a54b74", + "zh:04429b2b31a492d19e5ecf999b116d396dac0b24bba0d0fb19ecaefe193fdb8f", + "zh:26f8e51bb7c275c404ba6028c1b530312066009194db721a8427a7bc5cdbc83a", + "zh:772ff8dbdbef968651ab3ae76d04afd355c32f8a868d03244db3f8496e462690", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:898db5d2b6bd6ca5457dccb52eedbc7c5b1a71e4a4658381bcbb38cedbbda328", + "zh:8de913bf09a3fa7bedc29fec18c47c571d0c7a3d0644322c46f3aa648cf30cd8", + "zh:9402102c86a87bdfe7e501ffbb9c685c32bbcefcfcf897fd7d53df414c36877b", + "zh:b18b9bb1726bb8cfbefc0a29cf3657c82578001f514bcf4c079839b6776c47f0", + "zh:b9d31fdc4faecb909d7c5ce41d2479dd0536862a963df434be4b16e8e4edc94d", + "zh:c951e9f39cca3446c060bd63933ebb89cedde9523904813973fbc3d11863ba75", + "zh:e5b773c0d07e962291be0e9b413c7a22c044b8c7b58c76e8aa91d1659990dfb5", + ] +} diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 7dc9c30b..32b9962a 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -125,12 +125,14 @@ resource "random_string" "key_vault_key_name" { numeric = false } +resource "time_static" "expiry" {} + resource "azurerm_key_vault_key" "key_vault_key" { name = "${var.prefix}-key-${random_string.key_vault_key_name.result}" key_vault_id = azurerm_key_vault.key_vault.id key_type = "RSA-HSM" key_size = 2048 - expiration_date = timeadd(timestamp(), "8760h") + expiration_date = var.expiration_date != null ? var.expiration_date : timeadd(time_static.expiry.rfc3339, "8760h") key_opts = [ "decrypt", diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index 11a6127d..63c9f1d2 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -161,3 +161,9 @@ variable "network_acls_default_action" { description = "The default action for the AKS cluster's network ACLs" default = "Allow" } + +variable "expiration_date" { + type = string + description = "The expiration date for the AKS cluster's key vault" + default = null +} From 88c44e5ea6d3ac1dead76592e3f8b305a1b1c2a9 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Tue, 6 May 2025 15:37:21 +0200 Subject: [PATCH 13/18] Remove unnecessary variables --- terraform/cluster/azure-aks/main.tf | 3 --- terraform/cluster/azure-aks/variables.tf | 16 ---------------- terraform/network/azure-vpc/main.tf | 3 --- terraform/network/azure-vpc/variables.tf | 16 ---------------- 4 files changed, 38 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 32b9962a..78209da6 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -17,9 +17,6 @@ terraform { #----------------------------------------------------------------------------------------------------------------------- provider "azurerm" { - use_oidc = var.azure_use_oidc - client_id = var.azure_client_id - tenant_id = var.azure_tenant_id subscription_id = var.azure_subscription_id features { resource_group { diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index 63c9f1d2..80873524 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -32,22 +32,6 @@ variable "context_path" { default = "" } -variable "azure_use_oidc" { - type = bool - description = "Whether to use OIDC for the AKS cluster" - default = false -} - -variable "azure_client_id" { - type = string - description = "Client ID for the AKS cluster" -} - -variable "azure_tenant_id" { - type = string - description = "Tenant ID for the AKS cluster" -} - variable "azure_subscription_id" { type = string description = "Subscription ID for the AKS cluster" diff --git a/terraform/network/azure-vpc/main.tf b/terraform/network/azure-vpc/main.tf index a3bafc52..559dae4a 100644 --- a/terraform/network/azure-vpc/main.tf +++ b/terraform/network/azure-vpc/main.tf @@ -17,9 +17,6 @@ terraform { #----------------------------------------------------------------------------------------------------------------------- provider "azurerm" { - use_oidc = var.azure_use_oidc - client_id = var.azure_client_id - tenant_id = var.azure_tenant_id subscription_id = var.azure_subscription_id features {} } diff --git a/terraform/network/azure-vpc/variables.tf b/terraform/network/azure-vpc/variables.tf index 8bc6cacb..c82361de 100644 --- a/terraform/network/azure-vpc/variables.tf +++ b/terraform/network/azure-vpc/variables.tf @@ -39,22 +39,6 @@ variable "vpc_subnets" { } } -variable "azure_use_oidc" { - type = bool - description = "Whether to use OIDC for the AKS cluster" - default = false -} - -variable "azure_client_id" { - type = string - description = "Client ID for the AKS cluster" -} - -variable "azure_tenant_id" { - type = string - description = "Tenant ID for the AKS cluster" -} - variable "azure_subscription_id" { type = string description = "Subscription ID for the AKS cluster" From 9223f7a028ea0bf78fccecc141eb2cf4cbbc3081 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Tue, 6 May 2025 17:09:30 +0200 Subject: [PATCH 14/18] Addresses many of the points outlined in our call --- .../cluster/azure-aks/.terraform.lock.hcl | 19 -------- terraform/cluster/azure-aks/main.tf | 42 +++++++--------- terraform/cluster/azure-aks/variables.tf | 25 +++++++--- terraform/network/azure-vpc/main.tf | 48 +++++++++++-------- terraform/network/azure-vpc/variables.tf | 33 ++++++++----- 5 files changed, 85 insertions(+), 82 deletions(-) diff --git a/terraform/cluster/azure-aks/.terraform.lock.hcl b/terraform/cluster/azure-aks/.terraform.lock.hcl index d6600340..2788af76 100644 --- a/terraform/cluster/azure-aks/.terraform.lock.hcl +++ b/terraform/cluster/azure-aks/.terraform.lock.hcl @@ -40,25 +40,6 @@ provider "registry.terraform.io/hashicorp/local" { ] } -provider "registry.terraform.io/hashicorp/random" { - version = "3.7.2" - hashes = [ - "h1:KG4NuIBl1mRWU0KD/BGfCi1YN/j3F7H4YgeeM7iSdNs=", - "zh:14829603a32e4bc4d05062f059e545a91e27ff033756b48afbae6b3c835f508f", - "zh:1527fb07d9fea400d70e9e6eb4a2b918d5060d604749b6f1c361518e7da546dc", - "zh:1e86bcd7ebec85ba336b423ba1db046aeaa3c0e5f921039b3f1a6fc2f978feab", - "zh:24536dec8bde66753f4b4030b8f3ef43c196d69cccbea1c382d01b222478c7a3", - "zh:29f1786486759fad9b0ce4fdfbbfece9343ad47cd50119045075e05afe49d212", - "zh:4d701e978c2dd8604ba1ce962b047607701e65c078cb22e97171513e9e57491f", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7b8434212eef0f8c83f5a90c6d76feaf850f6502b61b53c329e85b3b281cba34", - "zh:ac8a23c212258b7976e1621275e3af7099e7e4a3d4478cf8d5d2a27f3bc3e967", - "zh:b516ca74431f3df4c6cf90ddcdb4042c626e026317a33c53f0b445a3d93b720d", - "zh:dc76e4326aec2490c1600d6871a95e78f9050f9ce427c71707ea412a2f2f1a62", - "zh:eac7b63e86c749c7d48f527671c7aee5b4e26c10be6ad7232d6860167f99dbb0", - ] -} - provider "registry.terraform.io/hashicorp/time" { version = "0.13.1" hashes = [ diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 78209da6..f88c38ed 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -17,7 +17,6 @@ terraform { #----------------------------------------------------------------------------------------------------------------------- provider "azurerm" { - subscription_id = var.azure_subscription_id features { resource_group { prevent_deletion_if_contains_resources = false @@ -32,20 +31,22 @@ provider "azurerm" { data "azurerm_client_config" "current" {} #----------------------------------------------------------------------------------------------------------------------- -# Resource Groups +# Locals #----------------------------------------------------------------------------------------------------------------------- -resource "azurerm_resource_group" "aks" { - name = "${var.prefix}-aks-rg" - location = var.region +locals { + kubeconfig_path = "${var.context_path}/.kube/config" + rg_name = var.resource_group_name == null ? "${var.context_id}-aks-rg" : var.resource_group_name + cluster_name = var.cluster_name == null ? "${var.context_id}-aks-cluster" : var.cluster_name } #----------------------------------------------------------------------------------------------------------------------- -# Locals +# Resource Groups #----------------------------------------------------------------------------------------------------------------------- -locals { - kubeconfig_path = "${var.context_path}/.kube/config" +resource "azurerm_resource_group" "aks" { + name = local.rg_name + location = var.region } #----------------------------------------------------------------------------------------------------------------------- @@ -54,7 +55,7 @@ locals { resource "azurerm_key_vault" "key_vault" { # checkov:skip=CKV2_AZURE_32: We are using a public cluster for testing, there is no need for private endpoints. - name = "${var.prefix}-keyvault" + name = "aks-keyvault-${var.context_id}" location = azurerm_resource_group.aks.location resource_group_name = azurerm_resource_group.aks.name tenant_id = data.azurerm_client_config.current.tenant_id @@ -115,17 +116,10 @@ resource "azurerm_key_vault_access_policy" "key_vault_access_policy_disk" { ] } -resource "random_string" "key_vault_key_name" { - length = 6 - special = false - upper = false - numeric = false -} - resource "time_static" "expiry" {} resource "azurerm_key_vault_key" "key_vault_key" { - name = "${var.prefix}-key-${random_string.key_vault_key_name.result}" + name = "aks-key-${var.context_id}" key_vault_id = azurerm_key_vault.key_vault.id key_type = "RSA-HSM" key_size = 2048 @@ -151,7 +145,7 @@ resource "azurerm_key_vault_key" "key_vault_key" { } resource "azurerm_disk_encryption_set" "main" { - name = "${var.prefix}-des-${random_string.key_vault_key_name.result}" + name = "des-${var.context_id}" resource_group_name = azurerm_resource_group.aks.name location = azurerm_resource_group.aks.location key_vault_key_id = azurerm_key_vault_key.key_vault_key.id @@ -166,7 +160,7 @@ resource "azurerm_disk_encryption_set" "main" { #----------------------------------------------------------------------------------------------------------------------- resource "azurerm_log_analytics_workspace" "aks_logs" { - name = "${var.prefix}-aks-logs" + name = "aks-logs-${var.context_id}" location = azurerm_resource_group.aks.location resource_group_name = azurerm_resource_group.aks.name sku = "PerGB2018" @@ -178,16 +172,16 @@ resource "azurerm_log_analytics_workspace" "aks_logs" { #----------------------------------------------------------------------------------------------------------------------- data "azurerm_subnet" "private" { - name = "${var.prefix}-priv-subnet-1" - resource_group_name = "${var.prefix}-vpc-rg" - virtual_network_name = "${var.prefix}-vpc" + name = "${var.context_id}-private-1" + resource_group_name = var.vnet_resource_group_name == null ? "windsor-vnet-rg-${var.context_id}" : var.vnet_resource_group_name + virtual_network_name = var.vnet_name == null ? "windsor-vnet-${var.context_id}" : var.vnet_name } resource "azurerm_kubernetes_cluster" "main" { - name = "${var.prefix}-${var.cluster_name}" + name = var.cluster_name == null ? "aks-cluster-${var.context_id}" : var.cluster_name location = azurerm_resource_group.aks.location resource_group_name = azurerm_resource_group.aks.name - dns_prefix = "${var.prefix}-${var.cluster_name}" + dns_prefix = var.cluster_name == null ? "aks-cluster-${var.context_id}" : var.cluster_name kubernetes_version = var.kubernetes_version role_based_access_control_enabled = var.role_based_access_control_enabled automatic_upgrade_channel = var.automatic_upgrade_channel diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index 80873524..8b41c3c1 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -2,10 +2,22 @@ # Variables #----------------------------------------------------------------------------------------------------------------------- -variable "prefix" { - description = "Prefix for the resources" +variable "resource_group_name" { + description = "Name of the resource group" type = string - default = "windsor" + default = null +} + +variable "vnet_resource_group_name" { + description = "Name of the VNET resource group" + type = string + default = null +} + +variable "vnet_name" { + description = "Name of the VNET" + type = string + default = null } variable "region" { @@ -17,7 +29,7 @@ variable "region" { variable "cluster_name" { description = "Name of the AKS cluster" type = string - default = "aks-cluster" + default = null } variable "kubernetes_version" { @@ -32,9 +44,10 @@ variable "context_path" { default = "" } -variable "azure_subscription_id" { +variable "context_id" { + description = "Context ID for the resources" type = string - description = "Subscription ID for the AKS cluster" + default = null } variable "role_based_access_control_enabled" { diff --git a/terraform/network/azure-vpc/main.tf b/terraform/network/azure-vpc/main.tf index 559dae4a..aede4282 100644 --- a/terraform/network/azure-vpc/main.tf +++ b/terraform/network/azure-vpc/main.tf @@ -17,16 +17,24 @@ terraform { #----------------------------------------------------------------------------------------------------------------------- provider "azurerm" { - subscription_id = var.azure_subscription_id features {} } +#----------------------------------------------------------------------------------------------------------------------- +# Locals +#----------------------------------------------------------------------------------------------------------------------- + +locals { + vnet_name = var.vnet_name == null ? "windsor-vnet-${var.context_id}" : var.vnet_name + rg_name = var.resource_group_name == null ? "windsor-vnet-rg-${var.context_id}" : var.resource_group_name +} + #----------------------------------------------------------------------------------------------------------------------- # Resource Group #----------------------------------------------------------------------------------------------------------------------- resource "azurerm_resource_group" "main" { - name = "${var.prefix}-vpc-rg" + name = local.rg_name location = var.region } @@ -35,8 +43,8 @@ resource "azurerm_resource_group" "main" { #----------------------------------------------------------------------------------------------------------------------- resource "azurerm_virtual_network" "main" { - name = "${var.prefix}-vpc" - address_space = [var.vpc_cidr] + name = local.vnet_name + address_space = [var.vnet_cidr] location = azurerm_resource_group.main.location resource_group_name = azurerm_resource_group.main.name } @@ -47,29 +55,29 @@ resource "azurerm_virtual_network" "main" { # Public subnets resource "azurerm_subnet" "public" { - count = length(var.vpc_subnets["public"]) > 0 ? length(var.vpc_subnets["public"]) : var.zones - name = "${var.prefix}-pub-subnet-${count.index + 1}" + count = length(var.vnet_subnets["public"]) > 0 ? length(var.vnet_subnets["public"]) : var.vnet_zones + name = "${var.context_id}-public-${count.index + 1}" resource_group_name = azurerm_resource_group.main.name virtual_network_name = azurerm_virtual_network.main.name - address_prefixes = length(var.vpc_subnets["public"]) > 0 ? [var.vpc_subnets["public"][count.index]] : ["${join(".", slice(split(".", var.vpc_cidr), 0, 2))}.${count.index + 1}.0/24"] + address_prefixes = length(var.vnet_subnets["public"]) > 0 ? [var.vnet_subnets["public"][count.index]] : ["${join(".", slice(split(".", var.vnet_cidr), 0, 2))}.${count.index + 1}.0/24"] } # Private subnets resource "azurerm_subnet" "private" { - count = length(var.vpc_subnets["private"]) > 0 ? length(var.vpc_subnets["private"]) : var.zones - name = "${var.prefix}-priv-subnet-${count.index + 1}" + count = length(var.vnet_subnets["private"]) > 0 ? length(var.vnet_subnets["private"]) : var.vnet_zones + name = "${var.context_id}-private-${count.index + 1}" resource_group_name = azurerm_resource_group.main.name virtual_network_name = azurerm_virtual_network.main.name - address_prefixes = length(var.vpc_subnets["private"]) > 0 ? [var.vpc_subnets["private"][count.index]] : ["${join(".", slice(split(".", var.vpc_cidr), 0, 2))}.1${count.index + 1}.0/24"] + address_prefixes = length(var.vnet_subnets["private"]) > 0 ? [var.vnet_subnets["private"][count.index]] : ["${join(".", slice(split(".", var.vnet_cidr), 0, 2))}.1${count.index + 1}.0/24"] } # Data subnets resource "azurerm_subnet" "data" { - count = length(var.vpc_subnets["data"]) > 0 ? length(var.vpc_subnets["data"]) : var.zones - name = "${var.prefix}-data-subnet-${count.index + 1}" + count = length(var.vnet_subnets["data"]) > 0 ? length(var.vnet_subnets["data"]) : var.vnet_zones + name = "${var.context_id}-data-${count.index + 1}" resource_group_name = azurerm_resource_group.main.name virtual_network_name = azurerm_virtual_network.main.name - address_prefixes = length(var.vpc_subnets["data"]) > 0 ? [var.vpc_subnets["data"][count.index]] : ["${join(".", slice(split(".", var.vpc_cidr), 0, 2))}.2${count.index + 1}.0/24"] + address_prefixes = length(var.vnet_subnets["data"]) > 0 ? [var.vnet_subnets["data"][count.index]] : ["${join(".", slice(split(".", var.vnet_cidr), 0, 2))}.2${count.index + 1}.0/24"] } #----------------------------------------------------------------------------------------------------------------------- @@ -78,8 +86,8 @@ resource "azurerm_subnet" "data" { # Public IP for NAT Gateway resource "azurerm_public_ip" "nat" { - count = var.zones - name = "${var.prefix}-nat-gw-ip-${count.index + 1}" + count = var.vnet_zones + name = "${var.context_id}-nat-gw-ip-${count.index + 1}" location = azurerm_resource_group.main.location resource_group_name = azurerm_resource_group.main.name allocation_method = "Static" @@ -88,8 +96,8 @@ resource "azurerm_public_ip" "nat" { # NAT Gateway resource "azurerm_nat_gateway" "main" { - count = var.zones - name = "${var.prefix}-nat-gw-${count.index + 1}" + count = var.vnet_zones + name = "${var.context_id}-nat-gw-${count.index + 1}" location = azurerm_resource_group.main.location resource_group_name = azurerm_resource_group.main.name sku_name = "Standard" @@ -97,21 +105,21 @@ resource "azurerm_nat_gateway" "main" { # Associate public IP with NAT Gateway resource "azurerm_nat_gateway_public_ip_association" "main" { - count = var.zones + count = var.vnet_zones nat_gateway_id = azurerm_nat_gateway.main[count.index].id public_ip_address_id = azurerm_public_ip.nat[count.index].id } # Associate NAT Gateway with private subnet resource "azurerm_subnet_nat_gateway_association" "private" { - count = var.zones + count = var.vnet_zones subnet_id = azurerm_subnet.private[count.index].id nat_gateway_id = azurerm_nat_gateway.main[count.index].id } # Associate NAT Gateway with data subnet resource "azurerm_subnet_nat_gateway_association" "data" { - count = var.zones + count = var.vnet_zones subnet_id = azurerm_subnet.data[count.index].id nat_gateway_id = azurerm_nat_gateway.main[count.index].id } diff --git a/terraform/network/azure-vpc/variables.tf b/terraform/network/azure-vpc/variables.tf index c82361de..39ad62e6 100644 --- a/terraform/network/azure-vpc/variables.tf +++ b/terraform/network/azure-vpc/variables.tf @@ -1,31 +1,37 @@ # Variables -variable "prefix" { - description = "Prefix for the resources" - type = string - default = "windsor" -} - variable "region" { description = "Region for the resources" type = string default = "eastus" } -variable "zones" { +variable "resource_group_name" { + description = "Name of the resource group" + type = string + default = null +} + +variable "vnet_name" { + description = "Name of the VNET" + type = string + default = null +} + +variable "vnet_zones" { description = "Number of availability zones to create" type = number default = 1 } -variable "vpc_cidr" { - description = "CIDR block for VPC" +variable "vnet_cidr" { + description = "CIDR block for VNET" type = string default = "10.20.0.0/16" } -variable "vpc_subnets" { - description = "Subnets to create in the VPC" +variable "vnet_subnets" { + description = "Subnets to create in the VNET" type = map(list(string)) # example: { # public = ["10.20.1.0/24", "10.20.2.0/24", "10.20.3.0/24"] @@ -39,7 +45,8 @@ variable "vpc_subnets" { } } -variable "azure_subscription_id" { +variable "context_id" { + description = "Context ID for the resources" type = string - description = "Subscription ID for the AKS cluster" + default = null } From f9619f7b27947825bba8308ed37c2e67c1b56a25 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Tue, 6 May 2025 17:43:09 +0200 Subject: [PATCH 15/18] Some more refactoring --- terraform/cluster/azure-aks/main.tf | 59 ++++++------- terraform/cluster/azure-aks/variables.tf | 105 ++++++++++++++--------- 2 files changed, 96 insertions(+), 68 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index f88c38ed..e5dd6fa9 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -36,8 +36,8 @@ data "azurerm_client_config" "current" {} locals { kubeconfig_path = "${var.context_path}/.kube/config" - rg_name = var.resource_group_name == null ? "${var.context_id}-aks-rg" : var.resource_group_name - cluster_name = var.cluster_name == null ? "${var.context_id}-aks-cluster" : var.cluster_name + rg_name = var.resource_group_name == null ? "windsor-aks-rg-${var.context_id}" : var.resource_group_name + cluster_name = var.cluster_name == null ? "windsor-aks-cluster-${var.context_id}" : var.cluster_name } #----------------------------------------------------------------------------------------------------------------------- @@ -186,8 +186,8 @@ resource "azurerm_kubernetes_cluster" "main" { role_based_access_control_enabled = var.role_based_access_control_enabled automatic_upgrade_channel = var.automatic_upgrade_channel sku_tier = var.sku_tier - # checkov:skip=CKV_AZURE_6: This feature is in preview - # api_server_authorized_ip_ranges = var.api_server_authorized_ip_ranges + # checkov:skip=CKV_AZURE_6: This feature is in preview, we are using a public cluster for testing + # api_server_authorized_ip_ranges = [0.0.0.0/0] # checkov:skip=CKV_AZURE_115: We are using a public cluster for testing # private clusters are encouraged for production private_cluster_enabled = var.private_cluster_enabled @@ -202,33 +202,33 @@ resource "azurerm_kubernetes_cluster" "main" { } default_node_pool { - name = "system" - node_count = 1 - vm_size = "Standard_D2s_v3" + name = var.default_node_pool.name + node_count = var.default_node_pool.node_count + vm_size = var.default_node_pool.vm_size vnet_subnet_id = data.azurerm_subnet.private.id orchestrator_version = var.kubernetes_version only_critical_addons_enabled = true # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs - os_disk_type = var.os_disk_type - host_encryption_enabled = var.host_encryption_enabled - max_pods = var.max_pods + os_disk_type = var.default_node_pool.os_disk_type + host_encryption_enabled = var.default_node_pool.host_encryption_enabled + max_pods = var.default_node_pool.max_pods } auto_scaler_profile { - balance_similar_node_groups = var.auto_scaler_profile["balance_similar_node_groups"] - max_graceful_termination_sec = var.auto_scaler_profile["max_graceful_termination_sec"] - scale_down_delay_after_add = var.auto_scaler_profile["scale_down_delay_after_add"] - scale_down_delay_after_delete = var.auto_scaler_profile["scale_down_delay_after_delete"] - scale_down_delay_after_failure = var.auto_scaler_profile["scale_down_delay_after_failure"] - scan_interval = var.auto_scaler_profile["scan_interval"] - scale_down_unneeded = var.auto_scaler_profile["scale_down_unneeded"] - scale_down_unready = var.auto_scaler_profile["scale_down_unready"] - scale_down_utilization_threshold = var.auto_scaler_profile["scale_down_utilization_threshold"] + balance_similar_node_groups = var.auto_scaler_profile.balance_similar_node_groups + max_graceful_termination_sec = var.auto_scaler_profile.max_graceful_termination_sec + scale_down_delay_after_add = var.auto_scaler_profile.scale_down_delay_after_add + scale_down_delay_after_delete = var.auto_scaler_profile.scale_down_delay_after_delete + scale_down_delay_after_failure = var.auto_scaler_profile.scale_down_delay_after_failure + scan_interval = var.auto_scaler_profile.scan_interval + scale_down_unneeded = var.auto_scaler_profile.scale_down_unneeded + scale_down_unready = var.auto_scaler_profile.scale_down_unready + scale_down_utilization_threshold = var.auto_scaler_profile.scale_down_utilization_threshold } workload_autoscaler_profile { - keda_enabled = var.workload_autoscaler_profile["keda_enabled"] - vertical_pod_autoscaler_enabled = var.workload_autoscaler_profile["vertical_pod_autoscaler_enabled"] + keda_enabled = var.workload_autoscaler_profile.keda_enabled + vertical_pod_autoscaler_enabled = var.workload_autoscaler_profile.vertical_pod_autoscaler_enabled } network_profile { @@ -252,19 +252,20 @@ resource "azurerm_kubernetes_cluster" "main" { } resource "azurerm_kubernetes_cluster_node_pool" "autoscaled" { - name = "autoscaled" + count = var.autoscaled_node_pool.enabled ? 1 : 0 + name = var.autoscaled_node_pool.name kubernetes_cluster_id = azurerm_kubernetes_cluster.main.id - vm_size = "Standard_D2s_v3" - mode = "User" + vm_size = var.autoscaled_node_pool.vm_size + mode = var.autoscaled_node_pool.mode auto_scaling_enabled = true - min_count = var.min_count - max_count = var.max_count + min_count = var.autoscaled_node_pool.min_count + max_count = var.autoscaled_node_pool.max_count vnet_subnet_id = data.azurerm_subnet.private.id orchestrator_version = var.kubernetes_version # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs - os_disk_type = var.os_disk_type - max_pods = var.max_pods - host_encryption_enabled = var.host_encryption_enabled + os_disk_type = var.autoscaled_node_pool.os_disk_type + max_pods = var.autoscaled_node_pool.max_pods + host_encryption_enabled = var.autoscaled_node_pool.host_encryption_enabled } resource "local_file" "kube_config" { diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index 8b41c3c1..6e2c7de6 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -50,6 +50,56 @@ variable "context_id" { default = null } +variable "default_node_pool" { + description = "Configuration for the default node pool" + type = object({ + name = string + vm_size = string + os_disk_type = string + max_pods = number + host_encryption_enabled = bool + min_count = number + max_count = number + node_count = number + }) + default = { + name = "system" + vm_size = "Standard_D2s_v3" + os_disk_type = "Managed" + max_pods = 30 + host_encryption_enabled = true + min_count = 1 + max_count = 3 + node_count = 1 + } +} + +variable "autoscaled_node_pool" { + description = "Configuration for the autoscaled node pool" + type = object({ + enabled = bool + name = string + vm_size = string + mode = string + os_disk_type = string + max_pods = number + host_encryption_enabled = bool + min_count = number + max_count = number + }) + default = { + enabled = true + name = "autoscaled" + vm_size = "Standard_D2s_v3" + mode = "User" + os_disk_type = "Managed" + max_pods = 30 + host_encryption_enabled = true + min_count = 1 + max_count = 3 + } +} + variable "role_based_access_control_enabled" { type = bool description = "Whether to enable role-based access control for the AKS cluster" @@ -57,7 +107,17 @@ variable "role_based_access_control_enabled" { } variable "auto_scaler_profile" { - type = map(string) + type = object({ + balance_similar_node_groups = bool + max_graceful_termination_sec = number + scale_down_delay_after_add = string + scale_down_delay_after_delete = string + scale_down_delay_after_failure = string + scan_interval = string + scale_down_unneeded = string + scale_down_unready = string + scale_down_utilization_threshold = string + }) description = "Configuration for the AKS cluster's auto-scaler" default = { balance_similar_node_groups = true @@ -67,13 +127,16 @@ variable "auto_scaler_profile" { scale_down_delay_after_failure = "3m" scan_interval = "10s" scale_down_unneeded = "10m" - scale_down_unready = "20m" + scale_down_unready = "20m" scale_down_utilization_threshold = "0.5" } } variable "workload_autoscaler_profile" { - type = map(string) + type = object({ + keda_enabled = bool + vertical_pod_autoscaler_enabled = bool + }) description = "Configuration for the AKS cluster's workload autoscaler" default = { keda_enabled = false @@ -93,12 +156,6 @@ variable "sku_tier" { default = "Standard" } -variable "api_server_authorized_ip_ranges" { - type = list(string) - description = "The API server authorized IP ranges for the AKS cluster" - default = ["0.0.0.0/0"] -} - variable "private_cluster_enabled" { type = bool description = "Whether to enable private cluster for the AKS cluster" @@ -117,36 +174,6 @@ variable "local_account_disabled" { default = false } -variable "os_disk_type" { - type = string - description = "The type of OS disk for the AKS cluster" - default = "Managed" -} - -variable "host_encryption_enabled" { - type = bool - description = "Whether to enable host encryption for the AKS cluster" - default = true -} - -variable "max_pods" { - type = number - description = "The maximum number of pods for the AKS cluster" - default = 50 -} - -variable "min_count" { - type = number - description = "The minimum number of nodes for the AKS cluster" - default = 1 -} - -variable "max_count" { - type = number - description = "The maximum number of nodes for the AKS cluster" - default = 3 -} - variable "public_network_access_enabled" { type = bool description = "Whether to enable public network access for the AKS cluster" From 5690797b14415bde3eb1c1dc1245c45d2422a0de Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Tue, 6 May 2025 17:43:23 +0200 Subject: [PATCH 16/18] Fix formatting --- terraform/cluster/azure-aks/variables.tf | 64 ++++++++++++------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index 6e2c7de6..b71ca6ae 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -53,50 +53,50 @@ variable "context_id" { variable "default_node_pool" { description = "Configuration for the default node pool" type = object({ - name = string - vm_size = string - os_disk_type = string - max_pods = number + name = string + vm_size = string + os_disk_type = string + max_pods = number host_encryption_enabled = bool - min_count = number - max_count = number - node_count = number + min_count = number + max_count = number + node_count = number }) default = { - name = "system" - vm_size = "Standard_D2s_v3" - os_disk_type = "Managed" - max_pods = 30 + name = "system" + vm_size = "Standard_D2s_v3" + os_disk_type = "Managed" + max_pods = 30 host_encryption_enabled = true - min_count = 1 - max_count = 3 - node_count = 1 + min_count = 1 + max_count = 3 + node_count = 1 } } variable "autoscaled_node_pool" { description = "Configuration for the autoscaled node pool" type = object({ - enabled = bool - name = string - vm_size = string - mode = string - os_disk_type = string - max_pods = number + enabled = bool + name = string + vm_size = string + mode = string + os_disk_type = string + max_pods = number host_encryption_enabled = bool - min_count = number - max_count = number + min_count = number + max_count = number }) default = { - enabled = true - name = "autoscaled" - vm_size = "Standard_D2s_v3" - mode = "User" - os_disk_type = "Managed" - max_pods = 30 + enabled = true + name = "autoscaled" + vm_size = "Standard_D2s_v3" + mode = "User" + os_disk_type = "Managed" + max_pods = 30 host_encryption_enabled = true - min_count = 1 - max_count = 3 + min_count = 1 + max_count = 3 } } @@ -115,7 +115,7 @@ variable "auto_scaler_profile" { scale_down_delay_after_failure = string scan_interval = string scale_down_unneeded = string - scale_down_unready = string + scale_down_unready = string scale_down_utilization_threshold = string }) description = "Configuration for the AKS cluster's auto-scaler" @@ -127,7 +127,7 @@ variable "auto_scaler_profile" { scale_down_delay_after_failure = "3m" scan_interval = "10s" scale_down_unneeded = "10m" - scale_down_unready = "20m" + scale_down_unready = "20m" scale_down_utilization_threshold = "0.5" } } From d945e385074eae47ee46d02176c08cf293778221 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Tue, 6 May 2025 18:20:42 +0200 Subject: [PATCH 17/18] more fixes --- terraform/cluster/azure-aks/main.tf | 9 +++++---- terraform/cluster/azure-aks/variables.tf | 6 ++++++ .../{azure-vpc => azure-vnet}/.terraform.lock.hcl | 0 terraform/network/{azure-vpc => azure-vnet}/main.tf | 0 terraform/network/{azure-vpc => azure-vnet}/outputs.tf | 0 terraform/network/{azure-vpc => azure-vnet}/variables.tf | 0 6 files changed, 11 insertions(+), 4 deletions(-) rename terraform/network/{azure-vpc => azure-vnet}/.terraform.lock.hcl (100%) rename terraform/network/{azure-vpc => azure-vnet}/main.tf (100%) rename terraform/network/{azure-vpc => azure-vnet}/outputs.tf (100%) rename terraform/network/{azure-vpc => azure-vnet}/variables.tf (100%) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index e5dd6fa9..201624e6 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -172,16 +172,17 @@ resource "azurerm_log_analytics_workspace" "aks_logs" { #----------------------------------------------------------------------------------------------------------------------- data "azurerm_subnet" "private" { + count = var.vnet_subnet_id == null ? 1 : 0 name = "${var.context_id}-private-1" resource_group_name = var.vnet_resource_group_name == null ? "windsor-vnet-rg-${var.context_id}" : var.vnet_resource_group_name virtual_network_name = var.vnet_name == null ? "windsor-vnet-${var.context_id}" : var.vnet_name } resource "azurerm_kubernetes_cluster" "main" { - name = var.cluster_name == null ? "aks-cluster-${var.context_id}" : var.cluster_name + name = local.cluster_name location = azurerm_resource_group.aks.location resource_group_name = azurerm_resource_group.aks.name - dns_prefix = var.cluster_name == null ? "aks-cluster-${var.context_id}" : var.cluster_name + dns_prefix = local.cluster_name kubernetes_version = var.kubernetes_version role_based_access_control_enabled = var.role_based_access_control_enabled automatic_upgrade_channel = var.automatic_upgrade_channel @@ -205,7 +206,7 @@ resource "azurerm_kubernetes_cluster" "main" { name = var.default_node_pool.name node_count = var.default_node_pool.node_count vm_size = var.default_node_pool.vm_size - vnet_subnet_id = data.azurerm_subnet.private.id + vnet_subnet_id = coalesce(var.vnet_subnet_id, try(data.azurerm_subnet.private[0].id, null)) orchestrator_version = var.kubernetes_version only_critical_addons_enabled = true # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs @@ -260,7 +261,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "autoscaled" { auto_scaling_enabled = true min_count = var.autoscaled_node_pool.min_count max_count = var.autoscaled_node_pool.max_count - vnet_subnet_id = data.azurerm_subnet.private.id + vnet_subnet_id = coalesce(var.vnet_subnet_id, try(data.azurerm_subnet.private[0].id, null)) orchestrator_version = var.kubernetes_version # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs os_disk_type = var.autoscaled_node_pool.os_disk_type diff --git a/terraform/cluster/azure-aks/variables.tf b/terraform/cluster/azure-aks/variables.tf index b71ca6ae..d269fe5f 100644 --- a/terraform/cluster/azure-aks/variables.tf +++ b/terraform/cluster/azure-aks/variables.tf @@ -20,6 +20,12 @@ variable "vnet_name" { default = null } +variable "vnet_subnet_id" { + description = "ID of the subnet" + type = string + default = null +} + variable "region" { description = "Region for the resources" type = string diff --git a/terraform/network/azure-vpc/.terraform.lock.hcl b/terraform/network/azure-vnet/.terraform.lock.hcl similarity index 100% rename from terraform/network/azure-vpc/.terraform.lock.hcl rename to terraform/network/azure-vnet/.terraform.lock.hcl diff --git a/terraform/network/azure-vpc/main.tf b/terraform/network/azure-vnet/main.tf similarity index 100% rename from terraform/network/azure-vpc/main.tf rename to terraform/network/azure-vnet/main.tf diff --git a/terraform/network/azure-vpc/outputs.tf b/terraform/network/azure-vnet/outputs.tf similarity index 100% rename from terraform/network/azure-vpc/outputs.tf rename to terraform/network/azure-vnet/outputs.tf diff --git a/terraform/network/azure-vpc/variables.tf b/terraform/network/azure-vnet/variables.tf similarity index 100% rename from terraform/network/azure-vpc/variables.tf rename to terraform/network/azure-vnet/variables.tf From cc6cd1e9177f52fc1907ca1efc55597862313bc8 Mon Sep 17 00:00:00 2001 From: Hernan Dominguez Date: Tue, 6 May 2025 18:46:32 +0200 Subject: [PATCH 18/18] Fixes --- terraform/cluster/azure-aks/main.tf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/terraform/cluster/azure-aks/main.tf b/terraform/cluster/azure-aks/main.tf index 201624e6..70fcbb81 100644 --- a/terraform/cluster/azure-aks/main.tf +++ b/terraform/cluster/azure-aks/main.tf @@ -172,7 +172,7 @@ resource "azurerm_log_analytics_workspace" "aks_logs" { #----------------------------------------------------------------------------------------------------------------------- data "azurerm_subnet" "private" { - count = var.vnet_subnet_id == null ? 1 : 0 + count = var.vnet_subnet_id == null ? 1 : 0 name = "${var.context_id}-private-1" resource_group_name = var.vnet_resource_group_name == null ? "windsor-vnet-rg-${var.context_id}" : var.vnet_resource_group_name virtual_network_name = var.vnet_name == null ? "windsor-vnet-${var.context_id}" : var.vnet_name @@ -212,7 +212,8 @@ resource "azurerm_kubernetes_cluster" "main" { # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs os_disk_type = var.default_node_pool.os_disk_type host_encryption_enabled = var.default_node_pool.host_encryption_enabled - max_pods = var.default_node_pool.max_pods + # checkov:skip=CKV_AZURE_168: This is set in the variable by default to 50 + max_pods = var.default_node_pool.max_pods } auto_scaler_profile { @@ -263,8 +264,9 @@ resource "azurerm_kubernetes_cluster_node_pool" "autoscaled" { max_count = var.autoscaled_node_pool.max_count vnet_subnet_id = coalesce(var.vnet_subnet_id, try(data.azurerm_subnet.private[0].id, null)) orchestrator_version = var.kubernetes_version - # checkov:skip=CKV_AZURE_226: we are using the managed disk type to reduce costs - os_disk_type = var.autoscaled_node_pool.os_disk_type + # checkov:skip=CKV_AZURE_226: We are using the managed disk type to reduce costs + os_disk_type = var.autoscaled_node_pool.os_disk_type + # checkov:skip=CKV_AZURE_168: This is set in the variable by default to 50 max_pods = var.autoscaled_node_pool.max_pods host_encryption_enabled = var.autoscaled_node_pool.host_encryption_enabled }