From e7caa0db8a2287e78a4eb06cffa52fe4e0f13294 Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Thu, 1 May 2025 18:19:29 +0530 Subject: [PATCH 01/13] tf for vault config in azure --- .../dz-control-plane-deps/values/vault.yaml | 24 +++ terraform/examples/azure/base-cluster/main.tf | 153 ++++++++++++++++++ .../examples/azure/base-cluster/outputs.tf | 25 +++ .../azure/base-cluster/terraform.tfvars | 34 ++++ .../examples/azure/base-cluster/variables.tf | 114 +++++++++++++ .../examples/azure/cluster-extensions/main.tf | 51 ++++++ .../modules/azure/cluster_extensions/main.tf | 69 ++++++++ .../azure/cluster_extensions/outputs.tf | 0 .../azure/cluster_extensions/variables.tf | 15 ++ 9 files changed, 485 insertions(+) create mode 100644 terraform/examples/azure/base-cluster/main.tf create mode 100644 terraform/examples/azure/base-cluster/outputs.tf create mode 100644 terraform/examples/azure/base-cluster/terraform.tfvars create mode 100644 terraform/examples/azure/base-cluster/variables.tf create mode 100644 terraform/examples/azure/cluster-extensions/main.tf create mode 100644 terraform/modules/azure/cluster_extensions/main.tf create mode 100644 terraform/modules/azure/cluster_extensions/outputs.tf create mode 100644 terraform/modules/azure/cluster_extensions/variables.tf diff --git a/charts/dz-control-plane-deps/values/vault.yaml b/charts/dz-control-plane-deps/values/vault.yaml index f86044df..62e2c0f6 100644 --- a/charts/dz-control-plane-deps/values/vault.yaml +++ b/charts/dz-control-plane-deps/values/vault.yaml @@ -19,6 +19,21 @@ server: dataStorage: enabled: true + # The following is an example of how to configure the Vault server to use Azure Key Vault for auto-unsealing. + # Before using this configuration, you need to create a secret in the Kubernetes cluster that contains the Azure Key Vault credentials: + # kubectl create secret generic vault-azure-creds --from-literal=AZURE_TENANT_ID= --from-literal=AZURE_CLIENT_ID= --from-literal=AZURE_CLIENT_SECRET= -n devzero + + # extraSecretEnvironmentVars: + # - envName: AZURE_CLIENT_ID + # secretName: vault-azure-creds + # secretKey: AZURE_CLIENT_ID + # - envName: AZURE_CLIENT_SECRET + # secretName: vault-azure-creds + # secretKey: AZURE_CLIENT_SECRET + # - envName: AZURE_TENANT_ID + # secretName: vault-azure-creds + # secretKey: AZURE_TENANT_ID + # Disable vault anti-affinity that requires one pod on each node. This allows vault to run on a single node cluster. affinity: "" @@ -71,6 +86,15 @@ server: # key_ring = "GCP_KEY_RING" # crypto_key = "GCP_CRYPTO_KEY" # } + + # This is used to configure the Vault server to use Azure Key Vault for auto-unsealing. + # seal "azurekeyvault" { + # tenant_id = "" + # client_id = "" + # client_secret = "" + # vault_name = "" + # key_name = "" + # } ingress: enabled: true diff --git a/terraform/examples/azure/base-cluster/main.tf b/terraform/examples/azure/base-cluster/main.tf new file mode 100644 index 00000000..d2913cf4 --- /dev/null +++ b/terraform/examples/azure/base-cluster/main.tf @@ -0,0 +1,153 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.106.1, < 4.0.0" + } + } +} + +provider "azurerm" { + features {} +} + +provider "azuread" {} + +locals { + public_subnet_names = ["public-subnet-1"] + private_subnet_names = ["private-subnet-1"] + + calculated_public_subnets_cidrs = ["10.240.1.0/24"] + calculated_private_subnets_cidrs = ["10.240.101.0/24"] +} + +data "azurerm_client_config" "current" {} + +################################################################################ +# VNET (updated to Azure/network/azurerm v5.3.0 with extended support) +################################################################################ + +module "vnet" { + source = "Azure/network/azurerm" + version = "5.3.0" + + resource_group_name = var.resource_group_name + use_for_each = false + + vnet_name = "${var.cluster_name}-vnet" + address_space = var.cidr + + subnet_names = concat(local.public_subnet_names, local.private_subnet_names) + subnet_prefixes = concat( + local.calculated_public_subnets_cidrs, + local.calculated_private_subnets_cidrs + ) + + subnet_enforce_private_link_endpoint_network_policies = { + for s in concat(local.public_subnet_names, local.private_subnet_names) : s => false + } + + tags = var.tags +} + + +################################################################################ +# AKS CLUSTER (updated to valid parameters for v9.4.1 module) +################################################################################ + +module "aks" { + source = "Azure/aks/azurerm" + version = "9.4.1" + + resource_group_name = var.resource_group_name + location = var.location + cluster_name = var.cluster_name + kubernetes_version = var.cluster_version + + vnet_subnet_id = module.vnet.vnet_subnets[0] + network_plugin = "azure" + network_policy = "azure" + private_cluster_enabled = var.enable_private_cluster + api_server_authorized_ip_ranges = [] + + identity_type = "SystemAssigned" + + rbac_aad = var.enable_rbac + rbac_aad_managed = var.enable_rbac + rbac_aad_admin_group_object_ids = var.enable_rbac ? var.admin_group_object_ids : null + role_based_access_control_enabled = var.enable_rbac + + log_analytics_workspace_enabled = true + log_retention_in_days = 30 + cluster_log_analytics_workspace_name = var.cluster_log_analytics_workspace_name + prefix = var.prefix + + key_vault_secrets_provider_enabled = true + secret_rotation_enabled = true + secret_rotation_interval = "2m" + + agents_pool_name = "systemnp" + agents_size = var.instance_type + enable_auto_scaling = var.enable_cluster_autoscaler + agents_min_count = var.enable_cluster_autoscaler ? var.min_size : null + agents_max_count = var.enable_cluster_autoscaler ? var.max_size : null + agents_count = var.enable_cluster_autoscaler ? null : var.node_count + temporary_name_for_rotation = "rotation" + agents_max_pods = 100 + + tags = var.tags +} + +################################################################################ +# Azure Key Vault for Vault Auto-Unseal +################################################################################ + +resource "azurerm_key_vault" "vault_auto_unseal" { + count = var.create_vault_auto_unseal_key ? 1 : 0 + + name = "${var.cluster_name}-vault-kv" + location = var.location + resource_group_name = var.resource_group_name + tenant_id = data.azurerm_client_config.current.tenant_id + sku_name = "standard" + purge_protection_enabled = true + soft_delete_retention_days = 10 + enable_rbac_authorization = true +} + +resource "azurerm_key_vault_key" "vault_auto_unseal" { + count = var.create_vault_auto_unseal_key ? 1 : 0 + + name = "${var.cluster_name}-auto-unseal" + key_vault_id = azurerm_key_vault.vault_auto_unseal[0].id + key_type = "RSA" + key_size = 2048 + key_opts = ["decrypt", "encrypt", "sign", "verify", "wrapKey", "unwrapKey"] + + depends_on = [azurerm_key_vault.vault_auto_unseal] +} + +resource "azurerm_role_assignment" "vault_key_usage" { + count = var.create_vault_auto_unseal_key ? 1 : 0 + + principal_id = azuread_service_principal.vault[0].object_id + role_definition_name = "Key Vault Crypto User" + scope = azurerm_key_vault.vault_auto_unseal[0].id +} + +resource "azuread_application" "vault" { + count = var.create_vault_auto_unseal_key ? 1 : 0 + display_name = "${var.cluster_name}-vault-app" +} + +resource "azuread_service_principal" "vault" { + count = var.create_vault_auto_unseal_key ? 1 : 0 + client_id = azuread_application.vault[0].client_id +} + +resource "azuread_application_password" "vault" { + count = var.create_vault_auto_unseal_key ? 1 : 0 + application_id = azuread_application.vault[0].id + display_name = "Vault SP Password" + end_date_relative = "8760h" +} \ No newline at end of file diff --git a/terraform/examples/azure/base-cluster/outputs.tf b/terraform/examples/azure/base-cluster/outputs.tf new file mode 100644 index 00000000..e44bd8d6 --- /dev/null +++ b/terraform/examples/azure/base-cluster/outputs.tf @@ -0,0 +1,25 @@ +output "vault_sp_client_id" { + value = var.create_vault_auto_unseal_key ? azuread_application.vault[0].client_id : null + description = "The client ID for Vault's Azure Key Vault integration." +} + +output "vault_sp_client_secret" { + value = var.create_vault_auto_unseal_key ? azuread_application_password.vault[0].value : null + description = "The client secret for Vault's Azure Key Vault integration." + sensitive = true +} + +output "vault_tenant_id" { + value = var.create_vault_auto_unseal_key ? data.azurerm_client_config.current.tenant_id : null + description = "The Azure Tenant ID." +} + +output "vault_key_name" { + value = var.create_vault_auto_unseal_key ? azurerm_key_vault_key.vault_auto_unseal[0].name : null + description = "The name of the Key Vault key." +} + +output "vault_keyvault_name" { + value = var.create_vault_auto_unseal_key ? azurerm_key_vault.vault_auto_unseal[0].name : null + description = "The name of the Key Vault instance." +} diff --git a/terraform/examples/azure/base-cluster/terraform.tfvars b/terraform/examples/azure/base-cluster/terraform.tfvars new file mode 100644 index 00000000..2cab4bbc --- /dev/null +++ b/terraform/examples/azure/base-cluster/terraform.tfvars @@ -0,0 +1,34 @@ +resource_group_name = "dev-test" +location = "eastus" +cluster_name = "devzero-aks" +cluster_version = "1.30" +cidr = "10.240.0.0/16" + +cluster_endpoint_public_access = true +enable_private_cluster = false +enable_rbac = false +admin_group_object_ids = [ + "00000000-0000-0000-0000-000000000000" +] + +instance_type = "Standard_D8s_v3" +enable_cluster_autoscaler = false +node_count = 2 +min_size = 1 +max_size = 3 + +enable_nat_gateway = true +single_nat_gateway = true + +# Optional tagging + +tags = { + environment = "dev" + project = "devzero" + owner = "garvit" +} + +cluster_log_analytics_workspace_name = "devzero-logs" +prefix = "devzero" + +create_vault_auto_unseal_key = true \ No newline at end of file diff --git a/terraform/examples/azure/base-cluster/variables.tf b/terraform/examples/azure/base-cluster/variables.tf new file mode 100644 index 00000000..5ca14990 --- /dev/null +++ b/terraform/examples/azure/base-cluster/variables.tf @@ -0,0 +1,114 @@ +variable "resource_group_name" { + description = "Name of the resource group in which resources will be created" + type = string +} + +variable "prefix" {} + +variable "location" { + description = "Azure region where the resources will be deployed" + type = string +} + +variable "cluster_name" { + description = "Name of the AKS cluster" + type = string +} + +variable "cluster_version" { + description = "Kubernetes version to use for the AKS cluster" + type = string +} + +variable "cidr" { + description = "CIDR block for the virtual network" + type = string +} + +variable "subnet_id" { + description = "Subnet ID used by AKS cluster" + type = string +} + +variable "cluster_endpoint_public_access" { + description = "Whether the AKS API server should be publicly accessible" + type = bool +} + +variable "cluster_endpoint_public_access_cidrs" { + description = "List of CIDRs allowed to access the API server if public access is enabled" + type = list(string) + default = [] +} + +variable "enable_rbac" { + type = bool +} + +variable "enable_private_cluster" { + type = bool +} + +variable "admin_group_object_ids" { + description = "List of AAD group object IDs with admin access to the cluster" + type = list(string) +} + +variable "instance_type" { + description = "VM size for the default node pool" + type = string +} + +variable "enable_cluster_autoscaler" { + type = bool +} + + +variable "node_count" { + type = number +} + +variable "min_size" { + description = "Minimum number of nodes in the default node pool" + type = number +} + +variable "max_size" { + description = "Maximum number of nodes in the default node pool" + type = number +} + +variable "desired_size" { + description = "Desired number of nodes in the default node pool (ignored when autoscaling is enabled)" + type = number + default = null +} + +variable "enable_nat_gateway" { + description = "Whether to enable NAT Gateway" + type = bool + default = false +} + +variable "single_nat_gateway" { + description = "Whether to use a single NAT gateway for all subnets" + type = bool + default = false +} + +variable "cluster_log_analytics_workspace_name" {} + +variable "tags" { + description = "Tags to apply to all resources" + type = map(string) + default = {} +} + +################################################################################ +# Vault +################################################################################ +variable "create_vault_auto_unseal_key" { + description = "Whether or not to create a KMS key for Vault auto unseal" + type = bool + default = false +} \ No newline at end of file diff --git a/terraform/examples/azure/cluster-extensions/main.tf b/terraform/examples/azure/cluster-extensions/main.tf new file mode 100644 index 00000000..e1a4f73a --- /dev/null +++ b/terraform/examples/azure/cluster-extensions/main.tf @@ -0,0 +1,51 @@ +################################################################################ +# Providers +################################################################################ + +provider "azurerm" { + features {} +} + +provider "azuread" {} + +data "azurerm_client_config" "current" {} + +provider "kubernetes" { + host = data.azurerm_kubernetes_cluster.this.kube_config[0].host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate) +} + +provider "helm" { + kubernetes { + host = data.azurerm_kubernetes_cluster.this.kube_config[0].host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate) + } +} + +################################################################################ +# AKS Cluster Data +################################################################################ + +data "azurerm_kubernetes_cluster" "this" { + name = var.cluster_name + resource_group_name = var.resource_group_name +} + +################################################################################ +# Cluster Extensions Module (Azure parity) +################################################################################ + +module "cluster_extensions" { + source = "../../../modules/azure/cluster_extensions" + + cluster_name = var.cluster_name + resource_group_name = var.resource_group_name + location = var.location + enable_external_secrets = var.enable_external_secrets + enable_azure_files = var.enable_azure_files + tags = var.tags +} diff --git a/terraform/modules/azure/cluster_extensions/main.tf b/terraform/modules/azure/cluster_extensions/main.tf new file mode 100644 index 00000000..6b7f85d1 --- /dev/null +++ b/terraform/modules/azure/cluster_extensions/main.tf @@ -0,0 +1,69 @@ +################################################################################ +# AKS Cluster Details +################################################################################ + +data "azurerm_kubernetes_cluster" "this" { + name = var.cluster_name + resource_group_name = var.resource_group_name +} + +provider "kubernetes" { + host = data.azurerm_kubernetes_cluster.this.kube_config[0].host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate) +} + +provider "helm" { + kubernetes { + host = data.azurerm_kubernetes_cluster.this.kube_config[0].host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate) + } +} + +################################################################################ +# External Secrets Operator +################################################################################ + +resource "helm_release" "external_secrets" { + count = var.enable_external_secrets ? 1 : 0 + + name = "external-secrets" + namespace = "external-secrets" + repository = "https://charts.external-secrets.io" + chart = "external-secrets" + version = "0.9.13" + + create_namespace = true + + set { + name = "installCRDs" + value = "true" + } +} + +################################################################################ +# Azure Files CSI StorageClass +################################################################################ + +resource "kubernetes_storage_class" "azurefiles_csi" { + count = var.enable_azure_files ? 1 : 0 + + metadata { + name = "azurefiles-csi" + annotations = { + "storageclass.kubernetes.io/is-default-class" = "true" + } + } + + storage_provisioner = "file.csi.azure.com" + reclaim_policy = "Delete" + volume_binding_mode = "WaitForFirstConsumer" + allow_volume_expansion = true + + parameters = { + skuName = "Standard_LRS" + } +} \ No newline at end of file diff --git a/terraform/modules/azure/cluster_extensions/outputs.tf b/terraform/modules/azure/cluster_extensions/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/terraform/modules/azure/cluster_extensions/variables.tf b/terraform/modules/azure/cluster_extensions/variables.tf new file mode 100644 index 00000000..a6ed7385 --- /dev/null +++ b/terraform/modules/azure/cluster_extensions/variables.tf @@ -0,0 +1,15 @@ +variable "cluster_name" {} +variable "resource_group_name" {} +variable "location" {} +variable "enable_external_secrets" { + type = bool + default = false +} +variable "enable_azure_files" { + type = bool + default = false +} +variable "tags" { + type = map(string) + default = {} +} \ No newline at end of file From 1b2a5bc70c167a044ac60cb09737a5b83f0eb863 Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Tue, 6 May 2025 22:34:13 +0530 Subject: [PATCH 02/13] derp and cluster extensions for azure --- terraform/examples/azure/README.md | 117 ++++++++++++++++ .../azure/base-cluster/daemonset.yaml | 107 +++++++++++++++ .../examples/azure/base-cluster/kata-sa.yaml | 42 ++++++ terraform/examples/azure/base-cluster/main.tf | 27 +++- .../examples/azure/base-cluster/outputs.tf | 20 +++ .../azure/base-cluster/terraform.tfvars | 12 +- .../examples/azure/base-cluster/variables.tf | 24 ++-- .../examples/azure/cluster-extensions/main.tf | 9 +- .../azure/cluster-extensions/terraform.tfvars | 13 ++ .../azure/cluster-extensions/variables.tf | 37 +++++ terraform/examples/gcp/README.md | 20 +-- .../modules/azure/cluster_extensions/main.tf | 51 ++++++- terraform/modules/azure/derp/derp-init.tpl | 54 ++++++++ terraform/modules/azure/derp/main.tf | 129 ++++++++++++++++++ terraform/modules/azure/derp/variables.tf | 50 +++++++ 15 files changed, 669 insertions(+), 43 deletions(-) create mode 100644 terraform/examples/azure/README.md create mode 100644 terraform/examples/azure/base-cluster/daemonset.yaml create mode 100644 terraform/examples/azure/base-cluster/kata-sa.yaml create mode 100644 terraform/examples/azure/cluster-extensions/terraform.tfvars create mode 100644 terraform/examples/azure/cluster-extensions/variables.tf create mode 100644 terraform/modules/azure/derp/derp-init.tpl create mode 100644 terraform/modules/azure/derp/main.tf create mode 100644 terraform/modules/azure/derp/variables.tf diff --git a/terraform/examples/azure/README.md b/terraform/examples/azure/README.md new file mode 100644 index 00000000..b3198e65 --- /dev/null +++ b/terraform/examples/azure/README.md @@ -0,0 +1,117 @@ +# DevZero Self-Hosted - Terraform Setup - Azure + +This document provides a step-by-step guide for setting up the infrastructure required to self-host the DevZero Control Plane and Data Plane using Terraform. The infrastructure can be deployed on cloud platforms like Azure. + +## Pre-reading + +For readers experienced with Terraform deployments at their companies, we have some examples under [./examples](./examples/) that you can reference to see how to run a full DevZero deployment. +If you have your own terraform environment and want to reuse our modules, you can refer to the [./modules](./modules/) directory to use whichever components you need. + +## Overview + +The `terraform/` directory contains Infrastructure as Code (IaC) configurations that automate the provisioning of essential cloud resources such as VNet, AKS clusters, load balancers, and VPNs. + +## Prerequisites + +### Tools Required +- [Terraform](https://www.terraform.io/) (for managing infrastructure as code) +- [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) (for interacting with Azure resources) +- Access credentials for your Azure Account + +### Permissions Required +Ensure your IAM user or service account has sufficient permissions to create resources like VNet, subnets, AKS clusters, VMs, and Key Vault. + +## Infrastructure Setup Guide + +### 1. Clone the Repository + +```bash +git clone https://github.com/devzero-inc/self-hosted.git +``` + +### 2. Navigate to the Base Cluster Directory + +```bash +cd self-hosted/terraform/examples/azure/base-cluster +``` + +### 3. Configure Terraform Variables + +Update `terraform.tfvars`. + +#### Cluster Endpoint Access +- Set `cluster_endpoint_public_access = true` to allow public access. +- Set it to `false` for private access. + +### 4. Initialise and Apply Terraform + +```bash +terraform init +terraform apply +``` + +- This will create Azure resources such as VNet, AKS, VM, Key Vault, etc. +- Copy the output values like subscription_id, resource_group_name, location, and cluster_name for the next steps. + +## Extending the Cluster + +### 5. Navigate to the Cluster Extensions Directory + +```bash +cd ../cluster-extensions +``` + +### 6. Update `terraform.tfvars` + +- Add the subscription_id, resource_group_name, location, and cluster_name from the previous step. + +### 7. Apply Terraform for Storage + +```bash +terraform init +terraform apply +``` + +- This will create StorageClasses, and Azure Files. + +## Post-Deployment Steps + +### 8. Update kubeconfig + +```bash +az aks get-credentials --resource-group --name +``` + +### 9. Install Kata in AKS Node + +```bash +kubectl apply -f kata-sa.yaml +kubectl apply -f daemonset.yaml +``` + +### 10. Add the Labels + +```bash +kubectl get nodes +kubectl label node kata-runtime=running +kubectl label node node-role.kubernetes.io/kata-devpod-node=1 +``` + +Or you can automatically label all your nodes like this: + +```bash +for NODE in $(kubectl get nodes -o name); do + kubectl label "$NODE" kata-runtime=running --overwrite + kubectl label "$NODE" node-role.kubernetes.io/kata-devpod-node=1 --overwrite +done +``` + +### 11. Install DevZero Self-Hosted + +Refer to the [Charts README](../charts/README.md) for further steps to deploy the Control Plane and Data Plane. + +## Troubleshooting + +- Verify cloud credentials and permissions. +- Check Terraform state files for resource management. +- Use `terraform plan` to preview changes before applying. diff --git a/terraform/examples/azure/base-cluster/daemonset.yaml b/terraform/examples/azure/base-cluster/daemonset.yaml new file mode 100644 index 00000000..1953569f --- /dev/null +++ b/terraform/examples/azure/base-cluster/daemonset.yaml @@ -0,0 +1,107 @@ +# This is a DaemonSet configuration for deploying Kata Containers on Kubernetes nodes. Find it here: https://github.com/kata-containers/kata-containers/blob/main/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kata-deploy + namespace: kube-system +spec: + selector: + matchLabels: + name: kata-deploy + template: + metadata: + labels: + name: kata-deploy + spec: + serviceAccountName: kata-deploy-sa + hostPID: true + containers: + - name: kube-kata + image: quay.io/kata-containers/kata-deploy:latest + imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"] + command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install"] + # NOTE: Please don't change the order of the environment variables below. + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DEBUG + value: "false" + - name: SHIMS + value: "cloud-hypervisor" + - name: DEFAULT_SHIM + value: "cloud-hypervisor" + - name: CREATE_RUNTIMECLASSES + value: "false" + - name: CREATE_DEFAULT_RUNTIMECLASS + value: "false" + - name: ALLOWED_HYPERVISOR_ANNOTATIONS + value: "" + - name: SNAPSHOTTER_HANDLER_MAPPING + value: "" + - name: AGENT_HTTPS_PROXY + value: "" + - name: AGENT_NO_PROXY + value: "" + - name: PULL_TYPE_MAPPING + value: "" + - name: INSTALLATION_PREFIX + value: "" + - name: MULTI_INSTALL_SUFFIX + value: "" + securityContext: + privileged: true + volumeMounts: + - name: crio-conf + mountPath: /etc/crio/ + - name: containerd-conf + mountPath: /etc/containerd/ + - name: host + mountPath: /host/ + - name: patch-containerd-handler + image: alpine:3.18 + securityContext: + privileged: true + volumeMounts: + - name: host + mountPath: /host + command: + - /bin/sh + - -c + - | + apk add --no-cache util-linux + + echo "Waiting for kata-cloud-hypervisor config block..." + while ! grep -q '\[plugins.*kata-cloud-hypervisor\]' /host/etc/containerd/config.toml; do + sleep 2 + done + + echo "Patching config.toml to rename kata-cloud-hypervisor → kata..." + sed -i 's/kata-cloud-hypervisor/kata/g' /host/etc/containerd/config.toml + + echo "Restarting containerd..." + nsenter --target 1 --mount --uts --ipc --net --pid systemctl restart containerd + + # exit to let the sidecar terminate + echo "Patch done. Exiting sidecar." + volumes: + - name: crio-conf + hostPath: + path: /etc/crio/ + - name: containerd-conf + hostPath: + path: /etc/containerd/ + - name: host + hostPath: + path: / + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate \ No newline at end of file diff --git a/terraform/examples/azure/base-cluster/kata-sa.yaml b/terraform/examples/azure/base-cluster/kata-sa.yaml new file mode 100644 index 00000000..6b0c1308 --- /dev/null +++ b/terraform/examples/azure/base-cluster/kata-sa.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kata-deploy-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole + + +metadata: + name: kata-deploy-role +rules: +- apiGroups: [""] + resources: ["serviceaccounts", "pods", "namespaces", "nodes"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles", "clusterrolebindings"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +- apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kata-deploy-rb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kata-deploy-role +subjects: +- kind: ServiceAccount + name: kata-deploy-sa + namespace: kube-system diff --git a/terraform/examples/azure/base-cluster/main.tf b/terraform/examples/azure/base-cluster/main.tf index d2913cf4..f9e5517d 100644 --- a/terraform/examples/azure/base-cluster/main.tf +++ b/terraform/examples/azure/base-cluster/main.tf @@ -63,6 +63,7 @@ module "aks" { location = var.location cluster_name = var.cluster_name kubernetes_version = var.cluster_version + prefix = var.cluster_name vnet_subnet_id = module.vnet.vnet_subnets[0] network_plugin = "azure" @@ -79,8 +80,7 @@ module "aks" { log_analytics_workspace_enabled = true log_retention_in_days = 30 - cluster_log_analytics_workspace_name = var.cluster_log_analytics_workspace_name - prefix = var.prefix + cluster_log_analytics_workspace_name = "${var.cluster_name}-logs" key_vault_secrets_provider_enabled = true secret_rotation_enabled = true @@ -98,6 +98,29 @@ module "aks" { tags = var.tags } +################################################################################ +# Custom DERP server in Azure +################################################################################ + +module "derp" { + source = "../../../modules/azure/derp" + + count = var.create_derp ? 1 : 0 + + name_prefix = var.cluster_name + location = var.location + resource_group_name = var.resource_group_name + subnet_id = module.vnet.vnet_subnets[0] + public_derp = var.public_derp + existing_ip = "" + firewall_rule_name = "" + ingress_cidr_blocks = ["0.0.0.0/0"] + hostname = "derp.devzero.net" + vm_size = "Standard_B2s" + volume_size = 30 + tags = var.tags +} + ################################################################################ # Azure Key Vault for Vault Auto-Unseal ################################################################################ diff --git a/terraform/examples/azure/base-cluster/outputs.tf b/terraform/examples/azure/base-cluster/outputs.tf index e44bd8d6..9cc725fa 100644 --- a/terraform/examples/azure/base-cluster/outputs.tf +++ b/terraform/examples/azure/base-cluster/outputs.tf @@ -1,3 +1,23 @@ +output "subscription_id" { + description = "Azure Subscription ID" + value = data.azurerm_client_config.current.subscription_id +} + +output "resource_group_name" { + description = "The name of the resource group used for the AKS cluster." + value = var.resource_group_name +} + +output "cluster_name" { + description = "The name of the AKS cluster." + value = var.cluster_name +} + +output "location" { + description = "The Azure region where the AKS cluster is deployed." + value = var.location +} + output "vault_sp_client_id" { value = var.create_vault_auto_unseal_key ? azuread_application.vault[0].client_id : null description = "The client ID for Vault's Azure Key Vault integration." diff --git a/terraform/examples/azure/base-cluster/terraform.tfvars b/terraform/examples/azure/base-cluster/terraform.tfvars index 2cab4bbc..06f052a0 100644 --- a/terraform/examples/azure/base-cluster/terraform.tfvars +++ b/terraform/examples/azure/base-cluster/terraform.tfvars @@ -1,6 +1,6 @@ resource_group_name = "dev-test" location = "eastus" -cluster_name = "devzero-aks" +cluster_name = "devzero" cluster_version = "1.30" cidr = "10.240.0.0/16" @@ -24,11 +24,11 @@ single_nat_gateway = true tags = { environment = "dev" - project = "devzero" - owner = "garvit" -} + owner = "devzero" + createdBy = "terraform" +} -cluster_log_analytics_workspace_name = "devzero-logs" -prefix = "devzero" +create_derp = true +public_derp = true create_vault_auto_unseal_key = true \ No newline at end of file diff --git a/terraform/examples/azure/base-cluster/variables.tf b/terraform/examples/azure/base-cluster/variables.tf index 5ca14990..254c90bf 100644 --- a/terraform/examples/azure/base-cluster/variables.tf +++ b/terraform/examples/azure/base-cluster/variables.tf @@ -3,8 +3,6 @@ variable "resource_group_name" { type = string } -variable "prefix" {} - variable "location" { description = "Azure region where the resources will be deployed" type = string @@ -25,11 +23,6 @@ variable "cidr" { type = string } -variable "subnet_id" { - description = "Subnet ID used by AKS cluster" - type = string -} - variable "cluster_endpoint_public_access" { description = "Whether the AKS API server should be publicly accessible" type = bool @@ -96,14 +89,27 @@ variable "single_nat_gateway" { default = false } -variable "cluster_log_analytics_workspace_name" {} - variable "tags" { description = "Tags to apply to all resources" type = map(string) default = {} } +################################################################################ +# Custom DERP server +################################################################################ +variable "create_derp" { + description = "Create custom DERP server" + type = bool + default = false +} + +variable "public_derp" { + type = bool + default = false + description = "Whether to make the DERP server public" +} + ################################################################################ # Vault ################################################################################ diff --git a/terraform/examples/azure/cluster-extensions/main.tf b/terraform/examples/azure/cluster-extensions/main.tf index e1a4f73a..06d20998 100644 --- a/terraform/examples/azure/cluster-extensions/main.tf +++ b/terraform/examples/azure/cluster-extensions/main.tf @@ -4,6 +4,7 @@ provider "azurerm" { features {} + subscription_id = var.subscription_id } provider "azuread" {} @@ -42,10 +43,10 @@ data "azurerm_kubernetes_cluster" "this" { module "cluster_extensions" { source = "../../../modules/azure/cluster_extensions" - cluster_name = var.cluster_name - resource_group_name = var.resource_group_name - location = var.location + cluster_name = var.cluster_name + resource_group_name = var.resource_group_name + location = var.location enable_external_secrets = var.enable_external_secrets enable_azure_files = var.enable_azure_files tags = var.tags -} +} \ No newline at end of file diff --git a/terraform/examples/azure/cluster-extensions/terraform.tfvars b/terraform/examples/azure/cluster-extensions/terraform.tfvars new file mode 100644 index 00000000..3003f9c2 --- /dev/null +++ b/terraform/examples/azure/cluster-extensions/terraform.tfvars @@ -0,0 +1,13 @@ +subscription_id = "a32b188c-43fd-4e28-8f67-c95649ab3119" +cluster_name = "devzero" +resource_group_name = "dev-test" +location = "eastus" + +enable_external_secrets = false +enable_azure_files = true + +tags = { + environment = "dev" + owner = "devzero" + createdBy = "terraform" +} diff --git a/terraform/examples/azure/cluster-extensions/variables.tf b/terraform/examples/azure/cluster-extensions/variables.tf new file mode 100644 index 00000000..93bf60a9 --- /dev/null +++ b/terraform/examples/azure/cluster-extensions/variables.tf @@ -0,0 +1,37 @@ +variable "subscription_id" { + type = string + description = "Azure Subscription ID" +} + +variable "cluster_name" { + description = "The name of the AKS cluster." + type = string +} + +variable "resource_group_name" { + description = "The resource group containing the AKS cluster." + type = string +} + +variable "location" { + description = "Azure region where resources will be created." + type = string +} + +variable "enable_external_secrets" { + description = "Whether to deploy the External Secrets Operator." + type = bool + default = false +} + +variable "enable_azure_files" { + description = "Whether to enable and set up the Azure Files CSI driver with default StorageClass." + type = bool + default = true +} + +variable "tags" { + description = "Tags to apply to all resources." + type = map(string) + default = {} +} diff --git a/terraform/examples/gcp/README.md b/terraform/examples/gcp/README.md index 1ecb7bdd..6cb55db1 100644 --- a/terraform/examples/gcp/README.md +++ b/terraform/examples/gcp/README.md @@ -9,7 +9,7 @@ If you have your own terraform environment and want to reuse our modules, you ca ## Overview -The `terraform/` directory contains Infrastructure as Code (IaC) configurations that automate the provisioning of essential cloud resources such as VPCs, EKS clusters, load balancers, and VPNs. +The `terraform/` directory contains Infrastructure as Code (IaC) configurations that automate the provisioning of essential cloud resources such as VPCs, GKE clusters, load balancers, and VPNs. ## Prerequisites @@ -37,14 +37,7 @@ cd self-hosted/terraform/examples/gcp/base-cluster ### 3. Configure Terraform Variables -#### Using an Existing VPC -- Open the `terraform.tfvars` file. -- Update the VPC ID and subnet IDs. -- Set `create_vpc = false` to prevent Terraform from creating a new VPC. - -#### Let Terraform Create a New VPC -- Skip the above step. -- Ensure `create_vpc = true` (default setting). +Update `terraform.tfvars`. #### Cluster Endpoint Access - Set `cluster_endpoint_public_access = true` to allow public access. @@ -117,15 +110,6 @@ done Refer to the [Charts README](../charts/README.md) for further steps to deploy the Control Plane and Data Plane. -### 12. Update Kata Runtime - -After DSH installation, delete the default kata runtimeclass and create a new one: - -```bash -kubectl delete runtimeclass kata -kubectl apply -f runtimeclass.yaml -``` - ## Troubleshooting - Verify cloud credentials and permissions. diff --git a/terraform/modules/azure/cluster_extensions/main.tf b/terraform/modules/azure/cluster_extensions/main.tf index 6b7f85d1..973be920 100644 --- a/terraform/modules/azure/cluster_extensions/main.tf +++ b/terraform/modules/azure/cluster_extensions/main.tf @@ -45,14 +45,54 @@ resource "helm_release" "external_secrets" { } ################################################################################ -# Azure Files CSI StorageClass +# Azure Files Backing Resources +################################################################################ + +resource "azurerm_storage_account" "azure_files" { + count = var.enable_azure_files ? 1 : 0 + + name = lower(replace("${var.cluster_name}files", "[^a-z0-9]", "")) + resource_group_name = var.resource_group_name + location = var.location + account_tier = "Standard" + account_replication_type = "LRS" + account_kind = "StorageV2" + + tags = var.tags +} + +resource "azurerm_storage_share" "efs_etcd_share" { + count = var.enable_azure_files ? 1 : 0 + name = "efs-etcd" + storage_account_name = azurerm_storage_account.azure_files[0].name + quota = 100 +} + +resource "kubernetes_secret" "azure_files_secret" { + count = var.enable_azure_files ? 1 : 0 + + metadata { + name = "azure-files-secret" + namespace = "default" + } + + data = { + azurestorageaccountname = azurerm_storage_account.azure_files[0].name + azurestorageaccountkey = azurerm_storage_account.azure_files[0].primary_access_key + } + + type = "Opaque" +} + +################################################################################ +# Azure Files CSI StorageClass (named as `efs-etcd`) ################################################################################ resource "kubernetes_storage_class" "azurefiles_csi" { count = var.enable_azure_files ? 1 : 0 metadata { - name = "azurefiles-csi" + name = "efs-etcd" annotations = { "storageclass.kubernetes.io/is-default-class" = "true" } @@ -64,6 +104,9 @@ resource "kubernetes_storage_class" "azurefiles_csi" { allow_volume_expansion = true parameters = { - skuName = "Standard_LRS" + skuName = "Standard_LRS" + secretName = "azure-files-secret" + secretNamespace = "default" + shareName = "efs-etcd" } -} \ No newline at end of file +} diff --git a/terraform/modules/azure/derp/derp-init.tpl b/terraform/modules/azure/derp/derp-init.tpl new file mode 100644 index 00000000..1ad67660 --- /dev/null +++ b/terraform/modules/azure/derp/derp-init.tpl @@ -0,0 +1,54 @@ +#!/bin/bash +set -euxo pipefail +exec > /var/log/derp-init.log 2>&1 + +# Install dependencies +apt-get update +apt-get install -y curl git gcc make openssl certbot + +# Install Go (use official binary) +wget -q https://go.dev/dl/go1.21.6.linux-amd64.tar.gz +rm -rf /usr/local/go +tar -C /usr/local -xzf go1.21.6.linux-amd64.tar.gz + +# Set up Go env +export PATH=$PATH:/usr/local/go/bin +export GOPATH=/root/go +export PATH=$PATH:$GOPATH/bin +export HOME=/root +export GOCACHE=$HOME/.cache/go-build + +# Install derper +go install tailscale.com/cmd/derper@main +cp $GOPATH/bin/derper /usr/bin/ + +%{ if !public_derp } +export DERP_PRIVATE_IP=$(curl -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip) +mkdir -p /etc/derper +openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \ + -keyout /etc/derper/$DERP_PRIVATE_IP.key \ + -out /etc/derper/$DERP_PRIVATE_IP.crt \ + -subj "/CN=$DERP_PRIVATE_IP" \ + -addext "subjectAltName=IP:$DERP_PRIVATE_IP" +%{ endif } + +# Create systemd service +cat < /etc/systemd/system/derper.service +[Unit] +Description=Devzero DERP Server +After=network-online.target +Wants=network-online.target + +[Service] +User=root +ExecStart=/usr/bin/derper %{ if public_derp }-hostname ${hostname}%{ else }-hostname \$DERP_PRIVATE_IP -certmode manual -certdir /etc/derper/ %{ endif } +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOT + +systemctl daemon-reload +systemctl enable derper +systemctl start derper diff --git a/terraform/modules/azure/derp/main.tf b/terraform/modules/azure/derp/main.tf new file mode 100644 index 00000000..72b8d0b3 --- /dev/null +++ b/terraform/modules/azure/derp/main.tf @@ -0,0 +1,129 @@ +locals { + create_firewall_rule = var.firewall_rule_name == "" + create_static_ip = var.public_derp && var.existing_ip == "" + + ip_address = local.create_static_ip ? azurerm_public_ip.derp_ip[0].ip_address : var.existing_ip +} + +# Static Public IP (only if needed) +resource "azurerm_public_ip" "derp_ip" { + count = local.create_static_ip ? 1 : 0 + name = "${var.name_prefix}-derp-ip" + location = var.location + resource_group_name = var.resource_group_name + allocation_method = "Static" + sku = "Standard" +} + +# Network Security Group +resource "azurerm_network_security_group" "derp_nsg" { + count = local.create_firewall_rule ? 1 : 0 + name = "${var.name_prefix}-derp-nsg" + location = var.location + resource_group_name = var.resource_group_name + + security_rule { + name = "Allow-UDP-3478" + priority = 1001 + direction = "Inbound" + access = "Allow" + protocol = "Udp" + source_port_range = "*" + destination_port_range = "3478" + source_address_prefixes = var.ingress_cidr_blocks + destination_address_prefix = "*" + } + + security_rule { + name = "Allow-TCP-443" + priority = 1002 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefixes = var.ingress_cidr_blocks + destination_address_prefix = "*" + } + + security_rule { + name = "Allow-SSH" + priority = 1003 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefixes = var.ingress_cidr_blocks + destination_address_prefix = "*" + } +} + +# Network Interface +resource "azurerm_network_interface" "derp_nic" { + name = "${var.name_prefix}-nic" + location = var.location + resource_group_name = var.resource_group_name + + ip_configuration { + name = "internal" + subnet_id = var.subnet_id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = local.create_static_ip ? azurerm_public_ip.derp_ip[0].id : null + } + + tags = var.tags +} + +# NSG Association +resource "azurerm_network_interface_security_group_association" "derp_nsg_assoc" { + count = local.create_firewall_rule ? 1 : 0 + network_interface_id = azurerm_network_interface.derp_nic.id + network_security_group_id = azurerm_network_security_group.derp_nsg[0].id +} + +# SSH Key +resource "tls_private_key" "ssh_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +# VM +resource "azurerm_linux_virtual_machine" "derp_vm" { + name = "${var.name_prefix}-derp-vm" + resource_group_name = var.resource_group_name + location = var.location + size = var.vm_size + admin_username = "ubuntu" + network_interface_ids = [ + azurerm_network_interface.derp_nic.id, + ] + + disable_password_authentication = true + + admin_ssh_key { + username = "ubuntu" + public_key = tls_private_key.ssh_key.public_key_openssh + } + + source_image_reference { + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts" + version = "latest" + } + + os_disk { + caching = "ReadWrite" + storage_account_type = "Premium_LRS" + disk_size_gb = var.volume_size + name = "${var.name_prefix}-osdisk" + } + + custom_data = base64encode(templatefile("${path.module}/derp-init.tpl", { + hostname = var.hostname + public_derp = var.public_derp + })) + + tags = var.tags +} diff --git a/terraform/modules/azure/derp/variables.tf b/terraform/modules/azure/derp/variables.tf new file mode 100644 index 00000000..ffb89976 --- /dev/null +++ b/terraform/modules/azure/derp/variables.tf @@ -0,0 +1,50 @@ +variable "resource_group_name" { + type = string +} + +variable "location" { + type = string +} + +variable "subnet_id" { + type = string + description = "Use subnet ID from the shared VNet module" +} + +variable "firewall_rule_name" { + type = string + default = "" +} + +variable "public_derp" { + type = bool +} + +variable "existing_ip" { + type = string + default = "" +} + +variable "name_prefix" { + type = string +} + +variable "ingress_cidr_blocks" { + type = list(string) +} + +variable "hostname" { + type = string +} + +variable "tags" { + type = map(string) +} + +variable "vm_size" { + type = string +} + +variable "volume_size" { + type = number +} From be9aad0d97982aac89584bc2618e254b420f5655 Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Wed, 14 May 2025 18:04:37 +0530 Subject: [PATCH 03/13] tf config for vpn and nat gateway for dsh in azure --- terraform/examples/azure/base-cluster/main.tf | 113 ++++++++-- .../examples/azure/base-cluster/outputs.tf | 12 +- .../azure/base-cluster/root-devzero.ovpn | 89 ++++++++ .../azure/base-cluster/terraform.tfvars | 8 +- .../examples/azure/base-cluster/variables.tf | 19 +- .../azure/cluster_extensions/outputs.tf | 0 terraform/modules/azure/vpn/main.tf | 210 ++++++++++++++++++ terraform/modules/azure/vpn/variables.tf | 19 ++ 8 files changed, 438 insertions(+), 32 deletions(-) create mode 100644 terraform/examples/azure/base-cluster/root-devzero.ovpn delete mode 100644 terraform/modules/azure/cluster_extensions/outputs.tf create mode 100644 terraform/modules/azure/vpn/main.tf create mode 100644 terraform/modules/azure/vpn/variables.tf diff --git a/terraform/examples/azure/base-cluster/main.tf b/terraform/examples/azure/base-cluster/main.tf index f9e5517d..995c2af0 100644 --- a/terraform/examples/azure/base-cluster/main.tf +++ b/terraform/examples/azure/base-cluster/main.tf @@ -16,15 +16,34 @@ provider "azuread" {} locals { public_subnet_names = ["public-subnet-1"] private_subnet_names = ["private-subnet-1"] + gateway_subnet_names = var.create_vpn ? ["GatewaySubnet"] : [] calculated_public_subnets_cidrs = ["10.240.1.0/24"] calculated_private_subnets_cidrs = ["10.240.101.0/24"] + gateway_subnet_cidrs = var.create_vpn ? ["10.240.255.0/27"] : [] } data "azurerm_client_config" "current" {} +resource "azuread_application" "sp" { + count = (var.create_vault_auto_unseal_key || var.create_vpn) ? 1 : 0 + display_name = "${var.cluster_name}-devzero-app" +} + +resource "azuread_service_principal" "sp" { + count = (var.create_vault_auto_unseal_key || var.create_vpn) ? 1 : 0 + client_id = azuread_application.sp[0].client_id +} + +resource "azuread_application_password" "sp" { + count = (var.create_vault_auto_unseal_key || var.create_vpn) ? 1 : 0 + application_id = azuread_application.sp[0].id + display_name = "DevZero SP Password" + end_date_relative = "8760h" +} + ################################################################################ -# VNET (updated to Azure/network/azurerm v5.3.0 with extended support) +# VNET ################################################################################ module "vnet" { @@ -37,10 +56,11 @@ module "vnet" { vnet_name = "${var.cluster_name}-vnet" address_space = var.cidr - subnet_names = concat(local.public_subnet_names, local.private_subnet_names) + subnet_names = concat(local.public_subnet_names, local.private_subnet_names, local.gateway_subnet_names) subnet_prefixes = concat( local.calculated_public_subnets_cidrs, - local.calculated_private_subnets_cidrs + local.calculated_private_subnets_cidrs, + local.gateway_subnet_cidrs ) subnet_enforce_private_link_endpoint_network_policies = { @@ -50,9 +70,54 @@ module "vnet" { tags = var.tags } +################################################################################ +# NAT Gateway +################################################################################ + +resource "azurerm_public_ip" "nat" { + count = var.enable_nat_gateway ? 1 : 0 + name = "${var.cluster_name}-nat-ip" + location = var.location + resource_group_name = var.resource_group_name + allocation_method = "Static" + sku = "Standard" +} + +resource "azurerm_nat_gateway" "nat" { + count = var.enable_nat_gateway ? 1 : 0 + name = "${var.cluster_name}-nat-gateway" + location = var.location + resource_group_name = var.resource_group_name +} + +resource "azurerm_nat_gateway_public_ip_association" "nat" { + count = var.enable_nat_gateway ? 1 : 0 + nat_gateway_id = azurerm_nat_gateway.nat[count.index].id + public_ip_address_id = azurerm_public_ip.nat[count.index].id +} + +resource "azurerm_route_table" "private_route_table" { + count = var.enable_nat_gateway ? 1 : 0 + name = "${var.cluster_name}-private-route-table" + location = var.location + resource_group_name = var.resource_group_name + + route { + name = "private-subnet-nat-route" + address_prefix = "0.0.0.0/0" + next_hop_type = "Internet" + next_hop_in_ip_address = azurerm_public_ip.nat[count.index].ip_address + } +} + +resource "azurerm_subnet_route_table_association" "private_subnet_route_association" { + count = var.enable_nat_gateway ? length(local.private_subnet_names) : 0 + subnet_id = module.vnet.vnet_subnets[1] # Assuming the module outputs subnet_ids + route_table_id = azurerm_route_table.private_route_table[0].id +} ################################################################################ -# AKS CLUSTER (updated to valid parameters for v9.4.1 module) +# AKS CLUSTER ################################################################################ module "aks" { @@ -98,6 +163,27 @@ module "aks" { tags = var.tags } +################################################################################ +# VPN +################################################################################ + +module "vpn" { + count = var.create_vpn ? 1 : 0 + + source = "../../../modules/azure/vpn" + + name = var.cluster_name + location = var.location + resource_group_name = var.resource_group_name + gateway_subnet_id = module.vnet.vnet_subnets[2] + vpn_client_cidr = "172.16.0.0/24" + vpn_client_list = var.vpn_client_list + tenant_id = data.azurerm_client_config.current.tenant_id + sp_object_id = azuread_service_principal.sp[0].object_id + + depends_on = [module.vnet] +} + ################################################################################ # Custom DERP server in Azure ################################################################################ @@ -153,24 +239,7 @@ resource "azurerm_key_vault_key" "vault_auto_unseal" { resource "azurerm_role_assignment" "vault_key_usage" { count = var.create_vault_auto_unseal_key ? 1 : 0 - principal_id = azuread_service_principal.vault[0].object_id + principal_id = azuread_service_principal.sp[0].object_id role_definition_name = "Key Vault Crypto User" scope = azurerm_key_vault.vault_auto_unseal[0].id -} - -resource "azuread_application" "vault" { - count = var.create_vault_auto_unseal_key ? 1 : 0 - display_name = "${var.cluster_name}-vault-app" -} - -resource "azuread_service_principal" "vault" { - count = var.create_vault_auto_unseal_key ? 1 : 0 - client_id = azuread_application.vault[0].client_id -} - -resource "azuread_application_password" "vault" { - count = var.create_vault_auto_unseal_key ? 1 : 0 - application_id = azuread_application.vault[0].id - display_name = "Vault SP Password" - end_date_relative = "8760h" } \ No newline at end of file diff --git a/terraform/examples/azure/base-cluster/outputs.tf b/terraform/examples/azure/base-cluster/outputs.tf index 9cc725fa..25d2d8b0 100644 --- a/terraform/examples/azure/base-cluster/outputs.tf +++ b/terraform/examples/azure/base-cluster/outputs.tf @@ -18,19 +18,19 @@ output "location" { value = var.location } -output "vault_sp_client_id" { - value = var.create_vault_auto_unseal_key ? azuread_application.vault[0].client_id : null +output "sp_client_id" { + value = (var.create_vault_auto_unseal_key || var.create_vpn) ? azuread_application.sp[0].client_id : null description = "The client ID for Vault's Azure Key Vault integration." } -output "vault_sp_client_secret" { - value = var.create_vault_auto_unseal_key ? azuread_application_password.vault[0].value : null +output "sp_client_secret" { + value = (var.create_vault_auto_unseal_key || var.create_vpn) ? azuread_application_password.sp[0].value : null description = "The client secret for Vault's Azure Key Vault integration." sensitive = true } -output "vault_tenant_id" { - value = var.create_vault_auto_unseal_key ? data.azurerm_client_config.current.tenant_id : null +output "tenant_id" { + value = (var.create_vault_auto_unseal_key || var.create_vpn) ? data.azurerm_client_config.current.tenant_id : null description = "The Azure Tenant ID." } diff --git a/terraform/examples/azure/base-cluster/root-devzero.ovpn b/terraform/examples/azure/base-cluster/root-devzero.ovpn new file mode 100644 index 00000000..583ff83c --- /dev/null +++ b/terraform/examples/azure/base-cluster/root-devzero.ovpn @@ -0,0 +1,89 @@ +client +dev tun +proto udp +remote 172.191.212.28 1194 +remote-cert-tls server +data-ciphers AES-256-GCM:AES-128-GCM:CHACHA20-POLY1305 +data-ciphers-fallback AES-256-CBC +verify-x509-name "devzero-vpn-gw" name +persist-key +persist-tun +resolv-retry infinite +nobind +verb 3 +auth-nocache + +-----BEGIN CERTIFICATE----- +MIIDNzCCAh+gAwIBAgIQVGWv3CtnNS9iNmuu6RuqqDANBgkqhkiG9w0BAQsFADAr +MRAwDgYDVQQKEwdkZXZ6ZXJvMRcwFQYDVQQDEw5kZXZ6ZXJvLnZwbi5jYTAeFw0y +NTA1MTQwOTE3MTVaFw0zNTA1MTIwOTE3MTVaMCsxEDAOBgNVBAoTB2Rldnplcm8x +FzAVBgNVBAMTDmRldnplcm8udnBuLmNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEA7vIkEAWk7z8mtamd60H32nuZsd8BC5GWSe1NzMOK6akqyXOvdrvX +xlgYV6TyPgyEn1R6rE9KzQispDurfjST+TLJ/biDmUT/QWFCY/8Ho13hWNNbcDZ1 +zG93TUqqFHgC5PXIRJdDxDHWF2y+giT1RSohcKJ+JLtHyPae9eL8KL0b8TW3/FIj +paAMDW6/Eb7jtvdfozrZftL4nQqzzaeNNspooHaTn2hFnxnfzmYWdiXDjtbwtqY0 +oLa9PmQQmXe19I0RnEznqPpXrtRDAyHu8rWWrfGWybVIU1nihnkcsy8ZxEs2yV1P ++doJGiEjufBnc+SbBU+6UwnbBHHT9OQ59QIDAQABo1cwVTAOBgNVHQ8BAf8EBAMC +AaYwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQUkbnro8N9eSEg8sLoTQzEwFGbLYEwDQYJKoZIhvcNAQELBQADggEBAF/nhdNS +dNw4y7xOJP76+u/mG2YVvk0W1c/8So62XcP5A7FIsNsrQB9sD78h7LiuAtH9AatJ +Q2ciL4PFs3xKnHQahriG7u6e/wX0CHtPwp+Y1OC7RSQ4+4JT9bvTCl8FcGVW56Gp +2SnASIZr37zBZroixnWdoS1wWPznh5Qq/vfb1hR94XuSTcNcEjiu7wd0AzCwKBKq +Cv9T+90ZQUjj5MQUCZRp1qACRrvh9h1ok0VhQtz3olmKDXKwKHXCNf8snmeL+fuH +qoI2mgSJO18dePen+6Z+B4zTqQVJaonhQ7QDadIduG3N6/KHVegl997FuKV1rwMF +G2fBVdXxnnZYDrA= +-----END CERTIFICATE----- + + + +-----BEGIN CERTIFICATE----- +MIIDLDCCAhSgAwIBAgIQaY66tE73vbPEY3kAl5OD2TANBgkqhkiG9w0BAQsFADAr +MRAwDgYDVQQKEwdkZXZ6ZXJvMRcwFQYDVQQDEw5kZXZ6ZXJvLnZwbi5jYTAeFw0y +NTA1MTQwOTE3MTVaFw0zNTA1MTIwOTE3MTVaMCExEDAOBgNVBAoTB2Rldnplcm8x +DTALBgNVBAMTBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC9 +CPNbau9jvQRgJ30yZE6sr8xNBcM6MXxDOBrxgbdiPgq/uXS1ejJfvRd7CKt7B4qF +wCP2W2gV2Pvr4XoBHu0D3V98GoZPu59dKtLGQbVf9fFA0t3JQ9Iz/EI2R91AB+vQ +iz3R9KYXG86bubwwdnJWMlvHAqPrAMpcuRxpfrltjHMIXSx8IbCon4oOXv2GP89u +N4SF1n55GozGdQh1RLahR6CGt3s02PpWxIVHjmV2S6wUzhO7150idvVzKl69IlOR +Y7548FAnVKLjdh9qW0l3l2cuz0hZiFd/g6EFPcL3R6RwSz0vZuju/RCDsIZHwAUp +E5pmsyaEwmW4zDJpIWwhAgMBAAGjVjBUMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUE +DDAKBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB8GA1UdIwQYMBaAFJG566PDfXkh +IPLC6E0MxMBRmy2BMA0GCSqGSIb3DQEBCwUAA4IBAQAFoATg1P/eXJGiiQVEdzW7 +VjduTyyUafHKUeeTK6AMeLVSGEKyEehjrkR+rbAEXMyXVFBujMzZeX5ZDdv6r1ao +WRuExbX8DB106CtIwK/EJaRfTLdOq6oFXMqATz0Zq1F1f6hcZj0nHIKz/HCEmvnY +4XdRKgHuZotePQUgrYtx2P4Oz2n9kxGPUmpLuK+SbAtscmXGAutJx7dm22Eeh2iq +K6278hky2WqZ5jViwL67jt5y4/iXfnvPq2g6rEiePKT6KTY/IVCsr3Q3oosuGQ/9 +g82t5BrpYj32oKaUAR0FE5ZOAgO/mwW/pH+bAGNFgYGe5T4w3fXH7oGHwXCvcamo +-----END CERTIFICATE----- + + + +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAvQjzW2rvY70EYCd9MmROrK/MTQXDOjF8Qzga8YG3Yj4Kv7l0 +tXoyX70XewireweKhcAj9ltoFdj76+F6AR7tA91ffBqGT7ufXSrSxkG1X/XxQNLd +yUPSM/xCNkfdQAfr0Is90fSmFxvOm7m8MHZyVjJbxwKj6wDKXLkcaX65bYxzCF0s +fCGwqJ+KDl79hj/PbjeEhdZ+eRqMxnUIdUS2oUeghrd7NNj6VsSFR45ldkusFM4T +u9edInb1cypevSJTkWO+ePBQJ1Si43YfaltJd5dnLs9IWYhXf4OhBT3C90ekcEs9 +L2bo7v0Qg7CGR8AFKROaZrMmhMJluMwyaSFsIQIDAQABAoIBAB7NEw4NvgTj2w8p +0AoCcxlLKcBRFI7qWkN/pFA+oRlWX5z+xGf/ia602lsXPg21XFpPxOxS14Jb0SyN +mDoPhnIPQ4wurDzYpu5g2Ggqc7m3ksBLu4ycSudijo1Wp1PUFpXr4j5QHTH9u2v/ +Vw0Ud6vY18B1Cy1y+wYhuWHtHpydMopT8uxOTeJcJKazGj/VaJBjiFzf7fcjerwk +vN9FInAX9EP62ZpTUxxrBclgDB9ufTuu4hfHJn9nMSP64B4LXsjJJMQCrzpFkBZD +yvWluP8qeyT3tfJ9QN0VgFN3PiwOaVxo7U79DAQpHDeqtPoh+qWlCyELf3mnG/zH +0jSptpsCgYEAy/bSJtGchL+S7tnSYdac0P/ce40yX6KYw9kkHpHWGIIsnJjZseV8 +hxQhpcbK7YAfn7yqjamiIRYB1Mr5ClPmJMpDwfZ3I0VYAWUHyLovSbl8+QRnuduV +aHCSc4BJitHVFJpfFyZuDbfWdKRDLj16fIHf8sKATWAGKaGzvCXmQfcCgYEA7UMV +mClVhRNTYCBnz2kE7XybAIXZdAB2NXCPIt99AjqeGvULKoGG9/WSDQHHyKsoFrJZ +D8tiuk5qckU0xIAqe4NRYeCkXqP/I7KTd9IjR/02uTcyoyoMmdMn2e/WdwuaJ+pK +/n4xipNu85jHgSJm4taOgerJB3TdR1kuzjWdvKcCgYEAqx41a5n6LakoFbmrdqf+ +Qp4WlZV9SJz6e31Gf1ss+rzjvrHCGi0FT7KFFDb281DFt1cq49UIj6eN+8AaNh7z +e7QyKp2jkHTSA8GGSVJi8ABtp+GZQGXIMKoZREvB53Rpx3JICv0T4fB1PMC/GCE+ +DQ1NHLcvuzqvkOf4Cgbl6jsCgYEA34qMJwASvj32GSSPCmElLmhvVe/GcMAan8D8 +a1vr2RTHcBieO58sPiVWRQD/bbatH61LslDrJSPKsP05eUZUQGj6XWCdHTogok6X +GJaZfTM6wwkZoZd1pdzHuFN7WlBJHwBlRiA5pxm/uK2ikq2KCIEkZY2DeI4ICjgh +nFHx2xMCgYAddO7QIv83gJGKNWKKxUn9pA5j7f7MmKsa7o+OiOkeBSp4PcwxyBVq +DoHL0K/enHhIUYtaK2ADym12wRDaHgZcKwkvfk8MOkAH7ct/D4MfXR9AOs1gyML5 +HNSeHpArKSUiZBE7Nxy853jPsxPJW6LNsw0vajNbBlRdX4cjTSWpww== +-----END RSA PRIVATE KEY----- + + diff --git a/terraform/examples/azure/base-cluster/terraform.tfvars b/terraform/examples/azure/base-cluster/terraform.tfvars index 06f052a0..642342bc 100644 --- a/terraform/examples/azure/base-cluster/terraform.tfvars +++ b/terraform/examples/azure/base-cluster/terraform.tfvars @@ -28,7 +28,9 @@ tags = { createdBy = "terraform" } -create_derp = true -public_derp = true +create_derp = false +public_derp = false -create_vault_auto_unseal_key = true \ No newline at end of file +create_vault_auto_unseal_key = false + +create_vpn = true \ No newline at end of file diff --git a/terraform/examples/azure/base-cluster/variables.tf b/terraform/examples/azure/base-cluster/variables.tf index 254c90bf..d02b51ce 100644 --- a/terraform/examples/azure/base-cluster/variables.tf +++ b/terraform/examples/azure/base-cluster/variables.tf @@ -80,7 +80,7 @@ variable "desired_size" { variable "enable_nat_gateway" { description = "Whether to enable NAT Gateway" type = bool - default = false + default = true } variable "single_nat_gateway" { @@ -113,8 +113,25 @@ variable "public_derp" { ################################################################################ # Vault ################################################################################ + variable "create_vault_auto_unseal_key" { description = "Whether or not to create a KMS key for Vault auto unseal" type = bool default = false +} + +################################################################################ +# Vault +################################################################################ + +variable "create_vpn" { + description = "Controls if VPN gateway and VPN resources will be created." + type = bool + default = false +} + +variable "vpn_client_list" { + description = "Subnets" + type = set(string) + default = ["root"] } \ No newline at end of file diff --git a/terraform/modules/azure/cluster_extensions/outputs.tf b/terraform/modules/azure/cluster_extensions/outputs.tf deleted file mode 100644 index e69de29b..00000000 diff --git a/terraform/modules/azure/vpn/main.tf b/terraform/modules/azure/vpn/main.tf new file mode 100644 index 00000000..80eb0120 --- /dev/null +++ b/terraform/modules/azure/vpn/main.tf @@ -0,0 +1,210 @@ +resource "tls_private_key" "ca" { + algorithm = "RSA" +} + +resource "tls_self_signed_cert" "ca" { + private_key_pem = tls_private_key.ca.private_key_pem + subject { + common_name = "${var.name}.vpn.ca" + organization = var.name + } + validity_period_hours = 87600 + is_ca_certificate = true + allowed_uses = [ + "cert_signing", + "crl_signing", + "digital_signature", # Add this for client auth + "key_encipherment", # Add this for VPN usage + "client_auth", # Ensure VPN client compatibility + "server_auth" + ] +} + +locals { + root_cert_data = replace( + replace( + replace( + tls_self_signed_cert.ca.cert_pem, + "-----BEGIN CERTIFICATE-----\n", + "" + ), + "\n-----END CERTIFICATE-----", + "" + ), + "\n", + "" + ) +} + +resource "azurerm_public_ip" "vpn_gateway_ip" { + name = "${var.name}-vpn-ip" + location = var.location + resource_group_name = var.resource_group_name + allocation_method = "Static" + sku = "Standard" +} + +resource "azurerm_virtual_network_gateway" "vpn_gateway" { + name = "${var.name}-vpn-gw" + location = var.location + resource_group_name = var.resource_group_name + + type = "Vpn" + vpn_type = "RouteBased" + sku = "VpnGw1" + + active_active = false + enable_bgp = false + + ip_configuration { + name = "vpn-gateway-config" + public_ip_address_id = azurerm_public_ip.vpn_gateway_ip.id + private_ip_address_allocation = "Dynamic" + subnet_id = var.gateway_subnet_id + } + + vpn_client_configuration { + address_space = [var.vpn_client_cidr] + vpn_client_protocols = ["OpenVPN"] + root_certificate { + name = "vpn-root" + public_cert_data = local.root_cert_data + } + vpn_auth_types = ["Certificate"] + } + depends_on = [ + tls_self_signed_cert.ca + ] +} + +resource "azurerm_storage_account" "vpn_config" { + name = "${lower(var.name)}vpnstorage" + location = var.location + resource_group_name = var.resource_group_name + account_tier = "Standard" + account_replication_type = "LRS" +} + +resource "azurerm_storage_container" "vpn_config_container" { + name = "vpn-config-files" + storage_account_name = azurerm_storage_account.vpn_config.name + container_access_type = "private" +} + +resource "azurerm_key_vault" "vpn_keyvault" { + name = "${lower(var.name)}-vpn-kv" + location = var.location + resource_group_name = var.resource_group_name + enabled_for_disk_encryption = true + tenant_id = var.tenant_id + sku_name = "standard" + enable_rbac_authorization = true +} + +resource "azurerm_key_vault_access_policy" "vpn_access" { + key_vault_id = azurerm_key_vault.vpn_keyvault.id + tenant_id = var.tenant_id + object_id = var.sp_object_id + + secret_permissions = [ + "Get", "List", # Add "get" permission to allow secret retrieval + ] +} + +resource "azurerm_key_vault_secret" "vpn_ca_cert" { + name = "vpn-ca-cert" + value = tls_self_signed_cert.ca.cert_pem + key_vault_id = azurerm_key_vault.vpn_keyvault.id +} + +resource "azurerm_key_vault_secret" "vpn_ca_key" { + name = "vpn-ca-key" + value = tls_private_key.ca.private_key_pem + key_vault_id = azurerm_key_vault.vpn_keyvault.id +} + +resource "tls_private_key" "client" { + for_each = var.vpn_client_list + algorithm = "RSA" +} + +resource "tls_cert_request" "client" { + for_each = var.vpn_client_list + private_key_pem = tls_private_key.client[each.key].private_key_pem + subject { + common_name = each.key + organization = var.name + } +} + +resource "tls_locally_signed_cert" "client" { + for_each = var.vpn_client_list + cert_request_pem = tls_cert_request.client[each.key].cert_request_pem + ca_private_key_pem = tls_private_key.ca.private_key_pem + ca_cert_pem = tls_self_signed_cert.ca.cert_pem + validity_period_hours = 87600 + allowed_uses = [ + "key_encipherment", + "digital_signature", + "client_auth", + "server_auth" + ] +} + +resource "azurerm_key_vault_secret" "vpn_client_cert" { + for_each = var.vpn_client_list + name = "vpn-${each.key}-cert" + value = tls_locally_signed_cert.client[each.key].cert_pem + key_vault_id = azurerm_key_vault.vpn_keyvault.id +} + +resource "azurerm_key_vault_secret" "vpn_client_key" { + for_each = var.vpn_client_list + name = "vpn-${each.key}-key" + value = tls_private_key.client[each.key].private_key_pem + key_vault_id = azurerm_key_vault.vpn_keyvault.id +} + +resource "azurerm_storage_blob" "vpn_config_file" { + for_each = var.vpn_client_list + name = "${substr(each.key, 0, 30)}-${lower(var.name)}.ovpn" + storage_account_name = azurerm_storage_account.vpn_config.name + storage_container_name = azurerm_storage_container.vpn_config_container.name + type = "Block" + source_content = < +${azurerm_key_vault_secret.vpn_ca_cert.value} + + +${azurerm_key_vault_secret.vpn_client_cert[each.key].value} + + +${azurerm_key_vault_secret.vpn_client_key[each.key].value} + +EOT + + depends_on = [ + azurerm_key_vault_secret.vpn_client_cert, + azurerm_key_vault_secret.vpn_client_key + ] +} \ No newline at end of file diff --git a/terraform/modules/azure/vpn/variables.tf b/terraform/modules/azure/vpn/variables.tf new file mode 100644 index 00000000..2c80588b --- /dev/null +++ b/terraform/modules/azure/vpn/variables.tf @@ -0,0 +1,19 @@ +variable "tenant_id" {} +variable "name" {} +variable "location" {} +variable "resource_group_name" {} +variable "gateway_subnet_id" {} +variable "vpn_client_cidr" { + default = "172.16.0.0/24" +} + +variable "vpn_gateway_port" { + description = "The port for the VPN gateway" + type = string + default = "1194" +} +variable "vpn_client_list" { + type = set(string) +} + +variable "sp_object_id" {} \ No newline at end of file From ad463c213110350e797b29752b333761b8abf478 Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Wed, 14 May 2025 18:08:56 +0530 Subject: [PATCH 04/13] removed ovpn --- .../azure/base-cluster/root-devzero.ovpn | 89 ------------------- 1 file changed, 89 deletions(-) delete mode 100644 terraform/examples/azure/base-cluster/root-devzero.ovpn diff --git a/terraform/examples/azure/base-cluster/root-devzero.ovpn b/terraform/examples/azure/base-cluster/root-devzero.ovpn deleted file mode 100644 index 583ff83c..00000000 --- a/terraform/examples/azure/base-cluster/root-devzero.ovpn +++ /dev/null @@ -1,89 +0,0 @@ -client -dev tun -proto udp -remote 172.191.212.28 1194 -remote-cert-tls server -data-ciphers AES-256-GCM:AES-128-GCM:CHACHA20-POLY1305 -data-ciphers-fallback AES-256-CBC -verify-x509-name "devzero-vpn-gw" name -persist-key -persist-tun -resolv-retry infinite -nobind -verb 3 -auth-nocache - ------BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIQVGWv3CtnNS9iNmuu6RuqqDANBgkqhkiG9w0BAQsFADAr -MRAwDgYDVQQKEwdkZXZ6ZXJvMRcwFQYDVQQDEw5kZXZ6ZXJvLnZwbi5jYTAeFw0y -NTA1MTQwOTE3MTVaFw0zNTA1MTIwOTE3MTVaMCsxEDAOBgNVBAoTB2Rldnplcm8x -FzAVBgNVBAMTDmRldnplcm8udnBuLmNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEA7vIkEAWk7z8mtamd60H32nuZsd8BC5GWSe1NzMOK6akqyXOvdrvX -xlgYV6TyPgyEn1R6rE9KzQispDurfjST+TLJ/biDmUT/QWFCY/8Ho13hWNNbcDZ1 -zG93TUqqFHgC5PXIRJdDxDHWF2y+giT1RSohcKJ+JLtHyPae9eL8KL0b8TW3/FIj -paAMDW6/Eb7jtvdfozrZftL4nQqzzaeNNspooHaTn2hFnxnfzmYWdiXDjtbwtqY0 -oLa9PmQQmXe19I0RnEznqPpXrtRDAyHu8rWWrfGWybVIU1nihnkcsy8ZxEs2yV1P -+doJGiEjufBnc+SbBU+6UwnbBHHT9OQ59QIDAQABo1cwVTAOBgNVHQ8BAf8EBAMC -AaYwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E -FgQUkbnro8N9eSEg8sLoTQzEwFGbLYEwDQYJKoZIhvcNAQELBQADggEBAF/nhdNS -dNw4y7xOJP76+u/mG2YVvk0W1c/8So62XcP5A7FIsNsrQB9sD78h7LiuAtH9AatJ -Q2ciL4PFs3xKnHQahriG7u6e/wX0CHtPwp+Y1OC7RSQ4+4JT9bvTCl8FcGVW56Gp -2SnASIZr37zBZroixnWdoS1wWPznh5Qq/vfb1hR94XuSTcNcEjiu7wd0AzCwKBKq -Cv9T+90ZQUjj5MQUCZRp1qACRrvh9h1ok0VhQtz3olmKDXKwKHXCNf8snmeL+fuH -qoI2mgSJO18dePen+6Z+B4zTqQVJaonhQ7QDadIduG3N6/KHVegl997FuKV1rwMF -G2fBVdXxnnZYDrA= ------END CERTIFICATE----- - - - ------BEGIN CERTIFICATE----- -MIIDLDCCAhSgAwIBAgIQaY66tE73vbPEY3kAl5OD2TANBgkqhkiG9w0BAQsFADAr -MRAwDgYDVQQKEwdkZXZ6ZXJvMRcwFQYDVQQDEw5kZXZ6ZXJvLnZwbi5jYTAeFw0y -NTA1MTQwOTE3MTVaFw0zNTA1MTIwOTE3MTVaMCExEDAOBgNVBAoTB2Rldnplcm8x -DTALBgNVBAMTBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC9 -CPNbau9jvQRgJ30yZE6sr8xNBcM6MXxDOBrxgbdiPgq/uXS1ejJfvRd7CKt7B4qF -wCP2W2gV2Pvr4XoBHu0D3V98GoZPu59dKtLGQbVf9fFA0t3JQ9Iz/EI2R91AB+vQ -iz3R9KYXG86bubwwdnJWMlvHAqPrAMpcuRxpfrltjHMIXSx8IbCon4oOXv2GP89u -N4SF1n55GozGdQh1RLahR6CGt3s02PpWxIVHjmV2S6wUzhO7150idvVzKl69IlOR -Y7548FAnVKLjdh9qW0l3l2cuz0hZiFd/g6EFPcL3R6RwSz0vZuju/RCDsIZHwAUp -E5pmsyaEwmW4zDJpIWwhAgMBAAGjVjBUMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUE -DDAKBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB8GA1UdIwQYMBaAFJG566PDfXkh -IPLC6E0MxMBRmy2BMA0GCSqGSIb3DQEBCwUAA4IBAQAFoATg1P/eXJGiiQVEdzW7 -VjduTyyUafHKUeeTK6AMeLVSGEKyEehjrkR+rbAEXMyXVFBujMzZeX5ZDdv6r1ao -WRuExbX8DB106CtIwK/EJaRfTLdOq6oFXMqATz0Zq1F1f6hcZj0nHIKz/HCEmvnY -4XdRKgHuZotePQUgrYtx2P4Oz2n9kxGPUmpLuK+SbAtscmXGAutJx7dm22Eeh2iq -K6278hky2WqZ5jViwL67jt5y4/iXfnvPq2g6rEiePKT6KTY/IVCsr3Q3oosuGQ/9 -g82t5BrpYj32oKaUAR0FE5ZOAgO/mwW/pH+bAGNFgYGe5T4w3fXH7oGHwXCvcamo ------END CERTIFICATE----- - - - ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAvQjzW2rvY70EYCd9MmROrK/MTQXDOjF8Qzga8YG3Yj4Kv7l0 -tXoyX70XewireweKhcAj9ltoFdj76+F6AR7tA91ffBqGT7ufXSrSxkG1X/XxQNLd -yUPSM/xCNkfdQAfr0Is90fSmFxvOm7m8MHZyVjJbxwKj6wDKXLkcaX65bYxzCF0s -fCGwqJ+KDl79hj/PbjeEhdZ+eRqMxnUIdUS2oUeghrd7NNj6VsSFR45ldkusFM4T -u9edInb1cypevSJTkWO+ePBQJ1Si43YfaltJd5dnLs9IWYhXf4OhBT3C90ekcEs9 -L2bo7v0Qg7CGR8AFKROaZrMmhMJluMwyaSFsIQIDAQABAoIBAB7NEw4NvgTj2w8p -0AoCcxlLKcBRFI7qWkN/pFA+oRlWX5z+xGf/ia602lsXPg21XFpPxOxS14Jb0SyN -mDoPhnIPQ4wurDzYpu5g2Ggqc7m3ksBLu4ycSudijo1Wp1PUFpXr4j5QHTH9u2v/ -Vw0Ud6vY18B1Cy1y+wYhuWHtHpydMopT8uxOTeJcJKazGj/VaJBjiFzf7fcjerwk -vN9FInAX9EP62ZpTUxxrBclgDB9ufTuu4hfHJn9nMSP64B4LXsjJJMQCrzpFkBZD -yvWluP8qeyT3tfJ9QN0VgFN3PiwOaVxo7U79DAQpHDeqtPoh+qWlCyELf3mnG/zH -0jSptpsCgYEAy/bSJtGchL+S7tnSYdac0P/ce40yX6KYw9kkHpHWGIIsnJjZseV8 -hxQhpcbK7YAfn7yqjamiIRYB1Mr5ClPmJMpDwfZ3I0VYAWUHyLovSbl8+QRnuduV -aHCSc4BJitHVFJpfFyZuDbfWdKRDLj16fIHf8sKATWAGKaGzvCXmQfcCgYEA7UMV -mClVhRNTYCBnz2kE7XybAIXZdAB2NXCPIt99AjqeGvULKoGG9/WSDQHHyKsoFrJZ -D8tiuk5qckU0xIAqe4NRYeCkXqP/I7KTd9IjR/02uTcyoyoMmdMn2e/WdwuaJ+pK -/n4xipNu85jHgSJm4taOgerJB3TdR1kuzjWdvKcCgYEAqx41a5n6LakoFbmrdqf+ -Qp4WlZV9SJz6e31Gf1ss+rzjvrHCGi0FT7KFFDb281DFt1cq49UIj6eN+8AaNh7z -e7QyKp2jkHTSA8GGSVJi8ABtp+GZQGXIMKoZREvB53Rpx3JICv0T4fB1PMC/GCE+ -DQ1NHLcvuzqvkOf4Cgbl6jsCgYEA34qMJwASvj32GSSPCmElLmhvVe/GcMAan8D8 -a1vr2RTHcBieO58sPiVWRQD/bbatH61LslDrJSPKsP05eUZUQGj6XWCdHTogok6X -GJaZfTM6wwkZoZd1pdzHuFN7WlBJHwBlRiA5pxm/uK2ikq2KCIEkZY2DeI4ICjgh -nFHx2xMCgYAddO7QIv83gJGKNWKKxUn9pA5j7f7MmKsa7o+OiOkeBSp4PcwxyBVq -DoHL0K/enHhIUYtaK2ADym12wRDaHgZcKwkvfk8MOkAH7ct/D4MfXR9AOs1gyML5 -HNSeHpArKSUiZBE7Nxy853jPsxPJW6LNsw0vajNbBlRdX4cjTSWpww== ------END RSA PRIVATE KEY----- - - From d80eb9670933cbf2836440c95a1997c269bfcf59 Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Wed, 21 May 2025 11:37:05 +0530 Subject: [PATCH 05/13] ci for dsh-testing azure --- .github/workflows/azure-dsh-testing.yaml | 179 +++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 .github/workflows/azure-dsh-testing.yaml diff --git a/.github/workflows/azure-dsh-testing.yaml b/.github/workflows/azure-dsh-testing.yaml new file mode 100644 index 00000000..da8699d2 --- /dev/null +++ b/.github/workflows/azure-dsh-testing.yaml @@ -0,0 +1,179 @@ +name: "[Azure] DevZero self-hosted deployment" + +on: + push: + branches: + - garvit/azure-tf + workflow_dispatch: + +jobs: + setup-and-test: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: 'Az CLI login' + uses: azure/login@a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.11.3" + + - name: Install yq + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/v4.35.2/yq_linux_amd64 -O /usr/local/bin/yq + sudo chmod +x /usr/local/bin/yq + + - name : Add SHORT_SHA Environment Variable + id : short-sha + shell: bash + run : echo "SHORT_SHA=`git rev-parse --short HEAD`" >> $GITHUB_ENV + + - name : Generate unique job identifier + id : job-identifier + shell: bash + run : echo "JOB_IDENTIFIER=gh-azure-ci-${{ github.event.inputs.base_image }}-${SHORT_SHA}" >> $GITHUB_ENV + + - name: Add Backend Override (Base Cluster) + run: | + cd terraform/examples/azure/base-cluster + cat < backend_override.tf + terraform { + backend "azurerm" { + resource_group_name = "dev-test" + storage_account_name = "dshterraformstate" + container_name = "tfstate" + key = "${JOB_IDENTIFIER}/base-cluster/terraform.tfstate" + } + } + EOF + + - name: Initialize and Apply Terraform (Base Cluster) + run: | + cd terraform/examples/azure/base-cluster + terraform init + terraform apply -auto-approve -var="cluster_name=$JOB_IDENTIFIER" + + - name: Configure Kubernetes Access + run: | + az aks get-credentials --resource-group dev-test --name $JOB_IDENTIFIER + + - name: Set up Kata + run: | + cd terraform/examples/azure/base-cluster + kubectl apply -f kata-sa.yaml + kubectl apply -f daemonset.yaml + for NODE in $(kubectl get nodes -o name); do + kubectl label "$NODE" kata-runtime=running --overwrite + kubectl label "$NODE" node-role.kubernetes.io/kata-devpod-node=1 --overwrite + done + + - name: Deploy Control Plane Dependencies (and modify domains) + run: | + DEFAULT_SC=$(kubectl get sc -o jsonpath='{.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}') + kubectl get sc "$DEFAULT_SC" -o yaml | \ + sed "s/name: $DEFAULT_SC/name: gp2/" | \ + sed "/^ uid:/d; /^ resourceVersion:/d; /^ creationTimestamp:/d" | \ + kubectl apply -f - + + cd charts/dz-control-plane-deps + + find . -name "values.yaml" -print0 | while IFS= read -r -d '' file; do + yq e -i '.. |= select(tag == "!!str" and test("example\\.com")) |= sub("example\\.com"; env(JOB_IDENTIFIER) + ".ci.selfzero.net")' "$file" + done + + make install + + - name: Update values.yaml for dz-control-plane + env: + BACKEND_LICENSE_KEY: ${{ secrets.BACKEND_LICENSE_KEY }} + run: | + # setting credentials enable to false since we will explicitly feed the dockerhub creds to kubernetes api + # also setting image.pullsecrets to empty to make sure that each of the deployments dont try to pull their relevant OCI images from this registry + # backend license key is ... needed + + yq e '.credentials.enable = false | .backend.licenseKey = strenv(BACKEND_LICENSE_KEY) | .image.pullSecrets = []' -i charts/dz-control-plane/values.yaml + + - name: Deploy DevZero Control Plane (after configuring kubernetes to use dockerhub creds, and patching all the deployments to point to the right domain) + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + cd charts/dz-control-plane + make add-docker-creds + + find . -name "values.yaml" -print0 | while IFS= read -r -d '' file; do + yq e -i '.. |= select(tag == "!!str" and test("example\\.com")) |= sub("example\\.com"; env(JOB_IDENTIFIER) + ".ci.selfzero.net")' "$file" + done + + make install + + - name: Validate Control Plane + run: | + .github/scripts/dsh-pod-test.sh + + - name: Deploy Data Plane Dependencies + run: | + cd charts/dz-data-plane-deps + + find . -name "values.yaml" -print0 | while IFS= read -r -d '' file; do + yq e -i '.. |= select(tag == "!!str" and test("example\\.com")) |= sub("example\\.com"; env(JOB_IDENTIFIER) + ".ci.selfzero.net")' "$file" + done + + make install + + - name: Deploy DevZero Data Plane + run: | + cd charts/dz-data-plane + + find . -name "values.yaml" -print0 | while IFS= read -r -d '' file; do + yq e -i '.. |= select(tag == "!!str" and test("example\\.com")) |= sub("example\\.com"; env(JOB_IDENTIFIER) + ".ci.selfzero.net")' "$file" + done + + make install + + - name: Validate Data Plane + run: | + kubectl get pods -n devzero-self-hosted + kubectl get ingress -n devzero-self-hosted + + - name: '[helm] Destroy data-plane' + if: always() + run: | + cd charts/dz-data-plane + make delete + + - name: '[helm] Destroy data-plane-deps' + if: always() + run: | + cd charts/dz-data-plane-deps + make delete + + - name: '[helm] Destroy control-plane' + if: always() + run: | + cd charts/dz-control-plane + make delete + + - name: '[helm] Destroy control-plane-deps' + if: always() + run: | + cd charts/dz-control-plane-deps + make delete + + - name: '[terraform] Destroy base-cluster' + if: always() + run: | + cd terraform/examples/azure/base-cluster + terraform destroy -auto-approve -var="prefix=$JOB_IDENTIFIER" + + From a57af173fa706e6907f023c333863ed25045ec9c Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Wed, 21 May 2025 11:38:52 +0530 Subject: [PATCH 06/13] ci for dsh-testing azure --- .github/workflows/azure-dsh-testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/azure-dsh-testing.yaml b/.github/workflows/azure-dsh-testing.yaml index da8699d2..2ba2b303 100644 --- a/.github/workflows/azure-dsh-testing.yaml +++ b/.github/workflows/azure-dsh-testing.yaml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v4 - name: 'Az CLI login' - uses: azure/login@a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0 + uses: azure/login@v1.6.1 with: client-id: ${{ secrets.AZURE_CLIENT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }} From 7f892bc3c4030b18a83beb94fdaaa90b77e1d4a9 Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Wed, 21 May 2025 11:56:17 +0530 Subject: [PATCH 07/13] ci for dsh-testing azure --- .github/workflows/azure-dsh-testing.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/azure-dsh-testing.yaml b/.github/workflows/azure-dsh-testing.yaml index 2ba2b303..9a43f308 100644 --- a/.github/workflows/azure-dsh-testing.yaml +++ b/.github/workflows/azure-dsh-testing.yaml @@ -41,7 +41,9 @@ jobs: - name : Generate unique job identifier id : job-identifier shell: bash - run : echo "JOB_IDENTIFIER=gh-azure-ci-${{ github.event.inputs.base_image }}-${SHORT_SHA}" >> $GITHUB_ENV + run : | + SAFE_ID=$(echo "gha${SHORT_SHA}" | tr -cd 'a-z0-9' | cut -c1-20) + echo "JOB_IDENTIFIER=$SAFE_ID" >> $GITHUB_ENV - name: Add Backend Override (Base Cluster) run: | From f90eeca97de8224c99867a3fed7cf8e1c792596a Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Wed, 21 May 2025 12:35:49 +0530 Subject: [PATCH 08/13] ci for dsh-testing azure --- terraform/examples/azure/base-cluster/terraform.tfvars | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/examples/azure/base-cluster/terraform.tfvars b/terraform/examples/azure/base-cluster/terraform.tfvars index 642342bc..6693307c 100644 --- a/terraform/examples/azure/base-cluster/terraform.tfvars +++ b/terraform/examples/azure/base-cluster/terraform.tfvars @@ -17,8 +17,8 @@ node_count = 2 min_size = 1 max_size = 3 -enable_nat_gateway = true -single_nat_gateway = true +enable_nat_gateway = false +single_nat_gateway = false # Optional tagging @@ -33,4 +33,4 @@ public_derp = false create_vault_auto_unseal_key = false -create_vpn = true \ No newline at end of file +create_vpn = false \ No newline at end of file From a6f42fd329e168dab1e796e173b6afb608fe2a29 Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Wed, 21 May 2025 13:01:41 +0530 Subject: [PATCH 09/13] ci for dsh-testing azure --- terraform/examples/azure/base-cluster/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/terraform/examples/azure/base-cluster/main.tf b/terraform/examples/azure/base-cluster/main.tf index 995c2af0..e822b62d 100644 --- a/terraform/examples/azure/base-cluster/main.tf +++ b/terraform/examples/azure/base-cluster/main.tf @@ -13,6 +13,10 @@ provider "azurerm" { provider "azuread" {} +provider "azapi" { + use_oidc = true +} + locals { public_subnet_names = ["public-subnet-1"] private_subnet_names = ["private-subnet-1"] From e3e3dbb79ce883338a7b26d03bcba00e5533d9f3 Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Wed, 21 May 2025 13:07:33 +0530 Subject: [PATCH 10/13] ci for dsh-testing azure --- terraform/examples/azure/base-cluster/main.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/examples/azure/base-cluster/main.tf b/terraform/examples/azure/base-cluster/main.tf index e822b62d..688b973a 100644 --- a/terraform/examples/azure/base-cluster/main.tf +++ b/terraform/examples/azure/base-cluster/main.tf @@ -4,6 +4,9 @@ terraform { source = "hashicorp/azurerm" version = ">= 3.106.1, < 4.0.0" } + azapi = { + source = "azure/azapi" + } } } From 52b0f68615df813bd3a7de977ea4e6666ab74678 Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Wed, 21 May 2025 13:28:27 +0530 Subject: [PATCH 11/13] ci for dsh-testing azure --- .github/workflows/azure-dsh-testing.yaml | 6 ++++++ terraform/examples/azure/base-cluster/main.tf | 1 + 2 files changed, 7 insertions(+) diff --git a/.github/workflows/azure-dsh-testing.yaml b/.github/workflows/azure-dsh-testing.yaml index 9a43f308..5ca1da91 100644 --- a/.github/workflows/azure-dsh-testing.yaml +++ b/.github/workflows/azure-dsh-testing.yaml @@ -23,6 +23,12 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - name: Set Azure env for Terraform providers + run: | + echo "ARM_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}" >> $GITHUB_ENV + echo "ARM_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}" >> $GITHUB_ENV + echo "ARM_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> $GITHUB_ENV + - name: Set up Terraform uses: hashicorp/setup-terraform@v3 with: diff --git a/terraform/examples/azure/base-cluster/main.tf b/terraform/examples/azure/base-cluster/main.tf index 688b973a..f6bc048b 100644 --- a/terraform/examples/azure/base-cluster/main.tf +++ b/terraform/examples/azure/base-cluster/main.tf @@ -6,6 +6,7 @@ terraform { } azapi = { source = "azure/azapi" + version = ">= 1.15.0, < 2.0.0" } } } From 920df53e3b3a6b2124c510cbe6104568e10c251a Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Wed, 21 May 2025 14:01:55 +0530 Subject: [PATCH 12/13] ci for dsh-testing azure --- .github/workflows/azure-dsh-testing.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/azure-dsh-testing.yaml b/.github/workflows/azure-dsh-testing.yaml index 5ca1da91..57c3a7ce 100644 --- a/.github/workflows/azure-dsh-testing.yaml +++ b/.github/workflows/azure-dsh-testing.yaml @@ -182,6 +182,4 @@ jobs: if: always() run: | cd terraform/examples/azure/base-cluster - terraform destroy -auto-approve -var="prefix=$JOB_IDENTIFIER" - - + terraform destroy -auto-approve -var="cluster_name=$JOB_IDENTIFIER" \ No newline at end of file From 3ba7dfee5c66c3aabf12cf2f77958d4e3a372a67 Mon Sep 17 00:00:00 2001 From: garvit3835 Date: Wed, 21 May 2025 17:36:03 +0530 Subject: [PATCH 13/13] tf modules for dsh in azure --- terraform/examples/azure/base-cluster/main.tf | 73 +++---------------- .../examples/azure/base-cluster/outputs.tf | 4 +- terraform/modules/azure/nat/main.tf | 37 ++++++++++ terraform/modules/azure/nat/variables.tf | 19 +++++ terraform/modules/azure/vault/main.tf | 26 +++++++ terraform/modules/azure/vault/outputs.tf | 9 +++ terraform/modules/azure/vault/variables.tf | 24 ++++++ 7 files changed, 126 insertions(+), 66 deletions(-) create mode 100644 terraform/modules/azure/nat/main.tf create mode 100644 terraform/modules/azure/nat/variables.tf create mode 100644 terraform/modules/azure/vault/main.tf create mode 100644 terraform/modules/azure/vault/outputs.tf create mode 100644 terraform/modules/azure/vault/variables.tf diff --git a/terraform/examples/azure/base-cluster/main.tf b/terraform/examples/azure/base-cluster/main.tf index f6bc048b..ff1a351a 100644 --- a/terraform/examples/azure/base-cluster/main.tf +++ b/terraform/examples/azure/base-cluster/main.tf @@ -81,47 +81,13 @@ module "vnet" { ################################################################################ # NAT Gateway ################################################################################ - -resource "azurerm_public_ip" "nat" { - count = var.enable_nat_gateway ? 1 : 0 - name = "${var.cluster_name}-nat-ip" - location = var.location - resource_group_name = var.resource_group_name - allocation_method = "Static" - sku = "Standard" -} - -resource "azurerm_nat_gateway" "nat" { - count = var.enable_nat_gateway ? 1 : 0 - name = "${var.cluster_name}-nat-gateway" - location = var.location - resource_group_name = var.resource_group_name -} - -resource "azurerm_nat_gateway_public_ip_association" "nat" { +module "nat_gateway" { + source = "../../../modules/azure/nat" count = var.enable_nat_gateway ? 1 : 0 - nat_gateway_id = azurerm_nat_gateway.nat[count.index].id - public_ip_address_id = azurerm_public_ip.nat[count.index].id -} - -resource "azurerm_route_table" "private_route_table" { - count = var.enable_nat_gateway ? 1 : 0 - name = "${var.cluster_name}-private-route-table" + cluster_name = var.cluster_name location = var.location resource_group_name = var.resource_group_name - - route { - name = "private-subnet-nat-route" - address_prefix = "0.0.0.0/0" - next_hop_type = "Internet" - next_hop_in_ip_address = azurerm_public_ip.nat[count.index].ip_address - } -} - -resource "azurerm_subnet_route_table_association" "private_subnet_route_association" { - count = var.enable_nat_gateway ? length(local.private_subnet_names) : 0 - subnet_id = module.vnet.vnet_subnets[1] # Assuming the module outputs subnet_ids - route_table_id = azurerm_route_table.private_route_table[0].id + private_subnet_ids = [module.vnet.vnet_subnets[1]] # or a list of actual subnet IDs } ################################################################################ @@ -219,35 +185,14 @@ module "derp" { # Azure Key Vault for Vault Auto-Unseal ################################################################################ -resource "azurerm_key_vault" "vault_auto_unseal" { +module "vault" { + source = "../../../modules/azure/vault" + count = var.create_vault_auto_unseal_key ? 1 : 0 - name = "${var.cluster_name}-vault-kv" + cluster_name = var.cluster_name location = var.location resource_group_name = var.resource_group_name tenant_id = data.azurerm_client_config.current.tenant_id - sku_name = "standard" - purge_protection_enabled = true - soft_delete_retention_days = 10 - enable_rbac_authorization = true -} - -resource "azurerm_key_vault_key" "vault_auto_unseal" { - count = var.create_vault_auto_unseal_key ? 1 : 0 - - name = "${var.cluster_name}-auto-unseal" - key_vault_id = azurerm_key_vault.vault_auto_unseal[0].id - key_type = "RSA" - key_size = 2048 - key_opts = ["decrypt", "encrypt", "sign", "verify", "wrapKey", "unwrapKey"] - - depends_on = [azurerm_key_vault.vault_auto_unseal] -} - -resource "azurerm_role_assignment" "vault_key_usage" { - count = var.create_vault_auto_unseal_key ? 1 : 0 - - principal_id = azuread_service_principal.sp[0].object_id - role_definition_name = "Key Vault Crypto User" - scope = azurerm_key_vault.vault_auto_unseal[0].id + service_principal_object_id = azuread_service_principal.sp[0].object_id } \ No newline at end of file diff --git a/terraform/examples/azure/base-cluster/outputs.tf b/terraform/examples/azure/base-cluster/outputs.tf index 25d2d8b0..5c53c39c 100644 --- a/terraform/examples/azure/base-cluster/outputs.tf +++ b/terraform/examples/azure/base-cluster/outputs.tf @@ -35,11 +35,11 @@ output "tenant_id" { } output "vault_key_name" { - value = var.create_vault_auto_unseal_key ? azurerm_key_vault_key.vault_auto_unseal[0].name : null + value = var.create_vault_auto_unseal_key ? module.vault[0].vault_key_name : null description = "The name of the Key Vault key." } output "vault_keyvault_name" { - value = var.create_vault_auto_unseal_key ? azurerm_key_vault.vault_auto_unseal[0].name : null + value = var.create_vault_auto_unseal_key ? module.vault[0].vault_keyvault_name: null description = "The name of the Key Vault instance." } diff --git a/terraform/modules/azure/nat/main.tf b/terraform/modules/azure/nat/main.tf new file mode 100644 index 00000000..59c68c2f --- /dev/null +++ b/terraform/modules/azure/nat/main.tf @@ -0,0 +1,37 @@ +resource "azurerm_public_ip" "nat" { + name = "${var.cluster_name}-nat-ip" + location = var.location + resource_group_name = var.resource_group_name + allocation_method = "Static" + sku = "Standard" +} + +resource "azurerm_nat_gateway" "nat" { + name = "${var.cluster_name}-nat-gateway" + location = var.location + resource_group_name = var.resource_group_name +} + +resource "azurerm_nat_gateway_public_ip_association" "nat" { + nat_gateway_id = azurerm_nat_gateway.nat.id + public_ip_address_id = azurerm_public_ip.nat.id +} + +resource "azurerm_route_table" "private_route_table" { + name = "${var.cluster_name}-private-route-table" + location = var.location + resource_group_name = var.resource_group_name + + route { + name = "private-subnet-nat-route" + address_prefix = "0.0.0.0/0" + next_hop_type = "Internet" + # next_hop_in_ip_address = azurerm_public_ip.nat.ip_address + } +} + +resource "azurerm_subnet_route_table_association" "private_subnet_route_association" { + count = length(var.private_subnet_ids) + subnet_id = var.private_subnet_ids[count.index] + route_table_id = azurerm_route_table.private_route_table.id +} diff --git a/terraform/modules/azure/nat/variables.tf b/terraform/modules/azure/nat/variables.tf new file mode 100644 index 00000000..033aa664 --- /dev/null +++ b/terraform/modules/azure/nat/variables.tf @@ -0,0 +1,19 @@ +variable "cluster_name" { + description = "Cluster name used in naming NAT resources" + type = string +} + +variable "location" { + description = "Azure region" + type = string +} + +variable "resource_group_name" { + description = "Resource group to deploy NAT resources" + type = string +} + +variable "private_subnet_ids" { + description = "List of private subnet IDs to associate with route table" + type = list(string) +} diff --git a/terraform/modules/azure/vault/main.tf b/terraform/modules/azure/vault/main.tf new file mode 100644 index 00000000..29bde908 --- /dev/null +++ b/terraform/modules/azure/vault/main.tf @@ -0,0 +1,26 @@ +resource "azurerm_key_vault" "vault_auto_unseal" { + name = "${var.cluster_name}-vault-kv" + location = var.location + resource_group_name = var.resource_group_name + tenant_id = var.tenant_id + sku_name = "standard" + purge_protection_enabled = true + soft_delete_retention_days = 10 + enable_rbac_authorization = true +} + +resource "azurerm_key_vault_key" "vault_auto_unseal" { + name = "${var.cluster_name}-auto-unseal" + key_vault_id = azurerm_key_vault.vault_auto_unseal.id + key_type = "RSA" + key_size = 2048 + key_opts = ["decrypt", "encrypt", "sign", "verify", "wrapKey", "unwrapKey"] + + depends_on = [azurerm_key_vault.vault_auto_unseal] +} + +resource "azurerm_role_assignment" "vault_key_usage" { + principal_id = var.service_principal_object_id + role_definition_name = "Key Vault Crypto User" + scope = azurerm_key_vault.vault_auto_unseal.id +} diff --git a/terraform/modules/azure/vault/outputs.tf b/terraform/modules/azure/vault/outputs.tf new file mode 100644 index 00000000..923deecd --- /dev/null +++ b/terraform/modules/azure/vault/outputs.tf @@ -0,0 +1,9 @@ +output "vault_key_name" { + value = azurerm_key_vault_key.vault_auto_unseal.name + description = "Name of the created Key Vault key for Vault auto-unseal" +} + +output "vault_keyvault_name" { + value = azurerm_key_vault.vault_auto_unseal.name + description = "Name of the created Key Vault for Vault auto-unseal" +} diff --git a/terraform/modules/azure/vault/variables.tf b/terraform/modules/azure/vault/variables.tf new file mode 100644 index 00000000..a26d4a8e --- /dev/null +++ b/terraform/modules/azure/vault/variables.tf @@ -0,0 +1,24 @@ +variable "cluster_name" { + description = "The name of the AKS or Vault cluster" + type = string +} + +variable "location" { + description = "Azure region" + type = string +} + +variable "resource_group_name" { + description = "Resource group name" + type = string +} + +variable "tenant_id" { + description = "Azure AD tenant ID" + type = string +} + +variable "service_principal_object_id" { + description = "Object ID of the service principal to assign key access to" + type = string +}