diff --git a/.github/workflows/azure-dsh-testing.yaml b/.github/workflows/azure-dsh-testing.yaml new file mode 100644 index 00000000..57c3a7ce --- /dev/null +++ b/.github/workflows/azure-dsh-testing.yaml @@ -0,0 +1,185 @@ +name: "[Azure] DevZero self-hosted deployment" + +on: + push: + branches: + - garvit/azure-tf + workflow_dispatch: + +jobs: + setup-and-test: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: 'Az CLI login' + uses: azure/login@v1.6.1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Set Azure env for Terraform providers + run: | + echo "ARM_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}" >> $GITHUB_ENV + echo "ARM_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}" >> $GITHUB_ENV + echo "ARM_SUBSCRIPTION_ID=${{ secrets.AZURE_SUBSCRIPTION_ID }}" >> $GITHUB_ENV + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.11.3" + + - name: Install yq + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/v4.35.2/yq_linux_amd64 -O /usr/local/bin/yq + sudo chmod +x /usr/local/bin/yq + + - name : Add SHORT_SHA Environment Variable + id : short-sha + shell: bash + run : echo "SHORT_SHA=`git rev-parse --short HEAD`" >> $GITHUB_ENV + + - name : Generate unique job identifier + id : job-identifier + shell: bash + run : | + SAFE_ID=$(echo "gha${SHORT_SHA}" | tr -cd 'a-z0-9' | cut -c1-20) + echo "JOB_IDENTIFIER=$SAFE_ID" >> $GITHUB_ENV + + - name: Add Backend Override (Base Cluster) + run: | + cd terraform/examples/azure/base-cluster + cat < backend_override.tf + terraform { + backend "azurerm" { + resource_group_name = "dev-test" + storage_account_name = "dshterraformstate" + container_name = "tfstate" + key = "${JOB_IDENTIFIER}/base-cluster/terraform.tfstate" + } + } + EOF + + - name: Initialize and Apply Terraform (Base Cluster) + run: | + cd terraform/examples/azure/base-cluster + terraform init + terraform apply -auto-approve -var="cluster_name=$JOB_IDENTIFIER" + + - name: Configure Kubernetes Access + run: | + az aks get-credentials --resource-group dev-test --name $JOB_IDENTIFIER + + - name: Set up Kata + run: | + cd terraform/examples/azure/base-cluster + kubectl apply -f kata-sa.yaml + kubectl apply -f daemonset.yaml + for NODE in $(kubectl get nodes -o name); do + kubectl label "$NODE" kata-runtime=running --overwrite + kubectl label "$NODE" node-role.kubernetes.io/kata-devpod-node=1 --overwrite + done + + - name: Deploy Control Plane Dependencies (and modify domains) + run: | + DEFAULT_SC=$(kubectl get sc -o jsonpath='{.items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")].metadata.name}') + kubectl get sc "$DEFAULT_SC" -o yaml | \ + sed "s/name: $DEFAULT_SC/name: gp2/" | \ + sed "/^ uid:/d; /^ resourceVersion:/d; /^ creationTimestamp:/d" | \ + kubectl apply -f - + + cd charts/dz-control-plane-deps + + find . -name "values.yaml" -print0 | while IFS= read -r -d '' file; do + yq e -i '.. |= select(tag == "!!str" and test("example\\.com")) |= sub("example\\.com"; env(JOB_IDENTIFIER) + ".ci.selfzero.net")' "$file" + done + + make install + + - name: Update values.yaml for dz-control-plane + env: + BACKEND_LICENSE_KEY: ${{ secrets.BACKEND_LICENSE_KEY }} + run: | + # setting credentials enable to false since we will explicitly feed the dockerhub creds to kubernetes api + # also setting image.pullsecrets to empty to make sure that each of the deployments dont try to pull their relevant OCI images from this registry + # backend license key is ... needed + + yq e '.credentials.enable = false | .backend.licenseKey = strenv(BACKEND_LICENSE_KEY) | .image.pullSecrets = []' -i charts/dz-control-plane/values.yaml + + - name: Deploy DevZero Control Plane (after configuring kubernetes to use dockerhub creds, and patching all the deployments to point to the right domain) + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + cd charts/dz-control-plane + make add-docker-creds + + find . -name "values.yaml" -print0 | while IFS= read -r -d '' file; do + yq e -i '.. |= select(tag == "!!str" and test("example\\.com")) |= sub("example\\.com"; env(JOB_IDENTIFIER) + ".ci.selfzero.net")' "$file" + done + + make install + + - name: Validate Control Plane + run: | + .github/scripts/dsh-pod-test.sh + + - name: Deploy Data Plane Dependencies + run: | + cd charts/dz-data-plane-deps + + find . -name "values.yaml" -print0 | while IFS= read -r -d '' file; do + yq e -i '.. |= select(tag == "!!str" and test("example\\.com")) |= sub("example\\.com"; env(JOB_IDENTIFIER) + ".ci.selfzero.net")' "$file" + done + + make install + + - name: Deploy DevZero Data Plane + run: | + cd charts/dz-data-plane + + find . -name "values.yaml" -print0 | while IFS= read -r -d '' file; do + yq e -i '.. |= select(tag == "!!str" and test("example\\.com")) |= sub("example\\.com"; env(JOB_IDENTIFIER) + ".ci.selfzero.net")' "$file" + done + + make install + + - name: Validate Data Plane + run: | + kubectl get pods -n devzero-self-hosted + kubectl get ingress -n devzero-self-hosted + + - name: '[helm] Destroy data-plane' + if: always() + run: | + cd charts/dz-data-plane + make delete + + - name: '[helm] Destroy data-plane-deps' + if: always() + run: | + cd charts/dz-data-plane-deps + make delete + + - name: '[helm] Destroy control-plane' + if: always() + run: | + cd charts/dz-control-plane + make delete + + - name: '[helm] Destroy control-plane-deps' + if: always() + run: | + cd charts/dz-control-plane-deps + make delete + + - name: '[terraform] Destroy base-cluster' + if: always() + run: | + cd terraform/examples/azure/base-cluster + terraform destroy -auto-approve -var="cluster_name=$JOB_IDENTIFIER" \ No newline at end of file diff --git a/charts/dz-control-plane-deps/values/vault.yaml b/charts/dz-control-plane-deps/values/vault.yaml index f86044df..62e2c0f6 100644 --- a/charts/dz-control-plane-deps/values/vault.yaml +++ b/charts/dz-control-plane-deps/values/vault.yaml @@ -19,6 +19,21 @@ server: dataStorage: enabled: true + # The following is an example of how to configure the Vault server to use Azure Key Vault for auto-unsealing. + # Before using this configuration, you need to create a secret in the Kubernetes cluster that contains the Azure Key Vault credentials: + # kubectl create secret generic vault-azure-creds --from-literal=AZURE_TENANT_ID= --from-literal=AZURE_CLIENT_ID= --from-literal=AZURE_CLIENT_SECRET= -n devzero + + # extraSecretEnvironmentVars: + # - envName: AZURE_CLIENT_ID + # secretName: vault-azure-creds + # secretKey: AZURE_CLIENT_ID + # - envName: AZURE_CLIENT_SECRET + # secretName: vault-azure-creds + # secretKey: AZURE_CLIENT_SECRET + # - envName: AZURE_TENANT_ID + # secretName: vault-azure-creds + # secretKey: AZURE_TENANT_ID + # Disable vault anti-affinity that requires one pod on each node. This allows vault to run on a single node cluster. affinity: "" @@ -71,6 +86,15 @@ server: # key_ring = "GCP_KEY_RING" # crypto_key = "GCP_CRYPTO_KEY" # } + + # This is used to configure the Vault server to use Azure Key Vault for auto-unsealing. + # seal "azurekeyvault" { + # tenant_id = "" + # client_id = "" + # client_secret = "" + # vault_name = "" + # key_name = "" + # } ingress: enabled: true diff --git a/terraform/examples/azure/README.md b/terraform/examples/azure/README.md new file mode 100644 index 00000000..b3198e65 --- /dev/null +++ b/terraform/examples/azure/README.md @@ -0,0 +1,117 @@ +# DevZero Self-Hosted - Terraform Setup - Azure + +This document provides a step-by-step guide for setting up the infrastructure required to self-host the DevZero Control Plane and Data Plane using Terraform. The infrastructure can be deployed on cloud platforms like Azure. + +## Pre-reading + +For readers experienced with Terraform deployments at their companies, we have some examples under [./examples](./examples/) that you can reference to see how to run a full DevZero deployment. +If you have your own terraform environment and want to reuse our modules, you can refer to the [./modules](./modules/) directory to use whichever components you need. + +## Overview + +The `terraform/` directory contains Infrastructure as Code (IaC) configurations that automate the provisioning of essential cloud resources such as VNet, AKS clusters, load balancers, and VPNs. + +## Prerequisites + +### Tools Required +- [Terraform](https://www.terraform.io/) (for managing infrastructure as code) +- [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) (for interacting with Azure resources) +- Access credentials for your Azure Account + +### Permissions Required +Ensure your IAM user or service account has sufficient permissions to create resources like VNet, subnets, AKS clusters, VMs, and Key Vault. + +## Infrastructure Setup Guide + +### 1. Clone the Repository + +```bash +git clone https://github.com/devzero-inc/self-hosted.git +``` + +### 2. Navigate to the Base Cluster Directory + +```bash +cd self-hosted/terraform/examples/azure/base-cluster +``` + +### 3. Configure Terraform Variables + +Update `terraform.tfvars`. + +#### Cluster Endpoint Access +- Set `cluster_endpoint_public_access = true` to allow public access. +- Set it to `false` for private access. + +### 4. Initialise and Apply Terraform + +```bash +terraform init +terraform apply +``` + +- This will create Azure resources such as VNet, AKS, VM, Key Vault, etc. +- Copy the output values like subscription_id, resource_group_name, location, and cluster_name for the next steps. + +## Extending the Cluster + +### 5. Navigate to the Cluster Extensions Directory + +```bash +cd ../cluster-extensions +``` + +### 6. Update `terraform.tfvars` + +- Add the subscription_id, resource_group_name, location, and cluster_name from the previous step. + +### 7. Apply Terraform for Storage + +```bash +terraform init +terraform apply +``` + +- This will create StorageClasses, and Azure Files. + +## Post-Deployment Steps + +### 8. Update kubeconfig + +```bash +az aks get-credentials --resource-group --name +``` + +### 9. Install Kata in AKS Node + +```bash +kubectl apply -f kata-sa.yaml +kubectl apply -f daemonset.yaml +``` + +### 10. Add the Labels + +```bash +kubectl get nodes +kubectl label node kata-runtime=running +kubectl label node node-role.kubernetes.io/kata-devpod-node=1 +``` + +Or you can automatically label all your nodes like this: + +```bash +for NODE in $(kubectl get nodes -o name); do + kubectl label "$NODE" kata-runtime=running --overwrite + kubectl label "$NODE" node-role.kubernetes.io/kata-devpod-node=1 --overwrite +done +``` + +### 11. Install DevZero Self-Hosted + +Refer to the [Charts README](../charts/README.md) for further steps to deploy the Control Plane and Data Plane. + +## Troubleshooting + +- Verify cloud credentials and permissions. +- Check Terraform state files for resource management. +- Use `terraform plan` to preview changes before applying. diff --git a/terraform/examples/azure/base-cluster/daemonset.yaml b/terraform/examples/azure/base-cluster/daemonset.yaml new file mode 100644 index 00000000..1953569f --- /dev/null +++ b/terraform/examples/azure/base-cluster/daemonset.yaml @@ -0,0 +1,107 @@ +# This is a DaemonSet configuration for deploying Kata Containers on Kubernetes nodes. Find it here: https://github.com/kata-containers/kata-containers/blob/main/tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kata-deploy + namespace: kube-system +spec: + selector: + matchLabels: + name: kata-deploy + template: + metadata: + labels: + name: kata-deploy + spec: + serviceAccountName: kata-deploy-sa + hostPID: true + containers: + - name: kube-kata + image: quay.io/kata-containers/kata-deploy:latest + imagePullPolicy: Always + lifecycle: + preStop: + exec: + command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh cleanup"] + command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh install"] + # NOTE: Please don't change the order of the environment variables below. + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: DEBUG + value: "false" + - name: SHIMS + value: "cloud-hypervisor" + - name: DEFAULT_SHIM + value: "cloud-hypervisor" + - name: CREATE_RUNTIMECLASSES + value: "false" + - name: CREATE_DEFAULT_RUNTIMECLASS + value: "false" + - name: ALLOWED_HYPERVISOR_ANNOTATIONS + value: "" + - name: SNAPSHOTTER_HANDLER_MAPPING + value: "" + - name: AGENT_HTTPS_PROXY + value: "" + - name: AGENT_NO_PROXY + value: "" + - name: PULL_TYPE_MAPPING + value: "" + - name: INSTALLATION_PREFIX + value: "" + - name: MULTI_INSTALL_SUFFIX + value: "" + securityContext: + privileged: true + volumeMounts: + - name: crio-conf + mountPath: /etc/crio/ + - name: containerd-conf + mountPath: /etc/containerd/ + - name: host + mountPath: /host/ + - name: patch-containerd-handler + image: alpine:3.18 + securityContext: + privileged: true + volumeMounts: + - name: host + mountPath: /host + command: + - /bin/sh + - -c + - | + apk add --no-cache util-linux + + echo "Waiting for kata-cloud-hypervisor config block..." + while ! grep -q '\[plugins.*kata-cloud-hypervisor\]' /host/etc/containerd/config.toml; do + sleep 2 + done + + echo "Patching config.toml to rename kata-cloud-hypervisor → kata..." + sed -i 's/kata-cloud-hypervisor/kata/g' /host/etc/containerd/config.toml + + echo "Restarting containerd..." + nsenter --target 1 --mount --uts --ipc --net --pid systemctl restart containerd + + # exit to let the sidecar terminate + echo "Patch done. Exiting sidecar." + volumes: + - name: crio-conf + hostPath: + path: /etc/crio/ + - name: containerd-conf + hostPath: + path: /etc/containerd/ + - name: host + hostPath: + path: / + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate \ No newline at end of file diff --git a/terraform/examples/azure/base-cluster/kata-sa.yaml b/terraform/examples/azure/base-cluster/kata-sa.yaml new file mode 100644 index 00000000..6b0c1308 --- /dev/null +++ b/terraform/examples/azure/base-cluster/kata-sa.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kata-deploy-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole + + +metadata: + name: kata-deploy-role +rules: +- apiGroups: [""] + resources: ["serviceaccounts", "pods", "namespaces", "nodes"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["clusterroles", "clusterrolebindings"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +- apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kata-deploy-rb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kata-deploy-role +subjects: +- kind: ServiceAccount + name: kata-deploy-sa + namespace: kube-system diff --git a/terraform/examples/azure/base-cluster/main.tf b/terraform/examples/azure/base-cluster/main.tf new file mode 100644 index 00000000..ff1a351a --- /dev/null +++ b/terraform/examples/azure/base-cluster/main.tf @@ -0,0 +1,198 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.106.1, < 4.0.0" + } + azapi = { + source = "azure/azapi" + version = ">= 1.15.0, < 2.0.0" + } + } +} + +provider "azurerm" { + features {} +} + +provider "azuread" {} + +provider "azapi" { + use_oidc = true +} + +locals { + public_subnet_names = ["public-subnet-1"] + private_subnet_names = ["private-subnet-1"] + gateway_subnet_names = var.create_vpn ? ["GatewaySubnet"] : [] + + calculated_public_subnets_cidrs = ["10.240.1.0/24"] + calculated_private_subnets_cidrs = ["10.240.101.0/24"] + gateway_subnet_cidrs = var.create_vpn ? ["10.240.255.0/27"] : [] +} + +data "azurerm_client_config" "current" {} + +resource "azuread_application" "sp" { + count = (var.create_vault_auto_unseal_key || var.create_vpn) ? 1 : 0 + display_name = "${var.cluster_name}-devzero-app" +} + +resource "azuread_service_principal" "sp" { + count = (var.create_vault_auto_unseal_key || var.create_vpn) ? 1 : 0 + client_id = azuread_application.sp[0].client_id +} + +resource "azuread_application_password" "sp" { + count = (var.create_vault_auto_unseal_key || var.create_vpn) ? 1 : 0 + application_id = azuread_application.sp[0].id + display_name = "DevZero SP Password" + end_date_relative = "8760h" +} + +################################################################################ +# VNET +################################################################################ + +module "vnet" { + source = "Azure/network/azurerm" + version = "5.3.0" + + resource_group_name = var.resource_group_name + use_for_each = false + + vnet_name = "${var.cluster_name}-vnet" + address_space = var.cidr + + subnet_names = concat(local.public_subnet_names, local.private_subnet_names, local.gateway_subnet_names) + subnet_prefixes = concat( + local.calculated_public_subnets_cidrs, + local.calculated_private_subnets_cidrs, + local.gateway_subnet_cidrs + ) + + subnet_enforce_private_link_endpoint_network_policies = { + for s in concat(local.public_subnet_names, local.private_subnet_names) : s => false + } + + tags = var.tags +} + +################################################################################ +# NAT Gateway +################################################################################ +module "nat_gateway" { + source = "../../../modules/azure/nat" + count = var.enable_nat_gateway ? 1 : 0 + cluster_name = var.cluster_name + location = var.location + resource_group_name = var.resource_group_name + private_subnet_ids = [module.vnet.vnet_subnets[1]] # or a list of actual subnet IDs +} + +################################################################################ +# AKS CLUSTER +################################################################################ + +module "aks" { + source = "Azure/aks/azurerm" + version = "9.4.1" + + resource_group_name = var.resource_group_name + location = var.location + cluster_name = var.cluster_name + kubernetes_version = var.cluster_version + prefix = var.cluster_name + + vnet_subnet_id = module.vnet.vnet_subnets[0] + network_plugin = "azure" + network_policy = "azure" + private_cluster_enabled = var.enable_private_cluster + api_server_authorized_ip_ranges = [] + + identity_type = "SystemAssigned" + + rbac_aad = var.enable_rbac + rbac_aad_managed = var.enable_rbac + rbac_aad_admin_group_object_ids = var.enable_rbac ? var.admin_group_object_ids : null + role_based_access_control_enabled = var.enable_rbac + + log_analytics_workspace_enabled = true + log_retention_in_days = 30 + cluster_log_analytics_workspace_name = "${var.cluster_name}-logs" + + key_vault_secrets_provider_enabled = true + secret_rotation_enabled = true + secret_rotation_interval = "2m" + + agents_pool_name = "systemnp" + agents_size = var.instance_type + enable_auto_scaling = var.enable_cluster_autoscaler + agents_min_count = var.enable_cluster_autoscaler ? var.min_size : null + agents_max_count = var.enable_cluster_autoscaler ? var.max_size : null + agents_count = var.enable_cluster_autoscaler ? null : var.node_count + temporary_name_for_rotation = "rotation" + agents_max_pods = 100 + + tags = var.tags +} + +################################################################################ +# VPN +################################################################################ + +module "vpn" { + count = var.create_vpn ? 1 : 0 + + source = "../../../modules/azure/vpn" + + name = var.cluster_name + location = var.location + resource_group_name = var.resource_group_name + gateway_subnet_id = module.vnet.vnet_subnets[2] + vpn_client_cidr = "172.16.0.0/24" + vpn_client_list = var.vpn_client_list + tenant_id = data.azurerm_client_config.current.tenant_id + sp_object_id = azuread_service_principal.sp[0].object_id + + depends_on = [module.vnet] +} + +################################################################################ +# Custom DERP server in Azure +################################################################################ + +module "derp" { + source = "../../../modules/azure/derp" + + count = var.create_derp ? 1 : 0 + + name_prefix = var.cluster_name + location = var.location + resource_group_name = var.resource_group_name + subnet_id = module.vnet.vnet_subnets[0] + public_derp = var.public_derp + existing_ip = "" + firewall_rule_name = "" + ingress_cidr_blocks = ["0.0.0.0/0"] + hostname = "derp.devzero.net" + vm_size = "Standard_B2s" + volume_size = 30 + tags = var.tags +} + +################################################################################ +# Azure Key Vault for Vault Auto-Unseal +################################################################################ + +module "vault" { + source = "../../../modules/azure/vault" + + count = var.create_vault_auto_unseal_key ? 1 : 0 + + cluster_name = var.cluster_name + location = var.location + resource_group_name = var.resource_group_name + tenant_id = data.azurerm_client_config.current.tenant_id + service_principal_object_id = azuread_service_principal.sp[0].object_id +} \ No newline at end of file diff --git a/terraform/examples/azure/base-cluster/outputs.tf b/terraform/examples/azure/base-cluster/outputs.tf new file mode 100644 index 00000000..5c53c39c --- /dev/null +++ b/terraform/examples/azure/base-cluster/outputs.tf @@ -0,0 +1,45 @@ +output "subscription_id" { + description = "Azure Subscription ID" + value = data.azurerm_client_config.current.subscription_id +} + +output "resource_group_name" { + description = "The name of the resource group used for the AKS cluster." + value = var.resource_group_name +} + +output "cluster_name" { + description = "The name of the AKS cluster." + value = var.cluster_name +} + +output "location" { + description = "The Azure region where the AKS cluster is deployed." + value = var.location +} + +output "sp_client_id" { + value = (var.create_vault_auto_unseal_key || var.create_vpn) ? azuread_application.sp[0].client_id : null + description = "The client ID for Vault's Azure Key Vault integration." +} + +output "sp_client_secret" { + value = (var.create_vault_auto_unseal_key || var.create_vpn) ? azuread_application_password.sp[0].value : null + description = "The client secret for Vault's Azure Key Vault integration." + sensitive = true +} + +output "tenant_id" { + value = (var.create_vault_auto_unseal_key || var.create_vpn) ? data.azurerm_client_config.current.tenant_id : null + description = "The Azure Tenant ID." +} + +output "vault_key_name" { + value = var.create_vault_auto_unseal_key ? module.vault[0].vault_key_name : null + description = "The name of the Key Vault key." +} + +output "vault_keyvault_name" { + value = var.create_vault_auto_unseal_key ? module.vault[0].vault_keyvault_name: null + description = "The name of the Key Vault instance." +} diff --git a/terraform/examples/azure/base-cluster/terraform.tfvars b/terraform/examples/azure/base-cluster/terraform.tfvars new file mode 100644 index 00000000..6693307c --- /dev/null +++ b/terraform/examples/azure/base-cluster/terraform.tfvars @@ -0,0 +1,36 @@ +resource_group_name = "dev-test" +location = "eastus" +cluster_name = "devzero" +cluster_version = "1.30" +cidr = "10.240.0.0/16" + +cluster_endpoint_public_access = true +enable_private_cluster = false +enable_rbac = false +admin_group_object_ids = [ + "00000000-0000-0000-0000-000000000000" +] + +instance_type = "Standard_D8s_v3" +enable_cluster_autoscaler = false +node_count = 2 +min_size = 1 +max_size = 3 + +enable_nat_gateway = false +single_nat_gateway = false + +# Optional tagging + +tags = { + environment = "dev" + owner = "devzero" + createdBy = "terraform" +} + +create_derp = false +public_derp = false + +create_vault_auto_unseal_key = false + +create_vpn = false \ No newline at end of file diff --git a/terraform/examples/azure/base-cluster/variables.tf b/terraform/examples/azure/base-cluster/variables.tf new file mode 100644 index 00000000..d02b51ce --- /dev/null +++ b/terraform/examples/azure/base-cluster/variables.tf @@ -0,0 +1,137 @@ +variable "resource_group_name" { + description = "Name of the resource group in which resources will be created" + type = string +} + +variable "location" { + description = "Azure region where the resources will be deployed" + type = string +} + +variable "cluster_name" { + description = "Name of the AKS cluster" + type = string +} + +variable "cluster_version" { + description = "Kubernetes version to use for the AKS cluster" + type = string +} + +variable "cidr" { + description = "CIDR block for the virtual network" + type = string +} + +variable "cluster_endpoint_public_access" { + description = "Whether the AKS API server should be publicly accessible" + type = bool +} + +variable "cluster_endpoint_public_access_cidrs" { + description = "List of CIDRs allowed to access the API server if public access is enabled" + type = list(string) + default = [] +} + +variable "enable_rbac" { + type = bool +} + +variable "enable_private_cluster" { + type = bool +} + +variable "admin_group_object_ids" { + description = "List of AAD group object IDs with admin access to the cluster" + type = list(string) +} + +variable "instance_type" { + description = "VM size for the default node pool" + type = string +} + +variable "enable_cluster_autoscaler" { + type = bool +} + + +variable "node_count" { + type = number +} + +variable "min_size" { + description = "Minimum number of nodes in the default node pool" + type = number +} + +variable "max_size" { + description = "Maximum number of nodes in the default node pool" + type = number +} + +variable "desired_size" { + description = "Desired number of nodes in the default node pool (ignored when autoscaling is enabled)" + type = number + default = null +} + +variable "enable_nat_gateway" { + description = "Whether to enable NAT Gateway" + type = bool + default = true +} + +variable "single_nat_gateway" { + description = "Whether to use a single NAT gateway for all subnets" + type = bool + default = false +} + +variable "tags" { + description = "Tags to apply to all resources" + type = map(string) + default = {} +} + +################################################################################ +# Custom DERP server +################################################################################ +variable "create_derp" { + description = "Create custom DERP server" + type = bool + default = false +} + +variable "public_derp" { + type = bool + default = false + description = "Whether to make the DERP server public" +} + +################################################################################ +# Vault +################################################################################ + +variable "create_vault_auto_unseal_key" { + description = "Whether or not to create a KMS key for Vault auto unseal" + type = bool + default = false +} + +################################################################################ +# Vault +################################################################################ + +variable "create_vpn" { + description = "Controls if VPN gateway and VPN resources will be created." + type = bool + default = false +} + +variable "vpn_client_list" { + description = "Subnets" + type = set(string) + default = ["root"] +} \ No newline at end of file diff --git a/terraform/examples/azure/cluster-extensions/main.tf b/terraform/examples/azure/cluster-extensions/main.tf new file mode 100644 index 00000000..06d20998 --- /dev/null +++ b/terraform/examples/azure/cluster-extensions/main.tf @@ -0,0 +1,52 @@ +################################################################################ +# Providers +################################################################################ + +provider "azurerm" { + features {} + subscription_id = var.subscription_id +} + +provider "azuread" {} + +data "azurerm_client_config" "current" {} + +provider "kubernetes" { + host = data.azurerm_kubernetes_cluster.this.kube_config[0].host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate) +} + +provider "helm" { + kubernetes { + host = data.azurerm_kubernetes_cluster.this.kube_config[0].host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate) + } +} + +################################################################################ +# AKS Cluster Data +################################################################################ + +data "azurerm_kubernetes_cluster" "this" { + name = var.cluster_name + resource_group_name = var.resource_group_name +} + +################################################################################ +# Cluster Extensions Module (Azure parity) +################################################################################ + +module "cluster_extensions" { + source = "../../../modules/azure/cluster_extensions" + + cluster_name = var.cluster_name + resource_group_name = var.resource_group_name + location = var.location + enable_external_secrets = var.enable_external_secrets + enable_azure_files = var.enable_azure_files + tags = var.tags +} \ No newline at end of file diff --git a/terraform/examples/azure/cluster-extensions/terraform.tfvars b/terraform/examples/azure/cluster-extensions/terraform.tfvars new file mode 100644 index 00000000..3003f9c2 --- /dev/null +++ b/terraform/examples/azure/cluster-extensions/terraform.tfvars @@ -0,0 +1,13 @@ +subscription_id = "a32b188c-43fd-4e28-8f67-c95649ab3119" +cluster_name = "devzero" +resource_group_name = "dev-test" +location = "eastus" + +enable_external_secrets = false +enable_azure_files = true + +tags = { + environment = "dev" + owner = "devzero" + createdBy = "terraform" +} diff --git a/terraform/examples/azure/cluster-extensions/variables.tf b/terraform/examples/azure/cluster-extensions/variables.tf new file mode 100644 index 00000000..93bf60a9 --- /dev/null +++ b/terraform/examples/azure/cluster-extensions/variables.tf @@ -0,0 +1,37 @@ +variable "subscription_id" { + type = string + description = "Azure Subscription ID" +} + +variable "cluster_name" { + description = "The name of the AKS cluster." + type = string +} + +variable "resource_group_name" { + description = "The resource group containing the AKS cluster." + type = string +} + +variable "location" { + description = "Azure region where resources will be created." + type = string +} + +variable "enable_external_secrets" { + description = "Whether to deploy the External Secrets Operator." + type = bool + default = false +} + +variable "enable_azure_files" { + description = "Whether to enable and set up the Azure Files CSI driver with default StorageClass." + type = bool + default = true +} + +variable "tags" { + description = "Tags to apply to all resources." + type = map(string) + default = {} +} diff --git a/terraform/examples/gcp/README.md b/terraform/examples/gcp/README.md index 1ecb7bdd..6cb55db1 100644 --- a/terraform/examples/gcp/README.md +++ b/terraform/examples/gcp/README.md @@ -9,7 +9,7 @@ If you have your own terraform environment and want to reuse our modules, you ca ## Overview -The `terraform/` directory contains Infrastructure as Code (IaC) configurations that automate the provisioning of essential cloud resources such as VPCs, EKS clusters, load balancers, and VPNs. +The `terraform/` directory contains Infrastructure as Code (IaC) configurations that automate the provisioning of essential cloud resources such as VPCs, GKE clusters, load balancers, and VPNs. ## Prerequisites @@ -37,14 +37,7 @@ cd self-hosted/terraform/examples/gcp/base-cluster ### 3. Configure Terraform Variables -#### Using an Existing VPC -- Open the `terraform.tfvars` file. -- Update the VPC ID and subnet IDs. -- Set `create_vpc = false` to prevent Terraform from creating a new VPC. - -#### Let Terraform Create a New VPC -- Skip the above step. -- Ensure `create_vpc = true` (default setting). +Update `terraform.tfvars`. #### Cluster Endpoint Access - Set `cluster_endpoint_public_access = true` to allow public access. @@ -117,15 +110,6 @@ done Refer to the [Charts README](../charts/README.md) for further steps to deploy the Control Plane and Data Plane. -### 12. Update Kata Runtime - -After DSH installation, delete the default kata runtimeclass and create a new one: - -```bash -kubectl delete runtimeclass kata -kubectl apply -f runtimeclass.yaml -``` - ## Troubleshooting - Verify cloud credentials and permissions. diff --git a/terraform/modules/azure/cluster_extensions/main.tf b/terraform/modules/azure/cluster_extensions/main.tf new file mode 100644 index 00000000..973be920 --- /dev/null +++ b/terraform/modules/azure/cluster_extensions/main.tf @@ -0,0 +1,112 @@ +################################################################################ +# AKS Cluster Details +################################################################################ + +data "azurerm_kubernetes_cluster" "this" { + name = var.cluster_name + resource_group_name = var.resource_group_name +} + +provider "kubernetes" { + host = data.azurerm_kubernetes_cluster.this.kube_config[0].host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate) +} + +provider "helm" { + kubernetes { + host = data.azurerm_kubernetes_cluster.this.kube_config[0].host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate) + } +} + +################################################################################ +# External Secrets Operator +################################################################################ + +resource "helm_release" "external_secrets" { + count = var.enable_external_secrets ? 1 : 0 + + name = "external-secrets" + namespace = "external-secrets" + repository = "https://charts.external-secrets.io" + chart = "external-secrets" + version = "0.9.13" + + create_namespace = true + + set { + name = "installCRDs" + value = "true" + } +} + +################################################################################ +# Azure Files Backing Resources +################################################################################ + +resource "azurerm_storage_account" "azure_files" { + count = var.enable_azure_files ? 1 : 0 + + name = lower(replace("${var.cluster_name}files", "[^a-z0-9]", "")) + resource_group_name = var.resource_group_name + location = var.location + account_tier = "Standard" + account_replication_type = "LRS" + account_kind = "StorageV2" + + tags = var.tags +} + +resource "azurerm_storage_share" "efs_etcd_share" { + count = var.enable_azure_files ? 1 : 0 + name = "efs-etcd" + storage_account_name = azurerm_storage_account.azure_files[0].name + quota = 100 +} + +resource "kubernetes_secret" "azure_files_secret" { + count = var.enable_azure_files ? 1 : 0 + + metadata { + name = "azure-files-secret" + namespace = "default" + } + + data = { + azurestorageaccountname = azurerm_storage_account.azure_files[0].name + azurestorageaccountkey = azurerm_storage_account.azure_files[0].primary_access_key + } + + type = "Opaque" +} + +################################################################################ +# Azure Files CSI StorageClass (named as `efs-etcd`) +################################################################################ + +resource "kubernetes_storage_class" "azurefiles_csi" { + count = var.enable_azure_files ? 1 : 0 + + metadata { + name = "efs-etcd" + annotations = { + "storageclass.kubernetes.io/is-default-class" = "true" + } + } + + storage_provisioner = "file.csi.azure.com" + reclaim_policy = "Delete" + volume_binding_mode = "WaitForFirstConsumer" + allow_volume_expansion = true + + parameters = { + skuName = "Standard_LRS" + secretName = "azure-files-secret" + secretNamespace = "default" + shareName = "efs-etcd" + } +} diff --git a/terraform/modules/azure/cluster_extensions/variables.tf b/terraform/modules/azure/cluster_extensions/variables.tf new file mode 100644 index 00000000..a6ed7385 --- /dev/null +++ b/terraform/modules/azure/cluster_extensions/variables.tf @@ -0,0 +1,15 @@ +variable "cluster_name" {} +variable "resource_group_name" {} +variable "location" {} +variable "enable_external_secrets" { + type = bool + default = false +} +variable "enable_azure_files" { + type = bool + default = false +} +variable "tags" { + type = map(string) + default = {} +} \ No newline at end of file diff --git a/terraform/modules/azure/derp/derp-init.tpl b/terraform/modules/azure/derp/derp-init.tpl new file mode 100644 index 00000000..1ad67660 --- /dev/null +++ b/terraform/modules/azure/derp/derp-init.tpl @@ -0,0 +1,54 @@ +#!/bin/bash +set -euxo pipefail +exec > /var/log/derp-init.log 2>&1 + +# Install dependencies +apt-get update +apt-get install -y curl git gcc make openssl certbot + +# Install Go (use official binary) +wget -q https://go.dev/dl/go1.21.6.linux-amd64.tar.gz +rm -rf /usr/local/go +tar -C /usr/local -xzf go1.21.6.linux-amd64.tar.gz + +# Set up Go env +export PATH=$PATH:/usr/local/go/bin +export GOPATH=/root/go +export PATH=$PATH:$GOPATH/bin +export HOME=/root +export GOCACHE=$HOME/.cache/go-build + +# Install derper +go install tailscale.com/cmd/derper@main +cp $GOPATH/bin/derper /usr/bin/ + +%{ if !public_derp } +export DERP_PRIVATE_IP=$(curl -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip) +mkdir -p /etc/derper +openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \ + -keyout /etc/derper/$DERP_PRIVATE_IP.key \ + -out /etc/derper/$DERP_PRIVATE_IP.crt \ + -subj "/CN=$DERP_PRIVATE_IP" \ + -addext "subjectAltName=IP:$DERP_PRIVATE_IP" +%{ endif } + +# Create systemd service +cat < /etc/systemd/system/derper.service +[Unit] +Description=Devzero DERP Server +After=network-online.target +Wants=network-online.target + +[Service] +User=root +ExecStart=/usr/bin/derper %{ if public_derp }-hostname ${hostname}%{ else }-hostname \$DERP_PRIVATE_IP -certmode manual -certdir /etc/derper/ %{ endif } +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOT + +systemctl daemon-reload +systemctl enable derper +systemctl start derper diff --git a/terraform/modules/azure/derp/main.tf b/terraform/modules/azure/derp/main.tf new file mode 100644 index 00000000..72b8d0b3 --- /dev/null +++ b/terraform/modules/azure/derp/main.tf @@ -0,0 +1,129 @@ +locals { + create_firewall_rule = var.firewall_rule_name == "" + create_static_ip = var.public_derp && var.existing_ip == "" + + ip_address = local.create_static_ip ? azurerm_public_ip.derp_ip[0].ip_address : var.existing_ip +} + +# Static Public IP (only if needed) +resource "azurerm_public_ip" "derp_ip" { + count = local.create_static_ip ? 1 : 0 + name = "${var.name_prefix}-derp-ip" + location = var.location + resource_group_name = var.resource_group_name + allocation_method = "Static" + sku = "Standard" +} + +# Network Security Group +resource "azurerm_network_security_group" "derp_nsg" { + count = local.create_firewall_rule ? 1 : 0 + name = "${var.name_prefix}-derp-nsg" + location = var.location + resource_group_name = var.resource_group_name + + security_rule { + name = "Allow-UDP-3478" + priority = 1001 + direction = "Inbound" + access = "Allow" + protocol = "Udp" + source_port_range = "*" + destination_port_range = "3478" + source_address_prefixes = var.ingress_cidr_blocks + destination_address_prefix = "*" + } + + security_rule { + name = "Allow-TCP-443" + priority = 1002 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefixes = var.ingress_cidr_blocks + destination_address_prefix = "*" + } + + security_rule { + name = "Allow-SSH" + priority = 1003 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "22" + source_address_prefixes = var.ingress_cidr_blocks + destination_address_prefix = "*" + } +} + +# Network Interface +resource "azurerm_network_interface" "derp_nic" { + name = "${var.name_prefix}-nic" + location = var.location + resource_group_name = var.resource_group_name + + ip_configuration { + name = "internal" + subnet_id = var.subnet_id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = local.create_static_ip ? azurerm_public_ip.derp_ip[0].id : null + } + + tags = var.tags +} + +# NSG Association +resource "azurerm_network_interface_security_group_association" "derp_nsg_assoc" { + count = local.create_firewall_rule ? 1 : 0 + network_interface_id = azurerm_network_interface.derp_nic.id + network_security_group_id = azurerm_network_security_group.derp_nsg[0].id +} + +# SSH Key +resource "tls_private_key" "ssh_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +# VM +resource "azurerm_linux_virtual_machine" "derp_vm" { + name = "${var.name_prefix}-derp-vm" + resource_group_name = var.resource_group_name + location = var.location + size = var.vm_size + admin_username = "ubuntu" + network_interface_ids = [ + azurerm_network_interface.derp_nic.id, + ] + + disable_password_authentication = true + + admin_ssh_key { + username = "ubuntu" + public_key = tls_private_key.ssh_key.public_key_openssh + } + + source_image_reference { + publisher = "Canonical" + offer = "0001-com-ubuntu-server-jammy" + sku = "22_04-lts" + version = "latest" + } + + os_disk { + caching = "ReadWrite" + storage_account_type = "Premium_LRS" + disk_size_gb = var.volume_size + name = "${var.name_prefix}-osdisk" + } + + custom_data = base64encode(templatefile("${path.module}/derp-init.tpl", { + hostname = var.hostname + public_derp = var.public_derp + })) + + tags = var.tags +} diff --git a/terraform/modules/azure/derp/variables.tf b/terraform/modules/azure/derp/variables.tf new file mode 100644 index 00000000..ffb89976 --- /dev/null +++ b/terraform/modules/azure/derp/variables.tf @@ -0,0 +1,50 @@ +variable "resource_group_name" { + type = string +} + +variable "location" { + type = string +} + +variable "subnet_id" { + type = string + description = "Use subnet ID from the shared VNet module" +} + +variable "firewall_rule_name" { + type = string + default = "" +} + +variable "public_derp" { + type = bool +} + +variable "existing_ip" { + type = string + default = "" +} + +variable "name_prefix" { + type = string +} + +variable "ingress_cidr_blocks" { + type = list(string) +} + +variable "hostname" { + type = string +} + +variable "tags" { + type = map(string) +} + +variable "vm_size" { + type = string +} + +variable "volume_size" { + type = number +} diff --git a/terraform/modules/azure/nat/main.tf b/terraform/modules/azure/nat/main.tf new file mode 100644 index 00000000..59c68c2f --- /dev/null +++ b/terraform/modules/azure/nat/main.tf @@ -0,0 +1,37 @@ +resource "azurerm_public_ip" "nat" { + name = "${var.cluster_name}-nat-ip" + location = var.location + resource_group_name = var.resource_group_name + allocation_method = "Static" + sku = "Standard" +} + +resource "azurerm_nat_gateway" "nat" { + name = "${var.cluster_name}-nat-gateway" + location = var.location + resource_group_name = var.resource_group_name +} + +resource "azurerm_nat_gateway_public_ip_association" "nat" { + nat_gateway_id = azurerm_nat_gateway.nat.id + public_ip_address_id = azurerm_public_ip.nat.id +} + +resource "azurerm_route_table" "private_route_table" { + name = "${var.cluster_name}-private-route-table" + location = var.location + resource_group_name = var.resource_group_name + + route { + name = "private-subnet-nat-route" + address_prefix = "0.0.0.0/0" + next_hop_type = "Internet" + # next_hop_in_ip_address = azurerm_public_ip.nat.ip_address + } +} + +resource "azurerm_subnet_route_table_association" "private_subnet_route_association" { + count = length(var.private_subnet_ids) + subnet_id = var.private_subnet_ids[count.index] + route_table_id = azurerm_route_table.private_route_table.id +} diff --git a/terraform/modules/azure/nat/variables.tf b/terraform/modules/azure/nat/variables.tf new file mode 100644 index 00000000..033aa664 --- /dev/null +++ b/terraform/modules/azure/nat/variables.tf @@ -0,0 +1,19 @@ +variable "cluster_name" { + description = "Cluster name used in naming NAT resources" + type = string +} + +variable "location" { + description = "Azure region" + type = string +} + +variable "resource_group_name" { + description = "Resource group to deploy NAT resources" + type = string +} + +variable "private_subnet_ids" { + description = "List of private subnet IDs to associate with route table" + type = list(string) +} diff --git a/terraform/modules/azure/vault/main.tf b/terraform/modules/azure/vault/main.tf new file mode 100644 index 00000000..29bde908 --- /dev/null +++ b/terraform/modules/azure/vault/main.tf @@ -0,0 +1,26 @@ +resource "azurerm_key_vault" "vault_auto_unseal" { + name = "${var.cluster_name}-vault-kv" + location = var.location + resource_group_name = var.resource_group_name + tenant_id = var.tenant_id + sku_name = "standard" + purge_protection_enabled = true + soft_delete_retention_days = 10 + enable_rbac_authorization = true +} + +resource "azurerm_key_vault_key" "vault_auto_unseal" { + name = "${var.cluster_name}-auto-unseal" + key_vault_id = azurerm_key_vault.vault_auto_unseal.id + key_type = "RSA" + key_size = 2048 + key_opts = ["decrypt", "encrypt", "sign", "verify", "wrapKey", "unwrapKey"] + + depends_on = [azurerm_key_vault.vault_auto_unseal] +} + +resource "azurerm_role_assignment" "vault_key_usage" { + principal_id = var.service_principal_object_id + role_definition_name = "Key Vault Crypto User" + scope = azurerm_key_vault.vault_auto_unseal.id +} diff --git a/terraform/modules/azure/vault/outputs.tf b/terraform/modules/azure/vault/outputs.tf new file mode 100644 index 00000000..923deecd --- /dev/null +++ b/terraform/modules/azure/vault/outputs.tf @@ -0,0 +1,9 @@ +output "vault_key_name" { + value = azurerm_key_vault_key.vault_auto_unseal.name + description = "Name of the created Key Vault key for Vault auto-unseal" +} + +output "vault_keyvault_name" { + value = azurerm_key_vault.vault_auto_unseal.name + description = "Name of the created Key Vault for Vault auto-unseal" +} diff --git a/terraform/modules/azure/vault/variables.tf b/terraform/modules/azure/vault/variables.tf new file mode 100644 index 00000000..a26d4a8e --- /dev/null +++ b/terraform/modules/azure/vault/variables.tf @@ -0,0 +1,24 @@ +variable "cluster_name" { + description = "The name of the AKS or Vault cluster" + type = string +} + +variable "location" { + description = "Azure region" + type = string +} + +variable "resource_group_name" { + description = "Resource group name" + type = string +} + +variable "tenant_id" { + description = "Azure AD tenant ID" + type = string +} + +variable "service_principal_object_id" { + description = "Object ID of the service principal to assign key access to" + type = string +} diff --git a/terraform/modules/azure/vpn/main.tf b/terraform/modules/azure/vpn/main.tf new file mode 100644 index 00000000..80eb0120 --- /dev/null +++ b/terraform/modules/azure/vpn/main.tf @@ -0,0 +1,210 @@ +resource "tls_private_key" "ca" { + algorithm = "RSA" +} + +resource "tls_self_signed_cert" "ca" { + private_key_pem = tls_private_key.ca.private_key_pem + subject { + common_name = "${var.name}.vpn.ca" + organization = var.name + } + validity_period_hours = 87600 + is_ca_certificate = true + allowed_uses = [ + "cert_signing", + "crl_signing", + "digital_signature", # Add this for client auth + "key_encipherment", # Add this for VPN usage + "client_auth", # Ensure VPN client compatibility + "server_auth" + ] +} + +locals { + root_cert_data = replace( + replace( + replace( + tls_self_signed_cert.ca.cert_pem, + "-----BEGIN CERTIFICATE-----\n", + "" + ), + "\n-----END CERTIFICATE-----", + "" + ), + "\n", + "" + ) +} + +resource "azurerm_public_ip" "vpn_gateway_ip" { + name = "${var.name}-vpn-ip" + location = var.location + resource_group_name = var.resource_group_name + allocation_method = "Static" + sku = "Standard" +} + +resource "azurerm_virtual_network_gateway" "vpn_gateway" { + name = "${var.name}-vpn-gw" + location = var.location + resource_group_name = var.resource_group_name + + type = "Vpn" + vpn_type = "RouteBased" + sku = "VpnGw1" + + active_active = false + enable_bgp = false + + ip_configuration { + name = "vpn-gateway-config" + public_ip_address_id = azurerm_public_ip.vpn_gateway_ip.id + private_ip_address_allocation = "Dynamic" + subnet_id = var.gateway_subnet_id + } + + vpn_client_configuration { + address_space = [var.vpn_client_cidr] + vpn_client_protocols = ["OpenVPN"] + root_certificate { + name = "vpn-root" + public_cert_data = local.root_cert_data + } + vpn_auth_types = ["Certificate"] + } + depends_on = [ + tls_self_signed_cert.ca + ] +} + +resource "azurerm_storage_account" "vpn_config" { + name = "${lower(var.name)}vpnstorage" + location = var.location + resource_group_name = var.resource_group_name + account_tier = "Standard" + account_replication_type = "LRS" +} + +resource "azurerm_storage_container" "vpn_config_container" { + name = "vpn-config-files" + storage_account_name = azurerm_storage_account.vpn_config.name + container_access_type = "private" +} + +resource "azurerm_key_vault" "vpn_keyvault" { + name = "${lower(var.name)}-vpn-kv" + location = var.location + resource_group_name = var.resource_group_name + enabled_for_disk_encryption = true + tenant_id = var.tenant_id + sku_name = "standard" + enable_rbac_authorization = true +} + +resource "azurerm_key_vault_access_policy" "vpn_access" { + key_vault_id = azurerm_key_vault.vpn_keyvault.id + tenant_id = var.tenant_id + object_id = var.sp_object_id + + secret_permissions = [ + "Get", "List", # Add "get" permission to allow secret retrieval + ] +} + +resource "azurerm_key_vault_secret" "vpn_ca_cert" { + name = "vpn-ca-cert" + value = tls_self_signed_cert.ca.cert_pem + key_vault_id = azurerm_key_vault.vpn_keyvault.id +} + +resource "azurerm_key_vault_secret" "vpn_ca_key" { + name = "vpn-ca-key" + value = tls_private_key.ca.private_key_pem + key_vault_id = azurerm_key_vault.vpn_keyvault.id +} + +resource "tls_private_key" "client" { + for_each = var.vpn_client_list + algorithm = "RSA" +} + +resource "tls_cert_request" "client" { + for_each = var.vpn_client_list + private_key_pem = tls_private_key.client[each.key].private_key_pem + subject { + common_name = each.key + organization = var.name + } +} + +resource "tls_locally_signed_cert" "client" { + for_each = var.vpn_client_list + cert_request_pem = tls_cert_request.client[each.key].cert_request_pem + ca_private_key_pem = tls_private_key.ca.private_key_pem + ca_cert_pem = tls_self_signed_cert.ca.cert_pem + validity_period_hours = 87600 + allowed_uses = [ + "key_encipherment", + "digital_signature", + "client_auth", + "server_auth" + ] +} + +resource "azurerm_key_vault_secret" "vpn_client_cert" { + for_each = var.vpn_client_list + name = "vpn-${each.key}-cert" + value = tls_locally_signed_cert.client[each.key].cert_pem + key_vault_id = azurerm_key_vault.vpn_keyvault.id +} + +resource "azurerm_key_vault_secret" "vpn_client_key" { + for_each = var.vpn_client_list + name = "vpn-${each.key}-key" + value = tls_private_key.client[each.key].private_key_pem + key_vault_id = azurerm_key_vault.vpn_keyvault.id +} + +resource "azurerm_storage_blob" "vpn_config_file" { + for_each = var.vpn_client_list + name = "${substr(each.key, 0, 30)}-${lower(var.name)}.ovpn" + storage_account_name = azurerm_storage_account.vpn_config.name + storage_container_name = azurerm_storage_container.vpn_config_container.name + type = "Block" + source_content = < +${azurerm_key_vault_secret.vpn_ca_cert.value} + + +${azurerm_key_vault_secret.vpn_client_cert[each.key].value} + + +${azurerm_key_vault_secret.vpn_client_key[each.key].value} + +EOT + + depends_on = [ + azurerm_key_vault_secret.vpn_client_cert, + azurerm_key_vault_secret.vpn_client_key + ] +} \ No newline at end of file diff --git a/terraform/modules/azure/vpn/variables.tf b/terraform/modules/azure/vpn/variables.tf new file mode 100644 index 00000000..2c80588b --- /dev/null +++ b/terraform/modules/azure/vpn/variables.tf @@ -0,0 +1,19 @@ +variable "tenant_id" {} +variable "name" {} +variable "location" {} +variable "resource_group_name" {} +variable "gateway_subnet_id" {} +variable "vpn_client_cidr" { + default = "172.16.0.0/24" +} + +variable "vpn_gateway_port" { + description = "The port for the VPN gateway" + type = string + default = "1194" +} +variable "vpn_client_list" { + type = set(string) +} + +variable "sp_object_id" {} \ No newline at end of file