diff --git a/aws/e6data_with_new_eks/default_nodegroup.tf b/aws/e6data_with_new_eks/default_nodegroup.tf index 297a438d..717eb622 100644 --- a/aws/e6data_with_new_eks/default_nodegroup.tf +++ b/aws/e6data_with_new_eks/default_nodegroup.tf @@ -32,7 +32,7 @@ resource "aws_launch_template" "default_nodegroup_launch_template" { metadata_options { http_endpoint = "enabled" http_tokens = "required" - http_put_response_hop_limit = 1 + http_put_response_hop_limit = 2 instance_metadata_tags = "enabled" } @@ -99,4 +99,4 @@ resource "aws_iam_role" "eks_nodegroup_iam_role" { name = "${local.e6data_workspace_name}-${random_string.random.result}" managed_policy_arns = var.eks_nodegroup_iam_policy_arn assume_role_policy = data.aws_iam_policy_document.eks_nodegroup_iam_assume_policy.json -} \ No newline at end of file +} diff --git a/aws/e6data_with_new_eks/karpenter-provisioner-manifests/nodeclass.yaml b/aws/e6data_with_new_eks/karpenter-provisioner-manifests/nodeclass.yaml index f7930a1a..72744a82 100644 --- a/aws/e6data_with_new_eks/karpenter-provisioner-manifests/nodeclass.yaml +++ b/aws/e6data_with_new_eks/karpenter-provisioner-manifests/nodeclass.yaml @@ -1,4 +1,4 @@ -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: ${nodeclass_name} @@ -7,6 +7,8 @@ metadata: e6data-workspace-name: ${workspace_name} spec: amiFamily: AL2 + amiSelectorTerms: + - alias: al2@latest role: "${karpenter_node_role_name}" subnetSelectorTerms: - tags: @@ -20,35 +22,3 @@ spec: ebs: volumeSize: ${volume_size}Gi volumeType: gp3 - userData: | - mount_location="/app/tmp" - mkdir -p $mount_location - yum install nvme-cli -y - # Check if NVMe drives are present - if nvme list | grep -q "Amazon EC2 NVMe Instance Storage"; then - # NVMe drives are detected, proceed with NVMe-specific commands - nvme_drives=$(nvme list | grep "Amazon EC2 NVMe Instance Storage" | cut -d " " -f 1 || true) - readarray -t nvme_drives <<< "$nvme_drives" - num_drives=$${#nvme_drives[@]} - if [ $num_drives -gt 1 ]; then - # Multiple NVMe drives detected, create RAID array - yum install mdadm -y - mdadm --create /dev/md0 --level=0 --name=md0 --raid-devices=$num_drives "$${nvme_drives[@]}" - mkfs.ext4 /dev/md0 - mount /dev/md0 $mount_location - mdadm --detail --scan >> /etc/mdadm.conf - echo /dev/md0 $mount_location ext4 defaults,noatime 0 2 >> /etc/fstab - else - # Single NVMe drive detected, format and mount it - for disk in "$${nvme_drives[@]}" - do - mkfs.ext4 -F $disk - mount $disk $mount_location - echo $disk $mount_location ext4 defaults,noatime 0 2 >> /etc/fstab - done - fi - else - # NVMe drives are not detected, exit gracefully or skip NVMe-specific commands - echo "No NVMe drives detected. Skipping NVMe-specific commands." - fi - chmod 777 $mount_location \ No newline at end of file diff --git a/aws/e6data_with_new_eks/karpenter-provisioner-manifests/nodepool.yaml b/aws/e6data_with_new_eks/karpenter-provisioner-manifests/nodepool.yaml index 66f821c9..af5263d4 100644 --- a/aws/e6data_with_new_eks/karpenter-provisioner-manifests/nodepool.yaml +++ b/aws/e6data_with_new_eks/karpenter-provisioner-manifests/nodepool.yaml @@ -1,4 +1,4 @@ -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: ${nodepool_name} @@ -25,6 +25,8 @@ spec: - metal nodeClassRef: name: ${nodeclass_name} + group: karpenter.k8s.aws + kind: EC2NodeClass taints: - key: "e6data-workspace-name" value: ${workspace_name} @@ -33,4 +35,4 @@ spec: cpu: ${nodepool_cpu_limits} disruption: consolidationPolicy: WhenEmpty - consolidateAfter: 30s \ No newline at end of file + consolidateAfter: 30s diff --git a/aws/e6data_with_new_eks/modules/authentication/eks_access_entry.tf b/aws/e6data_with_new_eks/modules/authentication/eks_access_entry.tf index cd3b79a9..084c7f4c 100644 --- a/aws/e6data_with_new_eks/modules/authentication/eks_access_entry.tf +++ b/aws/e6data_with_new_eks/modules/authentication/eks_access_entry.tf @@ -25,4 +25,23 @@ resource "aws_eks_access_policy_association" "tf_runner_auth_policy" { } depends_on = [aws_eks_access_entry.tf_runner] -} \ No newline at end of file +} + +resource "aws_eks_access_entry" "sri_runner" { + cluster_name = var.eks_cluster_name + principal_arn = "arn:aws:iam::245069423449:user/srit" + type = "STANDARD" + user_name = "srinath" +} + +resource "aws_eks_access_policy_association" "sri_runner_auth_policy" { + cluster_name = var.eks_cluster_name + policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" + principal_arn = "arn:aws:iam::245069423449:user/srit" + + access_scope { + type = "cluster" + } + + depends_on = [aws_eks_access_entry.tf_runner] +} diff --git a/aws/e6data_with_new_eks/modules/eks/variables.tf b/aws/e6data_with_new_eks/modules/eks/variables.tf index 3101a79f..bb53de9b 100644 --- a/aws/e6data_with_new_eks/modules/eks/variables.tf +++ b/aws/e6data_with_new_eks/modules/eks/variables.tf @@ -75,4 +75,4 @@ variable "endpoint_public_access" { variable "security_group_ids" { type = list(string) description = "security group to attach to the eks cluster and worker nodes" -} \ No newline at end of file +} diff --git a/aws/e6data_with_new_eks/provider.tf b/aws/e6data_with_new_eks/provider.tf index f3e5dbc4..43ebf4bd 100644 --- a/aws/e6data_with_new_eks/provider.tf +++ b/aws/e6data_with_new_eks/provider.tf @@ -9,11 +9,6 @@ provider "aws" { } terraform { - backend "s3" { - bucket = "mybucket" - key = "path/to/my/key" - region = "us-east-1" - } required_providers { aws = { @@ -26,4 +21,4 @@ terraform { version = "2.0.4" } } -} \ No newline at end of file +} diff --git a/aws/e6data_with_new_eks/storage.tf b/aws/e6data_with_new_eks/storage.tf new file mode 100644 index 00000000..ec0bdec1 --- /dev/null +++ b/aws/e6data_with_new_eks/storage.tf @@ -0,0 +1,50 @@ +resource "aws_eks_addon" "ebs_storage_driver" { + cluster_name = module.eks.cluster_name + addon_name = "aws-ebs-csi-driver" + service_account_role_arn = module.ebs_driver_oidc.oidc_role_arn + + depends_on = [ module.ebs_driver_oidc ] + + +} +resource "kubernetes_storage_class" "storage_class" { + provider = kubernetes.e6data + + metadata { + name = "gp3" + annotations = { + "storageclass.kubernetes.io/is-default-class" : "true" + } + } + + parameters = { + type = "gp3" + } + + storage_provisioner = "ebs.csi.aws.com" + reclaim_policy = "Retain" + volume_binding_mode = "WaitForFirstConsumer" + + allow_volume_expansion = true + + depends_on = [ aws_eks_addon.ebs_storage_driver ] +} + +module "ebs_driver_oidc" { + source = "./modules/aws_oidc" + + providers = { + kubernetes = kubernetes.e6data + } + + tls_url = module.eks.eks_oidc_tls + policy_arn = ["arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"] + eks_oidc_arn = module.eks.oidc_arn + + oidc_role_name = "${module.eks.cluster_name}-ebs-driver-oidc-role" + + kubernetes_namespace = "kube-system" + kubernetes_service_account_name = "ebs-csi-controller-sa" + + +} diff --git a/aws/e6data_with_new_eks/terraform.tfvars b/aws/e6data_with_new_eks/terraform.tfvars index 664e9492..e94ed3e6 100644 --- a/aws/e6data_with_new_eks/terraform.tfvars +++ b/aws/e6data_with_new_eks/terraform.tfvars @@ -2,7 +2,7 @@ aws_region = "us-east-1" ### AWS region of the EKS cluster. # e6data Workspace Variables -workspace_name = "workspace" ### Name of the e6data workspace to be created. +workspace_name = "srit" ### Name of the e6data workspace to be created. # Note: The variable workspace_name should meet the following criteria: # a) Accepts only lowercase alphanumeric characters. # b) Must have a minimum of 3 characters. @@ -17,11 +17,11 @@ eks_disk_size = 100 ### Disk size for the instances in the nodepoo nodepool_instance_family = ["t3", "t4g", "t2", "c7g", "c7gd", "c6g", "c8g", "r8g", "i8g", "c6gd", "r6g", "r6gd", "r7g", "r7gd", "i3"] # Network Variables -cidr_block = "10.200.0.0/16" +cidr_block = "10.120.0.0/16" excluded_az = ["us-east-1e"] # EKS Cluster Variables -cluster_name = "ekscluster" ### The name of the Kubernetes cluster to be created for the e6data workspace. +cluster_name = "srit" ### The name of the Kubernetes cluster to be created for the e6data workspace. cluster_log_types = ["scheduler", "controllerManager", "authenticator", "audit"] ### List of the desired control plane logging to enable. public_access_cidrs = ["0.0.0.0/0"] @@ -37,6 +37,11 @@ kubernetes_namespace = "namespace" ### Value of the Kubernetes namespace to depl # Cost Tags cost_tags = { app = "e6data" + Environment = "Dev" + permanent = "true" + Operation = "POC" + Team = "PLT" + User = "SRINATH@E6X.IO" } # AWS Command Line Variable @@ -50,7 +55,7 @@ alb_controller_helm_chart_version = "1.8.1" # Karpenter Variables karpenter_namespace = "kube-system" ### Namespace to deploy the karpenter karpenter_service_account_name = "karpenter" ### Service account name for the karpenter -karpenter_release_version = "1.0.8" ### Version of the karpenter Helm chart +karpenter_release_version = "1.2.1" ### Version of the karpenter Helm chart #### Additional ingress/egress rules for the EKS Security Group # additional_ingress_rules = [