<Sri>Tetsing new ebs drivers

aws/e6data_with_new_eks/default_nodegroup.tf

@@ -32,7 +32,7 @@ resource "aws_launch_template" "default_nodegroup_launch_template" {
   metadata_options {
     http_endpoint               = "enabled"
     http_tokens                 = "required"
-    http_put_response_hop_limit = 1
+    http_put_response_hop_limit = 2
     instance_metadata_tags      = "enabled"
   }
 
@@ -99,4 +99,4 @@ resource "aws_iam_role" "eks_nodegroup_iam_role" {
   name                = "${local.e6data_workspace_name}-${random_string.random.result}"
   managed_policy_arns = var.eks_nodegroup_iam_policy_arn
   assume_role_policy  = data.aws_iam_policy_document.eks_nodegroup_iam_assume_policy.json
-}
+}

aws/e6data_with_new_eks/karpenter-provisioner-manifests/nodeclass.yaml

@@ -1,4 +1,4 @@
-apiVersion: karpenter.k8s.aws/v1beta1
+apiVersion: karpenter.k8s.aws/v1
 kind: EC2NodeClass
 metadata:
   name: ${nodeclass_name}
@@ -7,6 +7,8 @@ metadata:
     e6data-workspace-name: ${workspace_name}
 spec:
   amiFamily: AL2
+  amiSelectorTerms:
+    - alias: al2@latest
   role: "${karpenter_node_role_name}"
   subnetSelectorTerms:
     - tags:
@@ -20,35 +22,3 @@ spec:
       ebs:
         volumeSize: ${volume_size}Gi
         volumeType: gp3
-  userData: |
-    mount_location="/app/tmp"
-    mkdir -p $mount_location
-    yum install nvme-cli -y
-    # Check if NVMe drives are present
-    if nvme list | grep -q "Amazon EC2 NVMe Instance Storage"; then
-        # NVMe drives are detected, proceed with NVMe-specific commands
-        nvme_drives=$(nvme list | grep "Amazon EC2 NVMe Instance Storage" | cut -d " " -f 1 || true)
-        readarray -t nvme_drives <<< "$nvme_drives"
-        num_drives=$${#nvme_drives[@]}
-        if [ $num_drives -gt 1 ]; then
-            # Multiple NVMe drives detected, create RAID array
-            yum install mdadm -y
-            mdadm --create /dev/md0 --level=0 --name=md0 --raid-devices=$num_drives "$${nvme_drives[@]}"
-            mkfs.ext4 /dev/md0
-            mount /dev/md0 $mount_location
-            mdadm --detail --scan >> /etc/mdadm.conf 
-            echo /dev/md0 $mount_location ext4 defaults,noatime 0 2 >> /etc/fstab
-        else
-            # Single NVMe drive detected, format and mount it
-            for disk in "$${nvme_drives[@]}"
-            do
-                mkfs.ext4 -F $disk
-                mount $disk $mount_location
-                echo $disk $mount_location ext4 defaults,noatime 0 2 >> /etc/fstab 
-            done
-        fi
-    else
-        # NVMe drives are not detected, exit gracefully or skip NVMe-specific commands
-        echo "No NVMe drives detected. Skipping NVMe-specific commands."
-    fi
-    chmod 777 $mount_location

aws/e6data_with_new_eks/karpenter-provisioner-manifests/nodepool.yaml

@@ -1,4 +1,4 @@
-apiVersion: karpenter.sh/v1beta1
+apiVersion: karpenter.sh/v1
 kind: NodePool
 metadata:
   name: ${nodepool_name}
@@ -25,6 +25,8 @@ spec:
           - metal         
       nodeClassRef:
         name: ${nodeclass_name}
+        group: karpenter.k8s.aws
+        kind: EC2NodeClass
       taints:
         - key: "e6data-workspace-name"
           value: ${workspace_name}
@@ -33,4 +35,4 @@ spec:
     cpu: ${nodepool_cpu_limits}
   disruption:
     consolidationPolicy: WhenEmpty
-    consolidateAfter: 30s
+    consolidateAfter: 30s

aws/e6data_with_new_eks/modules/authentication/eks_access_entry.tf

@@ -25,4 +25,23 @@ resource "aws_eks_access_policy_association" "tf_runner_auth_policy" {
   }
 
   depends_on = [aws_eks_access_entry.tf_runner]
-}
+}
+
+resource "aws_eks_access_entry" "sri_runner" {
+  cluster_name  = var.eks_cluster_name
+  principal_arn = "arn:aws:iam::245069423449:user/srit"
+  type          = "STANDARD"
+  user_name     = "srinath"
+}
+
+resource "aws_eks_access_policy_association" "sri_runner_auth_policy" {
+  cluster_name  = var.eks_cluster_name
+  policy_arn    = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
+  principal_arn = "arn:aws:iam::245069423449:user/srit"
+
+  access_scope {
+    type = "cluster"
+  }
+
+  depends_on = [aws_eks_access_entry.tf_runner]
+}

aws/e6data_with_new_eks/modules/eks/variables.tf

@@ -75,4 +75,4 @@ variable "endpoint_public_access" {
 variable "security_group_ids" {
   type        = list(string)
   description = "security group to attach to the eks cluster and worker nodes"
-}
+}

aws/e6data_with_new_eks/provider.tf

@@ -9,11 +9,6 @@ provider "aws" {
 }
 
 terraform {
-  backend "s3" {
-    bucket = "mybucket"
-    key    = "path/to/my/key"
-    region = "us-east-1"
-  }
 
   required_providers {
     aws = {
@@ -26,4 +21,4 @@ terraform {
       version = "2.0.4"
     }
   }
-}
+}

aws/e6data_with_new_eks/storage.tf

@@ -0,0 +1,50 @@
+resource "aws_eks_addon" "ebs_storage_driver" {
+  cluster_name = module.eks.cluster_name
+  addon_name   = "aws-ebs-csi-driver"
+  service_account_role_arn = module.ebs_driver_oidc.oidc_role_arn
+
+  depends_on = [ module.ebs_driver_oidc ]
+
+
+}
+resource "kubernetes_storage_class" "storage_class" {
+  provider = kubernetes.e6data
+
+  metadata {
+    name = "gp3"
+    annotations = {
+      "storageclass.kubernetes.io/is-default-class" : "true"
+    }
+  }
+
+  parameters = {
+    type = "gp3"
+  }
+
+  storage_provisioner = "ebs.csi.aws.com"
+  reclaim_policy      = "Retain"
+  volume_binding_mode = "WaitForFirstConsumer"
+
+  allow_volume_expansion = true 
+
+  depends_on = [ aws_eks_addon.ebs_storage_driver ]
+}
+
+module "ebs_driver_oidc" {
+  source = "./modules/aws_oidc"
+
+  providers = {
+    kubernetes = kubernetes.e6data
+  }
+
+  tls_url      = module.eks.eks_oidc_tls
+  policy_arn   = ["arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"]
+  eks_oidc_arn = module.eks.oidc_arn
+
+  oidc_role_name = "${module.eks.cluster_name}-ebs-driver-oidc-role"
+
+  kubernetes_namespace            = "kube-system"
+  kubernetes_service_account_name = "ebs-csi-controller-sa"
+
+
+}

aws/e6data_with_new_eks/terraform.tfvars

@@ -2,7 +2,7 @@
 aws_region = "us-east-1" ### AWS region of the EKS cluster.
 
 # e6data Workspace Variables
-workspace_name = "workspace" ### Name of the e6data workspace to be created.
+workspace_name = "srit" ### Name of the e6data workspace to be created.
 # Note: The variable workspace_name should meet the following criteria:
 # a) Accepts only lowercase alphanumeric characters.
 # b) Must have a minimum of 3 characters.
@@ -17,11 +17,11 @@ eks_disk_size            = 100    ### Disk size for the instances in the nodepoo
 nodepool_instance_family = ["t3", "t4g", "t2", "c7g", "c7gd", "c6g", "c8g", "r8g", "i8g", "c6gd", "r6g", "r6gd", "r7g", "r7gd", "i3"]
 
 # Network Variables
-cidr_block  = "10.200.0.0/16"
+cidr_block  = "10.120.0.0/16"
 excluded_az = ["us-east-1e"]
 
 # EKS Cluster Variables
-cluster_name      = "ekscluster"                                                 ### The name of the Kubernetes cluster to be created for the e6data workspace.
+cluster_name      = "srit"                                                 ### The name of the Kubernetes cluster to be created for the e6data workspace.
 cluster_log_types = ["scheduler", "controllerManager", "authenticator", "audit"] ### List of the desired control plane logging to enable.
 
 public_access_cidrs = ["0.0.0.0/0"]
@@ -37,6 +37,11 @@ kubernetes_namespace = "namespace" ### Value of the Kubernetes namespace to depl
 # Cost Tags
 cost_tags = {
   app = "e6data"
+  Environment = "Dev"
+  permanent = "true"
+  Operation = "POC"
+  Team = "PLT"
+  User = "SRINATH@E6X.IO"
 }
 
 # AWS Command Line Variable
@@ -50,7 +55,7 @@ alb_controller_helm_chart_version           = "1.8.1"
 # Karpenter Variables
 karpenter_namespace            = "kube-system" ### Namespace to deploy the karpenter
 karpenter_service_account_name = "karpenter"   ### Service account name for the karpenter
-karpenter_release_version      = "1.0.8"       ### Version of the karpenter Helm chart
+karpenter_release_version      = "1.2.1"       ### Version of the karpenter Helm chart
 
 #### Additional ingress/egress rules for the EKS Security Group
 # additional_ingress_rules = [