Internal

aws/e6data_with_existing_eks/e6data_engine_iam.tf

@@ -17,6 +17,28 @@ data "aws_iam_policy_document" "oidc_assume_role_policy" {
   }
 }
 
+data "aws_iam_policy_document" "system_tables_policy" {
+   statement {
+    sid    = "AssumeRole"
+    effect = "Allow"
+
+    actions = [
+      "sts:AssumeRole"
+    ]
+    resources = ["arn:aws:iam::${local.cross_account_id}:role/e6-system-tables-*"]
+  }
+
+  statement {
+    sid    = "TagSession"
+    effect = "Allow"
+
+    actions = [
+      "sts:TagSession"
+    ]
+    resources = ["*"]
+  }
+}
+
 data "aws_iam_policy_document" "engine_iam_glue_s3readAccess_doc" {
   statement {
     sid    = "glueReadOnlyAccess"
@@ -60,9 +82,16 @@ resource "aws_iam_policy" "e6data_engine_s3_glue_policy" {
   policy      = data.aws_iam_policy_document.engine_iam_glue_s3readAccess_doc.json
 }
 
+resource "aws_iam_policy" "e6data_engine_system_tables_policy" {
+  name        = "${local.e6data_workspace_name}-engine-system-tables-${random_string.random.result}"
+  description = "Allows assume the role for system tables"
+  policy      = data.aws_iam_policy_document.system_tables_policy.json
+}
+
+
 # Create an IAM role for the engine, allowing it to assume the role with specified policies attached
 resource "aws_iam_role" "e6data_engine_role" {
   name                = "${local.e6data_workspace_name}-engine-role-${random_string.random.result}"
   assume_role_policy  = data.aws_iam_policy_document.oidc_assume_role_policy.json
-  managed_policy_arns = [aws_iam_policy.e6data_engine_s3_glue_policy.arn, aws_iam_policy.e6data_s3_read_write_policy.arn]
+  managed_policy_arns = [aws_iam_policy.e6data_engine_s3_glue_policy.arn, aws_iam_policy.e6data_s3_read_write_policy.arn, aws_iam_policy.e6data_engine_system_tables_policy.arn]
 }

aws/e6data_with_existing_eks/helm.tf

@@ -12,8 +12,5 @@ resource "helm_release" "e6data_workspace_deployment" {
 
   values = [local.helm_values_file]
 
-  lifecycle {
-    ignore_changes = [values]
-  }
   # depends_on = [aws_eks_access_policy_association.tf_runner_auth_policy]
 }

aws/e6data_with_existing_eks/karpenter-provisioner-manifests/nodeclass.yaml

@@ -22,7 +22,6 @@ spec:
         volumeType: gp3
   userData: |
     echo "$(jq '.allowedUnsafeSysctls += ["net.core.somaxconn","net.ipv4.ip_local_port_range"]' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json
-    echo "$(jq '.cpuManagerPolicy = "static"' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json  
     mount_location="/app/tmp"
     mkdir -p $mount_location
     yum install nvme-cli -y

aws/e6data_with_existing_eks/support.tf

@@ -14,10 +14,7 @@ locals {
       type               = "AWS"
       oidc_value         = aws_iam_role.e6data_engine_role.arn
       control_plane_user = ["e6data-${var.workspace_name}-user"]
-    }
-    karpenter = {
-      nodepool  = local.e6data_nodepool_name
-      nodeclass = local.e6data_nodeclass_name
+      debug_namespaces   = var.debug_namespaces
     }
   })
   mapUsers    = try(data.kubernetes_config_map_v1.aws_auth_read.data["mapUsers"], "")
@@ -74,37 +71,29 @@ data "aws_eks_node_group" "current" {
   node_group_name = tolist(data.aws_eks_node_groups.current.names)[0]
 }
 
+data "aws_eks_cluster_auth" "target_eks_auth" {
+  name = data.aws_eks_cluster.current.name
+}
+
 provider "kubernetes" {
   alias                  = "eks_e6data"
   host                   = data.aws_eks_cluster.current.endpoint
   cluster_ca_certificate = base64decode(data.aws_eks_cluster.current.certificate_authority[0].data)
-  exec {
-    api_version = "client.authentication.k8s.io/v1beta1"
-    args        = ["eks", "get-token", "--cluster-name", var.eks_cluster_name]
-    command     = var.aws_command_line_path
-  }
+  token                  = data.aws_eks_cluster_auth.target_eks_auth.token
 }
 
 provider "kubectl" {
   host                   = data.aws_eks_cluster.current.endpoint
   cluster_ca_certificate = base64decode(data.aws_eks_cluster.current.certificate_authority[0].data)
   load_config_file       = false
-  exec {
-    api_version = "client.authentication.k8s.io/v1beta1"
-    args        = ["eks", "get-token", "--cluster-name", var.eks_cluster_name]
-    command     = var.aws_command_line_path
-  }
+  token                  = data.aws_eks_cluster_auth.target_eks_auth.token
 }
 
 provider "helm" {
   alias = "eks_e6data"
   kubernetes {
     host                   = data.aws_eks_cluster.current.endpoint
     cluster_ca_certificate = base64decode(data.aws_eks_cluster.current.certificate_authority[0].data)
-    exec {
-      api_version = "client.authentication.k8s.io/v1beta1"
-      args        = ["eks", "get-token", "--cluster-name", var.eks_cluster_name]
-      command     = var.aws_command_line_path
-    }
+    token                  = data.aws_eks_cluster_auth.target_eks_auth.token
   }
-}
+}

aws/e6data_with_existing_eks/terraform.tfvars

@@ -18,6 +18,7 @@ bucket_names = ["*"] ### List of bucket names that the e6data engine queries and
 kubernetes_namespace = "e6data" ### Value of the Kubernetes namespace to deploy the e6data workspace.
 helm_chart_version   = "2.1.7"  ### e6data workspace Helm chart version to be used.
 
+debug_namespaces = ["kube-system"]
 
 ### Below are the tags which will be applied to all the resources created by this Terraform script.
 cost_tags = {

aws/e6data_with_existing_eks/variables.tf

@@ -83,3 +83,12 @@ variable "nodepool_cpu_limits" {
   default     = 100000
 }
 
+variable "debug_namespaces" {
+  type        = list(string)
+  description = "kaprneter and alb controller namespaces"
+  default     = ["kube-system"]
+}
+
+locals {
+  cross_account_id = split(":", var.e6data_cross_oidc_role_arn[0])[4]
+}

aws/e6data_with_existing_vpc/e6data_engine_iam.tf

@@ -53,16 +53,44 @@ data "aws_iam_policy_document" "engine_iam_glue_s3readAccess_doc" {
   }
 }
 
+data "aws_iam_policy_document" "system_tables_policy" {
+   statement {
+    sid    = "AssumeRole"
+    effect = "Allow"
+
+    actions = [
+      "sts:AssumeRole"
+    ]
+    resources = ["arn:aws:iam::${local.cross_account_id}:role/e6-system-tables-*"]
+  }
+
+  statement {
+    sid    = "TagSession"
+    effect = "Allow"
+
+    actions = [
+      "sts:TagSession"
+    ]
+    resources = ["*"]
+  }
+}
+
 # Create an IAM policy that grants read access to S3 buckets and the Glue catalog
 resource "aws_iam_policy" "e6data_engine_s3_glue_policy" {
   name        = "${local.e6data_workspace_name}-engine-s3-glue-${random_string.random.result}"
   description = "Allows read access for s3 buckets and glue catalog"
   policy      = data.aws_iam_policy_document.engine_iam_glue_s3readAccess_doc.json
 }
 
+resource "aws_iam_policy" "e6data_engine_system_tables_policy" {
+  name        = "${local.e6data_workspace_name}-engine-system-tables-${random_string.random.result}"
+  description = "Allows assume the role for system tables"
+  policy      = data.aws_iam_policy_document.system_tables_policy.json
+}
+
 # Create an IAM role for the engine, allowing it to assume the role with specified policies attached
 resource "aws_iam_role" "e6data_engine_role" {
   name                = "${local.e6data_workspace_name}-engine-role-${random_string.random.result}"
   assume_role_policy  = data.aws_iam_policy_document.oidc_assume_role_policy.json
-  managed_policy_arns = [aws_iam_policy.e6data_engine_s3_glue_policy.arn, aws_iam_policy.e6data_s3_read_write_policy.arn]
+  managed_policy_arns = [aws_iam_policy.e6data_engine_s3_glue_policy.arn, aws_iam_policy.e6data_s3_read_write_policy.arn, aws_iam_policy.e6data_engine_system_tables_policy.arn]
 }

aws/e6data_with_existing_vpc/eks.tf

@@ -38,38 +38,34 @@ resource "aws_ec2_tag" "cluster_primary_security_group" {
   value       = "e6data"
 }
 
+data "aws_eks_cluster_auth" "target_eks_auth" {
+  name = module.eks.cluster_name
+
+  depends_on = [ 
+    module.eks
+  ]
+}
+
 provider "kubernetes" {
   alias                  = "e6data"
   host                   = module.eks.eks_endpoint
   cluster_ca_certificate = base64decode(module.eks.eks_certificate_data)
-  exec {
-    api_version = "client.authentication.k8s.io/v1beta1"
-    args        = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
-    command     = var.aws_command_line_path
-  }
+  token                  = data.aws_eks_cluster_auth.target_eks_auth.token
 }
 
 provider "kubectl" {
   host                   = module.eks.eks_endpoint
   cluster_ca_certificate = base64decode(module.eks.eks_certificate_data)
   load_config_file       = false
-  exec {
-    api_version = "client.authentication.k8s.io/v1beta1"
-    args        = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
-    command     = var.aws_command_line_path
-  }
+  token                  = data.aws_eks_cluster_auth.target_eks_auth.token
 }
 
 provider "helm" {
   alias = "e6data"
   kubernetes {
     host                   = module.eks.eks_endpoint
     cluster_ca_certificate = base64decode(module.eks.eks_certificate_data)
-    exec {
-      api_version = "client.authentication.k8s.io/v1beta1"
-      args        = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
-      command     = var.aws_command_line_path
-    }
+    token                  = data.aws_eks_cluster_auth.target_eks_auth.token
   }
 }
 

aws/e6data_with_existing_vpc/helm.tf

@@ -12,9 +12,5 @@ resource "helm_release" "e6data_workspace_deployment" {
 
   values = [local.helm_values_file]
 
-  lifecycle {
-    ignore_changes = [values]
-  }
-
   depends_on = [module.eks, aws_eks_node_group.default_node_group, module.e6data_authentication]
 }

aws/e6data_with_existing_vpc/karpenter-provisioner-manifests/nodeclass.yaml

@@ -22,7 +22,6 @@ spec:
         volumeType: gp3
   userData: |
     echo "$(jq '.allowedUnsafeSysctls += ["net.core.somaxconn","net.ipv4.ip_local_port_range"]' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json
-    echo "$(jq '.cpuManagerPolicy = "static"' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json
     mount_location="/app/tmp"
     mkdir -p $mount_location
     yum install nvme-cli -y

aws/e6data_with_existing_vpc/support.tf

@@ -17,10 +17,7 @@ locals {
       type               = "AWS"
       oidc_value         = aws_iam_role.e6data_engine_role.arn
       control_plane_user = ["e6data-${var.workspace_name}-user"]
-    }
-    karpenter = {
-      nodepool  = local.e6data_nodepool_name
-      nodeclass = local.e6data_nodeclass_name
+      debug_namespaces   = var.debug_namespaces
     }
   })
 }

aws/e6data_with_existing_vpc/terraform.tfvars

@@ -10,8 +10,8 @@ workspace_name = "workspace" ### Name of the e6data workspace to be created.
 helm_chart_version = "2.1.7" ### e6data workspace Helm chart version to be used.
 
 # Kubernetes Variables
-kube_version             = "1.31" ### The Kubernetes cluster version. Version 1.24 or higher is required.
-default_nodegroup_kube_version = "1.31"
+kube_version             = "1.32" ### The Kubernetes cluster version. Version 1.24 or higher is required.
+default_nodegroup_kube_version = "1.32"
 
 eks_disk_size            = 100    ### Disk size for the instances in the nodepool. A minimum of 100 GB is required.
 nodepool_instance_family = ["t3", "t4g", "t2", "c7g", "c7gd", "c6g", "c8g", "r8g", "i8g", "c6gd", "r6g", "r6gd", "r7g", "r7gd", "i3"]
@@ -52,6 +52,8 @@ karpenter_namespace            = "kube-system" ### Namespace to deploy the karpe
 karpenter_service_account_name = "karpenter"   ### Service account name for the karpenter
 karpenter_release_version      = "1.0.8"       ### Version of the karpenter Helm chart
 
+debug_namespaces = ["kube-system"]
+
 #### Additional ingress/egress rules for the EKS Security Group
 # additional_ingress_rules = [
 #   {

aws/e6data_with_existing_vpc/variables.tf

@@ -263,6 +263,11 @@ variable "additional_egress_rules" {
   default = []
 }
 
+variable "debug_namespaces" {
+  type        = list(string)
+  description = "karpenter and alb controller namespaces"
+  default     = ["kube-system"]
+}
 variable "vpc_cni_version" {
   description = "Version of the VPC CNI to use"
   type        = string
@@ -285,4 +290,8 @@ variable "minimum_ip_target" {
   description = "Minimum number of IP addresses to keep available for pod assignment."
   type        = number
   default     = 12
+}
+
+locals {
+  cross_account_id = split(":", var.e6data_cross_oidc_role_arn[0])[4]
 }

aws/e6data_with_new_eks/default_nodegroup.tf

@@ -32,7 +32,7 @@ resource "aws_launch_template" "default_nodegroup_launch_template" {
   metadata_options {
     http_endpoint               = "enabled"
     http_tokens                 = "required"
-    http_put_response_hop_limit = 1
+    http_put_response_hop_limit = 2
     instance_metadata_tags      = "enabled"
   }
 
@@ -99,4 +99,4 @@ resource "aws_iam_role" "eks_nodegroup_iam_role" {
   name                = "${local.e6data_workspace_name}-${random_string.random.result}"
   managed_policy_arns = var.eks_nodegroup_iam_policy_arn
   assume_role_policy  = data.aws_iam_policy_document.eks_nodegroup_iam_assume_policy.json
-}
+}

aws/e6data_with_new_eks/e6data_engine_iam.tf

@@ -53,16 +53,44 @@ data "aws_iam_policy_document" "engine_iam_glue_s3readAccess_doc" {
   }
 }
 
+data "aws_iam_policy_document" "system_tables_policy" {
+   statement {
+    sid    = "AssumeRole"
+    effect = "Allow"
+
+    actions = [
+      "sts:AssumeRole"
+    ]
+    resources = ["arn:aws:iam::${local.cross_account_id}:role/e6-system-tables-*"]
+  }
+
+  statement {
+    sid    = "TagSession"
+    effect = "Allow"
+
+    actions = [
+      "sts:TagSession"
+    ]
+    resources = ["*"]
+  }
+}
+
 # Create an IAM policy that grants read access to S3 buckets and the Glue catalog
 resource "aws_iam_policy" "e6data_engine_s3_glue_policy" {
   name        = "${local.e6data_workspace_name}-engine-s3-glue-policy-${random_string.random.result}"
   description = "Allows read access for s3 buckets and glue catalog"
   policy      = data.aws_iam_policy_document.engine_iam_glue_s3readAccess_doc.json
 }
 
+resource "aws_iam_policy" "e6data_engine_system_tables_policy" {
+  name        = "${local.e6data_workspace_name}-engine-system-tables-${random_string.random.result}"
+  description = "Allows assume the role for system tables"
+  policy      = data.aws_iam_policy_document.system_tables_policy.json
+}
+
 # Create an IAM role for the engine, allowing it to assume the role with specified policies attached
 resource "aws_iam_role" "e6data_engine_role" {
   name                = "${local.e6data_workspace_name}-engine-role-${random_string.random.result}"
   assume_role_policy  = data.aws_iam_policy_document.oidc_assume_role_policy.json
-  managed_policy_arns = [aws_iam_policy.e6data_engine_s3_glue_policy.arn, aws_iam_policy.e6data_s3_read_write_policy.arn]
+  managed_policy_arns = [aws_iam_policy.e6data_engine_s3_glue_policy.arn, aws_iam_policy.e6data_s3_read_write_policy.arn, aws_iam_policy.e6data_engine_system_tables_policy.arn]
 }