From 22293b34231a288703e54b87964534305f20070a Mon Sep 17 00:00:00 2001 From: Thomas Z Date: Thu, 13 Feb 2020 10:53:57 -0800 Subject: [PATCH 1/5] cleanup: removed templates, commented out code --- cmd/create.go | 1 - internal/config/create_example.go | 15 +- internal/generate/generate_infrastructure.go | 15 - internal/generate/generate_modules.go | 1 - pkg/util/flog/log.go | 2 - templates/commit0/.gitignore | 1 - templates/commit0/commit0.tmpl | 51 --- .../environments/development/main.tf | 41 -- .../terraform/environments/production/main.tf | 41 -- .../terraform/environments/staging/main.tf | 41 -- .../terraform/modules/kubernetes/README.md | 27 -- .../modules/kubernetes/example/README.md | 0 .../modules/kubernetes/example/main.tf | 30 -- .../modules/kubernetes/example/variables.tf | 0 .../modules/kubernetes/external_dns.tf | 143 ------- .../modules/kubernetes/ingress/README.md | 4 - .../modules/kubernetes/ingress/main.tf | 371 ------------------ .../modules/kubernetes/ingress/variables.tf | 11 - .../modules/kubernetes/kube2iam/README.md | 6 - .../modules/kubernetes/kube2iam/main.tf | 105 ----- .../modules/kubernetes/kube2iam/variables.tf | 7 - .../terraform/modules/kubernetes/main.tf | 20 - .../kubernetes/monitoring/cloudwatch_agent.tf | 218 ---------- .../monitoring/docs/test-logging-app.yaml | 10 - .../monitoring/files/containers.conf | 44 --- .../monitoring/files/cwagentconfig.json.tpl | 21 - .../kubernetes/monitoring/files/fluent.conf | 10 - .../kubernetes/monitoring/files/host.conf | 69 ---- .../kubernetes/monitoring/files/systemd.conf | 75 ---- .../modules/kubernetes/monitoring/fluentd.tf | 224 ----------- .../modules/kubernetes/monitoring/main.tf | 24 -- .../kubernetes/monitoring/variables.tf | 15 - .../terraform/modules/kubernetes/provider.tf | 20 - .../terraform/modules/kubernetes/variables.tf | 28 -- templates/terraform/README.md | 101 ----- .../bootstrap/create-users/.gitignore | 2 - .../terraform/bootstrap/create-users/main.tf | 23 -- .../terraform/bootstrap/remote-state/main.tf | 37 -- .../environments/development/main.tf | 58 --- .../terraform/environments/production/main.tf | 58 --- .../terraform/environments/staging/main.tf | 61 --- templates/terraform/modules/cognito/main.tf | 40 -- .../terraform/modules/cognito/variables.tf | 6 - templates/terraform/modules/ecr/main.tf | 43 -- templates/terraform/modules/ecr/variables.tf | 14 - templates/terraform/modules/ecr/versions.tf | 4 - templates/terraform/modules/eks/main.tf | 51 --- templates/terraform/modules/eks/outputs.tf | 19 - templates/terraform/modules/eks/variables.tf | 45 --- templates/terraform/modules/eks/versions.tf | 4 - .../terraform/modules/environment/main.tf | 90 ----- .../terraform/modules/environment/provider.tf | 5 - .../modules/environment/variables.tf | 53 --- .../terraform/modules/environment/versions.tf | 3 - .../terraform/modules/kube2iam/README.md | 6 - templates/terraform/modules/kube2iam/main.tf | 130 ------ .../terraform/modules/kube2iam/variables.tf | 15 - .../terraform/modules/kube2iam/versions.tf | 4 - .../terraform/modules/s3_hosting/main.tf | 153 -------- .../terraform/modules/s3_hosting/variables.tf | 13 - templates/terraform/modules/vpc/main.tf | 37 -- templates/terraform/modules/vpc/outputs.tf | 35 -- templates/terraform/modules/vpc/variables.tf | 15 - templates/terraform/modules/vpc/versions.tf | 4 - 64 files changed, 8 insertions(+), 2812 deletions(-) delete mode 100644 templates/commit0/.gitignore delete mode 100644 templates/commit0/commit0.tmpl delete mode 100644 templates/kubernetes/terraform/environments/development/main.tf delete mode 100644 templates/kubernetes/terraform/environments/production/main.tf delete mode 100644 templates/kubernetes/terraform/environments/staging/main.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/README.md delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/example/README.md delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/example/main.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/example/variables.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/external_dns.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/ingress/README.md delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/ingress/main.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/ingress/variables.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/kube2iam/README.md delete mode 100755 templates/kubernetes/terraform/modules/kubernetes/kube2iam/main.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/kube2iam/variables.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/main.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/docs/test-logging-app.yaml delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/files/containers.conf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/files/cwagentconfig.json.tpl delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/files/fluent.conf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/files/host.conf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/files/systemd.conf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/provider.tf delete mode 100644 templates/kubernetes/terraform/modules/kubernetes/variables.tf delete mode 100644 templates/terraform/README.md delete mode 100644 templates/terraform/bootstrap/create-users/.gitignore delete mode 100644 templates/terraform/bootstrap/create-users/main.tf delete mode 100644 templates/terraform/bootstrap/remote-state/main.tf delete mode 100644 templates/terraform/environments/development/main.tf delete mode 100644 templates/terraform/environments/production/main.tf delete mode 100644 templates/terraform/environments/staging/main.tf delete mode 100644 templates/terraform/modules/cognito/main.tf delete mode 100644 templates/terraform/modules/cognito/variables.tf delete mode 100644 templates/terraform/modules/ecr/main.tf delete mode 100644 templates/terraform/modules/ecr/variables.tf delete mode 100644 templates/terraform/modules/ecr/versions.tf delete mode 100644 templates/terraform/modules/eks/main.tf delete mode 100644 templates/terraform/modules/eks/outputs.tf delete mode 100644 templates/terraform/modules/eks/variables.tf delete mode 100644 templates/terraform/modules/eks/versions.tf delete mode 100644 templates/terraform/modules/environment/main.tf delete mode 100644 templates/terraform/modules/environment/provider.tf delete mode 100644 templates/terraform/modules/environment/variables.tf delete mode 100644 templates/terraform/modules/environment/versions.tf delete mode 100644 templates/terraform/modules/kube2iam/README.md delete mode 100644 templates/terraform/modules/kube2iam/main.tf delete mode 100644 templates/terraform/modules/kube2iam/variables.tf delete mode 100644 templates/terraform/modules/kube2iam/versions.tf delete mode 100644 templates/terraform/modules/s3_hosting/main.tf delete mode 100644 templates/terraform/modules/s3_hosting/variables.tf delete mode 100644 templates/terraform/modules/vpc/main.tf delete mode 100644 templates/terraform/modules/vpc/outputs.tf delete mode 100644 templates/terraform/modules/vpc/variables.tf delete mode 100644 templates/terraform/modules/vpc/versions.tf diff --git a/cmd/create.go b/cmd/create.go index 265d5c22e..9fe0cb3aa 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -22,7 +22,6 @@ var createCmd = &cobra.Command{ projectName := args[0] context.Create(projectName, "./") - // TODO create a commit0.yml config.CreateExample(projectName) }, } diff --git a/internal/config/create_example.go b/internal/config/create_example.go index 7038f44d3..5b77a7c60 100644 --- a/internal/config/create_example.go +++ b/internal/config/create_example.go @@ -8,15 +8,16 @@ import ( ) const exampleConfig = `name: %s - -# Context will populated automatically or could be added manually + +# Context is normally populated automatically but could be used to inject global params context: -modules: - # module can be in any format the go-getter supports (path, github, url, etc.) - # supports https://github.com/hashicorp/go-getter#url-format - # - source: "../tests/test_data/modules/ci" - - source: "github.com/zthomas/react-mui-kit"` +# module can be in any format the go-getter supports (path, github, url, etc.) +# supports https://github.com/hashicorp/go-getter#url-format +# - source: "../tests/test_data/modules/ci" + +modules: + - source: "github.com/zthomas/commit0-terraform-basic"` func CreateExample(projectName string) { content := []byte(fmt.Sprintf(exampleConfig, projectName)) diff --git a/internal/generate/generate_infrastructure.go b/internal/generate/generate_infrastructure.go index d9f4fc860..bbe4abf91 100644 --- a/internal/generate/generate_infrastructure.go +++ b/internal/generate/generate_infrastructure.go @@ -21,21 +21,6 @@ var amiLookup = map[string]string{ "eu-central-1": "ami-0e21bc066a9dbabfa", } -// @TODO deprecate & remove -// func GenerateInfrastructure(t *templator.Templator, cfg *config.Commit0Config, wg *sync.WaitGroup, pathPrefix string) { -// if cfg.Infrastructure.AWS.EKS.WorkerAMI == "" { -// ami, found := amiLookup[cfg.Infrastructure.AWS.Region] -// if !found { -// log.Fatalln(aurora.Red(emoji.Sprintf(":exclamation: Unable to look up an AMI for the chosen region"))) -// } - -// cfg.Infrastructure.AWS.EKS.WorkerAMI = ami -// } -// data := templator.GenericTemplateData{Config: *cfg} - -// t.Terraform.TemplateFiles(data, false, wg, pathPrefix) -// } - // GetOutputs captures the terraform output for the specific variables func GetOutputs(cfg *config.Commit0Config, pathPrefix string, outputs []string) map[string]string { outputsMap := make(map[string]string) diff --git a/internal/generate/generate_modules.go b/internal/generate/generate_modules.go index a4edc5cba..d601d7b8a 100644 --- a/internal/generate/generate_modules.go +++ b/internal/generate/generate_modules.go @@ -163,7 +163,6 @@ func ExecuteTemplates(templates []*TemplateConfig, data interface{}, delimiters // @TODO if strict mode then only copy file name := path.Base(source) template, err := template.New(name).Delims(leftDelim, rightDelim).Funcs(util.FuncMap).ParseFiles(source) - // flog.Infof("Templating %s: %s => %s", name, source, dest) err = template.Execute(f, data) if err != nil { diff --git a/pkg/util/flog/log.go b/pkg/util/flog/log.go index 07bdb1276..7166026b1 100644 --- a/pkg/util/flog/log.go +++ b/pkg/util/flog/log.go @@ -7,8 +7,6 @@ import ( "github.com/logrusorgru/aurora" ) -// TODO support log levels - // Warnf logs a formatted error message func Infof(format string, a ...interface{}) { log.Println(aurora.Cyan(emoji.Sprintf(format, a...))) diff --git a/templates/commit0/.gitignore b/templates/commit0/.gitignore deleted file mode 100644 index e984de722..000000000 --- a/templates/commit0/.gitignore +++ /dev/null @@ -1 +0,0 @@ -secrets.yml \ No newline at end of file diff --git a/templates/commit0/commit0.tmpl b/templates/commit0/commit0.tmpl deleted file mode 100644 index 1eabf0792..000000000 --- a/templates/commit0/commit0.tmpl +++ /dev/null @@ -1,51 +0,0 @@ -organization: {{.Organization}} -name: {{.ProjectName}} -description: {{.Description}} -maintainers: - {{range .Maintainers}} - # - name: {{.Name}} - # email: {{.Email}} - {{end}} - -infrastructure: - aws: - accountId: {{ .Infrastructure.AWS.AccountID }} - region: {{ .Infrastructure.AWS.Region }} - eks: - clusterName: {{.ProjectName}}-staging-{{ .Infrastructure.AWS.Region }} - cognito: - enabled: true - s3Hosting: - enabled: true - -frontend: - framework: {{.FrontendFramework }} - ci: - system: github - app: - name: {{.ProjectName }} - -services: - {{range .Services}} - - name: {{.Name}} - description: {{.Description}} - language: {{.Language}} - gitRepo: {{.GitRepo}} - dockerRepo: - ci: - system: circleci - buildImage: golang/golang - buildTag: 1.12 - buildCommand: make build - testCommand: make test - network: - grpc: - host: 0.0.0.0 - port: 3000 - http: - enabled: true - port: 8080 - web: - enabled: true - port: 8090 - {{end}} diff --git a/templates/kubernetes/terraform/environments/development/main.tf b/templates/kubernetes/terraform/environments/development/main.tf deleted file mode 100644 index fbf7e5065..000000000 --- a/templates/kubernetes/terraform/environments/development/main.tf +++ /dev/null @@ -1,41 +0,0 @@ -terraform { - backend "s3" { - bucket = "{{ .Config.Name }}-development-terraform-state" - key = "infrastructure/terraform/environments/development/kubernetes" - encrypt = true - region = "{{ .Config.Infrastructure.AWS.Region }}" - dynamodb_table = "{{ .Config.Name }}-development-terraform-state-locks" - } -} - -# Provision kubernetes resources required to run services/applications -module "kubernetes" { - source = "../../modules/kubernetes" - - environment = "development" - region = "{{ .Config.Infrastructure.AWS.Region }}" - - # Authenticate with the EKS cluster via the cluster id - cluster_name = "{{ .Config.Infrastructure.AWS.EKS.ClusterName }}" - - # Assume-role policy used by monitoring fluentd daemonset - assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json - - external_dns_zone = "{{ .Config.Frontend.Hostname }}" - external_dns_owner_id = "{{ GenerateUUID }}" - external_dns_assume_roles = [ "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/k8s-{{ .Config.Infrastructure.AWS.EKS.ClusterName }}-workers" ] -} - -# Data sources for EKS IAM -data "aws_caller_identity" "current" {} - -data "aws_iam_policy_document" "assumerole_root_policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] - } - } -} diff --git a/templates/kubernetes/terraform/environments/production/main.tf b/templates/kubernetes/terraform/environments/production/main.tf deleted file mode 100644 index d3fc76c1f..000000000 --- a/templates/kubernetes/terraform/environments/production/main.tf +++ /dev/null @@ -1,41 +0,0 @@ -terraform { - backend "s3" { - bucket = "{{ .Config.Name }}-production-terraform-state" - key = "infrastructure/terraform/environments/production/kubernetes" - encrypt = true - region = "{{ .Config.Infrastructure.AWS.Region }}" - dynamodb_table = "{{ .Config.Name }}-production-terraform-state-locks" - } -} - -# Provision kubernetes resources required to run services/applications -module "kubernetes" { - source = "../../modules/kubernetes" - - environment = "production" - region = "{{ .Config.Infrastructure.AWS.Region }}" - - # Authenticate with the EKS cluster via the cluster id - cluster_name = "{{ .Config.Infrastructure.AWS.EKS.ClusterName }}" - - # Assume-role policy used by monitoring fluentd daemonset - assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json - - external_dns_zone = "{{ .Config.Frontend.Hostname }}" - external_dns_owner_id = "{{ GenerateUUID }}" - external_dns_assume_roles = [ "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/k8s-{{ .Config.Infrastructure.AWS.EKS.ClusterName }}-workers" ] -} - -# Data sources for EKS IAM -data "aws_caller_identity" "current" {} - -data "aws_iam_policy_document" "assumerole_root_policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] - } - } -} diff --git a/templates/kubernetes/terraform/environments/staging/main.tf b/templates/kubernetes/terraform/environments/staging/main.tf deleted file mode 100644 index af6345866..000000000 --- a/templates/kubernetes/terraform/environments/staging/main.tf +++ /dev/null @@ -1,41 +0,0 @@ -terraform { - backend "s3" { - bucket = "{{ .Config.Name }}-staging-terraform-state" - key = "infrastructure/terraform/environments/staging/kubernetes" - encrypt = true - region = "{{ .Config.Infrastructure.AWS.Region }}" - dynamodb_table = "{{ .Config.Name }}-staging-terraform-state-locks" - } -} - -# Provision kubernetes resources required to run services/applications -module "kubernetes" { - source = "../../modules/kubernetes" - - environment = "staging" - region = "{{ .Config.Infrastructure.AWS.Region }}" - - # Authenticate with the EKS cluster via the cluster id - cluster_name = "{{ .Config.Infrastructure.AWS.EKS.ClusterName }}" - - # Assume-role policy used by monitoring fluentd daemonset - assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json - - external_dns_zone = "{{ .Config.Frontend.Hostname }}" - external_dns_owner_id = "{{ GenerateUUID }}" - external_dns_assume_roles = [ "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/k8s-{{ .Config.Infrastructure.AWS.EKS.ClusterName }}-workers" ] -} - -# Data sources for EKS IAM -data "aws_caller_identity" "current" {} - -data "aws_iam_policy_document" "assumerole_root_policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] - } - } -} diff --git a/templates/kubernetes/terraform/modules/kubernetes/README.md b/templates/kubernetes/terraform/modules/kubernetes/README.md deleted file mode 100644 index 5ef98d709..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# kubernetes tf module - -## Introduction - -This Terraform module contains configuration to provision kubernetes resources. - -## Organization - -``` - main.tf - Configuration entrypoint. - ingress/ - Confioguration required to provision nginx-ingress-controller. - main.tf - provider.tf - variables.tf - monitoring/ - Configuration required to provision cluster monitoring. - main.tf - provider.tf - variables.tf - fluentd/ - main.tf - files/ - ... - cloudwatch/ - main.tf - files/ - ... -``` \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/example/README.md b/templates/kubernetes/terraform/modules/kubernetes/example/README.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/templates/kubernetes/terraform/modules/kubernetes/example/main.tf b/templates/kubernetes/terraform/modules/kubernetes/example/main.tf deleted file mode 100644 index ac3c4b8ed..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/example/main.tf +++ /dev/null @@ -1,30 +0,0 @@ -## Commit (example) service -## NOT CURRENTLY IMPORTED / INSTANTIATED ANYWHERE - -# Create a role and establish a trust relationship with the worker nodes -resource "aws_iam_role" "k8s_worker_commit_service_role" { - name = "k8s-service-commit" - assume_role_policy = data.aws_iam_policy_document.k8s_worker_assumerole_policy.json - force_detach_policies = true -} - -# Policy allowing access to specific AWS resources -data "aws_iam_policy_document" "k8s_commit_service_access_policy" { - statement { - actions = ["s3:ListBucket"] - resources = ["arn:aws:s3:::processed-data-files"] - } - - statement { - actions = ["s3:GetObject"] - resources = ["arn:aws:s3:::processed-data-files/*"] - } -} - -# Add the above policy to the created role -resource "aws_iam_role_policy" "k8s_commit_service_role_policy" { - name = "worker-commit-service-policy" - role = aws_iam_role.k8s_worker_commit_service_role.id - policy = data.aws_iam_policy_document.k8s_commit_service_access_policy.json -} - diff --git a/templates/kubernetes/terraform/modules/kubernetes/example/variables.tf b/templates/kubernetes/terraform/modules/kubernetes/example/variables.tf deleted file mode 100644 index e69de29bb..000000000 diff --git a/templates/kubernetes/terraform/modules/kubernetes/external_dns.tf b/templates/kubernetes/terraform/modules/kubernetes/external_dns.tf deleted file mode 100644 index a31405fdc..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/external_dns.tf +++ /dev/null @@ -1,143 +0,0 @@ -# Trust relationship -data "aws_iam_policy_document" "external_dns_trust_relationship" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["ec2.amazonaws.com"] - } - } - - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - identifiers = var.external_dns_assume_roles - } - } -} - -# external-dns role -resource "aws_iam_role" "external_dns_role" { - name = "k8s-external-dns-role" - assume_role_policy = data.aws_iam_policy_document.external_dns_trust_relationship.json -} - -data "aws_iam_policy_document" "external_dns_policy_doc" { - statement { - sid = "k8sExternalDnsRead" - effect = "Allow" - - actions = [ - "route53:ListHostedZones", - "route53:ListResourceRecordSets", - ] - - resources = ["*"] - } - - statement { - sid = "k8sExternalDnsWrite" - effect = "Allow" - - actions = ["route53:ChangeResourceRecordSets"] - - resources = ["arn:aws:route53:::hostedzone/*"] - } -} - -resource "aws_iam_role_policy" "external_dns_policy" { - name = "k8s-external-dns-policy" - role = aws_iam_role.external_dns_role.id - policy = data.aws_iam_policy_document.external_dns_policy_doc.json -} - -resource "kubernetes_service_account" "external_dns" { - metadata { - name = "external-dns" - namespace = "kube-system" - } -} - -resource "kubernetes_cluster_role" "external_dns" { - metadata { - name = "external-dns" - } - rule { - verbs = ["get", "list", "watch"] - api_groups = [""] - resources = ["pods", "services"] - } - rule { - verbs = ["get", "list", "watch"] - api_groups = ["extensions"] - resources = ["ingresses"] - } -rule { - verbs = ["list"] - api_groups = [""] - resources = ["nodes"] - } -} - -resource "kubernetes_cluster_role_binding" "external_dns" { - metadata { - name = "external-dns" - } - subject { - kind = "ServiceAccount" - name = "external-dns" - namespace = "kube-system" - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "ClusterRole" - name = "external-dns" - } -} - -resource "kubernetes_deployment" "external_dns" { - metadata { - name = "external-dns" - namespace = "kube-system" - } - spec { - replicas = 1 - selector { - match_labels = { - "app" = "external-dns", - } - } - template { - metadata { - labels = { - "app" = "external-dns", - } - annotations = { - "iam.amazonaws.com/role" = "k8s-external-dns-role", - } - } - spec { - container { - name = "external-dns" - image = "registry.opensource.zalan.do/teapot/external-dns:latest" - args = [ - "--source=service", - "--source=ingress", - "--domain-filter=${var.external_dns_zone}", # Give access only to the specified zone - "--provider=aws", - "--aws-zone-type=public", - "--policy=upsert-only", # Prevent ExternalDNS from deleting any records - "--registry=txt", - "--txt-owner-id=${var.external_dns_owner_id}", # ID of txt record to manage state - ] - } - - service_account_name = "external-dns" - automount_service_account_token = true - } - } - } -} diff --git a/templates/kubernetes/terraform/modules/kubernetes/ingress/README.md b/templates/kubernetes/terraform/modules/kubernetes/ingress/README.md deleted file mode 100644 index 42c3fd9ba..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/ingress/README.md +++ /dev/null @@ -1,4 +0,0 @@ -ingress -= - -Configuration of the nginx-ingress-controller. \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/ingress/main.tf b/templates/kubernetes/terraform/modules/kubernetes/ingress/main.tf deleted file mode 100644 index b2baee84e..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/ingress/main.tf +++ /dev/null @@ -1,371 +0,0 @@ -resource "kubernetes_namespace" "ingress_nginx" { - metadata { - name = "ingress-nginx" - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } -} - -resource "kubernetes_config_map" "nginx_configuration" { - metadata { - name = "nginx-configuration" - namespace = "ingress-nginx" - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } - data = { - proxy-real-ip-cidr = "0.0.0.0/0", - use-forwarded-headers = "true", - use-proxy-protocol = "false" - } - depends_on = [kubernetes_namespace.ingress_nginx] -} - -# resource "kubernetes_config_map" "nginx_configuration" { -# metadata { -# name = "nginx-configuration" -# namespace = "ingress-nginx" -# labels = { -# "app.kubernetes.io/name" = "ingress-nginx", -# "app.kubernetes.io/part-of" = "ingress-nginx" -# } -# } -# depends_on = [kubernetes_namespace.ingress_nginx] -# } - -resource "kubernetes_config_map" "tcp_services" { - metadata { - name = "tcp-services" - namespace = "ingress-nginx" - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } - depends_on = [kubernetes_namespace.ingress_nginx] -} - -resource "kubernetes_config_map" "udp_services" { - metadata { - name = "udp-services" - namespace = "ingress-nginx" - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } - depends_on = [kubernetes_namespace.ingress_nginx] -} - -resource "kubernetes_service_account" "nginx_ingress_serviceaccount" { - metadata { - name = "nginx-ingress-serviceaccount" - namespace = "ingress-nginx" - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } - depends_on = [kubernetes_namespace.ingress_nginx] -} - -resource "kubernetes_cluster_role" "nginx_ingress_clusterrole" { - metadata { - name = "nginx-ingress-clusterrole" - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } - rule { - verbs = ["list", "watch"] - api_groups = [""] - resources = ["configmaps", "endpoints", "nodes", "pods", "secrets"] - } - rule { - verbs = ["get"] - api_groups = [""] - resources = ["nodes"] - } - rule { - verbs = ["get", "list", "watch"] - api_groups = [""] - resources = ["services"] - } - rule { - verbs = ["create", "patch"] - api_groups = [""] - resources = ["events"] - } - rule { - verbs = ["get", "list", "watch"] - api_groups = ["extensions", "networking.k8s.io"] - resources = ["ingresses"] - } - rule { - verbs = ["update"] - api_groups = ["extensions", "networking.k8s.io"] - resources = ["ingresses/status"] - } -} - -resource "kubernetes_role" "nginx_ingress_role" { - metadata { - name = "nginx-ingress-role" - namespace = "ingress-nginx" - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } - rule { - verbs = ["get"] - api_groups = [""] - resources = ["configmaps", "pods", "secrets", "namespaces"] - } - rule { - verbs = ["get", "update"] - api_groups = [""] - resources = ["configmaps"] - resource_names = ["ingress-controller-leader-nginx"] - } - rule { - verbs = ["create"] - api_groups = [""] - resources = ["configmaps"] - } - rule { - verbs = ["get"] - api_groups = [""] - resources = ["endpoints"] - } - depends_on = [kubernetes_namespace.ingress_nginx] -} - -resource "kubernetes_role_binding" "nginx_ingress_role_nisa_binding" { - metadata { - name = "nginx-ingress-role-nisa-binding" - namespace = "ingress-nginx" - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } - subject { - kind = "ServiceAccount" - name = "nginx-ingress-serviceaccount" - namespace = "ingress-nginx" - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "Role" - name = "nginx-ingress-role" - } - depends_on = [kubernetes_namespace.ingress_nginx] -} - -resource "kubernetes_cluster_role_binding" "nginx_ingress_clusterrole_nisa_binding" { - metadata { - name = "nginx-ingress-clusterrole-nisa-binding" - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } - subject { - kind = "ServiceAccount" - name = "nginx-ingress-serviceaccount" - namespace = "ingress-nginx" - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "ClusterRole" - name = "nginx-ingress-clusterrole" - } -} - -resource "kubernetes_deployment" "nginx_ingress_controller" { - depends_on = [ - kubernetes_config_map.tcp_services, - kubernetes_config_map.udp_services, - ] - metadata { - name = "nginx-ingress-controller" - namespace = "ingress-nginx" - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } - spec { - replicas = 1 - selector { - match_labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } - template { - metadata { - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - # annotations = { - # "prometheus.io/port" = "10254", - # "prometheus.io/scrape" = "true" - # } - } - spec { - container { - name = "nginx-ingress-controller" - image = "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1" - args = [ - "/nginx-ingress-controller", - "--configmap=$(POD_NAMESPACE)/nginx-configuration", - "--tcp-services-configmap=$(POD_NAMESPACE)/tcp-services", - "--udp-services-configmap=$(POD_NAMESPACE)/udp-services", - "--publish-service=$(POD_NAMESPACE)/ingress-nginx", - "--annotations-prefix=nginx.ingress.kubernetes.io" - ] - port { - name = "http" - container_port = 80 - } - port { - name = "https" - container_port = 443 - } - env { - name = "POD_NAME" - value_from { - field_ref { - field_path = "metadata.name" - } - } - } - env { - name = "POD_NAMESPACE" - value_from { - field_ref { - field_path = "metadata.namespace" - } - } - } - liveness_probe { - http_get { - path = "/healthz" - port = "10254" - scheme = "HTTP" - } - initial_delay_seconds = 10 - timeout_seconds = 10 - period_seconds = 10 - success_threshold = 1 - failure_threshold = 3 - } - readiness_probe { - http_get { - path = "/healthz" - port = "10254" - scheme = "HTTP" - } - timeout_seconds = 10 - period_seconds = 10 - success_threshold = 1 - failure_threshold = 3 - } - lifecycle { - pre_stop { - exec { - command = ["/wait-shutdown"] - } - } - } - security_context { - run_as_user = 33 - allow_privilege_escalation = true - capabilities { - add = ["NET_BIND_SERVICE"] - drop = ["ALL"] - } - } - } - termination_grace_period_seconds = 300 - node_selector = { "kubernetes.io/os" = "linux" } - service_account_name = "nginx-ingress-serviceaccount" - automount_service_account_token = true - } - } - } -} - -resource "kubernetes_service" "ingress_nginx" { - metadata { - name = "ingress-nginx" - namespace = "ingress-nginx" - labels = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - } - spec { - port { - name = "http" - port = 80 - target_port = "http" - } - port { - name = "https" - port = 443 - target_port = "https" - } - selector = { - "app.kubernetes.io/name" = "ingress-nginx", - "app.kubernetes.io/part-of" = "ingress-nginx" - } - type = "LoadBalancer" - external_traffic_policy = "Local" - } - depends_on = [kubernetes_namespace.ingress_nginx] -} - -# HTTPS Load balancer -# resource "kubernetes_service" "ingress_nginx" { -# metadata { -# name = "ingress-nginx" -# namespace = "ingress-nginx" -# labels = { -# "app.kubernetes.io/name" = "ingress-nginx", -# "app.kubernetes.io/part-of" = "ingress-nginx" -# } -# annotations = { -# "service.beta.kubernetes.io/aws-load-balancer-backend-protocol" = "http", -# "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout" = "60", -# "service.beta.kubernetes.io/aws-load-balancer-ssl-cert" = var.load_balancer_ssl_cert_arn, -# "service.beta.kubernetes.io/aws-load-balancer-ssl-ports" = "https" -# } -# } -# spec { -# port { -# name = "http" -# port = 80 -# target_port = "http" -# } -# port { -# name = "https" -# port = 443 -# target_port = "http" -# } -# selector = { -# "app.kubernetes.io/name" = "ingress-nginx", -# "app.kubernetes.io/part-of" = "ingress-nginx" -# } -# type = "LoadBalancer" -# } -# depends_on = [kubernetes_namespace.ingress_nginx] -# } diff --git a/templates/kubernetes/terraform/modules/kubernetes/ingress/variables.tf b/templates/kubernetes/terraform/modules/kubernetes/ingress/variables.tf deleted file mode 100644 index 3d6239424..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/ingress/variables.tf +++ /dev/null @@ -1,11 +0,0 @@ -variable "environment" { - description = "The environment (dev/staging/prod)" -} - -variable "region" { - description = "The AWS region" -} - -variable "load_balancer_ssl_cert_arn" { - description = "Load Balancer SSL certificate ARN" -} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/kube2iam/README.md b/templates/kubernetes/terraform/modules/kubernetes/kube2iam/README.md deleted file mode 100644 index dcfe37fa4..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/kube2iam/README.md +++ /dev/null @@ -1,6 +0,0 @@ -kube2iam -= - -Provide IAM credentials to containers running inside a kubernetes cluster based on annotations. - -Will be deprecated soon in favour of: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/. \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/kube2iam/main.tf b/templates/kubernetes/terraform/modules/kubernetes/kube2iam/main.tf deleted file mode 100755 index 75d16d8d7..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/kube2iam/main.tf +++ /dev/null @@ -1,105 +0,0 @@ -resource "kubernetes_service_account" "kube2iam" { - metadata { - name = "kube2iam" - namespace = "kube-system" - } -} - -resource "kubernetes_cluster_role" "kube2iam" { - metadata { - name = "kube2iam" - } - rule { - verbs = ["get", "watch", "list"] - api_groups = [""] - resources = ["namespaces", "pods"] - } -} - -resource "kubernetes_cluster_role_binding" "kube2iam" { - metadata { - name = "kube2iam" - } - subject { - kind = "ServiceAccount" - name = "kube2iam" - namespace = "kube-system" - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "ClusterRole" - name = "kube2iam" - } -} - -resource "kubernetes_daemonset" "kube2iam" { - metadata { - name = "kube2iam" - namespace = "kube-system" - labels = { - app = "kube2iam" - } - } - spec { - selector { - match_labels = { - name = "kube2iam" - } - } - template { - metadata { - labels = { - name = "kube2iam" - } - } - spec { - container { - name = "kube2iam" - image = "jtblin/kube2iam:0.10.8" - args = [ - "--auto-discover-base-arn", - "--auto-discover-default-role", - "--iptables=true", - "--host-ip=$(HOST_IP)", - "--host-interface=eni+", - # "--node=$(NODE_NAME)", - "--use-regional-sts-endpoint", - "--log-level=info" - ] - port { - name = "http" - host_port = 8181 - container_port = 8181 - } - env { - name = "HOST_IP" - value_from { - field_ref { - field_path = "status.podIP" - } - } - } - # env { - # name = "NODE_NAME" - # value_from { - # field_ref { - # field_path = "spec.nodeName" - # } - # } - # } - env { - name = "AWS_REGION" - value = var.region - } - security_context { - privileged = true - } - } - service_account_name = "kube2iam" - automount_service_account_token = true - host_network = true - } - } - } -} - diff --git a/templates/kubernetes/terraform/modules/kubernetes/kube2iam/variables.tf b/templates/kubernetes/terraform/modules/kubernetes/kube2iam/variables.tf deleted file mode 100644 index dc049cfa6..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/kube2iam/variables.tf +++ /dev/null @@ -1,7 +0,0 @@ -variable "region" { - description = "AWS Region" -} - -variable "environment" { - description = "Environment" -} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/main.tf b/templates/kubernetes/terraform/modules/kubernetes/main.tf deleted file mode 100644 index 8b3bd2b20..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/main.tf +++ /dev/null @@ -1,20 +0,0 @@ -module "kube2iam" { - source = "./kube2iam" - environment = var.environment - region = var.region -} - -module "monitoring" { - source = "./monitoring" - environment = var.environment - region = var.region - assume_role_policy = var.assume_role_policy - cluster_name = var.cluster_name -} - -module "ingress" { - source = "./ingress" - environment = var.environment - region = var.region - load_balancer_ssl_cert_arn = "" -} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent.tf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent.tf deleted file mode 100644 index a58c7911b..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/monitoring/cloudwatch_agent.tf +++ /dev/null @@ -1,218 +0,0 @@ -resource "kubernetes_service_account" "cloudwatch_agent" { - metadata { - name = "cloudwatch-agent" - namespace = "amazon-cloudwatch" - } - depends_on = [kubernetes_namespace.amazon_cloudwatch] -} - -resource "kubernetes_cluster_role" "cloudwatch_agent_role" { - metadata { - name = "cloudwatch-agent-role" - } - rule { - verbs = ["list", "watch"] - api_groups = [""] - resources = ["pods", "nodes", "endpoints"] - } - rule { - verbs = ["list", "watch"] - api_groups = ["apps"] - resources = ["replicasets"] - } - rule { - verbs = ["list", "watch"] - api_groups = ["batch"] - resources = ["jobs"] - } - rule { - verbs = ["get"] - api_groups = [""] - resources = ["nodes/proxy"] - } - rule { - verbs = ["create"] - api_groups = [""] - resources = ["nodes/stats", "configmaps", "events"] - } - rule { - verbs = ["get", "update"] - api_groups = [""] - resources = ["configmaps"] - resource_names = ["cwagent-clusterleader"] - } -} - -resource "kubernetes_cluster_role_binding" "cloudwatch_agent_role_binding" { - metadata { - name = "cloudwatch-agent-role-binding" - } - subject { - kind = "ServiceAccount" - name = "cloudwatch-agent" - namespace = "amazon-cloudwatch" - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "ClusterRole" - name = "cloudwatch-agent-role" - } -} - -resource "kubernetes_config_map" "cwagentconfig" { - metadata { - name = "cwagentconfig" - namespace = "amazon-cloudwatch" - } - data = { - "cwagentconfig.json" = templatefile( - "${path.module}/files/cwagentconfig.json.tpl", - { - region = var.region, - cluster_name = var.cluster_name - } - ) - } - depends_on = [kubernetes_namespace.amazon_cloudwatch] -} - -resource "kubernetes_daemonset" "cloudwatch_agent" { - # Explicitly declare dependency on config map - depends_on = [ - kubernetes_config_map.cwagentconfig - ] - metadata { - name = "cloudwatch-agent" - namespace = "amazon-cloudwatch" - } - spec { - selector { - match_labels = { name = "cloudwatch-agent" } - } - template { - metadata { - labels = { name = "cloudwatch-agent" } - annotations = { - "iam.amazonaws.com/role" = "k8s-${var.environment}-monitoring" - } - } - spec { - volume { - name = "cwagentconfig" - config_map { - name = "cwagentconfig" - } - } - volume { - name = "rootfs" - host_path { - path = "/" - } - } - volume { - name = "dockersock" - host_path { - path = "/var/run/docker.sock" - } - } - volume { - name = "varlibdocker" - host_path { - path = "/var/lib/docker" - } - } - volume { - name = "sys" - host_path { - path = "/sys" - } - } - volume { - name = "devdisk" - host_path { - path = "/dev/disk/" - } - } - container { - name = "cloudwatch-agent" - image = "amazon/cloudwatch-agent:latest" - port { - container_port = 8125 - host_port = 8125 - protocol = "UDP" - } - env { - name = "HOST_IP" - value_from { - field_ref { - field_path = "status.hostIP" - } - } - } - env { - name = "HOST_NAME" - value_from { - field_ref { - field_path = "spec.nodeName" - } - } - } - env { - name = "K8S_NAMESPACE" - value_from { - field_ref { - field_path = "metadata.namespace" - } - } - } - env { - name = "CI_VERSION" - value = "k8s/1.0.0" - } - resources { - limits { - cpu = "200m" - memory = "200Mi" - } - requests { - memory = "200Mi" - cpu = "200m" - } - } - volume_mount { - name = "cwagentconfig" - mount_path = "/etc/cwagentconfig" - } - volume_mount { - name = "rootfs" - read_only = true - mount_path = "/rootfs" - } - volume_mount { - name = "dockersock" - read_only = true - mount_path = "/var/run/docker.sock" - } - volume_mount { - name = "varlibdocker" - read_only = true - mount_path = "/var/lib/docker" - } - volume_mount { - name = "sys" - read_only = true - mount_path = "/sys" - } - volume_mount { - name = "devdisk" - read_only = true - mount_path = "/dev/disk" - } - } - termination_grace_period_seconds = 60 - service_account_name = "cloudwatch-agent" - automount_service_account_token = true - } - } - } -} diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/docs/test-logging-app.yaml b/templates/kubernetes/terraform/modules/kubernetes/monitoring/docs/test-logging-app.yaml deleted file mode 100644 index fb13b9d93..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/monitoring/docs/test-logging-app.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: counter -spec: - containers: - - name: count - image: busybox - args: [/bin/sh, -c, - 'i=0; while true; do echo "$i: $(date)"; i=$((i+1)); sleep 1; done'] \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/containers.conf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/containers.conf deleted file mode 100644 index db5971f2d..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/containers.conf +++ /dev/null @@ -1,44 +0,0 @@ - - @type tail - @id in_tail_container_logs - @label @containers - path /var/log/containers/*.log - pos_file /var/log/fluentd-containers.log.pos - tag * - read_from_head true - - @type json - time_format %Y-%m-%dT%H:%M:%S.%NZ - - - - \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/cwagentconfig.json.tpl b/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/cwagentconfig.json.tpl deleted file mode 100644 index 0a969ddba..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/cwagentconfig.json.tpl +++ /dev/null @@ -1,21 +0,0 @@ -{ - "agent": { - "region": "${region}" - }, - "logs": { - "metrics_collected": { - "kubernetes": { - "cluster_name": "${cluster_name}", - "metrics_collection_interval": 60 - } - }, - "force_flush_interval": 5 - }, - "metrics": { - "metrics_collected": { - "statsd": { - "service_address": ":8125" - } - } - } -} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/fluent.conf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/fluent.conf deleted file mode 100644 index 31bc30944..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/fluent.conf +++ /dev/null @@ -1,10 +0,0 @@ -@include containers.conf -@include systemd.conf -@include host.conf - - - @type null - - - log_level debug - \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/host.conf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/host.conf deleted file mode 100644 index a43a2aa2a..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/host.conf +++ /dev/null @@ -1,69 +0,0 @@ - - @type tail - @id in_tail_dmesg - @label @hostlogs - path /var/log/dmesg - pos_file /var/log/dmesg.log.pos - tag host.dmesg - read_from_head true - - @type syslog - - - - - @type tail - @id in_tail_secure - @label @hostlogs - path /var/log/secure - pos_file /var/log/secure.log.pos - tag host.secure - read_from_head true - - @type syslog - - - - - @type tail - @id in_tail_messages - @label @hostlogs - path /var/log/messages - pos_file /var/log/messages.log.pos - tag host.messages - read_from_head true - - @type syslog - - - - \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/systemd.conf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/systemd.conf deleted file mode 100644 index c1a6e8240..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/monitoring/files/systemd.conf +++ /dev/null @@ -1,75 +0,0 @@ - - @type systemd - @id in_systemd_kubelet - @label @systemd - filters [{ "_SYSTEMD_UNIT": "kubelet.service" }] - - field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} - field_map_strict true - - path /var/log/journal - pos_file /var/log/fluentd-journald-kubelet.pos - read_from_head true - tag kubelet.service - - - - @type systemd - @id in_systemd_kubeproxy - @label @systemd - filters [{ "_SYSTEMD_UNIT": "kubeproxy.service" }] - - field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} - field_map_strict true - - path /var/log/journal - pos_file /var/log/fluentd-journald-kubeproxy.pos - read_from_head true - tag kubeproxy.service - - - - @type systemd - @id in_systemd_docker - @label @systemd - filters [{ "_SYSTEMD_UNIT": "docker.service" }] - - field_map {"MESSAGE": "message", "_HOSTNAME": "hostname", "_SYSTEMD_UNIT": "systemd_unit"} - field_map_strict true - - path /var/log/journal - pos_file /var/log/fluentd-journald-docker.pos - read_from_head true - tag docker.service - - - \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd.tf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd.tf deleted file mode 100644 index 24ceca0dd..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/monitoring/fluentd.tf +++ /dev/null @@ -1,224 +0,0 @@ -data "local_file" "containers" { - filename = "${path.module}/files/containers.conf" -} - -data "local_file" "fluent" { - filename = "${path.module}/files/fluent.conf" -} - -data "local_file" "host" { - filename = "${path.module}/files/host.conf" -} - -data "local_file" "systemd" { - filename = "${path.module}/files/systemd.conf" -} - -resource "kubernetes_config_map" "cluster_info" { - metadata { - name = "cluster-info" - namespace = "amazon-cloudwatch" - } - data = { - "cluster.name" = var.cluster_name - "logs.region" = var.region - } - depends_on = [kubernetes_namespace.amazon_cloudwatch] -} - -resource "kubernetes_service_account" "fluentd" { - metadata { - name = "fluentd" - namespace = "amazon-cloudwatch" - } - depends_on = [kubernetes_namespace.amazon_cloudwatch] -} - -resource "kubernetes_cluster_role" "fluentd_role" { - metadata { - name = "fluentd-role" - } - rule { - verbs = ["get", "list", "watch"] - api_groups = [""] - resources = ["namespaces", "pods", "pods/logs"] - } -} - -resource "kubernetes_cluster_role_binding" "fluentd_role_binding" { - metadata { - name = "fluentd-role-binding" - } - subject { - kind = "ServiceAccount" - name = "fluentd" - namespace = "amazon-cloudwatch" - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "ClusterRole" - name = "fluentd-role" - } - depends_on = [kubernetes_service_account.fluentd] -} - -resource "kubernetes_config_map" "fluentd_config" { - metadata { - name = "fluentd-config" - namespace = "amazon-cloudwatch" - labels = { k8s-app = "fluentd-cloudwatch" } - } - data = { - "containers.conf" = data.local_file.containers.content - "fluent.conf" = data.local_file.fluent.content - "host.conf" = data.local_file.host.content - "systemd.conf" = data.local_file.systemd.content - } - depends_on = [kubernetes_namespace.amazon_cloudwatch] -} - -resource "kubernetes_daemonset" "fluentd_cloudwatch" { - depends_on = [ - kubernetes_config_map.cluster_info, - kubernetes_config_map.fluentd_config - ] - metadata { - name = "fluentd-cloudwatch" - namespace = "amazon-cloudwatch" - labels = { - k8s-app = "fluentd-cloudwatch" - } - } - spec { - selector { - match_labels = { - k8s-app = "fluentd-cloudwatch" - } - } - template { - metadata { - labels = { - k8s-app = "fluentd-cloudwatch" - } - annotations = { - configHash = "8915de4cf9c3551a8dc74c0137a3e83569d28c71044b0359c2578d2e0461825", - "iam.amazonaws.com/role" = "k8s-${var.environment}-monitoring" - } - } - spec { - volume { - name = "config-volume" - config_map { - name = "fluentd-config" - } - } - volume { - name = "fluentdconf" - } - volume { - name = "varlog" - host_path { - path = "/var/log" - } - } - volume { - name = "varlibdockercontainers" - host_path { - path = "/var/lib/docker/containers" - } - } - volume { - name = "runlogjournal" - host_path { - path = "/run/log/journal" - } - } - volume { - name = "dmesg" - host_path { - path = "/var/log/dmesg" - } - } - init_container { - name = "copy-fluentd-config" - image = "busybox" - command = ["sh", "-c", "cp /config-volume/..data/* /fluentd/etc"] - volume_mount { - name = "config-volume" - mount_path = "/config-volume" - } - volume_mount { - name = "fluentdconf" - mount_path = "/fluentd/etc" - } - } - init_container { - name = "update-log-driver" - image = "busybox" - command = ["sh", "-c", ""] - } - container { - name = "fluentd-cloudwatch" - image = "fluent/fluentd-kubernetes-daemonset:v1.3.3-debian-cloudwatch-1.4" - env { - name = "REGION" - value_from { - config_map_key_ref { - name = "cluster-info" - key = "logs.region" - } - } - } - env { - name = "CLUSTER_NAME" - value_from { - config_map_key_ref { - name = "cluster-info" - key = "cluster.name" - } - } - } - resources { - limits { - memory = "200Mi" - } - requests { - cpu = "100m" - memory = "200Mi" - } - } - volume_mount { - name = "config-volume" - mount_path = "/config-volume" - } - volume_mount { - name = "fluentdconf" - mount_path = "/fluentd/etc" - } - volume_mount { - name = "varlog" - mount_path = "/var/log" - } - volume_mount { - name = "varlibdockercontainers" - read_only = true - mount_path = "/var/lib/docker/containers" - } - volume_mount { - name = "runlogjournal" - read_only = true - mount_path = "/run/log/journal" - } - volume_mount { - name = "dmesg" - read_only = true - mount_path = "/var/log/dmesg" - } - } - termination_grace_period_seconds = 30 - service_account_name = "fluentd" - automount_service_account_token = true - } - } - } -} diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf deleted file mode 100644 index 6c395ab50..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/monitoring/main.tf +++ /dev/null @@ -1,24 +0,0 @@ -resource "aws_iam_role" "k8s_monitoring" { - name = "{{ .Config.Name }}-k8s-${var.environment}-monitoring" - assume_role_policy = var.assume_role_policy - force_detach_policies = true -} - -# Create amazon-cloudwatch kubernetes namespace for fluentd/cloudwatchagent -resource "kubernetes_namespace" "amazon_cloudwatch" { - metadata { - name = "amazon-cloudwatch" - labels = { - name = "amazon-cloudwatch" - } - } -} - -data "aws_iam_policy" "CloudWatchAgentServerPolicy" { - arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy" -} - -resource "aws_iam_role_policy_attachment" "k8s_monitoring_role_policy" { - role = aws_iam_role.k8s_monitoring.id - policy_arn = data.aws_iam_policy.CloudWatchAgentServerPolicy.arn -} diff --git a/templates/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf b/templates/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf deleted file mode 100644 index 93e911d14..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/monitoring/variables.tf +++ /dev/null @@ -1,15 +0,0 @@ -variable "region" { - description = "AWS Region" -} - -variable "environment" { - description = "Environment" -} - -variable "assume_role_policy" { - description = "Assume-role policy" -} - -variable "cluster_name" { - description = "Cluster name" -} \ No newline at end of file diff --git a/templates/kubernetes/terraform/modules/kubernetes/provider.tf b/templates/kubernetes/terraform/modules/kubernetes/provider.tf deleted file mode 100644 index f33146f77..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/provider.tf +++ /dev/null @@ -1,20 +0,0 @@ -# https://github.com/terraform-providers/terraform-provider-kubernetes/issues/161#issuecomment-461190931 - -data "aws_eks_cluster" "cluster" { - name = var.cluster_name -} - -data "aws_eks_cluster_auth" "cluster_auth" { - name = data.aws_eks_cluster.cluster.name -} - -provider "aws" { - region = var.region -} - -provider "kubernetes" { - host = data.aws_eks_cluster.cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) - token = data.aws_eks_cluster_auth.cluster_auth.token - load_config_file = false -} diff --git a/templates/kubernetes/terraform/modules/kubernetes/variables.tf b/templates/kubernetes/terraform/modules/kubernetes/variables.tf deleted file mode 100644 index f75de7f35..000000000 --- a/templates/kubernetes/terraform/modules/kubernetes/variables.tf +++ /dev/null @@ -1,28 +0,0 @@ -variable "region" { - description = "AWS Region" -} - -variable "environment" { - description = "Environment" -} - -variable "cluster_name" { - description = "Kubernetes cluster name" -} - -variable "assume_role_policy" { - description = "Assume-role policy for monitoring" -} - -variable "external_dns_zone" { - description = "R53 zone that external-dns will have access to" -} - -variable "external_dns_owner_id" { - description = "Unique id of the TXT record that external-dns will use to store state (can just be a uuid)" -} - -variable "external_dns_assume_roles" { - description = "List of roles that should be able to assume the external dns role (most likely the role of the cluster worker nodes)" - type = list(string) -} diff --git a/templates/terraform/README.md b/templates/terraform/README.md deleted file mode 100644 index 6685a1faf..000000000 --- a/templates/terraform/README.md +++ /dev/null @@ -1,101 +0,0 @@ -## Guidelines & Style Convention Summary - -- All Terraform configuration should be formatted with `terraform fmt` before being accepted into this repository. -- This repository is Terraform version >= 0.12, as such, leverage features from this release whenever possible. - See https://www.terraform.io/upgrade-guides/0-12.html for more information. -- Leverage community-maintained Terraform modules whenever possible. -- Attempt to minimize duplication whenever possible, but only within reason -- sometimes duplication is an acceptable solution. -- Follow style conventions described in `docs/guide.pdf` whenever possible. -- Whenever possible, inject resources down versus referencing resources across modules. This has been made easier with new features in v0.12. -- Whenever possible, define the types of variables. - -### Module Conventions - -- All modules should contain the following: - - `README.md`: A description of the module. - `main.tf`: Module entrypoint where instantiation of resources happens. - `variables.tf`: Module variables. - `outputs.tf`: Output values (optional). - `files/`: Any / all files required by the module. - -- All module variables must have a description. -- Again, leverage community-maintained Terraform modules whenever possible. -- Avoid writing a module that is simply a wrapper of a Terraform resource unless absolutely necessary. - -### Environment Conventions - -- All environments should contain the following: - - `main.tf`: Toplevel terraform configuration file that instantiates the `environment` module. - -- Configuration should be pushed "top->down" from the `environment` module to it's submodules. - -### The Environment Module - -- The `environment` module can be considered the top-level module, all other modules are imported from this module. -- Environment-specific variables should be exposed via the `variables.tf` file in this module, where they will be set from within the appropriate environment in the `environments/` directory. -- The `environment` module contains the following: - - `main.tf`: Module entrypoint where instantiation of resources happens. - `backend.tf`: Terraform remote state configuration. - `provider.tf`: Provider configuration. - `variables.tf`: Environment-specific variables are desclared here. - `versions.tf`: Terraform version information. - `files/`: (DEPRECATED) - -## Directory Structure - -``` - README.md - environments/ - production/ - main.tf - staging/ - main.tf - development/ - main.tf - docs/ - guide.pdf - modules/ - environment/ - ... - / - files/ - scripts/ - main.tf - outputs.tf - variables.tf - / - ... -``` - -## AWS Guidelines - -- TODO: Identity/Access Management (IAM) Guidelines - -## Kubernetes Guidelines - -- When to use the Terraform Kuberenetes Provider and when to use manifests? - - - Use the Terraform Kubernetes Provider (`provider "kubernetes"`) whenever you are provisioning a resource that could be considered relatively static (think Ingress, RoleBinding, CluterRoleBinding, etc). - - - Use conventional Kubernetes manifests / `kubectl` when provisioning resouirces that could be considered dynamic (think Deployments). - -## Application - - 1. Set up a profile for your project with your credentials in a specific profile in `~/.aws/credentials` and then export the following env var: - `export AWS_PROFILE=` - - 2. Run the following from the appropriate environment directory under `environments/`: - - ``` - environment/development$ terraform init - environment/development$ terraform plan - ``` - -## To use kubectl with the created EKS cluster: - - Exchange your aws credentials for kubernetes credentials. - This will add a new context to your kubeconfig. - `aws eks update-kubeconfig --name --region ` diff --git a/templates/terraform/bootstrap/create-users/.gitignore b/templates/terraform/bootstrap/create-users/.gitignore deleted file mode 100644 index f2eacdf5c..000000000 --- a/templates/terraform/bootstrap/create-users/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# The state will have keys in it which can be ignored -terraform.tfstate* diff --git a/templates/terraform/bootstrap/create-users/main.tf b/templates/terraform/bootstrap/create-users/main.tf deleted file mode 100644 index cd12fa78e..000000000 --- a/templates/terraform/bootstrap/create-users/main.tf +++ /dev/null @@ -1,23 +0,0 @@ -provider "aws" { - region = "{{ .Config.Infrastructure.AWS.Region }}" -} - -# Create the CI User -resource "aws_iam_user" "ci_user" { - name = "ci-user" -} - -# Create a keypair to be used by CI systems -resource "aws_iam_access_key" "ci_user" { - user = aws_iam_user.ci_user.name -} - -# Add the keys to AWS secrets manager -resource "aws_secretsmanager_secret" "ci_user_keys" { - name = "ci-user-keys" -} - -resource "aws_secretsmanager_secret_version" "ci_user_keys" { - secret_id = aws_secretsmanager_secret.ci_user_keys.id - secret_string = jsonencode(map("access_key_id", aws_iam_access_key.ci_user.id, "secret_key", aws_iam_access_key.ci_user.secret)) -} diff --git a/templates/terraform/bootstrap/remote-state/main.tf b/templates/terraform/bootstrap/remote-state/main.tf deleted file mode 100644 index 5965bbc42..000000000 --- a/templates/terraform/bootstrap/remote-state/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -provider "aws" { - region = "{{ .Config.Infrastructure.AWS.Region }}" -} - -resource "aws_s3_bucket" "terraform_remote_state" { - bucket = "{{ .Config.Name }}-${var.environment}-terraform-state" - acl = "private" - - versioning { - enabled = true - } -} - -resource "aws_s3_bucket_public_access_block" "terraform_remote_state" { - bucket = aws_s3_bucket.terraform_remote_state.id - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -resource "aws_dynamodb_table" "terraform_state_locks" { - name = "{{ .Config.Name }}-${var.environment}-terraform-state-locks" - read_capacity = 2 - write_capacity = 2 - hash_key = "LockID" - - attribute { - name = "LockID" - type = "S" - } -} - -variable "environment" { - description = "The environment (development/staging/production)" -} diff --git a/templates/terraform/environments/development/main.tf b/templates/terraform/environments/development/main.tf deleted file mode 100644 index 9c2530ded..000000000 --- a/templates/terraform/environments/development/main.tf +++ /dev/null @@ -1,58 +0,0 @@ -terraform { - required_version = ">= 0.12" - backend "s3" { - bucket = "{{ .Config.Name }}-development-terraform-state" - key = "infrastructure/terraform/environments/development/main" - encrypt = true - region = "{{ .Config.Infrastructure.AWS.Region }}" - dynamodb_table = "{{ .Config.Name }}-development-terraform-state-locks" - } -} - -# Instantiate the development environment -module "development" { - source = "../../modules/environment" - environment = "development" - - # Project configuration - project = "{{ .Config.Name }}" - region = "{{ .Config.Infrastructure.AWS.Region }}" - allowed_account_ids = ["{{ .Config.Infrastructure.AWS.AccountID }}"] - - {{- if ne .Config.Infrastructure.AWS.EKS.ClusterName "" }} - # ECR configuration - ecr_repositories = ["{{ .Config.Infrastructure.AWS.EKS.ClusterName }}"] - - # EKS configuration - eks_worker_instance_type = "t2.small" - eks_worker_asg_min_size = 1 - eks_worker_asg_max_size = 3 - - # EKS-Optimized AMI for your region: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html - # https://us-east-1.console.aws.amazon.com/systems-manager/parameters/%252Faws%252Fservice%252Feks%252Foptimized-ami%252F1.14%252Famazon-linux-2%252Frecommended%252Fimage_id/description?region=us-east-1 - eks_worker_ami = "{{ .Config.Infrastructure.AWS.EKS.WorkerAMI }}" - {{- end }} - - {{- if .Config.Infrastructure.AWS.Cognito.Enabled }} - # Cognito configuration - user_pool = "{{ .Config.Name }}-development" - hostname = "{{ .Config.Frontend.Hostname }}" - {{- end}} - - # Hosting configuration - s3_hosting_buckets = [ - "{{ .Config.Name }}-development" - ] - s3_hosting_cert_domain = "{{ .Config.Frontend.Hostname}}" - -} - -{{- if .Config.Infrastructure.AWS.Cognito.Enabled }} -output "cognito_client_id" { - value = module.staging.cognito.cognito_client_id -} - -output "cognito_pool_id" { - value = module.staging.cognito.cognito_pool_id -} -{{- end}} diff --git a/templates/terraform/environments/production/main.tf b/templates/terraform/environments/production/main.tf deleted file mode 100644 index 66ee42f46..000000000 --- a/templates/terraform/environments/production/main.tf +++ /dev/null @@ -1,58 +0,0 @@ -terraform { - required_version = ">= 0.12" - backend "s3" { - bucket = "{{ .Config.Name }}-production-terraform-state" - key = "infrastructure/terraform/environments/production/main" - encrypt = true - region = "{{ .Config.Infrastructure.AWS.Region }}" - dynamodb_table = "{{ .Config.Name }}-production-terraform-state-locks" - } -} - -# Instantiate the production environment -module "production" { - source = "../../modules/environment" - environment = "production" - - # Project configuration - project = "{{ .Config.Name }}" - region = "{{ .Config.Infrastructure.AWS.Region }}" - allowed_account_ids = ["{{ .Config.Infrastructure.AWS.AccountID }}"] - - {{- if ne .Config.Infrastructure.AWS.EKS.ClusterName "" }} - # ECR configuration - ecr_repositories = ["{{ .Config.Infrastructure.AWS.EKS.ClusterName }}"] - - # EKS configuration - eks_worker_instance_type = "m4.large" - eks_worker_asg_min_size = 3 - eks_worker_asg_max_size = 6 - - # EKS-Optimized AMI for your region: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html - # https://us-east-1.console.aws.amazon.com/systems-manager/parameters/%252Faws%252Fservice%252Feks%252Foptimized-ami%252F1.14%252Famazon-linux-2%252Frecommended%252Fimage_id/description?region=us-east-1 - eks_worker_ami = "{{ .Config.Infrastructure.AWS.EKS.WorkerAMI }}" - {{- end }} - - {{- if .Config.Infrastructure.AWS.Cognito.Enabled }} - # Cognito configuration - user_pool = "{{ .Config.Name }}-production" - hostname = "{{ .Config.Frontend.Hostname }}" - {{- end}} - - # Hosting configuration - s3_hosting_buckets = [ - "{{ .Config.Name }}-production" - ] - s3_hosting_cert_domain = "{{ .Config.Frontend.Hostname}}" - -} - -{{- if .Config.Infrastructure.AWS.Cognito.Enabled }} -output "cognito_client_id" { - value = module.staging.cognito.cognito_client_id -} - -output "cognito_pool_id" { - value = module.staging.cognito.cognito_pool_id -} -{{- end}} diff --git a/templates/terraform/environments/staging/main.tf b/templates/terraform/environments/staging/main.tf deleted file mode 100644 index 4da6cdc50..000000000 --- a/templates/terraform/environments/staging/main.tf +++ /dev/null @@ -1,61 +0,0 @@ -terraform { - required_version = ">= 0.12" - backend "s3" { - bucket = "{{ .Config.Name }}-staging-terraform-state" - key = "infrastructure/terraform/environments/staging/main" - encrypt = true - region = "{{ .Config.Infrastructure.AWS.Region }}" - dynamodb_table = "{{ .Config.Name }}-staging-terraform-state-locks" - } -} - -# Instantiate the staging environment -module "staging" { - source = "../../modules/environment" - environment = "staging" - - # Project configuration - project = "{{ .Config.Name }}" - region = "{{ .Config.Infrastructure.AWS.Region }}" - allowed_account_ids = ["{{ .Config.Infrastructure.AWS.AccountID }}"] - -{{- if ne .Config.Infrastructure.AWS.EKS.ClusterName "" }} - # ECR configuration - ecr_repositories = [ - {{- range .Config.Services }} - "{{ .Name }}", - {{- end }} - ] - - # EKS configuration - eks_worker_instance_type = "t2.small" - eks_worker_asg_min_size = 2 - eks_worker_asg_max_size = 6 - - # EKS-Optimized AMI for your region: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html - # https://us-east-1.console.aws.amazon.com/systems-manager/parameters/%252Faws%252Fservice%252Feks%252Foptimized-ami%252F1.14%252Famazon-linux-2%252Frecommended%252Fimage_id/description?region=us-east-1 - eks_worker_ami = "{{ .Config.Infrastructure.AWS.EKS.WorkerAMI }}" -{{- end }} - - {{- if .Config.Infrastructure.AWS.Cognito.Enabled }} - # Cognito configuration - user_pool = "{{ .Config.Name }}-staging" - hostname = "{{ .Config.Frontend.Hostname }}" - {{- end }} - - # Hosting configuration - s3_hosting_buckets = [ - "{{ .Config.Name }}-staging" - ] - s3_hosting_cert_domain = "{{ .Config.Frontend.Hostname}}" -} - -{{- if .Config.Infrastructure.AWS.Cognito.Enabled }} -output "cognito_client_id" { - value = module.staging.cognito.cognito_client_id -} - -output "cognito_pool_id" { - value = module.staging.cognito.cognito_pool_id -} -{{- end}} diff --git a/templates/terraform/modules/cognito/main.tf b/templates/terraform/modules/cognito/main.tf deleted file mode 100644 index dbca7db3e..000000000 --- a/templates/terraform/modules/cognito/main.tf +++ /dev/null @@ -1,40 +0,0 @@ -resource "aws_cognito_user_pool" "users" { - name = "${var.user_pool}-user-pool" - - username_attributes = [ - "email", - ] - - # auto_verified_attributes = ["email"] -} - -resource "aws_cognito_user_pool_client" "client" { - name = "${var.user_pool}-cognito-client" - - user_pool_id = "${aws_cognito_user_pool.users.id}" - generate_secret = false - - allowed_oauth_flows_user_pool_client = true - allowed_oauth_flows = ["code", "implicit"] - allowed_oauth_scopes = ["profile", "openid"] - - supported_identity_providers = ["COGNITO"] - refresh_token_validity = "14" - - explicit_auth_flows = [ - "ADMIN_NO_SRP_AUTH", - "USER_PASSWORD_AUTH", - ] - - write_attributes = ["email"] - - callback_urls = ["https://auth.${var.hostname}","https://auth.${var.hostname}/oauth2/idpresponse"] - logout_urls = ["https://auth.${var.hostname}/logout"] -} - -output "cognito_pool_id" { - value = aws_cognito_user_pool.users.id -} -output "cognito_client_id" { - value = aws_cognito_user_pool_client.client.id -} diff --git a/templates/terraform/modules/cognito/variables.tf b/templates/terraform/modules/cognito/variables.tf deleted file mode 100644 index faa8b1c5b..000000000 --- a/templates/terraform/modules/cognito/variables.tf +++ /dev/null @@ -1,6 +0,0 @@ -variable "user_pool" { - description = "AWS Cognito pool name" -} -variable "hostname" { - description = "AWS Cognito pool name" -} diff --git a/templates/terraform/modules/ecr/main.tf b/templates/terraform/modules/ecr/main.tf deleted file mode 100644 index 9cbd9ecf1..000000000 --- a/templates/terraform/modules/ecr/main.tf +++ /dev/null @@ -1,43 +0,0 @@ -resource "aws_ecr_repository" "ecr_repository" { - for_each = var.ecr_repositories - name = each.value - - tags = { - environment = var.environment - } -} - -data "aws_iam_policy_document" "ecr_fullaccess" { - count = var.enabled ? 1 : 0 - - statement { - sid = "FullAccess" - effect = "Allow" - - principals { - type = "AWS" - - identifiers = var.ecr_principals - } - - actions = [ - "ecr:GetAuthorizationToken", - "ecr:InitiateLayerUpload", - "ecr:UploadLayerPart", - "ecr:CompleteLayerUpload", - "ecr:PutImage", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:GetRepositoryPolicy", - "ecr:DescribeRepositories", - "ecr:ListImages", - "ecr:DescribeImages", - "ecr:BatchGetImage", - ] - } -} - -resource "aws_ecr_repository_policy" "default" { - repository = join("", aws_ecr_repository.ecr_repository.*.name) - policy = data.aws_iam_policy_document.ecr_fullaccess.json -} diff --git a/templates/terraform/modules/ecr/variables.tf b/templates/terraform/modules/ecr/variables.tf deleted file mode 100644 index 67150990b..000000000 --- a/templates/terraform/modules/ecr/variables.tf +++ /dev/null @@ -1,14 +0,0 @@ -variable "environment" { - description = "The environment (dev/staging/prod)" -} - -variable "ecr_repositories" { - description = "List of ECR repository names to create" - type = list(string) -} - -variable "ecr_principals" { - description = "List of principals (most likely users) to give full access to the created ECR repositories" - type = list(string) -} - diff --git a/templates/terraform/modules/ecr/versions.tf b/templates/terraform/modules/ecr/versions.tf deleted file mode 100644 index ac97c6ac8..000000000 --- a/templates/terraform/modules/ecr/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.12" -} diff --git a/templates/terraform/modules/eks/main.tf b/templates/terraform/modules/eks/main.tf deleted file mode 100644 index c2164ef74..000000000 --- a/templates/terraform/modules/eks/main.tf +++ /dev/null @@ -1,51 +0,0 @@ -# Create KubernetesAdmin role for aws-iam-authenticator -resource "aws_iam_role" "kubernetes_admin_role" { - name = "{{ .Config.Name }}-kubernetes-admin" - assume_role_policy = var.assume_role_policy - description = "Kubernetes administrator role (for AWS IAM Authenticator)" -} - -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "6.0.2" - - cluster_name = var.cluster_name - cluster_version = "1.14" - subnets = var.private_subnets - vpc_id = var.vpc_id - - worker_groups = [ - { - instance_type = var.worker_instance_type - asg_min_size = var.worker_asg_min_size - asg_desired_capacity = var.worker_asg_min_size - asg_max_size = var.worker_asg_max_size - ami_id = var.worker_ami - tags = [{ - key = "environment" - value = var.environment - propagate_at_launch = true - }] - }, - ] - - map_roles = [ - { - rolearn = "arn:aws:iam::${var.iam_account_id}:role/kubernetes-admin" - username = "kubernetes-admin" - groups = ["system:masters"] - }, - ] - cluster_iam_role_name = "k8s-${var.cluster_name}-cluster" - workers_role_name = "k8s-${var.cluster_name}-workers" - - # TODO, determine if this should be true/false - manage_aws_auth = true - - write_kubeconfig = false - write_aws_auth_config = false - - tags = { - environment = var.environment - } -} diff --git a/templates/terraform/modules/eks/outputs.tf b/templates/terraform/modules/eks/outputs.tf deleted file mode 100644 index 4fc0ad76c..000000000 --- a/templates/terraform/modules/eks/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -output "cluster_id" { - description = "Identifier of the EKS cluster" - value = module.eks.cluster_id -} - -output "worker_iam_role_arn" { - description = "The ARN of the EKS worker IAM role" - value = module.eks.worker_iam_role_arn -} - -output "worker_iam_role_name" { - description = "The name of the EKS worker IAM role" - value = module.eks.worker_iam_role_name -} - -output "worker_security_group_id" { - description = "The security group of the EKS worker" - value = module.eks.worker_security_group_id -} \ No newline at end of file diff --git a/templates/terraform/modules/eks/variables.tf b/templates/terraform/modules/eks/variables.tf deleted file mode 100644 index 780b774ec..000000000 --- a/templates/terraform/modules/eks/variables.tf +++ /dev/null @@ -1,45 +0,0 @@ -variable "project" { - description = "Name of the project" -} - -variable "environment" { - description = "The environment (dev/staging/prod)" -} - -variable "cluster_name" { - description = "Name to be given to the EKS cluster" -} - -variable "assume_role_policy" { - description = "IAM policy document for AssumeRole" -} - -variable "private_subnets" { - description = "VPC subnets for the EKS cluster" - # type = list(string) -} - -variable "vpc_id" { - description = "VPC ID for EKS cluster" -} - -variable "worker_instance_type" { - description = "Instance type for the EKS workers" -} - -variable "worker_asg_min_size" { - description = "Minimum number of instances for the EKS ASG" -} - -variable "worker_asg_max_size" { - description = "Maximum number of instances for the EKS ASG" -} - -variable "worker_ami" { - description = "The (EKS-optimized) AMI for EKS worker instances" -} - -variable "iam_account_id" { - description = "Account ID of the current IAM user" -} - diff --git a/templates/terraform/modules/eks/versions.tf b/templates/terraform/modules/eks/versions.tf deleted file mode 100644 index ac97c6ac8..000000000 --- a/templates/terraform/modules/eks/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.12" -} diff --git a/templates/terraform/modules/environment/main.tf b/templates/terraform/modules/environment/main.tf deleted file mode 100644 index 231ca2f2c..000000000 --- a/templates/terraform/modules/environment/main.tf +++ /dev/null @@ -1,90 +0,0 @@ -# Environment entrypoint - -locals { - kubernetes_cluster_name = "${var.project}-${var.environment}-${var.region}" -} - -module "vpc" { - source = "../../modules/vpc" - project = var.project - environment = var.environment - region = var.region - kubernetes_cluster_name = local.kubernetes_cluster_name -} - -# Data sources for EKS IAM -data "aws_caller_identity" "current" {} - -data "aws_iam_policy_document" "assumerole_root_policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"] - } - } -} - -{{- if ne .Config.Infrastructure.AWS.EKS.ClusterName "" }} -# Provision the EKS cluster -module "eks" { - source = "../../modules/eks" - project = var.project - environment = var.environment - cluster_name = local.kubernetes_cluster_name - iam_account_id = data.aws_caller_identity.current.account_id - - assume_role_policy = data.aws_iam_policy_document.assumerole_root_policy.json - private_subnets = module.vpc.private_subnets - vpc_id = module.vpc.vpc_id - - worker_instance_type = var.eks_worker_instance_type - worker_asg_min_size = var.eks_worker_asg_min_size - worker_asg_max_size = var.eks_worker_asg_max_size - worker_ami = var.eks_worker_ami # EKS-Optimized AMI for your region: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html -} - -module "kube2iam" { - source = "../../modules/kube2iam" - environment = var.environment - eks_worker_iam_role_arn = module.eks.worker_iam_role_arn - eks_worker_iam_role_name = module.eks.worker_iam_role_name - iam_account_id = data.aws_caller_identity.current.account_id -} -{{- end}} - -data "aws_iam_user" "ci_user" { - user_name = "ci-user" # Should have been created in the bootstrap process -} -{{- if .Config.Services }} - -# Set up ECR repositories -module "ecr" { - source = "../../modules/ecr" - environment = var.environment - ecr_repositories = var.ecr_repositories - ecr_principals = [aws_iam_user.id] -} -{{- end}} - -{{- if .Config.Infrastructure.AWS.Cognito.Enabled }} -module "cognito" { - source = "../../modules/cognito" - user_pool = var.user_pool - hostname = var.hostname -} - -output "cognito" { - value = module.cognito -} -{{- end}} - -{{ if .Config.Infrastructure.AWS.S3Hosting.Enabled }} -module "s3_hosting" { - source = "../../modules/s3_hosting" - buckets = var.s3_hosting_buckets - cert_domain = var.s3_hosting_cert_domain - project = var.project -} -{{- end}} diff --git a/templates/terraform/modules/environment/provider.tf b/templates/terraform/modules/environment/provider.tf deleted file mode 100644 index 8072c8566..000000000 --- a/templates/terraform/modules/environment/provider.tf +++ /dev/null @@ -1,5 +0,0 @@ -provider "aws" { - region = var.region - allowed_account_ids = var.allowed_account_ids -} - diff --git a/templates/terraform/modules/environment/variables.tf b/templates/terraform/modules/environment/variables.tf deleted file mode 100644 index 2ff27eae7..000000000 --- a/templates/terraform/modules/environment/variables.tf +++ /dev/null @@ -1,53 +0,0 @@ -variable "project" { - description = "The name of the project, mostly for tagging" -} - -variable "environment" { - description = "The environment (dev/staging/prod)" -} - -variable "region" { - description = "The AWS region" -} - -variable "allowed_account_ids" { - description = "The IDs of AWS accounts for this project, to protect against mistakenly applying to the wrong env" - type = list(string) -} - -variable "ecr_repositories" { - description = "List of ECR repository names to create" - type = list(string) -} - -variable "eks_worker_instance_type" { - description = "Instance type for the EKS workers" -} - -variable "eks_worker_asg_min_size" { - description = "Minimum number of instances for the EKS ASG" -} - -variable "eks_worker_asg_max_size" { - description = "Maximum number of instances for the EKS ASG" -} - -variable "eks_worker_ami" { - description = "The (EKS-optimized) AMI for EKS worker instances" -} - -{{- if .Config.Infrastructure.AWS.Cognito.Enabled }} -variable "hostname" { - description = "Application hostname" -} -{{- end }} -variable "s3_hosting_buckets" { - description = "S3 hosting buckets" - type = set(string) -} - -variable "s3_hosting_cert_domain" { - description = "Domain of the ACM certificate to lookup for Cloudfront to use" - type = string -} - diff --git a/templates/terraform/modules/environment/versions.tf b/templates/terraform/modules/environment/versions.tf deleted file mode 100644 index d9b6f790b..000000000 --- a/templates/terraform/modules/environment/versions.tf +++ /dev/null @@ -1,3 +0,0 @@ -terraform { - required_version = ">= 0.12" -} diff --git a/templates/terraform/modules/kube2iam/README.md b/templates/terraform/modules/kube2iam/README.md deleted file mode 100644 index dcfe37fa4..000000000 --- a/templates/terraform/modules/kube2iam/README.md +++ /dev/null @@ -1,6 +0,0 @@ -kube2iam -= - -Provide IAM credentials to containers running inside a kubernetes cluster based on annotations. - -Will be deprecated soon in favour of: https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/. \ No newline at end of file diff --git a/templates/terraform/modules/kube2iam/main.tf b/templates/terraform/modules/kube2iam/main.tf deleted file mode 100644 index deb7c971f..000000000 --- a/templates/terraform/modules/kube2iam/main.tf +++ /dev/null @@ -1,130 +0,0 @@ -# Allow the worker nodes to assume a role we are creating below -data "aws_iam_policy_document" "k8s_worker_assumerole_policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "AWS" - identifiers = [var.eks_worker_iam_role_arn] - } - } -} - -# Policy to allow worker nodes to assume roles starting with "k8s-" -data "aws_iam_policy_document" "node_assume_kube2iam_role" { - statement { - effect = "Allow" - actions = ["sts:AssumeRole"] - resources = ["arn:aws:iam::${var.iam_account_id}:role/k8s-*"] - } -} - -# Add the above policy to the worker role -resource "aws_iam_role_policy" "node_kube2iam_policy" { - name = "eks-node-kube2iam-policy" - role = var.eks_worker_iam_role_name - policy = data.aws_iam_policy_document.node_assume_kube2iam_role.json -} - -# This is now done with the kubernetes terraform provider, see the kubernetes/kube2iam module. -# # Execute the kubernetes manifest required to create the kube2iam daemonset -# resource "null_resource" "kube2iam" { -# provisioner "local-exec" { -# command = "kubectl apply -f ${path.root}/kubernetes/kube2iam.yaml --kubeconfig ${path.root}/output/kubeconfig_${var.environment}" -# } -# # TODO: Module-aware dependencies not yet supported - https://github.com/hashicorp/terraform/issues/17101 -# # depends_on = ["module.eks"] -# } - -### Kube2IAM roles to map to pods ### -# These can be referenced in an annotation in a kubernetes deployment manifest file - -## ALB Ingress Controller -# Create a role and establish a trust relationship with the worker nodes -resource "aws_iam_role" "k8s_worker_alb_ingress_controller_role" { - name = "{{ .Config.Name }}-k8s-alb-ingress-controller" - assume_role_policy = data.aws_iam_policy_document.k8s_worker_assumerole_policy.json - force_detach_policies = true -} - -# Policy allowing access to specific AWS resources -data "aws_iam_policy_document" "k8s_alb_ingress_controller_access_policy" { - statement { - actions = [ - "acm:DescribeCertificate", - "acm:ListCertificates", - "acm:GetCertificate", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:CreateSecurityGroup", - "ec2:CreateTags", - "ec2:DeleteTags", - "ec2:DeleteSecurityGroup", - "ec2:DescribeAccountAttributes", - "ec2:DescribeInstances", - "ec2:DescribeInstanceStatus", - "ec2:DescribeInternetGateways", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSubnets", - "ec2:DescribeTags", - "ec2:DescribeVpcs", - "ec2:ModifyInstanceAttribute", - "ec2:ModifyNetworkInterfaceAttribute", - "ec2:RevokeSecurityGroupIngress", - "elasticloadbalancing:AddTags", - "elasticloadbalancing:CreateListener", - "elasticloadbalancing:CreateLoadBalancer", - "elasticloadbalancing:CreateRule", - "elasticloadbalancing:CreateTargetGroup", - "elasticloadbalancing:DeleteListener", - "elasticloadbalancing:DeleteLoadBalancer", - "elasticloadbalancing:DeleteRule", - "elasticloadbalancing:DeleteTargetGroup", - "elasticloadbalancing:DeregisterTargets", - "elasticloadbalancing:DescribeListenerCertificates", - "elasticloadbalancing:DescribeListeners", - "elasticloadbalancing:DescribeLoadBalancers", - "elasticloadbalancing:DescribeLoadBalancerAttributes", - "elasticloadbalancing:DescribeRules", - "elasticloadbalancing:DescribeSSLPolicies", - "elasticloadbalancing:DescribeTags", - "elasticloadbalancing:DescribeTargetGroups", - "elasticloadbalancing:DescribeTargetGroupAttributes", - "elasticloadbalancing:DescribeTargetHealth", - "elasticloadbalancing:ModifyListener", - "elasticloadbalancing:ModifyLoadBalancerAttributes", - "elasticloadbalancing:ModifyRule", - "elasticloadbalancing:ModifyTargetGroup", - "elasticloadbalancing:ModifyTargetGroupAttributes", - "elasticloadbalancing:RegisterTargets", - "elasticloadbalancing:RemoveTags", - "elasticloadbalancing:SetIpAddressType", - "elasticloadbalancing:SetSecurityGroups", - "elasticloadbalancing:SetSubnets", - "elasticloadbalancing:SetWebACL", - "iam:GetServerCertificate", - "iam:ListServerCertificates", - "waf-regional:GetWebACLForResource", - "waf-regional:GetWebACL", - "waf-regional:AssociateWebACL", - "waf-regional:DisassociateWebACL", - "waf:GetWebACL", - "tag:GetResources", - "tag:TagResources", - "cognito-idp:DescribeUserPoolClient", - ] - - resources = ["*"] - } - - statement { - actions = ["iam:CreateServiceLinkedRole"] - resources = ["arn:aws:iam::${var.iam_account_id}:role/aws-service-role/elasticloadbalancing.amazonaws.com/AWSServiceRoleForElasticLoadBalancing"] - } -} - -# Add the above policy to the created role -resource "aws_iam_role_policy" "k8s_worker_alb_ingress_controller_role_policy" { - name = "worker-alb-ingress-controller-policy" - role = aws_iam_role.k8s_worker_alb_ingress_controller_role.id - policy = data.aws_iam_policy_document.k8s_alb_ingress_controller_access_policy.json -} diff --git a/templates/terraform/modules/kube2iam/variables.tf b/templates/terraform/modules/kube2iam/variables.tf deleted file mode 100644 index 5f210b9b1..000000000 --- a/templates/terraform/modules/kube2iam/variables.tf +++ /dev/null @@ -1,15 +0,0 @@ -variable "eks_worker_iam_role_arn" { - description = "The ARN of the EKS worker IAM role" -} - -variable "eks_worker_iam_role_name" { - description = "The name of the EKS worker IAM role" -} - -variable "iam_account_id" { - description = "Account ID of the current IAM user" -} - -variable "environment" { - description = "The environment (dev/staging/prod)" -} \ No newline at end of file diff --git a/templates/terraform/modules/kube2iam/versions.tf b/templates/terraform/modules/kube2iam/versions.tf deleted file mode 100644 index ac97c6ac8..000000000 --- a/templates/terraform/modules/kube2iam/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.12" -} diff --git a/templates/terraform/modules/s3_hosting/main.tf b/templates/terraform/modules/s3_hosting/main.tf deleted file mode 100644 index 19721db79..000000000 --- a/templates/terraform/modules/s3_hosting/main.tf +++ /dev/null @@ -1,153 +0,0 @@ -locals { - assets_access_identity = "${var.project}-client-assets" -} - -resource "aws_s3_bucket" "client_assets" { - for_each = var.buckets - - // Our bucket's name is going to be the same as our site's domain name. - bucket = each.value - acl = "private" // The contents will be available through cloudfront, they should not be accessible publicly - - - // S3 understands what it means to host a website. - website { - // Here we tell S3 what to use when a request comes in to the root - index_document = "index.html" - error_document = "404.html" - } -} - -# Deny public access to this bucket -resource "aws_s3_bucket_public_access_block" "client_assets" { - for_each = var.buckets - - bucket = each.value - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -# Access identity for CF access to S3 -resource "aws_cloudfront_origin_access_identity" "client_assets" { - comment = local.assets_access_identity -} - -# Policy to allow CF access to S3 -data "aws_iam_policy_document" "assets_origin" { - for_each = var.buckets - - statement { - actions = ["s3:GetObject"] - resources = ["arn:aws:s3:::${each.value}/*"] - - principals { - type = "AWS" - identifiers = [aws_cloudfront_origin_access_identity.client_assets.iam_arn] - } - } - - statement { - actions = ["s3:ListBucket"] - resources = ["arn:aws:s3:::${each.value}"] - - principals { - type = "AWS" - identifiers = [aws_cloudfront_origin_access_identity.client_assets.iam_arn] - } - } -} - -# Attach the policy to the bucket -resource "aws_s3_bucket_policy" "client_assets" { - for_each = var.buckets - - bucket = each.value - policy = data.aws_iam_policy_document.assets_origin[each.value].json -} - -# To use an ACM cert with CF it has to exist in us-east-1 -provider "aws" { - region = "us-east-1" - alias = "east1" -} - -# Find an already created ACM cert for this domain -data "aws_acm_certificate" "wildcard_cert" { - provider = "aws.east1" - domain = var.cert_domain - most_recent = "true" -} - -# Create the cloudfront distribution -resource "aws_cloudfront_distribution" "client_assets_distribution" { - for_each = var.buckets - - // origin is where CloudFront gets its content from. - origin { - domain_name = aws_s3_bucket.client_assets[each.value].bucket_domain_name - origin_id = local.assets_access_identity - s3_origin_config { - origin_access_identity = aws_cloudfront_origin_access_identity.client_assets.cloudfront_access_identity_path - } - } - - enabled = true - is_ipv6_enabled = true - default_root_object = "index.html" # Render this when you hit the root - - // All values are defaults from the AWS console. - default_cache_behavior { - target_origin_id = local.assets_access_identity - viewer_protocol_policy = "redirect-to-https" - compress = true - allowed_methods = ["GET", "HEAD"] - cached_methods = ["GET", "HEAD"] - min_ttl = 0 - default_ttl = 86400 - max_ttl = 31536000 - - forwarded_values { - query_string = false - cookies { - forward = "none" - } - } - } - - aliases = [ - each.value, - ] - - restrictions { - geo_restriction { - restriction_type = "none" - } - } - - # Use our cert - viewer_certificate { - acm_certificate_arn = data.aws_acm_certificate.wildcard_cert.arn - minimum_protocol_version = "TLSv1" - ssl_support_method = "sni-only" - } - -} - -# Find the route53 zone -data "aws_route53_zone" "public" { - name = "${var.cert_domain}." - private_zone = false -} - -# Subdomain to point at CF -resource "aws_route53_record" "client_assets" { - for_each = var.buckets - - zone_id = data.aws_route53_zone.public.zone_id - name = each.value - type = "CNAME" - ttl = "120" - records = [aws_cloudfront_distribution.client_assets_distribution[each.value].domain_name] -} diff --git a/templates/terraform/modules/s3_hosting/variables.tf b/templates/terraform/modules/s3_hosting/variables.tf deleted file mode 100644 index dd0742ab7..000000000 --- a/templates/terraform/modules/s3_hosting/variables.tf +++ /dev/null @@ -1,13 +0,0 @@ -variable "project" { - description = "The name of the project, mostly for tagging" -} - -variable "buckets" { - description = "S3 hosting buckets" - type = set(string) -} - -variable "cert_domain" { - description = "Domain of the ACM certificate to lookup for Cloudfront to use" - type = string -} diff --git a/templates/terraform/modules/vpc/main.tf b/templates/terraform/modules/vpc/main.tf deleted file mode 100644 index 97baa5603..000000000 --- a/templates/terraform/modules/vpc/main.tf +++ /dev/null @@ -1,37 +0,0 @@ -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - - name = "${var.project}-${var.environment}-vpc" - cidr = "10.10.0.0/16" - - azs = ["${var.region}a", "${var.region}b", "${var.region}c"] # Most regions have 3+ azs - private_subnets = ["10.10.32.0/19", "10.10.64.0/19", "10.10.96.0/19"] - public_subnets = ["10.10.1.0/24", "10.10.2.0/24", "10.10.3.0/24"] - database_subnets = ["10.10.10.0/24", "10.10.11.0/24", "10.10.12.0/24"] - - # Allow kubernetes ALB ingress controller to auto-detect - private_subnet_tags = { - "kubernetes.io/cluster/${var.kubernetes_cluster_name}" = "owned" - "kubernetes.io/role/internal-elb" = "1" - } - - public_subnet_tags = { - "kubernetes.io/cluster/${var.kubernetes_cluster_name}" = "owned" - "kubernetes.io/role/elb" = "1" - } - - enable_nat_gateway = true - enable_vpn_gateway = false - enable_dns_hostnames = true - - create_database_subnet_group = true - create_database_subnet_route_table = true - - tags = { - environment = var.environment - } - - vpc_tags = { - "kubernetes.io/cluster/${var.kubernetes_cluster_name}" = "shared" - } -} diff --git a/templates/terraform/modules/vpc/outputs.tf b/templates/terraform/modules/vpc/outputs.tf deleted file mode 100644 index cd9f19e60..000000000 --- a/templates/terraform/modules/vpc/outputs.tf +++ /dev/null @@ -1,35 +0,0 @@ -output "vpc_id" { - description = "The ID of the created VPC" - value = module.vpc.vpc_id -} - -output "vpc_cidr_block" { - description = "The CIDR block of the VPC" - value = module.vpc.vpc_cidr_block -} - -output "azs" { - description = "Availability zones for the VPC" - value = module.vpc.azs -} - -output "private_subnets" { - description = "List of private subnets" - value = module.vpc.private_subnets -} - -output "public_subnets" { - description = "List of public subnets" - value = module.vpc.public_subnets -} - -output "database_subnets" { - description = "List of public subnets" - value = module.vpc.database_subnets -} - -output "database_subnet_group" { - description = "List of subnet groups" - value = module.vpc.database_subnet_group -} - diff --git a/templates/terraform/modules/vpc/variables.tf b/templates/terraform/modules/vpc/variables.tf deleted file mode 100644 index 044299b06..000000000 --- a/templates/terraform/modules/vpc/variables.tf +++ /dev/null @@ -1,15 +0,0 @@ -variable "project" { - description = "The name of the project, mostly for tagging" -} - -variable "environment" { - description = "The environment (development/staging/production)" -} - -variable "region" { - description = "The AWS region" -} - -variable "kubernetes_cluster_name" { - description = "Kubernetes cluster name used to associate with subnets for auto LB placement" -} diff --git a/templates/terraform/modules/vpc/versions.tf b/templates/terraform/modules/vpc/versions.tf deleted file mode 100644 index ac97c6ac8..000000000 --- a/templates/terraform/modules/vpc/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.12" -} From bea53409f5969c5c89971f756efe25660bbc843c Mon Sep 17 00:00:00 2001 From: Thomas Z Date: Thu, 13 Feb 2020 11:05:40 -0800 Subject: [PATCH 2/5] updated docs with new module output params support and moved references into main readme file --- README.md | 146 ++++++++++++++++++++++++++++++ docs/commit0-reference.md | 134 --------------------------- internal/config/create_example.go | 4 +- 3 files changed, 149 insertions(+), 135 deletions(-) delete mode 100644 docs/commit0-reference.md diff --git a/README.md b/README.md index a37976148..67e9f3328 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,152 @@ The best way then to use this is to add an alias, then you can use the CLI as if 5) You can go to each project repo and follow the project readme to start the service 6) `commit0 ui` launches the locally hosted web UI (Static SPA) and the API server + +# Configuring Commit0 + +This is a guide on how to configure your project manually with a single file `commit0.yml`. Simply write this file to the root of your project directory and run the commit0 CLI tool against it to generate your project files. + + +# Table of Contents + +* [commit0.yml](#commit0-yaml) + * [name*](#name) + * [context](#context) + * [modules*](#modules) + * [source*](#module-source) + * [params*](#module-params) + * [output](#module-output) +* [commit0.module.yml](#commit0-module-yaml) + * [name*](#module-name) + * [description](#module-description) + * [template](#template) + * [extension](#template-extension) + * [delimiters](#template-delimiters) + * [output](#template-output) + +# Commit0.yaml +Your project config file. It describes the project +Example: +``` +name: hello-world +context: + cognitoPoolID: xxx +modules: + #- source: "../tests/modules/ci" + - source: "github.com/zthomas/react-mui-kit" + - output: "web-app" +``` + +## name +Name of your project. This will be used to name the github repos as well as in other parts of the generated code. + +[]() | | +--- | --- +Required | True +Type | String + +## context +A key value map of global context parameters to use in the templates. + +[]() | | +--- | --- +Required | False +Type | Map[String] + +## modules +List of modules template modules to import + +[]() | | +--- | --- +Required | True +Type | Map[Module] + +## source +We are using go-getter to parse the sources, we you can use any URL or file formats that [go-getter](https://github.com/hashicorp/go-getter#url-format) supports. + +[]() | | +--- | --- +Required | True +Type | String + +## module +Module parameters to use during templating + +[]() | | +--- | --- +Required | True +Type | String + +## output +Template output directory that the current module should write to. + +[]() | | +--- | --- +Required | False +Type | String + + +# Commit0.module.yaml +The module config file. You can configure how the templating engine should process the files in the current repository. +Example: +``` +name: react-mui-kit +template: + extension: '.tmplt' + delimiters: + - '<%' + - '%>' + output: web-app +``` + +## name +Name of your module. This will be used as the default module directory as well as a display name in the prompts. + +[]() | | +--- | --- +Required | True +Type | String + +## description +Short description of the module + +[]() | | +--- | --- +Required | False +Type | String + +## template +Template configurations +[]() | | +--- | --- +Required | False +Type | Map + +## extension +File extension to signify that a file is a template. If this is defined, non-template files will not be parsed and will be copied over directly. The default value is `.tmplt` + +[]() | | +--- | --- +Required | False +Type | Map + +## delimiters +An pair of delimiters that the template engine should use. The default values are: `{{`, `}}` + +[]() | | +--- | --- +Required | False +Type | Map[String] + +## output +The default template output directory that you want the template engine to write to. This will be overwritten by the + +[]() | | +--- | --- +Required | False +Type | String + + ## Development We are looking for contributors! diff --git a/docs/commit0-reference.md b/docs/commit0-reference.md deleted file mode 100644 index 357b7b3d9..000000000 --- a/docs/commit0-reference.md +++ /dev/null @@ -1,134 +0,0 @@ - -# Configuring Commit0 - -This is a guide on how to configure your project manually with a single file `commit0.yml`. Simply write this file to the root of your project directory and run the commit0 CLI tool against it to generate your project files. - -You can see a complete commit0.yml in our [full example](TODO: Create example). - -# Table of Contents - -* [commit0.yml](#commit0-yaml) - * [name*](#name) - * [context](#context) - * [modules*](#modules) - * [source*](#module-source) - * [params*](#module-params) -* [commit0.module.yml](#commit0-module-yaml) - * [name*](#module-name) - * [description](#module-description) - * [template](#template) - * [extension](#template-extension) - * [delimiters](#template-delimiters) - * [output](#template-output) - -# Commit0.yaml -Your project config file. It describes the project -Example: -``` -name: hello-world -context: - cognitoPoolID: xxx -modules: - #- source: "../tests/modules/ci" - - source: "github.com/zthomas/react-mui-kit" -``` - -## name -Name of your project. This will be used to name the github repos as well as in other parts of the generated code. - -[]() | | ---- | --- -Required | True -Type | String - -## context -A key value map of global context parameters to use in the templates. - -[]() | | ---- | --- -Required | False -Type | Map[String] - -## modules -List of modules template modules to import - -[]() | | ---- | --- -Required | True -Type | Map[Module] - -## source -We are using go-getter to parse the sources, we you can use any URL or file formats that [go-getter](https://github.com/hashicorp/go-getter#url-format) supports. - -[]() | | ---- | --- -Required | True -Type | String - -## module -Module parameters to use during templating - -[]() | | ---- | --- -Required | True -Type | String - -# Commit0.module.yaml -The module config file. You can configure how the templating engine should process the files in the current repository. -Example: -``` -name: react-mui-kit -template: - extension: '.tmplt' - delimiters: - - '<%' - - '%>' - output: web-app -``` - -## name -Name of your module. This will be used as the default module directory as well as a display name in the prompts. - -[]() | | ---- | --- -Required | True -Type | String - -## description -Short description of the module - -[]() | | ---- | --- -Required | False -Type | String - -## template -Template configurations -[]() | | ---- | --- -Required | False -Type | Map - -## extension -File extension to signify that a file is a template. If this is defined, non-template files will not be parsed and will be copied over directly. The default value is `.tmplt` - -[]() | | ---- | --- -Required | False -Type | Map - -## delimiters -An pair of delimiters that the template engine should use. The default values are: `{{`, `}}` - -[]() | | ---- | --- -Required | False -Type | Map[String] - -## output -Template output directory that you want the template engine to write to. - -[]() | | ---- | --- -Required | False -Type | String diff --git a/internal/config/create_example.go b/internal/config/create_example.go index 5b77a7c60..72c682b0f 100644 --- a/internal/config/create_example.go +++ b/internal/config/create_example.go @@ -14,7 +14,9 @@ context: # module can be in any format the go-getter supports (path, github, url, etc.) # supports https://github.com/hashicorp/go-getter#url-format -# - source: "../tests/test_data/modules/ci" +# Example: +# - source: "../development/modules/ci" +# - output: "github-actions" modules: - source: "github.com/zthomas/commit0-terraform-basic"` From 3bc3feb0569e4b1251f6a835cd3a747fbbca10fa Mon Sep 17 00:00:00 2001 From: Thomas Z Date: Thu, 13 Feb 2020 13:33:21 -0800 Subject: [PATCH 3/5] added config features and updated readme --- .gitignore | 1 + README.md | 77 ++++++++++++++++++++------- internal/config/generator_config.go | 8 +-- internal/config/module_config.go | 2 +- internal/generate/generate_modules.go | 7 ++- internal/module/module.go | 2 + 6 files changed, 73 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 5619020e5..d2d9415a1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ packrd /commit0 .history/ tmp +.vscode diff --git a/README.md b/README.md index 67e9f3328..96bacc5c9 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ As there some dependencies it will be easier to use this tool within the provide The best way then to use this is to add an alias, then you can use the CLI as if it was installed as usual on your machine: `alias commit0='docker run -it -v "$(pwd):/project" -v "${HOME}/.aws:/root/.aws" commit0:v0'` -## Usage +# Usage 1) To create a project run `commit0 create [PROJECT_NAME]` 2) It will prompt you to select a cloud provider and an account profile to use @@ -31,13 +31,10 @@ The best way then to use this is to add an alias, then you can use the CLI as if 6) `commit0 ui` launches the locally hosted web UI (Static SPA) and the API server -# Configuring Commit0 +## Configuring Commit0 This is a guide on how to configure your project manually with a single file `commit0.yml`. Simply write this file to the root of your project directory and run the commit0 CLI tool against it to generate your project files. - -# Table of Contents - * [commit0.yml](#commit0-yaml) * [name*](#name) * [context](#context) @@ -45,25 +42,30 @@ This is a guide on how to configure your project manually with a single file `co * [source*](#module-source) * [params*](#module-params) * [output](#module-output) + * [overwrite](#module-overwrite) * [commit0.module.yml](#commit0-module-yaml) * [name*](#module-name) * [description](#module-description) + * [prompts](#module-prompts) + * [field*](#prompt-field) + * [label](#prompt-label) + * [options](#prompt-options) * [template](#template) * [extension](#template-extension) * [delimiters](#template-delimiters) * [output](#template-output) -# Commit0.yaml +## Commit0.yaml Your project config file. It describes the project Example: ``` -name: hello-world +name: foobar context: - cognitoPoolID: xxx + cognitoPoolID: xxx modules: - #- source: "../tests/modules/ci" - - source: "github.com/zthomas/react-mui-kit" - - output: "web-app" + #- source: "../tests/modules/ci" + - source: "github.com/zthomas/react-mui-kit" + - output: "web-app" ``` ## name @@ -114,18 +116,27 @@ Template output directory that the current module should write to. Required | False Type | String +## output +Whether to overwrite existing files when generating files from templates + +[]() | | +--- | --- +Required | False +Type | Boolean +Default | False + -# Commit0.module.yaml +## Commit0.module.yaml The module config file. You can configure how the templating engine should process the files in the current repository. Example: ``` name: react-mui-kit template: - extension: '.tmplt' - delimiters: - - '<%' - - '%>' - output: web-app + extension: '.tmplt' + delimiters: + - '<%' + - '%>' + output: web-app ``` ## name @@ -175,8 +186,38 @@ The default template output directory that you want the template engine to write Required | False Type | String +## Prompts +User prompts to generate to collect additional module specific params +[]() | | +--- | --- +Required | False +Type | Map + +## Field +The name of the field that the param should be written to + +[]() | | +--- | --- +Required | True +Type | String + +## Label +The message that will be presented to the user + +[]() | | +--- | --- +Required | False +Type | String + +## options +A list of options to select from. If not given, then it will be rendered as a text input prompt. + +[]() | | +--- | --- +Required | False +Type | Map[String] -## Development +# Development We are looking for contributors! Building from the source diff --git a/internal/config/generator_config.go b/internal/config/generator_config.go index a948c0a55..6c1521fa8 100644 --- a/internal/config/generator_config.go +++ b/internal/config/generator_config.go @@ -15,8 +15,10 @@ type GeneratorConfig struct { } type ModuleInstance struct { - Source string - Params map[string]string + Source string + Output string + Overwrite bool + Params map[string]string } func LoadGeneratorConfig(filePath string) *GeneratorConfig { @@ -31,6 +33,6 @@ func LoadGeneratorConfig(filePath string) *GeneratorConfig { log.Panicf("failed to parse config: %v", err) } - pp.Println(config) + pp.Println("Generator Config:", config) return config } diff --git a/internal/config/module_config.go b/internal/config/module_config.go index 4c3c33c6e..31e0f020d 100644 --- a/internal/config/module_config.go +++ b/internal/config/module_config.go @@ -40,6 +40,6 @@ func LoadModuleConfig(filePath string) *ModuleConfig { if err != nil { log.Panicf("failed to parse config: %v", err) } - pp.Println(config) + pp.Println("Module Config:", config) return config } diff --git a/internal/generate/generate_modules.go b/internal/generate/generate_modules.go index d601d7b8a..05f91fff3 100644 --- a/internal/generate/generate_modules.go +++ b/internal/generate/generate_modules.go @@ -56,8 +56,11 @@ type TemplateParams struct { func Generate(mod *module.TemplateModule, generatorCfg *config.GeneratorConfig) error { moduleDir := module.GetSourceDir(mod.Source) delimiters := mod.Config.Template.Delimiters - overwrite := true // @TODO get from configs - outputDir := mod.Config.Template.Output + overwrite := mod.Overwrite + outputDir := mod.Output + if outputDir == "" { + outputDir = mod.Config.Template.Output + } templateData := TemplateParams{} templateData.Name = generatorCfg.Name diff --git a/internal/module/module.go b/internal/module/module.go index d71fd4380..8483c4f86 100644 --- a/internal/module/module.go +++ b/internal/module/module.go @@ -32,6 +32,8 @@ func NewTemplateModule(moduleCfg config.ModuleInstance) (*TemplateModule, error) var templateModule TemplateModule templateModule.Source = moduleCfg.Source templateModule.Params = moduleCfg.Params + templateModule.Overwrite = moduleCfg.Overwrite + templateModule.Output = moduleCfg.Output p := &ProgressTracking{} sourcePath := GetSourceDir(templateModule.Source) From a731d450642720b9224b8b8faf0ae14fa4ceda87 Mon Sep 17 00:00:00 2001 From: Thomas Z Date: Thu, 13 Feb 2020 15:56:05 -0800 Subject: [PATCH 4/5] updated readme --- README.md | 44 ++++++++++++++++++++++++++-------- pkg/credentials/credentials.go | 3 ++- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 96bacc5c9..1beda6e83 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,37 @@ Status: Proof of Concept -Commit0 is an open source Push-To-Deploy tool designed to provide an amazing deployment process for developers while not compromising on dev ops best practices. Instead of using a Platform as a Service that simplifies your development but locks you in, we recreate the entire seamless workflow using open source technolgies and generate the infrastructure code for you while providing you with a simple interface. +## About Commit0 +Commit0 is a project skaffolding framework and ecosystem created to: + +1. Maximize knowledge sharing across an organization +2. Easily maintain a state of the art and easily reusable implementations of production grade solutions to recurring problems +3. Re-create the seamless deployment experience offered by PaaS solutions but with a fully open source stack that follows industry best practices With Commit0: -- You get the same simple Push-To-Deploy workflow that you are accustomed to with premium PaaS offerings -- Based on your configurations we'll generate all the infrastructure code that is needed to deploy and scale your application (Kubenetes manifests, Terraform, CI/CI configs etc.) and deploy to your own cloud provider. -- There's no vendor lock-in. It's all done with open source tools and generated code -- You don't need to know any dev ops to use Commit0 but if you are a dev ops engineer you can rest assured that you have a solid starting point and you can customize it as your project grows. -- We also include a set of commonly used open source microservices for tasks like authentication, user management, image resizing etc. so you can start developing the core application right away. +- Easily deploy and integrate various boilerplate solutions +- Instantly integrate commonly used open source microservices for authentication, user management, file encryption, image resizing etc. +- Get a simple Push-To-Deploy workflow that you are accustomed to with popular PaaS solutions [TODO] +- There's no vendor lock-in. It's all implemented with open source tools and deployed to your own cloud provider. + +## Commit0 Generator CLI +Commit0 CLI is an opinionated, yet fully modular code generation tool with declarative syntax that allows developers to easily integrate user prompts and interactions. + +Problems we encountered: +- It was tedious to creating reusable templates and hard to maintain +- Lack of standardization and integration interface between the templates +- Difficult to integrate multiple templated codebase + +How we aim to address those issues: +- Make templating behaviour simple and declarative +- Clear strategy and guideline around what are clear and reusable templates +- Standardize how templated code should get dependent parameters and start up + +This is inspired by: +- [Yeoman Generator](https://github.com/yeoman/generator) +- [JHipster](https://github.com/jhipster/generator-jhipster) +- [Boilr](https://github.com/tmrts/boilr) + ## Installation @@ -59,13 +82,14 @@ This is a guide on how to configure your project manually with a single file `co Your project config file. It describes the project Example: ``` -name: foobar +name: newProject context: cognitoPoolID: xxx modules: - #- source: "../tests/modules/ci" - - source: "github.com/zthomas/react-mui-kit" - - output: "web-app" + - source: "github.com/zthomas/commit0-terraform-basic" + output: "infrastructure" + - source: "github.com/zthomas/react-mui-kit" + output: "web-app" ``` ## name diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go index 1bf378bd0..e2052e03e 100644 --- a/pkg/credentials/credentials.go +++ b/pkg/credentials/credentials.go @@ -94,7 +94,8 @@ func GetSecrets(baseDir string) Secrets { } } - writeSecrets(secretsFile, secrets) + // @TODO deprecate + // writeSecrets(secretsFile, secrets) return secrets } } From 0726cdb0b263f4c592ff7e5c47ddac5ed350acb6 Mon Sep 17 00:00:00 2001 From: Thomas Z Date: Thu, 13 Feb 2020 15:57:58 -0800 Subject: [PATCH 5/5] fixed typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1beda6e83..04dc00f3c 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,9 @@ With Commit0: Commit0 CLI is an opinionated, yet fully modular code generation tool with declarative syntax that allows developers to easily integrate user prompts and interactions. Problems we encountered: -- It was tedious to creating reusable templates and hard to maintain +- It was tedious to create reusable templates and hard to maintain - Lack of standardization and integration interface between the templates -- Difficult to integrate multiple templated codebase +- Difficult to integrate individually templated codebases How we aim to address those issues: - Make templating behaviour simple and declarative