[AIRFLOW-2947] Added Helm chart for Kubernetes executor

scripts/ci/kubernetes/helm/airflow/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

scripts/ci/kubernetes/helm/airflow/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: airflow
+version: 0.1.0
+appVersion: 1.10.0

scripts/ci/kubernetes/helm/airflow/requirements.yaml

@@ -0,0 +1,5 @@
+dependencies:
+- name: postgresql
+  version: 0.15.0
+  repository: https://kubernetes-charts.storage.googleapis.com/
+  condition: postgresql.enabled

scripts/ci/kubernetes/helm/airflow/templates/NOTES.txt

@@ -0,0 +1,19 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range .Values.ingress.hosts }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }}{{ $.Values.ingress.path }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "airflow.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get svc -w {{ template "airflow.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "airflow.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "airflow.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl port-forward $POD_NAME 8080:80
+{{- end }}

scripts/ci/kubernetes/helm/airflow/templates/_helpers.tpl

@@ -0,0 +1,42 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "airflow.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "airflow.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "airflow.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified postgresql name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "airflow.postgresql.fullname" -}}
+{{- $name := default "postgresql" .Values.postgresql.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+

scripts/ci/kubernetes/kube/configmaps.yaml → scripts/ci/kubernetes/helm/airflow/templates/configmaps.yaml

@@ -17,23 +17,30 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: airflow-configmap
+  name: {{ template "airflow.fullname" . }}-config
+  labels:
+    app: {{ template "airflow.name" . }}
+    chart: {{ template "airflow.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
 data:
   airflow.cfg: |
     [core]
-    airflow_home = /root/airflow
-    dags_folder = /root/airflow/dags
-    base_log_folder = /root/airflow/logs
+    airflow_home = {{ .Values.airflow.core.airflow_home }}
+    dags_folder = {{ .Values.airflow.core.dags_folder }}
+    base_log_folder = {{ .Values.airflow.core.base_log_folder }}
     logging_level = INFO
+    logging_config_class = {{ .Values.airflow.core.logging_config_class }}
     executor = KubernetesExecutor
     parallelism = 32
-    load_examples = True
-    plugins_folder = /root/airflow/plugins
+    load_examples = {{ default "True" .Values.airflow.core.load_examples }}
+    plugins_folder = {{ .Values.airflow.core.plugins_folder }}
     sql_alchemy_conn = $SQL_ALCHEMY_CONN
+    encrypt_s3_logs = False
 
     [scheduler]
     dag_dir_list_interval = 300
-    child_process_log_directory = /root/airflow/logs/scheduler
+    child_process_log_directory = {{ .Values.airflow.scheduler.child_process_log_directory }}
     # Task instances listen for external kill signal (when you clear tasks
     # from the CLI or the UI), this defines the frequency at which they should
     # listen (in seconds).
@@ -57,6 +64,9 @@ data:
     statsd_port = 8125
     statsd_prefix = airflow
 
+    # How many seconds to wait between file-parsing loops to prevent the logs from being spammed.
+    min_file_parsing_loop_time = 1
+
     print_stats_interval = 30
     scheduler_zombie_task_threshold = 300
     max_tis_per_query = 0
@@ -172,37 +182,32 @@ data:
     smtp_mail_from = airflow@example.com
 
     [kubernetes]
-    airflow_configmap = airflow-configmap
-    worker_container_repository = airflow
-    worker_container_tag = latest
+    airflow_configmap = {{ template "airflow.fullname" . }}-config
+    worker_container_repository = {{ .Values.image.repository }}
+    worker_container_tag = {{ .Values.image.tag }}
     worker_container_image_pull_policy = IfNotPresent
-    worker_dags_folder = /tmp/dags
-    delete_worker_pods = True
-    git_repo = https://github.com/apache/incubator-airflow.git
-    git_branch = master
-    git_subpath = airflow/example_dags/
-    git_user =
-    git_password =
-    dags_volume_claim = airflow-dags
-    dags_volume_subpath =
-    logs_volume_claim = airflow-logs
-    logs_volume_subpath =
-    in_cluster = True
-    namespace = default
-    gcp_service_account_keys =
+    delete_worker_pods = {{ default "False" .Values.airflow.kubernetes.delete_worker_pods }}
+    git_repo = {{ .Values.airflow.kubernetes.git_repo }}
+    git_branch = {{ .Values.airflow.kubernetes.git_branch }}
+    git_subpath = {{ .Values.airflow.kubernetes.git_subpath }}
+    git_user = {{ .Values.airflow.kubernetes.git_user }}
+    git_password = {{ .Values.airflow.kubernetes.git_password }}
+    dags_volume_claim = {{ .Values.airflow.kubernetes.dags_volume_claim }}
+    dags_volume_subpath = {{ .Values.airflow.kubernetes.dags_volume_subpath }}
+    logs_volume_claim = {{ .Values.airflow.kubernetes.logs_volume_claim }}
+    logs_volume_subpath = {{ .Values.airflow.kubernetes.logs_volume_subpath }}
+    in_cluster = {{ default "True" .Values.airflow.kubernetes.in_cluster }}
+    namespace = {{ .Release.Namespace }}
+    gcp_service_account_keys = {{ .Values.airflow.kubernetes.gcp_service_account_keys }}
+    worker_service_account_name = {{ template "airflow.fullname" . }}-worker
 
     # For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync
     git_sync_container_repository = gcr.io/google-containers/git-sync-amd64
-    git_sync_container_tag = v2.0.5
+    git_sync_container_tag = v2.0.6
     git_sync_init_container_name = git-sync-clone
 
-    [kubernetes_node_selectors]
-    # The Key-value pairs to be given to worker pods.
-    # The worker pods will be scheduled to the nodes of the specified key-value pairs.
-    # Should be supplied in the format: key = value
-
     [kubernetes_secrets]
-    SQL_ALCHEMY_CONN = airflow-secrets=sql_alchemy_conn
+    SQL_ALCHEMY_CONN = {{ template "airflow.fullname" . }}-secrets=sql_alchemy_conn
 
     [hive]
     # Default mapreduce queue for HiveOperator tasks

scripts/ci/kubernetes/helm/airflow/templates/deployment-scheduler.yaml

@@ -0,0 +1,70 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: {{ template "airflow.fullname" . }}-scheduler
+  labels:
+    app: {{ template "airflow.name" . }}-scheduler
+    chart: {{ template "airflow.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ template "airflow.name" . }}-scheduler
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "airflow.name" . }}-scheduler
+        release: {{ .Release.Name }}
+    spec:
+      serviceAccountName: {{ template "airflow.fullname" . }}-scheduler
+      containers:
+        - name: {{ .Chart.Name }}-scheduler
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args: ["scheduler"]
+          env:
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: POSTGRES_HOST
+            value: {{ template "airflow.postgresql.fullname" . }}
+          - name: POSTGRES_NAME
+            value: {{ .Values.postgresql.postgresDatabase }}
+          - name: POSTGRES_PASSWORD
+            value: {{ .Values.postgresql.postgresPassword }}
+          - name: POSTGRES_USERNAME
+            value: {{ .Values.postgresql.postgresUser }}
+          - name: SQL_ALCHEMY_CONN
+            value: postgresql+psycopg2://{{ .Values.postgresql.postgresUser }}:{{ .Values.postgresql.postgresPassword }}@{{ template "airflow.postgresql.fullname" . }}:5432/{{ .Values.postgresql.postgresDatabase }}
+          - name: TZ
+            value: Etc/UTC
+          volumeMounts:
+          - name: airflow-configmap
+            mountPath: /home/airflow/airflow.cfg
+            subPath: airflow.cfg
+          - name: airflow-logs
+            mountPath: /home/airflow/logs
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+      volumes:
+      - name: airflow-configmap
+        configMap:
+          name: {{ template "airflow.fullname" . }}-config
+      - name: airflow-logs
+        emptyDir: {}
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}

scripts/ci/kubernetes/helm/airflow/templates/deployment-web.yaml

@@ -0,0 +1,95 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: {{ template "airflow.fullname" . }}-web
+  labels:
+    app: {{ template "airflow.name" . }}-web
+    chart: {{ template "airflow.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ template "airflow.name" . }}-web
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "airflow.name" . }}-web
+        release: {{ .Release.Name }}
+    spec:
+      initContainers:
+        - name: "init"
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+          - name: airflow-configmap
+            mountPath: /home/airflow/airflow.cfg
+            subPath: airflow.cfg
+          env:
+          - name: SQL_ALCHEMY_CONN
+            value: postgresql+psycopg2://{{ .Values.postgresql.postgresUser }}:{{ .Values.postgresql.postgresPassword }}@{{ template "airflow.postgresql.fullname" . }}:5432/{{ .Values.postgresql.postgresDatabase }}
+          command:
+            - "bash"
+          args:
+            - "-cx"
+            - "mkdir -pv /home/airflow/dags/ && ./home/airflow/airflow-init.sh"
+      containers:
+        - name: {{ .Chart.Name }}-web
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: webserver
+              containerPort: 8080
+              protocol: TCP
+          args: ["webserver"]
+          livenessProbe:
+            httpGet:
+              path: /login/
+              port: webserver
+          readinessProbe:
+            httpGet:
+              path: /login/
+              port: webserver
+          env:
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: POSTGRES_HOST
+            value: {{ template "airflow.postgresql.fullname" . }}
+          - name: POSTGRES_NAME
+            value: {{ .Values.postgresql.postgresDatabase }}
+          - name: POSTGRES_PASSWORD
+            value: {{ .Values.postgresql.postgresPassword }}
+          - name: POSTGRES_USERNAME
+            value: {{ .Values.postgresql.postgresUser }}
+          - name: SQL_ALCHEMY_CONN
+            value: postgresql+psycopg2://{{ .Values.postgresql.postgresUser }}:{{ .Values.postgresql.postgresPassword }}@{{ template "airflow.postgresql.fullname" . }}:5432/{{ .Values.postgresql.postgresDatabase }}
+          - name: TZ
+            value: Etc/UTC
+          volumeMounts:
+          - name: airflow-configmap
+            mountPath: /home/airflow/airflow.cfg
+            subPath: airflow.cfg
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+      volumes:
+      - name: airflow-configmap
+        configMap:
+          name: {{ template "airflow.fullname" . }}-config
+      - name: airflow-logs
+        emptyDir: {}
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}