[AIRFLOW-2952] Splits CI into k8s + docker-compose

.travis.yml

@@ -19,21 +19,23 @@
 sudo: true
 dist: trusty
 language: python
+services:
+  - postgresql
 env:
   global:
     - DOCKER_COMPOSE_VERSION=1.20.0
     - SLUGIFY_USES_TEXT_UNIDECODE=yes
     - TRAVIS_CACHE=$HOME/.travis_cache/
   matrix:
     - TOX_ENV=flake8
-    - TOX_ENV=py27-backend_mysql
-    - TOX_ENV=py27-backend_sqlite
-    - TOX_ENV=py27-backend_postgres
-    - TOX_ENV=py35-backend_mysql PYTHON_VERSION=3
-    - TOX_ENV=py35-backend_sqlite PYTHON_VERSION=3
-    - TOX_ENV=py35-backend_postgres PYTHON_VERSION=3
-    - TOX_ENV=py27-backend_postgres KUBERNETES_VERSION=v1.9.0
-    - TOX_ENV=py35-backend_postgres KUBERNETES_VERSION=v1.10.0 PYTHON_VERSION=3
+    - TOX_ENV=py27-backend_mysql-env_docker
+    - TOX_ENV=py27-backend_sqlite-env_docker
+    - TOX_ENV=py27-backend_postgres-env_docker
+    - TOX_ENV=py35-backend_mysql-env_docker PYTHON_VERSION=3
+    - TOX_ENV=py35-backend_sqlite-env_docker PYTHON_VERSION=3
+    - TOX_ENV=py35-backend_postgres-env_docker PYTHON_VERSION=3
+    - TOX_ENV=py27-backend_postgres_k8s-env_kubernetes KUBERNETES_VERSION=v1.9.0
+    - TOX_ENV=py35-backend_postgres_k8s-env_kubernetes KUBERNETES_VERSION=v1.10.0 PYTHON_VERSION=3
 cache:
   directories:
     - $HOME/.wheelhouse/
@@ -43,12 +45,19 @@ before_install:
   - sudo ls -lh $HOME/.cache/pip/
   - sudo rm -rf $HOME/.cache/pip/* $HOME/.wheelhouse/*
   - sudo chown -R travis:travis $HOME/.cache/pip
+  - ./scripts/ci/docker-registry.sh
 install:
   # Use recent docker-compose version
   - sudo rm /usr/local/bin/docker-compose
   - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
   - chmod +x docker-compose
   - sudo mv docker-compose /usr/local/bin
+  - psql -c 'create database airflow;' -U postgres
   - pip install --upgrade pip
+  - pip install tox
+  - pip install codecov
 script:
-  - docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml run airflow-testing /app/scripts/ci/run-ci.sh
+  - ./scripts/ci/run-ci.sh
+after_success:
+  - sudo chown -R travis.travis .
+  - codecov

scripts/ci/daemon.json

@@ -0,0 +1,3 @@
+{
+  "insecure-registries" : ["10.192.0.1:5000"]
+}

scripts/ci/docker-registry.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+_UNAME_OUT=$(uname -s)
+case "${_UNAME_OUT}" in
+    Linux*)     _MY_OS=linux;;
+    Darwin*)    _MY_OS=darwin;;
+    *)          echo "${_UNAME_OUT} is unsupported."
+                exit 1;;
+esac
+
+if [ "$_MY_OS" = "linux" ]; then
+    export _REGISTRY_IP=10.192.0.1
+else
+    export _REGISTRY_IP=`ipconfig getifaddr en0`
+fi
+echo "Deploying insecure registry"
+if [ "$_MY_OS" = "linux" ]; then
+    DIRNAME=$(cd "$(dirname "$0")"; pwd)
+#    sudo rm /etc/docker/daemon.json
+#    sudo cp $DIRNAME/daemon.json /etc/docker/
+    sudo sed -i "s/\DOCKER_OPTS=\"/DOCKER_OPTS=\"--insecure-registry=$_REGISTRY_IP:5000 /g" /etc/default/docker
+    sudo cat /etc/default/docker
+    sudo service docker restart
+fi
+
+
+

scripts/ci/kubernetes/docker/build.sh

@@ -22,17 +22,32 @@ TAG=${2:-latest}
 DIRNAME=$(cd "$(dirname "$0")"; pwd)
 AIRFLOW_ROOT="$DIRNAME/../../../.."
 
-ENVCONFIG=$(minikube docker-env)
-if [ $? -eq 0 ]; then
-  eval $ENVCONFIG
-fi
+#ENVCONFIG=$(minikube docker-env)
+#if [ $? -eq 0 ]; then
+#  eval $ENVCONFIG
+#fi
+_UNAME_OUT=$(uname -s)
+case "${_UNAME_OUT}" in
+    Linux*)     _MY_OS=linux;;
+    Darwin*)    _MY_OS=darwin;;
+    *)          echo "${_UNAME_OUT} is not unsupported."
+                exit 1;;
+esac
+echo "Local OS is ${_MY_OS}"
 
+if [ "$_MY_OS" = "linux" ]; then
+    export _REGISTRY_IP=10.192.0.1
+else
+    export _REGISTRY_IP=`ipconfig getifaddr en0`
+fi
 echo "Airflow directory $AIRFLOW_ROOT"
 echo "Airflow Docker directory $DIRNAME"
 
 cd $AIRFLOW_ROOT
 python setup.py sdist -q
 echo "Copy distro $AIRFLOW_ROOT/dist/*.tar.gz ${DIRNAME}/airflow.tar.gz"
 cp $AIRFLOW_ROOT/dist/*.tar.gz ${DIRNAME}/airflow.tar.gz
-cd $DIRNAME && docker build --pull $DIRNAME --tag=${IMAGE}:${TAG}
+cd $DIRNAME && docker build --pull $DIRNAME --tag=${_REGISTRY_IP}:5000/${IMAGE}:${TAG}
+docker push ${_REGISTRY_IP}:5000/${IMAGE}:${TAG}
+docker exec kube-node-1 docker pull ${_REGISTRY_IP}:5000/${IMAGE}:${TAG}
 rm $DIRNAME/airflow.tar.gz

scripts/ci/kubernetes/kube/airflow.yaml → scripts/ci/kubernetes/kube/airflow.yaml.template

@@ -43,7 +43,7 @@ spec:
     spec:
       initContainers:
       - name: "init"
-        image: airflow
+        image: {REG_IP}:5000/airflow
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: airflow-configmap
@@ -65,8 +65,8 @@ spec:
           - "-cx"
           - "./tmp/airflow-test-env-init.sh"
       containers:
-      - name: webserver
-        image: airflow
+      - name: webserv
+        image: {REG_IP}:5000/airflow
         imagePullPolicy: IfNotPresent
         ports:
         - name: webserver
@@ -105,7 +105,7 @@ spec:
 #            path: /login
 #            port: 8080
       - name: scheduler
-        image: airflow
+        image: {REG_IP}:5000/airflow
         imagePullPolicy: IfNotPresent
         args: ["scheduler"]
         env:

scripts/ci/kubernetes/kube/deploy.sh

@@ -16,10 +16,27 @@
 #  KIND, either express or implied.  See the License for the    *
 #  specific language governing permissions and limitations      *
 #  under the License.                                           *
+_UNAME_OUT=$(uname -s)
+case "${_UNAME_OUT}" in
+    Linux*)     _MY_OS=linux;;
+    Darwin*)    _MY_OS=darwin;;
+    *)          echo "${_UNAME_OUT} is not unsupported."
+                exit 1;;
+esac
+echo "Local OS is ${_MY_OS}"
+
+if [ "$_MY_OS" = "linux" ]; then
+    export _REGISTRY_IP=10.192.0.1
+else
+    export _REGISTRY_IP=`ipconfig getifaddr en0`
+fi
 
 IMAGE=${1:-airflow/ci}
 TAG=${2:-latest}
 DIRNAME=$(cd "$(dirname "$0")"; pwd)
+rm $DIRNAME/airflow.yaml
+sed -e s/{REG_IP}/$_REGISTRY_IP/g $DIRNAME/airflow.yaml.template > $DIRNAME/airflow.yaml
+cat $DIRNAME/airflow.yaml
 
 kubectl delete -f $DIRNAME/postgres.yaml
 kubectl delete -f $DIRNAME/airflow.yaml

scripts/ci/kubernetes/kube/volumes.yaml

@@ -35,7 +35,7 @@ metadata:
   name: airflow-dags
 spec:
   accessModes:
-    - ReadWriteMany
+    - ReadWriteOnce
   resources:
     requests:
       storage: 2Gi
@@ -81,7 +81,7 @@ metadata:
   name: test-volume
 spec:
   accessModes:
-    - ReadWriteMany
+    - ReadWriteOnce
   resources:
     requests:
       storage: 2Gi

scripts/ci/kubernetes/minikube/start_kubeadm.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+_MY_SCRIPT="${BASH_SOURCE[0]}"
+_MY_DIR=$(cd "$(dirname "$_MY_SCRIPT")" && pwd)
+_UNAME_OUT=$(uname -s)
+case "${_UNAME_OUT}" in
+    Linux*)     _MY_OS=linux;;
+    Darwin*)    _MY_OS=darwin;;
+    *)          echo "${_UNAME_OUT} is not unsupported."
+                exit 1;;
+esac
+echo "Local OS is ${_MY_OS}"
+_KUBERNETES_VERSION="${KUBERNETES_VERSION}"
+if [ "$_MY_OS" = "linux" ]; then
+    export _REGISTRY_IP="10.192.0.1"
+else
+    export _REGISTRY_IP=`ipconfig getifaddr en0`
+fi
+
+cd $_MY_DIR
+rm -rf tmp
+mkdir -p bin tmp
+
+rm $DIRNAME/dind-cluster-*
+wget https://cdn.rawgit.com/kubernetes-sigs/kubeadm-dind-cluster/master/fixed/dind-cluster-v1.10.sh
+
+chmod +x $_MY_DIR/dind-cluster-v1.10.sh
+$_MY_DIR/dind-cluster-v1.10.sh clean
+echo "registries are "[\"${_REGISTRY_IP}:5000\"]""
+DIND_INSECURE_REGISTRIES="[\"${_REGISTRY_IP}:5000\"]" DIND_SUBNET="10.192.0.0" DIND_SUBNET_SIZE=16 $_MY_DIR/dind-cluster-v1.10.sh up
+export PATH="$HOME/.kubeadm-dind-cluster:$PATH"
+
+if [[ ! -x /usr/local/bin/kubectl ]]; then
+  echo Downloading kubectl, which is a requirement for using minikube.
+  curl -Lo bin/kubectl  \
+    https://storage.googleapis.com/kubernetes-release/release/${_KUBERNETES_VERSION}/bin/${_MY_OS}/amd64/kubectl
+  chmod +x bin/kubectl
+  sudo mv bin/kubectl /usr/local/bin/kubectl
+fi
+
+REG=`docker ps -f name=registry -q`
+
+if [ -n "$REG" ]; then
+    docker stop $REG; docker rm $REG
+fi
+
+if [ "$_MY_OS" = "linux" ]; then
+    docker run -d -p :5000:5000 --restart=always --name registry registry:2
+else
+    docker run -d -p 5000:5000 --restart=always --name registry registry:2
+fi

scripts/ci/kubernetes/setup_kubernetes.sh

@@ -22,8 +22,7 @@ echo "This script downloads minikube, starts a driver=None minikube cluster, bui
 echo "For development, start minikube yourself (ie: minikube start) then run this script as you probably do not want a driver=None minikube cluster"
 
 DIRNAME=$(cd "$(dirname "$0")"; pwd)
-
-$DIRNAME/minikube/start_minikube.sh
+$DIRNAME/minikube/start_kubeadm.sh
 $DIRNAME/docker/build.sh
 $DIRNAME/kube/deploy.sh
 

scripts/ci/run-ci-docker.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+set -x -e
+
+DIRNAME=$(cd "$(dirname "$0")"; pwd)
+AIRFLOW_ROOT="$DIRNAME/../.."
+
+# Fix file permissions
+sudo chown -R airflow.airflow . $HOME/.wheelhouse/ $HOME/.cache/pip
+
+if [[ $PYTHON_VERSION == '3' ]]; then
+  PIP=pip3
+else
+  PIP=pip
+fi
+
+sudo $PIP install --upgrade pip
+sudo $PIP install tox
+
+cd $AIRFLOW_ROOT && $PIP --version && tox --version
+
+tox -e $TOX_ENV

scripts/ci/run-ci-kubernetes.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+#
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing,
+#  software distributed under the License is distributed on an
+#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#  KIND, either express or implied.  See the License for the
+#  specific language governing permissions and limitations
+#  under the License.
+
+set -x -e
+
+DIRNAME=$(cd "$(dirname "$0")"; pwd)
+AIRFLOW_ROOT="$DIRNAME/../.."
+
+# Fix file permissions
+sudo chown -R travis.travis . $HOME/.wheelhouse/ $HOME/.cache/pip
+sudo apt-get install python3-pip
+
+if [[ $PYTHON_VERSION == '3' ]]; then
+  PIP=pip3
+else
+  PIP=pip
+fi
+
+sudo $PIP install --upgrade pip
+sudo $PIP install tox
+
+KUBERNETES_VERSION=${KUBERNETES_VERSION} $DIRNAME/kubernetes/setup_kubernetes.sh && \
+tox -e $TOX_ENV -- tests.contrib.minikube \
+                 --with-coverage \
+                 --cover-erase \
+                 --cover-html \
+                 --cover-package=airflow \
+                 --cover-html-dir=airflow/www/static/coverage \
+                 --with-ignore-docstrings \
+                 --rednose \
+                 --with-timer \
+                 -v \
+                 --logging-level=DEBUG

scripts/ci/run-ci.sh

@@ -19,38 +19,11 @@
 #  under the License.
 
 set -x
-
 DIRNAME=$(cd "$(dirname "$0")"; pwd)
-AIRFLOW_ROOT="$DIRNAME/../.."
-
-# Fix file permissions
-sudo chown -R airflow.airflow . $HOME/.wheelhouse/ $HOME/.cache/pip
-
-if [[ $PYTHON_VERSION == '3' ]]; then
-  PIP=pip3
-else
-  PIP=pip
-fi
-
-sudo $PIP install --upgrade pip
-sudo $PIP install tox
-
-cd $AIRFLOW_ROOT && $PIP --version && tox --version
 
 if [ -z "$KUBERNETES_VERSION" ];
 then
-  tox -e $TOX_ENV
+  docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml run airflow-testing /app/scripts/ci/run-ci-docker.sh
 else
-  KUBERNETES_VERSION=${KUBERNETES_VERSION} $DIRNAME/kubernetes/setup_kubernetes.sh && \
-  tox -e $TOX_ENV -- tests.contrib.minikube \
-                     --with-coverage \
-                     --cover-erase \
-                     --cover-html \
-                     --cover-package=airflow \
-                     --cover-html-dir=airflow/www/static/coverage \
-                     --with-ignore-docstrings \
-                     --rednose \
-                     --with-timer \
-                     -v \
-                     --logging-level=DEBUG
+  $DIRNAME/run-ci-kubernetes.sh
 fi