From 107bcc6cdacf3a11efad31d2324eb420fe57c29a Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Wed, 16 Dec 2020 16:03:22 -0800
Subject: [PATCH 01/12] zk-less druid cluster in k8s build

---
 .../script/build_run_k8s_cluster.sh           |  4 +-
 integration-tests/tiny-cluster-zk.yaml        | 97 -------------------
 integration-tests/tiny-cluster.yaml           | 16 ++-
 3 files changed, 12 insertions(+), 105 deletions(-)
 delete mode 100644 integration-tests/tiny-cluster-zk.yaml

diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh
index 5c9428072088..8bd25b8f8012 100755
--- a/integration-tests/script/build_run_k8s_cluster.sh
+++ b/integration-tests/script/build_run_k8s_cluster.sh
@@ -59,16 +59,14 @@ git checkout -b druid-operator-$DRUID_OPERATOR_VERSION druid-operator-$DRUID_OPE
 cd ..
 sed -i "s|REPLACE_IMAGE|druidio/druid-operator:$DRUID_OPERATOR_VERSION|g" druid-operator/deploy/operator.yaml
 cp integration-tests/tiny-cluster.yaml druid-operator/examples/
-cp integration-tests/tiny-cluster-zk.yaml druid-operator/examples/
 sed -i "s|REPLACE_VOLUMES|`pwd`|g" druid-operator/examples/tiny-cluster.yaml
 
-# Create ZK, Historical, MiddleManager, Overlord-coordiantor, Broker and Router pods using statefulset
+# Deploy Druid Operator and Druid CR spec
 sudo /usr/local/bin/kubectl create -f druid-operator/deploy/service_account.yaml
 sudo /usr/local/bin/kubectl create -f druid-operator/deploy/role.yaml
 sudo /usr/local/bin/kubectl create -f druid-operator/deploy/role_binding.yaml
 sudo /usr/local/bin/kubectl create -f druid-operator/deploy/crds/druid.apache.org_druids_crd.yaml
 sudo /usr/local/bin/kubectl create -f druid-operator/deploy/operator.yaml
-sudo /usr/local/bin/kubectl apply -f druid-operator/examples/tiny-cluster-zk.yaml
 sudo /usr/local/bin/kubectl apply -f druid-operator/examples/tiny-cluster.yaml
 
 # Wait 4 * 15 seconds to launch pods.
diff --git a/integration-tests/tiny-cluster-zk.yaml b/integration-tests/tiny-cluster-zk.yaml
deleted file mode 100644
index d83612e78fa0..000000000000
--- a/integration-tests/tiny-cluster-zk.yaml
+++ /dev/null
@@ -1,97 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: tiny-cluster-zk
-spec:
-  clusterIP: None
-  ports:
-    - name: zk-client-port
-      port: 2181
-    - name: zk-fwr-port
-      port: 2888
-    - name: zk-elec-port
-      port: 3888
-  selector:
-    zk_cluster: tiny-cluster-zk
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: tiny-cluster-zk-nodeport
-spec:
-  type: NodePort
-  ports:
-    - name: zk-service-port
-      nodePort: 30600
-      port: 2181
-      protocol: TCP
-      targetPort: 2181
-  selector:
-    zk_cluster: tiny-cluster-zk
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  labels:
-    zk_cluster: tiny-cluster-zk
-  name: tiny-cluster-zk
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      zk_cluster: tiny-cluster-zk
-  serviceName: tiny-cluster-zk
-  template:
-    metadata:
-      labels:
-        zk_cluster: tiny-cluster-zk
-    spec:
-      containers:
-        - env:
-            - name: ZOO_SERVERS
-              value: server.0=tiny-cluster-zk-0.tiny-cluster-zk:2888:3888
-            - name: SERVER_JVMFLAGS
-              value: -Xms256m -Xmx256m
-          image: zookeeper:3.4.13
-          name: tiny-cluster-zk
-          command: ["/bin/sh"]
-          args: ["-c", "ZOO_MY_ID=$(echo `hostname` | cut -d '-' -f2) /docker-entrypoint.sh zkServer.sh start-foreground"]
-          ports:
-            - containerPort: 2181
-              name: zk-client-port
-            - containerPort: 2888
-              name: zk-fwr-port
-            - containerPort: 3888
-              name: zk-elec-port
-          resources:
-            limits:
-              cpu: 1
-              memory: 512Mi
-            requests:
-              cpu: 1
-              memory: 512Mi
-          volumeMounts:
-            - mountPath: /data
-              name: druid-test-zk-data
-            - mountPath: /datalog
-              name: druid-test-zk-data-log
-      volumes:
-        - name: druid-test-zk-data
-          emptyDir: {}
-        - name: druid-test-zk-data-log
-          emptyDir: {}
diff --git a/integration-tests/tiny-cluster.yaml b/integration-tests/tiny-cluster.yaml
index d3b89b2a5ba9..2639618ad514 100644
--- a/integration-tests/tiny-cluster.yaml
+++ b/integration-tests/tiny-cluster.yaml
@@ -60,10 +60,15 @@ spec:
     </Configuration>
   common.runtime.properties: |
 
-    # Zookeeper
-    druid.zk.service.host=tiny-cluster-zk-0.tiny-cluster-zk
-    druid.zk.paths.base=/druid
-    druid.zk.service.compress=false
+    #
+    # Zookeeper-less Druid Cluster
+    #
+    druid.zk.service.enabled=false
+    druid.discovery.type=k8s
+    druid.discovery.k8s.clusterIdentifier=druid-it
+    druid.serverview.type=http
+    druid.coordinator.loadqueuepeon.type=http
+    druid.indexer.runner.type=httpRemote
 
     # Metadata Store
     druid.metadata.storage.type=derby
@@ -79,13 +84,14 @@ spec:
     #
     # Extensions
     #
-    druid.extensions.loadList=["druid-avro-extensions","druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches"]
+    druid.extensions.loadList=["druid-avro-extensions","druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-kubernetes-extensions"]
 
     #
     # Service discovery
     #
     druid.selectors.indexing.serviceName=druid/overlord
     druid.selectors.coordinator.serviceName=druid/coordinator
+
   nodes:
     brokers:
       # Optionally specify for running broker as Deployment

From 04c488e52c98a49107eeff586c81b838ee69110f Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Thu, 17 Dec 2020 08:41:01 -0800
Subject: [PATCH 02/12] attempt to fix build and use http based remote task
 management

---
 .travis.yml                                    |  2 +-
 .../k8s/discovery/K8sDruidNodeAnnouncer.java   |  2 +-
 .../script/build_run_k8s_cluster.sh            |  2 +-
 integration-tests/tiny-cluster.yaml            | 18 ++++++++++++++++--
 4 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 38258665162d..6bd71de7eb3c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -554,7 +554,7 @@ jobs:
       jdk: openjdk8
       services: &integration_test_services_k8s
         - docker
-      env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest'
+      env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest' POD_NAME=int-test POD_NAMESPACE=default
       before_script: integration-tests/script/build_run_k8s_cluster.sh
       script: &run_integration_test_k8s
         - ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP}
diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java
index 29c06f443a3e..d43c93ed17b1 100644
--- a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java
+++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java
@@ -120,7 +120,7 @@ public void announce(DiscoveryDruidNode discoveryDruidNode)
       LOGGER.info("Announced DiscoveryDruidNode[%s]", discoveryDruidNode);
     }
     catch (Exception ex) {
-      throw new RE(ex, "Failed to announce DiscoveryDruidNode[%s]", discoveryDruidNode);
+      throw new RE(ex, "Failed to announce DiscoveryDruidNode[%s], Does the Pod have atleast one label and annotation?", discoveryDruidNode);
     }
   }
 
diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh
index 8bd25b8f8012..645d54478c06 100755
--- a/integration-tests/script/build_run_k8s_cluster.sh
+++ b/integration-tests/script/build_run_k8s_cluster.sh
@@ -79,4 +79,4 @@ sleep 120
 sudo /usr/local/bin/kubectl get pod
 sudo /usr/local/bin/kubectl get svc
 docker images
-sudo /usr/local/bin/kubectl describe pod druid-tiny-cluster-middlemanagers-0
\ No newline at end of file
+sudo /usr/local/bin/kubectl describe pod druid-tiny-cluster-middlemanagers-0
diff --git a/integration-tests/tiny-cluster.yaml b/integration-tests/tiny-cluster.yaml
index 2639618ad514..f7163534ae7d 100644
--- a/integration-tests/tiny-cluster.yaml
+++ b/integration-tests/tiny-cluster.yaml
@@ -26,6 +26,8 @@ spec:
   podLabels:
     environment: stage
     release: alpha
+  podAnnotations:
+    dummy: k8s_extn_needs_atleast_one_annotation
   securityContext:
     fsGroup: 0
     runAsUser: 0
@@ -92,6 +94,19 @@ spec:
     druid.selectors.indexing.serviceName=druid/overlord
     druid.selectors.coordinator.serviceName=druid/coordinator
 
+    druid.indexer.logs.type=file
+    druid.indexer.logs.directory=/druid/data/task-logs
+    druid.indexer.task.baseDir=/druid/data/task-base
+  env:
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+
   nodes:
     brokers:
       # Optionally specify for running broker as Deployment
@@ -180,7 +195,6 @@ spec:
         druid.coordinator.asOverlord.enabled=true
         druid.coordinator.asOverlord.overlordService=druid/overlord
         druid.indexer.queue.startDelay=PT30S
-        druid.indexer.runner.type=local
       extra.jvm.options: |-
         -Xmx800m
         -Xms800m
@@ -323,4 +337,4 @@ spec:
         requests:
           memory: "3G"
         limits:
-          memory: "3G"
\ No newline at end of file
+          memory: "3G"

From d8f3f2177c4ee6c9fe63b0bf143fe89995afbb20 Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Thu, 17 Dec 2020 14:55:39 -0800
Subject: [PATCH 03/12] mm/router logs for debugging

---
 integration-tests/script/build_run_k8s_cluster.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh
index 645d54478c06..24573cc4b216 100755
--- a/integration-tests/script/build_run_k8s_cluster.sh
+++ b/integration-tests/script/build_run_k8s_cluster.sh
@@ -80,3 +80,12 @@ sudo /usr/local/bin/kubectl get pod
 sudo /usr/local/bin/kubectl get svc
 docker images
 sudo /usr/local/bin/kubectl describe pod druid-tiny-cluster-middlemanagers-0
+
+echo "MM Logs............"
+sudo /usr/local/bin/kubectl logs --previous druid-tiny-cluster-middlemanagers-0
+
+echo "Router Logs........."
+sudo /usr/local/bin/kubectl logs --previous druid-tiny-cluster-routers-0
+
+
+

From b769b0e961dfae401820102d38a56bf87f5a448a Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Fri, 18 Dec 2020 13:32:30 -0800
Subject: [PATCH 04/12] add default account k8s role and binding for pod,
 configMap access

---
 integration-tests/k8s/role-and-binding.yaml   | 24 +++++++++++++++++++
 integration-tests/{ => k8s}/tiny-cluster.yaml |  4 ++++
 .../script/build_run_k8s_cluster.sh           | 12 ++++------
 3 files changed, 32 insertions(+), 8 deletions(-)
 create mode 100644 integration-tests/k8s/role-and-binding.yaml
 rename integration-tests/{ => k8s}/tiny-cluster.yaml (99%)

diff --git a/integration-tests/k8s/role-and-binding.yaml b/integration-tests/k8s/role-and-binding.yaml
new file mode 100644
index 000000000000..044e5b630be4
--- /dev/null
+++ b/integration-tests/k8s/role-and-binding.yaml
@@ -0,0 +1,24 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: druid-cluster
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - configmaps
+  verbs:
+  - '*'
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: druid-cluster
+subjects:
+- kind: ServiceAccount
+  name: default
+roleRef:
+  kind: Role
+  name: druid-cluster
+  apiGroup: rbac.authorization.k8s.io
diff --git a/integration-tests/tiny-cluster.yaml b/integration-tests/k8s/tiny-cluster.yaml
similarity index 99%
rename from integration-tests/tiny-cluster.yaml
rename to integration-tests/k8s/tiny-cluster.yaml
index f7163534ae7d..8529ab0dbb7d 100644
--- a/integration-tests/tiny-cluster.yaml
+++ b/integration-tests/k8s/tiny-cluster.yaml
@@ -28,6 +28,10 @@ spec:
     release: alpha
   podAnnotations:
     dummy: k8s_extn_needs_atleast_one_annotation
+  readinessProbe:
+    httpGet:
+      path: /status/health
+      port: 8088
   securityContext:
     fsGroup: 0
     runAsUser: 0
diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh
index 24573cc4b216..f9ff3663f90e 100755
--- a/integration-tests/script/build_run_k8s_cluster.sh
+++ b/integration-tests/script/build_run_k8s_cluster.sh
@@ -58,8 +58,6 @@ cd druid-operator
 git checkout -b druid-operator-$DRUID_OPERATOR_VERSION druid-operator-$DRUID_OPERATOR_VERSION
 cd ..
 sed -i "s|REPLACE_IMAGE|druidio/druid-operator:$DRUID_OPERATOR_VERSION|g" druid-operator/deploy/operator.yaml
-cp integration-tests/tiny-cluster.yaml druid-operator/examples/
-sed -i "s|REPLACE_VOLUMES|`pwd`|g" druid-operator/examples/tiny-cluster.yaml
 
 # Deploy Druid Operator and Druid CR spec
 sudo /usr/local/bin/kubectl create -f druid-operator/deploy/service_account.yaml
@@ -67,7 +65,10 @@ sudo /usr/local/bin/kubectl create -f druid-operator/deploy/role.yaml
 sudo /usr/local/bin/kubectl create -f druid-operator/deploy/role_binding.yaml
 sudo /usr/local/bin/kubectl create -f druid-operator/deploy/crds/druid.apache.org_druids_crd.yaml
 sudo /usr/local/bin/kubectl create -f druid-operator/deploy/operator.yaml
-sudo /usr/local/bin/kubectl apply -f druid-operator/examples/tiny-cluster.yaml
+
+sudo /usr/local/bin/kubectl apply -f integration-tests/k8s/role-and-binding.yaml
+sed -i "s|REPLACE_VOLUMES|`pwd`|g" integration-tests/k8s/tiny-cluster.yaml
+sudo /usr/local/bin/kubectl apply -f integration-tests/k8s/tiny-cluster.yaml
 
 # Wait 4 * 15 seconds to launch pods.
 #count=0
@@ -78,11 +79,6 @@ sleep 120
 
 sudo /usr/local/bin/kubectl get pod
 sudo /usr/local/bin/kubectl get svc
-docker images
-sudo /usr/local/bin/kubectl describe pod druid-tiny-cluster-middlemanagers-0
-
-echo "MM Logs............"
-sudo /usr/local/bin/kubectl logs --previous druid-tiny-cluster-middlemanagers-0
 
 echo "Router Logs........."
 sudo /usr/local/bin/kubectl logs --previous druid-tiny-cluster-routers-0

From d17489c358588dff678b4a08c9eb2cdb0e9fe175 Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Fri, 18 Dec 2020 15:30:48 -0800
Subject: [PATCH 05/12] fix issue

---
 integration-tests/script/build_run_k8s_cluster.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh
index f9ff3663f90e..ad0e474cc388 100755
--- a/integration-tests/script/build_run_k8s_cluster.sh
+++ b/integration-tests/script/build_run_k8s_cluster.sh
@@ -80,8 +80,3 @@ sleep 120
 sudo /usr/local/bin/kubectl get pod
 sudo /usr/local/bin/kubectl get svc
 
-echo "Router Logs........."
-sudo /usr/local/bin/kubectl logs --previous druid-tiny-cluster-routers-0
-
-
-

From 263dd77e4d4663fc39a8446711996a879ecfbd3d Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Sat, 19 Dec 2020 05:33:06 -0800
Subject: [PATCH 06/12] change router port to 8088 for common readinessProbe

---
 integration-tests/k8s/tiny-cluster.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/integration-tests/k8s/tiny-cluster.yaml b/integration-tests/k8s/tiny-cluster.yaml
index 8529ab0dbb7d..91d1d23d97b3 100644
--- a/integration-tests/k8s/tiny-cluster.yaml
+++ b/integration-tests/k8s/tiny-cluster.yaml
@@ -261,16 +261,16 @@ spec:
 
     routers:
       nodeType: "router"
-      druid.port: 8888
+      druid.port: 8088
       services:
         - spec:
             type: NodePort
             ports:
               - name: router-service-port
                 nodePort: 30400
-                port: 8888
+                port: 8088
                 protocol: TCP
-                targetPort: 8888
+                targetPort: 8088
           selector:
             nodeSpecUniqueStr: druid-tiny-cluster-routers
           metadata:
@@ -282,7 +282,7 @@ spec:
       replicas: 1
       runtime.properties: |
         druid.service=druid/router
-        druid.plaintextPort=8888
+        druid.plaintextPort=8088
 
         # HTTP proxy
         druid.router.http.numConnections=50

From f8bb7b119583e3c483e7034bba654364f3988f97 Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Sat, 19 Dec 2020 07:23:55 -0800
Subject: [PATCH 07/12] break build_run_k8s_cluster.sh into separate scripts

---
 .travis.yml                                   |  5 ++-
 ...n_k8s_cluster.sh => setup_druid_on_k8s.sh} | 34 ++---------------
 .../script/setup_druid_operator_on_k8s.sh     | 37 +++++++++++++++++++
 integration-tests/script/setup_k8s_cluster.sh | 35 ++++++++++++++++++
 4 files changed, 79 insertions(+), 32 deletions(-)
 rename integration-tests/script/{build_run_k8s_cluster.sh => setup_druid_on_k8s.sh} (54%)
 create mode 100755 integration-tests/script/setup_druid_operator_on_k8s.sh
 create mode 100755 integration-tests/script/setup_k8s_cluster.sh

diff --git a/.travis.yml b/.travis.yml
index 6bd71de7eb3c..31b3890ac4a9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -555,7 +555,10 @@ jobs:
       services: &integration_test_services_k8s
         - docker
       env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest' POD_NAME=int-test POD_NAMESPACE=default
-      before_script: integration-tests/script/build_run_k8s_cluster.sh
+      before_script:
+        - integration-tests/script/setup_k8s_cluster.sh
+        - integration-tests/script/setup_druid_operator_on_k8s.sh
+        - integration-tests/script/setup_druid_on_k8s.sh
       script: &run_integration_test_k8s
         - ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP}
       after_script: integration-tests/script/stop_k8s_cluster.sh
diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/setup_druid_on_k8s.sh
similarity index 54%
rename from integration-tests/script/build_run_k8s_cluster.sh
rename to integration-tests/script/setup_druid_on_k8s.sh
index ad0e474cc388..54a0b82cbcd3 100755
--- a/integration-tests/script/build_run_k8s_cluster.sh
+++ b/integration-tests/script/setup_druid_on_k8s.sh
@@ -16,7 +16,7 @@
 
 set -e
 
-export DRUID_OPERATOR_VERSION=0.0.3
+export KUBECTL="sudo /usr/local/bin/kubectl"
 
 # setup client keystore
 cd integration-tests
@@ -34,41 +34,13 @@ mvn -B -ff -q dependency:go-offline \
 
 docker build -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced .
 
-# Set Necessary ENV
-export CHANGE_MINIKUBE_NONE_USER=true
-export MINIKUBE_WANTUPDATENOTIFICATION=false
-export MINIKUBE_WANTREPORTERRORPROMPT=false
-export MINIKUBE_HOME=$HOME
-export KUBECONFIG=$HOME/.kube/config
-sudo apt install -y conntrack
-
 # This tmp dir is used for MiddleManager pod and Historical Pod to cache segments.
 mkdir tmp
 chmod 777 tmp
 
-# Lacunch K8S cluster
-curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.18.1/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
-curl -Lo minikube https://storage.googleapis.com/minikube/releases/v1.8.1/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
-sudo /usr/local/bin/minikube start --profile=minikube --vm-driver=none --kubernetes-version=v1.18.1
-sudo /usr/local/bin/minikube update-context
-
-# Prepare For Druid-Operator
-git clone https://github.com/druid-io/druid-operator.git
-cd druid-operator
-git checkout -b druid-operator-$DRUID_OPERATOR_VERSION druid-operator-$DRUID_OPERATOR_VERSION
-cd ..
-sed -i "s|REPLACE_IMAGE|druidio/druid-operator:$DRUID_OPERATOR_VERSION|g" druid-operator/deploy/operator.yaml
-
-# Deploy Druid Operator and Druid CR spec
-sudo /usr/local/bin/kubectl create -f druid-operator/deploy/service_account.yaml
-sudo /usr/local/bin/kubectl create -f druid-operator/deploy/role.yaml
-sudo /usr/local/bin/kubectl create -f druid-operator/deploy/role_binding.yaml
-sudo /usr/local/bin/kubectl create -f druid-operator/deploy/crds/druid.apache.org_druids_crd.yaml
-sudo /usr/local/bin/kubectl create -f druid-operator/deploy/operator.yaml
-
-sudo /usr/local/bin/kubectl apply -f integration-tests/k8s/role-and-binding.yaml
+$KUBECTL apply -f integration-tests/k8s/role-and-binding.yaml
 sed -i "s|REPLACE_VOLUMES|`pwd`|g" integration-tests/k8s/tiny-cluster.yaml
-sudo /usr/local/bin/kubectl apply -f integration-tests/k8s/tiny-cluster.yaml
+$KUBECTL apply -f integration-tests/k8s/tiny-cluster.yaml
 
 # Wait 4 * 15 seconds to launch pods.
 #count=0
diff --git a/integration-tests/script/setup_druid_operator_on_k8s.sh b/integration-tests/script/setup_druid_operator_on_k8s.sh
new file mode 100755
index 000000000000..392841976634
--- /dev/null
+++ b/integration-tests/script/setup_druid_operator_on_k8s.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+export DRUID_OPERATOR_VERSION=0.0.3
+export KUBECTL="sudo /usr/local/bin/kubectl"
+
+
+# Prepare For Druid-Operator
+git clone https://github.com/druid-io/druid-operator.git
+cd druid-operator
+git checkout -b druid-operator-$DRUID_OPERATOR_VERSION druid-operator-$DRUID_OPERATOR_VERSION
+cd ..
+sed -i "s|REPLACE_IMAGE|druidio/druid-operator:$DRUID_OPERATOR_VERSION|g" druid-operator/deploy/operator.yaml
+
+# Deploy Druid Operator and Druid CR spec
+$KUBECTL create -f druid-operator/deploy/service_account.yaml
+$KUBECTL create -f druid-operator/deploy/role.yaml
+$KUBECTL create -f druid-operator/deploy/role_binding.yaml
+$KUBECTL create -f druid-operator/deploy/crds/druid.apache.org_druids_crd.yaml
+$KUBECTL create -f druid-operator/deploy/operator.yaml
+
+echo "Setup Druid Operator on K8S Done!"
diff --git a/integration-tests/script/setup_k8s_cluster.sh b/integration-tests/script/setup_k8s_cluster.sh
new file mode 100755
index 000000000000..4fd962e84e11
--- /dev/null
+++ b/integration-tests/script/setup_k8s_cluster.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Set Necessary ENV
+export CHANGE_MINIKUBE_NONE_USER=true
+export MINIKUBE_WANTUPDATENOTIFICATION=false
+export MINIKUBE_WANTREPORTERRORPROMPT=false
+export MINIKUBE_HOME=$HOME
+export KUBECONFIG=$HOME/.kube/config
+export KUBECTL="sudo /usr/local/bin/kubectl"
+
+sudo apt install -y conntrack
+
+# Lacunch K8S cluster
+curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.18.1/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
+curl -Lo minikube https://storage.googleapis.com/minikube/releases/v1.8.1/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/
+sudo /usr/local/bin/minikube start --profile=minikube --vm-driver=none --kubernetes-version=v1.18.1
+sudo /usr/local/bin/minikube update-context
+
+echo "Setup K8S Cluster Done!"

From a0d43c722aa2413b82fece8ca641fbb274775ac1 Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Sat, 19 Dec 2020 07:24:40 -0800
Subject: [PATCH 08/12] revert changes to K8sDruidNodeAnnouncer.java

---
 .../org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java
index d43c93ed17b1..29c06f443a3e 100644
--- a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java
+++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java
@@ -120,7 +120,7 @@ public void announce(DiscoveryDruidNode discoveryDruidNode)
       LOGGER.info("Announced DiscoveryDruidNode[%s]", discoveryDruidNode);
     }
     catch (Exception ex) {
-      throw new RE(ex, "Failed to announce DiscoveryDruidNode[%s], Does the Pod have atleast one label and annotation?", discoveryDruidNode);
+      throw new RE(ex, "Failed to announce DiscoveryDruidNode[%s]", discoveryDruidNode);
     }
   }
 

From d1debfff434fa8360575e867dab3524606bb4e2b Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Sat, 19 Dec 2020 07:43:01 -0800
Subject: [PATCH 09/12] k8s extension doc update

---
 .../development/extensions-core/kubernetes.md | 35 +++++++++++++++++--
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/docs/development/extensions-core/kubernetes.md b/docs/development/extensions-core/kubernetes.md
index 513a98d9c655..daab608890c2 100644
--- a/docs/development/extensions-core/kubernetes.md
+++ b/docs/development/extensions-core/kubernetes.md
@@ -39,9 +39,7 @@ This extension works together with HTTP based segment and task management in Dru
 `druid.indexer.runner.type=httpRemote`
 `druid.discovery.type=k8s`
 
-For Node Discovery, Each Druid process running inside a pod "announces" itself by adding few "labels" and "annotations" in the pod spec. So, to add those...
-- Druid process needs to be aware of pod name and namespace which it reads from environment variables `POD_NAME` and `POD_NAMESPACE`. These variable names can be changed, see configuration below. But in the end, each pod needs to have pod name and namespace added as environment variables.
-- Label/Annotation path in the pod spec must exist, which is easily satisfied if there is at least one label/annotation in the pod spec already. This limitation may be removed in future.
+For Node Discovery, Each Druid process running inside a pod "announces" itself by adding few "labels" and "annotations" in the pod spec. Druid process needs to be aware of pod name and namespace which it reads from environment variables `POD_NAME` and `POD_NAMESPACE`. These variable names can be changed, see configuration below. But in the end, each pod needs to have self pod name and namespace added as environment variables.
 
 Additionally, this extension has following configuration.
 
@@ -57,3 +55,34 @@ Additionally, this extension has following configuration.
 |`druid.discovery.k8s.renewDeadline`|`Duration`|Lease renewal period used by Leader.|PT17S|No|
 |`druid.discovery.k8s.retryPeriod`|`Duration`|Retry wait used by Leader Election algorithm on failed operations.|PT5S|No|
 
+### Gotchas
+
+- Label/Annotation path in each pod spec MUST EXIST, which is easily satisfied if there is at least one label/annotation in the pod spec already. This limitation may be removed in future.
+- Druid Pods need permissions to be able to add labels to self-pod, List and Watch other Pods, create ConfigMap for leader election. Assuming, "default" service account is used by Druid pods, you might need to add following or something similar Kubernetes Role and Role Binding.
+
+```
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: druid-cluster
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - configmaps
+  verbs:
+  - '*'
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: druid-cluster
+subjects:
+- kind: ServiceAccount
+  name: default
+roleRef:
+  kind: Role
+  name: druid-cluster
+  apiGroup: rbac.authorization.k8s.io
+```
\ No newline at end of file

From 32277c770f322963553585fefea86ae59583d7c0 Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Sat, 19 Dec 2020 12:08:32 -0800
Subject: [PATCH 10/12] add license to new file

---
 integration-tests/k8s/role-and-binding.yaml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/integration-tests/k8s/role-and-binding.yaml b/integration-tests/k8s/role-and-binding.yaml
index 044e5b630be4..ef15b6b74ee1 100644
--- a/integration-tests/k8s/role-and-binding.yaml
+++ b/integration-tests/k8s/role-and-binding.yaml
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:

From a69f199064ecaeb8a9182183b1f9dbc4355e85c5 Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Mon, 28 Dec 2020 09:03:48 -0800
Subject: [PATCH 11/12] address review comments

---
 integration-tests/script/setup_druid_on_k8s.sh | 8 +++-----
 integration-tests/script/setup_k8s_cluster.sh  | 1 -
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/integration-tests/script/setup_druid_on_k8s.sh b/integration-tests/script/setup_druid_on_k8s.sh
index 54a0b82cbcd3..aab018f238b1 100755
--- a/integration-tests/script/setup_druid_on_k8s.sh
+++ b/integration-tests/script/setup_druid_on_k8s.sh
@@ -42,13 +42,11 @@ $KUBECTL apply -f integration-tests/k8s/role-and-binding.yaml
 sed -i "s|REPLACE_VOLUMES|`pwd`|g" integration-tests/k8s/tiny-cluster.yaml
 $KUBECTL apply -f integration-tests/k8s/tiny-cluster.yaml
 
-# Wait 4 * 15 seconds to launch pods.
-#count=0
-#JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until sudo /usr/local/bin/kubectl -n default get pods -lapp=travis-example -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 4;if [ $count -eq 15 ];then break 2 ;else let "count++";fi;echo $i;echo "waiting for travis-example deployment to be available"; sudo /usr/local/bin/kubectl get pods -n default; done
+# Wait a bit
 sleep 120
 
 ## Debug And FastFail
 
-sudo /usr/local/bin/kubectl get pod
-sudo /usr/local/bin/kubectl get svc
+$KUBECTL get pod
+$KUBECTL get svc
 
diff --git a/integration-tests/script/setup_k8s_cluster.sh b/integration-tests/script/setup_k8s_cluster.sh
index 4fd962e84e11..3313def4bca3 100755
--- a/integration-tests/script/setup_k8s_cluster.sh
+++ b/integration-tests/script/setup_k8s_cluster.sh
@@ -22,7 +22,6 @@ export MINIKUBE_WANTUPDATENOTIFICATION=false
 export MINIKUBE_WANTREPORTERRORPROMPT=false
 export MINIKUBE_HOME=$HOME
 export KUBECONFIG=$HOME/.kube/config
-export KUBECTL="sudo /usr/local/bin/kubectl"
 
 sudo apt install -y conntrack
 

From f6fd694d2e7a2a1ac186eb413ccfdeec86cfbe10 Mon Sep 17 00:00:00 2001
From: Himanshu Gupta <g.himanshu@gmail.com>
Date: Tue, 5 Jan 2021 13:41:16 -0800
Subject: [PATCH 12/12] do not try to load lookups at startup to improve
 cluster startup time

---
 integration-tests/k8s/tiny-cluster.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/integration-tests/k8s/tiny-cluster.yaml b/integration-tests/k8s/tiny-cluster.yaml
index 91d1d23d97b3..98dc6231fac9 100644
--- a/integration-tests/k8s/tiny-cluster.yaml
+++ b/integration-tests/k8s/tiny-cluster.yaml
@@ -101,6 +101,9 @@ spec:
     druid.indexer.logs.type=file
     druid.indexer.logs.directory=/druid/data/task-logs
     druid.indexer.task.baseDir=/druid/data/task-base
+
+    druid.lookup.enableLookupSyncOnStartup=false
+
   env:
     - name: POD_NAME
       valueFrom: