From 9b3a2ad3d7cf73bcc98122a7b959e0975ef03fa5 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 11 Dec 2020 15:49:34 +0800 Subject: [PATCH 01/11] add a travls job to do integrate test on K8s --- .idea/misc.xml | 4 +- .travis.yml | 19 ++ integration-tests/build_run_cluster.sh | 17 +- integration-tests/build_run_k8s_cluster.sh | 24 ++ integration-tests/k8s_run_config_file.json | 12 + integration-tests/pom.xml | 51 +++ .../script/build_docker_image_for_k8s.sh | 18 ++ integration-tests/script/k8s_run_cluster.sh | 58 ++++ integration-tests/stop_k8s_cluster.sh | 26 ++ integration-tests/tiny-cluster.yaml | 300 ++++++++++++++++++ 10 files changed, 526 insertions(+), 3 deletions(-) create mode 100644 integration-tests/build_run_k8s_cluster.sh create mode 100644 integration-tests/k8s_run_config_file.json create mode 100644 integration-tests/script/build_docker_image_for_k8s.sh create mode 100755 integration-tests/script/k8s_run_cluster.sh create mode 100755 integration-tests/stop_k8s_cluster.sh create mode 100644 integration-tests/tiny-cluster.yaml diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index f787f99c34e0..9c329c622265 100644 --- a/.travis.yml +++ b/.travis.yml @@ -549,6 +549,25 @@ jobs: env: TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' # END - Integration tests for Compile with Java 8 and Run with Java 11 + - &integration_batch_index_k8s + name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) batch index integration test running on K8s" + jdk: openjdk8 + services: &integration_test_services_k8s + - docker + env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' K8S_RUN_SKIP=false CONFIG_FILE='k8s_run_config_file.json' + before_script: + - sudo su + script: &run_integration_test_k8s + - ${MVN} verify -P int-tests-config-file ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} -Dk8s.run.skip=${K8S_RUN_SKIP} ${MAVEN_SKIP} + after_failure: &integration_test_diags_k8s + - for v in ~/shared/logs/*.log ; do + echo $v logtail ======================== ; tail -100 $v ; + done + - for v in broker middlemanager router coordinator historical ; do + echo $v dmesg ======================== ; + /usr/local/bin/kubectl exec -it druid-tiny-cluster-"$v"s-0 sh -c 'dmesg | tail -3' ; + done + - name: "security vulnerabilities" stage: cron install: skip diff --git a/integration-tests/build_run_cluster.sh b/integration-tests/build_run_cluster.sh index 97176e5ae865..575d46837776 100755 --- a/integration-tests/build_run_cluster.sh +++ b/integration-tests/build_run_cluster.sh @@ -26,16 +26,31 @@ export SHARED_DIR=${HOME}/shared # so docker IP addr will be known during docker build echo ${DOCKER_IP:=127.0.0.1} > $DOCKERDIR/docker_ip + +if !($DRUID_INTEGRATION_TEST_SKIP_RUN_K8S); then + bash ./stop_k8s_cluster.sh + bash ./script/k8s_run_cluster.sh + bash ./script/build_docker_image_for_k8s.sh + exit 0 +fi + + +# DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER default false if !($DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER); then bash ./script/copy_resources.sh bash ./script/docker_build_containers.sh fi +# DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER default fasle if !($DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER); then bash ./stop_cluster.sh bash ./script/docker_run_cluster.sh +elif !($DRUID_INTEGRATION_TEST_SKIP_RUN_K8S); then + bash ./stop_k8s_cluster.sh + bash ./script/k8s_run_cluster.sh fi +# DRUID_INTEGRATION_TEST_START_HADOOP_DOCKER default false if ($DRUID_INTEGRATION_TEST_START_HADOOP_DOCKER); then bash ./script/copy_hadoop_resources.sh -fi +fi \ No newline at end of file diff --git a/integration-tests/build_run_k8s_cluster.sh b/integration-tests/build_run_k8s_cluster.sh new file mode 100644 index 000000000000..7e65e63c6e79 --- /dev/null +++ b/integration-tests/build_run_k8s_cluster.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if !($DRUID_INTEGRATION_TEST_SKIP_RUN_K8S); then + bash ./stop_k8s_cluster.sh + bash ./script/k8s_run_cluster.sh + bash ./script/build_docker_image_for_k8s.sh + exit 0 +fi \ No newline at end of file diff --git a/integration-tests/k8s_run_config_file.json b/integration-tests/k8s_run_config_file.json new file mode 100644 index 000000000000..fa4d5df53960 --- /dev/null +++ b/integration-tests/k8s_run_config_file.json @@ -0,0 +1,12 @@ +{ + "broker_host": "localhost", + "broker_port": "30100", + "router_host": "localhost", + "router_port": "30400", + "indexer_host": "localhost", + "indexer_port": "8081", + "coordinator_host": "localhost", + "coordinator_port": "30200", + "middlemanager_host": "localhost", + "zookeeper_hosts": "localhost" +} \ No newline at end of file diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 57f78b0029d9..d0f6284a041f 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -369,6 +369,7 @@ false false + true false middleManager @@ -398,6 +399,7 @@ ${resource.file.dir.path} ${docker.build.skip} ${docker.run.skip} + ${k8s.run.skip} ${it.indexer} ${project.basedir}/build_run_cluster.sh @@ -477,8 +479,57 @@ int-tests-config-file + + false + true + false + middleManager + + + + \ %Россия\ 한국\ 中国!? + + + org.codehaus.mojo + exec-maven-plugin + + + k8s-package + + exec + + pre-integration-test + + + ${start.hadoop.docker} + ${jvm.runtime} + ${groups} + ${override.config.path} + ${resource.file.dir.path} + ${docker.build.skip} + ${k8s.run.skip} + ${it.indexer} + + ${project.basedir}/build_run_k8s_cluster.sh + + + + stop-druid-cluster-k8s + + exec + + post-integration-test + + + ${k8s.run.skip} + + ${project.basedir}/stop_k8s_cluster.sh + + + + org.apache.maven.plugins maven-failsafe-plugin diff --git a/integration-tests/script/build_docker_image_for_k8s.sh b/integration-tests/script/build_docker_image_for_k8s.sh new file mode 100644 index 000000000000..a0de62e5d008 --- /dev/null +++ b/integration-tests/script/build_docker_image_for_k8s.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mvn clean install -Pdist -DskipTests -Dgpg.skip -Dcheckstyle.skip +docker build -t druid/cluster -f distribution/docker/Dockerfile . \ No newline at end of file diff --git a/integration-tests/script/k8s_run_cluster.sh b/integration-tests/script/k8s_run_cluster.sh new file mode 100755 index 000000000000..28f71febae0a --- /dev/null +++ b/integration-tests/script/k8s_run_cluster.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +if [ `whoami` != "root" ]; + then + echo "Need root when build K8S cluster !" + exit -1 +fi +# Setup minikube, kubectl and druid-operator + +# Export all the ENV needed +export CHANGE_MINIKUBE_NONE_USER=true +export MINIKUBE_WANTUPDATENOTIFICATION=false +export MINIKUBE_WANTREPORTERRORPROMPT=false +export MINIKUBE_HOME=$HOME +export KUBECONFIG=$HOME/.kube/config + +# Setup minikube and kubectl +apt-get -qq -y install conntrack +curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.18.1/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/ +curl -Lo minikube https://storage.googleapis.com/minikube/releases/v1.8.1/minikube-linux-amd64 && chmod +x minikube && mv minikube /usr/local/bin/ +mkdir -p $HOME/.kube $HOME/.minikube +touch $KUBECONFIG +/usr/local/bin/minikube start --profile=minikube --vm-driver=none --kubernetes-version=v1.18.1 + +# Setup druid-operator +git clone https://github.com/druid-io/druid-operator.git +docker pull druidio/druid-operator:0.0.3 +sed -i 's|REPLACE_IMAGE|druidio/druid-operator:0.0.3|g' druid-operator/deploy/operator.yaml +cp tiny-cluster.yaml druid-operator/examples/ + +# Create Everythings using POD +/usr/local/bin/kubectl create -f deploy/service_account.yaml +/usr/local/bin/kubectl create -f deploy/role.yaml +/usr/local/bin/kubectl create -f deploy/role_binding.yaml +/usr/local/bin/kubectl create -f deploy/crds/druid.apache.org_druids_crd.yaml +/usr/local/bin/kubectl create -f deploy/operator.yaml +/usr/local/bin/kubectl apply -f examples/tiny-cluster-zk.yaml +/usr/local/bin/kubectl apply -f examples/tiny-cluster.yaml + +# Wait 2 * 15 seconds to launch pods. +count=0 +JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n default get pods -lapp=travis-example -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 2;if [ $count -eq 15 ];then break 2 ;else let "count++";fi;echo $i;echo "waiting for travis-example deployment to be available"; kubectl get pods -n default; done + diff --git a/integration-tests/stop_k8s_cluster.sh b/integration-tests/stop_k8s_cluster.sh new file mode 100755 index 000000000000..028d8b8741f6 --- /dev/null +++ b/integration-tests/stop_k8s_cluster.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Skip stopping docker if flag set (For use during development) +if [ -n "$DRUID_INTEGRATION_TEST_SKIP_RUN_K8S" ] && [ "$DRUID_INTEGRATION_TEST_SKIP_RUN_K8S" == true ] + then + exit 0 +fi + +/usr/local/bin/minikube delete +rm -rf /tmp/druid diff --git a/integration-tests/tiny-cluster.yaml b/integration-tests/tiny-cluster.yaml new file mode 100644 index 000000000000..446230eb45e5 --- /dev/null +++ b/integration-tests/tiny-cluster.yaml @@ -0,0 +1,300 @@ +apiVersion: "druid.apache.org/v1alpha1" +kind: "Druid" +metadata: + name: tiny-cluster +spec: + image: druid/cluster + # Optionally specify image for all nodes. Can be specify on nodes also + # imagePullSecrets: + # - name: tutu + startScript: /druid.sh + podLabels: + environment: stage + release: alpha + securityContext: + fsGroup: 0 + runAsUser: 0 + runAsGroup: 0 + containerSecurityContext: + privileged: true + services: + - spec: + type: ClusterIP + clusterIP: None + commonConfigMountPath: "/opt/druid/conf/druid/cluster/_common" + jvm.options: |- + -server + -XX:MaxDirectMemorySize=10240g + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Dlog4j.debug + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + log4j.config: |- + + + + + + + + + + + + + + common.runtime.properties: | + + # Zookeeper + druid.zk.service.host=tiny-cluster-zk-0.tiny-cluster-zk + druid.zk.paths.base=/druid + druid.zk.service.compress=false + + # Metadata Store + druid.metadata.storage.type=derby + druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true + druid.metadata.storage.connector.host=localhost + druid.metadata.storage.connector.port=1527 + druid.metadata.storage.connector.createTables=true + + # Deep Storage + druid.storage.type=local + druid.storage.storageDirectory=/druid/data/deepstorage + + # + # Extensions + # + druid.extensions.loadList=["druid-avro-extensions","druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches"] + + # + # Service discovery + # + druid.selectors.indexing.serviceName=druid/overlord + druid.selectors.coordinator.serviceName=druid/coordinator + nodes: + brokers: + # Optionally specify for running broker as Deployment + # kind: Deployment + nodeType: "broker" + # Optionally specify for broker nodes + # imagePullSecrets: + # - name: tutu + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: broker-service-port + nodePort: 30100 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-brokers + metadata: + name: broker-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/broker" + replicas: 1 + runtime.properties: | + druid.service=druid/broker + + # HTTP server threads + druid.broker.http.numConnections=5 + druid.server.http.numThreads=10 + + # Processing threads and buffers + druid.processing.buffer.sizeBytes=1 + druid.processing.numMergeBuffers=1 + druid.processing.numThreads=1 + druid.sql.enable=true + extra.jvm.options: |- + -Xmx1G + -Xms1G + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: /tmp/druid + resources: + requests: + memory: "2G" + cpu: "2" + limits: + memory: "2G" + cpu: "2" + + coordinators: + # Optionally specify for running coordinator as Deployment + # kind: Deployment + nodeType: "coordinator" + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: coordinator-service-port + nodePort: 30200 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-coordinators + metadata: + name: coordinator-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/master/coordinator-overlord" + replicas: 1 + runtime.properties: | + druid.service=druid/coordinator + + # HTTP server threads + druid.coordinator.startDelay=PT30S + druid.coordinator.period=PT30S + + # Configure this coordinator to also run as Overlord + druid.coordinator.asOverlord.enabled=true + druid.coordinator.asOverlord.overlordService=druid/overlord + druid.indexer.queue.startDelay=PT30S + druid.indexer.runner.type=local + extra.jvm.options: |- + -Xmx1G + -Xms1G + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: /tmp/druid + resources: + requests: + memory: "2G" + cpu: "2" + limits: + memory: "2G" + cpu: "2" + + historicals: + nodeType: "historical" + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: historical-service-port + nodePort: 30300 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-historicals + metadata: + name: historical-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/historical" + replicas: 1 + runtime.properties: | + druid.service=druid/historical + druid.server.http.numThreads=5 + druid.processing.buffer.sizeBytes=1 + druid.processing.numMergeBuffers=1 + druid.processing.numThreads=1 + # Segment storage + druid.segmentCache.locations=[{"path":"/druid/data/segments","maxSize":10737418240}] + druid.server.maxSize=10737418240 + extra.jvm.options: |- + -Xmx1G + -Xms1G + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: /tmp/druid + resources: + requests: + memory: "2G" + cpu: "2" + limits: + memory: "2G" + cpu: "2" + + routers: + nodeType: "router" + druid.port: 8888 + services: + - spec: + type: NodePort + ports: + - name: router-service-port + nodePort: 30400 + port: 8888 + protocol: TCP + targetPort: 8888 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-routers + metadata: + name: router-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/router" + replicas: 1 + runtime.properties: | + druid.service=druid/router + druid.plaintextPort=8888 + + # HTTP proxy + druid.router.http.numConnections=50 + druid.router.http.readTimeout=PT5M + druid.router.http.numMaxThreads=100 + druid.server.http.numThreads=100 + + # Service discovery + druid.router.defaultBrokerServiceName=druid/broker + druid.router.coordinatorServiceName=druid/coordinator + + # Management proxy to coordinator / overlord: required for unified web console. + druid.router.managementProxy.enabled=true + + middlemanagers: + nodeType: "middleManager" + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/middleManager" + druid.port: 8080 + replicas: 1 + runtime.properties: | + druid.service=druid/middleManager + druid.worker.capacity=1 + druid.indexer.runner.javaOpts=-server -XX:MaxDirectMemorySize=10240g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/druid/data/tmp -Dlog4j.debug -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=50 -XX:GCLogFileSize=10m -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:+UseG1GC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Xloggc:/druid/data/logs/peon.gc.%t.%p.log -XX:HeapDumpPath=/druid/data/logs/peon.%t.%p.hprof -Xms1G -Xmx1G + druid.indexer.task.baseTaskDir=/druid/data/baseTaskDir + druid.server.http.numThreads=10 + druid.indexer.fork.property.druid.processing.buffer.sizeBytes=268435456 + druid.indexer.fork.property.druid.processing.numMergeBuffers=1 + druid.indexer.fork.property.druid.processing.numThreads=1 + extra.jvm.options: |- + -Xmx1G + -Xms1G + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: /tmp/druid + resources: + requests: + memory: "2G" + cpu: "2" + limits: + memory: "2G" + cpu: "2" \ No newline at end of file From 3ca937f5d33be1d9828db4ff2cf4866928109d00 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 11 Dec 2020 15:55:09 +0800 Subject: [PATCH 02/11] revert build_run_cluster.sh --- integration-tests/build_run_cluster.sh | 17 +---------------- integration-tests/build_run_k8s_cluster.sh | 1 - .../script/build_docker_image_for_k8s.sh | 2 ++ 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/integration-tests/build_run_cluster.sh b/integration-tests/build_run_cluster.sh index 575d46837776..97176e5ae865 100755 --- a/integration-tests/build_run_cluster.sh +++ b/integration-tests/build_run_cluster.sh @@ -26,31 +26,16 @@ export SHARED_DIR=${HOME}/shared # so docker IP addr will be known during docker build echo ${DOCKER_IP:=127.0.0.1} > $DOCKERDIR/docker_ip - -if !($DRUID_INTEGRATION_TEST_SKIP_RUN_K8S); then - bash ./stop_k8s_cluster.sh - bash ./script/k8s_run_cluster.sh - bash ./script/build_docker_image_for_k8s.sh - exit 0 -fi - - -# DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER default false if !($DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER); then bash ./script/copy_resources.sh bash ./script/docker_build_containers.sh fi -# DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER default fasle if !($DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER); then bash ./stop_cluster.sh bash ./script/docker_run_cluster.sh -elif !($DRUID_INTEGRATION_TEST_SKIP_RUN_K8S); then - bash ./stop_k8s_cluster.sh - bash ./script/k8s_run_cluster.sh fi -# DRUID_INTEGRATION_TEST_START_HADOOP_DOCKER default false if ($DRUID_INTEGRATION_TEST_START_HADOOP_DOCKER); then bash ./script/copy_hadoop_resources.sh -fi \ No newline at end of file +fi diff --git a/integration-tests/build_run_k8s_cluster.sh b/integration-tests/build_run_k8s_cluster.sh index 7e65e63c6e79..702a55f25e04 100644 --- a/integration-tests/build_run_k8s_cluster.sh +++ b/integration-tests/build_run_k8s_cluster.sh @@ -20,5 +20,4 @@ if !($DRUID_INTEGRATION_TEST_SKIP_RUN_K8S); then bash ./stop_k8s_cluster.sh bash ./script/k8s_run_cluster.sh bash ./script/build_docker_image_for_k8s.sh - exit 0 fi \ No newline at end of file diff --git a/integration-tests/script/build_docker_image_for_k8s.sh b/integration-tests/script/build_docker_image_for_k8s.sh index a0de62e5d008..365396f41560 100644 --- a/integration-tests/script/build_docker_image_for_k8s.sh +++ b/integration-tests/script/build_docker_image_for_k8s.sh @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +set -e mvn clean install -Pdist -DskipTests -Dgpg.skip -Dcheckstyle.skip + docker build -t druid/cluster -f distribution/docker/Dockerfile . \ No newline at end of file From 1183b70c5d0f9a2ade0c106f54092ee3ed2a04b1 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 11 Dec 2020 15:56:16 +0800 Subject: [PATCH 03/11] revert msic --- .idea/misc.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + From 29f1d40d72c93bb0e66af2a885df9f426ccfb026 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Sun, 13 Dec 2020 02:01:29 +0800 Subject: [PATCH 04/11] run IT test --- .travis.yml | 13 ++--- integration-tests/build_run_k8s_cluster.sh | 45 ++++++++++++-- integration-tests/pom.xml | 51 ---------------- .../script/build_docker_image_for_k8s.sh | 20 ------- integration-tests/script/k8s_run_cluster.sh | 58 ------------------- integration-tests/stop_k8s_cluster.sh | 11 +--- integration-tests/tiny-cluster.yaml | 31 +++++++--- 7 files changed, 70 insertions(+), 159 deletions(-) delete mode 100644 integration-tests/script/build_docker_image_for_k8s.sh delete mode 100755 integration-tests/script/k8s_run_cluster.sh diff --git a/.travis.yml b/.travis.yml index 9c329c622265..5f25a039b491 100644 --- a/.travis.yml +++ b/.travis.yml @@ -550,19 +550,18 @@ jobs: # END - Integration tests for Compile with Java 8 and Run with Java 11 - &integration_batch_index_k8s - name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) batch index integration test running on K8s" + name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITIndexer integration test running on K8s" jdk: openjdk8 services: &integration_test_services_k8s - docker - env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' K8S_RUN_SKIP=false CONFIG_FILE='k8s_run_config_file.json' + env: CONFIG_FILE='k8s_run_config_file.json' ${IT_TEST}='-Dit.test=ITIndexerTest' before_script: - - sudo su + - ./build_run_k8s_cluster.sh script: &run_integration_test_k8s - - ${MVN} verify -P int-tests-config-file ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} -Dk8s.run.skip=${K8S_RUN_SKIP} ${MAVEN_SKIP} + - ${MVN} verify -P int-tests-config-file ${IT_TEST} + after_script: + - ./stop_k8s_cluster.sh after_failure: &integration_test_diags_k8s - - for v in ~/shared/logs/*.log ; do - echo $v logtail ======================== ; tail -100 $v ; - done - for v in broker middlemanager router coordinator historical ; do echo $v dmesg ======================== ; /usr/local/bin/kubectl exec -it druid-tiny-cluster-"$v"s-0 sh -c 'dmesg | tail -3' ; diff --git a/integration-tests/build_run_k8s_cluster.sh b/integration-tests/build_run_k8s_cluster.sh index 702a55f25e04..99a6273ea797 100644 --- a/integration-tests/build_run_k8s_cluster.sh +++ b/integration-tests/build_run_k8s_cluster.sh @@ -16,8 +16,43 @@ set -e -if !($DRUID_INTEGRATION_TEST_SKIP_RUN_K8S); then - bash ./stop_k8s_cluster.sh - bash ./script/k8s_run_cluster.sh - bash ./script/build_docker_image_for_k8s.sh -fi \ No newline at end of file +# Build Docker images for pods +mvn clean install -Pdist -DskipTests -Dgpg.skip -Dcheckstyle.skip +docker build -t druid/cluster -f distribution/docker/Dockerfile . + +# Set Necessary ENV +export CHANGE_MINIKUBE_NONE_USER=true +export MINIKUBE_WANTUPDATENOTIFICATION=false +export MINIKUBE_WANTREPORTERRORPROMPT=false +export MINIKUBE_HOME=$HOME +export KUBECONFIG=$HOME/.kube/config +sudo apt install -y conntrack + +# This tmp dir is used for MiddleManager pod and Historical Pod to cache segments. +mkdir tmp +chmod 777 tmp + +# Lacunch K8S cluster +curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.18.1/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ +curl -Lo minikube https://storage.googleapis.com/minikube/releases/v1.8.1/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ +sudo /usr/local/bin/minikube start --profile=minikube --vm-driver=none --kubernetes-version=v1.18.1 +sudo /usr/local/bin/minikube update-context + +# Prepare For Druid-Operator +git clone https://github.com/druid-io/druid-operator.git +sed -i 's|REPLACE_IMAGE|druidio/druid-operator:0.0.3|g' druid-operator/deploy/operator.yaml +cp tiny-cluster.yaml druid-operator/druid-operator/examples/ +sed -i "s|REPLACE_VOLUMES|`pwd`|g" druid-operator/examples/tiny-cluster.yaml + +# Create ZK, Historical, MiddleManager, Overlord-coordiantor, Broker and Router pods using statefulset +sudo /usr/local/bin/kubectl create -f druid-operator/deploy/service_account.yaml +sudo /usr/local/bin/kubectl create -f druid-operator/deploy/role.yaml +sudo /usr/local/bin/kubectl create -f druid-operator/deploy/role_binding.yaml +sudo /usr/local/bin/kubectl create -f druid-operator/deploy/crds/druid.apache.org_druids_crd.yaml +sudo /usr/local/bin/kubectl create -f druid-operator/deploy/operator.yaml +sudo /usr/local/bin/kubectl apply -f druid-operator/examples/tiny-cluster-zk.yaml +sudo /usr/local/bin/kubectl apply -f druid-operator/examples/tiny-cluster.yaml + +# Wait 4 * 15 seconds to launch pods. +count=0 +JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n default get pods -lapp=travis-example -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 4;if [ $count -eq 15 ];then break 2 ;else let "count++";fi;echo $i;echo "waiting for travis-example deployment to be available"; kubectl get pods -n default; done diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index d0f6284a041f..57f78b0029d9 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -369,7 +369,6 @@ false false - true false middleManager @@ -399,7 +398,6 @@ ${resource.file.dir.path} ${docker.build.skip} ${docker.run.skip} - ${k8s.run.skip} ${it.indexer} ${project.basedir}/build_run_cluster.sh @@ -479,57 +477,8 @@ int-tests-config-file - - false - true - false - middleManager - - - - \ %Россия\ 한국\ 中国!? - - - org.codehaus.mojo - exec-maven-plugin - - - k8s-package - - exec - - pre-integration-test - - - ${start.hadoop.docker} - ${jvm.runtime} - ${groups} - ${override.config.path} - ${resource.file.dir.path} - ${docker.build.skip} - ${k8s.run.skip} - ${it.indexer} - - ${project.basedir}/build_run_k8s_cluster.sh - - - - stop-druid-cluster-k8s - - exec - - post-integration-test - - - ${k8s.run.skip} - - ${project.basedir}/stop_k8s_cluster.sh - - - - org.apache.maven.plugins maven-failsafe-plugin diff --git a/integration-tests/script/build_docker_image_for_k8s.sh b/integration-tests/script/build_docker_image_for_k8s.sh deleted file mode 100644 index 365396f41560..000000000000 --- a/integration-tests/script/build_docker_image_for_k8s.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -e - -mvn clean install -Pdist -DskipTests -Dgpg.skip -Dcheckstyle.skip - -docker build -t druid/cluster -f distribution/docker/Dockerfile . \ No newline at end of file diff --git a/integration-tests/script/k8s_run_cluster.sh b/integration-tests/script/k8s_run_cluster.sh deleted file mode 100755 index 28f71febae0a..000000000000 --- a/integration-tests/script/k8s_run_cluster.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -if [ `whoami` != "root" ]; - then - echo "Need root when build K8S cluster !" - exit -1 -fi -# Setup minikube, kubectl and druid-operator - -# Export all the ENV needed -export CHANGE_MINIKUBE_NONE_USER=true -export MINIKUBE_WANTUPDATENOTIFICATION=false -export MINIKUBE_WANTREPORTERRORPROMPT=false -export MINIKUBE_HOME=$HOME -export KUBECONFIG=$HOME/.kube/config - -# Setup minikube and kubectl -apt-get -qq -y install conntrack -curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.18.1/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/ -curl -Lo minikube https://storage.googleapis.com/minikube/releases/v1.8.1/minikube-linux-amd64 && chmod +x minikube && mv minikube /usr/local/bin/ -mkdir -p $HOME/.kube $HOME/.minikube -touch $KUBECONFIG -/usr/local/bin/minikube start --profile=minikube --vm-driver=none --kubernetes-version=v1.18.1 - -# Setup druid-operator -git clone https://github.com/druid-io/druid-operator.git -docker pull druidio/druid-operator:0.0.3 -sed -i 's|REPLACE_IMAGE|druidio/druid-operator:0.0.3|g' druid-operator/deploy/operator.yaml -cp tiny-cluster.yaml druid-operator/examples/ - -# Create Everythings using POD -/usr/local/bin/kubectl create -f deploy/service_account.yaml -/usr/local/bin/kubectl create -f deploy/role.yaml -/usr/local/bin/kubectl create -f deploy/role_binding.yaml -/usr/local/bin/kubectl create -f deploy/crds/druid.apache.org_druids_crd.yaml -/usr/local/bin/kubectl create -f deploy/operator.yaml -/usr/local/bin/kubectl apply -f examples/tiny-cluster-zk.yaml -/usr/local/bin/kubectl apply -f examples/tiny-cluster.yaml - -# Wait 2 * 15 seconds to launch pods. -count=0 -JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n default get pods -lapp=travis-example -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 2;if [ $count -eq 15 ];then break 2 ;else let "count++";fi;echo $i;echo "waiting for travis-example deployment to be available"; kubectl get pods -n default; done - diff --git a/integration-tests/stop_k8s_cluster.sh b/integration-tests/stop_k8s_cluster.sh index 028d8b8741f6..6e8969bd12fc 100755 --- a/integration-tests/stop_k8s_cluster.sh +++ b/integration-tests/stop_k8s_cluster.sh @@ -15,12 +15,5 @@ # limitations under the License. set -e - -# Skip stopping docker if flag set (For use during development) -if [ -n "$DRUID_INTEGRATION_TEST_SKIP_RUN_K8S" ] && [ "$DRUID_INTEGRATION_TEST_SKIP_RUN_K8S" == true ] - then - exit 0 -fi - -/usr/local/bin/minikube delete -rm -rf /tmp/druid +sudo /usr/local/bin/minikube delete +sudo rm -rf `pwd`/tmp diff --git a/integration-tests/tiny-cluster.yaml b/integration-tests/tiny-cluster.yaml index 446230eb45e5..6ac781dbf891 100644 --- a/integration-tests/tiny-cluster.yaml +++ b/integration-tests/tiny-cluster.yaml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + apiVersion: "druid.apache.org/v1alpha1" kind: "Druid" metadata: @@ -12,11 +27,9 @@ spec: environment: stage release: alpha securityContext: - fsGroup: 0 - runAsUser: 0 - runAsGroup: 0 - containerSecurityContext: - privileged: true + fsGroup: 1000 + runAsUser: 1000 + runAsGroup: 1000 services: - spec: type: ClusterIP @@ -119,7 +132,7 @@ spec: volumes: - name: data-volume hostPath: - path: /tmp/druid + path: REPLACE_VOLUMES/tmp resources: requests: memory: "2G" @@ -172,7 +185,7 @@ spec: volumes: - name: data-volume hostPath: - path: /tmp/druid + path: REPLACE_VOLUMES/tmp resources: requests: memory: "2G" @@ -220,7 +233,7 @@ spec: volumes: - name: data-volume hostPath: - path: /tmp/druid + path: REPLACE_VOLUMES/tmp resources: requests: memory: "2G" @@ -290,7 +303,7 @@ spec: volumes: - name: data-volume hostPath: - path: /tmp/druid + path: REPLACE_VOLUMES/tmp resources: requests: memory: "2G" From 2aef19ddc5971ead8006d77ac57eb7af36a544e3 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Sun, 13 Dec 2020 02:44:25 +0800 Subject: [PATCH 05/11] ready to test --- .travis.yml | 7 +++---- integration-tests/build_run_k8s_cluster.sh | 5 +++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f25a039b491..00973b944db9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -550,15 +550,15 @@ jobs: # END - Integration tests for Compile with Java 8 and Run with Java 11 - &integration_batch_index_k8s - name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITIndexer integration test running on K8s" + name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) batch-index integration test" jdk: openjdk8 services: &integration_test_services_k8s - docker - env: CONFIG_FILE='k8s_run_config_file.json' ${IT_TEST}='-Dit.test=ITIndexerTest' + env: CONFIG_FILE='k8s_run_config_file.json' TESTNG_GROUPS='-Dgroups=batch-index' before_script: - ./build_run_k8s_cluster.sh script: &run_integration_test_k8s - - ${MVN} verify -P int-tests-config-file ${IT_TEST} + - ${MVN} verify -P int-tests-config-file ${TESTNG_GROUPS} ${MAVEN_SKIP} after_script: - ./stop_k8s_cluster.sh after_failure: &integration_test_diags_k8s @@ -566,7 +566,6 @@ jobs: echo $v dmesg ======================== ; /usr/local/bin/kubectl exec -it druid-tiny-cluster-"$v"s-0 sh -c 'dmesg | tail -3' ; done - - name: "security vulnerabilities" stage: cron install: skip diff --git a/integration-tests/build_run_k8s_cluster.sh b/integration-tests/build_run_k8s_cluster.sh index 99a6273ea797..8f078225198d 100644 --- a/integration-tests/build_run_k8s_cluster.sh +++ b/integration-tests/build_run_k8s_cluster.sh @@ -54,5 +54,6 @@ sudo /usr/local/bin/kubectl apply -f druid-operator/examples/tiny-cluster-zk.yam sudo /usr/local/bin/kubectl apply -f druid-operator/examples/tiny-cluster.yaml # Wait 4 * 15 seconds to launch pods. -count=0 -JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n default get pods -lapp=travis-example -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 4;if [ $count -eq 15 ];then break 2 ;else let "count++";fi;echo $i;echo "waiting for travis-example deployment to be available"; kubectl get pods -n default; done +#count=0 +#JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until sudo /usr/local/bin/kubectl -n default get pods -lapp=travis-example -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 4;if [ $count -eq 15 ];then break 2 ;else let "count++";fi;echo $i;echo "waiting for travis-example deployment to be available"; sudo /usr/local/bin/kubectl get pods -n default; done +sleep 60 \ No newline at end of file From 34901b27ce6b25c4c0b2f5509f7c4382e68af461 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Sun, 13 Dec 2020 10:33:03 +0800 Subject: [PATCH 06/11] modify before/after script --- .idea/misc.xml | 4 ++-- .travis.yml | 6 ++---- integration-tests/{ => script}/build_run_k8s_cluster.sh | 0 integration-tests/{ => script}/stop_k8s_cluster.sh | 0 4 files changed, 4 insertions(+), 6 deletions(-) rename integration-tests/{ => script}/build_run_k8s_cluster.sh (100%) rename integration-tests/{ => script}/stop_k8s_cluster.sh (100%) diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 00973b944db9..386d44f63101 100644 --- a/.travis.yml +++ b/.travis.yml @@ -555,12 +555,10 @@ jobs: services: &integration_test_services_k8s - docker env: CONFIG_FILE='k8s_run_config_file.json' TESTNG_GROUPS='-Dgroups=batch-index' - before_script: - - ./build_run_k8s_cluster.sh + before_script: integration-tests/script/build_run_k8s_cluster.sh script: &run_integration_test_k8s - ${MVN} verify -P int-tests-config-file ${TESTNG_GROUPS} ${MAVEN_SKIP} - after_script: - - ./stop_k8s_cluster.sh + after_script: integration-tests/script/stop_k8s_cluster.sh after_failure: &integration_test_diags_k8s - for v in broker middlemanager router coordinator historical ; do echo $v dmesg ======================== ; diff --git a/integration-tests/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh similarity index 100% rename from integration-tests/build_run_k8s_cluster.sh rename to integration-tests/script/build_run_k8s_cluster.sh diff --git a/integration-tests/stop_k8s_cluster.sh b/integration-tests/script/stop_k8s_cluster.sh similarity index 100% rename from integration-tests/stop_k8s_cluster.sh rename to integration-tests/script/stop_k8s_cluster.sh From 3a57cf2331080341302c716f7d31fd51ba0be5de Mon Sep 17 00:00:00 2001 From: yuezhang Date: Mon, 14 Dec 2020 21:34:18 +0800 Subject: [PATCH 07/11] done --- .idea/misc.xml | 4 +- .travis.yml | 10 +- .../docker/DockerfileBuildTarAdvanced | 62 ++++++++++++ integration-tests/k8s_run_config_file.json | 22 +++-- .../script/build_run_k8s_cluster.sh | 32 +++++- integration-tests/tiny-cluster-zk.yaml | 97 +++++++++++++++++++ integration-tests/tiny-cluster.yaml | 83 ++++++++-------- 7 files changed, 252 insertions(+), 58 deletions(-) create mode 100644 distribution/docker/DockerfileBuildTarAdvanced create mode 100644 integration-tests/tiny-cluster-zk.yaml diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + diff --git a/.travis.yml b/.travis.yml index 386d44f63101..38258665162d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -550,19 +550,19 @@ jobs: # END - Integration tests for Compile with Java 8 and Run with Java 11 - &integration_batch_index_k8s - name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) batch-index integration test" + name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test" jdk: openjdk8 services: &integration_test_services_k8s - docker - env: CONFIG_FILE='k8s_run_config_file.json' TESTNG_GROUPS='-Dgroups=batch-index' + env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest' before_script: integration-tests/script/build_run_k8s_cluster.sh script: &run_integration_test_k8s - - ${MVN} verify -P int-tests-config-file ${TESTNG_GROUPS} ${MAVEN_SKIP} + - ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} after_script: integration-tests/script/stop_k8s_cluster.sh after_failure: &integration_test_diags_k8s - for v in broker middlemanager router coordinator historical ; do - echo $v dmesg ======================== ; - /usr/local/bin/kubectl exec -it druid-tiny-cluster-"$v"s-0 sh -c 'dmesg | tail -3' ; + echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------"; + sudo /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0; done - name: "security vulnerabilities" stage: cron diff --git a/distribution/docker/DockerfileBuildTarAdvanced b/distribution/docker/DockerfileBuildTarAdvanced new file mode 100644 index 000000000000..a58087f4e51a --- /dev/null +++ b/distribution/docker/DockerfileBuildTarAdvanced @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Using For IT on K8s +FROM maven:3-jdk-8-slim as builder + +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get -qq update \ + && apt-get -qq -y install --no-install-recommends python3 python3-yaml + +COPY . /src +WORKDIR /src + +RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate \ + -Dexpression=project.version -DforceStdout=true \ + ) \ + && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \ + && ln -s /opt/apache-druid-${VERSION} /opt/druid + +FROM amd64/busybox:1.30.0-glibc as busybox + +FROM gcr.io/distroless/java:8 +LABEL maintainer="Apache Druid Developers " + +COPY --from=busybox /bin/busybox /busybox/busybox +RUN ["/busybox/busybox", "--install", "/bin"] + +RUN mkdir -p /shared/wikiticker-it + +COPY examples/quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz /shared/wikiticker-it/wikiticker-2015-09-12-sampled.json.gz +COPY integration-tests/docker/wiki-simple-lookup.json /shared/wikiticker-it/wiki-simple-lookup.json + +RUN addgroup -S -g 1000 druid \ + && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid \ + && mkdir -p /opt/druid/var \ + && chown -R druid:druid /opt \ + && chmod 775 /opt/druid/var + +COPY --chown=druid:druid --from=builder /opt /opt +COPY distribution/docker/druid.sh /druid.sh + +USER druid +VOLUME /opt/druid/var +WORKDIR /opt/druid + +ENTRYPOINT ["/druid.sh"] diff --git a/integration-tests/k8s_run_config_file.json b/integration-tests/k8s_run_config_file.json index fa4d5df53960..249e44b19ca3 100644 --- a/integration-tests/k8s_run_config_file.json +++ b/integration-tests/k8s_run_config_file.json @@ -1,12 +1,16 @@ { - "broker_host": "localhost", - "broker_port": "30100", - "router_host": "localhost", - "router_port": "30400", - "indexer_host": "localhost", - "indexer_port": "8081", - "coordinator_host": "localhost", - "coordinator_port": "30200", + "broker_host" : "localhost", + "broker_port" : "30400", + "broker_tls_url" : "http://localhost:30100", + "router_host" : "localhost", + "router_port" : "30400", + "router_tls_url" : "http://localhost:30400", + "indexer_host" : "localhost", + "indexer_port" : "30400", + "historical_host" : "localhost", + "historical_port" : "30300", + "coordinator_host" : "localhost", + "coordinator_port" : "30400", "middlemanager_host": "localhost", - "zookeeper_hosts": "localhost" + "zookeeper_hosts": "localhost:30600" } \ No newline at end of file diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh index 8f078225198d..4bf092a7947e 100644 --- a/integration-tests/script/build_run_k8s_cluster.sh +++ b/integration-tests/script/build_run_k8s_cluster.sh @@ -16,9 +16,22 @@ set -e +# setup client keystore +export SHARED_DIR=${HOME}/shared +cd integration-tests +./docker/tls/generate-client-certs-and-keystores.sh +rm -rf docker/client_tls +cp -r client_tls docker/client_tls +cd .. + # Build Docker images for pods -mvn clean install -Pdist -DskipTests -Dgpg.skip -Dcheckstyle.skip -docker build -t druid/cluster -f distribution/docker/Dockerfile . +mvn -B -ff -q dependency:go-offline \ + install \ + -Pdist,bundle-contrib-exts \ + -Pskip-static-checks,skip-tests \ + -Dmaven.javadoc.skip=true + +docker build -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced . # Set Necessary ENV export CHANGE_MINIKUBE_NONE_USER=true @@ -40,8 +53,12 @@ sudo /usr/local/bin/minikube update-context # Prepare For Druid-Operator git clone https://github.com/druid-io/druid-operator.git +cd druid-operator +git checkout -b druid-operator-0.0.3 druid-operator-0.0.3 +cd .. sed -i 's|REPLACE_IMAGE|druidio/druid-operator:0.0.3|g' druid-operator/deploy/operator.yaml -cp tiny-cluster.yaml druid-operator/druid-operator/examples/ +cp integration-tests/tiny-cluster.yaml druid-operator/examples/ +cp integration-tests/tiny-cluster-zk.yaml druid-operator/examples/ sed -i "s|REPLACE_VOLUMES|`pwd`|g" druid-operator/examples/tiny-cluster.yaml # Create ZK, Historical, MiddleManager, Overlord-coordiantor, Broker and Router pods using statefulset @@ -56,4 +73,11 @@ sudo /usr/local/bin/kubectl apply -f druid-operator/examples/tiny-cluster.yaml # Wait 4 * 15 seconds to launch pods. #count=0 #JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until sudo /usr/local/bin/kubectl -n default get pods -lapp=travis-example -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 4;if [ $count -eq 15 ];then break 2 ;else let "count++";fi;echo $i;echo "waiting for travis-example deployment to be available"; sudo /usr/local/bin/kubectl get pods -n default; done -sleep 60 \ No newline at end of file +sleep 120 + +## Debug And FastFail + +sudo /usr/local/bin/kubectl get pod +sudo /usr/local/bin/kubectl get svc +docker images +sudo /usr/local/bin/kubectl describe pod druid-tiny-cluster-middlemanagers-0 \ No newline at end of file diff --git a/integration-tests/tiny-cluster-zk.yaml b/integration-tests/tiny-cluster-zk.yaml new file mode 100644 index 000000000000..d83612e78fa0 --- /dev/null +++ b/integration-tests/tiny-cluster-zk.yaml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: v1 +kind: Service +metadata: + name: tiny-cluster-zk +spec: + clusterIP: None + ports: + - name: zk-client-port + port: 2181 + - name: zk-fwr-port + port: 2888 + - name: zk-elec-port + port: 3888 + selector: + zk_cluster: tiny-cluster-zk +--- +apiVersion: v1 +kind: Service +metadata: + name: tiny-cluster-zk-nodeport +spec: + type: NodePort + ports: + - name: zk-service-port + nodePort: 30600 + port: 2181 + protocol: TCP + targetPort: 2181 + selector: + zk_cluster: tiny-cluster-zk +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + zk_cluster: tiny-cluster-zk + name: tiny-cluster-zk +spec: + replicas: 1 + selector: + matchLabels: + zk_cluster: tiny-cluster-zk + serviceName: tiny-cluster-zk + template: + metadata: + labels: + zk_cluster: tiny-cluster-zk + spec: + containers: + - env: + - name: ZOO_SERVERS + value: server.0=tiny-cluster-zk-0.tiny-cluster-zk:2888:3888 + - name: SERVER_JVMFLAGS + value: -Xms256m -Xmx256m + image: zookeeper:3.4.13 + name: tiny-cluster-zk + command: ["/bin/sh"] + args: ["-c", "ZOO_MY_ID=$(echo `hostname` | cut -d '-' -f2) /docker-entrypoint.sh zkServer.sh start-foreground"] + ports: + - containerPort: 2181 + name: zk-client-port + - containerPort: 2888 + name: zk-fwr-port + - containerPort: 3888 + name: zk-elec-port + resources: + limits: + cpu: 1 + memory: 512Mi + requests: + cpu: 1 + memory: 512Mi + volumeMounts: + - mountPath: /data + name: druid-test-zk-data + - mountPath: /datalog + name: druid-test-zk-data-log + volumes: + - name: druid-test-zk-data + emptyDir: {} + - name: druid-test-zk-data-log + emptyDir: {} diff --git a/integration-tests/tiny-cluster.yaml b/integration-tests/tiny-cluster.yaml index 6ac781dbf891..d3b89b2a5ba9 100644 --- a/integration-tests/tiny-cluster.yaml +++ b/integration-tests/tiny-cluster.yaml @@ -18,7 +18,7 @@ kind: "Druid" metadata: name: tiny-cluster spec: - image: druid/cluster + image: druid/cluster:v1 # Optionally specify image for all nodes. Can be specify on nodes also # imagePullSecrets: # - name: tutu @@ -27,9 +27,11 @@ spec: environment: stage release: alpha securityContext: - fsGroup: 1000 - runAsUser: 1000 - runAsGroup: 1000 + fsGroup: 0 + runAsUser: 0 + runAsGroup: 0 + containerSecurityContext: + privileged: true services: - spec: type: ClusterIP @@ -116,16 +118,15 @@ spec: # HTTP server threads druid.broker.http.numConnections=5 - druid.server.http.numThreads=10 + druid.server.http.numThreads=40 # Processing threads and buffers - druid.processing.buffer.sizeBytes=1 - druid.processing.numMergeBuffers=1 + druid.processing.buffer.sizeBytes=25000000 druid.processing.numThreads=1 druid.sql.enable=true extra.jvm.options: |- - -Xmx1G - -Xms1G + -Xmx512m + -Xms512m volumeMounts: - mountPath: /druid/data name: data-volume @@ -135,11 +136,9 @@ spec: path: REPLACE_VOLUMES/tmp resources: requests: - memory: "2G" - cpu: "2" + memory: "800Mi" limits: - memory: "2G" - cpu: "2" + memory: "800Mi" coordinators: # Optionally specify for running coordinator as Deployment @@ -177,8 +176,8 @@ spec: druid.indexer.queue.startDelay=PT30S druid.indexer.runner.type=local extra.jvm.options: |- - -Xmx1G - -Xms1G + -Xmx800m + -Xms800m volumeMounts: - mountPath: /druid/data name: data-volume @@ -188,11 +187,9 @@ spec: path: REPLACE_VOLUMES/tmp resources: requests: - memory: "2G" - cpu: "2" + memory: "1G" limits: - memory: "2G" - cpu: "2" + memory: "1G" historicals: nodeType: "historical" @@ -217,16 +214,14 @@ spec: replicas: 1 runtime.properties: | druid.service=druid/historical - druid.server.http.numThreads=5 - druid.processing.buffer.sizeBytes=1 - druid.processing.numMergeBuffers=1 - druid.processing.numThreads=1 + druid.processing.buffer.sizeBytes=25000000 + druid.processing.numThreads=2 # Segment storage druid.segmentCache.locations=[{"path":"/druid/data/segments","maxSize":10737418240}] druid.server.maxSize=10737418240 extra.jvm.options: |- - -Xmx1G - -Xms1G + -Xmx512m + -Xms512m volumeMounts: - mountPath: /druid/data name: data-volume @@ -236,11 +231,9 @@ spec: path: REPLACE_VOLUMES/tmp resources: requests: - memory: "2G" - cpu: "2" + memory: "1G" limits: - memory: "2G" - cpu: "2" + memory: "1G" routers: nodeType: "router" @@ -283,20 +276,36 @@ spec: middlemanagers: nodeType: "middleManager" nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/middleManager" - druid.port: 8080 + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: middlemanager-service-port + nodePort: 30500 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-middleManagers + metadata: + name: middlemanager-%s-service + - spec: + type: ClusterIP + clusterIP: None replicas: 1 runtime.properties: | druid.service=druid/middleManager - druid.worker.capacity=1 - druid.indexer.runner.javaOpts=-server -XX:MaxDirectMemorySize=10240g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/druid/data/tmp -Dlog4j.debug -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=50 -XX:GCLogFileSize=10m -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:+UseG1GC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Xloggc:/druid/data/logs/peon.gc.%t.%p.log -XX:HeapDumpPath=/druid/data/logs/peon.%t.%p.hprof -Xms1G -Xmx1G + druid.worker.capacity=10 + druid.indexer.runner.javaOpts=-server -XX:MaxDirectMemorySize=10240g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/druid/data/tmp -Dlog4j.debug -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=50 -XX:GCLogFileSize=10m -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:+UseG1GC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Xloggc:/druid/data/logs/peon.gc.%t.%p.log -XX:HeapDumpPath=/druid/data/logs/peon.%t.%p.hprof -Xms256m -Xmx256m druid.indexer.task.baseTaskDir=/druid/data/baseTaskDir druid.server.http.numThreads=10 druid.indexer.fork.property.druid.processing.buffer.sizeBytes=268435456 druid.indexer.fork.property.druid.processing.numMergeBuffers=1 druid.indexer.fork.property.druid.processing.numThreads=1 extra.jvm.options: |- - -Xmx1G - -Xms1G + -Xmx256m + -Xms256m volumeMounts: - mountPath: /druid/data name: data-volume @@ -306,8 +315,6 @@ spec: path: REPLACE_VOLUMES/tmp resources: requests: - memory: "2G" - cpu: "2" + memory: "3G" limits: - memory: "2G" - cpu: "2" \ No newline at end of file + memory: "3G" \ No newline at end of file From 389d8905494bf38a9f076102f470eb884bd05931 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 15 Dec 2020 07:55:46 +0800 Subject: [PATCH 08/11] change mod for script --- integration-tests/script/build_run_k8s_cluster.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 integration-tests/script/build_run_k8s_cluster.sh diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh old mode 100644 new mode 100755 From b7443be1f4dfc3ae9335bc87e4b420c0d059e400 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 15 Dec 2020 21:45:51 +0800 Subject: [PATCH 09/11] done --- integration-tests/script/build_run_k8s_cluster.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh index 4bf092a7947e..50033370d37a 100755 --- a/integration-tests/script/build_run_k8s_cluster.sh +++ b/integration-tests/script/build_run_k8s_cluster.sh @@ -17,7 +17,6 @@ set -e # setup client keystore -export SHARED_DIR=${HOME}/shared cd integration-tests ./docker/tls/generate-client-certs-and-keystores.sh rm -rf docker/client_tls From 375c253671c8e9dfacc69d84153e6a3af7422b8c Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 16 Dec 2020 10:52:51 +0800 Subject: [PATCH 10/11] add env DRUID_OPERATOR_VERSION=0.0.3 --- integration-tests/script/build_run_k8s_cluster.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh index 50033370d37a..fe4a3525ea95 100755 --- a/integration-tests/script/build_run_k8s_cluster.sh +++ b/integration-tests/script/build_run_k8s_cluster.sh @@ -16,6 +16,8 @@ set -e +export DRUID_OPERATOR_VERSION=0.0.3 + # setup client keystore cd integration-tests ./docker/tls/generate-client-certs-and-keystores.sh @@ -55,7 +57,7 @@ git clone https://github.com/druid-io/druid-operator.git cd druid-operator git checkout -b druid-operator-0.0.3 druid-operator-0.0.3 cd .. -sed -i 's|REPLACE_IMAGE|druidio/druid-operator:0.0.3|g' druid-operator/deploy/operator.yaml +sed -i "s|REPLACE_IMAGE|druidio/druid-operator:$DRUID_OPERATOR_VERSION|g" druid-operator/deploy/operator.yaml cp integration-tests/tiny-cluster.yaml druid-operator/examples/ cp integration-tests/tiny-cluster-zk.yaml druid-operator/examples/ sed -i "s|REPLACE_VOLUMES|`pwd`|g" druid-operator/examples/tiny-cluster.yaml From 1a42794714ad1bce3d80ca612e015c66781235d9 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 16 Dec 2020 14:43:51 +0800 Subject: [PATCH 11/11] change version --- integration-tests/script/build_run_k8s_cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh index fe4a3525ea95..5c9428072088 100755 --- a/integration-tests/script/build_run_k8s_cluster.sh +++ b/integration-tests/script/build_run_k8s_cluster.sh @@ -55,7 +55,7 @@ sudo /usr/local/bin/minikube update-context # Prepare For Druid-Operator git clone https://github.com/druid-io/druid-operator.git cd druid-operator -git checkout -b druid-operator-0.0.3 druid-operator-0.0.3 +git checkout -b druid-operator-$DRUID_OPERATOR_VERSION druid-operator-$DRUID_OPERATOR_VERSION cd .. sed -i "s|REPLACE_IMAGE|druidio/druid-operator:$DRUID_OPERATOR_VERSION|g" druid-operator/deploy/operator.yaml cp integration-tests/tiny-cluster.yaml druid-operator/examples/