diff --git a/.travis.yml b/.travis.yml index f787f99c34e0..38258665162d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -549,6 +549,21 @@ jobs: env: TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' # END - Integration tests for Compile with Java 8 and Run with Java 11 + - &integration_batch_index_k8s + name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test" + jdk: openjdk8 + services: &integration_test_services_k8s + - docker + env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest' + before_script: integration-tests/script/build_run_k8s_cluster.sh + script: &run_integration_test_k8s + - ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} + after_script: integration-tests/script/stop_k8s_cluster.sh + after_failure: &integration_test_diags_k8s + - for v in broker middlemanager router coordinator historical ; do + echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------"; + sudo /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0; + done - name: "security vulnerabilities" stage: cron install: skip diff --git a/distribution/docker/DockerfileBuildTarAdvanced b/distribution/docker/DockerfileBuildTarAdvanced new file mode 100644 index 000000000000..a58087f4e51a --- /dev/null +++ b/distribution/docker/DockerfileBuildTarAdvanced @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Using For IT on K8s +FROM maven:3-jdk-8-slim as builder + +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get -qq update \ + && apt-get -qq -y install --no-install-recommends python3 python3-yaml + +COPY . /src +WORKDIR /src + +RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate \ + -Dexpression=project.version -DforceStdout=true \ + ) \ + && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \ + && ln -s /opt/apache-druid-${VERSION} /opt/druid + +FROM amd64/busybox:1.30.0-glibc as busybox + +FROM gcr.io/distroless/java:8 +LABEL maintainer="Apache Druid Developers " + +COPY --from=busybox /bin/busybox /busybox/busybox +RUN ["/busybox/busybox", "--install", "/bin"] + +RUN mkdir -p /shared/wikiticker-it + +COPY examples/quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz /shared/wikiticker-it/wikiticker-2015-09-12-sampled.json.gz +COPY integration-tests/docker/wiki-simple-lookup.json /shared/wikiticker-it/wiki-simple-lookup.json + +RUN addgroup -S -g 1000 druid \ + && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid \ + && mkdir -p /opt/druid/var \ + && chown -R druid:druid /opt \ + && chmod 775 /opt/druid/var + +COPY --chown=druid:druid --from=builder /opt /opt +COPY distribution/docker/druid.sh /druid.sh + +USER druid +VOLUME /opt/druid/var +WORKDIR /opt/druid + +ENTRYPOINT ["/druid.sh"] diff --git a/integration-tests/k8s_run_config_file.json b/integration-tests/k8s_run_config_file.json new file mode 100644 index 000000000000..249e44b19ca3 --- /dev/null +++ b/integration-tests/k8s_run_config_file.json @@ -0,0 +1,16 @@ +{ + "broker_host" : "localhost", + "broker_port" : "30400", + "broker_tls_url" : "http://localhost:30100", + "router_host" : "localhost", + "router_port" : "30400", + "router_tls_url" : "http://localhost:30400", + "indexer_host" : "localhost", + "indexer_port" : "30400", + "historical_host" : "localhost", + "historical_port" : "30300", + "coordinator_host" : "localhost", + "coordinator_port" : "30400", + "middlemanager_host": "localhost", + "zookeeper_hosts": "localhost:30600" +} \ No newline at end of file diff --git a/integration-tests/script/build_run_k8s_cluster.sh b/integration-tests/script/build_run_k8s_cluster.sh new file mode 100755 index 000000000000..5c9428072088 --- /dev/null +++ b/integration-tests/script/build_run_k8s_cluster.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +export DRUID_OPERATOR_VERSION=0.0.3 + +# setup client keystore +cd integration-tests +./docker/tls/generate-client-certs-and-keystores.sh +rm -rf docker/client_tls +cp -r client_tls docker/client_tls +cd .. + +# Build Docker images for pods +mvn -B -ff -q dependency:go-offline \ + install \ + -Pdist,bundle-contrib-exts \ + -Pskip-static-checks,skip-tests \ + -Dmaven.javadoc.skip=true + +docker build -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced . + +# Set Necessary ENV +export CHANGE_MINIKUBE_NONE_USER=true +export MINIKUBE_WANTUPDATENOTIFICATION=false +export MINIKUBE_WANTREPORTERRORPROMPT=false +export MINIKUBE_HOME=$HOME +export KUBECONFIG=$HOME/.kube/config +sudo apt install -y conntrack + +# This tmp dir is used for MiddleManager pod and Historical Pod to cache segments. +mkdir tmp +chmod 777 tmp + +# Lacunch K8S cluster +curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.18.1/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ +curl -Lo minikube https://storage.googleapis.com/minikube/releases/v1.8.1/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ +sudo /usr/local/bin/minikube start --profile=minikube --vm-driver=none --kubernetes-version=v1.18.1 +sudo /usr/local/bin/minikube update-context + +# Prepare For Druid-Operator +git clone https://github.com/druid-io/druid-operator.git +cd druid-operator +git checkout -b druid-operator-$DRUID_OPERATOR_VERSION druid-operator-$DRUID_OPERATOR_VERSION +cd .. +sed -i "s|REPLACE_IMAGE|druidio/druid-operator:$DRUID_OPERATOR_VERSION|g" druid-operator/deploy/operator.yaml +cp integration-tests/tiny-cluster.yaml druid-operator/examples/ +cp integration-tests/tiny-cluster-zk.yaml druid-operator/examples/ +sed -i "s|REPLACE_VOLUMES|`pwd`|g" druid-operator/examples/tiny-cluster.yaml + +# Create ZK, Historical, MiddleManager, Overlord-coordiantor, Broker and Router pods using statefulset +sudo /usr/local/bin/kubectl create -f druid-operator/deploy/service_account.yaml +sudo /usr/local/bin/kubectl create -f druid-operator/deploy/role.yaml +sudo /usr/local/bin/kubectl create -f druid-operator/deploy/role_binding.yaml +sudo /usr/local/bin/kubectl create -f druid-operator/deploy/crds/druid.apache.org_druids_crd.yaml +sudo /usr/local/bin/kubectl create -f druid-operator/deploy/operator.yaml +sudo /usr/local/bin/kubectl apply -f druid-operator/examples/tiny-cluster-zk.yaml +sudo /usr/local/bin/kubectl apply -f druid-operator/examples/tiny-cluster.yaml + +# Wait 4 * 15 seconds to launch pods. +#count=0 +#JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until sudo /usr/local/bin/kubectl -n default get pods -lapp=travis-example -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 4;if [ $count -eq 15 ];then break 2 ;else let "count++";fi;echo $i;echo "waiting for travis-example deployment to be available"; sudo /usr/local/bin/kubectl get pods -n default; done +sleep 120 + +## Debug And FastFail + +sudo /usr/local/bin/kubectl get pod +sudo /usr/local/bin/kubectl get svc +docker images +sudo /usr/local/bin/kubectl describe pod druid-tiny-cluster-middlemanagers-0 \ No newline at end of file diff --git a/integration-tests/script/stop_k8s_cluster.sh b/integration-tests/script/stop_k8s_cluster.sh new file mode 100755 index 000000000000..6e8969bd12fc --- /dev/null +++ b/integration-tests/script/stop_k8s_cluster.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +sudo /usr/local/bin/minikube delete +sudo rm -rf `pwd`/tmp diff --git a/integration-tests/tiny-cluster-zk.yaml b/integration-tests/tiny-cluster-zk.yaml new file mode 100644 index 000000000000..d83612e78fa0 --- /dev/null +++ b/integration-tests/tiny-cluster-zk.yaml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: v1 +kind: Service +metadata: + name: tiny-cluster-zk +spec: + clusterIP: None + ports: + - name: zk-client-port + port: 2181 + - name: zk-fwr-port + port: 2888 + - name: zk-elec-port + port: 3888 + selector: + zk_cluster: tiny-cluster-zk +--- +apiVersion: v1 +kind: Service +metadata: + name: tiny-cluster-zk-nodeport +spec: + type: NodePort + ports: + - name: zk-service-port + nodePort: 30600 + port: 2181 + protocol: TCP + targetPort: 2181 + selector: + zk_cluster: tiny-cluster-zk +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + zk_cluster: tiny-cluster-zk + name: tiny-cluster-zk +spec: + replicas: 1 + selector: + matchLabels: + zk_cluster: tiny-cluster-zk + serviceName: tiny-cluster-zk + template: + metadata: + labels: + zk_cluster: tiny-cluster-zk + spec: + containers: + - env: + - name: ZOO_SERVERS + value: server.0=tiny-cluster-zk-0.tiny-cluster-zk:2888:3888 + - name: SERVER_JVMFLAGS + value: -Xms256m -Xmx256m + image: zookeeper:3.4.13 + name: tiny-cluster-zk + command: ["/bin/sh"] + args: ["-c", "ZOO_MY_ID=$(echo `hostname` | cut -d '-' -f2) /docker-entrypoint.sh zkServer.sh start-foreground"] + ports: + - containerPort: 2181 + name: zk-client-port + - containerPort: 2888 + name: zk-fwr-port + - containerPort: 3888 + name: zk-elec-port + resources: + limits: + cpu: 1 + memory: 512Mi + requests: + cpu: 1 + memory: 512Mi + volumeMounts: + - mountPath: /data + name: druid-test-zk-data + - mountPath: /datalog + name: druid-test-zk-data-log + volumes: + - name: druid-test-zk-data + emptyDir: {} + - name: druid-test-zk-data-log + emptyDir: {} diff --git a/integration-tests/tiny-cluster.yaml b/integration-tests/tiny-cluster.yaml new file mode 100644 index 000000000000..d3b89b2a5ba9 --- /dev/null +++ b/integration-tests/tiny-cluster.yaml @@ -0,0 +1,320 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: "druid.apache.org/v1alpha1" +kind: "Druid" +metadata: + name: tiny-cluster +spec: + image: druid/cluster:v1 + # Optionally specify image for all nodes. Can be specify on nodes also + # imagePullSecrets: + # - name: tutu + startScript: /druid.sh + podLabels: + environment: stage + release: alpha + securityContext: + fsGroup: 0 + runAsUser: 0 + runAsGroup: 0 + containerSecurityContext: + privileged: true + services: + - spec: + type: ClusterIP + clusterIP: None + commonConfigMountPath: "/opt/druid/conf/druid/cluster/_common" + jvm.options: |- + -server + -XX:MaxDirectMemorySize=10240g + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Dlog4j.debug + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + log4j.config: |- + + + + + + + + + + + + + + common.runtime.properties: | + + # Zookeeper + druid.zk.service.host=tiny-cluster-zk-0.tiny-cluster-zk + druid.zk.paths.base=/druid + druid.zk.service.compress=false + + # Metadata Store + druid.metadata.storage.type=derby + druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true + druid.metadata.storage.connector.host=localhost + druid.metadata.storage.connector.port=1527 + druid.metadata.storage.connector.createTables=true + + # Deep Storage + druid.storage.type=local + druid.storage.storageDirectory=/druid/data/deepstorage + + # + # Extensions + # + druid.extensions.loadList=["druid-avro-extensions","druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches"] + + # + # Service discovery + # + druid.selectors.indexing.serviceName=druid/overlord + druid.selectors.coordinator.serviceName=druid/coordinator + nodes: + brokers: + # Optionally specify for running broker as Deployment + # kind: Deployment + nodeType: "broker" + # Optionally specify for broker nodes + # imagePullSecrets: + # - name: tutu + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: broker-service-port + nodePort: 30100 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-brokers + metadata: + name: broker-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/broker" + replicas: 1 + runtime.properties: | + druid.service=druid/broker + + # HTTP server threads + druid.broker.http.numConnections=5 + druid.server.http.numThreads=40 + + # Processing threads and buffers + druid.processing.buffer.sizeBytes=25000000 + druid.processing.numThreads=1 + druid.sql.enable=true + extra.jvm.options: |- + -Xmx512m + -Xms512m + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: REPLACE_VOLUMES/tmp + resources: + requests: + memory: "800Mi" + limits: + memory: "800Mi" + + coordinators: + # Optionally specify for running coordinator as Deployment + # kind: Deployment + nodeType: "coordinator" + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: coordinator-service-port + nodePort: 30200 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-coordinators + metadata: + name: coordinator-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/master/coordinator-overlord" + replicas: 1 + runtime.properties: | + druid.service=druid/coordinator + + # HTTP server threads + druid.coordinator.startDelay=PT30S + druid.coordinator.period=PT30S + + # Configure this coordinator to also run as Overlord + druid.coordinator.asOverlord.enabled=true + druid.coordinator.asOverlord.overlordService=druid/overlord + druid.indexer.queue.startDelay=PT30S + druid.indexer.runner.type=local + extra.jvm.options: |- + -Xmx800m + -Xms800m + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: REPLACE_VOLUMES/tmp + resources: + requests: + memory: "1G" + limits: + memory: "1G" + + historicals: + nodeType: "historical" + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: historical-service-port + nodePort: 30300 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-historicals + metadata: + name: historical-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/historical" + replicas: 1 + runtime.properties: | + druid.service=druid/historical + druid.processing.buffer.sizeBytes=25000000 + druid.processing.numThreads=2 + # Segment storage + druid.segmentCache.locations=[{"path":"/druid/data/segments","maxSize":10737418240}] + druid.server.maxSize=10737418240 + extra.jvm.options: |- + -Xmx512m + -Xms512m + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: REPLACE_VOLUMES/tmp + resources: + requests: + memory: "1G" + limits: + memory: "1G" + + routers: + nodeType: "router" + druid.port: 8888 + services: + - spec: + type: NodePort + ports: + - name: router-service-port + nodePort: 30400 + port: 8888 + protocol: TCP + targetPort: 8888 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-routers + metadata: + name: router-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/router" + replicas: 1 + runtime.properties: | + druid.service=druid/router + druid.plaintextPort=8888 + + # HTTP proxy + druid.router.http.numConnections=50 + druid.router.http.readTimeout=PT5M + druid.router.http.numMaxThreads=100 + druid.server.http.numThreads=100 + + # Service discovery + druid.router.defaultBrokerServiceName=druid/broker + druid.router.coordinatorServiceName=druid/coordinator + + # Management proxy to coordinator / overlord: required for unified web console. + druid.router.managementProxy.enabled=true + + middlemanagers: + nodeType: "middleManager" + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/middleManager" + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: middlemanager-service-port + nodePort: 30500 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-middleManagers + metadata: + name: middlemanager-%s-service + - spec: + type: ClusterIP + clusterIP: None + replicas: 1 + runtime.properties: | + druid.service=druid/middleManager + druid.worker.capacity=10 + druid.indexer.runner.javaOpts=-server -XX:MaxDirectMemorySize=10240g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/druid/data/tmp -Dlog4j.debug -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=50 -XX:GCLogFileSize=10m -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:+UseG1GC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Xloggc:/druid/data/logs/peon.gc.%t.%p.log -XX:HeapDumpPath=/druid/data/logs/peon.%t.%p.hprof -Xms256m -Xmx256m + druid.indexer.task.baseTaskDir=/druid/data/baseTaskDir + druid.server.http.numThreads=10 + druid.indexer.fork.property.druid.processing.buffer.sizeBytes=268435456 + druid.indexer.fork.property.druid.processing.numMergeBuffers=1 + druid.indexer.fork.property.druid.processing.numThreads=1 + extra.jvm.options: |- + -Xmx256m + -Xms256m + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: REPLACE_VOLUMES/tmp + resources: + requests: + memory: "3G" + limits: + memory: "3G" \ No newline at end of file