diff --git a/.travis.yml b/.travis.yml
index a25cc34bf2af..fe9528e14e3d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,7 +26,6 @@ cache:
env:
global:
- - ZK_VERSION=3.5 # default ZK version for integration tests
- DOCKER_IP=127.0.0.1 # for integration tests
- MVN="mvn -B"
- > # Various options to make execution of maven goals faster (e.g., mvn install)
@@ -153,6 +152,11 @@ jobs:
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *package
+ name: "(openjdk15) packaging check"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_processing_module
name: "(openjdk8) processing module test"
stage: Tests - phase 1
@@ -219,6 +223,11 @@ jobs:
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_processing_module
+ name: "(openjdk15) processing module test"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_processing_module_sqlcompat
<<: *test_processing_module
name: "(openjdk8) processing module test (SQL Compatibility)"
@@ -231,6 +240,11 @@ jobs:
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_processing_module_sqlcompat
+ name: "(openjdk15) processing module test (SQL Compatibility)"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_indexing_module
<<: *test_processing_module
name: "(openjdk8) indexing modules test"
@@ -242,6 +256,11 @@ jobs:
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_indexing_module
+ name: "(openjdk15) indexing modules test"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_indexing_module_sqlcompat
<<: *test_indexing_module
name: "(openjdk8) indexing modules test (SQL Compatibility)"
@@ -253,6 +272,11 @@ jobs:
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_indexing_module_sqlcompat
+ name: "(openjdk15) indexing modules test (SQL Compatibility)"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_server_module
<<: *test_processing_module
name: "(openjdk8) server module test"
@@ -264,6 +288,11 @@ jobs:
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_server_module
+ name: "(openjdk15) server module test"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_server_module_sqlcompat
<<: *test_server_module
name: "(openjdk8) server module test (SQL Compatibility)"
@@ -274,6 +303,11 @@ jobs:
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_server_module_sqlcompat
+ name: "(openjdk15) server module test (SQL Compatibility)"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_other_modules
<<: *test_processing_module
name: "(openjdk8) other modules test"
@@ -285,6 +319,11 @@ jobs:
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_other_modules
+ name: "(openjdk15) other modules test"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- &test_other_modules_sqlcompat
<<: *test_other_modules
name: "(openjdk8) other modules test (SQL Compatibility)"
@@ -295,6 +334,11 @@ jobs:
stage: Tests - phase 2
jdk: openjdk11
+ - <<: *test_other_modules_sqlcompat
+ name: "(openjdk15) other modules test (SQL Compatibility)"
+ stage: Tests - phase 2
+ jdk: openjdk15
+
- name: "web console"
install: skip
stage: Tests - phase 1
@@ -342,7 +386,7 @@ jobs:
- docker
env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
script: &run_integration_test
- - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} -Dzk.version=${ZK_VERSION} ${MAVEN_SKIP}
+ - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} ${MAVEN_SKIP}
after_failure: &integration_test_diags
- for v in ~/shared/logs/*.log ; do
echo $v logtail ======================== ; tail -100 $v ;
@@ -554,16 +598,6 @@ jobs:
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
- # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
- - <<: *integration_tests
- name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests"
- jdk: openjdk8
- env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
- - <<: *integration_kafka_format_tests
- name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats"
- env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
# END - Integration tests for Compile with Java 8 and Run with Java 8
# START - Integration tests for Compile with Java 8 and Run with Java 11
@@ -637,12 +671,6 @@ jobs:
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
- # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
- - <<: *integration_tests
- name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests"
- jdk: openjdk8
- env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
# END - Integration tests for Compile with Java 8 and Run with Java 11
- &integration_batch_index_k8s
diff --git a/docs/dependencies/zookeeper.md b/docs/dependencies/zookeeper.md
index 0855a376fef8..ec6814e24626 100644
--- a/docs/dependencies/zookeeper.md
+++ b/docs/dependencies/zookeeper.md
@@ -23,7 +23,17 @@ title: "ZooKeeper"
-->
-Apache Druid uses [Apache ZooKeeper](http://zookeeper.apache.org/) (ZK) for management of current cluster state. The operations that happen over ZK are
+Apache Druid uses [Apache ZooKeeper](http://zookeeper.apache.org/) (ZK) for management of current cluster state.
+
+## Minimum ZooKeeper versions
+
+Apache Druid supports ZooKeeper versions 3.5.x and above.
+
+> Note: Starting with Apache Druid 0.22.0, support for ZooKeeper 3.4.x has been removed
+
+## ZooKeeper Operations
+
+The operations that happen over ZK are
1. [Coordinator](../design/coordinator.md) leader election
2. Segment "publishing" protocol from [Historical](../design/historical.md)
diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml
index a5962e24a59d..a5f0b4374c06 100644
--- a/extensions-core/kafka-extraction-namespace/pom.xml
+++ b/extensions-core/kafka-extraction-namespace/pom.xml
@@ -132,8 +132,7 @@
org.apache.zookeeper
zookeeper
-
- 3.5.8
+ ${zookeeper.version}
test
diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml
index 545b0d808acd..17848a307830 100644
--- a/extensions-core/kafka-indexing-service/pom.xml
+++ b/extensions-core/kafka-indexing-service/pom.xml
@@ -144,8 +144,7 @@
org.apache.zookeeper
zookeeper
-
- 3.5.8
+ ${zookeeper.version}
test
diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml
index ef58bd6a6482..5c1d55aa7e48 100644
--- a/indexing-service/pom.xml
+++ b/indexing-service/pom.xml
@@ -137,6 +137,10 @@
org.apache.zookeeper
zookeeper
+
+ org.apache.zookeeper
+ zookeeper-jute
+
javax.servlet
javax.servlet-api
diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile
index b3af96bfa51c..80762401874e 100644
--- a/integration-tests/docker/Dockerfile
+++ b/integration-tests/docker/Dockerfile
@@ -21,6 +21,9 @@ FROM openjdk:$JDK_VERSION as druidbase
COPY base-setup.sh /root/base-setup.sh
ARG KAFKA_VERSION
+# ZooKeeper version to install in the base image
+# This is passed in by maven at build time to align with the client version we depend on in the pom file
+ARG ZK_VERSION
ARG APACHE_ARCHIVE_MIRROR_HOST=https://archive.apache.org
RUN APACHE_ARCHIVE_MIRROR_HOST=${APACHE_ARCHIVE_MIRROR_HOST} /root/base-setup.sh && rm -f /root/base-setup.sh
diff --git a/integration-tests/docker/base-setup.sh b/integration-tests/docker/base-setup.sh
index b2aa33e1284d..2a258664fb7b 100755
--- a/integration-tests/docker/base-setup.sh
+++ b/integration-tests/docker/base-setup.sh
@@ -40,15 +40,9 @@ install_zk() {
rm /tmp/$ZK_TAR.tar.gz
}
-ZK_VERSION=3.4.14
-ZK_TAR=zookeeper-$ZK_VERSION
-install_zk
-ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4
-
-ZK_VERSION=3.5.9
ZK_TAR=apache-zookeeper-$ZK_VERSION-bin
install_zk
-ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5
+ln -s /usr/local/$ZK_TAR /usr/local/zookeeper
# Kafka
# KAFKA_VERSION is defined by docker build arguments
diff --git a/integration-tests/docker/docker-compose.base.yml b/integration-tests/docker/docker-compose.base.yml
index 9267b1a26000..11d7962d993f 100644
--- a/integration-tests/docker/docker-compose.base.yml
+++ b/integration-tests/docker/docker-compose.base.yml
@@ -53,8 +53,6 @@ services:
- ./service-supervisords/kafka.conf:/usr/lib/druid/conf/kafka.conf
env_file:
- ./environment-configs/common
- environment:
- - ZK_VERSION
druid-metadata-storage:
image: druid/cluster
diff --git a/integration-tests/docker/service-supervisords/zookeeper.conf b/integration-tests/docker/service-supervisords/zookeeper.conf
index 6039e2630484..1ee5247817f5 100644
--- a/integration-tests/docker/service-supervisords/zookeeper.conf
+++ b/integration-tests/docker/service-supervisords/zookeeper.conf
@@ -1,5 +1,5 @@
[program:zookeeper]
-command=/usr/local/zookeeper-%(ENV_ZK_VERSION)s/bin/zkServer.sh start-foreground
+command=/usr/local/zookeeper/bin/zkServer.sh start-foreground
user=daemon
priority=0
stdout_logfile=/shared/logs/zookeeper.log
diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
index dc58ee55eea4..bd5803d0e043 100644
--- a/integration-tests/pom.xml
+++ b/integration-tests/pom.xml
@@ -462,7 +462,6 @@
false
false
middleManager
- 3.5
@@ -495,7 +494,7 @@
${mysql.version}
5.5.1
${apache.kafka.version}
- ${zk.version}
+ ${zookeeper.version}
${project.basedir}/build_run_cluster.sh
diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh
index 0819e3a10f2d..8e93d4f64271 100755
--- a/integration-tests/script/docker_build_containers.sh
+++ b/integration-tests/script/docker_build_containers.sh
@@ -22,17 +22,21 @@ set -e
if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ]
then
echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Building druid-cluster with default Java version"
- docker build -t druid/cluster --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION $SHARED_DIR/docker
+ docker build -t druid/cluster --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION $SHARED_DIR/docker
else
echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}"
case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in
8)
echo "Build druid-cluster with Java 8"
- docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
+ docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
;;
11)
echo "Build druid-cluster with Java 11"
- docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
+ docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
+ ;;
+ 15)
+ echo "Build druid-cluster with Java 15"
+ docker build -t druid/cluster --build-arg JDK_VERSION=15-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
;;
*)
echo "Invalid JVM Runtime given. Stopping"
diff --git a/licenses.yaml b/licenses.yaml
index 01f7bfa47c74..2eb60d9600f8 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -1902,9 +1902,10 @@ name: Apache Zookeeper
license_category: binary
module: java-core
license_name: Apache License version 2.0
-version: 3.4.14
+version: 3.5.9
libraries:
- org.apache.zookeeper: zookeeper
+ - org.apache.zookeeper: zookeeper-jute
notices:
- zookeeper: |
Apache ZooKeeper
diff --git a/pom.xml b/pom.xml
index 251b57a2c3c9..af6676465699 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,7 +99,6 @@
5.1.48
3.10.6.Final
1.3.1
-
4.1.63.Final
v10.24.0
6.14.11
@@ -113,11 +112,8 @@
1.11.884
2.8.0
0.8.6
-
- 3.4.14
+ 3.5.9
2.5.7
1.26.0
v1-rev20190607-${com.google.apis.client.version}
@@ -315,12 +311,21 @@
io.netty
netty
+
+ io.netty
+ netty-transport-native-epoll
+
com.github.spotbugs
spotbugs-annotations
+
+ org.apache.zookeeper
+ zookeeper-jute
+ ${zookeeper.version}
+
org.apache.curator
curator-client
@@ -349,11 +354,6 @@
-
org.apache.curator
curator-x-discovery
${apache.curator.version}
diff --git a/server/pom.xml b/server/pom.xml
index 2c5ad5b16161..cf0030d640f8 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -74,6 +74,10 @@
org.apache.zookeeper
zookeeper
+
+ org.apache.zookeeper
+ zookeeper-jute
+
org.apache.curator
curator-framework
diff --git a/website/.spelling b/website/.spelling
index 1219f567e677..5aed9139a277 100644
--- a/website/.spelling
+++ b/website/.spelling
@@ -459,6 +459,8 @@ druid-s3-extensions
BasicDataSource
- ../docs/dependencies/zookeeper.md
LeadershipLatch
+3.5.x
+3.4.x
- ../docs/design/auth.md
AllowAll
AuthenticationResult
@@ -1881,4 +1883,4 @@ TiB
PiB
protobuf
Golang
-multiValueHandling
\ No newline at end of file
+multiValueHandling