remove ZooKeeper 3.4 support + pass tests with Java 15

.travis.yml

@@ -26,7 +26,6 @@ cache:
 
 env:
   global:
-    - ZK_VERSION=3.5 # default ZK version for integration tests
     - DOCKER_IP=127.0.0.1  # for integration tests
     - MVN="mvn -B"
     - > # Various options to make execution of maven goals faster (e.g., mvn install)
@@ -153,6 +152,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *package
+      name: "(openjdk15) packaging check"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_processing_module
       name: "(openjdk8) processing module test"
       stage: Tests - phase 1
@@ -219,6 +223,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_processing_module
+      name: "(openjdk15) processing module test"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_processing_module_sqlcompat
       <<: *test_processing_module
       name: "(openjdk8) processing module test (SQL Compatibility)"
@@ -231,6 +240,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_processing_module_sqlcompat
+      name: "(openjdk15) processing module test (SQL Compatibility)"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_indexing_module
       <<: *test_processing_module
       name: "(openjdk8) indexing modules test"
@@ -242,6 +256,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_indexing_module
+      name: "(openjdk15) indexing modules test"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_indexing_module_sqlcompat
       <<: *test_indexing_module
       name: "(openjdk8) indexing modules test (SQL Compatibility)"
@@ -253,6 +272,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_indexing_module_sqlcompat
+      name: "(openjdk15) indexing modules test (SQL Compatibility)"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_server_module
       <<: *test_processing_module
       name: "(openjdk8) server module test"
@@ -264,6 +288,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_server_module
+      name: "(openjdk15) server module test"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_server_module_sqlcompat
       <<: *test_server_module
       name: "(openjdk8) server module test (SQL Compatibility)"
@@ -274,6 +303,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_server_module_sqlcompat
+      name: "(openjdk15) server module test (SQL Compatibility)"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_other_modules
       <<: *test_processing_module
       name: "(openjdk8) other modules test"
@@ -285,6 +319,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_other_modules
+      name: "(openjdk15) other modules test"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_other_modules_sqlcompat
       <<: *test_other_modules
       name: "(openjdk8) other modules test (SQL Compatibility)"
@@ -295,6 +334,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_other_modules_sqlcompat
+      name: "(openjdk15) other modules test (SQL Compatibility)"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - name: "web console"
       install: skip
       stage: Tests - phase 1
@@ -342,7 +386,7 @@ jobs:
         - docker
       env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
       script: &run_integration_test
-        - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} -Dzk.version=${ZK_VERSION} ${MAVEN_SKIP}
+        - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} ${MAVEN_SKIP}
       after_failure: &integration_test_diags
         - for v in ~/shared/logs/*.log ; do
           echo $v logtail ======================== ; tail -100 $v ;
@@ -554,16 +598,6 @@ jobs:
       jdk: openjdk8
       env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
 
-    # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
-    - <<: *integration_tests
-      name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests"
-      jdk: openjdk8
-      env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
-    - <<: *integration_kafka_format_tests
-      name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats"
-      env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
     # END - Integration tests for Compile with Java 8 and Run with Java 8
 
     # START - Integration tests for Compile with Java 8 and Run with Java 11
@@ -637,12 +671,6 @@ jobs:
       jdk: openjdk8
       env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
 
-    # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
-    - <<: *integration_tests
-      name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests"
-      jdk: openjdk8
-      env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
     # END - Integration tests for Compile with Java 8 and Run with Java 11
 
     - &integration_batch_index_k8s

docs/dependencies/zookeeper.md

@@ -23,7 +23,17 @@ title: "ZooKeeper"
   -->
 
 
-Apache Druid uses [Apache ZooKeeper](http://zookeeper.apache.org/) (ZK) for management of current cluster state. The operations that happen over ZK are
+Apache Druid uses [Apache ZooKeeper](http://zookeeper.apache.org/) (ZK) for management of current cluster state.
+
+## Minimum ZooKeeper versions
+
+Apache Druid supports ZooKeeper versions 3.5.x and above.
+
+> Note: Starting with Apache Druid 0.22.0, support for ZooKeeper 3.4.x has been removed
+
+## ZooKeeper Operations
+
+The operations that happen over ZK are
 
 1.  [Coordinator](../design/coordinator.md) leader election
 2.  Segment "publishing" protocol from [Historical](../design/historical.md)

extensions-core/kafka-extraction-namespace/pom.xml

@@ -132,8 +132,7 @@
     <dependency>
       <groupId>org.apache.zookeeper</groupId>
       <artifactId>zookeeper</artifactId>
-      <!-- Kafka brokers require ZooKeeper 3.5.x clients for testing -->
-      <version>3.5.8</version>
+      <version>${zookeeper.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>

extensions-core/kafka-indexing-service/pom.xml

@@ -144,8 +144,7 @@
     <dependency>
       <groupId>org.apache.zookeeper</groupId>
       <artifactId>zookeeper</artifactId>
-      <!-- Kafka brokers require ZooKeeper 3.5.x clients for testing -->
-      <version>3.5.8</version>
+      <version>${zookeeper.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>

indexing-service/pom.xml

@@ -137,6 +137,10 @@
             <groupId>org.apache.zookeeper</groupId>
             <artifactId>zookeeper</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.zookeeper</groupId>
+            <artifactId>zookeeper-jute</artifactId>
+        </dependency>
         <dependency>
             <groupId>javax.servlet</groupId>
             <artifactId>javax.servlet-api</artifactId>

integration-tests/docker/Dockerfile

@@ -21,6 +21,9 @@ FROM openjdk:$JDK_VERSION as druidbase
 
 COPY base-setup.sh /root/base-setup.sh
 ARG KAFKA_VERSION
+# ZooKeeper version to install in the base image
+# This is passed in by maven at build time to align with the client version we depend on in the pom file
+ARG ZK_VERSION
 ARG APACHE_ARCHIVE_MIRROR_HOST=https://archive.apache.org
 RUN APACHE_ARCHIVE_MIRROR_HOST=${APACHE_ARCHIVE_MIRROR_HOST} /root/base-setup.sh && rm -f /root/base-setup.sh
 

integration-tests/docker/base-setup.sh

@@ -40,15 +40,9 @@ install_zk() {
   rm /tmp/$ZK_TAR.tar.gz
 }
 
-ZK_VERSION=3.4.14
-ZK_TAR=zookeeper-$ZK_VERSION
-install_zk
-ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4
-
-ZK_VERSION=3.5.9
 ZK_TAR=apache-zookeeper-$ZK_VERSION-bin
 install_zk
-ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5
+ln -s /usr/local/$ZK_TAR /usr/local/zookeeper
 
 # Kafka
 # KAFKA_VERSION is defined by docker build arguments

integration-tests/docker/docker-compose.base.yml

@@ -53,8 +53,6 @@ services:
       - ./service-supervisords/kafka.conf:/usr/lib/druid/conf/kafka.conf
     env_file:
       - ./environment-configs/common
-    environment:
-      - ZK_VERSION
 
   druid-metadata-storage:
     image: druid/cluster

integration-tests/docker/service-supervisords/zookeeper.conf

@@ -1,5 +1,5 @@
 [program:zookeeper]
-command=/usr/local/zookeeper-%(ENV_ZK_VERSION)s/bin/zkServer.sh start-foreground
+command=/usr/local/zookeeper/bin/zkServer.sh start-foreground
 user=daemon
 priority=0
 stdout_logfile=/shared/logs/zookeeper.log

integration-tests/pom.xml

@@ -462,7 +462,6 @@
                 <docker.build.skip>false</docker.build.skip>
                 <docker.build.hadoop>false</docker.build.hadoop>
                 <it.indexer>middleManager</it.indexer>
-                <zk.version>3.5</zk.version>
                 <override.config.path />
                 <resource.file.dir.path />
 
@@ -495,7 +494,7 @@
                                         <MYSQL_VERSION>${mysql.version}</MYSQL_VERSION>
                                         <CONFLUENT_VERSION>5.5.1</CONFLUENT_VERSION>
                                         <KAFKA_VERSION>${apache.kafka.version}</KAFKA_VERSION>
-                                        <ZK_VERSION>${zk.version}</ZK_VERSION>
+                                        <ZK_VERSION>${zookeeper.version}</ZK_VERSION>
                                     </environmentVariables>
                                     <executable>${project.basedir}/build_run_cluster.sh</executable>
                                 </configuration>

integration-tests/script/docker_build_containers.sh

@@ -22,17 +22,21 @@ set -e
 if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ]
 then
   echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Building druid-cluster with default Java version"
-  docker build -t druid/cluster --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION $SHARED_DIR/docker
+  docker build -t druid/cluster --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION $SHARED_DIR/docker
 else
   echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}"
   case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in
   8)
     echo "Build druid-cluster with Java 8"
-    docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
+    docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
     ;;
   11)
     echo "Build druid-cluster with Java 11"
-    docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
+    docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
+    ;;
+  15)
+    echo "Build druid-cluster with Java 15"
+    docker build -t druid/cluster --build-arg JDK_VERSION=15-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
     ;;
   *)
     echo "Invalid JVM Runtime given. Stopping"

licenses.yaml

@@ -1902,9 +1902,10 @@ name: Apache Zookeeper
 license_category: binary
 module: java-core
 license_name: Apache License version 2.0
-version: 3.4.14
+version: 3.5.9
 libraries:
   - org.apache.zookeeper: zookeeper
+  - org.apache.zookeeper: zookeeper-jute
 notices:
   - zookeeper: |
       Apache ZooKeeper

pom.xml

@@ -99,7 +99,6 @@
         <mysql.version>5.1.48</mysql.version>
         <netty3.version>3.10.6.Final</netty3.version>
         <resilience4j.version>1.3.1</resilience4j.version>
-        <!-- Spark updated in https://github.com/apache/spark/pull/19884 -->
         <netty4.version>4.1.63.Final</netty4.version>
         <node.version>v10.24.0</node.version>
         <npm.version>6.14.11</npm.version>
@@ -113,11 +112,8 @@
         <aws.sdk.version>1.11.884</aws.sdk.version>
         <caffeine.version>2.8.0</caffeine.version>
         <jacoco.version>0.8.6</jacoco.version>
-        <!-- Curator requires 3.4.x ZooKeeper clients to maintain compatibility with 3.4.x ZooKeeper servers,
-             If we upgrade to 3.5.x clients, curator requires 3.5.x servers, which would break backwards compatibility
-             see http://curator.apache.org/zk-compatibility.html -->
         <!-- When upgrading ZK, edit docs and integration tests as well (integration-tests/docker-base/setup.sh) -->
-        <zookeeper.version>3.4.14</zookeeper.version>
+        <zookeeper.version>3.5.9</zookeeper.version>
         <checkerframework.version>2.5.7</checkerframework.version>
         <com.google.apis.client.version>1.26.0</com.google.apis.client.version>
         <com.google.apis.compute.version>v1-rev20190607-${com.google.apis.client.version}</com.google.apis.compute.version>
@@ -315,12 +311,21 @@
                         <groupId>io.netty</groupId>
                         <artifactId>netty</artifactId>
                     </exclusion>
+                    <exclusion>
+                        <groupId>io.netty</groupId>
+                        <artifactId>netty-transport-native-epoll</artifactId>
+                    </exclusion>
                     <exclusion>
                         <groupId>com.github.spotbugs</groupId>
                         <artifactId>spotbugs-annotations</artifactId>
                     </exclusion>
                 </exclusions>
             </dependency>
+            <dependency>
+                <groupId>org.apache.zookeeper</groupId>
+                <artifactId>zookeeper-jute</artifactId>
+                <version>${zookeeper.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.apache.curator</groupId>
                 <artifactId>curator-client</artifactId>
@@ -349,11 +354,6 @@
                 </exclusions>
             </dependency>
             <dependency>
-              <!--
-                ~ TODO: This dependency transitively pulls in org.codehaus.jackson:jackson-mapper.asl.1.9.13 which has
-                ~ security vulnerability CVE-2017-7525. https://github.com/apache/druid/pull/8177 tracks
-                ~ upgrading to a newer curator version.
-              -->
               <groupId>org.apache.curator</groupId>
               <artifactId>curator-x-discovery</artifactId>
               <version>${apache.curator.version}</version>

server/pom.xml

@@ -74,6 +74,10 @@
             <groupId>org.apache.zookeeper</groupId>
             <artifactId>zookeeper</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.zookeeper</groupId>
+            <artifactId>zookeeper-jute</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-framework</artifactId>

website/.spelling

@@ -459,6 +459,8 @@ druid-s3-extensions
 BasicDataSource
  - ../docs/dependencies/zookeeper.md
 LeadershipLatch
+3.5.x
+3.4.x
  - ../docs/design/auth.md
 AllowAll
 AuthenticationResult
@@ -1881,4 +1883,4 @@ TiB
 PiB
 protobuf
 Golang
-multiValueHandling
+multiValueHandling