remove ZooKeeper 3.4.x support and build with Java 15

.travis.yml

@@ -19,14 +19,15 @@ dist: xenial
 
 jdk:
   - openjdk8
+  - openjdk11
+  - openjdk15
 
 cache:
   directories:
     - $HOME/.m2
 
 env:
   global:
-    - ZK_VERSION=3.5 # default ZK version for integration tests
     - DOCKER_IP=127.0.0.1  # for integration tests
     - MVN="mvn -B"
     - > # Various options to make execution of maven goals faster (e.g., mvn install)
@@ -153,6 +154,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *package
+      name: "(openjdk15) packaging check"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_processing_module
       name: "(openjdk8) processing module test"
       stage: Tests - phase 1
@@ -219,6 +225,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_processing_module
+      name: "(openjdk15) processing module test"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_processing_module_sqlcompat
       <<: *test_processing_module
       name: "(openjdk8) processing module test (SQL Compatibility)"
@@ -231,6 +242,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+     - <<: *test_processing_module_sqlcompat
+      name: "(openjdk15) processing module test (SQL Compatibility)"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_indexing_module
       <<: *test_processing_module
       name: "(openjdk8) indexing modules test"
@@ -242,6 +258,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_indexing_module
+      name: "(openjdk15) indexing modules test"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_indexing_module_sqlcompat
       <<: *test_indexing_module
       name: "(openjdk8) indexing modules test (SQL Compatibility)"
@@ -253,6 +274,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_indexing_module_sqlcompat
+      name: "(openjdk15) indexing modules test (SQL Compatibility)"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_server_module
       <<: *test_processing_module
       name: "(openjdk8) server module test"
@@ -264,6 +290,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_server_module
+      name: "(openjdk15) server module test"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_server_module_sqlcompat
       <<: *test_server_module
       name: "(openjdk8) server module test (SQL Compatibility)"
@@ -274,6 +305,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_server_module_sqlcompat
+      name: "(openjdk15) server module test (SQL Compatibility)"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_other_modules
       <<: *test_processing_module
       name: "(openjdk8) other modules test"
@@ -285,6 +321,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_other_modules
+      name: "(openjdk15) other modules test"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - &test_other_modules_sqlcompat
       <<: *test_other_modules
       name: "(openjdk8) other modules test (SQL Compatibility)"
@@ -295,6 +336,11 @@ jobs:
       stage: Tests - phase 2
       jdk: openjdk11
 
+    - <<: *test_other_modules_sqlcompat
+      name: "(openjdk15) other modules test (SQL Compatibility)"
+      stage: Tests - phase 2
+      jdk: openjdk15
+
     - name: "web console"
       install: skip
       stage: Tests - phase 1
@@ -340,7 +386,7 @@ jobs:
         - docker
       env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
       script: &run_integration_test
-        - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} -Dzk.version=${ZK_VERSION} ${MAVEN_SKIP}
+        - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} ${MAVEN_SKIP}
       after_failure: &integration_test_diags
         - for v in ~/shared/logs/*.log ; do
           echo $v logtail ======================== ; tail -100 $v ;
@@ -543,16 +589,6 @@ jobs:
       jdk: openjdk8
       env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
 
-    # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
-    - <<: *integration_tests
-      name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests"
-      jdk: openjdk8
-      env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
-    - <<: *integration_kafka_format_tests
-      name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats"
-      env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
     # END - Integration tests for Compile with Java 8 and Run with Java 8
 
     # START - Integration tests for Compile with Java 8 and Run with Java 11
@@ -621,12 +657,6 @@ jobs:
       jdk: openjdk8
       env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
 
-    # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
-    - <<: *integration_tests
-      name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests"
-      jdk: openjdk8
-      env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4
-
     # END - Integration tests for Compile with Java 8 and Run with Java 11
 
     - &integration_batch_index_k8s

extensions-core/kafka-extraction-namespace/pom.xml

@@ -132,8 +132,7 @@
     <dependency>
       <groupId>org.apache.zookeeper</groupId>
       <artifactId>zookeeper</artifactId>
-      <!-- Kafka brokers require ZooKeeper 3.5.x clients for testing -->
-      <version>3.5.8</version>
+      <version>${zookeeper.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>

extensions-core/kafka-indexing-service/pom.xml

@@ -144,8 +144,7 @@
     <dependency>
       <groupId>org.apache.zookeeper</groupId>
       <artifactId>zookeeper</artifactId>
-      <!-- Kafka brokers require ZooKeeper 3.5.x clients for testing -->
-      <version>3.5.8</version>
+      <version>${zookeeper.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>

indexing-service/pom.xml

@@ -137,6 +137,10 @@
             <groupId>org.apache.zookeeper</groupId>
             <artifactId>zookeeper</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.zookeeper</groupId>
+            <artifactId>zookeeper-jute</artifactId>
+        </dependency>
         <dependency>
             <groupId>javax.servlet</groupId>
             <artifactId>javax.servlet-api</artifactId>

integration-tests/docker/base-setup.sh

@@ -40,15 +40,10 @@ install_zk() {
   rm /tmp/$ZK_TAR.tar.gz
 }
 
-ZK_VERSION=3.4.14
-ZK_TAR=zookeeper-$ZK_VERSION
-install_zk
-ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4
-
 ZK_VERSION=3.5.9
 ZK_TAR=apache-zookeeper-$ZK_VERSION-bin
 install_zk
-ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5
+ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-$ZK_VERSION
 
 # Kafka
 # Match the version to the Kafka client used by KafkaSupervisor

integration-tests/pom.xml

@@ -435,7 +435,6 @@
                 <docker.build.skip>false</docker.build.skip>
                 <docker.build.hadoop>false</docker.build.hadoop>
                 <it.indexer>middleManager</it.indexer>
-                <zk.version>3.5</zk.version>
                 <override.config.path />
                 <resource.file.dir.path />
 
@@ -466,7 +465,7 @@
                                         <DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>${docker.run.skip}</DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>
                                         <DRUID_INTEGRATION_TEST_INDEXER>${it.indexer}</DRUID_INTEGRATION_TEST_INDEXER>
                                         <MYSQL_VERSION>${mysql.version}</MYSQL_VERSION>
-                                        <ZK_VERSION>${zk.version}</ZK_VERSION>
+                                        <ZK_VERSION>${zookeeper.version}</ZK_VERSION>
                                     </environmentVariables>
                                     <executable>${project.basedir}/build_run_cluster.sh</executable>
                                 </configuration>

integration-tests/script/docker_build_containers.sh

@@ -34,6 +34,10 @@ else
     echo "Build druid-cluster with Java 11"
     docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
     ;;
+  15)
+    echo "Build druid-cluster with Java 15"
+    docker build -t druid/cluster --build-arg JDK_VERSION=15-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker
+    ;;
   *)
     echo "Invalid JVM Runtime given. Stopping"
     exit 1

licenses.yaml

@@ -1235,7 +1235,7 @@ name: Netty
 license_category: binary
 module: java-core
 license_name: Apache License version 2.0
-version: 4.1.48.Final
+version: 4.1.63.Final
 libraries:
   - io.netty: netty-buffer
   - io.netty: netty-codec
@@ -1902,9 +1902,10 @@ name: Apache Zookeeper
 license_category: binary
 module: java-core
 license_name: Apache License version 2.0
-version: 3.4.14
+version: 3.5.9
 libraries:
   - org.apache.zookeeper: zookeeper
+  - org.apache.zookeeper: zookeeper-jute
 notices:
   - zookeeper: |
       Apache ZooKeeper

pom.xml

@@ -99,8 +99,7 @@
         <mysql.version>5.1.48</mysql.version>
         <netty3.version>3.10.6.Final</netty3.version>
         <resilience4j.version>1.3.1</resilience4j.version>
-        <!-- Spark updated in https://github.com/apache/spark/pull/19884 -->
-        <netty4.version>4.1.48.Final</netty4.version>
+        <netty4.version>4.1.63.Final</netty4.version>
         <node.version>v10.24.0</node.version>
         <npm.version>6.14.11</npm.version>
         <postgresql.version>42.2.14</postgresql.version>
@@ -113,11 +112,8 @@
         <aws.sdk.version>1.11.884</aws.sdk.version>
         <caffeine.version>2.8.0</caffeine.version>
         <jacoco.version>0.8.6</jacoco.version>
-        <!-- Curator requires 3.4.x ZooKeeper clients to maintain compatibility with 3.4.x ZooKeeper servers,
-             If we upgrade to 3.5.x clients, curator requires 3.5.x servers, which would break backwards compatibility
-             see http://curator.apache.org/zk-compatibility.html -->
         <!-- When upgrading ZK, edit docs and integration tests as well (integration-tests/docker-base/setup.sh) -->
-        <zookeeper.version>3.4.14</zookeeper.version>
+        <zookeeper.version>3.5.9</zookeeper.version>
         <checkerframework.version>2.5.7</checkerframework.version>
         <com.google.apis.client.version>1.26.0</com.google.apis.client.version>
         <com.google.apis.compute.version>v1-rev20190607-${com.google.apis.client.version}</com.google.apis.compute.version>
@@ -315,12 +311,21 @@
                         <groupId>io.netty</groupId>
                         <artifactId>netty</artifactId>
                     </exclusion>
+                    <exclusion>
+                        <groupId>io.netty</groupId>
+                        <artifactId>netty-transport-native-epoll</artifactId>
+                    </exclusion>
                     <exclusion>
                         <groupId>com.github.spotbugs</groupId>
                         <artifactId>spotbugs-annotations</artifactId>
                     </exclusion>
                 </exclusions>
             </dependency>
+            <dependency>
+                <groupId>org.apache.zookeeper</groupId>
+                <artifactId>zookeeper-jute</artifactId>
+                <version>${zookeeper.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.apache.curator</groupId>
                 <artifactId>curator-client</artifactId>
@@ -349,11 +354,6 @@
                 </exclusions>
             </dependency>
             <dependency>
-              <!--
-                ~ TODO: This dependency transitively pulls in org.codehaus.jackson:jackson-mapper.asl.1.9.13 which has
-                ~ security vulnerability CVE-2017-7525. https://github.com/apache/druid/pull/8177 tracks
-                ~ upgrading to a newer curator version.
-              -->
               <groupId>org.apache.curator</groupId>
               <artifactId>curator-x-discovery</artifactId>
               <version>${apache.curator.version}</version>

server/pom.xml

@@ -74,6 +74,10 @@
             <groupId>org.apache.zookeeper</groupId>
             <artifactId>zookeeper</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.zookeeper</groupId>
+            <artifactId>zookeeper-jute</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-framework</artifactId>