diff --git a/.travis.yml b/.travis.yml index a25cc34bf2af..fe9528e14e3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,6 @@ cache: env: global: - - ZK_VERSION=3.5 # default ZK version for integration tests - DOCKER_IP=127.0.0.1 # for integration tests - MVN="mvn -B" - > # Various options to make execution of maven goals faster (e.g., mvn install) @@ -153,6 +152,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *package + name: "(openjdk15) packaging check" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_processing_module name: "(openjdk8) processing module test" stage: Tests - phase 1 @@ -219,6 +223,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_processing_module + name: "(openjdk15) processing module test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_processing_module_sqlcompat <<: *test_processing_module name: "(openjdk8) processing module test (SQL Compatibility)" @@ -231,6 +240,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_processing_module_sqlcompat + name: "(openjdk15) processing module test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_indexing_module <<: *test_processing_module name: "(openjdk8) indexing modules test" @@ -242,6 +256,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_indexing_module + name: "(openjdk15) indexing modules test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_indexing_module_sqlcompat <<: *test_indexing_module name: "(openjdk8) indexing modules test (SQL Compatibility)" @@ -253,6 +272,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_indexing_module_sqlcompat + name: "(openjdk15) indexing modules test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_server_module <<: *test_processing_module name: "(openjdk8) server module test" @@ -264,6 +288,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_server_module + name: "(openjdk15) server module test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_server_module_sqlcompat <<: *test_server_module name: "(openjdk8) server module test (SQL Compatibility)" @@ -274,6 +303,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_server_module_sqlcompat + name: "(openjdk15) server module test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_other_modules <<: *test_processing_module name: "(openjdk8) other modules test" @@ -285,6 +319,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_other_modules + name: "(openjdk15) other modules test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_other_modules_sqlcompat <<: *test_other_modules name: "(openjdk8) other modules test (SQL Compatibility)" @@ -295,6 +334,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_other_modules_sqlcompat + name: "(openjdk15) other modules test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - name: "web console" install: skip stage: Tests - phase 1 @@ -342,7 +386,7 @@ jobs: - docker env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' script: &run_integration_test - - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} -Dzk.version=${ZK_VERSION} ${MAVEN_SKIP} + - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} ${MAVEN_SKIP} after_failure: &integration_test_diags - for v in ~/shared/logs/*.log ; do echo $v logtail ======================== ; tail -100 $v ; @@ -554,16 +598,6 @@ jobs: jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' - # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility - - <<: *integration_tests - name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests" - jdk: openjdk8 - env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4 - - - <<: *integration_kafka_format_tests - name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats" - env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4 - # END - Integration tests for Compile with Java 8 and Run with Java 8 # START - Integration tests for Compile with Java 8 and Run with Java 11 @@ -637,12 +671,6 @@ jobs: jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' - # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility - - <<: *integration_tests - name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests" - jdk: openjdk8 - env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4 - # END - Integration tests for Compile with Java 8 and Run with Java 11 - &integration_batch_index_k8s diff --git a/docs/dependencies/zookeeper.md b/docs/dependencies/zookeeper.md index 0855a376fef8..ec6814e24626 100644 --- a/docs/dependencies/zookeeper.md +++ b/docs/dependencies/zookeeper.md @@ -23,7 +23,17 @@ title: "ZooKeeper" --> -Apache Druid uses [Apache ZooKeeper](http://zookeeper.apache.org/) (ZK) for management of current cluster state. The operations that happen over ZK are +Apache Druid uses [Apache ZooKeeper](http://zookeeper.apache.org/) (ZK) for management of current cluster state. + +## Minimum ZooKeeper versions + +Apache Druid supports ZooKeeper versions 3.5.x and above. + +> Note: Starting with Apache Druid 0.22.0, support for ZooKeeper 3.4.x has been removed + +## ZooKeeper Operations + +The operations that happen over ZK are 1. [Coordinator](../design/coordinator.md) leader election 2. Segment "publishing" protocol from [Historical](../design/historical.md) diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index a5962e24a59d..a5f0b4374c06 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -132,8 +132,7 @@ org.apache.zookeeper zookeeper - - 3.5.8 + ${zookeeper.version} test diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 545b0d808acd..17848a307830 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -144,8 +144,7 @@ org.apache.zookeeper zookeeper - - 3.5.8 + ${zookeeper.version} test diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index ef58bd6a6482..5c1d55aa7e48 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -137,6 +137,10 @@ org.apache.zookeeper zookeeper + + org.apache.zookeeper + zookeeper-jute + javax.servlet javax.servlet-api diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index b3af96bfa51c..80762401874e 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -21,6 +21,9 @@ FROM openjdk:$JDK_VERSION as druidbase COPY base-setup.sh /root/base-setup.sh ARG KAFKA_VERSION +# ZooKeeper version to install in the base image +# This is passed in by maven at build time to align with the client version we depend on in the pom file +ARG ZK_VERSION ARG APACHE_ARCHIVE_MIRROR_HOST=https://archive.apache.org RUN APACHE_ARCHIVE_MIRROR_HOST=${APACHE_ARCHIVE_MIRROR_HOST} /root/base-setup.sh && rm -f /root/base-setup.sh diff --git a/integration-tests/docker/base-setup.sh b/integration-tests/docker/base-setup.sh index b2aa33e1284d..2a258664fb7b 100755 --- a/integration-tests/docker/base-setup.sh +++ b/integration-tests/docker/base-setup.sh @@ -40,15 +40,9 @@ install_zk() { rm /tmp/$ZK_TAR.tar.gz } -ZK_VERSION=3.4.14 -ZK_TAR=zookeeper-$ZK_VERSION -install_zk -ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4 - -ZK_VERSION=3.5.9 ZK_TAR=apache-zookeeper-$ZK_VERSION-bin install_zk -ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5 +ln -s /usr/local/$ZK_TAR /usr/local/zookeeper # Kafka # KAFKA_VERSION is defined by docker build arguments diff --git a/integration-tests/docker/docker-compose.base.yml b/integration-tests/docker/docker-compose.base.yml index 9267b1a26000..11d7962d993f 100644 --- a/integration-tests/docker/docker-compose.base.yml +++ b/integration-tests/docker/docker-compose.base.yml @@ -53,8 +53,6 @@ services: - ./service-supervisords/kafka.conf:/usr/lib/druid/conf/kafka.conf env_file: - ./environment-configs/common - environment: - - ZK_VERSION druid-metadata-storage: image: druid/cluster diff --git a/integration-tests/docker/service-supervisords/zookeeper.conf b/integration-tests/docker/service-supervisords/zookeeper.conf index 6039e2630484..1ee5247817f5 100644 --- a/integration-tests/docker/service-supervisords/zookeeper.conf +++ b/integration-tests/docker/service-supervisords/zookeeper.conf @@ -1,5 +1,5 @@ [program:zookeeper] -command=/usr/local/zookeeper-%(ENV_ZK_VERSION)s/bin/zkServer.sh start-foreground +command=/usr/local/zookeeper/bin/zkServer.sh start-foreground user=daemon priority=0 stdout_logfile=/shared/logs/zookeeper.log diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index dc58ee55eea4..bd5803d0e043 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -462,7 +462,6 @@ false false middleManager - 3.5 @@ -495,7 +494,7 @@ ${mysql.version} 5.5.1 ${apache.kafka.version} - ${zk.version} + ${zookeeper.version} ${project.basedir}/build_run_cluster.sh diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh index 0819e3a10f2d..8e93d4f64271 100755 --- a/integration-tests/script/docker_build_containers.sh +++ b/integration-tests/script/docker_build_containers.sh @@ -22,17 +22,21 @@ set -e if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ] then echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Building druid-cluster with default Java version" - docker build -t druid/cluster --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION $SHARED_DIR/docker + docker build -t druid/cluster --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION $SHARED_DIR/docker else echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in 8) echo "Build druid-cluster with Java 8" - docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker ;; 11) echo "Build druid-cluster with Java 11" - docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker + ;; + 15) + echo "Build druid-cluster with Java 15" + docker build -t druid/cluster --build-arg JDK_VERSION=15-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker ;; *) echo "Invalid JVM Runtime given. Stopping" diff --git a/licenses.yaml b/licenses.yaml index 01f7bfa47c74..2eb60d9600f8 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -1902,9 +1902,10 @@ name: Apache Zookeeper license_category: binary module: java-core license_name: Apache License version 2.0 -version: 3.4.14 +version: 3.5.9 libraries: - org.apache.zookeeper: zookeeper + - org.apache.zookeeper: zookeeper-jute notices: - zookeeper: | Apache ZooKeeper diff --git a/pom.xml b/pom.xml index 251b57a2c3c9..af6676465699 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,6 @@ 5.1.48 3.10.6.Final 1.3.1 - 4.1.63.Final v10.24.0 6.14.11 @@ -113,11 +112,8 @@ 1.11.884 2.8.0 0.8.6 - - 3.4.14 + 3.5.9 2.5.7 1.26.0 v1-rev20190607-${com.google.apis.client.version} @@ -315,12 +311,21 @@ io.netty netty + + io.netty + netty-transport-native-epoll + com.github.spotbugs spotbugs-annotations + + org.apache.zookeeper + zookeeper-jute + ${zookeeper.version} + org.apache.curator curator-client @@ -349,11 +354,6 @@ - org.apache.curator curator-x-discovery ${apache.curator.version} diff --git a/server/pom.xml b/server/pom.xml index 2c5ad5b16161..cf0030d640f8 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -74,6 +74,10 @@ org.apache.zookeeper zookeeper + + org.apache.zookeeper + zookeeper-jute + org.apache.curator curator-framework diff --git a/website/.spelling b/website/.spelling index 1219f567e677..5aed9139a277 100644 --- a/website/.spelling +++ b/website/.spelling @@ -459,6 +459,8 @@ druid-s3-extensions BasicDataSource - ../docs/dependencies/zookeeper.md LeadershipLatch +3.5.x +3.4.x - ../docs/design/auth.md AllowAll AuthenticationResult @@ -1881,4 +1883,4 @@ TiB PiB protobuf Golang -multiValueHandling \ No newline at end of file +multiValueHandling