From 8104258d769cc27f67d76ffe56a00bf6d95b1c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 5 Mar 2021 11:23:33 -0800 Subject: [PATCH 1/8] remove ZK 3.4 compatibility --- .../kafka-extraction-namespace/pom.xml | 3 +-- extensions-core/kafka-indexing-service/pom.xml | 3 +-- indexing-service/pom.xml | 4 ++++ licenses.yaml | 3 ++- pom.xml | 17 ++++++++++------- server/pom.xml | 4 ++++ 6 files changed, 22 insertions(+), 12 deletions(-) diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index a5962e24a59d..a5f0b4374c06 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -132,8 +132,7 @@ org.apache.zookeeper zookeeper - - 3.5.8 + ${zookeeper.version} test diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 545b0d808acd..17848a307830 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -144,8 +144,7 @@ org.apache.zookeeper zookeeper - - 3.5.8 + ${zookeeper.version} test diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index ef58bd6a6482..5c1d55aa7e48 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -137,6 +137,10 @@ org.apache.zookeeper zookeeper + + org.apache.zookeeper + zookeeper-jute + javax.servlet javax.servlet-api diff --git a/licenses.yaml b/licenses.yaml index 01f7bfa47c74..2eb60d9600f8 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -1902,9 +1902,10 @@ name: Apache Zookeeper license_category: binary module: java-core license_name: Apache License version 2.0 -version: 3.4.14 +version: 3.5.9 libraries: - org.apache.zookeeper: zookeeper + - org.apache.zookeeper: zookeeper-jute notices: - zookeeper: | Apache ZooKeeper diff --git a/pom.xml b/pom.xml index 251b57a2c3c9..c8cba2623599 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,6 @@ 5.1.48 3.10.6.Final 1.3.1 - 4.1.63.Final v10.24.0 6.14.11 @@ -117,7 +116,7 @@ If we upgrade to 3.5.x clients, curator requires 3.5.x servers, which would break backwards compatibility see http://curator.apache.org/zk-compatibility.html --> - 3.4.14 + 3.5.9 2.5.7 1.26.0 v1-rev20190607-${com.google.apis.client.version} @@ -315,12 +314,21 @@ io.netty netty + + io.netty + netty-transport-native-epoll + com.github.spotbugs spotbugs-annotations + + org.apache.zookeeper + zookeeper-jute + ${zookeeper.version} + org.apache.curator curator-client @@ -349,11 +357,6 @@ - org.apache.curator curator-x-discovery ${apache.curator.version} diff --git a/server/pom.xml b/server/pom.xml index 2c5ad5b16161..cf0030d640f8 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -74,6 +74,10 @@ org.apache.zookeeper zookeeper + + org.apache.zookeeper + zookeeper-jute + org.apache.curator curator-framework From b7c6a71b95a5ca7ac23249bdb6238baa5027e0eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 5 Mar 2021 11:30:22 -0800 Subject: [PATCH 2/8] sync integration tests with pom ZK version --- .travis.yml | 19 +------------------ integration-tests/docker/base-setup.sh | 7 +------ integration-tests/pom.xml | 3 +-- pom.xml | 3 --- 4 files changed, 3 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index a25cc34bf2af..d2737d6663b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,6 @@ cache: env: global: - - ZK_VERSION=3.5 # default ZK version for integration tests - DOCKER_IP=127.0.0.1 # for integration tests - MVN="mvn -B" - > # Various options to make execution of maven goals faster (e.g., mvn install) @@ -342,7 +341,7 @@ jobs: - docker env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' script: &run_integration_test - - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} -Dzk.version=${ZK_VERSION} ${MAVEN_SKIP} + - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} ${MAVEN_SKIP} after_failure: &integration_test_diags - for v in ~/shared/logs/*.log ; do echo $v logtail ======================== ; tail -100 $v ; @@ -554,16 +553,6 @@ jobs: jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' - # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility - - <<: *integration_tests - name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests" - jdk: openjdk8 - env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4 - - - <<: *integration_kafka_format_tests - name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats" - env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4 - # END - Integration tests for Compile with Java 8 and Run with Java 8 # START - Integration tests for Compile with Java 8 and Run with Java 11 @@ -637,12 +626,6 @@ jobs: jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' - # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility - - <<: *integration_tests - name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests" - jdk: openjdk8 - env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4 - # END - Integration tests for Compile with Java 8 and Run with Java 11 - &integration_batch_index_k8s diff --git a/integration-tests/docker/base-setup.sh b/integration-tests/docker/base-setup.sh index b2aa33e1284d..c1218f8df5cf 100755 --- a/integration-tests/docker/base-setup.sh +++ b/integration-tests/docker/base-setup.sh @@ -40,15 +40,10 @@ install_zk() { rm /tmp/$ZK_TAR.tar.gz } -ZK_VERSION=3.4.14 -ZK_TAR=zookeeper-$ZK_VERSION -install_zk -ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4 - ZK_VERSION=3.5.9 ZK_TAR=apache-zookeeper-$ZK_VERSION-bin install_zk -ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5 +ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-$ZK_VERSION # Kafka # KAFKA_VERSION is defined by docker build arguments diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index dc58ee55eea4..bd5803d0e043 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -462,7 +462,6 @@ false false middleManager - 3.5 @@ -495,7 +494,7 @@ ${mysql.version} 5.5.1 ${apache.kafka.version} - ${zk.version} + ${zookeeper.version} ${project.basedir}/build_run_cluster.sh diff --git a/pom.xml b/pom.xml index c8cba2623599..af6676465699 100644 --- a/pom.xml +++ b/pom.xml @@ -112,9 +112,6 @@ 1.11.884 2.8.0 0.8.6 - 3.5.9 2.5.7 From 7dd234b8ba97894fa0c125afbc3845497c2f49c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 8 Mar 2021 19:56:29 -0800 Subject: [PATCH 3/8] add jkd15 to integration test containers --- integration-tests/script/docker_build_containers.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh index 0819e3a10f2d..6c7ee1203952 100755 --- a/integration-tests/script/docker_build_containers.sh +++ b/integration-tests/script/docker_build_containers.sh @@ -34,6 +34,10 @@ else echo "Build druid-cluster with Java 11" docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker ;; + 15) + echo "Build druid-cluster with Java 15" + docker build -t druid/cluster --build-arg JDK_VERSION=15-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker + ;; *) echo "Invalid JVM Runtime given. Stopping" exit 1 From 6e9c6300f29780ba461fcf15623a8a6f7b8372ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 6 Apr 2021 11:28:19 -0700 Subject: [PATCH 4/8] run unit tests with jdk15 --- .travis.yml | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.travis.yml b/.travis.yml index d2737d6663b5..fe9528e14e3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -152,6 +152,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *package + name: "(openjdk15) packaging check" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_processing_module name: "(openjdk8) processing module test" stage: Tests - phase 1 @@ -218,6 +223,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_processing_module + name: "(openjdk15) processing module test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_processing_module_sqlcompat <<: *test_processing_module name: "(openjdk8) processing module test (SQL Compatibility)" @@ -230,6 +240,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_processing_module_sqlcompat + name: "(openjdk15) processing module test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_indexing_module <<: *test_processing_module name: "(openjdk8) indexing modules test" @@ -241,6 +256,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_indexing_module + name: "(openjdk15) indexing modules test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_indexing_module_sqlcompat <<: *test_indexing_module name: "(openjdk8) indexing modules test (SQL Compatibility)" @@ -252,6 +272,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_indexing_module_sqlcompat + name: "(openjdk15) indexing modules test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_server_module <<: *test_processing_module name: "(openjdk8) server module test" @@ -263,6 +288,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_server_module + name: "(openjdk15) server module test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_server_module_sqlcompat <<: *test_server_module name: "(openjdk8) server module test (SQL Compatibility)" @@ -273,6 +303,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_server_module_sqlcompat + name: "(openjdk15) server module test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_other_modules <<: *test_processing_module name: "(openjdk8) other modules test" @@ -284,6 +319,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_other_modules + name: "(openjdk15) other modules test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_other_modules_sqlcompat <<: *test_other_modules name: "(openjdk8) other modules test (SQL Compatibility)" @@ -294,6 +334,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_other_modules_sqlcompat + name: "(openjdk15) other modules test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - name: "web console" install: skip stage: Tests - phase 1 From 19666c0ad06b03e0b388c7842cfcd2eafb9bbc17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 20 Apr 2021 14:51:11 -0700 Subject: [PATCH 5/8] remove need for ZK env var at runtime --- integration-tests/docker/Dockerfile | 3 +++ integration-tests/docker/base-setup.sh | 3 +-- integration-tests/docker/docker-compose.base.yml | 2 -- .../docker/service-supervisords/zookeeper.conf | 2 +- integration-tests/script/docker_build_containers.sh | 8 ++++---- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index b3af96bfa51c..80762401874e 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -21,6 +21,9 @@ FROM openjdk:$JDK_VERSION as druidbase COPY base-setup.sh /root/base-setup.sh ARG KAFKA_VERSION +# ZooKeeper version to install in the base image +# This is passed in by maven at build time to align with the client version we depend on in the pom file +ARG ZK_VERSION ARG APACHE_ARCHIVE_MIRROR_HOST=https://archive.apache.org RUN APACHE_ARCHIVE_MIRROR_HOST=${APACHE_ARCHIVE_MIRROR_HOST} /root/base-setup.sh && rm -f /root/base-setup.sh diff --git a/integration-tests/docker/base-setup.sh b/integration-tests/docker/base-setup.sh index c1218f8df5cf..2a258664fb7b 100755 --- a/integration-tests/docker/base-setup.sh +++ b/integration-tests/docker/base-setup.sh @@ -40,10 +40,9 @@ install_zk() { rm /tmp/$ZK_TAR.tar.gz } -ZK_VERSION=3.5.9 ZK_TAR=apache-zookeeper-$ZK_VERSION-bin install_zk -ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-$ZK_VERSION +ln -s /usr/local/$ZK_TAR /usr/local/zookeeper # Kafka # KAFKA_VERSION is defined by docker build arguments diff --git a/integration-tests/docker/docker-compose.base.yml b/integration-tests/docker/docker-compose.base.yml index 9267b1a26000..11d7962d993f 100644 --- a/integration-tests/docker/docker-compose.base.yml +++ b/integration-tests/docker/docker-compose.base.yml @@ -53,8 +53,6 @@ services: - ./service-supervisords/kafka.conf:/usr/lib/druid/conf/kafka.conf env_file: - ./environment-configs/common - environment: - - ZK_VERSION druid-metadata-storage: image: druid/cluster diff --git a/integration-tests/docker/service-supervisords/zookeeper.conf b/integration-tests/docker/service-supervisords/zookeeper.conf index 6039e2630484..1ee5247817f5 100644 --- a/integration-tests/docker/service-supervisords/zookeeper.conf +++ b/integration-tests/docker/service-supervisords/zookeeper.conf @@ -1,5 +1,5 @@ [program:zookeeper] -command=/usr/local/zookeeper-%(ENV_ZK_VERSION)s/bin/zkServer.sh start-foreground +command=/usr/local/zookeeper/bin/zkServer.sh start-foreground user=daemon priority=0 stdout_logfile=/shared/logs/zookeeper.log diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh index 6c7ee1203952..8e93d4f64271 100755 --- a/integration-tests/script/docker_build_containers.sh +++ b/integration-tests/script/docker_build_containers.sh @@ -22,21 +22,21 @@ set -e if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ] then echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Building druid-cluster with default Java version" - docker build -t druid/cluster --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION $SHARED_DIR/docker + docker build -t druid/cluster --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION $SHARED_DIR/docker else echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in 8) echo "Build druid-cluster with Java 8" - docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker ;; 11) echo "Build druid-cluster with Java 11" - docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker ;; 15) echo "Build druid-cluster with Java 15" - docker build -t druid/cluster --build-arg JDK_VERSION=15-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=15-slim --build-arg ZK_VERSION --build-arg KAFKA_VERSION --build-arg CONFLUENT_VERSION --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker ;; *) echo "Invalid JVM Runtime given. Stopping" From b2d22d9fdc92d85b1f1c2ae7f4cf59d0929fbb89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 21 May 2021 10:59:29 -0700 Subject: [PATCH 6/8] add docs note on ZK support --- docs/dependencies/zookeeper.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/dependencies/zookeeper.md b/docs/dependencies/zookeeper.md index 0855a376fef8..ec6814e24626 100644 --- a/docs/dependencies/zookeeper.md +++ b/docs/dependencies/zookeeper.md @@ -23,7 +23,17 @@ title: "ZooKeeper" --> -Apache Druid uses [Apache ZooKeeper](http://zookeeper.apache.org/) (ZK) for management of current cluster state. The operations that happen over ZK are +Apache Druid uses [Apache ZooKeeper](http://zookeeper.apache.org/) (ZK) for management of current cluster state. + +## Minimum ZooKeeper versions + +Apache Druid supports ZooKeeper versions 3.5.x and above. + +> Note: Starting with Apache Druid 0.22.0, support for ZooKeeper 3.4.x has been removed + +## ZooKeeper Operations + +The operations that happen over ZK are 1. [Coordinator](../design/coordinator.md) leader election 2. Segment "publishing" protocol from [Historical](../design/historical.md) From 8d54ceb8a483bc172af42e065224d68782b9020d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 21 May 2021 12:48:40 -0700 Subject: [PATCH 7/8] make spellcheck happy --- website/.spelling | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/website/.spelling b/website/.spelling index 1219f567e677..121310f98ab0 100644 --- a/website/.spelling +++ b/website/.spelling @@ -489,6 +489,8 @@ partitionNum v9 - ../docs/development/build.md 3.x +3.5.x +3.4.x 8u92 DskipTests Papache-release @@ -1881,4 +1883,4 @@ TiB PiB protobuf Golang -multiValueHandling \ No newline at end of file +multiValueHandling From bd777c1d82401020d40b89c534d8f2b1b16de909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 21 May 2021 13:46:52 -0700 Subject: [PATCH 8/8] fix spelling --- website/.spelling | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/.spelling b/website/.spelling index 121310f98ab0..5aed9139a277 100644 --- a/website/.spelling +++ b/website/.spelling @@ -459,6 +459,8 @@ druid-s3-extensions BasicDataSource - ../docs/dependencies/zookeeper.md LeadershipLatch +3.5.x +3.4.x - ../docs/design/auth.md AllowAll AuthenticationResult @@ -489,8 +491,6 @@ partitionNum v9 - ../docs/development/build.md 3.x -3.5.x -3.4.x 8u92 DskipTests Papache-release