From 1089fb3109af8d3a78c96c5d85b29cd98a5715ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 5 Mar 2021 11:23:33 -0800 Subject: [PATCH 1/5] remove ZK 3.4 compatibility --- extensions-core/kafka-extraction-namespace/pom.xml | 3 +-- extensions-core/kafka-indexing-service/pom.xml | 3 +-- indexing-service/pom.xml | 4 ++++ licenses.yaml | 3 ++- pom.xml | 12 ++++++------ server/pom.xml | 4 ++++ 6 files changed, 18 insertions(+), 11 deletions(-) diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index a5962e24a59d..a5f0b4374c06 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -132,8 +132,7 @@ org.apache.zookeeper zookeeper - - 3.5.8 + ${zookeeper.version} test diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 545b0d808acd..17848a307830 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -144,8 +144,7 @@ org.apache.zookeeper zookeeper - - 3.5.8 + ${zookeeper.version} test diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index be8360b127b5..71cc336cc686 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -137,6 +137,10 @@ org.apache.zookeeper zookeeper + + org.apache.zookeeper + zookeeper-jute + javax.servlet javax.servlet-api diff --git a/licenses.yaml b/licenses.yaml index f4b9cf415eb7..91c8d6e10345 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -1902,9 +1902,10 @@ name: Apache Zookeeper license_category: binary module: java-core license_name: Apache License version 2.0 -version: 3.4.14 +version: 3.5.9 libraries: - org.apache.zookeeper: zookeeper + - org.apache.zookeeper: zookeeper-jute notices: - zookeeper: | Apache ZooKeeper diff --git a/pom.xml b/pom.xml index fbe6b3490865..6cb95d43fa16 100644 --- a/pom.xml +++ b/pom.xml @@ -117,7 +117,7 @@ If we upgrade to 3.5.x clients, curator requires 3.5.x servers, which would break backwards compatibility see http://curator.apache.org/zk-compatibility.html --> - 3.4.14 + 3.5.9 2.5.7 1.26.0 v1-rev20190607-${com.google.apis.client.version} @@ -321,6 +321,11 @@ + + org.apache.zookeeper + zookeeper-jute + ${zookeeper.version} + org.apache.curator curator-client @@ -349,11 +354,6 @@ - org.apache.curator curator-x-discovery ${apache.curator.version} diff --git a/server/pom.xml b/server/pom.xml index 2c5ad5b16161..cf0030d640f8 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -74,6 +74,10 @@ org.apache.zookeeper zookeeper + + org.apache.zookeeper + zookeeper-jute + org.apache.curator curator-framework From 8cd1f60852dad05c96931e57ad23a5f91fdfd4fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 9 Mar 2021 16:57:19 -0800 Subject: [PATCH 2/5] update netty to 4.1.60 --- licenses.yaml | 2 +- pom.xml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/licenses.yaml b/licenses.yaml index 91c8d6e10345..e6c895497af2 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -1235,7 +1235,7 @@ name: Netty license_category: binary module: java-core license_name: Apache License version 2.0 -version: 4.1.48.Final +version: 4.1.63.Final libraries: - io.netty: netty-buffer - io.netty: netty-codec diff --git a/pom.xml b/pom.xml index 6cb95d43fa16..c6bae6236932 100644 --- a/pom.xml +++ b/pom.xml @@ -99,8 +99,7 @@ 5.1.48 3.10.6.Final 1.3.1 - - 4.1.48.Final + 4.1.63.Final v10.24.0 6.14.11 42.2.14 @@ -315,6 +314,10 @@ io.netty netty + + io.netty + netty-transport-native-epoll + com.github.spotbugs spotbugs-annotations From 02a428f12190764c4acd63af55394a8939c04708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 5 Mar 2021 11:30:22 -0800 Subject: [PATCH 3/5] sync integration tests with pom ZK version --- .travis.yml | 19 +------------------ integration-tests/docker/base-setup.sh | 7 +------ integration-tests/pom.xml | 3 +-- pom.xml | 3 --- 4 files changed, 3 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 29bf971df075..be8cbaf21a6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,6 @@ cache: env: global: - - ZK_VERSION=3.5 # default ZK version for integration tests - DOCKER_IP=127.0.0.1 # for integration tests - MVN="mvn -B" - > # Various options to make execution of maven goals faster (e.g., mvn install) @@ -340,7 +339,7 @@ jobs: - docker env: TESTNG_GROUPS='-Dgroups=batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' script: &run_integration_test - - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} -Dzk.version=${ZK_VERSION} ${MAVEN_SKIP} + - ${MVN} verify -pl integration-tests -P integration-tests ${TESTNG_GROUPS} ${JVM_RUNTIME} -Dit.indexer=${USE_INDEXER} ${MAVEN_SKIP} after_failure: &integration_test_diags - for v in ~/shared/logs/*.log ; do echo $v logtail ======================== ; tail -100 $v ; @@ -543,16 +542,6 @@ jobs: jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' - # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility - - <<: *integration_tests - name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests" - jdk: openjdk8 - env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4 - - - <<: *integration_kafka_format_tests - name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats" - env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4 - # END - Integration tests for Compile with Java 8 and Run with Java 8 # START - Integration tests for Compile with Java 8 and Run with Java 11 @@ -621,12 +610,6 @@ jobs: jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' - # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility - - <<: *integration_tests - name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests" - jdk: openjdk8 - env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4 - # END - Integration tests for Compile with Java 8 and Run with Java 11 - &integration_batch_index_k8s diff --git a/integration-tests/docker/base-setup.sh b/integration-tests/docker/base-setup.sh index e68f6bcccdca..7d49fe22d133 100755 --- a/integration-tests/docker/base-setup.sh +++ b/integration-tests/docker/base-setup.sh @@ -40,15 +40,10 @@ install_zk() { rm /tmp/$ZK_TAR.tar.gz } -ZK_VERSION=3.4.14 -ZK_TAR=zookeeper-$ZK_VERSION -install_zk -ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4 - ZK_VERSION=3.5.9 ZK_TAR=apache-zookeeper-$ZK_VERSION-bin install_zk -ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5 +ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-$ZK_VERSION # Kafka # Match the version to the Kafka client used by KafkaSupervisor diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 2fb9adc239e9..ee08d6f0f169 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -435,7 +435,6 @@ false false middleManager - 3.5 @@ -466,7 +465,7 @@ ${docker.run.skip} ${it.indexer} ${mysql.version} - ${zk.version} + ${zookeeper.version} ${project.basedir}/build_run_cluster.sh diff --git a/pom.xml b/pom.xml index c6bae6236932..c27137ac96de 100644 --- a/pom.xml +++ b/pom.xml @@ -112,9 +112,6 @@ 1.11.884 2.8.0 0.8.6 - 3.5.9 2.5.7 From a74cb6a97c69399633418672cb912fda4bee51b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 8 Mar 2021 19:56:29 -0800 Subject: [PATCH 4/5] add jkd15 to integration test containers --- integration-tests/script/docker_build_containers.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh index ef3df477dec3..121c9539a99f 100755 --- a/integration-tests/script/docker_build_containers.sh +++ b/integration-tests/script/docker_build_containers.sh @@ -34,6 +34,10 @@ else echo "Build druid-cluster with Java 11" docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker ;; + 15) + echo "Build druid-cluster with Java 15" + docker build -t druid/cluster --build-arg JDK_VERSION=15-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker + ;; *) echo "Invalid JVM Runtime given. Stopping" exit 1 From d16871fb966b8df7a2f95f8425c7c26c6f422551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 6 Apr 2021 11:28:19 -0700 Subject: [PATCH 5/5] run unit tests with jdk15 --- .travis.yml | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/.travis.yml b/.travis.yml index be8cbaf21a6a..a6fed4e8572c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,8 @@ dist: xenial jdk: - openjdk8 + - openjdk11 + - openjdk15 cache: directories: @@ -152,6 +154,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *package + name: "(openjdk15) packaging check" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_processing_module name: "(openjdk8) processing module test" stage: Tests - phase 1 @@ -218,6 +225,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_processing_module + name: "(openjdk15) processing module test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_processing_module_sqlcompat <<: *test_processing_module name: "(openjdk8) processing module test (SQL Compatibility)" @@ -230,6 +242,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_processing_module_sqlcompat + name: "(openjdk15) processing module test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_indexing_module <<: *test_processing_module name: "(openjdk8) indexing modules test" @@ -241,6 +258,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_indexing_module + name: "(openjdk15) indexing modules test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_indexing_module_sqlcompat <<: *test_indexing_module name: "(openjdk8) indexing modules test (SQL Compatibility)" @@ -252,6 +274,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_indexing_module_sqlcompat + name: "(openjdk15) indexing modules test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_server_module <<: *test_processing_module name: "(openjdk8) server module test" @@ -263,6 +290,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_server_module + name: "(openjdk15) server module test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_server_module_sqlcompat <<: *test_server_module name: "(openjdk8) server module test (SQL Compatibility)" @@ -273,6 +305,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_server_module_sqlcompat + name: "(openjdk15) server module test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_other_modules <<: *test_processing_module name: "(openjdk8) other modules test" @@ -284,6 +321,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_other_modules + name: "(openjdk15) other modules test" + stage: Tests - phase 2 + jdk: openjdk15 + - &test_other_modules_sqlcompat <<: *test_other_modules name: "(openjdk8) other modules test (SQL Compatibility)" @@ -294,6 +336,11 @@ jobs: stage: Tests - phase 2 jdk: openjdk11 + - <<: *test_other_modules_sqlcompat + name: "(openjdk15) other modules test (SQL Compatibility)" + stage: Tests - phase 2 + jdk: openjdk15 + - name: "web console" install: skip stage: Tests - phase 1