From 4dc2e1a1f68f6be3a47c3410b2096da197a07f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 20 Jan 2021 16:31:10 -0800 Subject: [PATCH 01/11] move integration tests from ZooKeeper 3.4.x to 3.5.x --- integration-tests/docker-base/setup.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/integration-tests/docker-base/setup.sh b/integration-tests/docker-base/setup.sh index 3b9d27d6d4fc..733841d7c115 100644 --- a/integration-tests/docker-base/setup.sh +++ b/integration-tests/docker-base/setup.sh @@ -32,11 +32,8 @@ apt-get install -y supervisor # Zookeeper -#ZK_VERSION=3.5.8 -#ZK_TAR=apache-zookeeper-$ZK_VERSION-bin - -ZK_VERISON=3.4.14 -ZK_TAR=zookeeper-$ZK_VERSION +ZK_VERSION=3.5.9 +ZK_TAR=apache-zookeeper-$ZK_VERSION-bin wget -q -O /tmp/$ZK_TAR.tar.gz "https://apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz" tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local From 286c4ef20d9299d378d8b97b8716f6129f8e8989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Thu, 21 Jan 2021 11:48:09 -0800 Subject: [PATCH 02/11] remove need to build separate docker-base image - use multi-stage build for the base image - use openjdk base image instead of building our own JDK base - workaround Debian not including MySQL by using MariaDB - download mysql connector directly instead of using distro version --- integration-tests/docker-base/README.md | 31 ---------- .../docker-base/jdk11/Dockerfile | 62 ------------------- integration-tests/docker-base/jdk8/Dockerfile | 27 -------- integration-tests/docker/Dockerfile | 21 ++++--- .../setup.sh => docker/base-setup.sh} | 19 ++++-- .../script/docker_build_containers.sh | 8 +-- 6 files changed, 30 insertions(+), 138 deletions(-) delete mode 100644 integration-tests/docker-base/README.md delete mode 100644 integration-tests/docker-base/jdk11/Dockerfile delete mode 100644 integration-tests/docker-base/jdk8/Dockerfile rename integration-tests/{docker-base/setup.sh => docker/base-setup.sh} (80%) mode change 100644 => 100755 diff --git a/integration-tests/docker-base/README.md b/integration-tests/docker-base/README.md deleted file mode 100644 index 00923f99cf42..000000000000 --- a/integration-tests/docker-base/README.md +++ /dev/null @@ -1,31 +0,0 @@ - - -## Base Docker Image for Integration Tests - -### Building Docker Image for Integration Tests - -Run the following commands from /integration-tests/docker-base - -To build docker image for Java JDK8: -- docker build -t druidbase: -f jdk8/Dockerfile . - -To build docker image for Java JDK11: -- docker build -t druidbase: -f jdk11/Dockerfile . - diff --git a/integration-tests/docker-base/jdk11/Dockerfile b/integration-tests/docker-base/jdk11/Dockerfile deleted file mode 100644 index c3ea66d5396d..000000000000 --- a/integration-tests/docker-base/jdk11/Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -# Based on the following projects/files: -# - SequenceIQ hadoop-docker project hosted at https://github.com/sequenceiq/hadoop-docker -# - AdoptOpenJDK openjdk-docker project hosted at https://github.com/AdoptOpenJDK/openjdk-docker -# and modified at the Apache Software Foundation (ASF). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM ubuntu:16.04 - -# Install Java JDK 11 (OpenJDK 11.0.5) -# Sourced from AdoptOpenJDK openjdk-docker project (https://github.com/AdoptOpenJDK/openjdk-docker) -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl ca-certificates fontconfig locales \ - && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \ - && locale-gen en_US.UTF-8 \ - && rm -rf /var/lib/apt/lists/* - -RUN set -eux; \ - ARCH="$(dpkg --print-architecture)"; \ - case "${ARCH}" in \ - armhf) \ - ESUM='c6b1fda3f8807028cbfcc34a4ded2e8a5a6b6239d2bcc1f06673ea6b1530df94'; \ - BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_arm_linux_hotspot_11.0.5_10.tar.gz'; \ - ;; \ - ppc64el|ppc64le) \ - ESUM='d763481ddc29ac0bdefb24216b3a0bf9afbb058552682567a075f9c0f7da5814'; \ - BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_ppc64le_linux_hotspot_11.0.5_10.tar.gz'; \ - ;; \ - amd64|x86_64) \ - ESUM='6dd0c9c8a740e6c19149e98034fba8e368fd9aa16ab417aa636854d40db1a161'; \ - BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.5_10.tar.gz'; \ - ;; \ - *) \ - echo "Unsupported arch: ${ARCH}"; \ - exit 1; \ - ;; \ - esac; \ - curl -LfsSo /tmp/openjdk.tar.gz ${BINARY_URL}; \ - echo "${ESUM} */tmp/openjdk.tar.gz" | sha256sum -c -; \ - mkdir -p /opt/java/openjdk; \ - cd /opt/java/openjdk; \ - tar -xf /tmp/openjdk.tar.gz --strip-components=1; \ - rm -rf /tmp/openjdk.tar.gz; - -ENV JAVA_HOME=/opt/java/openjdk \ - PATH="/opt/java/openjdk/bin:$PATH" - -# Bundle everything into one script so cleanup can reduce image size. -# Otherwise docker's layered images mean that things are not actually deleted. - -COPY setup.sh /root/setup.sh -RUN chmod 0755 /root/setup.sh && /root/setup.sh diff --git a/integration-tests/docker-base/jdk8/Dockerfile b/integration-tests/docker-base/jdk8/Dockerfile deleted file mode 100644 index 7784bf5924b3..000000000000 --- a/integration-tests/docker-base/jdk8/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Based on the SequenceIQ hadoop-docker project hosted at -# https://github.com/sequenceiq/hadoop-docker, and modified at -# the Apache Software Foundation (ASF). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM ubuntu:16.04 - -# Install Java JDK 8 -RUN apt-get update \ - && apt-get install -y openjdk-8-jdk - -# Bundle everything into one script so cleanup can reduce image size. -# Otherwise docker's layered images mean that things are not actually deleted. - -COPY setup.sh /root/setup.sh -RUN chmod 0755 /root/setup.sh && /root/setup.sh \ No newline at end of file diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 54fab63b4fd9..1e8b52b27231 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -13,16 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +ARG JDK_VERSION=8-slim +FROM openjdk:$JDK_VERSION as druidbase + +# Bundle everything into one script so cleanup can reduce image size. +# Otherwise docker's layered images mean that things are not actually deleted. + +COPY base-setup.sh /root/base-setup.sh +RUN /root/base-setup.sh && rm -f /root/base-setup.sh # This is default value for base image in case DOCKER_IMAGE is not given when building -ARG DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 -# Base image is built from integration-tests/docker-base in the Druid repo -FROM $DOCKER_IMAGE +FROM druidbase +ARG MYSQL_CONNECTOR_VERSION=5.1.49 # Verify Java version -ARG DOCKER_IMAGE -ENV DOCKER_IMAGE_USED=$DOCKER_IMAGE -RUN echo "Built using base docker image DOCKER_IMAGE_USED=$DOCKER_IMAGE_USED" RUN java -version RUN echo "[mysqld]\ncharacter-set-server=utf8\ncollation-server=utf8_bin\n" >> /etc/mysql/my.cnf @@ -37,9 +41,8 @@ RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \ ADD lib/* /usr/local/druid/lib/ # Download the MySQL Java connector -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends apt-utils libmysql-java -RUN ln -sf /usr/share/java/mysql-connector-java.jar /usr/local/druid/lib/mysql-connector-java.jar +RUN wget "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_CONNECTOR_VERSION/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar" \ + -O /usr/local/druid/lib/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar # Add sample data # touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72. diff --git a/integration-tests/docker-base/setup.sh b/integration-tests/docker/base-setup.sh old mode 100644 new mode 100755 similarity index 80% rename from integration-tests/docker-base/setup.sh rename to integration-tests/docker/base-setup.sh index 733841d7c115..904d8d09a97b --- a/integration-tests/docker-base/setup.sh +++ b/integration-tests/docker/base-setup.sh @@ -25,21 +25,30 @@ apt-get update apt-get install -y wget # MySQL (Metadata store) -apt-get install -y mysql-server +apt-get install -y default-mysql-server # Supervisor apt-get install -y supervisor # Zookeeper +install_zk() { + wget -q -O /tmp/$ZK_TAR.tar.gz "https://archive.apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz" + tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local + cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg + rm /tmp/$ZK_TAR.tar.gz +} + +ZK_VERSION=3.4.14 +ZK_TAR=zookeeper-$ZK_VERSION +install_zk + ZK_VERSION=3.5.9 ZK_TAR=apache-zookeeper-$ZK_VERSION-bin +install_zk -wget -q -O /tmp/$ZK_TAR.tar.gz "https://apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz" -tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local -cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg +# set default ZK version ln -s /usr/local/$ZK_TAR /usr/local/zookeeper -rm /tmp/$ZK_TAR.tar.gz # Kafka # Match the version to the Kafka client used by KafkaSupervisor diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh index b578ca7ab2ca..94fe3d329783 100755 --- a/integration-tests/script/docker_build_containers.sh +++ b/integration-tests/script/docker_build_containers.sh @@ -21,18 +21,18 @@ set -e if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ] then - echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Build druid-cluster with Java 8" - docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker + echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Building druid-cluster with default Java version" + docker build -t druid/cluster $SHARED_DIR/docker else echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in 8) echo "Build druid-cluster with Java 8" - docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=8-slim $SHARED_DIR/docker ;; 11) echo "Build druid-cluster with Java 11" - docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-11.0.5-1 $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=11-slim $SHARED_DIR/docker ;; *) echo "Invalid JVM Runtime given. Stopping" From 942f8afc20f9489224f69eb35739916d06bdb856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Thu, 28 Jan 2021 23:50:16 -0800 Subject: [PATCH 03/11] add 3.4 ZK integration test --- .travis.yml | 11 +++++++++++ integration-tests/docker/Dockerfile | 3 +++ integration-tests/docker/base-setup.sh | 5 ++--- integration-tests/docker/docker-compose.base.yml | 3 +++ .../docker/service-supervisords/zookeeper.conf | 2 +- 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index aa03eee01466..710fc16e033b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -496,6 +496,11 @@ jobs: jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' + - <<: *integration_tests + name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests" + jdk: openjdk8 + env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4 + # END - Integration tests for Compile with Java 8 and Run with Java 8 # START - Integration tests for Compile with Java 8 and Run with Java 11 @@ -558,6 +563,12 @@ jobs: name: "(Compile=openjdk8, Run=openjdk11) leadership and high availability integration tests" jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' + + - <<: *integration_tests + name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests" + jdk: openjdk8 + env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4 + # END - Integration tests for Compile with Java 8 and Run with Java 11 - &integration_batch_index_k8s diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 1e8b52b27231..056d9ed8db2f 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -74,6 +74,9 @@ ADD tls tls ADD client_tls client_tls +# define default ZooKeeper version +ENV ZK_VERSION=3.5 + # Expose ports: # - 8081, 8281: HTTP, HTTPS (coordinator) # - 8082, 8282: HTTP, HTTPS (broker) diff --git a/integration-tests/docker/base-setup.sh b/integration-tests/docker/base-setup.sh index 904d8d09a97b..186874a16d38 100755 --- a/integration-tests/docker/base-setup.sh +++ b/integration-tests/docker/base-setup.sh @@ -42,13 +42,12 @@ install_zk() { ZK_VERSION=3.4.14 ZK_TAR=zookeeper-$ZK_VERSION install_zk +ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4 ZK_VERSION=3.5.9 ZK_TAR=apache-zookeeper-$ZK_VERSION-bin install_zk - -# set default ZK version -ln -s /usr/local/$ZK_TAR /usr/local/zookeeper +ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5 # Kafka # Match the version to the Kafka client used by KafkaSupervisor diff --git a/integration-tests/docker/docker-compose.base.yml b/integration-tests/docker/docker-compose.base.yml index a8f63be514fd..d5e75d18e842 100644 --- a/integration-tests/docker/docker-compose.base.yml +++ b/integration-tests/docker/docker-compose.base.yml @@ -53,6 +53,9 @@ services: - ./service-supervisords/kafka.conf:/usr/lib/druid/conf/kafka.conf env_file: - ./environment-configs/common + environment: + # ZK_VERSION will be passed through to the docker container + - ZK_VERSION druid-metadata-storage: image: druid/cluster diff --git a/integration-tests/docker/service-supervisords/zookeeper.conf b/integration-tests/docker/service-supervisords/zookeeper.conf index 1ee5247817f5..98c6242d3bcd 100644 --- a/integration-tests/docker/service-supervisords/zookeeper.conf +++ b/integration-tests/docker/service-supervisords/zookeeper.conf @@ -1,5 +1,5 @@ [program:zookeeper] -command=/usr/local/zookeeper/bin/zkServer.sh start-foreground +command=/usr/local/zookeeper-%(ZK_VERSION)/bin/zkServer.sh start-foreground user=daemon priority=0 stdout_logfile=/shared/logs/zookeeper.log From 7c7a1eb328904f86a0eba72afd8cb9479350dbb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 29 Jan 2021 00:14:48 -0800 Subject: [PATCH 04/11] fix incorrect openssl command --- .../docker/tls/generate-server-certs-and-keystores.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/docker/tls/generate-server-certs-and-keystores.sh b/integration-tests/docker/tls/generate-server-certs-and-keystores.sh index 28bf09a54616..931b6e4752c9 100755 --- a/integration-tests/docker/tls/generate-server-certs-and-keystores.sh +++ b/integration-tests/docker/tls/generate-server-certs-and-keystores.sh @@ -61,7 +61,7 @@ DNS.2 = localhost EOT # Generate a server certificate for this machine -openssl genrsa -out server.key 1024 -sha256 +openssl genrsa -out server.key 1024 openssl req -new -out server.csr -key server.key -reqexts req_ext -config csr.conf openssl x509 -req -days 3650 -in server.csr -CA root.pem -CAkey root.key -set_serial 0x22222222 -out server.pem -sha256 -extfile csr.conf -extensions req_ext From f08cca580d30313be54e65cc6613486153e8328c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 29 Jan 2021 00:30:51 -0800 Subject: [PATCH 05/11] fix ZK version env args --- .travis.yml | 1 + integration-tests/docker/docker-compose.base.yml | 1 - integration-tests/docker/service-supervisords/zookeeper.conf | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 710fc16e033b..88793aa12ef0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ cache: env: global: + - ZK_VERSION=3.5 # default ZK version for integration tests - DOCKER_IP=127.0.0.1 # for integration tests - MVN="mvn -B" - > # Various options to make execution of maven goals faster (e.g., mvn install) diff --git a/integration-tests/docker/docker-compose.base.yml b/integration-tests/docker/docker-compose.base.yml index d5e75d18e842..dfe6b898dfcb 100644 --- a/integration-tests/docker/docker-compose.base.yml +++ b/integration-tests/docker/docker-compose.base.yml @@ -54,7 +54,6 @@ services: env_file: - ./environment-configs/common environment: - # ZK_VERSION will be passed through to the docker container - ZK_VERSION druid-metadata-storage: diff --git a/integration-tests/docker/service-supervisords/zookeeper.conf b/integration-tests/docker/service-supervisords/zookeeper.conf index 98c6242d3bcd..6039e2630484 100644 --- a/integration-tests/docker/service-supervisords/zookeeper.conf +++ b/integration-tests/docker/service-supervisords/zookeeper.conf @@ -1,5 +1,5 @@ [program:zookeeper] -command=/usr/local/zookeeper-%(ZK_VERSION)/bin/zkServer.sh start-foreground +command=/usr/local/zookeeper-%(ENV_ZK_VERSION)s/bin/zkServer.sh start-foreground user=daemon priority=0 stdout_logfile=/shared/logs/zookeeper.log From 19e5d7010f980611fc39fe9216c482d8aae7994f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 29 Jan 2021 00:35:56 -0800 Subject: [PATCH 06/11] remove unnecessary docker env var --- integration-tests/docker/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 056d9ed8db2f..1e8b52b27231 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -74,9 +74,6 @@ ADD tls tls ADD client_tls client_tls -# define default ZooKeeper version -ENV ZK_VERSION=3.5 - # Expose ports: # - 8081, 8281: HTTP, HTTPS (coordinator) # - 8082, 8282: HTTP, HTTPS (broker) From f3c4ead2dc3c05a0014ccf4e15a4671a420b8464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 29 Jan 2021 00:37:08 -0800 Subject: [PATCH 07/11] remove extraneous comment --- integration-tests/docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 1e8b52b27231..b45842960e40 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -22,7 +22,6 @@ FROM openjdk:$JDK_VERSION as druidbase COPY base-setup.sh /root/base-setup.sh RUN /root/base-setup.sh && rm -f /root/base-setup.sh -# This is default value for base image in case DOCKER_IMAGE is not given when building FROM druidbase ARG MYSQL_CONNECTOR_VERSION=5.1.49 From e5c64b4206154380207d8f579ade071612abf7a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 29 Jan 2021 14:01:37 -0800 Subject: [PATCH 08/11] reduce wget verbosity --- integration-tests/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index b45842960e40..a71131d38dd4 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -40,7 +40,7 @@ RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \ ADD lib/* /usr/local/druid/lib/ # Download the MySQL Java connector -RUN wget "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_CONNECTOR_VERSION/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar" \ +RUN wget -q "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_CONNECTOR_VERSION/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar" \ -O /usr/local/druid/lib/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar # Add sample data From fd23419c7b65e333de6d928f6bec972a23386ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 29 Jan 2021 15:06:39 -0800 Subject: [PATCH 09/11] fix mysql jar path to match common classpath config --- integration-tests/docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index a71131d38dd4..957d6617afcc 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -40,8 +40,9 @@ RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \ ADD lib/* /usr/local/druid/lib/ # Download the MySQL Java connector +# target path must match the exact path referenced in environment-configs/common RUN wget -q "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_CONNECTOR_VERSION/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar" \ - -O /usr/local/druid/lib/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar + -O /usr/local/druid/lib/mysql-connector-java.jar # Add sample data # touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72. From bb851d3046f7e8fcd5f04f379e0853f7f422d663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Sat, 30 Jan 2021 17:14:36 -0800 Subject: [PATCH 10/11] keep mysql connector version in sync with extension --- integration-tests/docker/Dockerfile | 4 ++-- integration-tests/pom.xml | 1 + integration-tests/script/docker_build_containers.sh | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 957d6617afcc..10e82b986449 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -23,7 +23,7 @@ COPY base-setup.sh /root/base-setup.sh RUN /root/base-setup.sh && rm -f /root/base-setup.sh FROM druidbase -ARG MYSQL_CONNECTOR_VERSION=5.1.49 +ARG MYSQL_VERSION # Verify Java version RUN java -version @@ -41,7 +41,7 @@ ADD lib/* /usr/local/druid/lib/ # Download the MySQL Java connector # target path must match the exact path referenced in environment-configs/common -RUN wget -q "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_CONNECTOR_VERSION/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar" \ +RUN wget -q "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_VERSION/mysql-connector-java-$MYSQL_VERSION.jar" \ -O /usr/local/druid/lib/mysql-connector-java.jar # Add sample data diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 53c3b373c307..ddcdaaa3c042 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -416,6 +416,7 @@ ${docker.build.skip} ${docker.run.skip} ${it.indexer} + ${mysql.version} ${project.basedir}/build_run_cluster.sh diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh index 94fe3d329783..6bff5020acc6 100755 --- a/integration-tests/script/docker_build_containers.sh +++ b/integration-tests/script/docker_build_containers.sh @@ -22,17 +22,17 @@ set -e if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ] then echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Building druid-cluster with default Java version" - docker build -t druid/cluster $SHARED_DIR/docker + docker build -t druid/cluster --build-arg MYSQL_VERSION $SHARED_DIR/docker else echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in 8) echo "Build druid-cluster with Java 8" - docker build -t druid/cluster --build-arg JDK_VERSION=8-slim $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker ;; 11) echo "Build druid-cluster with Java 11" - docker build -t druid/cluster --build-arg JDK_VERSION=11-slim $SHARED_DIR/docker + docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker ;; *) echo "Invalid JVM Runtime given. Stopping" From 11115078514a8f16c6bd08ebf00ab9573a43c743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Sat, 30 Jan 2021 17:22:03 -0800 Subject: [PATCH 11/11] add Kafka indexing integration test with ZK 3.4.x --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 88793aa12ef0..166d6f6a64f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -497,11 +497,16 @@ jobs: jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' + # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility - <<: *integration_tests name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests" jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4 + - <<: *integration_kafka_format_tests + name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats" + env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4 + # END - Integration tests for Compile with Java 8 and Run with Java 8 # START - Integration tests for Compile with Java 8 and Run with Java 11 @@ -565,6 +570,7 @@ jobs: jdk: openjdk8 env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' + # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility - <<: *integration_tests name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests" jdk: openjdk8