move integration tests from ZooKeeper 3.4.x to 3.5.x

.travis.yml

@@ -27,6 +27,7 @@ cache:
 
 env:
   global:
+    - ZK_VERSION=3.5 # default ZK version for integration tests
     - DOCKER_IP=127.0.0.1  # for integration tests
     - MVN="mvn -B"
     - > # Various options to make execution of maven goals faster (e.g., mvn install)
@@ -496,6 +497,16 @@ jobs:
       jdk: openjdk8
       env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
 
+    # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
+    - <<: *integration_tests
+      name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests"
+      jdk: openjdk8
+      env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
+
+    - <<: *integration_kafka_format_tests
+      name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats"
+      env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
+
     # END - Integration tests for Compile with Java 8 and Run with Java 8
 
     # START - Integration tests for Compile with Java 8 and Run with Java 11
@@ -558,6 +569,13 @@ jobs:
       name: "(Compile=openjdk8, Run=openjdk11) leadership and high availability integration tests"
       jdk: openjdk8
       env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
+
+    # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
+    - <<: *integration_tests
+      name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests"
+      jdk: openjdk8
+      env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4
+
     # END - Integration tests for Compile with Java 8 and Run with Java 11
 
     - &integration_batch_index_k8s

integration-tests/docker/Dockerfile

@@ -13,16 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+ARG JDK_VERSION=8-slim
+FROM openjdk:$JDK_VERSION as druidbase
 
-# This is default value for base image in case DOCKER_IMAGE is not given when building
-ARG DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1
-# Base image is built from integration-tests/docker-base in the Druid repo
-FROM $DOCKER_IMAGE
+# Bundle everything into one script so cleanup can reduce image size.
+# Otherwise docker's layered images mean that things are not actually deleted.
+
+COPY base-setup.sh /root/base-setup.sh
+RUN /root/base-setup.sh && rm -f /root/base-setup.sh
+
+FROM druidbase
+ARG MYSQL_VERSION
 
 # Verify Java version
-ARG DOCKER_IMAGE
-ENV DOCKER_IMAGE_USED=$DOCKER_IMAGE
-RUN echo "Built using base docker image DOCKER_IMAGE_USED=$DOCKER_IMAGE_USED"
 RUN java -version
 
 RUN echo "[mysqld]\ncharacter-set-server=utf8\ncollation-server=utf8_bin\n" >> /etc/mysql/my.cnf
@@ -37,9 +40,9 @@ RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \
 ADD lib/* /usr/local/druid/lib/
 
 # Download the MySQL Java connector
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends apt-utils libmysql-java
-RUN ln -sf /usr/share/java/mysql-connector-java.jar /usr/local/druid/lib/mysql-connector-java.jar
+# target path must match the exact path referenced in environment-configs/common
+RUN wget -q "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_VERSION/mysql-connector-java-$MYSQL_VERSION.jar" \
+    -O /usr/local/druid/lib/mysql-connector-java.jar
 
 # Add sample data
 # touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72.

integration-tests/docker-base/setup.sh → integration-tests/docker/base-setup.sh

@@ -25,24 +25,29 @@ apt-get update
 apt-get install -y wget
 
 # MySQL (Metadata store)
-apt-get install -y mysql-server
+apt-get install -y default-mysql-server
 
 # Supervisor
 apt-get install -y supervisor
 
 # Zookeeper
 
-#ZK_VERSION=3.5.8
-#ZK_TAR=apache-zookeeper-$ZK_VERSION-bin
+install_zk() {
+  wget -q -O /tmp/$ZK_TAR.tar.gz "https://archive.apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz"
+  tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local
+  cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg
+  rm /tmp/$ZK_TAR.tar.gz
+}
 
-ZK_VERISON=3.4.14
+ZK_VERSION=3.4.14
 ZK_TAR=zookeeper-$ZK_VERSION
+install_zk
+ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4
 
-wget -q -O /tmp/$ZK_TAR.tar.gz "https://apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz"
-tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local
-cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg
-ln -s /usr/local/$ZK_TAR /usr/local/zookeeper
-rm /tmp/$ZK_TAR.tar.gz
+ZK_VERSION=3.5.9
+ZK_TAR=apache-zookeeper-$ZK_VERSION-bin
+install_zk
+ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5
 
 # Kafka
 # Match the version to the Kafka client used by KafkaSupervisor

integration-tests/docker/docker-compose.base.yml

@@ -53,6 +53,8 @@ services:
       - ./service-supervisords/kafka.conf:/usr/lib/druid/conf/kafka.conf
     env_file:
       - ./environment-configs/common
+    environment:
+      - ZK_VERSION
 
   druid-metadata-storage:
     image: druid/cluster

integration-tests/docker/service-supervisords/zookeeper.conf

@@ -1,5 +1,5 @@
 [program:zookeeper]
-command=/usr/local/zookeeper/bin/zkServer.sh start-foreground
+command=/usr/local/zookeeper-%(ENV_ZK_VERSION)s/bin/zkServer.sh start-foreground
 user=daemon
 priority=0
 stdout_logfile=/shared/logs/zookeeper.log

integration-tests/docker/tls/generate-server-certs-and-keystores.sh

@@ -61,7 +61,7 @@ DNS.2 = localhost
 EOT
 
 # Generate a server certificate for this machine
-openssl genrsa -out server.key 1024 -sha256
+openssl genrsa -out server.key 1024
 openssl req -new -out server.csr -key server.key -reqexts req_ext -config csr.conf
 openssl x509 -req -days 3650 -in server.csr -CA root.pem -CAkey root.key -set_serial 0x22222222 -out server.pem -sha256 -extfile csr.conf -extensions req_ext
 

integration-tests/pom.xml

@@ -416,6 +416,7 @@
                                         <DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER>${docker.build.skip}</DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER>
                                         <DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>${docker.run.skip}</DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>
                                         <DRUID_INTEGRATION_TEST_INDEXER>${it.indexer}</DRUID_INTEGRATION_TEST_INDEXER>
+                                        <MYSQL_VERSION>${mysql.version}</MYSQL_VERSION>
                                     </environmentVariables>
                                     <executable>${project.basedir}/build_run_cluster.sh</executable>
                                 </configuration>

integration-tests/script/docker_build_containers.sh

@@ -21,18 +21,18 @@ set -e
 
 if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ]
 then
-  echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Build druid-cluster with Java 8"
-  docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker
+  echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Building druid-cluster with default Java version"
+  docker build -t druid/cluster --build-arg MYSQL_VERSION $SHARED_DIR/docker
 else
   echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}"
   case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in
   8)
     echo "Build druid-cluster with Java 8"
-    docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker
+    docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker
     ;;
   11)
     echo "Build druid-cluster with Java 11"
-    docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-11.0.5-1 $SHARED_DIR/docker
+    docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker
     ;;
   *)
     echo "Invalid JVM Runtime given. Stopping"