Docker build for the revised ITs

.travis.yml

@@ -46,7 +46,7 @@ addons:
 # Add various options to make 'mvn install' fast and skip javascript compile (-Ddruid.console.skip=true) since it is not
 # needed. Depending on network speeds, "mvn -q install" may take longer than the default 10 minute timeout to print any
 # output.  To compensate, use travis_wait to extend the timeout.
-install: ./check_test_suite.py && travis_terminate 0  || echo 'Running Maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+install: ./check_test_suite.py && travis_terminate 0  || echo 'Running Maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution,!:it-tools,!:it-image' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
 
 # There are 3 stages of tests
 # 1. Tests - phase 1
@@ -72,7 +72,7 @@ jobs:
     - name: "animal sniffer checks"
       stage: Tests - phase 1
       script: ${MVN} animal-sniffer:check --fail-at-end
-
+      
     - name: "checkstyle"
       script: ${MVN} checkstyle:checkstyle --fail-at-end
 
@@ -347,7 +347,7 @@ jobs:
       <<: *test_processing_module
       name: "(openjdk8) other modules test"
       env:
-        - MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests'
+        - MAVEN_PROJECTS='!processing,!indexing-hadoop,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-core/kinesis-indexing-service,!server,!web-console,!integration-tests,!:it-image,!:it-tools'
 
     - <<: *test_other_modules
       name: "(openjdk11) other modules test"
@@ -755,6 +755,21 @@ jobs:
       env: TESTNG_GROUPS='-Dgroups=query' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' MYSQL_DRIVER_CLASSNAME='org.mariadb.jdbc.Driver' OVERRIDE_CONFIG_PATH='./environment-configs/test-groups/prepopulated-data'
 
     # END - Integration tests for Compile with Java 8 and Run with Java 11
+
+    # BEGIN - Revised integration tests
+
+    # Experimental build of the revised integration test Docker image.
+    # Actual tests will come later.
+    - name: "experimental docker tests"
+      stage: Tests - phase 2
+      # Uses the install defined above. Then, builds the test tools and docker image,
+      # and run the various IT tests. If tests fail, echos log lines of any of
+      # the Druid services that did not exit normally.
+      # Run though install to ensure the test tools are installed, and the docker
+      # image is built. The tests only need verify.
+      script: ${MVN} install -P dist,test-image -rf :distribution ${MAVEN_SKIP} -DskipUTs=true
+
+    # END - Revised integration tests
 
     - &integration_batch_index_k8s
       name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test"

integration-tests-ex/.gitignore

@@ -0,0 +1 @@
+/bin/

integration-tests-ex/README.md

@@ -0,0 +1,35 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+# Revised Integration Tests
+
+This directory builds a Docker image for Druid. Later revisions
+use the image to run revised integration tests.
+
+The `it-base` project is built as part of the normal build,
+though it is used only for the Docker image.
+
+To build the image:
+
+```bash
+mvn $USUAL_CAVEATS -P test-image
+```
+
+Where `$USUAL_CAVEATS` are your favorite options to turn
+off static checks, UTs, etc.

integration-tests-ex/assets/log4j2.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<Configuration status="WARN">
+  <Appenders>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="info">
+      <AppenderRef ref="Console"/>
+    </Root>
+  </Loggers>
+</Configuration>

integration-tests-ex/check-results.sh

@@ -0,0 +1,89 @@
+#! /bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#--------------------------------------------------------------------
+
+# Run from Travis which has no good way to attach logs to a
+# build. Instead, we check if any IT failed. If so, we append
+# the last 100 lines of each server log to stdout. We have to
+# stay wihtin the 4MB limit which Travis applies, so we only
+# emit logs for the first failure, and only for servers that
+# don't report normal completion.
+#
+# The only good way to check for test failures is to parse
+# the Failsafe summary for each test located in
+# <project>/target/failsafe-reports/failsafe-summary.xml
+#
+# This directory has many subdirectories, some of which are
+# tests. We rely on the fact that a test starts with "it-" AND
+# contains a failsafe report. (Some projects start with "it-"
+# but are not tests.)
+
+# Run in the docker-tests directory
+cd $(dirname $0)
+
+# Scan for candidate projects
+for PROJECT in it-*
+do
+	# Check if a failsafe report exists. It will exist if the directory is
+	# a test project and failsafe ran on that directory.
+	REPORTS="$PROJECT/target/failsafe-reports/failsafe-summary.xml"
+	if [ -f "$REPORTS" ]
+	then
+		# OK, so Bash isn't the world's best text processing language...
+		ERRS=1
+		FAILS=1
+		while IFS= read -r line
+	    do
+	      	if [ "$line" = "    <errors>0</errors>" ]
+	      	then
+	      		ERRS=0
+	      	fi
+	      	if [ "$line" = "    <failures>0</failures>" ]
+	      	then
+	      		FAILS=0
+	      	fi
+		done < "$REPORTS"
+		if [ $ERRS -eq 1 -o $FAILS -eq 1 ]
+	    then
+	    	FOUND_LOGS=0
+	    	echo "======= $PROJECT Failed =========="
+	    	# All logs except zookeeper
+	    	for log in $(ls $PROJECT/target/shared/logs/[a-y]*.log)
+	    	do
+	    		# We assume that a successful exit includes a line with the
+	    		# following:
+	    		# Stopping lifecycle [module] stage [INIT]
+	    		tail -5 "$log" | grep -Fq 'Stopping lifecycle [module] stage [INIT]'
+	    		if [ $? -ne 0 ]
+	    		then
+	    			# Assume failure and report tail
+	    		   echo $(basename $log) "logtail ========================"
+	    		   tail -100 "$log"
+	    		   FOUND_LOGS=1
+	    		fi
+	        done
+
+	        # Only emit the first failure to avoid output bloat
+	        if [ $FOUND_LOGS -eq 1 ]
+	        then
+	    		exit 0
+	    	else
+	    		echo "All Druid services exited normally."
+	    	fi
+	    fi
+	fi
+done

integration-tests-ex/it-image/build-image.sh

@@ -0,0 +1,49 @@
+#! /usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#--------------------------------------------------------------------
+
+# Prepare the image contents and build the Druid image.
+# Since Docker requires all contents to be in or below the
+# working directory, we assemble the contents in target/docker.
+
+# Fail fast on any error
+set -e
+
+# Enable for tracing
+#set -x
+
+# Fail on unset environment variables
+set -u
+
+SCRIPT_DIR=$(cd $(dirname $0) && pwd)
+
+# Copy environment variables to a file. Used for manual rebuilds
+# and by scripts that start the test cluster.
+
+cat > $TARGET_DIR/env.sh << EOF
+export ZK_VERSION=$ZK_VERSION
+export KAFKA_VERSION=$KAFKA_VERSION
+export DRUID_VERSION=$DRUID_VERSION
+export MYSQL_VERSION=$MYSQL_VERSION
+export MYSQL_IMAGE_VERSION=$MYSQL_IMAGE_VERSION
+export CONFLUENT_VERSION=$CONFLUENT_VERSION
+export MARIADB_VERSION=$MARIADB_VERSION
+export HADOOP_VERSION=$HADOOP_VERSION
+export MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME
+export DRUID_IT_IMAGE_NAME=$DRUID_IT_IMAGE_NAME
+EOF
+
+exec bash $SCRIPT_DIR/docker-build.sh

integration-tests-ex/it-image/docker-build.sh

@@ -0,0 +1,56 @@
+#! /usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#--------------------------------------------------------------------
+
+# Invokes Docker to build the image. The environment should have been
+# setup from Maven via build-image.sh or manually via quick-build.sh.
+
+# Print environment for debugging
+#env
+
+# Enable for tracing
+#set -x
+
+SCRIPT_DIR=$(cd $(dirname $0) && pwd)
+
+# Maven should have created the docker dir with the needed
+# dependency jars. If doing this by hand, run Maven once to
+# populate these jars.
+if [ ! -d $TARGET_DIR/docker ]; then
+	echo "$TARGET_DIR/docker does not exist. It should contain dependency jars" 1>&2
+	exit 1
+fi
+
+# Create the run-specific docker directory
+mkdir -p $TARGET_DIR/docker
+cp -r docker/* $TARGET_DIR/docker
+cd $TARGET_DIR/docker
+
+# Grab the distribution if needed (skipped if no change.)
+DISTRIB_FILE=apache-druid-$DRUID_VERSION-bin.tar.gz
+SOURCE_FILE=$PARENT_DIR/distribution/target/$DISTRIB_FILE
+if [[ ! -f $DISTRIB_FILE || $SOURCE_FILE -nt $DISTRIB_FILE ]]; then
+	cp $SOURCE_FILE .
+fi
+
+docker build -t $DRUID_IT_IMAGE_NAME \
+	--build-arg DRUID_VERSION=$DRUID_VERSION \
+	--build-arg MYSQL_VERSION=$MYSQL_VERSION \
+	--build-arg MARIADB_VERSION=$MARIADB_VERSION \
+	--build-arg CONFLUENT_VERSION=$CONFLUENT_VERSION \
+	--build-arg HADOOP_VERSION=$HADOOP_VERSION \
+	--build-arg MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME \
+	.

integration-tests-ex/it-image/docker/Dockerfile

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#-------------------------------------------------------------------------
+
+# Builds the Druid image for testing. Does not include dependent
+# tools such as MySQL, Zookeeper or Kafka: those reside in their own images.
+#
+# The script assumes that Maven as placed a Druid distribution and the
+# required extra jars in the same location as this file: in the target/docker
+# directory. You must run Maven once to populate these files after each
+# build. After that first image build, you can use rebuild.sh to do follow-on
+# image builds.
+
+# This Dockerfile prefers to use the COPY command over ADD.
+# See: https://phoenixnap.com/kb/docker-add-vs-copy
+
+ARG JDK_VERSION=8-slim-buster
+
+# The FROM image provides Java on top of Debian, and
+# thus provides bash, apt-get, etc.
+# See https://hub.docker.com/_/openjdk
+
+FROM openjdk:$JDK_VERSION
+
+ARG DRUID_VERSION
+ENV DRUID_VERSION=$DRUID_VERSION
+ARG CONFLUENT_VERSION
+ENV CONFLUENT_VERSION=$CONFLUENT_VERSION
+ARG MYSQL_VERSION
+ENV MYSQL_VERSION=$MYSQL_VERSION
+ARG MARIADB_VERSION
+ENV MARIADB_VERSION=$MARIADB_VERSION
+ARG MYSQL_DRIVER_CLASSNAME=com.mysql.jdbc.Driver
+ENV MYSQL_DRIVER_CLASSNAME=$MYSQL_DRIVER_CLASSNAME
+
+ENV DRUID_HOME=/usr/local/druid
+
+# Populate build artifacts
+
+COPY apache-druid-${DRUID_VERSION}-bin.tar.gz /usr/local/
+COPY it-tools-${DRUID_VERSION}.jar /tmp/druid/extensions/it-tools/
+COPY kafka-protobuf-provider-${CONFLUENT_VERSION}.jar /tmp/druid/lib/
+COPY mysql-connector-java-${MYSQL_VERSION}.jar /tmp/druid/lib/
+COPY mariadb-java-client-${MARIADB_VERSION}.jar /tmp/druid/lib/
+COPY test-setup.sh /
+COPY druid.sh /
+COPY launch.sh /
+
+# Do the setup tasks. The tasks are done within a script, rather than
+# here, so they are easier to describe and debug. Turn on the "-x" flag
+# within the script to trace the steps if needed for debugging.
+
+RUN bash /test-setup.sh
+
+# Start in the Druid directory.
+
+USER druid:druid
+WORKDIR /
+ENTRYPOINT [ "bash", "/launch.sh" ]