diff --git a/integration-tests-ex/cases/cluster.sh b/integration-tests-ex/cases/cluster.sh index f7c6cb92f0bf..dfe1037b43ca 100755 --- a/integration-tests-ex/cases/cluster.sh +++ b/integration-tests-ex/cases/cluster.sh @@ -27,15 +27,17 @@ set -e # Enable for debugging #set -x -export MODULE_DIR=$(cd $(dirname $0) && pwd) +export BASE_MODULE_DIR=$(cd $(dirname $0) && pwd) + +# The location of the tests, which may be different than +# the location of this file. +export MODULE_DIR=${IT_MODULE_DIR:-$BASE_MODULE_DIR} function usage { cat <&2 exit 1 @@ -127,33 +129,33 @@ function show_status { function build_shared_dir { mkdir -p $SHARED_DIR # Must start with an empty DB to keep MySQL happy - rm -rf $SHARED_DIR/db + sudo rm -rf $SHARED_DIR/db mkdir -p $SHARED_DIR/logs mkdir -p $SHARED_DIR/tasklogs mkdir -p $SHARED_DIR/db mkdir -p $SHARED_DIR/kafka mkdir -p $SHARED_DIR/resources - cp $MODULE_DIR/assets/log4j2.xml $SHARED_DIR/resources + cp $BASE_MODULE_DIR/assets/log4j2.xml $SHARED_DIR/resources # Permissions in some build setups are screwed up. See above. The user # which runs Docker does not have permission to write into the /shared # directory. Force ownership to allow writing. - chmod -R a+rwx $SHARED_DIR + sudo chmod -R a+rwx $SHARED_DIR } # Either generate the docker-compose file, or use "static" versions. function docker_file { - # If a template exists, generate the docker-compose.yaml file. Copy over the Common - # folder. - TEMPLATE_DIR=$MODULE_DIR/templates - TEMPLATE_SCRIPT=${DRUID_INTEGRATION_TEST_GROUP}.py - if [ -f "$TEMPLATE_DIR/$TEMPLATE_SCRIPT" ]; then + # If a template exists, generate the docker-compose.yaml file. + # Copy over the Common folder. + TEMPLATE_SCRIPT=docker-compose.py + if [ -f "$CLUSTER_DIR/$TEMPLATE_SCRIPT" ]; then + export PYTHONPATH=$BASE_MODULE_DIR/cluster export COMPOSE_DIR=$TARGET_DIR/cluster/$DRUID_INTEGRATION_TEST_GROUP mkdir -p $COMPOSE_DIR - pushd $TEMPLATE_DIR > /dev/null + pushd $CLUSTER_DIR > /dev/null python3 $TEMPLATE_SCRIPT popd > /dev/null - cp -r $MODULE_DIR/cluster/Common $TARGET_DIR/cluster + cp -r $BASE_MODULE_DIR/cluster/Common $TARGET_DIR/cluster else # Else, use the existing non-template file in place. if [ ! -d $CLUSTER_DIR ]; then @@ -205,6 +207,13 @@ function verify_docker_file { fi } +function run_setup { + SETUP_SCRIPT="$CLUSTER_DIR/setup.sh" + if [ -f "$SETUP_SCRIPT" ]; then + source "$SETUP_SCRIPT" + fi +} + # Determine if docker-compose is available. If not, assume Docker supports # the compose subcommand set +e @@ -219,17 +228,6 @@ set -e # Print environment for debugging #env -# Determine if docker-compose is available. If not, assume Docker supports -# the compose subcommand -set +e -if which docker-compose > /dev/null -then - DOCKER_COMPOSE='docker-compose' -else - DOCKER_COMPOSE='docker compose' -fi -set -e - case $CMD in "-h" ) usage @@ -238,17 +236,11 @@ case $CMD in usage $DOCKER_COMPOSE help ;; - "prepare" ) - check_env_file - category $* - build_shared_dir - docker_file - ;; "gen" ) category $* build_shared_dir docker_file - echo "Generated file is in $COMPOSE_DIR" + echo "Generated file is $COMPOSE_DIR/docker-compose.yaml" ;; "up" ) check_env_file @@ -256,6 +248,7 @@ case $CMD in echo "Starting cluster $DRUID_INTEGRATION_TEST_GROUP" build_shared_dir docker_file + run_setup cd $COMPOSE_DIR $DOCKER_COMPOSE $DOCKER_ARGS up -d # Enable the following for debugging diff --git a/integration-tests-ex/cases/templates/AzureDeepStorage.py b/integration-tests-ex/cases/cluster/AzureDeepStorage/docker-compose.py similarity index 100% rename from integration-tests-ex/cases/templates/AzureDeepStorage.py rename to integration-tests-ex/cases/cluster/AzureDeepStorage/docker-compose.py diff --git a/integration-tests-ex/cases/cluster/AzureDeepStorage/verify.sh b/integration-tests-ex/cases/cluster/AzureDeepStorage/verify.sh new file mode 100644 index 000000000000..f12a2901479f --- /dev/null +++ b/integration-tests-ex/cases/cluster/AzureDeepStorage/verify.sh @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#-------------------------------------------------------------------- + +require_env_var AZURE_ACCOUNT +require_env_var AZURE_KEY +require_env_var AZURE_CONTAINER diff --git a/integration-tests-ex/cases/templates/BatchIndex.py b/integration-tests-ex/cases/cluster/BatchIndex/docker-compose.py similarity index 100% rename from integration-tests-ex/cases/templates/BatchIndex.py rename to integration-tests-ex/cases/cluster/BatchIndex/docker-compose.py diff --git a/integration-tests-ex/cases/cluster/Common/dependencies.yaml b/integration-tests-ex/cases/cluster/Common/dependencies.yaml index 1f43dc5a751a..a4a37c7c979d 100644 --- a/integration-tests-ex/cases/cluster/Common/dependencies.yaml +++ b/integration-tests-ex/cases/cluster/Common/dependencies.yaml @@ -71,6 +71,7 @@ services: # platform: linux/x86_64 image: mysql:$MYSQL_IMAGE_VERSION container_name: metadata + restart: always command: - --character-set-server=utf8mb4 networks: @@ -79,7 +80,7 @@ services: ports: - 3306:3306 volumes: - - ${SHARED_DIR}/db:/var/lib/mysql + - ${SHARED_DIR}/db/init.sql:/docker-entrypoint-initdb.d/init.sql environment: MYSQL_ROOT_PASSWORD: driud MYSQL_DATABASE: druid diff --git a/integration-tests-ex/cases/templates/GcsDeepStorage.py b/integration-tests-ex/cases/cluster/GcsDeepStorage/docker-compose.py similarity index 100% rename from integration-tests-ex/cases/templates/GcsDeepStorage.py rename to integration-tests-ex/cases/cluster/GcsDeepStorage/docker-compose.py diff --git a/integration-tests-ex/cases/cluster/GcsDeepStorage/verify.sh b/integration-tests-ex/cases/cluster/GcsDeepStorage/verify.sh new file mode 100644 index 000000000000..a1e30ceba568 --- /dev/null +++ b/integration-tests-ex/cases/cluster/GcsDeepStorage/verify.sh @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#-------------------------------------------------------------------- + +require_env_var GOOGLE_BUCKET +require_env_var GOOGLE_PREFIX +require_env_var GOOGLE_APPLICATION_CREDENTIALS +if [ ! -f "$GOOGLE_APPLICATION_CREDENTIALS" ]; then + echo "Required file GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS is missing" 1>&2 + exit 1 +fi diff --git a/integration-tests-ex/cases/templates/HighAvailability.py b/integration-tests-ex/cases/cluster/HighAvailability/docker-compose.py similarity index 100% rename from integration-tests-ex/cases/templates/HighAvailability.py rename to integration-tests-ex/cases/cluster/HighAvailability/docker-compose.py diff --git a/integration-tests-ex/cases/templates/MultiStageQuery.py b/integration-tests-ex/cases/cluster/MultiStageQuery/docker-compose.py similarity index 100% rename from integration-tests-ex/cases/templates/MultiStageQuery.py rename to integration-tests-ex/cases/cluster/MultiStageQuery/docker-compose.py diff --git a/integration-tests-ex/cases/templates/S3DeepStorage.py b/integration-tests-ex/cases/cluster/S3DeepStorage/docker-compose.py similarity index 100% rename from integration-tests-ex/cases/templates/S3DeepStorage.py rename to integration-tests-ex/cases/cluster/S3DeepStorage/docker-compose.py diff --git a/integration-tests-ex/cases/cluster/S3DeepStorage/verify.sh b/integration-tests-ex/cases/cluster/S3DeepStorage/verify.sh new file mode 100644 index 000000000000..ba2b3e507ac1 --- /dev/null +++ b/integration-tests-ex/cases/cluster/S3DeepStorage/verify.sh @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#-------------------------------------------------------------------- + +require_env_var DRUID_CLOUD_BUCKET +require_env_var DRUID_CLOUD_PATH +require_env_var AWS_REGION +require_env_var AWS_ACCESS_KEY_ID +require_env_var AWS_SECRET_ACCESS_KEY diff --git a/integration-tests-ex/cases/templates/template.py b/integration-tests-ex/cases/cluster/template.py similarity index 97% rename from integration-tests-ex/cases/templates/template.py rename to integration-tests-ex/cases/cluster/template.py index 1be24ab032e4..015b5126e226 100644 --- a/integration-tests-ex/cases/templates/template.py +++ b/integration-tests-ex/cases/cluster/template.py @@ -23,7 +23,7 @@ PyYaml does the grunt work of converting the data structure to the YAML file. ''' -import yaml, os, os.path +import yaml, os from pathlib import Path # Constants used frequently in the template. @@ -49,15 +49,16 @@ def generate(template_path, template): ''' # Compute the cluster (test category) name from the template path which - # we assume to be module//