From 2349d156caeb6af8dd08b756a52febee7736f84e Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 26 Jul 2022 11:55:52 +0200 Subject: [PATCH] Fix too long names of files and docker compose projects When there is a change spanning multiple providers, the test type might be very long "Providers[google,microsoft.mssql......]". Test type is used in generating directory/file names as well as in determining docker compose project names and such long name might be just ... too long. But at any point in time we only run one Provider* test type, so we can simply truncate the "[*]" when we use TEST_TYPE to determine dir and docker-compose name. This PR does exactly this. --- Dockerfile.ci | 4 ++-- scripts/ci/testing/ci_run_airflow_testing.sh | 6 +++--- scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh | 6 +++--- scripts/docker/entrypoint_ci.sh | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index d48eb9eaf14db..c84c3804a30b5 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -740,7 +740,7 @@ if [[ "${RUN_TESTS}" != "true" ]]; then fi set -u -export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE}-${BACKEND}.xml" +export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE/\[*\]/}-${BACKEND}.xml" EXTRA_PYTEST_ARGS=( "--verbosity=0" @@ -782,7 +782,7 @@ if [[ ${ENABLE_TEST_COVERAGE:="false"} == "true" ]]; then EXTRA_PYTEST_ARGS+=( "--cov=airflow/" "--cov-config=.coveragerc" - "--cov-report=xml:/files/coverage-${TEST_TYPE}-${BACKEND}.xml" + "--cov-report=xml:/files/coverage-${TEST_TYPE/\[*\]/}-${BACKEND}.xml" ) fi diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh index 5c0c10c3f375a..fa5165b4704bb 100755 --- a/scripts/ci/testing/ci_run_airflow_testing.sh +++ b/scripts/ci/testing/ci_run_airflow_testing.sh @@ -39,9 +39,9 @@ function run_test_types_in_parallel() { for TEST_TYPE in ${test_types_to_run} do export TEST_TYPE - mkdir -p "${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${TEST_TYPE}" - export JOB_LOG="${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${TEST_TYPE}/stdout" - export PARALLEL_JOB_STATUS="${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${TEST_TYPE}/status" + mkdir -p "${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${TEST_TYPE/\[*\]/}" + export JOB_LOG="${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${TEST_TYPE/\[*\]/}/stdout" + export PARALLEL_JOB_STATUS="${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${TEST_TYPE/\[*\]/}/status" # Each test job will get SIGTERM followed by SIGTERM 200ms later and SIGKILL 200ms later after 45 mins # shellcheck disable=SC2086 parallel --ungroup --bg --semaphore --semaphorename "${SEMAPHORE_NAME}" \ diff --git a/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh b/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh index bc5bc041ed17d..f1afd2687f3e1 100755 --- a/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh +++ b/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh @@ -89,7 +89,7 @@ function run_airflow_testing_in_docker() { echo docker-compose -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ "${INTEGRATIONS[@]}" \ - --project-name "airflow-${TEST_TYPE,,}-${BACKEND}" \ + --project-name "airflow-${TEST_TYPE/\[*\]/}-${BACKEND}" \ down --remove-orphans \ --volumes --timeout 10 docker-compose --log-level INFO \ @@ -97,7 +97,7 @@ function run_airflow_testing_in_docker() { "${BACKEND_DOCKER_COMPOSE[@]}" \ "${INTEGRATIONS[@]}" \ "${DOCKER_COMPOSE_LOCAL[@]}" \ - --project-name "airflow-${TEST_TYPE,,}-${BACKEND}" \ + --project-name "airflow-${TEST_TYPE/\[*\]/}-${BACKEND}" \ run airflow "${@}" exit_code=$? docker ps @@ -112,7 +112,7 @@ function run_airflow_testing_in_docker() { docker-compose --log-level INFO -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ "${INTEGRATIONS[@]}" \ - --project-name "airflow-${TEST_TYPE,,}-${BACKEND}" \ + --project-name "airflow-${TEST_TYPE/\[*\]/}-${BACKEND}" \ down --remove-orphans \ --volumes --timeout 10 set -u diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index ef6d092779543..175c3ad8fd80d 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -244,7 +244,7 @@ if [[ "${RUN_TESTS}" != "true" ]]; then fi set -u -export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE}-${BACKEND}.xml" +export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE/\[*\]/}-${BACKEND}.xml" EXTRA_PYTEST_ARGS=( "--verbosity=0" @@ -286,7 +286,7 @@ if [[ ${ENABLE_TEST_COVERAGE:="false"} == "true" ]]; then EXTRA_PYTEST_ARGS+=( "--cov=airflow/" "--cov-config=.coveragerc" - "--cov-report=xml:/files/coverage-${TEST_TYPE}-${BACKEND}.xml" + "--cov-report=xml:/files/coverage-${TEST_TYPE/\[*\]/}-${BACKEND}.xml" ) fi