chore: define rules to reduce the running time of integration tests:

* run bash tests only when certain files(codebase excluding description files and docs) change
* remove only testing DLE Docker images
* reuse generated data for logical and physical modes
* increase timeout for the physical_basebackup case

Author: agneum

.gitlab-ci.yml

@@ -338,10 +338,23 @@ build-image-swagger-latest:
 
 
 .bash-test: &bash_test
-  <<: *only_feature
   stage: integration-test
   variables:
     IMAGE_TAG: "${CI_COMMIT_REF_SLUG}"
+  only:
+    refs:
+      - branches
+    variables:
+      - $CI_COMMIT_REF_SLUG != "master"
+    changes:
+      - cmd/**/*
+      - configs/**/*
+      - internal/**/*
+      - pkg/**/*
+      - test/**/*
+      - Dockerfile.*
+      - go.*
+      - .gitlab-ci.yml
   script:
     - bash test/1.synthetic.sh
     - bash test/2.logical_generic.sh

test/2.logical_generic.sh

@@ -12,7 +12,7 @@ export SOURCE_PORT="${SOURCE_PORT:-7432}"
 export SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
 export SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
 export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
-export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
+export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab_mount"
 export DLE_TEST_POOL_NAME="test_dblab_pool"
 export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
 export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
@@ -23,13 +23,7 @@ DIR=${0%/*}
 if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
 ### Step 0. Create source database
   TMP_DATA_DIR="/tmp/dle_test/logical_generic"
-  cleanup_testdata_dir() {
-    sudo rm -rf "${TMP_DATA_DIR}"/postgresql/"${POSTGRES_VERSION}"/test || true
-  }
-
-  trap cleanup_testdata_dir EXIT
 
-  cleanup_testdata_dir
   sudo docker rm postgres"${POSTGRES_VERSION}" || true
 
   sudo docker run \
@@ -59,12 +53,21 @@ if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
 
   check_database_readiness || (echo "test database is not ready" && exit 1)
 
-  # Generate data in the test database using pgbench
-  # 1,000,000 accounts, ~0.14 GiB of data.
-  sudo docker exec postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 "${SOURCE_DBNAME}"
+  check_data_existence(){
+    sudo docker exec postgres"${POSTGRES_VERSION}" psql -d "${SOURCE_DBNAME}" -U postgres --command 'select from pgbench_accounts' > /dev/null 2>&1
+    return $?
+  }
+
+  generate_data(){
+    # Generate data in the test database using pgbench
+    # 1,000,000 accounts, ~0.14 GiB of data.
+    sudo docker exec postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 "${SOURCE_DBNAME}"
+
+    # Database info
+    sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ ${SOURCE_DBNAME}"
+  }
 
-  # Database info
-  sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ ${SOURCE_DBNAME}"
+  check_data_existence || generate_data
 fi
 
 ### Step 1. Prepare a machine with disk, Docker, and ZFS

test/4.physical_basebackup.sh

@@ -11,7 +11,7 @@ export SOURCE_PORT="${SOURCE_PORT:-7432}"
 export SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
 export SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
 export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
-export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
+export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab_mount"
 export DLE_TEST_POOL_NAME="test_dblab_pool"
 export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
 export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
@@ -22,13 +22,7 @@ DIR=${0%/*}
 if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
 ### Step 0. Create source database
   TMP_DATA_DIR="/tmp/dle_test/physical_basebackup"
-  cleanup_testdata_dir() {
-    sudo rm -rf "${TMP_DATA_DIR}"/postgresql/"${POSTGRES_VERSION}"/test || true
-  }
-
-  trap cleanup_testdata_dir EXIT
 
-  cleanup_testdata_dir
   sudo docker rm postgres"${POSTGRES_VERSION}" || true
 
   sudo docker run \
@@ -74,12 +68,21 @@ if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
     done
   fi
 
-  # Generate data in the test database using pgbench
-  # 1,000,000 accounts, ~0.14 GiB of data.
-  sudo docker exec postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 test
+  check_data_existence(){
+    sudo docker exec postgres"${POSTGRES_VERSION}" psql -d test -U postgres --command 'select from pgbench_accounts' > /dev/null 2>&1
+    return $?
+  }
+
+  generate_data(){
+    # Generate data in the test database using pgbench
+    # 1,000,000 accounts, ~0.14 GiB of data.
+    sudo docker exec postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 test
+
+    # Database info
+    sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ test"
+  }
 
-  # Database info
-  sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ test"
+  check_data_existence || generate_data
 fi
 
 ### Step 1. Prepare a machine with disk, Docker, and ZFS
@@ -158,8 +161,8 @@ check_dle_readiness(){
   return 1
 }
 
-### Waiting for the Database Lab Engine initialization.
-for i in {1..30}; do
+### Waiting for the Database Lab Engine initialization (7 minutes).
+for i in {1..42}; do
   check_dle_readiness && break || echo "Database Lab Engine is not ready yet"
   sleep 10
 done

test/_cleanup.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euxo pipefail
 
-DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
+DLE_TEST_MOUNT_DIR="/var/lib/test/dblab_mount"
 DLE_TEST_POOL_NAME="test_dblab_pool"
 ZFS_FILE="$(pwd)/zfs_file"
 
@@ -10,7 +10,7 @@ sudo docker ps -aq --filter label="test_dblab_pool" | xargs --no-run-if-empty su
 sudo docker ps -aq --filter label="dblab_test" | xargs --no-run-if-empty sudo docker rm -f
 
 # Remove unused Docker images
-sudo docker images -q | xargs --no-run-if-empty sudo docker rmi || true
+sudo docker images --filter=reference='registry.gitlab.com/postgres-ai/database-lab/dblab-server:*' -q | xargs --no-run-if-empty sudo docker rmi || true
 
 # Clean up the data directory
 sudo rm -rf ${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/data/*

test/_zfs.file.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euxo pipefail
 
-DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
+DLE_TEST_MOUNT_DIR="/var/lib/test/dblab_mount"
 DLE_TEST_POOL_NAME="test_dblab_pool"
 ZFS_FILE="$(pwd)/zfs_file"