Skip check for the past dbversion's migration existing if it is squashed

.github/actions/todb-tests/entrypoint.sh

@@ -18,6 +18,15 @@
 
 set -e
 
+files_changed_in_pr() {
+	if [[ "$GITHUB_REF" == refs/pull/*/merge ]]; then
+		pr_number="$(<<<"$GITHUB_REF" grep -o '[0-9]\+')"
+		files_changed="$(curl "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${pr_number}/files" | jq -r .[].filename)"
+	else
+		files_changed="$(git diff-tree --no-commit-id --name-only -r "$GITHUB_SHA")"
+	fi
+}
+
 CODE=0;
 
 migration_dirs=(traffic_ops/app/db/migrations traffic_ops/app/db/trafficvault/migrations);
@@ -69,7 +78,7 @@ for migration_dir in ${migration_dirs[@]}; do
 	done
 	mtime_length=${#mtime_array[@]}
 
-	if [[ $LATEST_FILE_TIME != ${mtime_array[$mtime_length-1]} ]]; then
+	if [[ $LATEST_FILE_TIME != ${mtime_array[$mtime_length-1]} ]] && <<<"$(files_changed_in_pr)" grep -q "^${LATEST_FILE}$"; then
 		echo "ERROR: latest added/modified file: $LATEST_FILE is not in the right order" >&2;
 		CODE=1;
 	fi

traffic_ops/app/db/admin.go

@@ -34,7 +34,9 @@ import (
 	"github.com/apache/trafficcontrol/lib/go-log"
 
 	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
 	_ "github.com/golang-migrate/migrate/v4/database/postgres"
+	"github.com/golang-migrate/migrate/v4/source"
 	_ "github.com/golang-migrate/migrate/v4/source/file"
 	"gopkg.in/yaml.v2"
 )
@@ -129,16 +131,17 @@ var (
 	DBVersionDirty bool
 
 	// globals that are parsed out of DBConfigFile and used in commands
-	DBDriver      string
-	DBName        string
-	DBSuperUser   = DefaultDBSuperUser
-	DBUser        string
-	DBPassword    string
-	HostIP        string
-	HostPort      string
-	SSLMode       string
-	Migrate       *migrate.Migrate
-	MigrationName string
+	ConnectionString string
+	DBDriver         string
+	DBName           string
+	DBSuperUser      = DefaultDBSuperUser
+	DBUser           string
+	DBPassword       string
+	HostIP           string
+	HostPort         string
+	SSLMode          string
+	Migrate          *migrate.Migrate
+	MigrationName    string
 )
 
 func parseDBConfig() error {
@@ -330,13 +333,41 @@ func maybeMigrateFromGoose() bool {
 	if err := Migrate.Steps(1); err != nil {
 		die("Error migrating to Migrate from Goose: " + err.Error())
 	}
+	DBVersion, DBVersionDirty, _ = Migrate.Version()
 	return true
 }
 
+// runFirstMigration is essentially Migrate.Migrate(FirstMigrationTimestamp) but without the obligatory Migrate.versionExists() call.
+// If calling Migrate.versionExists() is made optional, runFirstMigration() can be replaced.
+func runFirstMigration() error {
+	sourceDriver, sourceDriverErr := source.Open(DBMigrationsSource)
+	if sourceDriverErr != nil {
+		return fmt.Errorf("opening the migration source driver: " + sourceDriverErr.Error())
+	}
+	dbDriver, dbDriverErr := database.Open(ConnectionString)
+	if dbDriverErr != nil {
+		return fmt.Errorf("opening the dbdriver: " + dbDriverErr.Error())
+	}
+	firstMigration, firstMigrationName, migrationReadErr := sourceDriver.ReadUp(FirstMigrationTimestamp)
+	if migrationReadErr != nil {
+		return fmt.Errorf("reading migration %s: %s", firstMigrationName, migrationReadErr.Error())
+	}
+	if setDirtyVersionErr := dbDriver.SetVersion(int(FirstMigrationTimestamp), true); setDirtyVersionErr != nil {
+		return fmt.Errorf("setting the dirty version: %s", setDirtyVersionErr.Error())
+	}
+	if migrateErr := dbDriver.Run(firstMigration); migrateErr != nil {
+		return fmt.Errorf("running the migration: %s", migrateErr.Error())
+	}
+	if setVersionErr := dbDriver.SetVersion(int(FirstMigrationTimestamp), false); setVersionErr != nil {
+		return fmt.Errorf("setting the version after successfully running the migration: %s", setVersionErr.Error())
+	}
+	return nil
+}
+
 func runMigrations() {
 	migratedFromGoose := initMigrate()
 	if !TrafficVault && DBVersion == LastSquashedMigrationTimestamp && !DBVersionDirty {
-		if migrateErr := Migrate.Migrate(FirstMigrationTimestamp); migrateErr != nil {
+		if migrateErr := runFirstMigration(); migrateErr != nil {
 			die(fmt.Sprintf("Error migrating from DB version %d to %d: %s", LastSquashedMigrationTimestamp, FirstMigrationTimestamp, migrateErr.Error()))
 		}
 	}
@@ -569,11 +600,11 @@ func main() {
 
 func initMigrate() bool {
 	var err error
-	connectionString := fmt.Sprintf("%s://%s:%s@%s:%s/%s?sslmode=%s", DBDriver, DBUser, DBPassword, HostIP, HostPort, DBName, SSLMode)
+	ConnectionString = fmt.Sprintf("%s://%s:%s@%s:%s/%s?sslmode=%s", DBDriver, DBUser, DBPassword, HostIP, HostPort, DBName, SSLMode)
 	if TrafficVault {
-		Migrate, err = migrate.New(TrafficVaultMigrationsSource, connectionString)
+		Migrate, err = migrate.New(TrafficVaultMigrationsSource, ConnectionString)
 	} else {
-		Migrate, err = migrate.New(DBMigrationsSource, connectionString)
+		Migrate, err = migrate.New(DBMigrationsSource, ConnectionString)
 	}
 	if err != nil {
 		die("Starting Migrate: " + err.Error())

traffic_ops_db/test/docker/Dockerfile-db-admin

@@ -23,25 +23,43 @@ FROM centos:7
 ARG POSTGRES_VERSION=13.2
 ENV POSTGRES_VERSION=$POSTGRES_VERSION
 
-# NOTE: temporary workaround for removal of golang packages from CentOS 7 base repo
 RUN yum install -y \
-    epel-release \
-    centos-release-scl-rh \
-    https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm \
+		epel-release \
+		centos-release-scl-rh \
+		https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm \
     git && \
-    yum -y install golang
+    yum -y install           \
+        cpanminus            \
+        expat-devel          \
+        libcap               \
+        libcurl-devel        \
+        libidn-devel         \
+        libpcap-devel        \
+        mkisofs              \
+        openssl-devel        \
+        perl-core            \
+        perl-Crypt-ScryptKDF \
+        perl-DBD-Pg          \
+        perl-DBI             \
+        perl-Digest-SHA1     \
+        perl-JSON            \
+        perl-libwww-perl     \
+        perl-TermReadKey     \
+        perl-Test-CPAN-Meta  \
+        perl-WWW-Curl        \
+        postgresql13         \
+        postgresql13-devel &&\
+    yum clean all
 
 # Override TRAFFIC_OPS_RPM arg to use a different one using --build-arg TRAFFIC_OPS_RPM=...  Can be local file or http://...
 ARG TRAFFIC_OPS_RPM=traffic_ops.rpm
 ADD $TRAFFIC_OPS_RPM /
-RUN yum install -y \
-        /$(basename $TRAFFIC_OPS_RPM) && \
-        rm /$(basename $TRAFFIC_OPS_RPM) && \
-    yum clean all
+RUN rpm -Uvh $(basename $TRAFFIC_OPS_RPM) && \
+    rm $(basename $TRAFFIC_OPS_RPM)
 
 WORKDIR /opt/traffic_ops/app
 
-ADD run-db-test.sh \
+COPY run-db-test.sh \
     db-config.sh \
     /
 

traffic_ops_db/test/docker/run-db-test.sh

@@ -132,7 +132,7 @@ fi
 # test full restoration of the initial DB dump
 for d in $(get_db_dumps); do
     echo "testing restoration of DB dump: $d"
-    dropdb --echo --if-exists < "$d" > /dev/null || echo "Dropping DB ${DB_NAME} failed: $d"
+    dropdb --echo --if-exists "$DB_NAME" < "$d" > /dev/null || echo "Dropping DB ${DB_NAME} failed: $d"
     createdb --echo < "$d" > /dev/null || echo "Creating DB ${DB_NAME} failed: $d"
     pg_restore --verbose --clean --if-exists --exit-on-error -d "$DB_NAME" < "$d" > /dev/null || { echo "DB restoration failed: $d"; exit 1; }
 done