From 47d608546d776dda88c59cfc6d1c6c274ace7bff Mon Sep 17 00:00:00 2001 From: w41ter Date: Fri, 27 Oct 2023 21:31:56 +0800 Subject: [PATCH] [cases](regression-test) Add backup & restore with multi tables test --- .../doris/regression/suite/Syncer.groovy | 14 ++- .../backup_restore/test_backup_restore.groovy | 29 +++--- .../test_backup_restore_multi_tables.groovy | 99 +++++++++++++++++++ 3 files changed, 125 insertions(+), 17 deletions(-) create mode 100644 regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy index c7f5ccdcb8de2a..cf6a6bd6af849b 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy @@ -350,8 +350,11 @@ class Syncer { return isCheckedOK } - Boolean checkSnapshotFinish() { - String checkSQL = "SHOW BACKUP FROM " + context.db + Boolean checkSnapshotFinish(String dbName = null) { + if (dbName == null) { + dbName = context.db + } + String checkSQL = "SHOW BACKUP FROM ${dbName}" def records = suite.sql(checkSQL) for (row in records) { logger.info("BACKUP row is ${row}") @@ -385,8 +388,11 @@ class Syncer { null } - Boolean checkAllRestoreFinish() { - String checkSQL = "SHOW RESTORE FROM ${context.db}" + Boolean checkAllRestoreFinish(String dbName = null) { + if (dbName == null) { + dbName = context.db + } + String checkSQL = "SHOW RESTORE FROM ${dbName}" def records = suite.sql(checkSQL) for (row in records) { logger.info("Restore row is ${row}") diff --git a/regression-test/suites/backup_restore/test_backup_restore.groovy b/regression-test/suites/backup_restore/test_backup_restore.groovy index 3e5aa92ffbe745..0fccd42befda6d 100644 --- a/regression-test/suites/backup_restore/test_backup_restore.groovy +++ b/regression-test/suites/backup_restore/test_backup_restore.groovy @@ -17,14 +17,16 @@ suite("test_backup_restore", "backup_restore") { String repoName = "test_backup_restore_repo" + String dbName = "backup_restore_db" + String tableName = "test_backup_restore_table" def syncer = getSyncer() syncer.createS3Repository(repoName) - String tableName = "test_backup_restore_table" - sql "DROP TABLE IF EXISTS ${tableName}" + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" sql """ - CREATE TABLE ${tableName} ( + CREATE TABLE ${dbName}.${tableName} ( `id` LARGEINT NOT NULL, `count` LARGEINT SUM DEFAULT "0") AGGREGATE KEY(`id`) @@ -36,31 +38,31 @@ suite("test_backup_restore", "backup_restore") { """ List values = [] - for (i = 1; i <= 10; ++i) { + for (int i = 1; i <= 10; ++i) { values.add("(${i}, ${i})") } - sql "INSERT INTO ${tableName} VALUES ${values.join(",")}" - def result = sql "SELECT * FROM ${tableName}" + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + def result = sql "SELECT * FROM ${dbName}.${tableName}" assertEquals(result.size(), values.size()); String snapshotName = "test_backup_restore_snapshot" sql """ - BACKUP SNAPSHOT ${snapshotName} + BACKUP SNAPSHOT ${dbName}.${snapshotName} TO `${repoName}` ON (${tableName}) """ - while (!syncer.checkSnapshotFinish()) { + while (!syncer.checkSnapshotFinish(dbName)) { Thread.sleep(3000) } snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) assertTrue(snapshot != null) - sql "TRUNCATE TABLE ${tableName}" + sql "TRUNCATE TABLE ${dbName}.${tableName}" sql """ - RESTORE SNAPSHOT ${snapshotName} + RESTORE SNAPSHOT ${dbName}.${snapshotName} FROM `${repoName}` ON ( `${tableName}`) PROPERTIES @@ -70,14 +72,15 @@ suite("test_backup_restore", "backup_restore") { ) """ - while (!syncer.checkAllRestoreFinish()) { + while (!syncer.checkAllRestoreFinish(dbName)) { Thread.sleep(3000) } - result = sql "SELECT * FROM ${tableName}" + result = sql "SELECT * FROM ${dbName}.${tableName}" assertEquals(result.size(), values.size()); - sql "DROP TABLE ${tableName} FORCE" + sql "DROP TABLE ${dbName}.${tableName} FORCE" + sql "DROP DATABASE ${dbName} FORCE" sql "DROP REPOSITORY `${repoName}`" } diff --git a/regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy b/regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy new file mode 100644 index 00000000000000..109a4a13042a03 --- /dev/null +++ b/regression-test/suites/backup_restore/test_backup_restore_multi_tables.groovy @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_multi_tables", "backup_restore") { + String dbName = "backup_restore_multi_tables_db" + String suiteName = "test_backup_restore_multi_tables" + String repoName = "${suiteName}_repo" + String snapshotName = "${suiteName}_snapshot" + String tableNamePrefix = "${suiteName}_tables" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + + int numTables = 10; + int numRows = 10; + List tables = [] + for (int i = 0; i < numTables; ++i) { + String tableName = "${tableNamePrefix}_${i}" + tables.add(tableName) + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `count` LARGEINT SUM DEFAULT "0" + ) + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "replication_num" = "1" + ) + """ + List values = [] + for (int j = 1; j <= numRows; ++j) { + values.add("(${j}, ${j})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), numRows); + } + + def backupTables = tables[0..5] + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${backupTables.join(",")}) + """ + + while (!syncer.checkSnapshotFinish(dbName)) { + Thread.sleep(3000) + } + + snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + for (def tableName in backupTables) { + sql "TRUNCATE TABLE ${dbName}.${tableName}" + } + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON (${backupTables.join(",")}) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "replication_num" = "1" + ) + """ + + while (!syncer.checkAllRestoreFinish(dbName)) { + Thread.sleep(3000) + } + + for (def tableName in tables) { + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), numRows); + sql "DROP TABLE ${dbName}.${tableName} FORCE" + } + + sql "DROP DATABASE ${dbName} FORCE" + sql "DROP REPOSITORY `${repoName}`" +} +