From 500a3360ca19d4b3ced3d8b0063b013a8789f06f Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 22 Oct 2024 18:44:54 +0800 Subject: [PATCH] [Test](bloom filter) enhance bloom filter test case with retry logic (#42243) ## Proposed changes Fix case error as below ``` 2024-10-18 09:44:27.415 ERROR [suite-thread-2] (ScriptContext.groovy:122) - Run test_bloom_filter_hit_with_renamed_column in /home/ubuntu/regression-test/doris_branch-2.1/NEREIDS_ASAN/p0/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit_with_renamed_column.groovy failed org.opentest4j.AssertionFailedError: expected: but was: ``` ``` Exception in bloom_filter_p0/test_bloom_filter_drop_column.groovy(line 43): sql """INSERT INTO ${table_name} values ('1', '1')""" qt_select """select * from ${table_name} order by a""" // drop column c1 sql """ALTER TABLE ${table_name} DROP COLUMN c1""" // show create table def res = sql """SHOW CREATE TABLE ${table_name}""" assert res[0][1].contains("\"bloom_filter_columns\" = \"\"") ^^^^^^^^^^^^^^^^^^^^^^^^^^ERROR LINE^^^^^^^^^^^^^^^^^^^^^^^^^^ // add new column c1 sql """ALTER TABLE ${table_name} ADD COLUMN c1 ARRAY""" // insert data sql """INSERT INTO ${table_name} values ('2', null)""" // select data qt_select """select * from ${table_name} order by a""" } Exception: Assertion failed: assert res[0][1].contains("\"bloom_filter_columns\" = \"\"") | | | | | | | false | | 'CREATE TABLE `test_bloom_filter_drop_column` (\n `a` varchar(150) NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`a`)\nDISTRIBUTED BY HASH(`a`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 3",\n"min_load_replica_num" = "-1",\n"bloom_filter_columns" = "c1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n);' | ['test_bloom_filter_drop_column', 'CREATE TABLE `test_bloom_filter_drop_column` (\n `a` varchar(150) NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`a`)\nDISTRIBUTED BY HASH(`a`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 3",\n"min_load_replica_num" = "-1",\n"bloom_filter_columns" = "c1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n);'] [['test_bloom_filter_drop_column', 'CREATE TABLE `test_bloom_filter_drop_column` (\n `a` varchar(150) NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`a`)\nDISTRIBUTED BY HASH(`a`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 3",\n"min_load_replica_num" = "-1",\n"bloom_filter_columns" = "c1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n);']] ``` --- .../test_bloom_filter_drop_column.groovy | 23 ++++++++ ...loom_filter_hit_with_renamed_column.groovy | 57 ++++++++++++++----- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy index a18ff9fdbe0d80..a2de2426832854 100644 --- a/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy @@ -31,19 +31,42 @@ suite("test_bloom_filter_drop_column") { "in_memory" = "false", "storage_format" = "V2" )""" + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + def wait_for_latest_op_on_table_finish = { tableName, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${tableName}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } sql """INSERT INTO ${table_name} values ('1', '1')""" qt_select """select * from ${table_name} order by a""" // drop column c1 sql """ALTER TABLE ${table_name} DROP COLUMN c1""" + wait_for_latest_op_on_table_finish(table_name, timeout) + // show create table def res = sql """SHOW CREATE TABLE ${table_name}""" + log.info("show table:{}", res); assert res[0][1].contains("\"bloom_filter_columns\" = \"\"") // add new column c1 sql """ALTER TABLE ${table_name} ADD COLUMN c1 ARRAY""" + wait_for_latest_op_on_table_finish(table_name, timeout) + // insert data sql """INSERT INTO ${table_name} values ('2', null)""" // select data diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit_with_renamed_column.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit_with_renamed_column.groovy index 5d174e702414d4..46d2e766109174 100644 --- a/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit_with_renamed_column.groovy +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit_with_renamed_column.groovy @@ -114,23 +114,52 @@ suite("test_bloom_filter_hit_with_renamed_column") { sql """ set parallel_scan_min_rows_per_scanner = 2097152; """ sql """ select C_COMMENT_NEW from ${tableName} where C_COMMENT_NEW='OK' """ - // get and check profile - def profileUrl = '/rest/v1/query_profile/' - def profiles = httpGet(profileUrl) - log.debug("profiles:{}", profiles); - profiles = new JsonSlurper().parseText(profiles) - assertEquals(0, profiles.code) - - def profileId = null; - for (def profile in profiles["data"]["rows"]) { - if (profile["Sql Statement"].contains("""select C_COMMENT_NEW from ${tableName} where C_COMMENT_NEW='OK'""")) { - profileId = profile["Profile ID"] - break; + // get and check profile with retry logic + def getProfileIdWithRetry = { query, maxRetries, waitSeconds -> + def profileUrl = '/rest/v1/query_profile/' + def profiles = null + def profileId = null + int attempt = 0 + + while (attempt < maxRetries) { + profiles = httpGet(profileUrl) + log.debug("profiles attempt ${attempt + 1}: {}", profiles) + if (profiles == null) { + log.warn("Failed to fetch profiles on attempt ${attempt + 1}") + } else { + def jsonProfiles = new JsonSlurper().parseText(profiles) + if (jsonProfiles.code == 0) { + for (def profile in jsonProfiles["data"]["rows"]) { + if (profile["Sql Statement"].contains(query)) { + profileId = profile["Profile ID"] + break + } + } + } else { + log.warn("Profile response code is not 0 on attempt ${attempt + 1}") + } + } + + if (profileId != null) { + break + } else { + attempt++ + if (attempt < maxRetries) { + log.info("profileId is null, retrying after ${waitSeconds} second(s)... (Attempt ${attempt + 1}/${maxRetries})") + sleep(waitSeconds * 1000) + } + } } + + assertTrue(profileId != null, "Failed to retrieve profileId after ${maxRetries} attempts") + return profileId } - log.info("profileId:{}", profileId); + + def query = """select C_COMMENT_NEW from ${tableName} where C_COMMENT_NEW='OK'""" + def profileId = getProfileIdWithRetry(query, 3, 1) + log.info("profileId:{}", profileId) def profileDetail = httpGet("/rest/v1/query_profile/" + profileId) - log.info("profileDetail:{}", profileDetail); + log.info("profileDetail:{}", profileDetail) assertTrue(profileDetail.contains("BloomFilterFiltered:  15.0K  (15000)")) //———————— clean table and disable profile ————————