From ea3959a6fdbd7dd970bb3fe4c32e777a1f79d8fa Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 22 Oct 2024 18:44:54 +0800 Subject: [PATCH 1/2] [Test](bloom filter) enhance bloom filter test case with retry logic (#42243) Fix case error as below ``` 2024-10-18 09:44:27.415 ERROR [suite-thread-2] (ScriptContext.groovy:122) - Run test_bloom_filter_hit_with_renamed_column in /home/ubuntu/regression-test/doris_branch-2.1/NEREIDS_ASAN/p0/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit_with_renamed_column.groovy failed org.opentest4j.AssertionFailedError: expected: but was: ``` ``` Exception in bloom_filter_p0/test_bloom_filter_drop_column.groovy(line 43): sql """INSERT INTO ${table_name} values ('1', '1')""" qt_select """select * from ${table_name} order by a""" // drop column c1 sql """ALTER TABLE ${table_name} DROP COLUMN c1""" // show create table def res = sql """SHOW CREATE TABLE ${table_name}""" assert res[0][1].contains("\"bloom_filter_columns\" = \"\"") ^^^^^^^^^^^^^^^^^^^^^^^^^^ERROR LINE^^^^^^^^^^^^^^^^^^^^^^^^^^ // add new column c1 sql """ALTER TABLE ${table_name} ADD COLUMN c1 ARRAY""" // insert data sql """INSERT INTO ${table_name} values ('2', null)""" // select data qt_select """select * from ${table_name} order by a""" } Exception: Assertion failed: assert res[0][1].contains("\"bloom_filter_columns\" = \"\"") | | | | | | | false | | 'CREATE TABLE `test_bloom_filter_drop_column` (\n `a` varchar(150) NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`a`)\nDISTRIBUTED BY HASH(`a`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 3",\n"min_load_replica_num" = "-1",\n"bloom_filter_columns" = "c1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n);' | ['test_bloom_filter_drop_column', 'CREATE TABLE `test_bloom_filter_drop_column` (\n `a` varchar(150) NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`a`)\nDISTRIBUTED BY HASH(`a`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 3",\n"min_load_replica_num" = "-1",\n"bloom_filter_columns" = "c1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n);'] [['test_bloom_filter_drop_column', 'CREATE TABLE `test_bloom_filter_drop_column` (\n `a` varchar(150) NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`a`)\nDISTRIBUTED BY HASH(`a`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 3",\n"min_load_replica_num" = "-1",\n"bloom_filter_columns" = "c1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n);']] ``` --- .../test_bloom_filter_drop_column.groovy | 74 +++++++++++++++++++ ...loom_filter_hit_with_renamed_column.groovy | 57 ++++++++++---- 2 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy new file mode 100644 index 00000000000000..a2de2426832854 --- /dev/null +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_bloom_filter_drop_column") { + def table_name = "test_bloom_filter_drop_column" + + sql """drop TABLE if exists ${table_name}""" + + sql """CREATE TABLE IF NOT EXISTS ${table_name} ( + `a` varchar(150) NULL, + `c1` varchar(10) + ) ENGINE=OLAP + DUPLICATE KEY(`a`) + DISTRIBUTED BY HASH(`a`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "bloom_filter_columns" = "c1", + "in_memory" = "false", + "storage_format" = "V2" + )""" + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + + def wait_for_latest_op_on_table_finish = { tableName, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${tableName}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + sql """INSERT INTO ${table_name} values ('1', '1')""" + + qt_select """select * from ${table_name} order by a""" + + // drop column c1 + sql """ALTER TABLE ${table_name} DROP COLUMN c1""" + wait_for_latest_op_on_table_finish(table_name, timeout) + + // show create table + def res = sql """SHOW CREATE TABLE ${table_name}""" + log.info("show table:{}", res); + assert res[0][1].contains("\"bloom_filter_columns\" = \"\"") + + // add new column c1 + sql """ALTER TABLE ${table_name} ADD COLUMN c1 ARRAY""" + wait_for_latest_op_on_table_finish(table_name, timeout) + + // insert data + sql """INSERT INTO ${table_name} values ('2', null)""" + // select data + qt_select """select * from ${table_name} order by a""" +} diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit_with_renamed_column.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit_with_renamed_column.groovy index 5d174e702414d4..46d2e766109174 100644 --- a/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit_with_renamed_column.groovy +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit_with_renamed_column.groovy @@ -114,23 +114,52 @@ suite("test_bloom_filter_hit_with_renamed_column") { sql """ set parallel_scan_min_rows_per_scanner = 2097152; """ sql """ select C_COMMENT_NEW from ${tableName} where C_COMMENT_NEW='OK' """ - // get and check profile - def profileUrl = '/rest/v1/query_profile/' - def profiles = httpGet(profileUrl) - log.debug("profiles:{}", profiles); - profiles = new JsonSlurper().parseText(profiles) - assertEquals(0, profiles.code) - - def profileId = null; - for (def profile in profiles["data"]["rows"]) { - if (profile["Sql Statement"].contains("""select C_COMMENT_NEW from ${tableName} where C_COMMENT_NEW='OK'""")) { - profileId = profile["Profile ID"] - break; + // get and check profile with retry logic + def getProfileIdWithRetry = { query, maxRetries, waitSeconds -> + def profileUrl = '/rest/v1/query_profile/' + def profiles = null + def profileId = null + int attempt = 0 + + while (attempt < maxRetries) { + profiles = httpGet(profileUrl) + log.debug("profiles attempt ${attempt + 1}: {}", profiles) + if (profiles == null) { + log.warn("Failed to fetch profiles on attempt ${attempt + 1}") + } else { + def jsonProfiles = new JsonSlurper().parseText(profiles) + if (jsonProfiles.code == 0) { + for (def profile in jsonProfiles["data"]["rows"]) { + if (profile["Sql Statement"].contains(query)) { + profileId = profile["Profile ID"] + break + } + } + } else { + log.warn("Profile response code is not 0 on attempt ${attempt + 1}") + } + } + + if (profileId != null) { + break + } else { + attempt++ + if (attempt < maxRetries) { + log.info("profileId is null, retrying after ${waitSeconds} second(s)... (Attempt ${attempt + 1}/${maxRetries})") + sleep(waitSeconds * 1000) + } + } } + + assertTrue(profileId != null, "Failed to retrieve profileId after ${maxRetries} attempts") + return profileId } - log.info("profileId:{}", profileId); + + def query = """select C_COMMENT_NEW from ${tableName} where C_COMMENT_NEW='OK'""" + def profileId = getProfileIdWithRetry(query, 3, 1) + log.info("profileId:{}", profileId) def profileDetail = httpGet("/rest/v1/query_profile/" + profileId) - log.info("profileDetail:{}", profileDetail); + log.info("profileDetail:{}", profileDetail) assertTrue(profileDetail.contains("BloomFilterFiltered:  15.0K  (15000)")) //———————— clean table and disable profile ———————— From 1fc32526795d8c103346098ba29cbb51d19987e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Tue, 22 Oct 2024 18:48:07 +0800 Subject: [PATCH 2/2] update --- .../test_bloom_filter_drop_column.groovy | 74 ------------------- 1 file changed, 74 deletions(-) delete mode 100644 regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy deleted file mode 100644 index a2de2426832854..00000000000000 --- a/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -suite("test_bloom_filter_drop_column") { - def table_name = "test_bloom_filter_drop_column" - - sql """drop TABLE if exists ${table_name}""" - - sql """CREATE TABLE IF NOT EXISTS ${table_name} ( - `a` varchar(150) NULL, - `c1` varchar(10) - ) ENGINE=OLAP - DUPLICATE KEY(`a`) - DISTRIBUTED BY HASH(`a`) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "bloom_filter_columns" = "c1", - "in_memory" = "false", - "storage_format" = "V2" - )""" - def timeout = 60000 - def delta_time = 1000 - def alter_res = "null" - def useTime = 0 - - def wait_for_latest_op_on_table_finish = { tableName, OpTimeout -> - for(int t = delta_time; t <= OpTimeout; t += delta_time){ - alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${tableName}" ORDER BY CreateTime DESC LIMIT 1;""" - alter_res = alter_res.toString() - if(alter_res.contains("FINISHED")) { - sleep(3000) // wait change table state to normal - logger.info(table_name + " latest alter job finished, detail: " + alter_res) - break - } - useTime = t - sleep(delta_time) - } - assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") - } - sql """INSERT INTO ${table_name} values ('1', '1')""" - - qt_select """select * from ${table_name} order by a""" - - // drop column c1 - sql """ALTER TABLE ${table_name} DROP COLUMN c1""" - wait_for_latest_op_on_table_finish(table_name, timeout) - - // show create table - def res = sql """SHOW CREATE TABLE ${table_name}""" - log.info("show table:{}", res); - assert res[0][1].contains("\"bloom_filter_columns\" = \"\"") - - // add new column c1 - sql """ALTER TABLE ${table_name} ADD COLUMN c1 ARRAY""" - wait_for_latest_op_on_table_finish(table_name, timeout) - - // insert data - sql """INSERT INTO ${table_name} values ('2', null)""" - // select data - qt_select """select * from ${table_name} order by a""" -}