diff --git a/regression-test/plugins/plugin_curl_requester.groovy b/regression-test/plugins/plugin_curl_requester.groovy index 77698a6a88fb31..531b2c192fb547 100644 --- a/regression-test/plugins/plugin_curl_requester.groovy +++ b/regression-test/plugins/plugin_curl_requester.groovy @@ -18,8 +18,99 @@ import org.apache.doris.regression.suite.Suite import org.apache.doris.regression.util.Http import org.apache.doris.regression.util.NodeType +@Grab(group='org.apache.httpcomponents', module='httpclient', version='4.5.13') +import org.apache.http.client.methods.* +import org.apache.http.impl.client.* +import org.apache.http.util.EntityUtils +import org.apache.http.client.config.RequestConfig +import org.apache.http.conn.ConnectTimeoutException +import org.apache.http.conn.HttpHostConnectException import org.codehaus.groovy.runtime.IOGroovyMethods +Suite.metaClass.http_client = { String method, String url /* param */ -> + Suite suite = delegate as Suite + if (method != "GET" && method != "POST") { + throw new Exception("Invalid method: ${method}") + } + if (!url || !(url =~ /^https?:\/\/.+/)) { + throw new Exception("Invalid url: ${url}") + } + + Integer timeout = 300 // seconds + Integer maxRetries = 10 + Integer retryCount = 0 + Integer sleepTime = 1000 // milliseconds + + logger.info("HTTP request: ${method} ${url}") + + CloseableHttpClient httpClient = HttpClients.custom() + .setRetryHandler(new DefaultHttpRequestRetryHandler(3, true)) + .build() + + int code + String err + String out + + try { + while (retryCount < maxRetries) { + HttpRequestBase request + if (method == "GET") { + request = new HttpGet(url) + } else if (method == "POST") { + request = new HttpPost(url) + } + + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(timeout * 1000) + .setSocketTimeout(timeout * 1000) + .build() + request.setConfig(requestConfig) + + try { + CloseableHttpResponse response = httpClient.execute(request) + try { + code = response.getStatusLine().getStatusCode() + out = EntityUtils.toString(response.getEntity()) + + if (code >= 200 && code < 300) { + code = 0 // to be compatible with the old curl function + err = "" + return [code, out, err] + } else if (code == 500) { + return [code, out, "Internal Server Error"] + } else { + logger.warn("HTTP request failed with status code ${code}, response ${out}, retrying (${++retryCount}/${maxRetries})") + } + } finally { + response.close() + } + } catch (ConnectTimeoutException | HttpHostConnectException e) { + logger.warn("Connection failed, retrying (${++retryCount}/${maxRetries}): ${e.message}") + } catch (SocketTimeoutException e) { + timeout = timeout + 10 + logger.warn("Read timed out, retrying (${++retryCount}/${maxRetries}): ${e.message}") + } catch (Exception e) { + code = 500 // Internal Server Error + logger.error("Error executing HTTP request: ${e.message}") + err = e.message + return [code, out, err] + } + + sleep(sleepTime) + sleepTime = Math.min(sleepTime * 2, 60000) + } + + logger.error("HTTP request failed after ${maxRetries} attempts") + err = "Failed after ${maxRetries} attempts" + code = 500 // Internal Server Error + return [code, out, err] + } finally { + httpClient.close() + } +} + +logger.info("Added 'http_client' function to Suite") + Suite.metaClass.curl = { String method, String url /* param */-> Suite suite = delegate as Suite if (method != "GET" && method != "POST") @@ -161,3 +252,43 @@ Suite.metaClass.be_report_task = { String ip, int port -> } logger.info("Added 'be_report_task' function to Suite") + +// check nested index file api +Suite.metaClass.check_nested_index_file = { ip, port, tablet_id, expected_rowsets_count, expected_indices_count, format -> + def (code, out, err) = http_client("GET", String.format("http://%s:%s/api/show_nested_index_file?tablet_id=%s", ip, port, tablet_id)) + logger.info("Run show_nested_index_file_on_tablet: code=" + code + ", out=" + out + ", err=" + err) + // only when the expected_indices_count is 0, the tablet may not have the index file. + if (code == 500 && expected_indices_count == 0) { + assertEquals("E-6003", parseJson(out.trim()).status) + assertTrue(parseJson(out.trim()).msg.contains("not found")) + return + } + assertTrue(code == 0) + assertEquals(tablet_id, parseJson(out.trim()).tablet_id.toString()) + def rowsets_count = parseJson(out.trim()).rowsets.size(); + assertEquals(expected_rowsets_count, rowsets_count) + def index_files_count = 0 + def segment_files_count = 0 + for (def rowset in parseJson(out.trim()).rowsets) { + assertEquals(format, rowset.index_storage_format) + for (int i = 0; i < rowset.segments.size(); i++) { + def segment = rowset.segments[i] + assertEquals(i, segment.segment_id) + def indices_count = segment.indices.size() + assertEquals(expected_indices_count, indices_count) + if (format == "V1") { + index_files_count += indices_count + } else { + index_files_count++ + } + } + segment_files_count += rowset.segments.size() + } + if (format == "V1") { + assertEquals(index_files_count, segment_files_count * expected_indices_count) + } else { + assertEquals(index_files_count, segment_files_count) + } +} + +logger.info("Added 'check_nested_index_file' function to Suite") \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy index 430c3a67e0a89b..dabb3534e32e20 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_add_build_index_with_format_v2.groovy @@ -20,9 +20,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_add_build_index_with_format_v2", "inverted_index_format_v2"){ def tableName = "test_add_build_index_with_format_v2" - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); @@ -115,22 +112,15 @@ suite("test_add_build_index_with_format_v2", "inverted_index_format_v2"){ String backend_id = tablets[0].BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) + // cloud mode is directly schema change, local mode is light schema change. // cloud mode is 12, local mode is 6 if (isCloudMode()) { - assertEquals("12", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 7, 2, "V2") qt_sql "SELECT * FROM $tableName WHERE name match 'andy' order by id, name, score;" return } else { - assertEquals("6", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 7, 0, "V2") } // build index @@ -139,15 +129,7 @@ suite("test_add_build_index_with_format_v2", "inverted_index_format_v2"){ """ wait_for_build_index_on_partition_finish(tableName, timeout) - (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - assertEquals("12", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 7, 1, "V2") // build index sql """ @@ -155,15 +137,7 @@ suite("test_add_build_index_with_format_v2", "inverted_index_format_v2"){ """ wait_for_build_index_on_partition_finish(tableName, timeout) - (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - assertEquals("12", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 7, 2, "V2") qt_sql "SELECT * FROM $tableName WHERE name match 'andy' order by id, name, score;" } diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_create_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_create_table_with_format_v2.groovy index dd87a214e2d356..cf31e2ca80fe71 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_create_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_create_table_with_format_v2.groovy @@ -20,9 +20,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_create_table_with_format_v2", "inverted_index_format_v2"){ def tableName = "test_create_table_with_format_v2" - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); @@ -60,13 +57,5 @@ suite("test_create_table_with_format_v2", "inverted_index_format_v2"){ String backend_id = tablets[0].BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - assertEquals("12", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 7, 2, "V2") } diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy index f8ae36175a58c0..65270ebe597d31 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy @@ -40,10 +40,6 @@ suite("test_cumulative_compaction_with_format_v2", "inverted_index_format_v2") { assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } - def calc_segment_count = { tablet -> int segment_count = 0 String tablet_id = tablet.TabletId @@ -173,16 +169,7 @@ suite("test_cumulative_compaction_with_format_v2", "inverted_index_format_v2") { String port = backendId_to_backendHttpPort.get(backend_id) int segment_count = calc_segment_count(tablet) logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" + o + ", err=" + e) - assertTrue(c == 0) - assertTrue(o.contains("crc_value")) - assertTrue(o.contains("used_time_ms")) - assertEquals("0", parseJson(o.trim()).start_version) - assertEquals("9", parseJson(o.trim()).end_version) - assertEquals("9", parseJson(o.trim()).rowset_count) - int file_count = segment_count * 2 - assertEquals(file_count, Integer.parseInt(parseJson(o.trim()).file_count)) + check_nested_index_file(ip, port, tablet_id, 9, 3, "V2") StringBuilder sb = new StringBuilder(); sb.append("curl -X POST http://") @@ -242,17 +229,7 @@ suite("test_cumulative_compaction_with_format_v2", "inverted_index_format_v2") { String port = backendId_to_backendHttpPort.get(backend_id) int segment_count = calc_segment_count(tablet) logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" + o + ", err=" + e) - assertTrue(c == 0) - assertTrue(o.contains("crc_value")) - assertTrue(o.contains("used_time_ms")) - assertEquals("0", parseJson(o.trim()).start_version) - assertEquals("9", parseJson(o.trim()).end_version) - // after compaction, there are 2 rwosets. - assertEquals("2", parseJson(o.trim()).rowset_count) - int file_count = segment_count * 2 - assertEquals(file_count, Integer.parseInt(parseJson(o.trim()).file_count)) + check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") } int segmentsCount = 0 diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy index 24ac5c6ee6fa55..1a75137e1e40ba 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy @@ -25,9 +25,6 @@ suite("test_drop_column_with_format_v2", "inverted_index_format_v2"){ } def tableName = "test_drop_column_with_format_v2" - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); @@ -106,50 +103,23 @@ suite("test_drop_column_with_format_v2", "inverted_index_format_v2"){ String backend_id = tablets[0].BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - assertEquals("12", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 7, 2, "V2") // drop column sql """ ALTER TABLE ${tableName} DROP COLUMN score; """ wait_for_latest_op_on_table_finish(tableName, timeout) - // select to sync rowset meta in cloud mode - sql """ select * from ${tableName} limit 1; """ - tablets = sql_return_maparray """ show tablets from ${tableName}; """ tablet_id = tablets[0].TabletId - (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - assertEquals("12", parseJson(out.trim()).file_count) + // when drop column, the index files will not be deleted, so the index files count is still 2 + check_nested_index_file(ip, port, tablet_id, 7, 2, "V2") sql """ ALTER TABLE ${tableName} DROP COLUMN name; """ wait_for_latest_op_on_table_finish(tableName, timeout) - // select to sync rowset meta in cloud mode - sql """ select * from ${tableName} limit 1; """ - tablets = sql_return_maparray """ show tablets from ${tableName}; """ tablet_id = tablets[0].TabletId - (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - assertEquals("6", parseJson(out.trim()).file_count) + // when drop column, the index files will not be deleted, so the index files count is still 2 + // when all index columns are dropped, the index files will be deleted by GC later + check_nested_index_file(ip, port, tablet_id, 7, 2, "V2") } diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy index c2ce451e857049..12877099f38da5 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy @@ -25,9 +25,6 @@ suite("test_drop_index_with_format_v2", "inverted_index_format_v2"){ } def tableName = "test_drop_index_with_format_v2" - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); @@ -85,44 +82,15 @@ suite("test_drop_index_with_format_v2", "inverted_index_format_v2"){ String backend_id = tablets[0].BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - assertEquals("12", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 7, 2, "V2") // drop index sql """ DROP INDEX index_name on ${tableName}; """ wait_for_latest_op_on_table_finish(tableName, timeout) - - (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - assertEquals("12", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 7, 1, "V2") // drop index sql """ DROP INDEX index_score on ${tableName}; """ wait_for_latest_op_on_table_finish(tableName, timeout) - - // select to sync rowset meta in cloud mode - sql """ select * from ${tableName} limit 1; """ - - (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - assertEquals("6", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 7, 0, "V2") } diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy index 700cb6abc32221..b20148e8bd595a 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy @@ -40,10 +40,6 @@ suite("test_mor_table_with_format_v2", "inverted_index_format_v2") { assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } - def calc_segment_count = { tablet -> int segment_count = 0 String tablet_id = tablet.TabletId @@ -175,16 +171,7 @@ suite("test_mor_table_with_format_v2", "inverted_index_format_v2") { String port = backendId_to_backendHttpPort.get(backend_id) int segment_count = calc_segment_count(tablet) logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" + o + ", err=" + e) - assertTrue(c == 0) - assertTrue(o.contains("crc_value")) - assertTrue(o.contains("used_time_ms")) - assertEquals("0", parseJson(o.trim()).start_version) - assertEquals("10", parseJson(o.trim()).end_version) - assertEquals("10", parseJson(o.trim()).rowset_count) - int file_count = segment_count * 2 - assertEquals(file_count, Integer.parseInt(parseJson(o.trim()).file_count)) + check_nested_index_file(ip, port, tablet_id, 10, 3, "V2") StringBuilder sb = new StringBuilder(); sb.append("curl -X POST http://") @@ -242,23 +229,12 @@ suite("test_mor_table_with_format_v2", "inverted_index_format_v2") { String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - int segment_count = calc_segment_count(tablet) - logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" + o + ", err=" + e) - assertTrue(c == 0) - assertTrue(o.contains("crc_value")) - assertTrue(o.contains("used_time_ms")) - assertEquals("0", parseJson(o.trim()).start_version) - assertEquals("10", parseJson(o.trim()).end_version) // after compaction, there are 1 rwoset in local mode and 2 rowsets in cloud mode. if (isCloudMode()) { - assertEquals("2", parseJson(o.trim()).rowset_count) + check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") } else { - assertEquals("1", parseJson(o.trim()).rowset_count) + check_nested_index_file(ip, port, tablet_id, 1, 3, "V2") } - int file_count = segment_count * 2 - assertEquals(file_count, Integer.parseInt(parseJson(o.trim()).file_count)) } int segmentsCount = 0 diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy index e38f7bfe5e0f1d..568990c8b4d18e 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy @@ -40,10 +40,6 @@ suite("test_mow_table_with_format_v2", "inverted_index_format_v2") { assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } - def calc_segment_count = { tablet -> int segment_count = 0 String tablet_id = tablet.TabletId @@ -173,16 +169,7 @@ suite("test_mow_table_with_format_v2", "inverted_index_format_v2") { String port = backendId_to_backendHttpPort.get(backend_id) int segment_count = calc_segment_count(tablet) logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" + o + ", err=" + e) - assertTrue(c == 0) - assertTrue(o.contains("crc_value")) - assertTrue(o.contains("used_time_ms")) - assertEquals("0", parseJson(o.trim()).start_version) - assertEquals("9", parseJson(o.trim()).end_version) - assertEquals("9", parseJson(o.trim()).rowset_count) - int file_count = segment_count * 2 - assertEquals(file_count, Integer.parseInt(parseJson(o.trim()).file_count)) + check_nested_index_file(ip, port, tablet_id, 9, 3, "V2") StringBuilder sb = new StringBuilder(); sb.append("curl -X POST http://") @@ -240,19 +227,7 @@ suite("test_mow_table_with_format_v2", "inverted_index_format_v2") { String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - int segment_count = calc_segment_count(tablet) - logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" + o + ", err=" + e) - assertTrue(c == 0) - assertTrue(o.contains("crc_value")) - assertTrue(o.contains("used_time_ms")) - assertEquals("0", parseJson(o.trim()).start_version) - assertEquals("9", parseJson(o.trim()).end_version) - // after compaction, there are 2 rwosets. - assertEquals("2", parseJson(o.trim()).rowset_count) - int file_count = segment_count * 2 - assertEquals(file_count, Integer.parseInt(parseJson(o.trim()).file_count)) + check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") } int segmentsCount = 0 diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_recover_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_recover_with_format_v2.groovy index ee109817c4cb08..8de145d7921cee 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_recover_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_recover_with_format_v2.groovy @@ -20,9 +20,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_recover_with_format_v2", "inverted_index_format_v2"){ def tableName = "test_recover_with_format_v2" - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); @@ -34,15 +31,7 @@ suite("test_recover_with_format_v2", "inverted_index_format_v2"){ String backend_id = tablets[i].BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("3", parseJson(out.trim()).end_version) - assertEquals("3", parseJson(out.trim()).rowset_count) - assertEquals("4", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 3, 2, "V2") } } @@ -78,6 +67,9 @@ suite("test_recover_with_format_v2", "inverted_index_format_v2"){ sql """ INSERT INTO ${tableName} VALUES (3, "andy", 100); """ sql """ INSERT INTO ${tableName} VALUES (3, "bason", 99); """ + // select to sync tablet meta in cloud mode + sql """ SELECT * FROM ${tableName}; """ + check_index_file() // drop table and recover diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy index 663eb8434a3576..38b03510a0ca7a 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy @@ -20,9 +20,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_rename_column_with_format_v2", "inverted_index_format_v2"){ def tableName = "test_rename_column_with_format_v2" - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); @@ -78,15 +75,7 @@ suite("test_rename_column_with_format_v2", "inverted_index_format_v2"){ String backend_id = tablets[0].BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - def (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - assertEquals("12", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 7, 1, "V2") // rename column sql """ ALTER TABLE ${tableName} RENAME COLUMN name name_new; """ @@ -94,30 +83,12 @@ suite("test_rename_column_with_format_v2", "inverted_index_format_v2"){ qt_sql "SELECT * FROM $tableName WHERE name_new match 'andy' order by id, name_new, score;" - (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - assertEquals("12", parseJson(out.trim()).file_count) + check_nested_index_file(ip, port, tablet_id, 7, 1, "V2") // drop column sql """ ALTER TABLE ${tableName} DROP COLUMN name_new; """ wait_for_latest_op_on_table_finish(tableName, timeout) - - (code, out, err) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertTrue(code == 0) - assertTrue(out.contains("crc_value")) - assertTrue(out.contains("used_time_ms")) - assertEquals("0", parseJson(out.trim()).start_version) - assertEquals("7", parseJson(out.trim()).end_version) - assertEquals("7", parseJson(out.trim()).rowset_count) - // 6 files are expected after dropping the column - // Since we do not support change light_schema_change from true to false, so we can not do directly schema change. - // As a result, the rowset schema will not remove the index info, so the file count will not decrease. - assertEquals("12", parseJson(out.trim()).file_count) + // when drop column, the index files will not be deleted, so the index files count is still 1 + // when all index columns are dropped, the index files will be deleted by GC later + check_nested_index_file(ip, port, tablet_id, 7, 1, "V2") } diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy index 453fcd91226d0a..7826702fa7e816 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy @@ -46,10 +46,6 @@ suite("test_single_replica_compaction_with_format_v2", "inverted_index_format_v2 assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } - def calc_file_crc_on_tablet = { ip, port, tablet -> - return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) - } - def calc_segment_count = { tablet -> int segment_count = 0 String tablet_id = tablet.TabletId @@ -175,18 +171,7 @@ suite("test_single_replica_compaction_with_format_v2", "inverted_index_format_v2 backend_id = tablet.BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - int segment_count = calc_segment_count(tablet) - logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" + o + ", err=" + e) - assertTrue(c == 0) - assertTrue(o.contains("crc_value")) - assertTrue(o.contains("used_time_ms")) - assertEquals("0", parseJson(o.trim()).start_version) - assertEquals("9", parseJson(o.trim()).end_version) - assertEquals("9", parseJson(o.trim()).rowset_count) - int file_count = segment_count * 2 - assertEquals(file_count, Integer.parseInt(parseJson(o.trim()).file_count)) + check_nested_index_file(ip, port, tablet_id, 9, 3, "V2") StringBuilder sb = new StringBuilder(); sb.append("curl -X POST http://") @@ -244,19 +229,7 @@ suite("test_single_replica_compaction_with_format_v2", "inverted_index_format_v2 String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - int segment_count = calc_segment_count(tablet) - logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - def (c, o, e) = calc_file_crc_on_tablet(ip, port, tablet_id) - logger.info("Run calc_file_crc_on_tablet: code=" + c + ", out=" + o + ", err=" + e) - assertTrue(c == 0) - assertTrue(o.contains("crc_value")) - assertTrue(o.contains("used_time_ms")) - assertEquals("0", parseJson(o.trim()).start_version) - assertEquals("9", parseJson(o.trim()).end_version) - // after compaction, there are 2 rwosets. - assertEquals("2", parseJson(o.trim()).rowset_count) - int file_count = segment_count * 2 - assertEquals(file_count, Integer.parseInt(parseJson(o.trim()).file_count)) + check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") } int segmentsCount = 0