From 58f3d197effdcd28849041b853ebb54a4e4f86ed Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Fri, 8 Apr 2022 18:38:41 +0800 Subject: [PATCH 1/3] enhancement for regression-test framework update update --- .../org/apache/doris/regression/Config.groovy | 4 +- .../regression/action/ExplainAction.groovy | 13 ++-- .../doris/regression/action/TestAction.groovy | 21 ++++-- .../doris/regression/suite/Suite.groovy | 14 ++-- .../doris/regression/util/JdbcUtils.groovy | 20 +++--- .../doris/regression/util/MetaHolder.groovy | 24 +++++++ .../doris/regression/util/OutputUtils.groovy | 71 +++++++++++++++++-- 7 files changed, 135 insertions(+), 32 deletions(-) create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MetaHolder.groovy diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index 69d8dfcc080b68..045c3af2d59cdb 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -24,6 +24,7 @@ import com.google.common.collect.Maps import org.apache.commons.cli.CommandLine import org.apache.doris.regression.util.FileUtils import org.apache.doris.regression.util.JdbcUtils +import org.apache.doris.regression.util.Metaholder import java.sql.Connection import java.sql.DriverManager @@ -273,10 +274,11 @@ class Config { void tryCreateDbIfNotExist() { // connect without specify default db try { + Metaholder holder = new Metaholder(); String sql = "CREATE DATABASE IF NOT EXISTS ${defaultDb}" log.info("Try to create db, sql: ${sql}".toString()) getConnection().withCloseable { conn -> - JdbcUtils.executeToList(conn, sql) + JdbcUtils.executeToList(conn, sql, holder) } } catch (Throwable t) { throw new IllegalStateException("Create database failed, jdbcUrl: ${jdbcUrl}", t) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy index 287d8136c0f0c6..3a04c22066df03 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy @@ -21,6 +21,7 @@ import groovy.transform.stc.ClosureParams import groovy.transform.stc.FromString import org.apache.doris.regression.suite.SuiteContext import org.apache.doris.regression.util.JdbcUtils +import org.apache.doris.regression.util.Metaholder import groovy.util.logging.Slf4j import java.util.stream.Collectors @@ -112,13 +113,15 @@ class ExplainAction implements SuiteAction { log.info("Execute sql:\n${explainSql}".toString()) long startTime = System.currentTimeMillis() String explainString = null + Metaholder holder = new Metaholder(); try { - explainString = JdbcUtils.executeToList(context.getConnection(), explainSql).stream() + + explainString = JdbcUtils.executeToList(context.getConnection(), explainSql, holder).stream() .map({row -> row.get(0).toString()}) .collect(Collectors.joining("\n")) - return new ActionResult(explainString, null, startTime, System.currentTimeMillis()) + return new ActionResult(explainString, null, startTime, System.currentTimeMillis(), holder) } catch (Throwable t) { - return new ActionResult(explainString, t, startTime, System.currentTimeMillis()) + return new ActionResult(explainString, t, startTime, System.currentTimeMillis(), holder) } } @@ -127,12 +130,14 @@ class ExplainAction implements SuiteAction { Throwable exception long startTime long endTime + Metaholder holder - ActionResult(String result, Throwable exception, long startTime, long endTime) { + ActionResult(String result, Throwable exception, long startTime, long endTime, Metaholder holder) { this.result = result this.exception = exception this.startTime = startTime this.endTime = endTime + this.holder = holder } } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy index 362b37c431e41a..2b4ff502597786 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy @@ -24,6 +24,7 @@ import groovy.util.logging.Slf4j import org.apache.commons.io.LineIterator import org.apache.doris.regression.util.DataUtils import org.apache.doris.regression.util.OutputUtils +import org.apache.doris.regression.util.Metaholder import org.apache.http.HttpStatus import org.apache.http.client.methods.CloseableHttpResponse import org.apache.http.client.methods.RequestBuilder @@ -94,10 +95,10 @@ class TestAction implements SuiteAction { if (row instanceof List) { return OutputUtils.toCsvString(row as List) } else { - return OutputUtils.columnToCsvString(row) + return OutputUtils.toCsvString(row as Object) } }, - { List row -> OutputUtils.toCsvString(row) }, "Check failed") + { List row -> OutputUtils.toCsvString(row) }, "Check failed", result.holder.meta) if (errorMsg != null) { throw new IllegalStateException(errorMsg) } @@ -109,7 +110,7 @@ class TestAction implements SuiteAction { String errMsg = OutputUtils.checkOutput(csvIt, result.result.iterator(), { List row -> OutputUtils.toCsvString(row as List) }, { List row -> OutputUtils.toCsvString(row) }, - "Check failed compare to") + "Check failed compare to", result.holder.meta) if (errMsg != null) { throw new IllegalStateException(errMsg) } @@ -136,7 +137,8 @@ class TestAction implements SuiteAction { if (!file.exists()) { log.warn("Result file not exists: ${file}".toString()) } - log.warn("Compare to local file: ${file}".toString()) + + log.info("Compare to local file: ${file}".toString()) file.newInputStream().withCloseable { inputStream -> checkFunc(inputStream) } @@ -150,11 +152,13 @@ class TestAction implements SuiteAction { ActionResult doRun(Connection conn) { List> result = null + Metaholder holder = new Metaholder(); Throwable ex = null + long startTime = System.currentTimeMillis() try { log.info("Execute ${isOrder ? "order_" : ""}sql:\n${sql}".toString()) - result = JdbcUtils.executeToList(conn, sql) + result = JdbcUtils.executeToList(conn, sql, holder) if (isOrder) { result = DataUtils.sortByToString(result) } @@ -162,7 +166,8 @@ class TestAction implements SuiteAction { ex = t } long endTime = System.currentTimeMillis() - return new ActionResult(result, ex, startTime, endTime) + + return new ActionResult(result, ex, startTime, endTime, holder) } void sql(String sql) { @@ -234,12 +239,14 @@ class TestAction implements SuiteAction { Throwable exception long startTime long endTime + Metaholder holder - ActionResult(List> result, Throwable exception, long startTime, long endTime) { + ActionResult(List> result, Throwable exception, long startTime, long endTime, Metaholder holder) { this.result = result this.exception = exception this.startTime = startTime this.endTime = endTime + this.holder = holder } } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index c8f2b03b3026e0..02a85a78c04292 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -29,6 +29,7 @@ import org.apache.doris.regression.action.StreamLoadAction import org.apache.doris.regression.action.SuiteAction import org.apache.doris.regression.action.TestAction import org.apache.doris.regression.util.JdbcUtils +import org.apache.doris.regression.util.Metaholder import org.junit.jupiter.api.Assertions import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -184,7 +185,8 @@ class Suite implements GroovyInterceptable { List> sql(String sqlStr, boolean isOrder = false) { logger.info("Execute ${isOrder ? "order_" : ""}sql: ${sqlStr}".toString()) - def result = JdbcUtils.executeToList(context.getConnection(), sqlStr) + Metaholder holder = new Metaholder(); + def result = JdbcUtils.executeToList(context.getConnection(), sqlStr, holder) if (isOrder) { result = DataUtils.sortByToString(result) } @@ -263,9 +265,10 @@ class Suite implements GroovyInterceptable { void quickTest(String tag, String sql, boolean isOrder = false) { logger.info("Execute tag: ${tag}, ${isOrder ? "order_" : ""}sql: ${sql}".toString()) + Metaholder holder = new Metaholder(); if (context.config.generateOutputFile || context.config.forceGenerateOutputFile) { - def result = JdbcUtils.executorToStringList(context.getConnection(), sql) + def result = JdbcUtils.executeToStringList(context.getConnection(), sql, holder) if (isOrder) { result = sortByToString(result) } @@ -283,7 +286,8 @@ class Suite implements GroovyInterceptable { } OutputUtils.TagBlockIterator expectCsvResults = context.getOutputIterator().next() - List> realResults = JdbcUtils.executorToStringList(context.getConnection(), sql) + + List> realResults = JdbcUtils.executeToStringList(context.getConnection(), sql, holder) if (isOrder) { realResults = sortByToString(realResults) } @@ -291,8 +295,8 @@ class Suite implements GroovyInterceptable { try { errorMsg = OutputUtils.checkOutput(expectCsvResults, realResults.iterator(), { row -> OutputUtils.toCsvString(row as List) }, - {row -> OutputUtils.toCsvString(row) }, - "Check tag '${tag}' failed") + { row -> OutputUtils.toCsvString(row) }, + "Check tag '${tag}' failed", holder.meta) } catch (Throwable t) { throw new IllegalStateException("Check tag '${tag}' failed, sql:\n${sql}", t) } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy index b0a0bcdecbde6a..7602180afc3ba1 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy @@ -18,37 +18,40 @@ package org.apache.doris.regression.util import com.google.common.collect.ImmutableList +import org.apache.doris.regression.util.Metaholder import java.sql.Connection import java.sql.ResultSet +import java.sql.ResultSetMetaData class JdbcUtils { - static List> executeToList(Connection conn, String sql) { + static List> executeToList(Connection conn, String sql, Metaholder holder) { conn.prepareStatement(sql).withCloseable { stmt -> boolean hasResultSet = stmt.execute() if (!hasResultSet) { return ImmutableList.of(ImmutableList.of(stmt.getUpdateCount())) } else { - toList(stmt.resultSet) + toList(stmt.resultSet, holder) } } } - static List> executorToStringList(Connection conn, String sql) { + static List> executeToStringList(Connection conn, String sql, Metaholder holder) { conn.prepareStatement(sql).withCloseable { stmt -> boolean hasResultSet = stmt.execute() if (!hasResultSet) { return ImmutableList.of(ImmutableList.of(stmt.getUpdateCount())) } else { - toStringList(stmt.resultSet) + toStringList(stmt.resultSet, holder) } } } - static List> toList(ResultSet resultSet) { + static List> toList(ResultSet resultSet, Metaholder holder) { resultSet.withCloseable { + holder.meta = resultSet.metaData List> rows = new ArrayList<>() - def columnCount = resultSet.metaData.columnCount + def columnCount = holder.meta.columnCount while (resultSet.next()) { def row = new ArrayList<>() for (int i = 1; i <= columnCount; ++i) { @@ -60,10 +63,11 @@ class JdbcUtils { } } - static List> toStringList(ResultSet resultSet) { + static List> toStringList(ResultSet resultSet, Metaholder holder) { resultSet.withCloseable { + holder.meta = resultSet.metaData List> rows = new ArrayList<>() - def columnCount = resultSet.metaData.columnCount + def columnCount = holder.meta.columnCount while (resultSet.next()) { def row = new ArrayList<>() for (int i = 1; i <= columnCount; ++i) { diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MetaHolder.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MetaHolder.groovy new file mode 100644 index 00000000000000..45ca2eeea73dc0 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MetaHolder.groovy @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import java.sql.ResultSetMetaData + +class Metaholder { + public ResultSetMetaData meta; +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy index c30c893474b3fb..ea71e0e7cce9b6 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy @@ -25,13 +25,22 @@ import org.apache.commons.csv.CSVRecord import org.apache.commons.io.LineIterator import java.util.function.Function +import java.sql.ResultSetMetaData @CompileStatic class OutputUtils { - static String columnToCsvString(Object column) { + private static List castList(Object obj) { + List result = new ArrayList(); + for (Object o: (List) obj) { + result.add(toCsvString(o)); + } + return result; + } + + static String toCsvString(Object cell) { StringWriter writer = new StringWriter() def printer = new CSVPrinter(new PrintWriter(writer), CSVFormat.MYSQL) - printer.print(column) + printer.print(cell) return writer.toString() } @@ -44,9 +53,36 @@ class OutputUtils { return writer.toString() } + static String checkCell(String info, int line, String expect_cell, String real_cell, String data_type) { + if(data_type == "FLOAT" || data_type == "DOUBLE") { + double expect_double = Double.parseDouble(expect_cell) + double real_double = Double.parseDouble(real_cell) + + double real_relative_error = Math.abs(expect_double - real_double) / real_double + double expect_relative_error = 1e-10 + + if(expect_relative_error < real_relative_error) { + return "${info}, line ${line}, ${data_type} result mismatch.\nExpect cell is: ${expect_cell}\nBut real is: ${real_cell}\nrelative error is: ${real_relative_error}, bigger than ${expect_relative_error}" + } + } else if(data_type == "DATE" || data_type =="DATETIME") { + expect_cell = expect_cell.replace("T", " ") + real_cell = real_cell.replace("T", " ") + + if(!expect_cell.equals(real_cell)) { + return "${info}, line ${line}, ${data_type} result mismatch.\nExpect cell is: ${expect_cell}\nBut real is: ${real_cell}" + } + } else { + if(!expect_cell.equals(real_cell)) { + return "${info}, line ${line}, ${data_type} result mismatch.\nExpect cell is: ${expect_cell}\nBut real is: ${real_cell}" + } + } + + return null + } + static String checkOutput(Iterator expect, Iterator real, Function transform1, Function transform2, - String info) { + String info, ResultSetMetaData meta) { int line = 1 while (true) { if (expect.hasNext() && !real.hasNext()) { @@ -59,11 +95,32 @@ class OutputUtils { break } - def expectCsvString = transform1.apply(expect.next()) - def realCsvString = transform2.apply(real.next()) - if (!expectCsvString.equals(realCsvString)) { - return "${info}, line ${line} mismatch.\nExpect line is: ${expectCsvString}\nBut real is : ${realCsvString}" + def expectRaw = expect.next() + def realRaw = real.next() + + if (expectRaw instanceof List && meta != null) { + List expectList = castList(expectRaw) + List realList = castList(realRaw) + + def columnCount = meta.columnCount + for (int i = 1; i <= columnCount; i++) { + String expect_cell = toCsvString(expectList[i - 1]) + String real_cell = toCsvString(realList[i - 1]) + String data_type = meta.getColumnTypeName(i) + + def res = checkCell(info, line, expect_cell, real_cell, data_type) + if(res != null) { + return res + } + } + } else { + def expectCsvString = transform1.apply(expectRaw) + def realCsvString = transform2.apply(realRaw) + if (!expectCsvString.equals(realCsvString)) { + return "${info}, line ${line} mismatch.\nExpect line is: ${expectCsvString}\nBut real is: ${realCsvString}" + } } + line++ } } From 2fa3bd3783b999b1d5236529f7ab742b161ff882 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Tue, 12 Apr 2022 14:14:05 +0800 Subject: [PATCH 2/3] use camel case --- .../doris/regression/util/OutputUtils.groovy | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy index ea71e0e7cce9b6..ce483d3b8544a3 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy @@ -53,27 +53,27 @@ class OutputUtils { return writer.toString() } - static String checkCell(String info, int line, String expect_cell, String real_cell, String data_type) { - if(data_type == "FLOAT" || data_type == "DOUBLE") { - double expect_double = Double.parseDouble(expect_cell) - double real_double = Double.parseDouble(real_cell) + static String checkCell(String info, int line, String expectCell, String realCell, String dataType) { + if(dataType == "FLOAT" || dataType == "DOUBLE") { + double expectDouble = Double.parseDouble(expectCell) + double realDouble = Double.parseDouble(realCell) - double real_relative_error = Math.abs(expect_double - real_double) / real_double - double expect_relative_error = 1e-10 + double realRelativeError = Math.abs(expectDouble - realDouble) / realDouble + double expectRelativeError = 1e-10 - if(expect_relative_error < real_relative_error) { - return "${info}, line ${line}, ${data_type} result mismatch.\nExpect cell is: ${expect_cell}\nBut real is: ${real_cell}\nrelative error is: ${real_relative_error}, bigger than ${expect_relative_error}" + if(expectRelativeError < realRelativeError) { + return "${info}, line ${line}, ${dataType} result mismatch.\nExpect cell is: ${expectCell}\nBut real is: ${realCell}\nrelative error is: ${realRelativeError}, bigger than ${expectRelativeError}" } - } else if(data_type == "DATE" || data_type =="DATETIME") { - expect_cell = expect_cell.replace("T", " ") - real_cell = real_cell.replace("T", " ") + } else if(dataType == "DATE" || dataType =="DATETIME") { + expectCell = expectCell.replace("T", " ") + realCell = realCell.replace("T", " ") - if(!expect_cell.equals(real_cell)) { - return "${info}, line ${line}, ${data_type} result mismatch.\nExpect cell is: ${expect_cell}\nBut real is: ${real_cell}" + if(!expectCell.equals(realCell)) { + return "${info}, line ${line}, ${dataType} result mismatch.\nExpect cell is: ${expectCell}\nBut real is: ${realCell}" } } else { - if(!expect_cell.equals(real_cell)) { - return "${info}, line ${line}, ${data_type} result mismatch.\nExpect cell is: ${expect_cell}\nBut real is: ${real_cell}" + if(!expectCell.equals(realCell)) { + return "${info}, line ${line}, ${dataType} result mismatch.\nExpect cell is: ${expectCell}\nBut real is: ${realCell}" } } @@ -104,11 +104,11 @@ class OutputUtils { def columnCount = meta.columnCount for (int i = 1; i <= columnCount; i++) { - String expect_cell = toCsvString(expectList[i - 1]) - String real_cell = toCsvString(realList[i - 1]) - String data_type = meta.getColumnTypeName(i) + String expectCell = toCsvString(expectList[i - 1]) + String realCell = toCsvString(realList[i - 1]) + String dataType = meta.getColumnTypeName(i) - def res = checkCell(info, line, expect_cell, real_cell, data_type) + def res = checkCell(info, line, expectCell, realCell, dataType) if(res != null) { return res } From 7cd79822311e8924f2ae91c220f47dff691e9625 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Tue, 12 Apr 2022 15:19:58 +0800 Subject: [PATCH 3/3] add update fix reset --- .../org/apache/doris/regression/Config.groovy | 4 +-- .../regression/action/ExplainAction.groovy | 22 +++++++------- .../doris/regression/action/TestAction.groovy | 19 ++++++------ .../doris/regression/suite/Suite.groovy | 11 +++---- .../doris/regression/util/JdbcUtils.groovy | 30 ++++++++----------- .../doris/regression/util/MetaHolder.groovy | 24 --------------- 6 files changed, 37 insertions(+), 73 deletions(-) delete mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MetaHolder.groovy diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index 045c3af2d59cdb..69d8dfcc080b68 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -24,7 +24,6 @@ import com.google.common.collect.Maps import org.apache.commons.cli.CommandLine import org.apache.doris.regression.util.FileUtils import org.apache.doris.regression.util.JdbcUtils -import org.apache.doris.regression.util.Metaholder import java.sql.Connection import java.sql.DriverManager @@ -274,11 +273,10 @@ class Config { void tryCreateDbIfNotExist() { // connect without specify default db try { - Metaholder holder = new Metaholder(); String sql = "CREATE DATABASE IF NOT EXISTS ${defaultDb}" log.info("Try to create db, sql: ${sql}".toString()) getConnection().withCloseable { conn -> - JdbcUtils.executeToList(conn, sql, holder) + JdbcUtils.executeToList(conn, sql) } } catch (Throwable t) { throw new IllegalStateException("Create database failed, jdbcUrl: ${jdbcUrl}", t) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy index 3a04c22066df03..58a979b465f3d7 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy @@ -21,9 +21,8 @@ import groovy.transform.stc.ClosureParams import groovy.transform.stc.FromString import org.apache.doris.regression.suite.SuiteContext import org.apache.doris.regression.util.JdbcUtils -import org.apache.doris.regression.util.Metaholder import groovy.util.logging.Slf4j - +import java.sql.ResultSetMetaData import java.util.stream.Collectors @Slf4j @@ -113,15 +112,14 @@ class ExplainAction implements SuiteAction { log.info("Execute sql:\n${explainSql}".toString()) long startTime = System.currentTimeMillis() String explainString = null - Metaholder holder = new Metaholder(); + ResultSetMetaData meta = null try { - - explainString = JdbcUtils.executeToList(context.getConnection(), explainSql, holder).stream() - .map({row -> row.get(0).toString()}) - .collect(Collectors.joining("\n")) - return new ActionResult(explainString, null, startTime, System.currentTimeMillis(), holder) + def temp = null + (temp, meta) = JdbcUtils.executeToList(context.getConnection(), explainSql) + explainString = temp.stream().map({row -> row.get(0).toString()}).collect(Collectors.joining("\n")) + return new ActionResult(explainString, null, startTime, System.currentTimeMillis(), meta) } catch (Throwable t) { - return new ActionResult(explainString, t, startTime, System.currentTimeMillis(), holder) + return new ActionResult(explainString, t, startTime, System.currentTimeMillis(), meta) } } @@ -130,14 +128,14 @@ class ExplainAction implements SuiteAction { Throwable exception long startTime long endTime - Metaholder holder + ResultSetMetaData meta - ActionResult(String result, Throwable exception, long startTime, long endTime, Metaholder holder) { + ActionResult(String result, Throwable exception, long startTime, long endTime, ResultSetMetaData meta) { this.result = result this.exception = exception this.startTime = startTime this.endTime = endTime - this.holder = holder + this.meta = meta } } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy index 2b4ff502597786..bd03b76d9ea2f1 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy @@ -24,7 +24,6 @@ import groovy.util.logging.Slf4j import org.apache.commons.io.LineIterator import org.apache.doris.regression.util.DataUtils import org.apache.doris.regression.util.OutputUtils -import org.apache.doris.regression.util.Metaholder import org.apache.http.HttpStatus import org.apache.http.client.methods.CloseableHttpResponse import org.apache.http.client.methods.RequestBuilder @@ -34,7 +33,7 @@ import org.apache.http.util.EntityUtils import javax.swing.text.html.parser.Entity import java.nio.charset.StandardCharsets import java.sql.Connection - +import java.sql.ResultSetMetaData import org.apache.doris.regression.suite.SuiteContext import org.apache.doris.regression.util.JdbcUtils import org.junit.Assert @@ -98,7 +97,7 @@ class TestAction implements SuiteAction { return OutputUtils.toCsvString(row as Object) } }, - { List row -> OutputUtils.toCsvString(row) }, "Check failed", result.holder.meta) + { List row -> OutputUtils.toCsvString(row) }, "Check failed", result.meta) if (errorMsg != null) { throw new IllegalStateException(errorMsg) } @@ -110,7 +109,7 @@ class TestAction implements SuiteAction { String errMsg = OutputUtils.checkOutput(csvIt, result.result.iterator(), { List row -> OutputUtils.toCsvString(row as List) }, { List row -> OutputUtils.toCsvString(row) }, - "Check failed compare to", result.holder.meta) + "Check failed compare to", result.meta) if (errMsg != null) { throw new IllegalStateException(errMsg) } @@ -152,13 +151,13 @@ class TestAction implements SuiteAction { ActionResult doRun(Connection conn) { List> result = null - Metaholder holder = new Metaholder(); + ResultSetMetaData meta = null Throwable ex = null long startTime = System.currentTimeMillis() try { log.info("Execute ${isOrder ? "order_" : ""}sql:\n${sql}".toString()) - result = JdbcUtils.executeToList(conn, sql, holder) + (result, meta) = JdbcUtils.executeToList(conn, sql) if (isOrder) { result = DataUtils.sortByToString(result) } @@ -167,7 +166,7 @@ class TestAction implements SuiteAction { } long endTime = System.currentTimeMillis() - return new ActionResult(result, ex, startTime, endTime, holder) + return new ActionResult(result, ex, startTime, endTime, meta) } void sql(String sql) { @@ -239,14 +238,14 @@ class TestAction implements SuiteAction { Throwable exception long startTime long endTime - Metaholder holder + ResultSetMetaData meta - ActionResult(List> result, Throwable exception, long startTime, long endTime, Metaholder holder) { + ActionResult(List> result, Throwable exception, long startTime, long endTime, ResultSetMetaData meta) { this.result = result this.exception = exception this.startTime = startTime this.endTime = endTime - this.holder = holder + this.meta = meta } } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 02a85a78c04292..17a0d152bfd95a 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -29,7 +29,6 @@ import org.apache.doris.regression.action.StreamLoadAction import org.apache.doris.regression.action.SuiteAction import org.apache.doris.regression.action.TestAction import org.apache.doris.regression.util.JdbcUtils -import org.apache.doris.regression.util.Metaholder import org.junit.jupiter.api.Assertions import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -185,8 +184,7 @@ class Suite implements GroovyInterceptable { List> sql(String sqlStr, boolean isOrder = false) { logger.info("Execute ${isOrder ? "order_" : ""}sql: ${sqlStr}".toString()) - Metaholder holder = new Metaholder(); - def result = JdbcUtils.executeToList(context.getConnection(), sqlStr, holder) + def (result, meta) = JdbcUtils.executeToList(context.getConnection(), sqlStr) if (isOrder) { result = DataUtils.sortByToString(result) } @@ -265,10 +263,9 @@ class Suite implements GroovyInterceptable { void quickTest(String tag, String sql, boolean isOrder = false) { logger.info("Execute tag: ${tag}, ${isOrder ? "order_" : ""}sql: ${sql}".toString()) - Metaholder holder = new Metaholder(); if (context.config.generateOutputFile || context.config.forceGenerateOutputFile) { - def result = JdbcUtils.executeToStringList(context.getConnection(), sql, holder) + def (result, meta) = JdbcUtils.executeToStringList(context.getConnection(), sql) if (isOrder) { result = sortByToString(result) } @@ -287,7 +284,7 @@ class Suite implements GroovyInterceptable { OutputUtils.TagBlockIterator expectCsvResults = context.getOutputIterator().next() - List> realResults = JdbcUtils.executeToStringList(context.getConnection(), sql, holder) + def (realResults, meta) = JdbcUtils.executeToStringList(context.getConnection(), sql) if (isOrder) { realResults = sortByToString(realResults) } @@ -296,7 +293,7 @@ class Suite implements GroovyInterceptable { errorMsg = OutputUtils.checkOutput(expectCsvResults, realResults.iterator(), { row -> OutputUtils.toCsvString(row as List) }, { row -> OutputUtils.toCsvString(row) }, - "Check tag '${tag}' failed", holder.meta) + "Check tag '${tag}' failed", meta) } catch (Throwable t) { throw new IllegalStateException("Check tag '${tag}' failed, sql:\n${sql}", t) } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy index 7602180afc3ba1..e7fb1901030537 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy @@ -18,40 +18,39 @@ package org.apache.doris.regression.util import com.google.common.collect.ImmutableList -import org.apache.doris.regression.util.Metaholder +import groovy.lang.Tuple2 import java.sql.Connection import java.sql.ResultSet import java.sql.ResultSetMetaData class JdbcUtils { - static List> executeToList(Connection conn, String sql, Metaholder holder) { + static Tuple2>, ResultSetMetaData> executeToList(Connection conn, String sql) { conn.prepareStatement(sql).withCloseable { stmt -> boolean hasResultSet = stmt.execute() if (!hasResultSet) { - return ImmutableList.of(ImmutableList.of(stmt.getUpdateCount())) + return [ImmutableList.of(ImmutableList.of(stmt.getUpdateCount())), null] } else { - toList(stmt.resultSet, holder) + return toList(stmt.resultSet) } } } - static List> executeToStringList(Connection conn, String sql, Metaholder holder) { + static Tuple2>, ResultSetMetaData> executeToStringList(Connection conn, String sql) { conn.prepareStatement(sql).withCloseable { stmt -> boolean hasResultSet = stmt.execute() if (!hasResultSet) { - return ImmutableList.of(ImmutableList.of(stmt.getUpdateCount())) + return [ImmutableList.of(ImmutableList.of(stmt.getUpdateCount())), null] } else { - toStringList(stmt.resultSet, holder) + return toStringList(stmt.resultSet) } } } - static List> toList(ResultSet resultSet, Metaholder holder) { + static Tuple2>, ResultSetMetaData> toList(ResultSet resultSet) { resultSet.withCloseable { - holder.meta = resultSet.metaData List> rows = new ArrayList<>() - def columnCount = holder.meta.columnCount + def columnCount = resultSet.metaData.columnCount while (resultSet.next()) { def row = new ArrayList<>() for (int i = 1; i <= columnCount; ++i) { @@ -59,20 +58,17 @@ class JdbcUtils { } rows.add(row) } - return rows + return [rows, resultSet.metaData] } } - static List> toStringList(ResultSet resultSet, Metaholder holder) { + static Tuple2>, ResultSetMetaData> toStringList(ResultSet resultSet) { resultSet.withCloseable { - holder.meta = resultSet.metaData List> rows = new ArrayList<>() - def columnCount = holder.meta.columnCount + def columnCount = resultSet.metaData.columnCount while (resultSet.next()) { def row = new ArrayList<>() for (int i = 1; i <= columnCount; ++i) { - // row.add(resultSet.getObject(i)) - // row.add(resultSet.getString(i)) try { row.add(resultSet.getObject(i)) } catch (Throwable t) { @@ -85,7 +81,7 @@ class JdbcUtils { } rows.add(row) } - return rows + return [rows, resultSet.metaData] } } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MetaHolder.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MetaHolder.groovy deleted file mode 100644 index 45ca2eeea73dc0..00000000000000 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/MetaHolder.groovy +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.regression.util - -import java.sql.ResultSetMetaData - -class Metaholder { - public ResultSetMetaData meta; -}