From 8eab215d7452bf9cc4537f8675cca3d6fc9b6a18 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Tue, 29 Mar 2022 16:57:01 +0800 Subject: [PATCH 1/2] enhancement for regression-test framework --- regression-test/data/aggregate/aggregate.out | 59 ++++++------- .../data/demo/select_union_all_action.out | 1 + regression-test/data/demo/thread_action.out | 4 +- .../sql/avg_decimal.sql.generated.out | 4 + .../doris/regression/suite/Suite.groovy | 9 +- .../doris/regression/util/JdbcUtils.groovy | 2 +- .../doris/regression/util/OutputUtils.groovy | 85 +++++++++++++++---- .../suites/aggregate/aggregate.groovy | 62 ++++++++------ run-regression-test.sh | 1 + 9 files changed, 148 insertions(+), 79 deletions(-) create mode 100644 regression-test/data/empty_table/sql/avg_decimal.sql.generated.out diff --git a/regression-test/data/aggregate/aggregate.out b/regression-test/data/aggregate/aggregate.out index 964b08331f8053..14f1027d0e815f 100644 --- a/regression-test/data/aggregate/aggregate.out +++ b/regression-test/data/aggregate/aggregate.out @@ -1,75 +1,76 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !aggregate -- +-- !aggregate_min_max -- TESTING AGAIN --- !aggregate -- +-- !aggregate_avg -- 3309.4 50.12646 --- !aggregate -- +-- !aggregate_avg2 -- 3566.3333333333335 50.65743333333333 --- !aggregate -- +-- !aggregate_multi_count -- 15 15 14 14 14 15 --- !aggregate -- +-- !aggregate_multi_distinct_count -- 9 9 10 8 7 2 --- !aggregate -- +-- !aggregate_multi_max -- 8996 99.8777 testing 1999-01-08 2010-01-02T05:09:06 --- !aggregate -- +-- !aggregate_multi_min -- 12 0.245 AGAIN 1922-04-02 1989-01-08T04:05:06 --- !aggregate -- +-- !aggregate_mix -- 14 99.8777 3309.4 --- !aggregate -- +-- !aggregate_stddev1 -- 3037.0783723835643 35.210273031541995 --- !aggregate -- -3145.0576394647455 35.21027303154199 +-- !aggregate_stddev2 -- +3145.057639464745 35.21027303154199 --- !aggregate -- +-- !aggregate_stddev3 -- 3037.0783723835643 40.09062957713574 --- !aggregate -- +-- !aggregate_stddev4 -- 3143.674778708147 36.44609513116206 --- !aggregate -- +-- !aggregate_stddev5 -- 3335.837376132116 36.446095131162046 --- !aggregate -- +-- !aggregate_stddev6 -- 3143.6747787081463 42.52253405404598 --- !aggregate -- +-- !aggregate_sum -- 49641 751.8969000000001 --- !aggregate -- +-- !aggregate_distinct_sum -- 32097 455.9169 --- !aggregate -- +-- !aggregate_variance -- 9223845.040000001 1239.7633269557336 --- !aggregate -- -9891387.555555556 1239.7633269557332 +-- !aggregate_variance2 -- +9891387.555555554 1239.7633269557332 --- !aggregate -- +-- !aggregate_variance3 -- 9223845.04 1607.2585798911111 --- !aggregate -- +-- !aggregate_variance4 -- 9882691.114285717 1328.3178503097147 --- !aggregate -- -1.1127811E7 1328.317850309714 +-- !aggregate_variance5 -- +1.1127810999999998E7 1328.317850309714 --- !aggregate -- +-- !aggregate_variance6 -- 9882691.114285713 1808.1659023775 --- !aggregate -- +-- !aggregate_variance7 -- 9223845.040000001 1239.7633269557336 --- !aggregate -- -9891387.555555556 1239.7633269557332 +-- !aggregate_variance8 -- +9891387.555555554 1239.7633269557332 --- !aggregate -- +-- !aggregate_variance9 -- 9223845.04 1607.2585798911111 + diff --git a/regression-test/data/demo/select_union_all_action.out b/regression-test/data/demo/select_union_all_action.out index 7cdd5df20e1139..e3b3bdcb70d8d2 100644 --- a/regression-test/data/demo/select_union_all_action.out +++ b/regression-test/data/demo/select_union_all_action.out @@ -8,3 +8,4 @@ 0 abc 1 123 2 \N + diff --git a/regression-test/data/demo/thread_action.out b/regression-test/data/demo/thread_action.out index 4f14b39933b4f6..dd9024b20ae9ee 100644 --- a/regression-test/data/demo/thread_action.out +++ b/regression-test/data/demo/thread_action.out @@ -1,7 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !diffrent_tag1 -- +-- !diffrent_tag2 -- 100 --- !diffrent_tag2 -- +-- !diffrent_tag1 -- 100 diff --git a/regression-test/data/empty_table/sql/avg_decimal.sql.generated.out b/regression-test/data/empty_table/sql/avg_decimal.sql.generated.out new file mode 100644 index 00000000000000..1a9a561cf06947 --- /dev/null +++ b/regression-test/data/empty_table/sql/avg_decimal.sql.generated.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !avg_decimal -- +\N + diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 8a36eb5c3a2dfd..ff06571308215d 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -257,7 +257,7 @@ abstract class Suite extends Script implements GroovyInterceptable { logger.info("Execute tag: ${tag}, sql: ${sql}".toString()) if (context.config.generateOutputFile || context.config.forceGenerateOutputFile) { - def result = JdbcUtils.executorToStringList(context.getConnection(), sql) + def result = JdbcUtils.executeToStringList(context.getConnection(), sql) if (order) { result = sortByToString(result) } @@ -281,13 +281,10 @@ abstract class Suite extends Script implements GroovyInterceptable { } OutputUtils.TagBlockIterator expectCsvResults = context.getOutputIterator().next() - List> realResults = JdbcUtils.executorToStringList(context.getConnection(), sql) - if (order) { - realResults = sortByToString(realResults) - } + String errorMsg = null try { - errorMsg = OutputUtils.checkOutput(expectCsvResults, realResults.iterator(), "Check tag '${tag}' failed") + errorMsg = OutputUtils.checkOutput(expectCsvResults, context.getConnection(), sql, order, "Check tag '${tag}' failed") } catch (Throwable t) { List excelContentList = [context.file.getName(), tag, sql.trim(), t] context.recorder.reportDiffResult(excelContentList) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy index b0a0bcdecbde6a..61b872ac37494d 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy @@ -34,7 +34,7 @@ class JdbcUtils { } } - static List> executorToStringList(Connection conn, String sql) { + static List> executeToStringList(Connection conn, String sql) { conn.prepareStatement(sql).withCloseable { stmt -> boolean hasResultSet = stmt.execute() if (!hasResultSet) { diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy index ba37e33215cc88..7fdacb91649a4a 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy @@ -24,8 +24,20 @@ import org.apache.commons.csv.CSVPrinter import org.apache.commons.csv.CSVRecord import org.apache.commons.io.LineIterator +import java.sql.Connection +import java.sql.ResultSet + +import static org.apache.doris.regression.util.DataUtils.sortByToString + @CompileStatic class OutputUtils { + static String toCsvString(Object cell) { + StringWriter writer = new StringWriter() + def printer = new CSVPrinter(new PrintWriter(writer), CSVFormat.MYSQL) + printer.print(cell) + return writer.toString() + } + static String toCsvString(List row) { StringWriter writer = new StringWriter() def printer = new CSVPrinter(new PrintWriter(writer), CSVFormat.MYSQL) @@ -35,22 +47,65 @@ class OutputUtils { return writer.toString() } - static String checkOutput(Iterator> expect, Iterator> real, String info) { - while (true) { - if (expect.hasNext() && !real.hasNext()) { - return "${info}, result mismatch, real line is empty, but expect is ${expect.next()}" - } - if (!expect.hasNext() && real.hasNext()) { - return "${info}, result mismatch, expect line is empty, but real is ${toCsvString(real.next())}" - } - if (!expect.hasNext() && !real.hasNext()) { - break - } + static String checkOutput(Iterator> expectIterator, Connection conn, String sql, boolean order, String info) { + conn.prepareStatement(sql).withCloseable { stmt -> + boolean hasResultSet = stmt.execute() + if (!hasResultSet) { + if(expectIterator.hasNext()) { + return "${info}, result mismatch, real row number is 0, but expect more than it" + } + } else { + def columnCount = stmt.resultSet.metaData.columnCount + + List> rows = new ArrayList<>() + while(stmt.resultSet.next()) { + def row = new ArrayList<>() + for (int i = 1; i <= columnCount; ++i) { + row.add(stmt.resultSet.getObject(i)) + } + rows.add(row) + } + if (order) { + rows = sortByToString(rows) + } - def expectCsvString = toCsvString(expect.next() as List) - def realCsvString = toCsvString(real.next()) - if (!expectCsvString.equals(realCsvString)) { - return "${info}, result mismatch.\nExpect line is: ${expectCsvString}\nBut real is : ${realCsvString}" + for(int i = 1; i <= rows.size(); i++) { + if(expectIterator.hasNext()) { + List expect_row = expectIterator.next() + for(int j = 1; j <= columnCount; j++) { + String expect_cell = toCsvString(expect_row[j-1]) + String real_cell = toCsvString(rows[i-1][j-1]) + String tp = stmt.resultSet.metaData.getColumnTypeName(j) + if(tp == "FLOAT" || tp == "DOUBLE") { + double expect_double = Double.parseDouble(expect_cell) + double real_double = Double.parseDouble(real_cell) + + double real_relative_error = Math.abs(expect_double - real_double) / real_double + double expect_relative_error = 1e-10 + + if(expect_relative_error < real_relative_error) { + return "${info}, ${tp} result mismatch.\nExpect cell is: ${expect_cell}\nBut real is: ${real_cell}\nrelative error is: ${real_relative_error}, bigger than ${expect_relative_error}" + } + } else if(tp == "DATE" || tp =="DATETIME") { + expect_cell = expect_cell.replace("T", " ") + real_cell = real_cell.replace("T", " ") + + if(!expect_cell.equals(real_cell)) { + return "${info}, ${tp} result mismatch.\nExpect cell is: ${expect_cell}\nBut real is: ${real_cell}" + } + } else { + if(!expect_cell.equals(real_cell)) { + return "${info}, ${tp} result mismatch.\nExpect cell is: ${expect_cell}\nBut real is: ${real_cell}" + } + } + } + } else { + return "${info}, result mismatch, real row number is ${rows.size()}, but expect less than it" + } + } + if(expectIterator.hasNext()) { + return "${info}, result mismatch, real row number is ${rows.size()}, but expect less than it" + } } } } diff --git a/regression-test/suites/aggregate/aggregate.groovy b/regression-test/suites/aggregate/aggregate.groovy index 948243e0873d4c..4e15d850dd0ec3 100644 --- a/regression-test/suites/aggregate/aggregate.groovy +++ b/regression-test/suites/aggregate/aggregate.groovy @@ -23,7 +23,7 @@ def tableName = "datetype" sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( + CREATE TABLE ${tableName} ( c_bigint bigint, c_double double, c_string string, @@ -74,28 +74,38 @@ streamLoad { } } -qt_aggregate """ select max(upper(c_string)), min(upper(c_string)) from ${tableName} """ -qt_aggregate """ select avg(c_bigint), avg(c_double) from ${tableName} """ -qt_aggregate """ select avg(distinct c_bigint), avg(distinct c_double) from ${tableName} """ -qt_aggregate """ select count(c_bigint),count(c_double),count(c_string),count(c_date),count(c_timestamp),count(c_boolean) from ${tableName} """ -qt_aggregate """ select count(distinct c_bigint),count(distinct c_double),count(distinct c_string),count(distinct c_date),count(distinct c_timestamp),count(distinct c_boolean) from ${tableName} """ -qt_aggregate """ select max(c_bigint), max(c_double),max(c_string), max(c_date), max(c_timestamp) from ${tableName} """ -qt_aggregate """ select min(c_bigint), min(c_double), min(c_string), min(c_date), min(c_timestamp) from ${tableName} """ -qt_aggregate """ select count(c_string), max(c_double), avg(c_bigint) from ${tableName} """ -qt_aggregate """ select stddev_pop(c_bigint), stddev_pop(c_double) from ${tableName} """ -qt_aggregate """ select stddev_pop(distinct c_bigint), stddev_pop(c_double) from ${tableName} """ -qt_aggregate """ select stddev_pop(c_bigint), stddev_pop(distinct c_double) from ${tableName} """ -qt_aggregate """ select stddev_samp(c_bigint), stddev_samp(c_double) from ${tableName} """ -qt_aggregate """ select stddev_samp(distinct c_bigint), stddev_samp(c_double) from ${tableName} """ -qt_aggregate """ select stddev_samp(c_bigint), stddev_samp(distinct c_double) from ${tableName} """ -qt_aggregate """ select sum(c_bigint), sum(c_double) from ${tableName} """ -qt_aggregate """ select sum(distinct c_bigint), sum(distinct c_double) from ${tableName} """ -qt_aggregate """ select var_pop(c_bigint), var_pop(c_double) from ${tableName} """ -qt_aggregate """ select var_pop(distinct c_bigint), var_pop(c_double) from ${tableName} """ -qt_aggregate """ select var_pop(c_bigint), var_pop(distinct c_double) from ${tableName} """ -qt_aggregate """ select var_samp(c_bigint), var_samp(c_double) from ${tableName} """ -qt_aggregate """ select var_samp(distinct c_bigint), var_samp(c_double) from ${tableName} """ -qt_aggregate """ select var_samp(c_bigint), var_samp(distinct c_double) from ${tableName} """ -qt_aggregate """ select variance(c_bigint), variance(c_double) from ${tableName} """ -qt_aggregate """ select variance(distinct c_bigint), variance(c_double) from ${tableName} """ -qt_aggregate """ select variance(c_bigint), variance(distinct c_double) from ${tableName} """ +qt_aggregate_min_max """ select max(upper(c_string)), min(upper(c_string)) from ${tableName} """ + +qt_aggregate_avg """ select avg(c_bigint), avg(c_double) from ${tableName} """ +qt_aggregate_avg2 """ select avg(distinct c_bigint), avg(distinct c_double) from ${tableName} """ + +qt_aggregate_multi_count """ select count(c_bigint),count(c_double),count(c_string),count(c_date),count(c_timestamp),count(c_boolean) from ${tableName} """ + +qt_aggregate_multi_distinct_count """ select count(distinct c_bigint),count(distinct c_double),count(distinct c_string),count(distinct c_date),count(distinct c_timestamp),count(distinct c_boolean) from ${tableName} """ + +qt_aggregate_multi_max """ select max(c_bigint), max(c_double),max(c_string), max(c_date), max(c_timestamp) from ${tableName} """ + +qt_aggregate_multi_min """ select min(c_bigint), min(c_double), min(c_string), min(c_date), min(c_timestamp) from ${tableName} """ + +qt_aggregate_mix """ select count(c_string), max(c_double), avg(c_bigint) from ${tableName} """ + +qt_aggregate_stddev1 """ select stddev_pop(c_bigint), stddev_pop(c_double) from ${tableName} """ +qt_aggregate_stddev2 """ select stddev_pop(distinct c_bigint), stddev_pop(c_double) from ${tableName} """ +qt_aggregate_stddev3 """ select stddev_pop(c_bigint), stddev_pop(distinct c_double) from ${tableName} """ +qt_aggregate_stddev4 """ select stddev_samp(c_bigint), stddev_samp(c_double) from ${tableName} """ +qt_aggregate_stddev5 """ select stddev_samp(distinct c_bigint), stddev_samp(c_double) from ${tableName} """ +qt_aggregate_stddev6 """ select stddev_samp(c_bigint), stddev_samp(distinct c_double) from ${tableName} """ + +qt_aggregate_sum """ select sum(c_bigint), sum(c_double) from ${tableName} """ + +qt_aggregate_distinct_sum """ select sum(distinct c_bigint), sum(distinct c_double) from ${tableName} """ + +qt_aggregate_variance """ select var_pop(c_bigint), var_pop(c_double) from ${tableName} """ +qt_aggregate_variance2 """ select var_pop(distinct c_bigint), var_pop(c_double) from ${tableName} """ +qt_aggregate_variance3 """ select var_pop(c_bigint), var_pop(distinct c_double) from ${tableName} """ +qt_aggregate_variance4 """ select var_samp(c_bigint), var_samp(c_double) from ${tableName} """ +qt_aggregate_variance5 """ select var_samp(distinct c_bigint), var_samp(c_double) from ${tableName} """ +qt_aggregate_variance6 """ select var_samp(c_bigint), var_samp(distinct c_double) from ${tableName} """ +qt_aggregate_variance7 """ select variance(c_bigint), variance(c_double) from ${tableName} """ +qt_aggregate_variance8 """ select variance(distinct c_bigint), variance(c_double) from ${tableName} """ +qt_aggregate_variance9 """ select variance(c_bigint), variance(distinct c_double) from ${tableName} """ diff --git a/run-regression-test.sh b/run-regression-test.sh index 840d149c3fea57..2a22a6ef1a6d31 100755 --- a/run-regression-test.sh +++ b/run-regression-test.sh @@ -133,6 +133,7 @@ RUN_JAR=${OUTPUT_DIR}/lib/regression-test-*.jar if [ ${CLEAN} -eq 1 ]; then rm -rf ${REGRESSION_TEST_BUILD_DIR} rm -rf ${OUTPUT_DIR} + find ${DORIS_HOME}/regression-test/suites/ -name "*.generated.groovy"| xargs -r rm fi if [ ${RUN} -ne 1 ]; then From 250ff22b63bd516cb48ad338e5f5b443ef26a250 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt <952130278@qq.com> Date: Wed, 6 Apr 2022 18:33:02 +0800 Subject: [PATCH 2/2] fix datetime --- be/src/vec/columns/column_vector.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/be/src/vec/columns/column_vector.h b/be/src/vec/columns/column_vector.h index 216e8fd0edde08..19cefb4e59c6c3 100644 --- a/be/src/vec/columns/column_vector.h +++ b/be/src/vec/columns/column_vector.h @@ -21,6 +21,7 @@ #pragma once #include +#include #include "olap/uint24.h" #include "vec/columns/column.h" @@ -188,7 +189,8 @@ class ColumnVector final : public COWHelper> for (int i = 0; i < num; i++) { const char* cur_ptr = data_ptr + value_size * i; uint64_t value = *reinterpret_cast(cur_ptr); - vectorized::VecDateTimeValue date(value); + vectorized::VecDateTimeValue date; + date.from_olap_datetime(value); this->insert_data(reinterpret_cast(&date), 0); } }