From cc2a287a4d381bd1d980492d7e270cc9393fcb46 Mon Sep 17 00:00:00 2001 From: Huajian Lan <924060929@qq.com> Date: Thu, 24 Mar 2022 23:14:45 +0800 Subject: [PATCH 1/4] [test][refactor] support suite block to specify multiple group, support test action with check iterator/file/http stream --- .../developer-guide/regression-testing.md | 421 ++++++++++-------- regression-test/conf/logback.xml | 15 +- regression-test/conf/regression-conf.groovy | 2 + regression-test/data/demo/test_action.csv | 4 + regression-test/data/demo/test_sql_file.out | 6 + .../data/demo/test_sql_file_order.out | 6 + .../data/empty_table/sql/avg_decimal.out | 4 + ..._output_as_right_tale_left_outer_order.out | 5 + regression-test/framework/pom.xml | 23 + .../org/apache/doris/regression/Config.groovy | 47 +- .../doris/regression/ConfigOptions.groovy | 24 +- .../doris/regression/RegressionTest.groovy | 260 +++++------ .../regression/action/ExplainAction.groovy | 8 - .../regression/action/StreamLoadAction.groovy | 30 +- .../doris/regression/action/TestAction.groovy | 108 ++++- .../TeamcityServiceMessageEncoder.groovy | 61 +++ .../regression/suite/ScriptContext.groovy | 175 ++++++++ .../doris/regression/suite/ScriptInfo.groovy | 29 ++ .../regression/suite/ScriptSource.groovy | 82 ++++ .../doris/regression/suite/Suite.groovy | 56 +-- .../regression/suite/SuiteContext.groovy | 110 ++++- .../{util => suite}/SuiteInfo.groovy | 2 +- .../doris/regression/suite/SuiteScript.groovy | 61 +++ .../suite/event/EventListener.groovy | 35 ++ .../suite/event/RecorderEventListener.groovy | 81 ++++ .../suite/event/StackEventListeners.groovy | 102 +++++ .../suite/event/TeamcityEventListener.groovy | 81 ++++ .../doris/regression/util/LoggerUtils.groovy | 43 ++ .../doris/regression/util/OutputUtils.groovy | 25 +- .../doris/regression/util/Recorder.groovy | 7 +- .../regression/util/TeamcityUtils.groovy | 104 +++++ .../framework/src/main/groovy/suite.gdsl | 82 ++-- .../suites/aggregate/aggregate.groovy | 140 +++--- .../correctness/test_select_constant.groovy | 4 +- .../suites/demo/connect_action.groovy | 31 +- .../suites/demo/event_action.groovy | 80 ++-- .../suites/demo/explain_action.groovy | 50 ++- .../suites/demo/lazyCheck_action.groovy | 53 +-- regression-test/suites/demo/qt_action.groovy | 32 +- .../demo/select_union_all_action.groovy | 16 +- regression-test/suites/demo/sql_action.groovy | 108 ++--- .../suites/demo/streamLoad_action.groovy | 117 ++--- .../suites/demo/test_action.groovy | 94 ++-- regression-test/suites/demo/test_sql_file.sql | 5 + .../suites/demo/test_sql_file_order.sql | 5 + .../suites/demo/thread_action.groovy | 82 ++-- .../suites/demo/timer_action.groovy | 18 +- .../suites/empty_table/load.groovy | 14 +- regression-test/suites/join/load.groovy | 20 +- ...output_as_right_tale_left_outer_order.sql} | 0 .../test_streamload_perfomance.groovy | 30 +- .../types/complex_types/basic_agg_test.groovy | 18 +- run-regression-test.sh | 19 +- 53 files changed, 2159 insertions(+), 876 deletions(-) create mode 100644 regression-test/data/demo/test_action.csv create mode 100644 regression-test/data/demo/test_sql_file.out create mode 100644 regression-test/data/demo/test_sql_file_order.out create mode 100644 regression-test/data/empty_table/sql/avg_decimal.out create mode 100644 regression-test/data/join/sql/agg_output_as_right_tale_left_outer_order.out create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/logger/TeamcityServiceMessageEncoder.groovy create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptContext.groovy create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptInfo.groovy create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy rename regression-test/framework/src/main/groovy/org/apache/doris/regression/{util => suite}/SuiteInfo.groovy (96%) create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteScript.groovy create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/EventListener.groovy create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/RecorderEventListener.groovy create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/StackEventListeners.groovy create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/TeamcityEventListener.groovy create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/util/LoggerUtils.groovy create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/util/TeamcityUtils.groovy create mode 100644 regression-test/suites/demo/test_sql_file.sql create mode 100644 regression-test/suites/demo/test_sql_file_order.sql rename regression-test/suites/join/sql/{agg_output_as_right_tale_left_outer.sql => agg_output_as_right_tale_left_outer_order.sql} (100%) diff --git a/docs/zh-CN/developer-guide/regression-testing.md b/docs/zh-CN/developer-guide/regression-testing.md index a962502cd99f01..01e8e81ecfcd7b 100644 --- a/docs/zh-CN/developer-guide/regression-testing.md +++ b/docs/zh-CN/developer-guide/regression-testing.md @@ -102,10 +102,12 @@ suitePath = "${DORIS_HOME}/regression-test/suites" dataPath = "${DORIS_HOME}/regression-test/data" // 默认会读所有的组,读多个组可以用半角逗号隔开，如: "demo,performance" -// 一般不需要在配置文件中修改，而是通过run-regression-test.sh来动态指定和覆盖 +// 一般不需要在配置文件中修改，而是通过run-regression-test.sh --run -g来动态指定和覆盖 testGroups = "" -// 默认会读所有的用例, 同样可以使用run-regression-test.sh来动态指定和覆盖 +// 默认会读所有的用例, 同样可以使用run-regression-test.sh --run -s来动态指定和覆盖 testSuites = "" +// 默认会加载的用例目录, 可以通过run-regression-test.sh --run -d来动态指定和覆盖 +testDirectories = "" // 其他自定义配置 customConf1 = "test_custom_conf_value" @@ -128,30 +130,31 @@ sql action用于提交sql并获取结果，如果查询失败则会抛出异常 下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/sql_action.groovy`: ```groovy -// execute sql and ignore result -sql "show databases" +suite("sql_action", "demo") { + // execute sql and ignore result + sql "show databases" -// execute sql and get result, outer List denote rows, inner List denote columns in a single row -List> tables = sql "show tables" + // execute sql and get result, outer List denote rows, inner List denote columns in a single row + List> tables = sql "show tables" -// assertXxx() will invoke junit5's Assertions.assertXxx() dynamically -assertTrue(tables.size() >= 0) // test rowCount >= 0 + // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically + assertTrue(tables.size() >= 0) // test rowCount >= 0 -// syntax error -try { - sql "a b c d e" - throw new IllegalStateException("Should be syntax error") -} catch (java.sql.SQLException t) { - assertTrue(true) -} + // syntax error + try { + sql "a b c d e" + throw new IllegalStateException("Should be syntax error") + } catch (java.sql.SQLException t) { + assertTrue(true) + } -def testTable = "test_sql_action1" + def testTable = "test_sql_action1" -try { - sql "DROP TABLE IF EXISTS ${testTable}" + try { + sql "DROP TABLE IF EXISTS ${testTable}" - // multi-line sql - def result1 = sql """ + // multi-line sql + def result1 = sql """ CREATE TABLE IF NOT EXISTS ${testTable} ( id int ) @@ -161,40 +164,40 @@ try { ) """ - // DDL/DML return 1 row and 1 column, the only value is update row count - assertTrue(result1.size() == 1) - assertTrue(result1[0].size() == 1) - assertTrue(result1[0][0] == 0, "Create table should update 0 rows") - - def result2 = sql "INSERT INTO test_sql_action1 values(1), (2), (3)" - assertTrue(result2.size() == 1) - assertTrue(result2[0].size() == 1) - assertTrue(result2[0][0] == 3, "Insert should update 3 rows") -} finally { - /** - * try_xxx(args) means: - * - * try { - * return xxx(args) - * } catch (Throwable t) { - * // do nothing - * return null - * } - */ - try_sql("DROP TABLE IF EXISTS ${testTable}") - - // you can see the error sql will not throw exception and return - try { - def errorSqlResult = try_sql("a b c d e f g") - assertTrue(errorSqlResult == null) - } catch (Throwable t) { - assertTrue(false, "Never catch exception") + // DDL/DML return 1 row and 1 column, the only value is update row count + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 0, "Create table should update 0 rows") + + def result2 = sql "INSERT INTO test_sql_action1 values(1), (2), (3)" + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 1) + assertTrue(result2[0][0] == 3, "Insert should update 3 rows") + } finally { + /** + * try_xxx(args) means: + * + * try { + * return xxx(args) + * } catch (Throwable t) { + * // do nothing + * return null + * } + */ + try_sql("DROP TABLE IF EXISTS ${testTable}") + + // you can see the error sql will not throw exception and return + try { + def errorSqlResult = try_sql("a b c d e f g") + assertTrue(errorSqlResult == null) + } catch (Throwable t) { + assertTrue(false, "Never catch exception") + } } -} -// order_sql(sqlStr) equals to sql(sqlStr, isOrder=true) -// sort result by string dict -def list = order_sql """ + // order_sql(sqlStr) equals to sql(sqlStr, isOrder=true) + // sort result by string dict + def list = order_sql """ select 2 union all select 1 @@ -205,11 +208,13 @@ def list = order_sql """ union all select 3 """ -assertEquals(null, list[0][0]) -assertEquals(1, list[1][0]) -assertEquals(15, list[2][0]) -assertEquals(2, list[3][0]) -assertEquals(3, list[4][0]) + + assertEquals(null, list[0][0]) + assertEquals(1, list[1][0]) + assertEquals(15, list[2][0]) + assertEquals(2, list[3][0]) + assertEquals(3, list[4][0]) +} ``` ### qt action @@ -219,22 +224,23 @@ qt action用于提交sql，并使用对应的.out TSV文件来校验结果 下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/qt_action.groovy`: ```groovy -/** - * qt_xxx sql equals to quickTest(xxx, sql) witch xxx is tag. - * the result will be compare to the relate file: ${DORIS_HOME}/regression_test/data/qt_action.out. - * - * if you want to generate .out tsv file for real execute result. you can run with -genOut or -forceGenOut option. - * e.g - * ${DORIS_HOME}/run-regression-test.sh --run qt_action -genOut - * ${DORIS_HOME}/run-regression-test.sh --run qt_action -forceGenOut - */ -qt_select "select 1, 'beijing' union all select 2, 'shanghai'" - -qt_select2 "select 2" - -// order result by string dict then compare to .out file. -// order_qt_xxx sql equals to quickTest(xxx, sql, true). -order_qt_union_all """ +suite("qt_action", "demo") { + /** + * qt_xxx sql equals to quickTest(xxx, sql) witch xxx is tag. + * the result will be compare to the relate file: ${DORIS_HOME}/regression_test/data/qt_action.out. + * + * if you want to generate .out tsv file for real execute result. you can run with -genOut or -forceGenOut option. + * e.g + * ${DORIS_HOME}/run-regression-test.sh --run qt_action -genOut + * ${DORIS_HOME}/run-regression-test.sh --run qt_action -forceGenOut + */ + qt_select "select 1, 'beijing' union all select 2, 'shanghai'" + + qt_select2 "select 2" + + // order result by string dict then compare to .out file. + // order_qt_xxx sql equals to quickTest(xxx, sql, true). + order_qt_union_all """ select 2 union all select 1 @@ -245,6 +251,7 @@ order_qt_union_all """ union all select 3 """ +} ``` ### test action @@ -252,7 +259,9 @@ test action可以使用更复杂的校验规则来测试，比如验证行数、 可用参数 - String sql: 输入的sql字符串 -- List> result: 提供一个List对象，用于校验真实查询结果对比是否相等 +- List> result: 提供一个List对象，用于比较真实查询结果与List对象是否相等 +- Iterator resultIterator: 提供一个Iterator对象，用于比较真实查询结果与Iterator是否相等 +- String resultFile: 提供一个文件Uri(可以是本地文件相对路径，或http(s)路径)，用于比较真实查询结果与http响应流是否相等，格式与.out文件格式类似，但没有块头和注释 - String exception: 校验抛出的异常是否包含某些字符串 - long rowNum: 验证结果行数 - long time: 验证执行时间是否小于这个值，单位是毫秒 @@ -260,14 +269,15 @@ test action可以使用更复杂的校验规则来测试，比如验证行数、 下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/qt_action.groovy`: ```groovy -test { - sql "abcdefg" - // check exception message contains - exception "errCode = 2, detailMessage = Syntax error" -} +suite("test_action", "demo") { + test { + sql "abcdefg" + // check exception message contains + exception "errCode = 2, detailMessage = Syntax error" + } -test { - sql """ + test { + sql """ select * from ( select 1 id @@ -276,33 +286,33 @@ test { ) a order by id""" - // multi check condition + // multi check condition - // check return 2 rows - rowNum 2 - // execute time must <= 5000 millisecond - time 5000 - // check result, must be 2 rows and 1 column, the first row is 1, second is 2 - result( - [[1], [2]] - ) -} + // check return 2 rows + rowNum 2 + // execute time must <= 5000 millisecond + time 5000 + // check result, must be 2 rows and 1 column, the first row is 1, second is 2 + result( + [[1], [2]] + ) + } -test { - sql "a b c d e f g" + test { + sql "a b c d e f g" - // other check will not work because already declared a check callback - exception "aaaaaaaaa" + // other check will not work because already declared a check callback + exception "aaaaaaaaa" - // callback - check { result, exception, startTime, endTime -> - // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically - assertTrue(exception != null) + // callback + check { result, exception, startTime, endTime -> + // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically + assertTrue(exception != null) + } } -} -test { - sql """ + test { + sql """ select 2 union all select 1 @@ -314,15 +324,34 @@ test { select 3 """ - check { result, ex, startTime, endTime -> - // same as order_sql(sqlStr) - result = sortRows(result) + check { result, ex, startTime, endTime -> + // same as order_sql(sqlStr) + result = sortRows(result) + + assertEquals(null, result[0][0]) + assertEquals(1, result[1][0]) + assertEquals(15, result[2][0]) + assertEquals(2, result[3][0]) + assertEquals(3, result[4][0]) + } + } + + // execute sql and order query result, then compare to iterator + def selectValues = [1, 2, 3, 4] + test { + order true + sql selectUnionAll(selectValues) + resultIterator(selectValues.iterator()) + } + + // compare to data/demo/test_action.csv + test { + order true + sql selectUnionAll(selectValues) - assertEquals(null, result[0][0]) - assertEquals(1, result[1][0]) - assertEquals(15, result[2][0]) - assertEquals(2, result[3][0]) - assertEquals(3, result[4][0]) + // you can set to http://xxx or https://xxx + // and compare to http response body + resultFile "test_action.csv" } } ``` @@ -339,35 +368,37 @@ explain action用来校验explain返回的字符串是否包含某些字符串 下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/explain_action.groovy`: ```groovy -explain { - sql("select 100") +suite("explain_action", "demo") { + explain { + sql("select 100") - // contains("OUTPUT EXPRS: 100\n") && contains("PARTITION: UNPARTITIONED\n") - contains "OUTPUT EXPRS: 100\n" - contains "PARTITION: UNPARTITIONED\n" -} + // contains("OUTPUT EXPRS: 100\n") && contains("PARTITION: UNPARTITIONED\n") + contains "OUTPUT EXPRS: 100\n" + contains "PARTITION: UNPARTITIONED\n" + } -explain { - sql("select 100") + explain { + sql("select 100") - // contains(" 100\n") && !contains("abcdefg") && !("1234567") - contains " 100\n" - notContains "abcdefg" - notContains "1234567" -} + // contains(" 100\n") && !contains("abcdefg") && !("1234567") + contains " 100\n" + notContains "abcdefg" + notContains "1234567" + } -explain { - sql("select 100") - // simple callback - check { explainStr -> explainStr.contains("abcdefg") || explainStr.contains(" 100\n") } -} + explain { + sql("select 100") + // simple callback + check { explainStr -> explainStr.contains("abcdefg") || explainStr.contains(" 100\n") } + } -explain { - sql("a b c d e") - // callback with exception and time - check { explainStr, exception, startTime, endTime -> - // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically - assertTrue(exception != null) + explain { + sql("a b c d e") + // callback with exception and time + check { explainStr, exception, startTime, endTime -> + // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically + assertTrue(exception != null) + } } } ``` @@ -387,63 +418,68 @@ streamLoad action用于导入数据 下面的样例代码存放于`${DORIS_HOME}/regression-test/suites/demo/streamLoad_action.groovy`: ```groovy -def tableName = "test_streamload_action1" - -sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - id int, - name varchar(255) - ) - DISTRIBUTED BY HASH(id) BUCKETS 1 - PROPERTIES ( - "replication_num" = "1" - ) -""" - -streamLoad { - // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy - // db 'regression_test' - table tableName - - // default label is UUID: - // set 'label' UUID.randomUUID().toString() - - // default column_separator is specify in doris fe config, usually is '\t'. - // this line change to ',' - set 'column_separator', ',' - - // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. - // also, you can stream load a http stream, e.g. http://xxx/some.csv - file 'streamload_input.csv' - - time 10000 // limit inflight 10s - - // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows -} +suite("streamLoad_action", "demo") { + def tableName = "test_streamload_action1" -// stream load 100 rows -def rowCount = 100 -def rowIt = java.util.stream.LongStream.range(0, rowCount) // [0, rowCount) - .mapToObj({i -> [i, "a_" + i]}) // change Long to List - .iterator() + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id int, + name varchar(255) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + streamLoad { + // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy + // db 'regression_test' + table tableName + + // default label is UUID: + // set 'label' UUID.randomUUID().toString() + + // default column_separator is specify in doris fe config, usually is '\t'. + // this line change to ',' + set 'column_separator', ',' + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file 'streamload_input.csv' + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + } -streamLoad { - table tableName - // also, you can upload a memory iterator - inputIterator rowIt - // if declared a check callback, the default check condition will ignore. - // So you must check all condition - check { result, exception, startTime, endTime -> - if (exception != null) { - throw exception + // stream load 100 rows + def rowCount = 100 + // range: [0, rowCount) + // or rangeClosed: [0, rowCount] + def rowIt = range(0, rowCount) + .mapToObj({i -> [i, "a_" + i]}) // change Long to List + .iterator() + + streamLoad { + table tableName + // also, you can upload a memory iterator + inputIterator rowIt + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } - log.info("Stream load result: ${result}".toString()) - def json = parseJson(result) - assertEquals("success", json.Status.toLowerCase()) - assertEquals(json.NumberTotalRows, json.NumberLoadedRows) - assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } } ``` @@ -478,8 +514,14 @@ thread, lazyCheck, events, connect, selectUnionAll # 测试demo group下的sql_action ./run-regression-test.sh --run -g demo -s sql_action +# 测试demo目录下的sql_action +./run-regression-test.sh --run -d demo -s sql_action + # 自定义配置 ./run-regression-test.sh --run -conf a=b + +# 并发执行 +./run-regression-test.sh --run -parallel 5 -suiteParallel 10 -actionParallel 20 ``` ## 使用查询结果自动生成.out文件 @@ -489,4 +531,11 @@ thread, lazyCheck, events, connect, selectUnionAll # 使用查询结果自动生成sql_action用例的.out文件，如果.out文件存在则覆盖 ./run-regression-test.sh --run sql_action -forceGenOut +``` + +## CI/CD的支持 +### TeamCity +可以使用--teamcity开启TeamCity Service Message. `-Dteamcity.enableStdErr=false`可以让错误日志也打印到stdout中，方便按顺序分析日志。 +```shell +JAVA_OPTS="-Dteamcity.enableStdErr=${enableStdErr}" ./run-regression-test.sh --teamcity --run -s ${suite} ``` \ No newline at end of file diff --git a/regression-test/conf/logback.xml b/regression-test/conf/logback.xml index 1fb1eb05a16d54..64160138ee183e 100644 --- a/regression-test/conf/logback.xml +++ b/regression-test/conf/logback.xml @@ -21,9 +21,18 @@ under the License. - - %d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] \(%F:%L\) - %msg%n - + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] \(%F:%L\) - %msg%n + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %level [%thread] \(%F:%L\) - %msg%n + + + diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 7c34d464f152ee..bfbd6ed875584b 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -38,5 +38,7 @@ dataPath = "${DORIS_HOME}/regression-test/data" testGroups = "" // empty suite will test all suite testSuites = "" +// empty directories will test all directories +testDirectories = "" customConf1 = "test_custom_conf_value" \ No newline at end of file diff --git a/regression-test/data/demo/test_action.csv b/regression-test/data/demo/test_action.csv new file mode 100644 index 00000000000000..b17865743d7739 --- /dev/null +++ b/regression-test/data/demo/test_action.csv @@ -0,0 +1,4 @@ +1 +2 +3 +4 \ No newline at end of file diff --git a/regression-test/data/demo/test_sql_file.out b/regression-test/data/demo/test_sql_file.out new file mode 100644 index 00000000000000..ab6eae27c79caa --- /dev/null +++ b/regression-test/data/demo/test_sql_file.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_sql_file -- +200 +100 +300 + diff --git a/regression-test/data/demo/test_sql_file_order.out b/regression-test/data/demo/test_sql_file_order.out new file mode 100644 index 00000000000000..6ce27676bce96f --- /dev/null +++ b/regression-test/data/demo/test_sql_file_order.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_sql_file_order -- +100 +200 +300 + diff --git a/regression-test/data/empty_table/sql/avg_decimal.out b/regression-test/data/empty_table/sql/avg_decimal.out new file mode 100644 index 00000000000000..1a9a561cf06947 --- /dev/null +++ b/regression-test/data/empty_table/sql/avg_decimal.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !avg_decimal -- +\N + diff --git a/regression-test/data/join/sql/agg_output_as_right_tale_left_outer_order.out b/regression-test/data/join/sql/agg_output_as_right_tale_left_outer_order.out new file mode 100644 index 00000000000000..e13e4599800ebf --- /dev/null +++ b/regression-test/data/join/sql/agg_output_as_right_tale_left_outer_order.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !agg_output_as_right_tale_left_outer_order -- +1 1 +2 2 + diff --git a/regression-test/framework/pom.xml b/regression-test/framework/pom.xml index bcf2e4e015acd2..8565668983fcca 100644 --- a/regression-test/framework/pom.xml +++ b/regression-test/framework/pom.xml @@ -130,6 +130,24 @@ under the License. + + false + + + *:* + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + @@ -215,5 +233,10 @@ under the License. guava 31.0.1-jre + + org.codehaus.janino + janino + 3.1.6 + \ No newline at end of file diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index b96b0e2dce013a..69d8dfcc080b68 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -27,6 +27,7 @@ import org.apache.doris.regression.util.JdbcUtils import java.sql.Connection import java.sql.DriverManager +import java.util.function.Predicate import static org.apache.doris.regression.ConfigOptions.* @@ -47,6 +48,7 @@ class Config { public String testGroups public String testSuites + public String testDirectories public boolean generateOutputFile public boolean forceGenerateOutputFile public boolean randomOrder @@ -55,8 +57,10 @@ class Config { public Set suiteWildcard = new HashSet<>() public Set groups = new HashSet<>() + public Set directories = new HashSet<>() public InetSocketAddress feHttpInetSocketAddress public Integer parallel + public Integer suiteParallel public Integer actionParallel public Integer times public boolean withOutLoadData @@ -65,7 +69,7 @@ class Config { Config(String defaultDb, String jdbcUrl, String jdbcUser, String jdbcPassword, String feHttpAddress, String feHttpUser, String feHttpPassword, - String suitePath, String dataPath, String testGroups, String testSuites) { + String suitePath, String dataPath, String testGroups, String testSuites, String testDirectories) { this.defaultDb = defaultDb this.jdbcUrl = jdbcUrl this.jdbcUser = jdbcUser @@ -77,6 +81,7 @@ class Config { this.dataPath = dataPath this.testGroups = testGroups this.testSuites = testSuites + this.testDirectories = testDirectories } static Config fromCommandLine(CommandLine cmd) { @@ -106,6 +111,11 @@ class Config { .collect({g -> g.trim()}) .findAll({g -> g != null && g.length() > 0}) .toSet() + config.directories = cmd.getOptionValue(directoriesOpt, config.testDirectories) + .split(",") + .collect({d -> d.trim()}) + .findAll({d -> d != null && d.length() > 0}) + .toSet() config.feHttpAddress = cmd.getOptionValue(feHttpAddressOpt, config.feHttpAddress) try { @@ -116,7 +126,7 @@ class Config { throw new IllegalStateException("Can not parse stream load address: ${config.feHttpAddress}", t) } - config.defaultDb = cmd.getOptionValue(jdbcOpt, config.defaultDb) + config.defaultDb = cmd.getOptionValue(defaultDbOpt, config.defaultDb) config.jdbcUrl = cmd.getOptionValue(jdbcOpt, config.jdbcUrl) config.jdbcUser = cmd.getOptionValue(userOpt, config.jdbcUser) config.jdbcPassword = cmd.getOptionValue(passwordOpt, config.jdbcPassword) @@ -125,6 +135,7 @@ class Config { config.generateOutputFile = cmd.hasOption(genOutOpt) config.forceGenerateOutputFile = cmd.hasOption(forceGenOutOpt) config.parallel = Integer.parseInt(cmd.getOptionValue(parallelOpt, "1")) + config.suiteParallel = Integer.parseInt(cmd.getOptionValue(suiteParallelOpt, "1")) config.actionParallel = Integer.parseInt(cmd.getOptionValue(actionParallelOpt, "10")) config.times = Integer.parseInt(cmd.getOptionValue(timesOpt, "1")) config.randomOrder = cmd.hasOption(randomOrderOpt) @@ -151,7 +162,8 @@ class Config { configToString(obj.suitePath), configToString(obj.dataPath), configToString(obj.testGroups), - configToString(obj.testSuites) + configToString(obj.testSuites), + configToString(obj.testDirectories) ) def declareFileNames = config.getClass() @@ -218,6 +230,11 @@ class Config { log.info("Set testGroups to '${config.testGroups}' because not specify.".toString()) } + if (config.testDirectories == null) { + config.testDirectories = "" + log.info("Set testDirectories to empty because not specify.".toString()) + } + if (config.testSuites == null) { config.testSuites = "" log.info("Set testSuites to empty because not specify.".toString()) @@ -228,6 +245,11 @@ class Config { log.info("Set parallel to 1 because not specify.".toString()) } + if (config.suiteParallel == null) { + config.suiteParallel = 1 + log.info("Set suiteParallel to 1 because not specify.".toString()) + } + if (config.actionParallel == null) { config.actionParallel = 10 log.info("Set actionParallel to 10 because not specify.".toString()) @@ -265,6 +287,25 @@ class Config { return DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword) } + Predicate getDirectoryFilter() { + return (Predicate) { String directoryName -> + if (directories.isEmpty()) { + return true + } + + String relativePath = new File(suitePath).relativePath(new File(directoryName)) + List allLevelPaths = new ArrayList<>() + String parentPath = "" + for (String pathName : relativePath.split(File.separator)) { + String currentPath = parentPath + pathName + allLevelPaths.add(currentPath) + parentPath = currentPath + File.separator + } + + return allLevelPaths.any {directories.contains(it) } + } + } + private void buildUrlWithDefaultDb() { String urlWithDb = jdbcUrl String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy index 4e89c1919c38b2..c0700dd859f7cc 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy @@ -39,10 +39,12 @@ class ConfigOptions { static Option dataOpt static Option suiteOpt static Option groupsOpt + static Option directoriesOpt static Option confOpt static Option genOutOpt static Option forceGenOutOpt static Option parallelOpt + static Option suiteParallelOpt static Option actionParallelOpt static Option randomOrderOpt static Option timesOpt @@ -130,6 +132,15 @@ class ConfigOptions { .longOpt("groups") .desc("the suite group to be test") .build() + directoriesOpt = Option.builder("d") + .argName("directories") + .required(false) + .hasArg(true) + .optionalArg(true) + .type(String.class) + .longOpt("directories") + .desc("only the use cases in these directories can be executed") + .build() feHttpAddressOpt = Option.builder("ha") .argName("address") .required(false) @@ -179,7 +190,15 @@ class ConfigOptions { .optionalArg(true) .type(String.class) .longOpt("parallel") - .desc("the num of threads running test") + .desc("the num of threads running scripts") + .build() + suiteParallelOpt = Option.builder("suiteParallel") + .argName("parallel") + .required(false) + .hasArg(true) + .type(String.class) + .longOpt("suiteParallel") + .desc("the num of threads running for suites") .build() actionParallelOpt = Option.builder("actionParallel") .argName("parallel") @@ -221,6 +240,7 @@ class ConfigOptions { .addOption(confOpt) .addOption(suiteOpt) .addOption(groupsOpt) + .addOption(directoriesOpt) .addOption(feHttpAddressOpt) .addOption(feHttpUserOpt) .addOption(feHttpPasswordOpt) @@ -228,8 +248,10 @@ class ConfigOptions { .addOption(confFileOpt) .addOption(forceGenOutOpt) .addOption(parallelOpt) + .addOption(suiteParallelOpt) .addOption(actionParallelOpt) .addOption(randomOrderOpt) + .addOption(timesOpt) .addOption(withOutLoadDataOpt) CommandLine cmd = new DefaultParser().parse(options, args, true) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy index 627a6b7732a2a4..f7674ef4b04478 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy @@ -14,22 +14,31 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + package org.apache.doris.regression +import com.google.common.collect.Lists import groovy.transform.CompileStatic import jodd.util.Wildcard -import org.apache.doris.regression.suite.Suite -import org.apache.doris.regression.suite.SuiteContext +import org.apache.doris.regression.suite.event.EventListener +import org.apache.doris.regression.suite.GroovyFileSource +import org.apache.doris.regression.suite.ScriptContext +import org.apache.doris.regression.suite.ScriptSource +import org.apache.doris.regression.suite.SqlFileSource +import org.apache.doris.regression.suite.event.RecorderEventListener +import org.apache.doris.regression.suite.event.StackEventListeners +import org.apache.doris.regression.suite.SuiteScript +import org.apache.doris.regression.suite.event.TeamcityEventListener import org.apache.doris.regression.util.Recorder import groovy.util.logging.Slf4j import org.apache.commons.cli.* -import org.apache.doris.regression.util.SuiteInfo import org.codehaus.groovy.control.CompilerConfiguration +import java.beans.Introspector import java.util.concurrent.Executors import java.util.concurrent.ExecutorService import java.util.concurrent.Future -import java.util.stream.Collectors +import java.util.function.Predicate @Slf4j @CompileStatic @@ -38,8 +47,11 @@ class RegressionTest { static ClassLoader classloader static CompilerConfiguration compileConfig static GroovyShell shell - static ExecutorService executorService - static ExecutorService actionExecutorService + static ExecutorService scriptExecutors + static ExecutorService suiteExecutors + static ExecutorService actionExecutors + static ThreadLocal threadLoadedClassNum = new ThreadLocal<>() + static final int cleanLoadedClassesThreshold = 20 static void main(String[] args) { CommandLine cmd = ConfigOptions.initCommands(args) @@ -49,150 +61,114 @@ class RegressionTest { Config config = Config.fromCommandLine(cmd) initGroovyEnv(config) + boolean success = true for (int i = 0; i < config.times; i++) { log.info("=== run ${i} time ===") - Recorder recorder = runSuites(config) - printResult(config, recorder) + Recorder recorder = runScripts(config) + success = printResult(config, recorder) + } + actionExecutors.shutdown() + suiteExecutors.shutdown() + scriptExecutors.shutdown() + log.info("Test finished") + if (!success) { + System.exit(1) } - actionExecutorService.shutdown() - executorService.shutdown() } static void initGroovyEnv(Config config) { - log.info("parallel = ${config.parallel}") + log.info("parallel = ${config.parallel}, suiteParallel = ${config.suiteParallel}, actionParallel = ${config.actionParallel}") classloader = new GroovyClassLoader() compileConfig = new CompilerConfiguration() - compileConfig.setScriptBaseClass((Suite as Class).name) + compileConfig.setScriptBaseClass((SuiteScript as Class).name) shell = new GroovyShell(classloader, new Binding(), compileConfig) - log.info("starting ${config.parallel} threads") - executorService = Executors.newFixedThreadPool(config.parallel) - actionExecutorService = Executors.newFixedThreadPool(config.actionParallel) + scriptExecutors = Executors.newFixedThreadPool(config.parallel) + suiteExecutors = Executors.newFixedThreadPool(config.suiteParallel) + actionExecutors = Executors.newFixedThreadPool(config.actionParallel) } - static List findSuiteFiles(String root) { + static List findScriptSources(String root, Predicate directoryFilter, + Predicate fileFilter) { if (root == null) { log.warn('Not specify suite path') - return new ArrayList() + return new ArrayList() } - List files = new ArrayList<>() + List sources = new ArrayList<>() // 1. generate groovy for sql, excluding ddl - new File(root).eachFileRecurse { f -> - if (f.isFile() && f.name.endsWith('.sql') && f.getParentFile().name != "ddl") { - genetate_groovy_from_sql(f) + def rootFile = new File(root) + rootFile.eachFileRecurse { f -> + if (f.isFile() && f.name.endsWith('.sql') && f.getParentFile().name != "ddl" + && fileFilter.test(f.name) && directoryFilter.test(f.getParent())) { + sources.add(new SqlFileSource(rootFile, f)) } } - // 2. collect groovy files. - new File(root).eachFileRecurse { f -> - if (f.isFile() && f.name.endsWith('.groovy')) { - files.add(f) + // 2. collect groovy sources. + rootFile.eachFileRecurse { f -> + if (f.isFile() && f.name.endsWith('.groovy') && fileFilter.test(f.name) + && directoryFilter.test(f.getParent())) { + sources.add(new GroovyFileSource(f)) } } - return files + return sources } - static void genetate_groovy_from_sql(File f) { - File groovy_file = new File(f.getAbsolutePath() + '.generated.groovy') - groovy_file.delete() - groovy_file.createNewFile() - int separatorIndex = f.name.lastIndexOf('.') - String action_name = f.name.substring(0, separatorIndex) - if (action_name.endsWith('order')) { - groovy_file.text = "order_qt_${action_name} \"\"\"${f.text}\"\"\"" - } else { - groovy_file.text = "qt_${action_name} \"\"\"${f.text}\"\"\"" + static void runScript(Config config, ScriptSource source, Recorder recorder) { + def suiteFilter = { String suiteName, String groupName -> + canRun(config, suiteName, groupName) } - } - - static String parseGroup(Config config, File suiteFile) { - // ./run-regression-test.sh -g group_name runs all groups - // whose name starting with ${group_name}. - String group = new File(config.suitePath).relativePath(suiteFile) - int separatorIndex = group.lastIndexOf(File.separator) - String groups = ","; - while (separatorIndex != -1) { - group = group.substring(0, separatorIndex) - groups += "${group}," - separatorIndex = group.lastIndexOf(File.separator) - } - // remove ',' at head and trail - groups = groups.substring(1, groups.length() - 1); - return groups; - } - - static Integer runSuite(Config config, SuiteFile sf, ExecutorService executorService, Recorder recorder) { - File file = sf.file - String suiteName = sf.suiteName - String group = sf.group - def suiteConn = config.getConnection() - new SuiteContext(file, suiteConn, executorService, config, recorder).withCloseable { context -> - Suite suite = null + def file = source.getFile() + def eventListeners = getEventListeners(config, recorder) + new ScriptContext(file, suiteExecutors, actionExecutors, + config, eventListeners, suiteFilter).start { scriptContext -> try { - log.info("Run ${suiteName} in $file".toString()) - suite = shell.parse(file) as Suite - suite.init(suiteName, group, context) - suite.run() - suite.doLazyCheck() - suite.successCallbacks.each { it() } - recorder.onSuccess(new SuiteInfo(file, group, suiteName)) - log.info("Run ${suiteName} in ${file.absolutePath} succeed".toString()) - } catch (Throwable t) { - if (suite != null) { - suite.failCallbacks.each { it() } - } - recorder.onFailure(new SuiteInfo(file, group, suiteName)) - log.error("Run ${suiteName} in ${file.absolutePath} failed".toString(), t) + SuiteScript suiteScript = source.toScript(scriptContext, shell) + suiteScript.run() } finally { - if (suite != null) { - suite.finishCallbacks.each { it() } - } + // avoid jvm metaspace oom + cleanLoadedClassesIfNecessary() } - shell.resetLoadedClasses() } - - return 0 } - static void runSuites(Config config, Recorder recorder, Closure suiteNameMatch) { - def files = findSuiteFiles(config.suitePath) - List runScripts = files.stream().map({ file -> - String suiteName = file.name.substring(0, file.name.lastIndexOf('.')) - String group = parseGroup(config, file) - return new SuiteFile(file, suiteName, group) - }).filter({ sf -> - suiteNameMatch(sf.suiteName) && canRun(config, sf.suiteName, sf.group) - }).collect(Collectors.toList()) - + static void runScripts(Config config, Recorder recorder, + Predicate directoryFilter, Predicate fileNameFilter) { + def scriptSources = findScriptSources(config.suitePath, directoryFilter, fileNameFilter) if (config.randomOrder) { - Collections.shuffle(files) + Collections.shuffle(scriptSources) } - log.info('Start to run suites') - int totalFile = runScripts.size() - def futures = new ArrayList() - runScripts.eachWithIndex { sf, i -> - log.info("[${i + 1}/${totalFile}] Run ${sf.suiteName} in ${sf.file}".toString()) - Future future = executorService.submit { - runSuite(config, sf, actionExecutorService, recorder) +// int totalFile = scriptSources.size() + + List futures = Lists.newArrayList() + scriptSources.eachWithIndex { source, i -> +// log.info("Prepare scripts [${i + 1}/${totalFile}]".toString()) + def future = scriptExecutors.submit { + runScript(config, source, recorder) } futures.add(future) } - for (Future future : futures) { + // wait all scripts + for (Future future : futures) { try { future.get() - } - catch (Throwable t) { - log.info(" exception ${t.toString()}") + } catch (Throwable t) { + // do nothing, because already save to Recorder } } } - static Recorder runSuites(Config config) { + static Recorder runScripts(Config config) { def recorder = new Recorder() + def directoryFilter = config.getDirectoryFilter() if (!config.withOutLoadData) { - runSuites(config, recorder, {suiteName -> suiteName == "load" }) + log.info('Start to run load scripts') + runScripts(config, recorder, directoryFilter, + { fileName -> fileName.substring(0, fileName.lastIndexOf(".")) == "load" }) } - runSuites(config, recorder, {suiteName -> suiteName != "load" }) + log.info('Start to run scripts') + runScripts(config, recorder, directoryFilter, + { fileName -> fileName.substring(0, fileName.lastIndexOf(".")) != "load" }) return recorder } @@ -211,28 +187,67 @@ class RegressionTest { return false } - static void printResult(Config config, Recorder recorder) { + static List getEventListeners(Config config, Recorder recorder) { + StackEventListeners listeners = new StackEventListeners() + + // RecorderEventListener **MUST BE** first listener + listeners.addListener(new RecorderEventListener(recorder)) + + // other listeners + String stdoutAppenderType = System.getProperty("stdoutAppenderType") + if (stdoutAppenderType != null && stdoutAppenderType.equalsIgnoreCase("teamcity")) { + listeners.addListener(new TeamcityEventListener()) + } + return [listeners] as List + } + + static void cleanLoadedClassesIfNecessary() { + Integer loadedClassNum = threadLoadedClassNum.get() + if (loadedClassNum == null) { + loadedClassNum = 0 + } + loadedClassNum += 1 + if (loadedClassNum >= cleanLoadedClassesThreshold) { + // release dynamic script class: ThreadGroupContext.getContext().beanInfoCache() + Introspector.flushCaches() + loadedClassNum = 0 + } + threadLoadedClassNum.set(loadedClassNum) + } + + static boolean printResult(Config config, Recorder recorder) { int allSuiteNum = recorder.successList.size() + recorder.failureList.size() int failedSuiteNum = recorder.failureList.size() - log.info("Test ${allSuiteNum} suites, failed ${failedSuiteNum} suites".toString()) + int fatalScriptNum = recorder.fatalScriptList.size() + log.info("Test ${allSuiteNum} suites, failed ${failedSuiteNum} suites, fatal ${fatalScriptNum} scripts".toString()) // print success list - { + if (!recorder.successList.isEmpty()) { String successList = recorder.successList.collect { info -> "${info.file.absolutePath}: group=${info.group}, name=${info.suiteName}" }.join('\n') - log.info("successList suites:\n${successList}".toString()) + log.info("SuccessList suites:\n${successList}".toString()) } // print failure list - if (!recorder.failureList.isEmpty()) { - def failureList = recorder.failureList.collect() { info -> - "${info.file.absolutePath}: group=${info.group}, name=${info.suiteName}" - }.join('\n') - log.info("Failure suites:\n${failureList}".toString()) + if (!recorder.failureList.isEmpty() || !recorder.fatalScriptList.isEmpty()) { + if (!recorder.failureList.isEmpty()) { + def failureList = recorder.failureList.collect() { info -> + "${info.file.absolutePath}: group=${info.group}, name=${info.suiteName}" + }.join('\n') + log.info("Failure suites:\n${failureList}".toString()) + } + if (!recorder.fatalScriptList.isEmpty()) { + def failureList = recorder.fatalScriptList.collect() { info -> + "${info.file.absolutePath}" + }.join('\n') + log.info("Fatal scripts:\n${failureList}".toString()) + } printFailed() + return false } else { printPassed() + return true } } @@ -255,19 +270,4 @@ class RegressionTest { ||_|/_/ \\_\\___|_____|_____|____/ |'''.stripMargin()) } - - static class SuiteFile { - - File file - String suiteName - String group - - SuiteFile(File file, String suiteName, String group) { - this.file = file - this.suiteName = suiteName - this.group = group - } - - } - } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy index 01eddff9f734f4..287d8136c0f0c6 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy @@ -80,15 +80,11 @@ class ExplainAction implements SuiteAction { } } catch (Throwable t) { log.error("Explain and custom check failed", t) - List resList = [context.file.getName(), 'explain', sql, t] - context.recorder.reportDiffResult(resList) throw t } } else if (result.exception != null) { String msg = "Explain failed" log.error(msg, result.exception) - List resList = [context.file.getName(), 'explain', sql, result.exception] - context.recorder.reportDiffResult(resList) throw new IllegalStateException(msg, result.exception) } else { for (String string : containsStrings) { @@ -97,8 +93,6 @@ class ExplainAction implements SuiteAction { + "but actual explain string is:\n${explainString}").toString() log.info(msg) def t = new IllegalStateException(msg) - List resList = [context.file.getName(), 'explain', sql, t] - context.recorder.reportDiffResult(resList) throw t } } @@ -108,8 +102,6 @@ class ExplainAction implements SuiteAction { + "but actual explain string is:\n${explainString}").toString() log.info(msg) def t = new IllegalStateException(msg) - List resList = [context.file.getName(), 'explain', sql, t] - context.recorder.reportDiffResult(resList) throw t } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy index 2b4348de94bb54..e3b90f2f2a9129 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy @@ -26,6 +26,8 @@ import org.apache.doris.regression.util.OutputUtils import groovy.json.JsonSlurper import groovy.util.logging.Slf4j import org.apache.http.HttpEntity +import org.apache.http.HttpStatus +import org.apache.http.client.methods.CloseableHttpResponse import org.apache.http.client.methods.RequestBuilder import org.apache.http.entity.FileEntity import org.apache.http.entity.InputStreamEntity @@ -153,7 +155,14 @@ class StreamLoadAction implements SuiteAction { } private InputStream httpGetStream(CloseableHttpClient client, String url) { - return client.execute(RequestBuilder.get(url).build()).getEntity().getContent() + CloseableHttpResponse resp = client.execute(RequestBuilder.get(url).build()) + int code = resp.getStatusLine().getStatusCode() + if (code != HttpStatus.SC_OK) { + String streamBody = EntityUtils.toString(resp.getEntity()) + throw new IllegalStateException("Get http stream failed, status code is ${code}, body:\n${streamBody}") + } + + return resp.getEntity().getContent() } private RequestBuilder prepareRequestHeader(RequestBuilder requestBuilder) { @@ -207,9 +216,6 @@ class StreamLoadAction implements SuiteAction { def respCode = resp.getStatusLine().getStatusCode() // should redirect to backend if (respCode != 307) { - List resList = [context.file.getName(), 'streamLoad', '', "Expect frontend stream load response code is 307, " + - "but meet ${respCode}\nbody: ${body}"] - context.recorder.reportDiffResult(resList) throw new IllegalStateException("Expect frontend stream load response code is 307, " + "but meet ${respCode}\nbody: ${body}") } @@ -230,10 +236,6 @@ class StreamLoadAction implements SuiteAction { String body = EntityUtils.toString(resp.getEntity()) def respCode = resp.getStatusLine().getStatusCode() if (respCode != 200) { - List resList = [context.file.getName(), 'streamLoad', '', "Expect backend stream load response code is 200, " + - "but meet ${respCode}\nbody: ${body}"] - context.recorder.reportDiffResult(resList) - throw new IllegalStateException("Expect backend stream load response code is 200, " + "but meet ${respCode}\nbody: ${body}") } @@ -242,8 +244,6 @@ class StreamLoadAction implements SuiteAction { } } catch (Throwable t) { log.info("StreamLoadAction Exception: ", t) - List resList = [context.file.getName(), 'streamLoad', '', t] - context.recorder.reportDiffResult(resList) } return responseText } @@ -253,8 +253,6 @@ class StreamLoadAction implements SuiteAction { check.call(responseText, ex, startTime, endTime) } else { if (ex != null) { - List resList = [context.file.getName(), 'streamLoad', '', ex] - context.recorder.reportDiffResult(resList) throw ex } @@ -267,19 +265,13 @@ class StreamLoadAction implements SuiteAction { String errorDetails = HttpClients.createDefault().withCloseable { client -> httpGetString(client, errorUrl) } - List resList = [context.file.getName(), 'streamLoad', '', "Stream load failed:\n${responseText}\n${errorDetails}"] - context.recorder.reportDiffResult(resList) throw new IllegalStateException("Stream load failed:\n${responseText}\n${errorDetails}") } - List resList = [context.file.getName(), 'streamLoad', '', "Stream load failed:\n${responseText}"] - context.recorder.reportDiffResult(resList) throw new IllegalStateException("Stream load failed:\n${responseText}") } long numberTotalRows = result.NumberTotalRows.toLong() long numberLoadedRows = result.NumberLoadedRows.toLong() if (numberTotalRows != numberLoadedRows) { - List resList = [context.file.getName(), 'streamLoad', '', "Stream load rows mismatch:\n${responseText}"] - context.recorder.reportDiffResult(resList) throw new IllegalStateException("Stream load rows mismatch:\n${responseText}") } @@ -289,8 +281,6 @@ class StreamLoadAction implements SuiteAction { try{ Assert.assertTrue("Expect elapsed <= ${time}, but meet ${elapsed}", elapsed <= time) } catch (Throwable t) { - List resList = [context.file.getName(), 'streamLoad', '', "Expect elapsed <= ${time}, but meet ${elapsed}"] - context.recorder.reportDiffResult(resList) throw new IllegalStateException("Expect elapsed <= ${time}, but meet ${elapsed}") } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy index f9f629b1745208..362b37c431e41a 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy @@ -17,18 +17,36 @@ package org.apache.doris.regression.action +import groovy.transform.CompileStatic import groovy.transform.stc.ClosureParams import groovy.transform.stc.FromString import groovy.util.logging.Slf4j +import org.apache.commons.io.LineIterator +import org.apache.doris.regression.util.DataUtils +import org.apache.doris.regression.util.OutputUtils +import org.apache.http.HttpStatus +import org.apache.http.client.methods.CloseableHttpResponse +import org.apache.http.client.methods.RequestBuilder +import org.apache.http.impl.client.HttpClients +import org.apache.http.util.EntityUtils + +import javax.swing.text.html.parser.Entity +import java.nio.charset.StandardCharsets import java.sql.Connection import org.apache.doris.regression.suite.SuiteContext import org.apache.doris.regression.util.JdbcUtils import org.junit.Assert +import java.util.function.Consumer + @Slf4j +@CompileStatic class TestAction implements SuiteAction { private String sql + private boolean isOrder + private String resultFileUri + private Iterator resultIterator private Object result private long time private long rowNum = -1 @@ -70,22 +88,76 @@ class TestAction implements SuiteAction { if (this.result != null) { Assert.assertEquals(this.result, result.result) } + if (this.resultIterator != null) { + String errorMsg = OutputUtils.checkOutput(this.resultIterator, result.result.iterator(), + { Object row -> + if (row instanceof List) { + return OutputUtils.toCsvString(row as List) + } else { + return OutputUtils.columnToCsvString(row) + } + }, + { List row -> OutputUtils.toCsvString(row) }, "Check failed") + if (errorMsg != null) { + throw new IllegalStateException(errorMsg) + } + } + if (this.resultFileUri != null) { + Consumer checkFunc = { InputStream inputStream -> + def lineIt = new LineIterator(new InputStreamReader(inputStream, StandardCharsets.UTF_8)) + def csvIt = new OutputUtils.CsvParserIterator(lineIt) + String errMsg = OutputUtils.checkOutput(csvIt, result.result.iterator(), + { List row -> OutputUtils.toCsvString(row as List) }, + { List row -> OutputUtils.toCsvString(row) }, + "Check failed compare to") + if (errMsg != null) { + throw new IllegalStateException(errMsg) + } + } + + if (this.resultFileUri.startsWith("http://") || this.resultFileUri.startsWith("https://")) { + log.info("Compare to http stream: ${this.resultFileUri}") + HttpClients.createDefault().withCloseable { client -> + client.execute(RequestBuilder.get(this.resultFileUri).build()).withCloseable { CloseableHttpResponse resp -> + int code = resp.getStatusLine().getStatusCode() + if (code != HttpStatus.SC_OK) { + String streamBody = EntityUtils.toString(resp.getEntity()) + throw new IllegalStateException("Get http stream failed, status code is ${code}, body:\n${streamBody}") + } + checkFunc(resp.entity.content) + } + } + } else { + String fileName = resultFileUri + if (!new File(fileName).isAbsolute()) { + fileName = new File(context.dataPath, fileName).getAbsolutePath() + } + def file = new File(fileName) + if (!file.exists()) { + log.warn("Result file not exists: ${file}".toString()) + } + log.warn("Compare to local file: ${file}".toString()) + file.newInputStream().withCloseable { inputStream -> + checkFunc(inputStream) + } + } + } } } catch (Throwable t) { - log.info("TestAction Exception: ", t) - List resList = [context.file.getName(), 'test', sql, t] - context.recorder.reportDiffResult(resList) - throw t + throw new IllegalStateException("TestAction failed, sql:\n${sql}", t) } } ActionResult doRun(Connection conn) { - Object result = null + List> result = null Throwable ex = null long startTime = System.currentTimeMillis() try { - log.info("Execute sql:\n${sql}".toString()) + log.info("Execute ${isOrder ? "order_" : ""}sql:\n${sql}".toString()) result = JdbcUtils.executeToList(conn, sql) + if (isOrder) { + result = DataUtils.sortByToString(result) + } } catch (Throwable t) { ex = t } @@ -101,6 +173,10 @@ class TestAction implements SuiteAction { this.sql = sqlSupplier.call() } + void order(boolean isOrder) { + this.isOrder = isOrder + } + void time(long time) { this.time = time } @@ -125,6 +201,22 @@ class TestAction implements SuiteAction { this.result = resultSupplier.call() } + void resultIterator(Iterator resultIterator) { + this.resultIterator = resultIterator + } + + void resultIterator(Closure> resultIteratorSupplier) { + this.resultIterator = resultIteratorSupplier.call() + } + + void resultFile(String resultFile) { + this.resultFileUri = resultFile + } + + void resultFile(Closure resultFileSupplier) { + this.resultFileUri = resultFileSupplier.call() + } + void exception(String exceptionMsg) { this.exception = exceptionMsg } @@ -138,12 +230,12 @@ class TestAction implements SuiteAction { } class ActionResult { - Object result + List> result Throwable exception long startTime long endTime - ActionResult(Object result, Throwable exception, long startTime, long endTime) { + ActionResult(List> result, Throwable exception, long startTime, long endTime) { this.result = result this.exception = exception this.startTime = startTime diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/logger/TeamcityServiceMessageEncoder.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/logger/TeamcityServiceMessageEncoder.groovy new file mode 100644 index 00000000000000..cfe9110c550100 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/logger/TeamcityServiceMessageEncoder.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.logger + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.encoder.PatternLayoutEncoder +import ch.qos.logback.classic.spi.ILoggingEvent +import org.apache.doris.regression.suite.SuiteContext +import org.apache.doris.regression.util.TeamcityUtils + +class TeamcityServiceMessageEncoder extends PatternLayoutEncoder { + static final ThreadLocal CURRENT_SUITE_CONTEXT = new ThreadLocal<>() + + private boolean enableStdErr + + TeamcityServiceMessageEncoder() { + enableStdErr = Boolean.valueOf(System.getProperty("teamcity.enableStdErr", "true")) + System.out.println("TeamcityServiceMessageEncoder: teamcity.enableStdErr=${enableStdErr}") + } + + @Override + byte[] encode(ILoggingEvent event) { + String originLog = layout.doLayout(event) + + SuiteContext suiteContext = CURRENT_SUITE_CONTEXT.get() + if (suiteContext == null) { + return convertToBytes(originLog) + } + + String serviceMessageLog + if (event.getLevel().levelInt == Level.ERROR.levelInt && enableStdErr) { + serviceMessageLog = TeamcityUtils.formatStdErr(suiteContext, originLog) + } else { + serviceMessageLog = TeamcityUtils.formatStdOut(suiteContext, originLog) + } + return convertToBytes(serviceMessageLog + "\n") + } + + private byte[] convertToBytes(String s) { + if (getCharset() == null) { + return s.getBytes() + } else { + return s.getBytes(getCharset()) + } + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptContext.groovy new file mode 100644 index 00000000000000..d994499b661e17 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptContext.groovy @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.doris.regression.Config +import org.apache.doris.regression.suite.event.EventListener + +import java.lang.reflect.UndeclaredThrowableException +import java.util.concurrent.ExecutorService +import java.util.concurrent.Phaser +import java.util.function.Function + +@Slf4j +@CompileStatic +class ScriptContext implements Closeable { + public final File file + public final Config config + public final File dataPath + public final File outputFile + public final String name + public final String flowName + public final String flowId + public final List eventListeners + public final ExecutorService suiteExecutors + public final ExecutorService actionExecutors + public final Closure suiteFilter + private long startTime + private long finishTime + private final Thread createContextThread + private final Phaser phaser = new Phaser(1) // register main script thread + + ScriptContext(File file, ExecutorService suiteExecutors, ExecutorService actionExecutors, Config config, + List eventListeners, Closure suiteFilter) { + this.file = file + this.config = config + this.name = this.flowId = new File(config.suitePath).relativePath(file) + this.flowName = '' // force set to empty string, for teamcity + this.eventListeners = eventListeners + this.suiteExecutors = suiteExecutors + this.actionExecutors = actionExecutors + this.suiteFilter = suiteFilter + this.createContextThread = Thread.currentThread() + + def path = new File(config.suitePath).relativePath(file) + def outputRelativePath = path.substring(0, path.lastIndexOf(".")) + ".out" + this.outputFile = new File(new File(config.dataPath), outputRelativePath) + this.dataPath = this.outputFile.getParentFile().getCanonicalFile() + } + + private final synchronized Suite newSuite(String suiteName, String group) { + if (Thread.currentThread() != createContextThread) { + throw new IllegalStateException("Can not create suite in another thread") + } + // if close scriptContext, the phaser will become to 1 + if (phaser.getPhase() > 0) { + throw new IllegalStateException("Can not create suite after close scriptContext") + } + + SuiteContext suiteContext = new SuiteContext(file, suiteName, group, this, + suiteExecutors, actionExecutors, config) { + @Override + void close() { + try { + super.close() + } finally { + // count down + phaser.arriveAndDeregister() + } + } + } + Suite suite = new Suite(suiteName, group, suiteContext) + // count up, register suite thread + phaser.register() + return suite + } + + synchronized final Suite createAndRunSuite(String suiteName, String group, Closure suiteBody) { + Suite suite = newSuite(suiteName, group) + + suiteExecutors.submit { + suite.context.start { + log.info("Run ${suiteName} in ${file}".toString()) + + try { + // delegate closure + suiteBody.setResolveStrategy(Closure.DELEGATE_FIRST) + suiteBody.setDelegate(suite) + + // execute body + suiteBody.call(suite) + + // check + suite.doLazyCheck() + + // success + try { + suite.successCallbacks.each { it(suite) } + } catch (Throwable t) { + throw new IllegalStateException("Run suite success callbacks failed", t) + } + + log.info("Run ${suiteName} in ${file.absolutePath} succeed".toString()) + } catch (Throwable t) { + log.error("Run ${suiteName} in ${file.absolutePath} failed".toString(), t) + try { + // fail + if (suite != null) { + suite.failCallbacks.each { it(suite) } + } + } catch (Throwable ex) { + log.error("Run suite fail callbacks failed", ex) + t.addSuppressed(ex) + } + throw t + } finally { + try { + // finish + if (suite != null) { + suite.finishCallbacks.each { it(suite) } + } + } catch (Throwable t) { + log.error("Run suite finish callbacks failed", t) + } + } + } + } + return suite + } + + public T start(Function action) { + this.startTime = System.currentTimeMillis() + eventListeners.each {it.onScriptStarted(this) } + this.withCloseable { scriptContext -> + try { + action.apply(scriptContext) + } catch (Throwable t) { + if (t instanceof UndeclaredThrowableException) { + t = ((UndeclaredThrowableException) t).undeclaredThrowable + } + log.error("Fatal: run SuiteScript in ${file} failed".toString(), t) + eventListeners.each { it.onScriptFailed(scriptContext, t) } + null + } + } + } + + synchronized void close() { + if (phaser.getPhase() > 0) { + throw new IllegalStateException("Can not close scriptContext twice") + } + // wait for all suite finished, phase += 1 + phaser.arriveAndAwaitAdvance() + + this.finishTime = System.currentTimeMillis() + long elapsed = finishTime - startTime + eventListeners.each {it.onScriptFinished(this, elapsed) } + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptInfo.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptInfo.groovy new file mode 100644 index 00000000000000..d83031f023fdb2 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptInfo.groovy @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite + +import groovy.transform.CompileStatic + +@CompileStatic +class ScriptInfo { + File file + + ScriptInfo(File file) { + this.file = file + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy new file mode 100644 index 00000000000000..297604a8467a8d --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite + +interface ScriptSource { + SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell) + File getFile() +} + +class GroovyFileSource implements ScriptSource { + private File file + + GroovyFileSource(File file) { + this.file = file + } + + @Override + SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell) { + SuiteScript suiteScript = shell.parse(file) as SuiteScript + suiteScript.init(scriptContext) + return suiteScript + } + + @Override + File getFile() { + return file + } +} + +class SqlFileSource implements ScriptSource { + private File suiteRoot + private File file + + SqlFileSource(File suiteRoot, File file) { + this.suiteRoot = suiteRoot + this.file = file + } + + String getGroup() { + return SuiteScript.getDefaultGroups(suiteRoot, file) + } + + @Override + SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell) { + String suiteName = file.name.substring(0, file.name.lastIndexOf(".")) + String groupName = getGroup() + boolean order = suiteName.endsWith("_order") + String tag = suiteName + String sql = file.text + + SuiteScript script = new SuiteScript() { + @Override + Object run() { + suite(suiteName, groupName) { + quickTest(tag, sql, order) + } + } + } + script.init(scriptContext) + return script + } + + @Override + File getFile() { + return file + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 8a36eb5c3a2dfd..767ee13fe0ffb4 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -41,11 +41,11 @@ import java.util.stream.LongStream import static org.apache.doris.regression.util.DataUtils.sortByToString -abstract class Suite extends Script implements GroovyInterceptable { - SuiteContext context - String name - String group - final Logger logger = LoggerFactory.getLogger(getClass()) +class Suite implements GroovyInterceptable { + final SuiteContext context + final String name + final String group + final Logger logger = LoggerFactory.getLogger(this.class) final List successCallbacks = new Vector<>() final List failCallbacks = new Vector<>() @@ -53,7 +53,7 @@ abstract class Suite extends Script implements GroovyInterceptable { final List lazyCheckExceptions = new Vector<>() final List lazyCheckFutures = new Vector<>() - void init(String name, String group, SuiteContext context) { + Suite(String name, String group, SuiteContext context) { this.name = name this.group = group this.context = context @@ -139,17 +139,25 @@ abstract class Suite extends Script implements GroovyInterceptable { } public ListenableFuture thread(String threadName = null, Closure actionSupplier) { - return MoreExecutors.listeningDecorator(context.executorService).submit((Callable) { + return MoreExecutors.listeningDecorator(context.actionExecutors).submit((Callable) { + long startTime = System.currentTimeMillis() def originThreadName = Thread.currentThread().name try { Thread.currentThread().setName(threadName == null ? originThreadName : threadName) + context.scriptContext.eventListeners.each { it.onThreadStarted(context) } + return actionSupplier.call() + } catch (Throwable t) { + context.scriptContext.eventListeners.each { it.onThreadFailed(context, t) } + throw t } finally { try { context.closeThreadLocal() } catch (Throwable t) { logger.warn("Close thread local context failed", t) } + long finishTime = System.currentTimeMillis() + context.scriptContext.eventListeners.each { it.onThreadFinished(context, finishTime - startTime) } Thread.currentThread().setName(originThreadName) } }) @@ -175,7 +183,7 @@ abstract class Suite extends Script implements GroovyInterceptable { } List> sql(String sqlStr, boolean isOrder = false) { - logger.info("Execute sql: ${sqlStr}".toString()) + logger.info("Execute ${isOrder ? "order_" : ""}sql: ${sqlStr}".toString()) def result = JdbcUtils.executeToList(context.getConnection(), sqlStr) if (isOrder) { result = DataUtils.sortByToString(result) @@ -249,16 +257,16 @@ abstract class Suite extends Script implements GroovyInterceptable { void runAction(SuiteAction action, Closure actionSupplier) { actionSupplier.setDelegate(action) actionSupplier.setResolveStrategy(Closure.DELEGATE_FIRST) - actionSupplier.call() + actionSupplier.call(action) action.run() } - void quickTest(String tag, String sql, boolean order = false) { - logger.info("Execute tag: ${tag}, sql: ${sql}".toString()) + void quickTest(String tag, String sql, boolean isOrder = false) { + logger.info("Execute tag: ${tag}, ${isOrder ? "order_" : ""}sql: ${sql}".toString()) if (context.config.generateOutputFile || context.config.forceGenerateOutputFile) { def result = JdbcUtils.executorToStringList(context.getConnection(), sql) - if (order) { + if (isOrder) { result = sortByToString(result) } Iterator> realResults = result.iterator() @@ -267,36 +275,29 @@ abstract class Suite extends Script implements GroovyInterceptable { writer.write(realResults, tag) } else { if (!context.outputFile.exists()) { - String res = "Missing outputFile: ${context.outputFile.getAbsolutePath()}" - List excelContentList = [context.file.getName(), context.file, context.file, res] - context.recorder.reportDiffResult(excelContentList) throw new IllegalStateException("Missing outputFile: ${context.outputFile.getAbsolutePath()}") } if (!context.getOutputIterator().hasNextTagBlock(tag)) { - String res = "Missing output block for tag '${tag}': ${context.outputFile.getAbsolutePath()}" - List excelContentList = [context.file.getName(), tag, context.file, res] - context.recorder.reportDiffResult(excelContentList) throw new IllegalStateException("Missing output block for tag '${tag}': ${context.outputFile.getAbsolutePath()}") } OutputUtils.TagBlockIterator expectCsvResults = context.getOutputIterator().next() List> realResults = JdbcUtils.executorToStringList(context.getConnection(), sql) - if (order) { + if (isOrder) { realResults = sortByToString(realResults) } String errorMsg = null try { - errorMsg = OutputUtils.checkOutput(expectCsvResults, realResults.iterator(), "Check tag '${tag}' failed") + errorMsg = OutputUtils.checkOutput(expectCsvResults, realResults.iterator(), + { row -> OutputUtils.toCsvString(row as List) }, + {row -> OutputUtils.toCsvString(row) }, + "Check tag '${tag}' failed") } catch (Throwable t) { - List excelContentList = [context.file.getName(), tag, sql.trim(), t] - context.recorder.reportDiffResult(excelContentList) - throw new IllegalStateException("Check tag '${tag}' failed", t) + throw new IllegalStateException("Check tag '${tag}' failed, sql:\n${sql}", t) } if (errorMsg != null) { - List excelContentList = [context.file.getName(), tag, sql.trim(), errorMsg] - context.recorder.reportDiffResult(excelContentList) - throw new IllegalStateException(errorMsg) + throw new IllegalStateException("Check tag '${tag}' failed:\n${errorMsg}\n\nsql:\n${sql}") } } } @@ -320,6 +321,9 @@ abstract class Suite extends Script implements GroovyInterceptable { return null } } else { + if (metaClass == null) { + println("eeee") + } // invoke origin method return metaClass.invokeMethod(this, name, args) } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy index 320a7187688ac8..e388ce937ca60a 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy @@ -20,50 +20,88 @@ package org.apache.doris.regression.suite import groovy.transform.CompileStatic import org.apache.doris.regression.Config import org.apache.doris.regression.util.OutputUtils -import org.apache.doris.regression.util.Recorder import groovy.util.logging.Slf4j +import java.lang.reflect.UndeclaredThrowableException import java.sql.Connection import java.sql.DriverManager import java.util.concurrent.ExecutorService +import java.util.function.Function @Slf4j @CompileStatic class SuiteContext implements Closeable { public final File file - private final Connection conn + public final String suiteName + public final String group public final ThreadLocal threadLocalConn = new ThreadLocal<>() public final Config config public final File dataPath public final File outputFile + public final ScriptContext scriptContext + public final String flowName + public final String flowId public final ThreadLocal threadLocalOutputIterator = new ThreadLocal<>() - public final ExecutorService executorService - public final Recorder recorder -// public final File tmpOutputPath + public final ExecutorService suiteExecutors + public final ExecutorService actionExecutors private volatile OutputUtils.OutputBlocksWriter outputBlocksWriter + private long startTime + private long finishTime + private volatile Throwable throwable - SuiteContext(File file, Connection conn, ExecutorService executorService, Config config, Recorder recorder) { + SuiteContext(File file, String suiteName, String group, ScriptContext scriptContext, + ExecutorService suiteExecutors, ExecutorService actionExecutors, Config config) { this.file = file - this.conn = conn + this.suiteName = suiteName + this.group = group this.config = config - this.executorService = executorService - this.recorder = recorder + this.scriptContext = scriptContext + + String packageName = getPackageName() + String className = getClassName() + this.flowName = "${packageName}.${className}.${suiteName}" + this.flowId = "${scriptContext.flowId}#${suiteName}" + this.suiteExecutors = suiteExecutors + this.actionExecutors = actionExecutors def path = new File(config.suitePath).relativePath(file) def outputRelativePath = path.substring(0, path.lastIndexOf(".")) + ".out" this.outputFile = new File(new File(config.dataPath), outputRelativePath) this.dataPath = this.outputFile.getParentFile().getCanonicalFile() -// def dataParentPath = new File(config.dataPath).parentFile.absolutePath -// def tmpOutputPath = "${dataParentPath}/tmp_output/${outputRelativePath}".toString() -// this.tmpOutputPath = new File(tmpOutputPath) + } + + String getPackageName() { + String packageName = scriptContext.name + int dirSplitPos = packageName.lastIndexOf(File.separator) + if (dirSplitPos != -1) { + packageName = packageName.substring(0, dirSplitPos) + } + packageName = packageName.replace(File.separator, ".") + return packageName + } + + String getClassName() { + String scriptFileName = scriptContext.file.name + int suffixPos = scriptFileName.lastIndexOf(".") + String className = scriptFileName + if (suffixPos != -1) { + className = scriptFileName.substring(0, suffixPos) + } + return className + } + + // compatible to context.conn + Connection getConn() { + return getConnection() } Connection getConnection() { def threadConn = threadLocalConn.get() - if (threadConn != null) { - return threadConn + if (threadConn == null) { + threadConn = config.getConnection() + threadLocalConn.set(threadConn) } - return this.conn + return threadConn } public T connect(String user, String password, String url, Closure actionSupplier) { @@ -122,8 +160,40 @@ class SuiteContext implements Closeable { void closeThreadLocal() { def outputIterator = threadLocalOutputIterator.get() if (outputIterator != null) { - outputIterator.close() threadLocalOutputIterator.remove() + try { + outputIterator.close() + } catch (Throwable t) { + log.warn("Close outputIterator failed", t) + } + } + + Connection conn = threadLocalConn.get() + if (conn != null) { + threadLocalConn.remove() + try { + conn.close() + } catch (Throwable t) { + log.warn("Close connection failed", t) + } + } + } + + public T start(Function func) { + this.startTime = System.currentTimeMillis() + scriptContext.eventListeners.each { it.onSuiteStarted(this) } + + this.withCloseable {suiteContext -> + try { + func.apply(suiteContext) + } catch (Throwable t) { + if (t instanceof UndeclaredThrowableException) { + t = ((UndeclaredThrowableException) t).undeclaredThrowable + } + scriptContext.eventListeners.each { it.onSuiteFailed(this, t) } + throwable = t + null + } } } @@ -135,10 +205,8 @@ class SuiteContext implements Closeable { outputBlocksWriter.close() } - try { - conn.close() - } catch (Throwable t) { - log.warn("Close connection failed", t) - } + this.finishTime = System.currentTimeMillis() + long elapsed = finishTime - startTime + scriptContext.eventListeners.each { it.onSuiteFinished(this, throwable == null, elapsed) } } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SuiteInfo.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteInfo.groovy similarity index 96% rename from regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SuiteInfo.groovy rename to regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteInfo.groovy index 589d5b882c91e5..2511854db59bca 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/SuiteInfo.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteInfo.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.regression.util +package org.apache.doris.regression.suite import groovy.transform.CompileStatic diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteScript.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteScript.groovy new file mode 100644 index 00000000000000..7122b37fc18eb2 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteScript.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite + +import groovy.transform.CompileStatic +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +@CompileStatic +abstract class SuiteScript extends Script { + public ScriptContext context + public final Logger logger = LoggerFactory.getLogger(getClass()) + + void init(ScriptContext scriptContext) { + this.context = scriptContext + } + + void suite(String suiteName, String group = getDefaultGroups(new File(context.config.suitePath), context.file), Closure suiteBody) { + if (!context.suiteFilter.call(suiteName, group)) { + return + } + + try { + context.createAndRunSuite(suiteName, group, suiteBody) + } catch (Throwable t) { + logger.warn("Unexcept exception when run ${suiteName} in ${context.file.absolutePath} failed", t) + } + } + + static String getDefaultGroups(File suiteRoot, File scriptFile) { + String path = suiteRoot.relativePath(scriptFile.parentFile) + List groups = ["default"] + + String parentGroup = "" + + path.split(File.separator) + .collect {it.trim()} + .findAll {it != "." && it != ".." && !it.isEmpty()} + .each { + String currentGroup = parentGroup + it + groups.add(currentGroup) + parentGroup = currentGroup + "/" + } + return groups.join(",") + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/EventListener.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/EventListener.groovy new file mode 100644 index 00000000000000..fad3c9646e0348 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/EventListener.groovy @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite.event + +import groovy.transform.CompileStatic +import org.apache.doris.regression.suite.ScriptContext +import org.apache.doris.regression.suite.SuiteContext + +@CompileStatic +interface EventListener { + void onScriptStarted(ScriptContext scriptContext) + void onScriptFailed(ScriptContext scriptContext, Throwable t) + void onScriptFinished(ScriptContext scriptContext, long elapsed) + void onSuiteStarted(SuiteContext suiteContext) + void onSuiteFailed(SuiteContext suiteContext, Throwable t) + void onSuiteFinished(SuiteContext suiteContext, boolean success, long elapsed) + void onThreadStarted(SuiteContext suiteContext) + void onThreadFailed(SuiteContext suiteContext, Throwable t) + void onThreadFinished(SuiteContext suiteContext, long elapsed) +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/RecorderEventListener.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/RecorderEventListener.groovy new file mode 100644 index 00000000000000..665a76bb280563 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/RecorderEventListener.groovy @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite.event + +import groovy.transform.CompileStatic +import org.apache.doris.regression.suite.ScriptContext +import org.apache.doris.regression.suite.ScriptInfo +import org.apache.doris.regression.suite.SuiteContext +import org.apache.doris.regression.suite.SuiteInfo +import org.apache.doris.regression.util.Recorder + +@CompileStatic +class RecorderEventListener implements EventListener { + private final Recorder recorder + + RecorderEventListener(Recorder recorder) { + this.recorder = recorder + } + + @Override + void onScriptStarted(ScriptContext scriptContext) { + + } + + @Override + void onScriptFailed(ScriptContext scriptContext, Throwable t) { + recorder.onFatal(new ScriptInfo(scriptContext.file)) + } + + @Override + void onScriptFinished(ScriptContext scriptContext, long elapsed) { + + } + + @Override + void onSuiteStarted(SuiteContext suiteContext) { + + } + + @Override + void onSuiteFailed(SuiteContext suiteContext, Throwable t) { + recorder.onFailure(new SuiteInfo(suiteContext.scriptContext.file, suiteContext.group, suiteContext.suiteName)) + } + + @Override + void onSuiteFinished(SuiteContext suiteContext, boolean success, long elapsed) { + if (success) { + recorder.onSuccess(new SuiteInfo(suiteContext.scriptContext.file, suiteContext.group, suiteContext.suiteName)) + } + } + + @Override + void onThreadStarted(SuiteContext suiteContext) { + + } + + @Override + void onThreadFailed(SuiteContext suiteContext, Throwable t) { + + } + + @Override + void onThreadFinished(SuiteContext suiteContext, long elapsed) { + + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/StackEventListeners.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/StackEventListeners.groovy new file mode 100644 index 00000000000000..236d3be1d637f6 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/StackEventListeners.groovy @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite.event + +import groovy.transform.CompileStatic +import org.apache.doris.regression.suite.ScriptContext +import org.apache.doris.regression.suite.SuiteContext + +import java.util.function.Consumer + +@CompileStatic +class StackEventListeners implements EventListener { + private final Stack listeners = new Stack<>() + + synchronized void addListener(EventListener eventListener) { + this.listeners.push(eventListener) + } + + synchronized void foreach(Consumer func) { + for (int i = listeners.size() - 1; i >= 0; i--) { + func.accept(listeners.get(i)) + } + } + + @Override + synchronized void onScriptStarted(ScriptContext scriptContext) { + foreach { + it.onScriptStarted(scriptContext) + } + } + + @Override + synchronized void onScriptFailed(ScriptContext scriptContext, Throwable t) { + foreach { + it.onScriptFailed(scriptContext, t) + } + } + + @Override + synchronized void onScriptFinished(ScriptContext scriptContext, long elapsed) { + foreach { + it.onScriptFinished(scriptContext, elapsed) + } + } + + @Override + synchronized void onSuiteStarted(SuiteContext suiteContext) { + foreach { + it.onSuiteStarted(suiteContext) + } + } + + @Override + synchronized void onSuiteFailed(SuiteContext suiteContext, Throwable t) { + foreach { + it.onSuiteFailed(suiteContext, t) + } + } + + @Override + synchronized void onSuiteFinished(SuiteContext suiteContext, boolean success, long elapsed) { + foreach { + it.onSuiteFinished(suiteContext, success, elapsed) + } + } + + @Override + synchronized void onThreadStarted(SuiteContext suiteContext) { + foreach { + it.onThreadStarted(suiteContext) + } + } + + @Override + synchronized void onThreadFailed(SuiteContext suiteContext, Throwable t) { + foreach { + it.onThreadFailed(suiteContext, t) + } + } + + @Override + synchronized void onThreadFinished(SuiteContext suiteContext, long elapsed) { + foreach { + it.onThreadFinished(suiteContext, elapsed) + } + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/TeamcityEventListener.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/TeamcityEventListener.groovy new file mode 100644 index 00000000000000..98768da1348acc --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/event/TeamcityEventListener.groovy @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.suite.event + +import com.google.common.base.Throwables +import groovy.transform.CompileStatic +import org.apache.doris.regression.logger.TeamcityServiceMessageEncoder +import org.apache.doris.regression.suite.ScriptContext +import org.apache.doris.regression.suite.SuiteContext +import org.apache.doris.regression.util.LoggerUtils +import org.apache.doris.regression.util.TeamcityUtils + +@CompileStatic +class TeamcityEventListener implements EventListener { + @Override + void onScriptStarted(ScriptContext scriptContext) { + // do nothing + } + + @Override + void onScriptFailed(ScriptContext scriptContext, Throwable t) { + // do nothing + } + + @Override + void onScriptFinished(ScriptContext scriptContext, long elapsed) { + // do nothing + } + + @Override + void onSuiteStarted(SuiteContext suiteContext) { + TeamcityServiceMessageEncoder.CURRENT_SUITE_CONTEXT.set(suiteContext) + TeamcityUtils.testStarted(suiteContext) + } + + @Override + void onSuiteFailed(SuiteContext suiteContext, Throwable t) { + def (Integer errorLine, String errorInfo) = LoggerUtils.getErrorInfo(t, suiteContext.scriptContext.file) + String errorMsg = errorInfo == null + ? "Exception in ${suiteContext.scriptContext.name}:" + : "Exception in ${suiteContext.scriptContext.name}(line ${errorLine}):\n\n${errorInfo}\n\nException:" + def stackTrace = Throwables.getStackTraceAsString(t) + TeamcityUtils.testFailed(suiteContext, errorMsg, stackTrace) + } + + @Override + void onSuiteFinished(SuiteContext suiteContext, boolean success, long elapsed) { + TeamcityServiceMessageEncoder.CURRENT_SUITE_CONTEXT.remove() + TeamcityUtils.testFinished(suiteContext, elapsed) + } + + @Override + void onThreadStarted(SuiteContext suiteContext) { + TeamcityServiceMessageEncoder.CURRENT_SUITE_CONTEXT.set(suiteContext) + } + + @Override + void onThreadFailed(SuiteContext suiteContext, Throwable t) { + // do nothing + } + + @Override + void onThreadFinished(SuiteContext suiteContext, long elapsed) { + TeamcityServiceMessageEncoder.CURRENT_SUITE_CONTEXT.remove() + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/LoggerUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/LoggerUtils.groovy new file mode 100644 index 00000000000000..2bc01e6f4ba70f --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/LoggerUtils.groovy @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +class LoggerUtils { + static Tuple2 getErrorInfo(Throwable t, File file) { + if (file.name.endsWith(".groovy")) { + int lineNumber = -1 + for (def st : t.getStackTrace()) { + if (Objects.equals(st.fileName, file.name)) { + lineNumber = st.getLineNumber() + break + } + } + if (lineNumber == -1) { + return new Tuple2(null, null) + } + + List lines = file.text.split("\n").toList() + String errorPrefixText = lines.subList(Math.max(0, lineNumber - 10), lineNumber).join("\n") + String errorSuffixText = lines.subList(lineNumber, Math.min(lines.size(), lineNumber + 10)).join("\n") + String errorText = "${errorPrefixText}\n^^^^^^^^^^^^^^^^^^^^^^^^^^ERROR LINE^^^^^^^^^^^^^^^^^^^^^^^^^^\n${errorSuffixText}".toString() + return new Tuple2(lineNumber, errorText) + } else { + return new Tuple2(null, null) + } + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy index ba37e33215cc88..c30c893474b3fb 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy @@ -24,8 +24,17 @@ import org.apache.commons.csv.CSVPrinter import org.apache.commons.csv.CSVRecord import org.apache.commons.io.LineIterator +import java.util.function.Function + @CompileStatic class OutputUtils { + static String columnToCsvString(Object column) { + StringWriter writer = new StringWriter() + def printer = new CSVPrinter(new PrintWriter(writer), CSVFormat.MYSQL) + printer.print(column) + return writer.toString() + } + static String toCsvString(List row) { StringWriter writer = new StringWriter() def printer = new CSVPrinter(new PrintWriter(writer), CSVFormat.MYSQL) @@ -35,23 +44,27 @@ class OutputUtils { return writer.toString() } - static String checkOutput(Iterator> expect, Iterator> real, String info) { + static String checkOutput(Iterator expect, Iterator real, + Function transform1, Function transform2, + String info) { + int line = 1 while (true) { if (expect.hasNext() && !real.hasNext()) { - return "${info}, result mismatch, real line is empty, but expect is ${expect.next()}" + return "${info}, line ${line} mismatch, real line is empty, but expect is ${transform1(expect.next())}" } if (!expect.hasNext() && real.hasNext()) { - return "${info}, result mismatch, expect line is empty, but real is ${toCsvString(real.next())}" + return "${info}, line ${line} mismatch, expect line is empty, but real is ${transform2(real.next())}" } if (!expect.hasNext() && !real.hasNext()) { break } - def expectCsvString = toCsvString(expect.next() as List) - def realCsvString = toCsvString(real.next()) + def expectCsvString = transform1.apply(expect.next()) + def realCsvString = transform2.apply(real.next()) if (!expectCsvString.equals(realCsvString)) { - return "${info}, result mismatch.\nExpect line is: ${expectCsvString}\nBut real is : ${realCsvString}" + return "${info}, line ${line} mismatch.\nExpect line is: ${expectCsvString}\nBut real is : ${realCsvString}" } + line++ } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy index 34f3e82e92cd29..2e976e3021392c 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy @@ -18,11 +18,14 @@ package org.apache.doris.regression.util import groovy.transform.CompileStatic +import org.apache.doris.regression.suite.ScriptInfo +import org.apache.doris.regression.suite.SuiteInfo @CompileStatic class Recorder { public final List successList = new Vector<>() public final List failureList = new Vector<>() + public final List fatalScriptList = new Vector<>() void onSuccess(SuiteInfo suiteInfo) { successList.add(suiteInfo) @@ -32,7 +35,7 @@ class Recorder { failureList.add(suiteInfo) } - void reportDiffResult(List res) { - // TODO + void onFatal(ScriptInfo scriptInfo) { + fatalScriptList.add(scriptInfo) } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/TeamcityUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/TeamcityUtils.groovy new file mode 100644 index 00000000000000..ace3253d53a865 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/TeamcityUtils.groovy @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import groovy.transform.CompileStatic +import org.apache.doris.regression.suite.ScriptContext +import org.apache.doris.regression.suite.SuiteContext +import org.apache.tools.ant.util.DateUtils + +@CompileStatic +class TeamcityUtils { + static String formatNow() { + return DateUtils.format(System.currentTimeMillis(), "yyyy-MM-dd'T'HH:mm:ss.SSSZ") + } + + static String formatStdOut(SuiteContext suiteContext, String msg) { + String timestamp = formatNow() + return "##teamcity[testStdOut name='${suiteContext.flowName}' out='${escape(msg)}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']" + } + + static String formatStdErr(SuiteContext suiteContext, String msg) { + String timestamp = formatNow() + return "##teamcity[testStdErr name='${suiteContext.flowName}' out='${escape(msg)}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']" + } + +// static void testSuiteStarted(ScriptContext scriptContext) { +// String timestamp = formatNow() +// println("##teamcity[flowStarted flowId='${scriptContext.flowId}' timestamp='${timestamp}']") +// println("##teamcity[testSuiteStarted name='${scriptContext.flowName}' flowId='${scriptContext.flowId}' timestamp='${timestamp}']") +// } + +// static void testSuiteFinished(ScriptContext scriptContext) { +// String timestamp = formatNow() +// println("##teamcity[testSuiteFinished name='${scriptContext.flowName}' flowId='${scriptContext.flowId}' timestamp='${timestamp}']") +// println("##teamcity[flowFinished flowId='${scriptContext.flowId}' timestamp='${timestamp}']") +// } + +// static void testStarted(SuiteContext suiteContext) { +// String timestamp = formatNow() +// println("##teamcity[flowStarted flowId='${suiteContext.flowId}' parent='${suiteContext.scriptContext.flowId}' timestamp='${timestamp}']") +// println("##teamcity[testStarted name='${suiteContext.flowName}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']") +// } + + static void testStarted(SuiteContext suiteContext) { + String timestamp = formatNow() + println("##teamcity[flowStarted flowId='${suiteContext.flowId}' timestamp='${timestamp}']") + println("##teamcity[testStarted name='${suiteContext.flowName}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']") + } + + static void testFailed(SuiteContext suiteContext, String msg, String details) { + String timestamp = formatNow() + println("##teamcity[testFailed name='${suiteContext.flowName}' message='${escape(msg)}' flowId='${suiteContext.flowId}' details='${escape(details)}' timestamp='${timestamp}']") + } + + static void testFinished(SuiteContext suiteContext, long elapsed) { + String timestamp = formatNow() + println("##teamcity[testFinished name='${suiteContext.flowName}' flowId='${suiteContext.flowId}' duration='${elapsed}' timestamp='${timestamp}']") + println("##teamcity[flowFinished flowId='${suiteContext.flowId}' timestamp='${timestamp}']") + } + + static String escape(String str) { + StringBuilder sb = new StringBuilder() + char[] chars = str.toCharArray() + for (int i = 0; i < chars.length; ++i) { + char c = chars[i] + + switch (c) { + case '|': sb.append("||"); break + case '\'': sb.append("|'"); break + case '[': sb.append("|["); break + case ']': sb.append("|]"); break + case '\n': sb.append("|n"); break + case '\r': sb.append("|r"); break + case '\\': + if (i + 5 < chars.length && chars[i + 1] == 'u' && isHex(chars[i + 2]) && isHex(chars[i + 3]) && isHex(chars[i + 4]) && isHex(chars[i + 5])) { + sb.append("|0x") + ++i + break + } + default: sb.append(c) + } + } + return sb.toString() + } + + static boolean isHex(char c) { + return (('0' as char) <= c && c <= ('9' as char)) || (('A' as char) <= c && c <= ('F' as char)) || (('a' as char) <= c && c <= ('f' as char)) + } +} diff --git a/regression-test/framework/src/main/groovy/suite.gdsl b/regression-test/framework/src/main/groovy/suite.gdsl index f39d6614a59fbe..a271168d120807 100644 --- a/regression-test/framework/src/main/groovy/suite.gdsl +++ b/regression-test/framework/src/main/groovy/suite.gdsl @@ -22,14 +22,24 @@ def suiteContext = context( filetypes: ["groovy"] ) +def suiteScriptClassName = "org.apache.doris.regression.suite.SuiteScript" def suiteClassName = "org.apache.doris.regression.suite.Suite" +// bind suite +contributor([suiteContext]) { + if (!enclosingCall("suite")) { + delegatesTo(findClass(suiteScriptClassName)) + } +} + def bindAction = { actionName, actionClassName -> def closureBody = context(scope: closureScope(isArg: false)) contributor([closureBody]) { - if (enclosingCall(actionName)) { - def actionClass = findClass(actionClassName) - delegatesTo(actionClass) + if (enclosingCall("suite")) { + if (enclosingCall(actionName)) { + def actionClass = findClass(actionClassName) + delegatesTo(actionClass) + } } } } @@ -40,44 +50,48 @@ bindAction("streamLoad", "org.apache.doris.regression.action.StreamLoadAction") // bind qt_xxx and order_qt_xxx methods contributor([suiteContext]) { - def place = getPlace() - if (place == null || !place.getClass().getName().contains("GrReferenceExpressionImpl")) { - return - } - def invokeMethodName = place.getQualifiedReferenceName() - if (invokeMethodName == null) { - return - } - if (invokeMethodName.startsWith("qt_") || invokeMethodName.startsWith("order_qt_")) { - def suiteClass = findClass(suiteClassName) - def quickTestMethods = suiteClass.findMethodsByName("quickTest") - method(name: invokeMethodName, bindsTo: quickTestMethods[0]) + if (enclosingCall("suite")) { + def place = getPlace() + if (place == null || !place.getClass().getName().contains("GrReferenceExpressionImpl")) { + return + } + def invokeMethodName = place.getQualifiedReferenceName() + if (invokeMethodName == null) { + return + } + if (invokeMethodName.startsWith("qt_") || invokeMethodName.startsWith("order_qt_")) { + def suiteClass = findClass(suiteClassName) + def quickTestMethods = suiteClass.findMethodsByName("quickTest") + method(name: invokeMethodName, bindsTo: quickTestMethods[0]) + } } } contributor([suiteContext]) { - // bind assertXxx - def assertionsClass = findClass("org.junit.jupiter.api.Assertions") - delegatesTo(assertionsClass) + if (enclosingCall("suite")) { + // bind assertXxx + def assertionsClass = findClass("org.junit.jupiter.api.Assertions") + delegatesTo(assertionsClass) - if (enclosingCall("check") || - (!enclosingCall("test") && - !enclosingCall("explain") && - !enclosingCall("streamLoad"))) { - // bind other suite method and field - def suiteClass = findClass(suiteClassName) - delegatesTo(suiteClass) + if (enclosingCall("check") || + (!enclosingCall("test") && + !enclosingCall("explain") && + !enclosingCall("streamLoad"))) { + // bind other suite method and field + def suiteClass = findClass(suiteClassName) + delegatesTo(suiteClass) - // bind try_xxx - suiteClass.methods.each { m -> - if (m.isConstructor()) { - return - } - def parameters = m.getParameterList().getParameters().collectEntries { p -> - [p.name, p.getType().getPresentableText()] + // bind try_xxx + suiteClass.methods.each { m -> + if (m.isConstructor()) { + return + } + def parameters = m.getParameterList().getParameters().collectEntries { p -> + [p.name, p.getType().getPresentableText()] + } + def returnType = m.returnType.getPresentableText() + method(name: "try_${m.name}", bindsTo: m, params: parameters, type: returnType) } - def returnType = m.returnType.getPresentableText() - method(name: "try_${m.name}", bindsTo: m, params: parameters, type: returnType) } } } \ No newline at end of file diff --git a/regression-test/suites/aggregate/aggregate.groovy b/regression-test/suites/aggregate/aggregate.groovy index 948243e0873d4c..0fa1d52518f39c 100644 --- a/regression-test/suites/aggregate/aggregate.groovy +++ b/regression-test/suites/aggregate/aggregate.groovy @@ -19,83 +19,85 @@ // /testing/trino-product-tests/src/main/resources/sql-tests/testcases/aggregate // and modified by Doris. -def tableName = "datetype" +suite("aggregate") { + def tableName = "datetype" -sql """ DROP TABLE IF EXISTS ${tableName} """ -sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - c_bigint bigint, - c_double double, - c_string string, - c_date date, - c_timestamp datetime, - c_boolean boolean, - c_short_decimal decimal(5,2), - c_long_decimal decimal(27,9) - ) - DUPLICATE KEY(c_bigint) - DISTRIBUTED BY HASH(c_bigint) BUCKETS 1 - PROPERTIES ( - "replication_num" = "1" - ) -""" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + c_bigint bigint, + c_double double, + c_string string, + c_date date, + c_timestamp datetime, + c_boolean boolean, + c_short_decimal decimal(5,2), + c_long_decimal decimal(27,9) + ) + DUPLICATE KEY(c_bigint) + DISTRIBUTED BY HASH(c_bigint) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ -streamLoad { - // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy - // db 'regression_test' - table tableName + streamLoad { + // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy + // db 'regression_test' + table tableName - // default label is UUID: - // set 'label' UUID.randomUUID().toString() + // default label is UUID: + // set 'label' UUID.randomUUID().toString() - // default column_separator is specify in doris fe config, usually is '\t'. - // this line change to ',' - set 'column_separator', '|' + // default column_separator is specify in doris fe config, usually is '\t'. + // this line change to ',' + set 'column_separator', '|' - // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. - // also, you can stream load a http stream, e.g. http://xxx/some.csv - file 'datetype.csv' + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file 'datetype.csv' - time 10000 // limit inflight 10s + time 10000 // limit inflight 10s - // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows - // if declared a check callback, the default check condition will ignore. - // So you must check all condition - check { result, exception, startTime, endTime -> - if (exception != null) { - throw exception + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } - log.info("Stream load result: ${result}".toString()) - def json = parseJson(result) - assertEquals("success", json.Status.toLowerCase()) - assertEquals(json.NumberTotalRows, json.NumberLoadedRows) - assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } -} -qt_aggregate """ select max(upper(c_string)), min(upper(c_string)) from ${tableName} """ -qt_aggregate """ select avg(c_bigint), avg(c_double) from ${tableName} """ -qt_aggregate """ select avg(distinct c_bigint), avg(distinct c_double) from ${tableName} """ -qt_aggregate """ select count(c_bigint),count(c_double),count(c_string),count(c_date),count(c_timestamp),count(c_boolean) from ${tableName} """ -qt_aggregate """ select count(distinct c_bigint),count(distinct c_double),count(distinct c_string),count(distinct c_date),count(distinct c_timestamp),count(distinct c_boolean) from ${tableName} """ -qt_aggregate """ select max(c_bigint), max(c_double),max(c_string), max(c_date), max(c_timestamp) from ${tableName} """ -qt_aggregate """ select min(c_bigint), min(c_double), min(c_string), min(c_date), min(c_timestamp) from ${tableName} """ -qt_aggregate """ select count(c_string), max(c_double), avg(c_bigint) from ${tableName} """ -qt_aggregate """ select stddev_pop(c_bigint), stddev_pop(c_double) from ${tableName} """ -qt_aggregate """ select stddev_pop(distinct c_bigint), stddev_pop(c_double) from ${tableName} """ -qt_aggregate """ select stddev_pop(c_bigint), stddev_pop(distinct c_double) from ${tableName} """ -qt_aggregate """ select stddev_samp(c_bigint), stddev_samp(c_double) from ${tableName} """ -qt_aggregate """ select stddev_samp(distinct c_bigint), stddev_samp(c_double) from ${tableName} """ -qt_aggregate """ select stddev_samp(c_bigint), stddev_samp(distinct c_double) from ${tableName} """ -qt_aggregate """ select sum(c_bigint), sum(c_double) from ${tableName} """ -qt_aggregate """ select sum(distinct c_bigint), sum(distinct c_double) from ${tableName} """ -qt_aggregate """ select var_pop(c_bigint), var_pop(c_double) from ${tableName} """ -qt_aggregate """ select var_pop(distinct c_bigint), var_pop(c_double) from ${tableName} """ -qt_aggregate """ select var_pop(c_bigint), var_pop(distinct c_double) from ${tableName} """ -qt_aggregate """ select var_samp(c_bigint), var_samp(c_double) from ${tableName} """ -qt_aggregate """ select var_samp(distinct c_bigint), var_samp(c_double) from ${tableName} """ -qt_aggregate """ select var_samp(c_bigint), var_samp(distinct c_double) from ${tableName} """ -qt_aggregate """ select variance(c_bigint), variance(c_double) from ${tableName} """ -qt_aggregate """ select variance(distinct c_bigint), variance(c_double) from ${tableName} """ -qt_aggregate """ select variance(c_bigint), variance(distinct c_double) from ${tableName} """ + qt_aggregate """ select max(upper(c_string)), min(upper(c_string)) from ${tableName} """ + qt_aggregate """ select avg(c_bigint), avg(c_double) from ${tableName} """ + qt_aggregate """ select avg(distinct c_bigint), avg(distinct c_double) from ${tableName} """ + qt_aggregate """ select count(c_bigint),count(c_double),count(c_string),count(c_date),count(c_timestamp),count(c_boolean) from ${tableName} """ + qt_aggregate """ select count(distinct c_bigint),count(distinct c_double),count(distinct c_string),count(distinct c_date),count(distinct c_timestamp),count(distinct c_boolean) from ${tableName} """ + qt_aggregate """ select max(c_bigint), max(c_double),max(c_string), max(c_date), max(c_timestamp) from ${tableName} """ + qt_aggregate """ select min(c_bigint), min(c_double), min(c_string), min(c_date), min(c_timestamp) from ${tableName} """ + qt_aggregate """ select count(c_string), max(c_double), avg(c_bigint) from ${tableName} """ + qt_aggregate """ select stddev_pop(c_bigint), stddev_pop(c_double) from ${tableName} """ + qt_aggregate """ select stddev_pop(distinct c_bigint), stddev_pop(c_double) from ${tableName} """ + qt_aggregate """ select stddev_pop(c_bigint), stddev_pop(distinct c_double) from ${tableName} """ + qt_aggregate """ select stddev_samp(c_bigint), stddev_samp(c_double) from ${tableName} """ + qt_aggregate """ select stddev_samp(distinct c_bigint), stddev_samp(c_double) from ${tableName} """ + qt_aggregate """ select stddev_samp(c_bigint), stddev_samp(distinct c_double) from ${tableName} """ + qt_aggregate """ select sum(c_bigint), sum(c_double) from ${tableName} """ + qt_aggregate """ select sum(distinct c_bigint), sum(distinct c_double) from ${tableName} """ + qt_aggregate """ select var_pop(c_bigint), var_pop(c_double) from ${tableName} """ + qt_aggregate """ select var_pop(distinct c_bigint), var_pop(c_double) from ${tableName} """ + qt_aggregate """ select var_pop(c_bigint), var_pop(distinct c_double) from ${tableName} """ + qt_aggregate """ select var_samp(c_bigint), var_samp(c_double) from ${tableName} """ + qt_aggregate """ select var_samp(distinct c_bigint), var_samp(c_double) from ${tableName} """ + qt_aggregate """ select var_samp(c_bigint), var_samp(distinct c_double) from ${tableName} """ + qt_aggregate """ select variance(c_bigint), variance(c_double) from ${tableName} """ + qt_aggregate """ select variance(distinct c_bigint), variance(c_double) from ${tableName} """ + qt_aggregate """ select variance(c_bigint), variance(distinct c_double) from ${tableName} """ +} \ No newline at end of file diff --git a/regression-test/suites/correctness/test_select_constant.groovy b/regression-test/suites/correctness/test_select_constant.groovy index 2bf7589167e586..6368d3208304b8 100644 --- a/regression-test/suites/correctness/test_select_constant.groovy +++ b/regression-test/suites/correctness/test_select_constant.groovy @@ -1 +1,3 @@ -qt_select1 'select 100, "test", date("2021-01-02")' \ No newline at end of file +suite("test_select_constant") { + qt_select1 'select 100, "test", date("2021-01-02")' +} \ No newline at end of file diff --git a/regression-test/suites/demo/connect_action.groovy b/regression-test/suites/demo/connect_action.groovy index 0db0372188fc14..ba85cda770e945 100644 --- a/regression-test/suites/demo/connect_action.groovy +++ b/regression-test/suites/demo/connect_action.groovy @@ -1,16 +1,19 @@ -def result1 = connect(user = 'admin', password = context.config.jdbcPassword, url = context.config.jdbcUrl) { - // execute sql with admin user - sql 'select 99 + 1' -} +suite("connect_action", "demo") { + logger.info("ok") + def result1 = connect(user = 'admin', password = context.config.jdbcPassword, url = context.config.jdbcUrl) { + // execute sql with admin user + sql 'select 99 + 1' + } -// if not specify , it will be set to context.config.jdbc -// -// user: 'root' -// password: context.config.jdbcPassword -// url: context.config.jdbcUrl -def result2 = connect('root') { - // execute sql with root user - sql 'select 50 + 50' -} + // if not specify , it will be set to context.config.jdbc + // + // user: 'root' + // password: context.config.jdbcPassword + // url: context.config.jdbcUrl + def result2 = connect('root') { + // execute sql with root user + sql 'select 50 + 50' + } -assertEquals(result1, result2) \ No newline at end of file + assertEquals(result1, result2) +} diff --git a/regression-test/suites/demo/event_action.groovy b/regression-test/suites/demo/event_action.groovy index 722ac6a454640e..45938729f35b6a 100644 --- a/regression-test/suites/demo/event_action.groovy +++ b/regression-test/suites/demo/event_action.groovy @@ -1,40 +1,42 @@ -def createTable = { tableName -> - sql """ - create table ${tableName} - (id int) - distributed by hash(id) - properties - ( - "replication_num"="1" - ) - """ +suite("event_action", "demo") { + def createTable = { tableName -> + sql """ + create table ${tableName} + (id int) + distributed by hash(id) + properties + ( + "replication_num"="1" + ) + """ + } + + def tableName = "test_events_table1" + createTable(tableName) + + // lazy drop table when execute this suite finished + onFinish { + try_sql "drop table if exists ${tableName}" + } + + + + // all event: success, fail, finish + // and you can listen event multiple times + + onSuccess { + try_sql "drop table if exists ${tableName}" + } + + onSuccess { + try_sql "drop table if exists ${tableName}_not_exist" + } + + onFail { + try_sql "drop table if exists ${tableName}" + } + + onFail { + try_sql "drop table if exists ${tableName}_not_exist" + } } - -def tableName = "test_events_table1" -createTable(tableName) - -// lazy drop table when execute this suite finished -onFinish { - try_sql "drop table if exists ${tableName}" -} - - - -// all event: success, fail, finish -// and you can listen event multiple times - -onSuccess { - try_sql "drop table if exists ${tableName}" -} - -onSuccess { - try_sql "drop table if exists ${tableName}_not_exist" -} - -onFail { - try_sql "drop table if exists ${tableName}" -} - -onFail { - try_sql "drop table if exists ${tableName}_not_exist" -} \ No newline at end of file diff --git a/regression-test/suites/demo/explain_action.groovy b/regression-test/suites/demo/explain_action.groovy index a5161a60eec8a7..ca0ec6e8b63515 100644 --- a/regression-test/suites/demo/explain_action.groovy +++ b/regression-test/suites/demo/explain_action.groovy @@ -1,31 +1,33 @@ -explain { - sql("select 100") +suite("explain_action", "demo") { + explain { + sql("select 100") - // contains("OUTPUT EXPRS: 100\n") && contains("PARTITION: UNPARTITIONED\n") - contains "OUTPUT EXPRS: 100\n" - contains "PARTITION: UNPARTITIONED\n" -} + // contains("OUTPUT EXPRS: 100\n") && contains("PARTITION: UNPARTITIONED\n") + contains "OUTPUT EXPRS: 100\n" + contains "PARTITION: UNPARTITIONED\n" + } -explain { - sql("select 100") + explain { + sql("select 100") - // contains(" 100\n") && !contains("abcdefg") && !("1234567") - contains " 100\n" - notContains "abcdefg" - notContains "1234567" -} + // contains(" 100\n") && !contains("abcdefg") && !("1234567") + contains " 100\n" + notContains "abcdefg" + notContains "1234567" + } -explain { - sql("select 100") - // simple callback - check { explainStr -> explainStr.contains("abcdefg") || explainStr.contains(" 100\n") } -} + explain { + sql("select 100") + // simple callback + check { explainStr -> explainStr.contains("abcdefg") || explainStr.contains(" 100\n") } + } -explain { - sql("a b c d e") - // callback with exception and time - check { explainStr, exception, startTime, endTime -> - // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically - assertTrue(exception != null) + explain { + sql("a b c d e") + // callback with exception and time + check { explainStr, exception, startTime, endTime -> + // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically + assertTrue(exception != null) + } } } \ No newline at end of file diff --git a/regression-test/suites/demo/lazyCheck_action.groovy b/regression-test/suites/demo/lazyCheck_action.groovy index 2902147267ac47..377217dc60a408 100644 --- a/regression-test/suites/demo/lazyCheck_action.groovy +++ b/regression-test/suites/demo/lazyCheck_action.groovy @@ -1,33 +1,36 @@ -/***** 1. lazy check exceptions *****/ +suite("lazyCheck_action_exceptions", "demo") { + /***** 1. lazy check exceptions *****/ -// will not throw exception immediately -def result = lazyCheck { - sql "a b c d e d" // syntax error -} -assertTrue(result == null) - -result = lazyCheck { - sql "select 100" -} -assertEquals(result[0][0], 100) + // will not throw exception immediately + def result = lazyCheck { + sql "a b c d e d" // syntax error + } + assertTrue(result == null) -logger.info("You will see this log") + result = lazyCheck { + sql "select 100" + } + assertEquals(result[0][0], 100) -// if you not clear the lazyCheckExceptions, and then, -// after this suite execute finished, the syntax error in the lazyCheck action will be thrown. -lazyCheckExceptions.clear() + logger.info("You will see this log") + // if you not clear the lazyCheckExceptions, and then, + // after this suite execute finished, the syntax error in the lazyCheck action will be thrown. + lazyCheckExceptions.clear() +} -/***** 2. lazy check futures *****/ +suite("lazyCheck_action_futures", "demo") { + /***** 2. lazy check futures *****/ -// start new thread and lazy check future -def futureResult = lazyCheckThread { - sql "a b c d e d" -} -assertTrue(futureResult instanceof java.util.concurrent.Future) + // start new thread and lazy check future + def futureResult = lazyCheckThread { + sql "a b c d e d f" + } + assertTrue(futureResult instanceof java.util.concurrent.Future) -logger.info("You will see this log too") + logger.info("You will see this log too") -// if you not clear the lazyCheckFutures, and then, -// after this suite execute finished, the syntax error in the lazyCheckThread action will be thrown. -lazyCheckFutures.clear() + // if you not clear the lazyCheckFutures, and then, + // after this suite execute finished, the syntax error in the lazyCheckThread action will be thrown. + lazyCheckFutures.clear() +} \ No newline at end of file diff --git a/regression-test/suites/demo/qt_action.groovy b/regression-test/suites/demo/qt_action.groovy index e7a79d8c714235..da6251377cfad0 100644 --- a/regression-test/suites/demo/qt_action.groovy +++ b/regression-test/suites/demo/qt_action.groovy @@ -1,19 +1,20 @@ -/** - * qt_xxx sql equals to quickTest(xxx, sql) witch xxx is tag. - * the result will be compare to the relate file: ${DORIS_HOME}/regression_test/data/qt_action.out. - * - * if you want to generate .out tsv file for real execute result. you can run with -genOut or -forceGenOut option. - * e.g - * ${DORIS_HOME}/run-regression-test.sh --run qt_action -genOut - * ${DORIS_HOME}/run-regression-test.sh --run qt_action -forceGenOut - */ -qt_select "select 1, 'beijing' union all select 2, 'shanghai'" +suite("qt_action", "demo") { + /** + * qt_xxx sql equals to quickTest(xxx, sql) witch xxx is tag. + * the result will be compare to the relate file: ${DORIS_HOME}/regression_test/data/qt_action.out. + * + * if you want to generate .out tsv file for real execute result. you can run with -genOut or -forceGenOut option. + * e.g + * ${DORIS_HOME}/run-regression-test.sh --run qt_action -genOut + * ${DORIS_HOME}/run-regression-test.sh --run qt_action -forceGenOut + */ + qt_select "select 1, 'beijing' union all select 2, 'shanghai'" -qt_select2 "select 2" + qt_select2 "select 2" -// order result by string dict then compare to .out file. -// order_qt_xxx sql equals to quickTest(xxx, sql, true). -order_qt_union_all """ + // order result by string dict then compare to .out file. + // order_qt_xxx sql equals to quickTest(xxx, sql, true). + order_qt_union_all """ select 2 union all select 1 @@ -23,4 +24,5 @@ order_qt_union_all """ select 15 union all select 3 - """ \ No newline at end of file + """ +} diff --git a/regression-test/suites/demo/select_union_all_action.groovy b/regression-test/suites/demo/select_union_all_action.groovy index d4915a75adebb3..1964b83582a96e 100644 --- a/regression-test/suites/demo/select_union_all_action.groovy +++ b/regression-test/suites/demo/select_union_all_action.groovy @@ -1,6 +1,7 @@ -// 3 rows and 1 column -def rows = [3, 1, 10] -order_qt_select_union_all1 """ +suite("select_union_all_action", "demo") { + // 3 rows and 1 column + def rows = [3, 1, 10] + order_qt_select_union_all1 """ select c1 from ( @@ -8,12 +9,13 @@ order_qt_select_union_all1 """ ) a """ -// 3 rows and 2 columns -rows = [[1, "123"], [2, null], [0, "abc"]] -order_qt_select_union_all2 """ + // 3 rows and 2 columns + rows = [[1, "123"], [2, null], [0, "abc"]] + order_qt_select_union_all2 """ select c1, c2 from ( ${selectUnionAll(rows)} ) b - """ \ No newline at end of file + """ +} \ No newline at end of file diff --git a/regression-test/suites/demo/sql_action.groovy b/regression-test/suites/demo/sql_action.groovy index 9ba6716ffc91fe..66802f366943bb 100644 --- a/regression-test/suites/demo/sql_action.groovy +++ b/regression-test/suites/demo/sql_action.groovy @@ -1,27 +1,28 @@ -// execute sql and ignore result -sql "show databases" +suite("sql_action", "demo") { + // execute sql and ignore result + sql "show databases" -// execute sql and get result, outer List denote rows, inner List denote columns in a single row -List> tables = sql "show tables" + // execute sql and get result, outer List denote rows, inner List denote columns in a single row + List> tables = sql "show tables" -// assertXxx() will invoke junit5's Assertions.assertXxx() dynamically -assertTrue(tables.size() >= 0) // test rowCount >= 0 + // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically + assertTrue(tables.size() >= 0) // test rowCount >= 0 -// syntax error -try { - sql "a b c d e" - throw new IllegalStateException("Should be syntax error") -} catch (java.sql.SQLException t) { - assertTrue(true) -} + // syntax error + try { + sql "a b c d e" + throw new IllegalStateException("Should be syntax error") + } catch (java.sql.SQLException t) { + assertTrue(true) + } -def testTable = "test_sql_action1" + def testTable = "test_sql_action1" -try { - sql "DROP TABLE IF EXISTS ${testTable}" + try { + sql "DROP TABLE IF EXISTS ${testTable}" - // multi-line sql - def result1 = sql """ + // multi-line sql + def result1 = sql """ CREATE TABLE IF NOT EXISTS ${testTable} ( id int ) @@ -31,40 +32,40 @@ try { ) """ - // DDL/DML return 1 row and 1 column, the only value is update row count - assertTrue(result1.size() == 1) - assertTrue(result1[0].size() == 1) - assertTrue(result1[0][0] == 0, "Create table should update 0 rows") + // DDL/DML return 1 row and 1 column, the only value is update row count + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 0, "Create table should update 0 rows") - def result2 = sql "INSERT INTO test_sql_action1 values(1), (2), (3)" - assertTrue(result2.size() == 1) - assertTrue(result2[0].size() == 1) - assertTrue(result2[0][0] == 3, "Insert should update 3 rows") -} finally { - /** - * try_xxx(args) means: - * - * try { - * return xxx(args) - * } catch (Throwable t) { - * // do nothing - * return null - * } - */ - try_sql("DROP TABLE IF EXISTS ${testTable}") + def result2 = sql "INSERT INTO test_sql_action1 values(1), (2), (3)" + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 1) + assertTrue(result2[0][0] == 3, "Insert should update 3 rows") + } finally { + /** + * try_xxx(args) means: + * + * try { + * return xxx(args) + * } catch (Throwable t) { + * // do nothing + * return null + * } + */ + try_sql("DROP TABLE IF EXISTS ${testTable}") - // you can see the error sql will not throw exception and return - try { - def errorSqlResult = try_sql("a b c d e f g") - assertTrue(errorSqlResult == null) - } catch (Throwable t) { - assertTrue(false, "Never catch exception") + // you can see the error sql will not throw exception and return + try { + def errorSqlResult = try_sql("a b c d e f g") + assertTrue(errorSqlResult == null) + } catch (Throwable t) { + assertTrue(false, "Never catch exception") + } } -} -// order_sql(sqlStr) equals to sql(sqlStr, isOrder=true) -// sort result by string dict -def list = order_sql """ + // order_sql(sqlStr) equals to sql(sqlStr, isOrder=true) + // sort result by string dict + def list = order_sql """ select 2 union all select 1 @@ -76,8 +77,9 @@ def list = order_sql """ select 3 """ -assertEquals(null, list[0][0]) -assertEquals(1, list[1][0]) -assertEquals(15, list[2][0]) -assertEquals(2, list[3][0]) -assertEquals(3, list[4][0]) \ No newline at end of file + assertEquals(null, list[0][0]) + assertEquals(1, list[1][0]) + assertEquals(15, list[2][0]) + assertEquals(2, list[3][0]) + assertEquals(3, list[4][0]) +} diff --git a/regression-test/suites/demo/streamLoad_action.groovy b/regression-test/suites/demo/streamLoad_action.groovy index f9ae5bf02ac523..cc131128d10e4b 100644 --- a/regression-test/suites/demo/streamLoad_action.groovy +++ b/regression-test/suites/demo/streamLoad_action.groovy @@ -1,61 +1,64 @@ -def tableName = "test_streamload_action1" - -sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - id int, - name varchar(255) - ) - DISTRIBUTED BY HASH(id) BUCKETS 1 - PROPERTIES ( - "replication_num" = "1" - ) -""" - -streamLoad { - // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy - // db 'regression_test' - table tableName - - // default label is UUID: - // set 'label' UUID.randomUUID().toString() - - // default column_separator is specify in doris fe config, usually is '\t'. - // this line change to ',' - set 'column_separator', ',' - - // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. - // also, you can stream load a http stream, e.g. http://xxx/some.csv - file 'streamload_input.csv' - - time 10000 // limit inflight 10s - - // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows -} +suite("streamLoad_action", "demo") { + + def tableName = "test_streamload_action1" + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + id int, + name varchar(255) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + streamLoad { + // you can skip declare db, because a default db already specify in ${DORIS_HOME}/conf/regression-conf.groovy + // db 'regression_test' + table tableName + + // default label is UUID: + // set 'label' UUID.randomUUID().toString() + + // default column_separator is specify in doris fe config, usually is '\t'. + // this line change to ',' + set 'column_separator', ',' + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file 'streamload_input.csv' + time 10000 // limit inflight 10s -// stream load 100 rows -def rowCount = 100 -// range: [0, rowCount) -// or rangeClosed: [0, rowCount] -def rowIt = range(0, rowCount) - .mapToObj({i -> [i, "a_" + i]}) // change Long to List - .iterator() - -streamLoad { - table tableName - // also, you can upload a memory iterator - inputIterator rowIt - - // if declared a check callback, the default check condition will ignore. - // So you must check all condition - check { result, exception, startTime, endTime -> - if (exception != null) { - throw exception + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + } + + + // stream load 100 rows + def rowCount = 100 + // range: [0, rowCount) + // or rangeClosed: [0, rowCount] + def rowIt = range(0, rowCount) + .mapToObj({i -> [i, "a_" + i]}) // change Long to List + .iterator() + + streamLoad { + table tableName + // also, you can upload a memory iterator + inputIterator rowIt + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } - log.info("Stream load result: ${result}".toString()) - def json = parseJson(result) - assertEquals("success", json.Status.toLowerCase()) - assertEquals(json.NumberTotalRows, json.NumberLoadedRows) - assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) } -} \ No newline at end of file +} diff --git a/regression-test/suites/demo/test_action.groovy b/regression-test/suites/demo/test_action.groovy index c84ea23b4054aa..54615f3ed44235 100644 --- a/regression-test/suites/demo/test_action.groovy +++ b/regression-test/suites/demo/test_action.groovy @@ -1,11 +1,12 @@ -test { - sql "abcdefg" - // check exception message contains - exception "errCode = 2, detailMessage = Syntax error" -} +suite("test_action", "demo") { + test { + sql "abcdefg" + // check exception message contains + exception "errCode = 2, detailMessage = Syntax error" + } -test { - sql """ + test { + sql """ select * from ( select 1 id @@ -14,33 +15,33 @@ test { ) a order by id""" - // multi check condition + // multi check condition - // check return 2 rows - rowNum 2 - // execute time must <= 5000 millisecond - time 5000 - // check result, must be 2 rows and 1 column, the first row is 1, second is 2 - result( - [[1], [2]] - ) -} + // check return 2 rows + rowNum 2 + // execute time must <= 5000 millisecond + time 5000 + // check result, must be 2 rows and 1 column, the first row is 1, second is 2 + result( + [[1], [2]] + ) + } -test { - sql "a b c d e f g" + test { + sql "a b c d e f g" - // other check will not work because already declared a check callback - exception "aaaaaaaaa" + // other check will not work because already declared a check callback + exception "aaaaaaaaa" - // callback - check { result, exception, startTime, endTime -> - // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically - assertTrue(exception != null) + // callback + check { result, exception, startTime, endTime -> + // assertXxx() will invoke junit5's Assertions.assertXxx() dynamically + assertTrue(exception != null) + } } -} -test { - sql """ + test { + sql """ select 2 union all select 1 @@ -52,14 +53,33 @@ test { select 3 """ - check { result, ex, startTime, endTime -> - // same as order_sql(sqlStr) - result = sortRows(result) + check { result, ex, startTime, endTime -> + // same as order_sql(sqlStr) + result = sortRows(result) - assertEquals(null, result[0][0]) - assertEquals(1, result[1][0]) - assertEquals(15, result[2][0]) - assertEquals(2, result[3][0]) - assertEquals(3, result[4][0]) + assertEquals(null, result[0][0]) + assertEquals(1, result[1][0]) + assertEquals(15, result[2][0]) + assertEquals(2, result[3][0]) + assertEquals(3, result[4][0]) + } } -} \ No newline at end of file + + // execute sql and order query result, then compare to iterator + def selectValues = [1, 2, 3, 4] + test { + order true + sql selectUnionAll(selectValues) + resultIterator(selectValues.iterator()) + } + + // compare to data/demo/test_action.csv + test { + order true + sql selectUnionAll(selectValues) + + // you can set to http://xxx or https://xxx + // and compare to http response body + resultFile "test_action.csv" + } +} diff --git a/regression-test/suites/demo/test_sql_file.sql b/regression-test/suites/demo/test_sql_file.sql new file mode 100644 index 00000000000000..a8b84dcd3bbdbe --- /dev/null +++ b/regression-test/suites/demo/test_sql_file.sql @@ -0,0 +1,5 @@ +select 200 +union all +select 100 +union all +select 300 \ No newline at end of file diff --git a/regression-test/suites/demo/test_sql_file_order.sql b/regression-test/suites/demo/test_sql_file_order.sql new file mode 100644 index 00000000000000..a8b84dcd3bbdbe --- /dev/null +++ b/regression-test/suites/demo/test_sql_file_order.sql @@ -0,0 +1,5 @@ +select 200 +union all +select 100 +union all +select 300 \ No newline at end of file diff --git a/regression-test/suites/demo/thread_action.groovy b/regression-test/suites/demo/thread_action.groovy index 7af7ff886e2b87..afa6765a457d4b 100644 --- a/regression-test/suites/demo/thread_action.groovy +++ b/regression-test/suites/demo/thread_action.groovy @@ -1,48 +1,50 @@ -def (_, elapsedMillis) = timer { - /** - * the default max thread num is 10, you can specify by 'actionParallel' param. - * e.g. ./run-regression-test.sh --run someSuite -actionParallel 10 - */ - def future1 = thread("threadName1") { - sleep(200) - sql"select 1" - } +suite("thread_action", "demo") { + def (_, elapsedMillis) = timer { + /** + * the default max thread num is 10, you can specify by 'actionParallel' param. + * e.g. ./run-regression-test.sh --run someSuite -actionParallel 10 + */ + def future1 = thread("threadName1") { + sleep(200) + sql"select 1" + } - // create new thread but not specify name - def future2 = thread { - sleep(200) - sql "select 2" - } + // create new thread but not specify name + def future2 = thread { + sleep(200) + sql "select 2" + } - def future3 = thread("threadName3") { - sleep(200) - sql "select 3" - } + def future3 = thread("threadName3") { + sleep(200) + sql "select 3" + } + + def future4 = thread { + sleep(200) + sql "select 4" + } - def future4 = thread { - sleep(200) - sql "select 4" + // equals to combineFutures([future1, future2, future3, future4]), which [] is a Iterable + def combineFuture = combineFutures(future1, future2, future3, future4) + // or you can use lazyCheckThread action(see lazyCheck_action.groovy), and not have to check exception from futures. + List>> result = combineFuture.get() + assertEquals(result[0][0][0], 1) + assertEquals(result[1][0][0], 2) + assertEquals(result[2][0][0], 3) + assertEquals(result[3][0][0], 4) } + assertTrue(elapsedMillis < 600) - // equals to combineFutures([future1, future2, future3, future4]), which [] is a Iterable - def combineFuture = combineFutures(future1, future2, future3, future4) - // or you can use lazyCheckThread action(see lazyCheck_action.groovy), and not have to check exception from futures. - List>> result = combineFuture.get() - assertEquals(result[0][0][0], 1) - assertEquals(result[1][0][0], 2) - assertEquals(result[2][0][0], 3) - assertEquals(result[3][0][0], 4) -} -assertTrue(elapsedMillis < 600) + // you can use qt action in thread action, and you **MUST** specify different tag, + // testing framework can compare different qt result in different order. + lazyCheckThread { + sleep(100) + qt_diffrent_tag1 "select 100" + } -// you can use qt action in thread action, and you **MUST** specify different tag, -// testing framework can compare different qt result in different order. -lazyCheckThread { - sleep(100) - qt_diffrent_tag1 "select 100" + lazyCheckThread("lazyCheckThread2") { + qt_diffrent_tag2 "select 100" + } } - -lazyCheckThread("lazyCheckThread2") { - qt_diffrent_tag2 "select 100" -} \ No newline at end of file diff --git a/regression-test/suites/demo/timer_action.groovy b/regression-test/suites/demo/timer_action.groovy index 14b84f58d1c47e..3d4f15069c6e43 100644 --- a/regression-test/suites/demo/timer_action.groovy +++ b/regression-test/suites/demo/timer_action.groovy @@ -1,7 +1,13 @@ -def (sumResult, elapsedMillis) = timer { - long sum = 0 - (1..10000).each {sum += it} - sum // return value -} +suite("timer_action", "demo") { + def (sumResult, elapsedMillis) = timer { + long sum = 0 + (1..10000).each {sum += it} + sum // return value + } -logger.info("sum: ${sumResult}, elapsed: ${elapsedMillis} ms") + // you should convert GString to String because the log of GString not contains the correct file name, + // e.g. (NativeMethodAccessorImpl.java:-2) - sum: 50005000, elapsed: 47 ms + // so, invoke toString and then we can get the log: + // (timer_action.groovy:10) - sum: 50005000, elapsed: 47 ms + logger.info("sum: ${sumResult}, elapsed: ${elapsedMillis} ms".toString()) +} diff --git a/regression-test/suites/empty_table/load.groovy b/regression-test/suites/empty_table/load.groovy index a8a554e36c8449..6b8cd7a3b7416b 100644 --- a/regression-test/suites/empty_table/load.groovy +++ b/regression-test/suites/empty_table/load.groovy @@ -20,12 +20,14 @@ // /testing/trino-product-tests/src/main/resources/sql-tests/testcases // and modified by Doris. -def tables=["empty"] +suite("load") { + def tables=["empty"] -for (String table in tables) { - sql """ DROP TABLE IF EXISTS $table """ -} + for (String table in tables) { + sql """ DROP TABLE IF EXISTS $table """ + } -for (String table in tables) { - sql new File("""${context.file.parent}/ddl/${table}.sql""").text + for (String table in tables) { + sql new File("""${context.file.parent}/ddl/${table}.sql""").text + } } diff --git a/regression-test/suites/join/load.groovy b/regression-test/suites/join/load.groovy index dc085ef4820363..404d85fc962249 100644 --- a/regression-test/suites/join/load.groovy +++ b/regression-test/suites/join/load.groovy @@ -20,15 +20,17 @@ // /testing/trino-product-tests/src/main/resources/sql-tests/testcases // and modified by Doris. -def tables=["test_join"] +suite("load") { + def tables=["test_join"] -for (String table in tables) { - sql """ DROP TABLE IF EXISTS $table """ -} + for (String table in tables) { + sql """ DROP TABLE IF EXISTS $table """ + } -for (String table in tables) { - sql new File("""${context.file.parent}/ddl/${table}.sql""").text -} + for (String table in tables) { + sql new File("""${context.file.parent}/ddl/${table}.sql""").text + } -sql """ insert into test_join select 1 """ -sql """ insert into test_join select 2 """ + sql """ insert into test_join select 1 """ + sql """ insert into test_join select 2 """ +} diff --git a/regression-test/suites/join/sql/agg_output_as_right_tale_left_outer.sql b/regression-test/suites/join/sql/agg_output_as_right_tale_left_outer_order.sql similarity index 100% rename from regression-test/suites/join/sql/agg_output_as_right_tale_left_outer.sql rename to regression-test/suites/join/sql/agg_output_as_right_tale_left_outer_order.sql diff --git a/regression-test/suites/performance/test_streamload_perfomance.groovy b/regression-test/suites/performance/test_streamload_perfomance.groovy index df30baa19fba0e..8a5ef4a16cbb4e 100644 --- a/regression-test/suites/performance/test_streamload_perfomance.groovy +++ b/regression-test/suites/performance/test_streamload_perfomance.groovy @@ -1,7 +1,8 @@ -def tableName = "test_streamload_performance1" +suite("test_streamload_perfomance", "performance") { + def tableName = "test_streamload_performance1" -try { - sql """ + try { + sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( id int, name varchar(255) @@ -12,16 +13,17 @@ try { ) """ - def rowCount = 10000 - def rowIt = java.util.stream.LongStream.range(0, rowCount) - .mapToObj({i -> [i, "a_" + i]}) - .iterator() + def rowCount = 10000 + def rowIt = java.util.stream.LongStream.range(0, rowCount) + .mapToObj({i -> [i, "a_" + i]}) + .iterator() - streamLoad { - table tableName - time 5000 - inputIterator rowIt + streamLoad { + table tableName + time 5000 + inputIterator rowIt + } + } finally { + try_sql "DROP TABLE IF EXISTS ${tableName}" } -} finally { - try_sql "DROP TABLE IF EXISTS ${tableName}" -} \ No newline at end of file +} diff --git a/regression-test/suites/types/complex_types/basic_agg_test.groovy b/regression-test/suites/types/complex_types/basic_agg_test.groovy index c489d94905101d..c7528246e7195e 100644 --- a/regression-test/suites/types/complex_types/basic_agg_test.groovy +++ b/regression-test/suites/types/complex_types/basic_agg_test.groovy @@ -15,14 +15,16 @@ // specific language governing permissions and limitations // under the License. -def tables=["bitmap_basic_agg","hll_basic_agg"] +suite("basic_agg_test") { + def tables=["bitmap_basic_agg","hll_basic_agg"] -for (String table in tables) { - sql """drop table if exists ${table};""" - sql new File("""regression-test/common/table/${table}.sql""").text - sql new File("""regression-test/common/load/${table}.sql""").text -} + for (String table in tables) { + sql """drop table if exists ${table};""" + sql new File("""regression-test/common/table/${table}.sql""").text + sql new File("""regression-test/common/load/${table}.sql""").text + } -qt_sql_bitmap """select * from bitmap_basic_agg;""" + qt_sql_bitmap """select * from bitmap_basic_agg;""" -qt_sql_hll """select * from hll_basic_agg;""" + qt_sql_hll """select * from hll_basic_agg;""" +} \ No newline at end of file diff --git a/run-regression-test.sh b/run-regression-test.sh index 840d149c3fea57..5c56696ede58f8 100755 --- a/run-regression-test.sh +++ b/run-regression-test.sh @@ -21,6 +21,7 @@ # Usage: $0 # Optional shell_options: # --clean clean output of regression test +# --teamcity print teamcity service messages # --run run regression test. build framework if necessary # # Optional framework_options @@ -49,6 +50,7 @@ usage() { Usage: $0 Optional shell_options: --clean clean output of regression test framework + --teamcity print teamcity service messages --run run regression test. build framework if necessary Optional framework_options: @@ -68,9 +70,11 @@ Usage: $0 $0 --run -s test_select run a suite which named as test_select $0 --run test_select -genOut generate output file for test_select if not exist $0 --run -g default run all suite in the group which named as default + $0 --run -d demo,correctness/tmp run all suite in the directories which named as demo and correctness/tmp $0 --clean clean output of regression test framework $0 --clean --run test_select clean output and build regression test framework and run a suite which named as test_select $0 --run -h print framework options + $0 --teamcity --run test_select print teamcity service messages and build regression test framework and run test_select Log path: \${DORIS_HOME}/output/regression-test/log Default config file: \${DORIS_HOME}/regression-test/conf/regression-conf.groovy @@ -80,20 +84,24 @@ Default config file: \${DORIS_HOME}/regression-test/conf/regression-conf.groovy CLEAN= WRONG_CMD= +TEAMCITY= RUN= if [ $# == 0 ] ; then #default CLEAN=0 WRONG_CMD=0 + TEAMCITY=0 RUN=1 else CLEAN=0 RUN=0 + TEAMCITY=0 WRONG_CMD=0 while true; do case "$1" in - --clean) CLEAN=1 ; shift ;; - --run) RUN=1 ; shift ;; + --clean) CLEAN=1 ; shift ;; + --teamcity) TEAMCITY=1 ; shift ;; + --run) RUN=1 ; shift ;; *) if [ ${RUN} -eq 0 ] && [ ${CLEAN} -eq 0 ]; then WRONG_CMD=1 @@ -171,10 +179,15 @@ fi echo "===== Run Regression Test =====" +JAVA_OPTS=${JAVA_OPTS} +if [ ${TEAMCITY} -eq 1 ]; then + JAVA_OPTS="$JAVA_OPTS -DstdoutAppenderType=teamcity" +fi + $JAVA -DDORIS_HOME=$DORIS_HOME \ -DLOG_PATH=$LOG_OUTPUT_FILE \ -Dlogback.configurationFile=${LOG_CONFIG_FILE} \ - -Xmx2048m \ + ${JAVA_OPTS} \ -jar ${RUN_JAR} \ -cf ${CONFIG_FILE} \ ${REGRESSION_OPTIONS_PREFIX} "$@" From 809c5fca888b8a8fab1f63906f45228f6acc5146 Mon Sep 17 00:00:00 2001 From: Huajian Lan <924060929@qq.com> Date: Fri, 1 Apr 2022 12:27:23 +0800 Subject: [PATCH 2/4] refactor regression tesing framework --- .../apache/doris/regression/suite/Suite.groovy | 3 --- .../doris/regression/util/TeamcityUtils.groovy | 18 ------------------ run-regression-test.sh | 6 +++--- 3 files changed, 3 insertions(+), 24 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 767ee13fe0ffb4..c8f2b03b3026e0 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -321,9 +321,6 @@ class Suite implements GroovyInterceptable { return null } } else { - if (metaClass == null) { - println("eeee") - } // invoke origin method return metaClass.invokeMethod(this, name, args) } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/TeamcityUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/TeamcityUtils.groovy index ace3253d53a865..dd713db2d1db25 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/TeamcityUtils.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/TeamcityUtils.groovy @@ -38,24 +38,6 @@ class TeamcityUtils { return "##teamcity[testStdErr name='${suiteContext.flowName}' out='${escape(msg)}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']" } -// static void testSuiteStarted(ScriptContext scriptContext) { -// String timestamp = formatNow() -// println("##teamcity[flowStarted flowId='${scriptContext.flowId}' timestamp='${timestamp}']") -// println("##teamcity[testSuiteStarted name='${scriptContext.flowName}' flowId='${scriptContext.flowId}' timestamp='${timestamp}']") -// } - -// static void testSuiteFinished(ScriptContext scriptContext) { -// String timestamp = formatNow() -// println("##teamcity[testSuiteFinished name='${scriptContext.flowName}' flowId='${scriptContext.flowId}' timestamp='${timestamp}']") -// println("##teamcity[flowFinished flowId='${scriptContext.flowId}' timestamp='${timestamp}']") -// } - -// static void testStarted(SuiteContext suiteContext) { -// String timestamp = formatNow() -// println("##teamcity[flowStarted flowId='${suiteContext.flowId}' parent='${suiteContext.scriptContext.flowId}' timestamp='${timestamp}']") -// println("##teamcity[testStarted name='${suiteContext.flowName}' flowId='${suiteContext.flowId}' timestamp='${timestamp}']") -// } - static void testStarted(SuiteContext suiteContext) { String timestamp = formatNow() println("##teamcity[flowStarted flowId='${suiteContext.flowId}' timestamp='${timestamp}']") diff --git a/run-regression-test.sh b/run-regression-test.sh index 5c56696ede58f8..37364afc634d4b 100755 --- a/run-regression-test.sh +++ b/run-regression-test.sh @@ -99,9 +99,9 @@ else WRONG_CMD=0 while true; do case "$1" in - --clean) CLEAN=1 ; shift ;; - --teamcity) TEAMCITY=1 ; shift ;; - --run) RUN=1 ; shift ;; + --clean) CLEAN=1 ; shift ;; + --teamcity) TEAMCITY=1 ; shift ;; + --run) RUN=1 ; shift ;; *) if [ ${RUN} -eq 0 ] && [ ${CLEAN} -eq 0 ]; then WRONG_CMD=1 From 812d9957289659ce538a551d871d62c0999375f5 Mon Sep 17 00:00:00 2001 From: Huajian Lan <924060929@qq.com> Date: Fri, 1 Apr 2022 14:22:01 +0800 Subject: [PATCH 3/4] change regression-testing.md document --- docs/zh-CN/developer-guide/regression-testing.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/zh-CN/developer-guide/regression-testing.md b/docs/zh-CN/developer-guide/regression-testing.md index 01e8e81ecfcd7b..0a9b352d34231c 100644 --- a/docs/zh-CN/developer-guide/regression-testing.md +++ b/docs/zh-CN/developer-guide/regression-testing.md @@ -535,7 +535,8 @@ thread, lazyCheck, events, connect, selectUnionAll ## CI/CD的支持 ### TeamCity -可以使用--teamcity开启TeamCity Service Message. `-Dteamcity.enableStdErr=false`可以让错误日志也打印到stdout中，方便按顺序分析日志。 +TeamCity可以通过stdout识别Service Message。当使用`--teamcity`参数启动回归测试框架时，回归测试框架就会在stdout打印TeamCity Service Message，TeamCity将会自动读取stdout中的事件日志，并在当前流水线中展示`Tests`，其中会展示测试的test及其日志。 +因此只需要配置下面一行启动回归测试框架的命令即可。其中`-Dteamcity.enableStdErr=false`可以让错误日志也打印到stdout中，方便按顺序分析日志。 ```shell -JAVA_OPTS="-Dteamcity.enableStdErr=${enableStdErr}" ./run-regression-test.sh --teamcity --run -s ${suite} +JAVA_OPTS="-Dteamcity.enableStdErr=${enableStdErr}" ./run-regression-test.sh --teamcity --run ``` \ No newline at end of file From 39d872f3338e5e9a8d0dbfcf98d295359764b32c Mon Sep 17 00:00:00 2001 From: Huajian Lan <924060929@qq.com> Date: Fri, 1 Apr 2022 14:23:12 +0800 Subject: [PATCH 4/4] change regression-testing.md document --- docs/zh-CN/developer-guide/regression-testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh-CN/developer-guide/regression-testing.md b/docs/zh-CN/developer-guide/regression-testing.md index 0a9b352d34231c..55fb5e8feadc0b 100644 --- a/docs/zh-CN/developer-guide/regression-testing.md +++ b/docs/zh-CN/developer-guide/regression-testing.md @@ -536,7 +536,7 @@ thread, lazyCheck, events, connect, selectUnionAll ## CI/CD的支持 ### TeamCity TeamCity可以通过stdout识别Service Message。当使用`--teamcity`参数启动回归测试框架时，回归测试框架就会在stdout打印TeamCity Service Message，TeamCity将会自动读取stdout中的事件日志，并在当前流水线中展示`Tests`，其中会展示测试的test及其日志。 -因此只需要配置下面一行启动回归测试框架的命令即可。其中`-Dteamcity.enableStdErr=false`可以让错误日志也打印到stdout中，方便按顺序分析日志。 +因此只需要配置下面一行启动回归测试框架的命令即可。其中`-Dteamcity.enableStdErr=false`可以让错误日志也打印到stdout中，方便按时间顺序分析日志。 ```shell JAVA_OPTS="-Dteamcity.enableStdErr=${enableStdErr}" ./run-regression-test.sh --teamcity --run ``` \ No newline at end of file