Skip to content

[Enhancement][Regression-Test] enhancement for regression-test framework #8930

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
HappenLee merged 3 commits into from
Apr 15, 2022
Merged

[Enhancement][Regression-Test] enhancement for regression-test framework #8930

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
58f3d19
enhancement for regression-test framework
BiteTheDDDDt Apr 8, 2022
2fa3bd3
use camel case
BiteTheDDDDt Apr 12, 2022
7cd7982
add update
BiteTheDDDDt Apr 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions ...on-test/framework/src/main/groovy/org/apache/doris/regression/action/ExplainAction.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import groovy.transform.stc.FromString
import org.apache.doris.regression.suite.SuiteContext
import org.apache.doris.regression.util.JdbcUtils
import groovy.util.logging.Slf4j

import java.sql.ResultSetMetaData
import java.util.stream.Collectors

@Slf4j
Expand Down Expand Up @@ -112,13 +112,14 @@ class ExplainAction implements SuiteAction {
log.info("Execute sql:\n${explainSql}".toString())
long startTime = System.currentTimeMillis()
String explainString = null
ResultSetMetaData meta = null
try {
explainString = JdbcUtils.executeToList(context.getConnection(), explainSql).stream()
.map({row -> row.get(0).toString()})
.collect(Collectors.joining("\n"))
return new ActionResult(explainString, null, startTime, System.currentTimeMillis())
def temp = null
(temp, meta) = JdbcUtils.executeToList(context.getConnection(), explainSql)
explainString = temp.stream().map({row -> row.get(0).toString()}).collect(Collectors.joining("\n"))
return new ActionResult(explainString, null, startTime, System.currentTimeMillis(), meta)
} catch (Throwable t) {
return new ActionResult(explainString, t, startTime, System.currentTimeMillis())
return new ActionResult(explainString, t, startTime, System.currentTimeMillis(), meta)
}
}

Expand All @@ -127,12 +128,14 @@ class ExplainAction implements SuiteAction {
Throwable exception
long startTime
long endTime
ResultSetMetaData meta

ActionResult(String result, Throwable exception, long startTime, long endTime) {
ActionResult(String result, Throwable exception, long startTime, long endTime, ResultSetMetaData meta) {
this.result = result
this.exception = exception
this.startTime = startTime
this.endTime = endTime
this.meta = meta
}
}
}
22 changes: 14 additions & 8 deletions ...ssion-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import org.apache.http.util.EntityUtils
import javax.swing.text.html.parser.Entity
import java.nio.charset.StandardCharsets
import java.sql.Connection

import java.sql.ResultSetMetaData
import org.apache.doris.regression.suite.SuiteContext
import org.apache.doris.regression.util.JdbcUtils
import org.junit.Assert
Expand Down Expand Up @@ -94,10 +94,10 @@ class TestAction implements SuiteAction {
if (row instanceof List) {
return OutputUtils.toCsvString(row as List)
} else {
return OutputUtils.columnToCsvString(row)
return OutputUtils.toCsvString(row as Object)
}
},
{ List<Object> row -> OutputUtils.toCsvString(row) }, "Check failed")
{ List<Object> row -> OutputUtils.toCsvString(row) }, "Check failed", result.meta)
if (errorMsg != null) {
throw new IllegalStateException(errorMsg)
}
Expand All @@ -109,7 +109,7 @@ class TestAction implements SuiteAction {
String errMsg = OutputUtils.checkOutput(csvIt, result.result.iterator(),
{ List<String> row -> OutputUtils.toCsvString(row as List<Object>) },
{ List<Object> row -> OutputUtils.toCsvString(row) },
"Check failed compare to")
"Check failed compare to", result.meta)
if (errMsg != null) {
throw new IllegalStateException(errMsg)
}
Expand All @@ -136,7 +136,8 @@ class TestAction implements SuiteAction {
if (!file.exists()) {
log.warn("Result file not exists: ${file}".toString())
}
log.warn("Compare to local file: ${file}".toString())

log.info("Compare to local file: ${file}".toString())
file.newInputStream().withCloseable { inputStream ->
checkFunc(inputStream)
}
Expand All @@ -150,19 +151,22 @@ class TestAction implements SuiteAction {

ActionResult doRun(Connection conn) {
List<List<Object>> result = null
ResultSetMetaData meta = null
Throwable ex = null

long startTime = System.currentTimeMillis()
try {
log.info("Execute ${isOrder ? "order_" : ""}sql:\n${sql}".toString())
result = JdbcUtils.executeToList(conn, sql)
(result, meta) = JdbcUtils.executeToList(conn, sql)
if (isOrder) {
result = DataUtils.sortByToString(result)
}
} catch (Throwable t) {
ex = t
}
long endTime = System.currentTimeMillis()
return new ActionResult(result, ex, startTime, endTime)

return new ActionResult(result, ex, startTime, endTime, meta)
}

void sql(String sql) {
Expand Down Expand Up @@ -234,12 +238,14 @@ class TestAction implements SuiteAction {
Throwable exception
long startTime
long endTime
ResultSetMetaData meta

ActionResult(List<List<Object>> result, Throwable exception, long startTime, long endTime) {
ActionResult(List<List<Object>> result, Throwable exception, long startTime, long endTime, ResultSetMetaData meta) {
this.result = result
this.exception = exception
this.startTime = startTime
this.endTime = endTime
this.meta = meta
}
}
}
11 changes: 6 additions & 5 deletions regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ class Suite implements GroovyInterceptable {

List<List<Object>> sql(String sqlStr, boolean isOrder = false) {
logger.info("Execute ${isOrder ? "order_" : ""}sql: ${sqlStr}".toString())
def result = JdbcUtils.executeToList(context.getConnection(), sqlStr)
def (result, meta) = JdbcUtils.executeToList(context.getConnection(), sqlStr)
if (isOrder) {
result = DataUtils.sortByToString(result)
}
Expand Down Expand Up @@ -265,7 +265,7 @@ class Suite implements GroovyInterceptable {
logger.info("Execute tag: ${tag}, ${isOrder ? "order_" : ""}sql: ${sql}".toString())

if (context.config.generateOutputFile || context.config.forceGenerateOutputFile) {
def result = JdbcUtils.executorToStringList(context.getConnection(), sql)
def (result, meta) = JdbcUtils.executeToStringList(context.getConnection(), sql)
if (isOrder) {
result = sortByToString(result)
}
Expand All @@ -283,16 +283,17 @@ class Suite implements GroovyInterceptable {
}

OutputUtils.TagBlockIterator expectCsvResults = context.getOutputIterator().next()
List<List<Object>> realResults = JdbcUtils.executorToStringList(context.getConnection(), sql)

def (realResults, meta) = JdbcUtils.executeToStringList(context.getConnection(), sql)
if (isOrder) {
realResults = sortByToString(realResults)
}
String errorMsg = null
try {
errorMsg = OutputUtils.checkOutput(expectCsvResults, realResults.iterator(),
{ row -> OutputUtils.toCsvString(row as List<Object>) },
{row -> OutputUtils.toCsvString(row) },
"Check tag '${tag}' failed")
{ row -> OutputUtils.toCsvString(row) },
"Check tag '${tag}' failed", meta)
} catch (Throwable t) {
throw new IllegalStateException("Check tag '${tag}' failed, sql:\n${sql}", t)
}
Expand Down
24 changes: 12 additions & 12 deletions regression-test/framework/src/main/groovy/org/apache/doris/regression/util/JdbcUtils.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,34 +18,36 @@
package org.apache.doris.regression.util

import com.google.common.collect.ImmutableList
import groovy.lang.Tuple2

import java.sql.Connection
import java.sql.ResultSet
import java.sql.ResultSetMetaData

class JdbcUtils {
static List<List<Object>> executeToList(Connection conn, String sql) {
static Tuple2<List<List<Object>>, ResultSetMetaData> executeToList(Connection conn, String sql) {
conn.prepareStatement(sql).withCloseable { stmt ->
Copy link
Contributor

@924060929 924060929 Apr 11, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggest new Metaholder in JdbcUtils and return, and you can extract the interest metadata out of the Connection life cycle safely.

e.g.

class Metadata {
  public final List<String> columnNames;
  public final List<String> columnTypes;
}

static Tuple2<List<List<Object>>, Metadata> executeToList(Connection conn, String sql) {
  def metadata = new Metadata()
  resultList = ...
  metadata.columnNames = ...
  metadata.columnTypes = ...
  return [resultList, metadata]
}

then, you can take result by this code

def (result, metadata) = JdbcUtils.executeToList(connnection, sql)

Copy link
Contributor Author

@BiteTheDDDDt BiteTheDDDDt Apr 12, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

boolean hasResultSet = stmt.execute()
if (!hasResultSet) {
return ImmutableList.of(ImmutableList.of(stmt.getUpdateCount()))
return [ImmutableList.of(ImmutableList.of(stmt.getUpdateCount())), null]
} else {
toList(stmt.resultSet)
return toList(stmt.resultSet)
}
}
}

static List<List<Object>> executorToStringList(Connection conn, String sql) {
static Tuple2<List<List<Object>>, ResultSetMetaData> executeToStringList(Connection conn, String sql) {
conn.prepareStatement(sql).withCloseable { stmt ->
boolean hasResultSet = stmt.execute()
if (!hasResultSet) {
return ImmutableList.of(ImmutableList.of(stmt.getUpdateCount()))
return [ImmutableList.of(ImmutableList.of(stmt.getUpdateCount())), null]
} else {
toStringList(stmt.resultSet)
return toStringList(stmt.resultSet)
}
}
}

static List<List<Object>> toList(ResultSet resultSet) {
static Tuple2<List<List<Object>>, ResultSetMetaData> toList(ResultSet resultSet) {
resultSet.withCloseable {
List<List<Object>> rows = new ArrayList<>()
def columnCount = resultSet.metaData.columnCount
Expand All @@ -56,19 +58,17 @@ class JdbcUtils {
}
rows.add(row)
}
return rows
return [rows, resultSet.metaData]
}
}

static List<List<Object>> toStringList(ResultSet resultSet) {
static Tuple2<List<List<Object>>, ResultSetMetaData> toStringList(ResultSet resultSet) {
resultSet.withCloseable {
List<List<Object>> rows = new ArrayList<>()
def columnCount = resultSet.metaData.columnCount
while (resultSet.next()) {
def row = new ArrayList<>()
for (int i = 1; i <= columnCount; ++i) {
// row.add(resultSet.getObject(i))
// row.add(resultSet.getString(i))
try {
row.add(resultSet.getObject(i))
} catch (Throwable t) {
Expand All @@ -81,7 +81,7 @@ class JdbcUtils {
}
rows.add(row)
}
return rows
return [rows, resultSet.metaData]
}
}
}
71 changes: 64 additions & 7 deletions ...ession-test/framework/src/main/groovy/org/apache/doris/regression/util/OutputUtils.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,22 @@ import org.apache.commons.csv.CSVRecord
import org.apache.commons.io.LineIterator

import java.util.function.Function
import java.sql.ResultSetMetaData

@CompileStatic
class OutputUtils {
static String columnToCsvString(Object column) {
private static List<String> castList(Object obj) {
List<String> result = new ArrayList<String>();
for (Object o: (List<Object>) obj) {
result.add(toCsvString(o));
}
return result;
}

static String toCsvString(Object cell) {
StringWriter writer = new StringWriter()
def printer = new CSVPrinter(new PrintWriter(writer), CSVFormat.MYSQL)
printer.print(column)
printer.print(cell)
return writer.toString()
}

Expand All @@ -44,9 +53,36 @@ class OutputUtils {
return writer.toString()
}

static String checkCell(String info, int line, String expectCell, String realCell, String dataType) {
if(dataType == "FLOAT" || dataType == "DOUBLE") {
double expectDouble = Double.parseDouble(expectCell)
double realDouble = Double.parseDouble(realCell)

double realRelativeError = Math.abs(expectDouble - realDouble) / realDouble
double expectRelativeError = 1e-10

if(expectRelativeError < realRelativeError) {
return "${info}, line ${line}, ${dataType} result mismatch.\nExpect cell is: ${expectCell}\nBut real is: ${realCell}\nrelative error is: ${realRelativeError}, bigger than ${expectRelativeError}"
}
} else if(dataType == "DATE" || dataType =="DATETIME") {
expectCell = expectCell.replace("T", " ")
realCell = realCell.replace("T", " ")

if(!expectCell.equals(realCell)) {
return "${info}, line ${line}, ${dataType} result mismatch.\nExpect cell is: ${expectCell}\nBut real is: ${realCell}"
}
} else {
if(!expectCell.equals(realCell)) {
return "${info}, line ${line}, ${dataType} result mismatch.\nExpect cell is: ${expectCell}\nBut real is: ${realCell}"
}
}

return null
}

static <T1, T2> String checkOutput(Iterator<T1> expect, Iterator<T2> real,
Function<T1, String> transform1, Function<T2, String> transform2,
String info) {
String info, ResultSetMetaData meta) {
int line = 1
while (true) {
if (expect.hasNext() && !real.hasNext()) {
Expand All @@ -59,11 +95,32 @@ class OutputUtils {
break
}

def expectCsvString = transform1.apply(expect.next())
def realCsvString = transform2.apply(real.next())
if (!expectCsvString.equals(realCsvString)) {
return "${info}, line ${line} mismatch.\nExpect line is: ${expectCsvString}\nBut real is : ${realCsvString}"
def expectRaw = expect.next()
def realRaw = real.next()

if (expectRaw instanceof List && meta != null) {
List<String> expectList = castList(expectRaw)
List<String> realList = castList(realRaw)

def columnCount = meta.columnCount
for (int i = 1; i <= columnCount; i++) {
String expectCell = toCsvString(expectList[i - 1])
String realCell = toCsvString(realList[i - 1])
String dataType = meta.getColumnTypeName(i)

def res = checkCell(info, line, expectCell, realCell, dataType)
if(res != null) {
return res
}
}
} else {
def expectCsvString = transform1.apply(expectRaw)
def realCsvString = transform2.apply(realRaw)
if (!expectCsvString.equals(realCsvString)) {
return "${info}, line ${line} mismatch.\nExpect line is: ${expectCsvString}\nBut real is: ${realCsvString}"
}
}

line++
}
}
Expand Down