From 204d419790c1286b5275c928c1120d2285ccc61b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A5=E5=89=91=E6=97=AD?= Date: Mon, 16 Oct 2023 11:56:23 +0800 Subject: [PATCH 1/8] update run-thirdparties-dorcker.sh for p0 testing --- docker/thirdparties/run-thirdparties-docker.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 97357361fd3115..28b410f4ac27a3 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -37,7 +37,7 @@ Usage: $0 --stop stop the specified components All valid components: - mysql,pg,oracle,sqlserver,clickhouse,es,hive,iceberg,hudi,trino + mysql,pg,oracle,sqlserver,clickhouse,es,hive,iceberg,kafka,hudi,trino " exit 1 } @@ -60,7 +60,7 @@ STOP=0 if [[ "$#" == 1 ]]; then # default - COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino" + COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,kafka,hudi,trino" else while true; do case "$1" in @@ -92,7 +92,7 @@ else done if [[ "${COMPONENTS}"x == ""x ]]; then if [[ "${STOP}" -eq 1 ]]; then - COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino" + COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg,kafka,hudi,trino" fi fi fi @@ -128,6 +128,7 @@ RUN_CLICKHOUSE=0 RUN_HIVE=0 RUN_ES=0 RUN_ICEBERG=0 +RUN_KAFKA=0 RUN_HUDI=0 RUN_TRINO=0 @@ -146,6 +147,8 @@ for element in "${COMPONENTS_ARR[@]}"; do RUN_ES=1 elif [[ "${element}"x == "hive"x ]]; then RUN_HIVE=1 + elif [[ "${element}"x == "kafka"x ]]; then + RUN_KAFKA=1 elif [[ "${element}"x == "iceberg"x ]]; then RUN_ICEBERG=1 elif [[ "${element}"x == "hudi"x ]]; then From 90edc92d4507c501d5e02712a08b2cd47841c7af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A5=E5=89=91=E6=97=AD?= Date: Fri, 15 Mar 2024 19:42:00 +0800 Subject: [PATCH 2/8] update ccr tc project --- .../org/apache/doris/regression/Config.groovy | 29 +++++++++++++++ .../doris/regression/suite/Suite.groovy | 35 +++++++++++++++++-- .../regression/suite/SuiteContext.groovy | 17 ++++++--- .../regression/suite/SyncerContext.groovy | 6 ++-- 4 files changed, 77 insertions(+), 10 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index 8fcba76f0f7c8b..e3da767c50196f 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -41,6 +41,10 @@ class Config { public String jdbcPassword public String defaultDb + public String downstreamUrl + public String downstreamUser + public String downstreamPassword + public String feSourceThriftAddress public String feTargetThriftAddress public String feSyncerUser @@ -309,6 +313,9 @@ class Config { configToString(obj.sslCertificatePath) ) + config.downstreamUrl = configToString(obj.downstreamUrl) + config.downstreamUser = configToString(obj.downstreamUser) + config.downstreamPassword = configToString(obj.downstreamPassword) config.image = configToString(obj.image) config.dockerEndDeleteFiles = configToBoolean(obj.dockerEndDeleteFiles) config.excludeDockerTest = configToBoolean(obj.excludeDockerTest) @@ -533,6 +540,28 @@ class Config { return DriverManager.getConnection(dbUrl, jdbcUser, jdbcPassword) } + Connection getConnectionByArrowFlightSql(String dbName) { + Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver") + String arrowFlightSqlHost = otherConfigs.get("extArrowFlightSqlHost") + String arrowFlightSqlPort = otherConfigs.get("extArrowFlightSqlPort") + String arrowFlightSqlUrl = "jdbc:arrow-flight-sql://${arrowFlightSqlHost}:${arrowFlightSqlPort}" + + "/?useServerPrepStmts=false&useSSL=false&useEncryption=false" + // TODO jdbc:arrow-flight-sql not support connect db + String dbUrl = buildUrlWithDbImpl(arrowFlightSqlUrl, dbName) + tryCreateDbIfNotExist(dbName) + log.info("connect to ${dbUrl}".toString()) + String arrowFlightSqlJdbcUser = otherConfigs.get("extArrowFlightSqlUser") + String arrowFlightSqlJdbcPassword = otherConfigs.get("extArrowFlightSqlPassword") + return DriverManager.getConnection(dbUrl, arrowFlightSqlJdbcUser, arrowFlightSqlJdbcPassword) + } + + Connection getDownstreamConnectionByDbName(String dbName) { + String dbUrl = buildUrlWithDb(downstreamUrl, dbName) + tryCreateDbIfNotExist(dbName) + log.info("connect to ${dbUrl}".toString()) + return DriverManager.getConnection(dbUrl, downstreamUser, downstreamPassword) + } + String getDbNameByFile(File suiteFile) { String dir = new File(suitePath).relativePath(suiteFile.parentFile) // We put sql files under sql dir, so dbs and tables used by cases diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index f347a6cc9413ac..a7f0a94c7c833a 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -51,6 +51,7 @@ import java.util.stream.Collectors import java.util.stream.LongStream import static org.apache.doris.regression.util.DataUtils.sortByToString +import java.sql.Connection import java.sql.DriverManager import java.sql.PreparedStatement import java.sql.ResultSetMetaData @@ -259,9 +260,34 @@ class Suite implements GroovyInterceptable { return result } - def sql_return_maparray(String sqlStr) { + List> insert_into_sql_impl(Connection conn, String sqlStr, int num) { + logger.info("insert into " + num + " records") + def (result, meta) = JdbcUtils.executeToList(conn, sqlStr) + return result + } + + List> jdbc_insert_into_sql(String sqlStr, int num) { + return insert_into_sql_impl(context.getConnection(), sqlStr, num) + } + + List> arrow_flight_insert_into_sql(String sqlStr, int num) { + return insert_into_sql_impl(context.getArrowFlightSqlConnection(), (String) ("USE ${context.dbName};" + sqlStr), num) + } + + List> insert_into_sql(String sqlStr, int num) { + if (context.useArrowFlightSql()) { + return arrow_flight_insert_into_sql(sqlStr, num) + } else { + return jdbc_insert_into_sql(sqlStr, num) + } + } + + def sql_return_maparray(String sqlStr, Connection conn = null) { logger.info("Execute sql: ${sqlStr}".toString()) - def (result, meta) = JdbcUtils.executeToList(context.getConnection(), sqlStr) + if (conn == null) { + conn = context.getConnection() + } + def (result, meta) = JdbcUtils.executeToList(conn, sqlStr) // get all column names as list List columnNames = new ArrayList<>() @@ -516,6 +542,11 @@ class Suite implements GroovyInterceptable { return lines; } + + Connection getTargetConnection() { + return context.getTargetConnection(this) + } + boolean deleteFile(String filePath) { def file = new File(filePath) file.delete() diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy index 9cc21faae0777c..14e6f2da42c31b 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy @@ -152,9 +152,14 @@ class SuiteContext implements Closeable { return subJdbc.substring(0, subJdbc.indexOf("/")) } - private Map getSpec() { + private String getDownstreamJdbcNetInfo() { + String subJdbc = config.downstreamUrl.substring(config.downstreamUrl.indexOf("://") + 3) + return subJdbc.substring(0, subJdbc.indexOf("/")) + } + + private Map getSpec(String[] jdbc) { Map spec = Maps.newHashMap() - String[] jdbc = getJdbcNetInfo().split(":") + spec.put("host", jdbc[0]) spec.put("port", jdbc[1]) spec.put("user", config.feSyncerUser) @@ -165,7 +170,8 @@ class SuiteContext implements Closeable { } Map getSrcSpec() { - Map spec = getSpec() + String[] jdbc = getJdbcNetInfo().split(":") + Map spec = getSpec(jdbc) spec.put("thrift_port", config.feSourceThriftNetworkAddress.port.toString()) spec.put("database", dbName) @@ -173,7 +179,8 @@ class SuiteContext implements Closeable { } Map getDestSpec() { - Map spec = getSpec() + String[] jdbc = getDownstreamJdbcNetInfo().split(":") + Map spec = getSpec(jdbc) spec.put("thrift_port", config.feTargetThriftNetworkAddress.port.toString()) spec.put("database", "TEST_" + dbName) @@ -203,7 +210,7 @@ class SuiteContext implements Closeable { Connection getTargetConnection(Suite suite) { def context = getSyncer(suite).context if (context.targetConnection == null) { - context.targetConnection = config.getConnectionByDbName("TEST_" + dbName) + context.targetConnection = config.getDownstreamConnectionByDbName("TEST_" + dbName) } return context.targetConnection } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy index 388904ec2da4d2..92214532fdcf4e 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SyncerContext.groovy @@ -147,8 +147,8 @@ class SyncerContext { return info } - FrontendClientImpl getMasterFrontClient() { - def result = suite.sql_return_maparray "select Host, RpcPort, IsMaster from frontends();" + FrontendClientImpl getMasterFrontClient(Connection conn) { + def result = suite.sql_return_maparray("select Host, RpcPort, IsMaster from frontends();", conn) logger.info("get master fe: ${result}") def masterHost = "" @@ -179,7 +179,7 @@ class SyncerContext { FrontendClientImpl getTargetFrontClient() { if (targetFrontendClient == null) { - targetFrontendClient = getMasterFrontClient() + targetFrontendClient = getMasterFrontClient(suite.getTargetConnection()) } return targetFrontendClient } From ca3d29295c59ae89b1e99f319e3856ef42a39b76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A5=E5=89=91=E6=97=AD?= Date: Thu, 21 Mar 2024 10:22:27 +0800 Subject: [PATCH 3/8] update ccr --- .../org/apache/doris/regression/Config.groovy | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index e3da767c50196f..3ad2fcb89e3f7f 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -540,6 +540,27 @@ class Config { return DriverManager.getConnection(dbUrl, jdbcUser, jdbcPassword) } + public static String buildUrlWithDbImpl(String jdbcUrl, String dbName) { + String urlWithDb = jdbcUrl + String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3) + if (urlWithoutSchema.indexOf("/") >= 0) { + if (jdbcUrl.contains("?")) { + // e.g: jdbc:mysql://locahost:8080/?a=b + urlWithDb = jdbcUrl.substring(0, jdbcUrl.lastIndexOf("?")) + urlWithDb = urlWithDb.substring(0, urlWithDb.lastIndexOf("/")) + urlWithDb += ("/" + dbName) + jdbcUrl.substring(jdbcUrl.lastIndexOf("?")) + } else { + // e.g: jdbc:mysql://locahost:8080/ + urlWithDb += dbName + } + } else { + // e.g: jdbc:mysql://locahost:8080 + urlWithDb += ("/" + dbName) + } + + return urlWithDb + } + Connection getConnectionByArrowFlightSql(String dbName) { Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver") String arrowFlightSqlHost = otherConfigs.get("extArrowFlightSqlHost") From 71b8080c812a83a8e7591b0d20c496dba74acc87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A5=E5=89=91=E6=97=AD?= Date: Thu, 21 Mar 2024 11:24:14 +0800 Subject: [PATCH 4/8] update ccr project --- regression-test/pipeline/p0/conf/regression-conf.groovy | 5 +++++ regression-test/pipeline/p1/conf/regression-conf.groovy | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy index 8cfe8811993c0f..5463e62fbc6889 100644 --- a/regression-test/pipeline/p0/conf/regression-conf.groovy +++ b/regression-test/pipeline/p0/conf/regression-conf.groovy @@ -25,6 +25,11 @@ targetJdbcUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLo jdbcUser = "root" jdbcPassword = "" +ccrDownstreamUrl = "jdbc:mysql://172.19.0.2:9132/?useLocalSessionState=true&allowLoadLocalInfile=true" +ccrDownstreamUser = "root" +ccrDownstreamPassword = "" +ccrDownstreamFeThriftAddress = "127.0.0.1:9020" + feSourceThriftAddress = "127.0.0.1:9020" feTargetThriftAddress = "127.0.0.1:9020" feSyncerUser = "root" diff --git a/regression-test/pipeline/p1/conf/regression-conf.groovy b/regression-test/pipeline/p1/conf/regression-conf.groovy index 8669e8fb5bd3ed..98f0d9173b218f 100644 --- a/regression-test/pipeline/p1/conf/regression-conf.groovy +++ b/regression-test/pipeline/p1/conf/regression-conf.groovy @@ -25,6 +25,11 @@ targetJdbcUrl = "jdbc:mysql://172.19.0.2:9132/?useLocalSessionState=true&allowLo jdbcUser = "root" jdbcPassword = "" +ccrDownstreamUrl = "jdbc:mysql://172.19.0.2:9132/?useLocalSessionState=true&allowLoadLocalInfile=true" +ccrDownstreamUser = "root" +ccrDownstreamPassword = "" +ccrDownstreamFeThriftAddress = "127.0.0.1:9020" + feSourceThriftAddress = "127.0.0.1:9020" feTargetThriftAddress = "127.0.0.1:9020" feSyncerUser = "root" From 345c2b14fb149640eb4ab21e64bde6293b514cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A5=E5=89=91=E6=97=AD?= Date: Thu, 21 Mar 2024 15:06:09 +0800 Subject: [PATCH 5/8] update ccr --- .../org/apache/doris/regression/Config.groovy | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index 3ad2fcb89e3f7f..44be413a7eb5c9 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -41,9 +41,9 @@ class Config { public String jdbcPassword public String defaultDb - public String downstreamUrl - public String downstreamUser - public String downstreamPassword + public String ccrccrDownstreamUrl + public String ccrccrDownstreamUser + public String ccrccrDownstreamPassword public String feSourceThriftAddress public String feTargetThriftAddress @@ -313,9 +313,9 @@ class Config { configToString(obj.sslCertificatePath) ) - config.downstreamUrl = configToString(obj.downstreamUrl) - config.downstreamUser = configToString(obj.downstreamUser) - config.downstreamPassword = configToString(obj.downstreamPassword) + config.ccrDownstreamUrl = configToString(obj.ccrDownstreamUrl) + config.ccrDownstreamUser = configToString(obj.ccrDownstreamUser) + config.ccrDownstreamPassword = configToString(obj.ccrDownstreamPassword) config.image = configToString(obj.image) config.dockerEndDeleteFiles = configToBoolean(obj.dockerEndDeleteFiles) config.excludeDockerTest = configToBoolean(obj.excludeDockerTest) @@ -560,7 +560,7 @@ class Config { return urlWithDb } - + Connection getConnectionByArrowFlightSql(String dbName) { Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver") String arrowFlightSqlHost = otherConfigs.get("extArrowFlightSqlHost") @@ -577,10 +577,10 @@ class Config { } Connection getDownstreamConnectionByDbName(String dbName) { - String dbUrl = buildUrlWithDb(downstreamUrl, dbName) + String dbUrl = buildUrlWithDb(ccrDownstreamUrl, dbName) tryCreateDbIfNotExist(dbName) log.info("connect to ${dbUrl}".toString()) - return DriverManager.getConnection(dbUrl, downstreamUser, downstreamPassword) + return DriverManager.getConnection(dbUrl, ccrDownstreamUser, ccrDownstreamPassword) } String getDbNameByFile(File suiteFile) { From 5b8e33ad4c0d5d42adb54c898625b23f385800b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A5=E5=89=91=E6=97=AD?= Date: Thu, 21 Mar 2024 15:08:43 +0800 Subject: [PATCH 6/8] update ccr --- .../main/groovy/org/apache/doris/regression/Config.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index 44be413a7eb5c9..f15cd81bc35cad 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -41,9 +41,9 @@ class Config { public String jdbcPassword public String defaultDb - public String ccrccrDownstreamUrl - public String ccrccrDownstreamUser - public String ccrccrDownstreamPassword + public String ccrDownstreamUrl + public String ccrDownstreamUser + public String ccrDownstreamPassword public String feSourceThriftAddress public String feTargetThriftAddress From 86dc45328e86a0fa2e1da8296d6dad379cbc6c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A5=E5=89=91=E6=97=AD?= Date: Thu, 21 Mar 2024 15:55:14 +0800 Subject: [PATCH 7/8] update ccr --- .../org/apache/doris/regression/suite/SuiteContext.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy index 14e6f2da42c31b..b2a2cb7ba1e393 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy @@ -153,7 +153,7 @@ class SuiteContext implements Closeable { } private String getDownstreamJdbcNetInfo() { - String subJdbc = config.downstreamUrl.substring(config.downstreamUrl.indexOf("://") + 3) + String subJdbc = config.ccrDownstreamUrl.substring(config.ccrDownstreamUrl.indexOf("://") + 3) return subJdbc.substring(0, subJdbc.indexOf("/")) } From 7df021fe69e16aee732578a25fae079928a6777b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A5=E5=89=91=E6=97=AD?= Date: Thu, 21 Mar 2024 16:58:56 +0800 Subject: [PATCH 8/8] update ccr --- regression-test/pipeline/p0/conf/regression-conf.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy index 5463e62fbc6889..615cb4f81c09d6 100644 --- a/regression-test/pipeline/p0/conf/regression-conf.groovy +++ b/regression-test/pipeline/p0/conf/regression-conf.groovy @@ -25,7 +25,7 @@ targetJdbcUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLo jdbcUser = "root" jdbcPassword = "" -ccrDownstreamUrl = "jdbc:mysql://172.19.0.2:9132/?useLocalSessionState=true&allowLoadLocalInfile=true" +ccrDownstreamUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLoadLocalInfile=true" ccrDownstreamUser = "root" ccrDownstreamPassword = "" ccrDownstreamFeThriftAddress = "127.0.0.1:9020"