From 180771ee2acd541600830534c7f11f44ecc2ec3e Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Tue, 14 Nov 2023 19:49:35 +0800 Subject: [PATCH] [fix](catalog) Fix ClickHouse DataTime64 precision parsing --- .../clickhouse/init/03-create-table.sql | 9 ++++++++ .../clickhouse/init/04-insert.sql | 2 +- .../jdbc/client/JdbcClickHouseClient.java | 20 ++++++------------ regression-test/conf/regression-conf.groovy | 8 +++---- .../jdbc/test_clickhouse_jdbc_catalog.out | Bin 4887 -> 4942 bytes .../jdbc/test_clickhouse_jdbc_catalog.groovy | 2 ++ 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql index b09c5f1543a350..34f03796e76381 100644 --- a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql +++ b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql @@ -279,4 +279,13 @@ CREATE TABLE doris_test.ts ts UInt64 ) ENGINE = MergeTree +ORDER BY id; + +CREATE TABLE doris_test.dt_with_tz +( + id Int64, + dt1 DateTime('Asia/Shanghai'), + dt2 DateTime64(6, 'Asia/Shanghai') +) +ENGINE = MergeTree ORDER BY id; \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql b/docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql index f8af30e80153f6..08b5d034aa160f 100644 --- a/docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql +++ b/docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql @@ -54,4 +54,4 @@ INSERT INTO doris_test.final_test Values (1, 'second'); INSERT INTO doris_test.ts values (1,1694438743); - +INSERT INTO doris_test.dt_with_tz values(1, '2022-01-02 00:00:00','2022-01-02 00:00:00.000000'); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java index f25fabcc3c8e98..a6088ad11caa60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java @@ -73,20 +73,14 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) { if (ckType.startsWith("DateTime(") || ckType.equals("DateTime")) { return ScalarType.createDatetimeV2Type(0); } else { - int indexStart = ckType.indexOf('('); - int indexEnd = ckType.indexOf(')'); - if (indexStart != -1 && indexEnd != -1) { - String scaleStr = ckType.substring(indexStart + 1, indexEnd); - int scale = Integer.parseInt(scaleStr); - if (scale > JDBC_DATETIME_SCALE) { - scale = JDBC_DATETIME_SCALE; - } - // return with the actual scale - return ScalarType.createDatetimeV2Type(scale); - } else { - // default precision if not specified - return ScalarType.createDatetimeV2Type(JDBC_DATETIME_SCALE); + // DateTime64 with millisecond precision + // Datetime64(6) / DateTime64(6, 'Asia/Shanghai') + String[] accuracy = ckType.substring(11, ckType.length() - 1).split(", "); + int precision = Integer.parseInt(accuracy[0]); + if (precision > 6) { + precision = JDBC_DATETIME_SCALE; } + return ScalarType.createDatetimeV2Type(precision); } } diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index dd3d1c29b34218..b0ff6169397658 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -24,8 +24,8 @@ defaultDb = "regression_test" // init cmd like: select @@session.tx_read_only // at each time we connect. // add allowLoadLocalInfile so that the jdbc can execute mysql load data from client. -jdbcUrl = "jdbc:mysql://127.0.0.1:9632/?useLocalSessionState=true&allowLoadLocalInfile=true" -targetJdbcUrl = "jdbc:mysql://127.0.0.1:9632/?useLocalSessionState=true&allowLoadLocalInfile=true" +jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true" +targetJdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true" jdbcUser = "root" jdbcPassword = "" @@ -35,7 +35,7 @@ syncerAddress = "127.0.0.1:9190" feSyncerUser = "root" feSyncerPassword = "" -feHttpAddress = "127.0.0.1:8632" +feHttpAddress = "127.0.0.1:8030" feHttpUser = "root" feHttpPassword = "" @@ -86,7 +86,7 @@ sk="" // jdbc connector test config // To enable jdbc test, you need first start mysql/pg container. // See `docker/thirdparties/start-thirdparties-docker.sh` -enableJdbcTest=true +enableJdbcTest=false mysql_57_port=3316 pg_14_port=5442 oracle_11_port=1521 diff --git a/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.out index 5f2d70696c3ab0ae76a76e5645797ab22bdca594..dcffcfe9ca7d21032c8da3aca5446b71a393ac7d 100644 GIT binary patch delta 34 pcmbQPc1~@BxUjIUu7YApNql)`Nk)80m4dD=m*HeaW+f&rE&#Wc2>t*7 delta 7 OcmX@7HeGFlxG(?=X#$J@ diff --git a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy index 987cbe940273bb..294ee63e58364e 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy @@ -91,6 +91,8 @@ suite("test_clickhouse_jdbc_catalog", "p0,external,clickhouse,external_docker,ex contains """QUERY: SELECT "id", "ts" FROM "doris_test"."ts" WHERE ("ts" <= toUnixTimestamp(FROM_UNIXTIME(ts, '%Y-%m-%d')))""" } + order_qt_dt_with_tz """ select * from dt_with_tz order by id; """ + sql """ drop catalog if exists ${catalog_name} """ } }