Skip to content

[fix](jdbc) fix clickhouse catalog arr nullable and add case #26639

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zy-kkk merged 2 commits into from
Nov 9, 2023
Merged

[fix](jdbc) fix clickhouse catalog arr nullable and add case #26639

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ea7fe64
[fix](jdbc) fix clickhouse catalog arr nullable and add case
zy-kkk Nov 9, 2023
eb43bd9
[fix](jdbc) fix clickhouse catalog arr nullable and add case
zy-kkk Nov 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 141 additions & 0 deletions docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,79 @@ CREATE TABLE doris_test.type
ENGINE = MergeTree
ORDER BY k1;

CREATE TABLE doris_test.type_null
(
`id` String,
`k1` Nullable(Bool),
`k2` Nullable(Date),
`k3` Nullable(Date32),
`k4` Nullable(Datetime),
`k5` Nullable(Datetime64),
`k6` Nullable(Float32),
`k7` Nullable(Float64),
`k8` Nullable(Int8),
`k9` Nullable(Int16),
`k10` Nullable(Int32),
`k11` Nullable(Int64),
`k12` Nullable(Int128),
`k13` Nullable(Int256),
`k14` Nullable(UInt8),
`k15` Nullable(UInt16),
`k16` Nullable(UInt32),
`k17` Nullable(UInt64),
`k18` Nullable(UInt128),
`k19` Nullable(UInt256),
`k20` Nullable(Decimal(9,2)),
`k21` Nullable(Decimal(18,2)),
`k22` Nullable(Decimal(38,2)),
`k23` Nullable(Decimal(76,2)),
`k24` Nullable(Enum('hello' = 1, 'world' = 2)),
`k25` Nullable(IPv4),
`k26` Nullable(IPv6),
`k27` Nullable(UUID),
`k28` Nullable(String),
`k29` Nullable(FixedString(2))
)
ENGINE = MergeTree
ORDER BY id;

CREATE TABLE doris_test.type_ins
(
`id` String,
`k1` Nullable(Bool),
`k2` Nullable(Date),
`k3` Nullable(Date32),
`k4` Nullable(Datetime),
`k5` Nullable(Datetime64),
`k6` Nullable(Float32),
`k7` Nullable(Float64),
`k8` Nullable(Int8),
`k9` Nullable(Int16),
`k10` Nullable(Int32),
`k11` Nullable(Int64),
`k12` Nullable(Int128),
`k13` Nullable(Int256),
`k14` Nullable(UInt8),
`k15` Nullable(UInt16),
`k16` Nullable(UInt32),
`k17` Nullable(UInt64),
`k18` Nullable(UInt128),
`k19` Nullable(UInt256),
`k20` Nullable(Decimal(9,2)),
`k21` Nullable(Decimal(18,2)),
`k22` Nullable(Decimal(38,2)),
`k23` Nullable(Decimal(76,2)),
`k24` Nullable(Enum('hello' = 1, 'world' = 2)),
`k25` Nullable(IPv4),
`k26` Nullable(IPv6),
`k27` Nullable(UUID),
`k28` Nullable(String),
`k29` Nullable(FixedString(2))
)
ENGINE = MergeTree
ORDER BY id;


CREATE TABLE doris_test.number
(
`k6` Float32,
Expand Down Expand Up @@ -115,6 +188,74 @@ CREATE TABLE doris_test.arr
ENGINE = MergeTree
ORDER BY id;

CREATE TABLE doris_test.arr_null
(
`id` String,
`arr1` Array(Nullable(Bool)),
`arr2` Array(Nullable(Date)),
`arr3` Array(Nullable(Date32)),
`arr4` Array(Nullable(Float32)),
`arr5` Array(Nullable(Float64)),
`arr6` Array(Nullable(Int8)),
`arr7` Array(Nullable(Int16)),
`arr8` Array(Nullable(Int32)),
`arr9` Array(Nullable(Int64)),
`arr10` Array(Nullable(Int128)),
`arr11` Array(Nullable(Int256)),
`arr12` Array(Nullable(UInt8)),
`arr13` Array(Nullable(UInt16)),
`arr14` Array(Nullable(UInt32)),
`arr15` Array(Nullable(UInt64)),
`arr16` Array(Nullable(UInt128)),
`arr17` Array(Nullable(UInt256)),
`arr18` Array(Nullable(Decimal(9,2))),
`arr19` Array(Nullable(Enum('hello' = 1, 'world' = 2))),
`arr20` Array(Nullable(IPv4)),
`arr21` Array(Nullable(IPv6)),
`arr22` Array(Nullable(UUID)),
`arr23` Array(Nullable(Int8)),
`arr24` Array(Nullable(String)),
`arr25` Array(LowCardinality(String)),
`arr26` Array(Nullable(Datetime)),
`arr27` Array(Nullable(Datetime64))
)
ENGINE = MergeTree
ORDER BY id;

CREATE TABLE doris_test.arr_ins
(
`id` String,
`arr1` Array(Nullable(Bool)),
`arr2` Array(Nullable(Date)),
`arr3` Array(Nullable(Date32)),
`arr4` Array(Nullable(Float32)),
`arr5` Array(Nullable(Float64)),
`arr6` Array(Nullable(Int8)),
`arr7` Array(Nullable(Int16)),
`arr8` Array(Nullable(Int32)),
`arr9` Array(Nullable(Int64)),
`arr10` Array(Nullable(Int128)),
`arr11` Array(Nullable(Int256)),
`arr12` Array(Nullable(UInt8)),
`arr13` Array(Nullable(UInt16)),
`arr14` Array(Nullable(UInt32)),
`arr15` Array(Nullable(UInt64)),
`arr16` Array(Nullable(UInt128)),
`arr17` Array(Nullable(UInt256)),
`arr18` Array(Nullable(Decimal(9,2))),
`arr19` Array(Nullable(Enum('hello' = 1, 'world' = 2))),
`arr20` Array(Nullable(IPv4)),
`arr21` Array(Nullable(IPv6)),
`arr22` Array(Nullable(UUID)),
`arr23` Array(Nullable(Int8)),
`arr24` Array(Nullable(String)),
`arr25` Array(LowCardinality(String)),
`arr26` Array(Nullable(Datetime)),
`arr27` Array(Nullable(Datetime64))
)
ENGINE = MergeTree
ORDER BY id;

set allow_experimental_object_type = 1;
CREATE TABLE doris_test.json
(
Expand Down
13 changes: 13 additions & 0 deletions docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,14 @@ INSERT INTO doris_test.type VALUES
INSERT INTO doris_test.type VALUES
(false, '2022-01-02','2022-01-02','2022-01-02 00:00:00','2022-01-02 00:00:00.000000000',2.2,2.2,2,2,2,2,2,2,2,2,2,2,2,2,2.2,2.2,2.2,2.2,2,'116.253.40.133','2a02:aa08:e000:3100::2','61f0c404-5cb3-11e7-907b-a6006ad3dba0','String','T');


INSERT INTO doris_test.type_null VALUES
('1',true, '2022-01-01','2022-01-01','2022-01-01 00:00:00','2022-01-01 00:00:00.000000000',1.1,1.1,1,1,1,1,1,1,1,1,1,1,1,1,1.1,1.1,1.1,1.1,1,'116.253.40.133','2a02:aa08:e000:3100::2','61f0c404-5cb3-11e7-907b-a6006ad3dba0','String','F');
INSERT INTO doris_test.type_null VALUES
('2',false, '2022-01-02','2022-01-02','2022-01-02 00:00:00','2022-01-02 00:00:00.000000000',2.2,2.2,2,2,2,2,2,2,2,2,2,2,2,2,2.2,2.2,2.2,2.2,2,'116.253.40.133','2a02:aa08:e000:3100::2','61f0c404-5cb3-11e7-907b-a6006ad3dba0','String','T');
INSERT INTO doris_test.type_null VALUES
('3',NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);

INSERT INTO doris_test.number
(`k6`, `k7`, `k8`, `k9`, `k10`, `k11`, `k12`, `k13`, `k14`, `k15`, `k16`, `k17`, `k18`, `k19`)
VALUES
Expand All @@ -34,6 +42,11 @@ INSERT INTO doris_test.student values (1, 'doris', 18), (2, 'alice', 19), (3, 'b
INSERT INTO doris_test.arr values
('1',[true],['2022-01-01'],['2022-01-01'],[1.1],[1.1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[2.2],[1],['116.253.40.133'],['2a02:aa08:e000:3100::2'],['61f0c404-5cb3-11e7-907b-a6006ad3dba0'],[1],['string'],['string'],['2022-01-01 00:00:00'],['2022-01-01 00:00:00']);

INSERT INTO doris_test.arr_null values
('1',[true],['2022-01-01'],['2022-01-01'],[1.1],[1.1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[2.2],[1],['116.253.40.133'],['2a02:aa08:e000:3100::2'],['61f0c404-5cb3-11e7-907b-a6006ad3dba0'],[1],['string'],['string'],['2022-01-01 00:00:00'],['2022-01-01 00:00:00']);
INSERT INTO doris_test.arr_null values
('2', [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL], [NULL]);

INSERT INTO doris_test.json VALUES ('1','{"a": 1, "b": { "c": 2, "d": [1, 2, 3] }}');

INSERT INTO doris_test.final_test Values (1, 'first');
Expand Down
38 changes: 33 additions & 5 deletions fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
import java.math.RoundingMode;
import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
Expand Down Expand Up @@ -1996,21 +1995,50 @@ private void copyBatchDecimalResult(BigInteger[] column, boolean isNullable, int
static {
CK_ARRAY_CONVERTERS.put(String[].class, res -> Arrays.toString((String[]) res));
CK_ARRAY_CONVERTERS.put(boolean[].class, res -> Arrays.toString((boolean[]) res));
CK_ARRAY_CONVERTERS.put(Boolean[].class, res -> Arrays.toString((Boolean[]) res));
CK_ARRAY_CONVERTERS.put(byte[].class, res -> Arrays.toString((byte[]) res));
CK_ARRAY_CONVERTERS.put(Byte[].class, res -> Arrays.toString((Byte[]) res));
CK_ARRAY_CONVERTERS.put(LocalDate[].class, res -> Arrays.toString((LocalDate[]) res));
CK_ARRAY_CONVERTERS.put(LocalDateTime[].class, res -> Arrays.toString((LocalDateTime[]) res));
CK_ARRAY_CONVERTERS.put(float[].class, res -> Arrays.toString((float[]) res));
CK_ARRAY_CONVERTERS.put(Float[].class, res -> Arrays.toString((Float[]) res));
CK_ARRAY_CONVERTERS.put(double[].class, res -> Arrays.toString((double[]) res));
CK_ARRAY_CONVERTERS.put(Double[].class, res -> Arrays.toString((Double[]) res));
CK_ARRAY_CONVERTERS.put(short[].class, res -> Arrays.toString((short[]) res));
CK_ARRAY_CONVERTERS.put(Short[].class, res -> Arrays.toString((Short[]) res));
CK_ARRAY_CONVERTERS.put(int[].class, res -> Arrays.toString((int[]) res));
CK_ARRAY_CONVERTERS.put(Integer[].class, res -> Arrays.toString((Integer[]) res));
CK_ARRAY_CONVERTERS.put(long[].class, res -> Arrays.toString((long[]) res));
CK_ARRAY_CONVERTERS.put(Long[].class, res -> Arrays.toString((Long[]) res));
CK_ARRAY_CONVERTERS.put(BigInteger[].class, res -> Arrays.toString((BigInteger[]) res));
CK_ARRAY_CONVERTERS.put(BigDecimal[].class, res -> Arrays.toString((BigDecimal[]) res));
CK_ARRAY_CONVERTERS.put(Inet4Address[].class, res -> Arrays.toString(Arrays.stream((Inet4Address[]) res)
.map(InetAddress::getHostAddress).toArray(String[]::new)));
CK_ARRAY_CONVERTERS.put(Inet6Address[].class, res -> Arrays.toString(Arrays.stream((Inet6Address[]) res)
.map(addr -> simplifyIPv6Address(addr.getHostAddress())).toArray(String[]::new)));
CK_ARRAY_CONVERTERS.put(Inet4Address[].class, res -> {
if (res == null) {
return "null";
} else {
return Arrays.toString(Arrays.stream((Inet4Address[]) res)
.map(addr -> addr == null ? "null" : addr.getHostAddress())
.toArray(String[]::new));
}
});

CK_ARRAY_CONVERTERS.put(Inet6Address[].class, res -> {
if (res == null) {
return "null";
} else {
return Arrays.toString(Arrays.stream((Inet6Address[]) res)
.map(addr -> addr == null ? "null" : simplifyIPv6Address(addr.getHostAddress()))
.toArray(String[]::new));
}
});
CK_ARRAY_CONVERTERS.put(com.clickhouse.data.value.UnsignedByte[].class,
res -> Arrays.toString((com.clickhouse.data.value.UnsignedByte[]) res));
CK_ARRAY_CONVERTERS.put(com.clickhouse.data.value.UnsignedShort[].class,
res -> Arrays.toString((com.clickhouse.data.value.UnsignedShort[]) res));
CK_ARRAY_CONVERTERS.put(com.clickhouse.data.value.UnsignedInteger[].class,
res -> Arrays.toString((com.clickhouse.data.value.UnsignedInteger[]) res));
CK_ARRAY_CONVERTERS.put(com.clickhouse.data.value.UnsignedLong[].class,
res -> Arrays.toString((com.clickhouse.data.value.UnsignedLong[]) res));
CK_ARRAY_CONVERTERS.put(UUID[].class, res -> Arrays.toString((UUID[]) res));
}

Expand Down
Binary file modified regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.out
View file
Open in desktop
Binary file not shown.
8 changes: 8 additions & 0 deletions regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,16 @@ suite("test_clickhouse_jdbc_catalog", "p0,external,clickhouse,external_docker,ex
sql """ use ${ex_db_name} """

order_qt_type """ select * from type order by k1; """
order_qt_type_null """ select * from type_null order by id; """
sql """drop table if exists internal.${internal_db_name}.ck_type_null """
order_qt_ctas_type_null """create table internal.${internal_db_name}.ck_type_null PROPERTIES("replication_num" = "1") as select * from type_null """;
order_qt_query_ctas_type_null """ select * from internal.${internal_db_name}.ck_type_null order by id; """
order_qt_number """ select * from number order by k6; """
order_qt_arr """ select * from arr order by id; """
order_qt_arr_null """ select * from arr_null order by id; """
sql """ drop table if exists internal.${internal_db_name}.ck_arr_null"""
order_qt_ctas_arr_null """create table internal.${internal_db_name}.ck_arr_null PROPERTIES("replication_num" = "1") as select * from arr_null """;
order_qt_query_ctas_arr_null """ select * from internal.${internal_db_name}.ck_arr_null order by id; """
sql """ insert into internal.${internal_db_name}.${inDorisTable} select * from student; """
order_qt_in_tb """ select id, name, age from internal.${internal_db_name}.${inDorisTable} order by id; """
order_qt_system """ show tables from `system`; """
Expand Down