From f67ae744f67fe650fdc3ae0f117d6e5b2042492b Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Fri, 11 Jun 2021 20:56:06 -0300 Subject: [PATCH 1/5] row_binary_fixed_string support --- .../util/ClickHouseRowBinaryStream.java | 14 ++++ .../integration/NativeStreamTest.java | 15 +++- .../util/ClickHouseRowBinaryStreamTest.java | 74 +++++++++++++++++++ 3 files changed, 100 insertions(+), 3 deletions(-) diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java index 626d84900..a5400ff61 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java @@ -93,6 +93,20 @@ public void writeString(String string) throws IOException { out.write(bytes); } + public void writeFixedString(String string) throws IOException { + byte[] bytes = Objects.requireNonNull(string).getBytes(StandardCharsets.UTF_8); + out.write(bytes); + } + + public void writeFixedString(String string, Integer len) throws IOException { + byte[] bytes = Objects.requireNonNull(string).getBytes(StandardCharsets.UTF_8); + Integer bl = bytes.length; + out.write(bytes, 0, Math.min(len, bl)); + for (int i = 0; i < len - bl; i++) { + out.write(0); + } + } + public void writeUInt8(boolean value) throws IOException { out.writeByte(value ? 1 : 0); } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java index ee7d6a7e8..256df0f8e 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java @@ -33,7 +33,11 @@ public void testLowCardinality() throws Exception{ final ClickHouseStatement statement = connection.createStatement(); connection.createStatement().execute("DROP TABLE IF EXISTS test.low_cardinality"); connection.createStatement().execute( - "CREATE TABLE test.low_cardinality (date Date, lowCardinality LowCardinality(String), string String) ENGINE = MergeTree(date, (date), 8192)" + "CREATE TABLE test.low_cardinality (date Date, " + + "lowCardinality LowCardinality(String), " + + "string String," + + "fixedString String) " + + "ENGINE = MergeTree(date, (date), 8192)" ); // Code: 368, e.displayText() = DB::Exception: Bad cast from type DB::ColumnString to DB::ColumnLowCardinality @@ -44,11 +48,11 @@ public void testLowCardinality() throws Exception{ final Date date1 = new Date(1497474018000L); statement.sendNativeStream( - "INSERT INTO test.low_cardinality (date, lowCardinality, string)", + "INSERT INTO test.low_cardinality (date, lowCardinality, string, fixedString)", new ClickHouseStreamCallback() { @Override public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { - stream.writeUnsignedLeb128(3); // Columns number + stream.writeUnsignedLeb128(4); // Columns number stream.writeUnsignedLeb128(1); // Rows number stream.writeString("date"); // Column name @@ -62,6 +66,10 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { stream.writeString("string"); // Column name stream.writeString("String"); // Column type stream.writeString("string"); // value + + stream.writeString("fixedString"); // Column name + stream.writeString("FixedString(3)"); // Column type + stream.writeFixedString("str"); // value } } ); @@ -71,5 +79,6 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { Assert.assertTrue(rs.next()); assertEquals(rs.getString("lowCardinality"), "string"); assertEquals(rs.getString("string"), "string"); + assertEquals(rs.getString("fixedString"), "str"); } } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStreamTest.java index b7469055d..fe0dd8b9f 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStreamTest.java @@ -262,6 +262,80 @@ public void write(ClickHouseRowBinaryStream stream) throws Exception { ); } + @Test + public void testFixedString() throws Exception { + check( + new StreamWriter() { + @Override + public void write(ClickHouseRowBinaryStream stream) throws Exception { + stream.writeFixedString( + "aaaa~����%20�&zzzzz" + ); + } + }, + new byte[]{ + 97, 97, 97, 97, 126, -17, -65, -67, -17, -65, -67, -17, -65, -67, -17, + -65, -67, 37, 50, 48, -17, -65, -67, 38, 122, 122, 122, 122, 122 + } + //clickhouse-client -q "select toFixedString('aaaa~����%20�&zzzzz', 29) format RowBinary"| od -vAn -td1 + ); + } + + @Test + public void testFixedStringLen() throws Exception { + check( + new StreamWriter() { + @Override + public void write(ClickHouseRowBinaryStream stream) throws Exception { + stream.writeFixedString( + "aaaa~����%20�&zzzzz", 32 + ); + } + }, + new byte[]{ + 97, 97, 97, 97, 126, -17, -65, -67, -17, -65, -67, -17, -65, -67, -17, + -65, -67, 37, 50, 48, -17, -65, -67, 38, 122, 122, 122, 122, 122, 0, 0, 0 + } + //clickhouse-client -q "select toFixedString('aaaa~����%20�&zzzzz', 32) format RowBinary"| od -vAn -td1 + ); + } + + @Test + public void testFixedStringLen1() throws Exception { + check( + new StreamWriter() { + @Override + public void write(ClickHouseRowBinaryStream stream) throws Exception { + stream.writeFixedString( + "", 5 + ); + } + }, + new byte[]{ + 0, 0, 0, 0, 0 + } + //clickhouse-client -q "select toFixedString('', 5) format RowBinary"| od -vAn -td1 + ); + } + + @Test + public void testFixedStringLen2() throws Exception { + check( + new StreamWriter() { + @Override + public void write(ClickHouseRowBinaryStream stream) throws Exception { + stream.writeFixedString( + "1234567890", 5 + ); + } + }, + new byte[]{ + 49, 50, 51, 52, 53 + } + //clickhouse-client -q "select toFixedString('1234567890', 5) format RowBinary"| od -vAn -td1 + ); + } + @Test public void testUUID() throws Exception { check( From 265db6dd14813839e0a108d9cc47030642c699eb Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Fri, 11 Jun 2021 21:01:22 -0300 Subject: [PATCH 2/5] row_binary_fixed_string support --- .../java/ru/yandex/clickhouse/integration/NativeStreamTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java index 256df0f8e..276c63947 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java @@ -36,7 +36,7 @@ public void testLowCardinality() throws Exception{ "CREATE TABLE test.low_cardinality (date Date, " + "lowCardinality LowCardinality(String), " + "string String," + - "fixedString String) " + + "fixedString FixedString(3)) " + "ENGINE = MergeTree(date, (date), 8192)" ); From b8ca0e989ce3c90ec43c2e105b1cfb9f901dafa5 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Fri, 11 Jun 2021 21:19:05 -0300 Subject: [PATCH 3/5] more tests --- .../clickhouse/integration/NativeStreamTest.java | 13 ++++++++++--- .../clickhouse/integration/RowBinaryStreamTest.java | 10 ++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java index 276c63947..5d7eaf3d5 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java @@ -36,8 +36,9 @@ public void testLowCardinality() throws Exception{ "CREATE TABLE test.low_cardinality (date Date, " + "lowCardinality LowCardinality(String), " + "string String," + - "fixedString FixedString(3)) " + - "ENGINE = MergeTree(date, (date), 8192)" + "fixedString FixedString(3)," + + "fixedStringLC LowCardinality(FixedString(6))" + + ") ENGINE = MergeTree(date, (date), 8192)" ); // Code: 368, e.displayText() = DB::Exception: Bad cast from type DB::ColumnString to DB::ColumnLowCardinality @@ -48,7 +49,7 @@ public void testLowCardinality() throws Exception{ final Date date1 = new Date(1497474018000L); statement.sendNativeStream( - "INSERT INTO test.low_cardinality (date, lowCardinality, string, fixedString)", + "INSERT INTO test.low_cardinality (date, lowCardinality, string, fixedString, fixedStringLC)", new ClickHouseStreamCallback() { @Override public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { @@ -70,6 +71,11 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { stream.writeString("fixedString"); // Column name stream.writeString("FixedString(3)"); // Column type stream.writeFixedString("str"); // value + + stream.writeString("fixedStringLC"); // Column name + stream.writeString("FixedString(6)"); // Column type + stream.writeFixedString("str1", 6); // value + } } ); @@ -80,5 +86,6 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(rs.getString("lowCardinality"), "string"); assertEquals(rs.getString("string"), "string"); assertEquals(rs.getString("fixedString"), "str"); + assertEquals(rs.getString("fixedStringLC"), "str1"); } } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java index b6a672f89..d0868aac5 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java @@ -83,7 +83,8 @@ private void createTable(String table) throws SQLException { "float32Array Array(Float32), " + "float64Array Array(Float64), " + "uuid UUID," + - "lowCardinality LowCardinality(String)" + + "lowCardinality LowCardinality(String)," + + "fixedString FixedString(15)" + ") ENGINE = MergeTree(date, (date), 8192)" ); } @@ -340,7 +341,7 @@ private void testRowBinaryStream(boolean rowBinaryResult) throws Exception { statement.sendRowBinaryStream( "INSERT INTO test.raw_binary " + - "(date, dateTime, string, int8, uInt8, int16, uInt16, int32, uInt32, int64, uInt64, float32, float64, dateArray, dateTimeArray, stringArray, int8Array, uInt8Array, int16Array, uInt16Array, int32Array, uInt32Array, int64Array, uInt64Array, float32Array, float64Array, uuid, lowCardinality)", + "(date, dateTime, string, int8, uInt8, int16, uInt16, int32, uInt32, int64, uInt64, float32, float64, dateArray, dateTimeArray, stringArray, int8Array, uInt8Array, int16Array, uInt16Array, int32Array, uInt32Array, int64Array, uInt64Array, float32Array, float64Array, uuid, lowCardinality, fixedString)", new ClickHouseStreamCallback() { @Override public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { @@ -373,6 +374,8 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { stream.writeFloat64Array(float64s1); stream.writeUUID(uuid1); stream.writeString("lowCardinality\n1"); + stream.writeString("lowCardinality\n1"); + stream.writeFixedString("fixedString1", 15); stream.writeDate(date2); stream.writeDateTime(date2); @@ -402,6 +405,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { stream.writeFloat64Array(new double[]{}); stream.writeUUID(uuid2); stream.writeString("lowCardinality\n2"); + stream.writeFixedString("fixedString2", 15); } } ); @@ -427,6 +431,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(rs.getDouble("float64"), 42.21); assertEquals(rs.getObject("uuid").toString(), "123e4567-e89b-12d3-a456-426655440000"); assertEquals(rs.getString("lowCardinality"), "lowCardinality\n1"); + assertEquals(rs.getString("fixedString"), "fixedString1"); Date[] expectedDates1 = new Date[dates1.length]; for (int i = 0; i < dates1.length; i++) { @@ -470,6 +475,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(rs.getDouble("float64"), 77.77); assertEquals(rs.getString("uuid"), "789e0123-e89b-12d3-a456-426655444444"); assertEquals(rs.getString("lowCardinality"), "lowCardinality\n2"); + assertEquals(rs.getString("fixedString"), "fixedString2"); Assert.assertFalse(rs.next()); } else { From 3c4862f7d104acbecd81b9989c42f5254424467a Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Fri, 11 Jun 2021 22:04:32 -0300 Subject: [PATCH 4/5] fix tests --- .../clickhouse/util/ClickHouseRowBinaryStream.java | 13 +++++++++++++ .../clickhouse/integration/NativeStreamTest.java | 7 +++---- .../clickhouse/integration/RowBinaryStreamTest.java | 8 ++++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java index a5400ff61..f5afcc0a0 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java @@ -93,11 +93,24 @@ public void writeString(String string) throws IOException { out.write(bytes); } + /** + * Write string with predefined proper length. + * + * @param string Input string + * @throws IOException in case if an I/O error occurs + */ public void writeFixedString(String string) throws IOException { byte[] bytes = Objects.requireNonNull(string).getBytes(StandardCharsets.UTF_8); out.write(bytes); } + /** + * Write string with any length, but it will be corrected, cut or extended to len + * + * @param string Input string + * @param len Length of FixedString + * @throws IOException in case if an I/O error occurs + */ public void writeFixedString(String string, Integer len) throws IOException { byte[] bytes = Objects.requireNonNull(string).getBytes(StandardCharsets.UTF_8); Integer bl = bytes.length; diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java index 5d7eaf3d5..84dc4892a 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java @@ -38,7 +38,7 @@ public void testLowCardinality() throws Exception{ "string String," + "fixedString FixedString(3)," + "fixedStringLC LowCardinality(FixedString(6))" + - ") ENGINE = MergeTree(date, (date), 8192)" + ") ENGINE = MergeTree partition by toYYYYMM(date) order by date" ); // Code: 368, e.displayText() = DB::Exception: Bad cast from type DB::ColumnString to DB::ColumnLowCardinality @@ -53,7 +53,7 @@ public void testLowCardinality() throws Exception{ new ClickHouseStreamCallback() { @Override public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { - stream.writeUnsignedLeb128(4); // Columns number + stream.writeUnsignedLeb128(5); // Columns number stream.writeUnsignedLeb128(1); // Rows number stream.writeString("date"); // Column name @@ -75,7 +75,6 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { stream.writeString("fixedStringLC"); // Column name stream.writeString("FixedString(6)"); // Column type stream.writeFixedString("str1", 6); // value - } } ); @@ -86,6 +85,6 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(rs.getString("lowCardinality"), "string"); assertEquals(rs.getString("string"), "string"); assertEquals(rs.getString("fixedString"), "str"); - assertEquals(rs.getString("fixedStringLC"), "str1"); + assertEquals(rs.getString("fixedStringLC"), "str1\0\0"); } } diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java index d0868aac5..bef85b9a1 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java @@ -85,7 +85,7 @@ private void createTable(String table) throws SQLException { "uuid UUID," + "lowCardinality LowCardinality(String)," + "fixedString FixedString(15)" + - ") ENGINE = MergeTree(date, (date), 8192)" + ") ENGINE = MergeTree partition by toYYYYMM(date) order by date" ); } @@ -374,7 +374,6 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { stream.writeFloat64Array(float64s1); stream.writeUUID(uuid1); stream.writeString("lowCardinality\n1"); - stream.writeString("lowCardinality\n1"); stream.writeFixedString("fixedString1", 15); stream.writeDate(date2); @@ -431,7 +430,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(rs.getDouble("float64"), 42.21); assertEquals(rs.getObject("uuid").toString(), "123e4567-e89b-12d3-a456-426655440000"); assertEquals(rs.getString("lowCardinality"), "lowCardinality\n1"); - assertEquals(rs.getString("fixedString"), "fixedString1"); + assertEquals(rs.getString("fixedString"), "fixedString1\0\0\0"); Date[] expectedDates1 = new Date[dates1.length]; for (int i = 0; i < dates1.length; i++) { @@ -475,7 +474,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(rs.getDouble("float64"), 77.77); assertEquals(rs.getString("uuid"), "789e0123-e89b-12d3-a456-426655444444"); assertEquals(rs.getString("lowCardinality"), "lowCardinality\n2"); - assertEquals(rs.getString("fixedString"), "fixedString2"); + assertEquals(rs.getString("fixedString"), "fixedString2\0\0\0"); Assert.assertFalse(rs.next()); } else { @@ -556,6 +555,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(is.readUUID(), uuid2); assertEquals(is.readString(), "lowCardinality\n2"); + assertEquals(is.readString(), "fixedString2\0\0\0"); // check EOF try { From 9bf36f1aeb583c7b28747445ae28e9df6f888ff2 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Fri, 11 Jun 2021 22:29:03 -0300 Subject: [PATCH 5/5] fix tests --- .../clickhouse/integration/RowBinaryStreamTest.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java index bef85b9a1..bc4d490c5 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java @@ -341,7 +341,10 @@ private void testRowBinaryStream(boolean rowBinaryResult) throws Exception { statement.sendRowBinaryStream( "INSERT INTO test.raw_binary " + - "(date, dateTime, string, int8, uInt8, int16, uInt16, int32, uInt32, int64, uInt64, float32, float64, dateArray, dateTimeArray, stringArray, int8Array, uInt8Array, int16Array, uInt16Array, int32Array, uInt32Array, int64Array, uInt64Array, float32Array, float64Array, uuid, lowCardinality, fixedString)", + "(date, dateTime, string, int8, uInt8, int16, uInt16, int32, uInt32, int64, uInt64, float32, " + + "float64, dateArray, dateTimeArray, stringArray, int8Array, uInt8Array, int16Array, uInt16Array, " + + "int32Array, uInt32Array, int64Array, uInt64Array, float32Array, float64Array, uuid, lowCardinality, " + + "fixedString)", new ClickHouseStreamCallback() { @Override public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { @@ -374,7 +377,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { stream.writeFloat64Array(float64s1); stream.writeUUID(uuid1); stream.writeString("lowCardinality\n1"); - stream.writeFixedString("fixedString1", 15); + stream.writeFixedString("fixedString1\0\0\0"); stream.writeDate(date2); stream.writeDateTime(date2); @@ -523,6 +526,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(is.readUUID(), uuid1); assertEquals(is.readString(), "lowCardinality\n1"); + assertEquals(is.readFixedString(15), "fixedString1\0\0\0"); assertEquals(is.readDate(), withTimeAtStartOfDay(date2)); assertEquals(is.readDateTime().getTime(), date2.getTime()); @@ -555,7 +559,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException { assertEquals(is.readUUID(), uuid2); assertEquals(is.readString(), "lowCardinality\n2"); - assertEquals(is.readString(), "fixedString2\0\0\0"); + assertEquals(is.readFixedString(15), "fixedString2\0\0\0"); // check EOF try {