Skip to content

row_binary_fixed_string support #673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zhicwu merged 5 commits into from
Jun 12, 2021
Merged

row_binary_fixed_string support #673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f67ae74
row_binary_fixed_string support
den-crane Jun 11, 2021
265db6d
row_binary_fixed_string support
den-crane Jun 12, 2021
b8ca0e9
more tests
den-crane Jun 12, 2021
3c4862f
fix tests
den-crane Jun 12, 2021
9bf36f1
fix tests
den-crane Jun 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,33 @@ public void writeString(String string) throws IOException {
out.write(bytes);
}

/**
* Write string with predefined proper length.
*
* @param string Input string
* @throws IOException in case if an I/O error occurs
*/
public void writeFixedString(String string) throws IOException {
byte[] bytes = Objects.requireNonNull(string).getBytes(StandardCharsets.UTF_8);
out.write(bytes);
}

/**
* Write string with any length, but it will be corrected, cut or extended to len
*
* @param string Input string
* @param len Length of FixedString
* @throws IOException in case if an I/O error occurs
*/
public void writeFixedString(String string, Integer len) throws IOException {
byte[] bytes = Objects.requireNonNull(string).getBytes(StandardCharsets.UTF_8);
Integer bl = bytes.length;
out.write(bytes, 0, Math.min(len, bl));
for (int i = 0; i < len - bl; i++) {
out.write(0);
}
}

public void writeUInt8(boolean value) throws IOException {
out.writeByte(value ? 1 : 0);
}
Expand Down
21 changes: 18 additions & 3 deletions clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/NativeStreamTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,12 @@ public void testLowCardinality() throws Exception{
final ClickHouseStatement statement = connection.createStatement();
connection.createStatement().execute("DROP TABLE IF EXISTS test.low_cardinality");
connection.createStatement().execute(
"CREATE TABLE test.low_cardinality (date Date, lowCardinality LowCardinality(String), string String) ENGINE = MergeTree(date, (date), 8192)"
"CREATE TABLE test.low_cardinality (date Date, " +
"lowCardinality LowCardinality(String), " +
"string String," +
"fixedString FixedString(3)," +
"fixedStringLC LowCardinality(FixedString(6))" +
") ENGINE = MergeTree partition by toYYYYMM(date) order by date"
);

// Code: 368, e.displayText() = DB::Exception: Bad cast from type DB::ColumnString to DB::ColumnLowCardinality
Expand All @@ -44,11 +49,11 @@ public void testLowCardinality() throws Exception{
final Date date1 = new Date(1497474018000L);

statement.sendNativeStream(
"INSERT INTO test.low_cardinality (date, lowCardinality, string)",
"INSERT INTO test.low_cardinality (date, lowCardinality, string, fixedString, fixedStringLC)",
new ClickHouseStreamCallback() {
@Override
public void writeTo(ClickHouseRowBinaryStream stream) throws IOException {
stream.writeUnsignedLeb128(3); // Columns number
stream.writeUnsignedLeb128(5); // Columns number
stream.writeUnsignedLeb128(1); // Rows number

stream.writeString("date"); // Column name
Expand All @@ -62,6 +67,14 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException {
stream.writeString("string"); // Column name
stream.writeString("String"); // Column type
stream.writeString("string"); // value

stream.writeString("fixedString"); // Column name
stream.writeString("FixedString(3)"); // Column type
stream.writeFixedString("str"); // value

stream.writeString("fixedStringLC"); // Column name
stream.writeString("FixedString(6)"); // Column type
stream.writeFixedString("str1", 6); // value
}
}
);
Expand All @@ -71,5 +84,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException {
Assert.assertTrue(rs.next());
assertEquals(rs.getString("lowCardinality"), "string");
assertEquals(rs.getString("string"), "string");
assertEquals(rs.getString("fixedString"), "str");
assertEquals(rs.getString("fixedStringLC"), "str1\0\0");
}
}
16 changes: 13 additions & 3 deletions clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/RowBinaryStreamTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,9 @@ private void createTable(String table) throws SQLException {
"float32Array Array(Float32), " +
"float64Array Array(Float64), " +
"uuid UUID," +
"lowCardinality LowCardinality(String)" +
") ENGINE = MergeTree(date, (date), 8192)"
"lowCardinality LowCardinality(String)," +
"fixedString FixedString(15)" +
") ENGINE = MergeTree partition by toYYYYMM(date) order by date"
);
}

Expand Down Expand Up @@ -340,7 +341,10 @@ private void testRowBinaryStream(boolean rowBinaryResult) throws Exception {

statement.sendRowBinaryStream(
"INSERT INTO test.raw_binary " +
"(date, dateTime, string, int8, uInt8, int16, uInt16, int32, uInt32, int64, uInt64, float32, float64, dateArray, dateTimeArray, stringArray, int8Array, uInt8Array, int16Array, uInt16Array, int32Array, uInt32Array, int64Array, uInt64Array, float32Array, float64Array, uuid, lowCardinality)",
"(date, dateTime, string, int8, uInt8, int16, uInt16, int32, uInt32, int64, uInt64, float32, " +
"float64, dateArray, dateTimeArray, stringArray, int8Array, uInt8Array, int16Array, uInt16Array, " +
"int32Array, uInt32Array, int64Array, uInt64Array, float32Array, float64Array, uuid, lowCardinality, " +
"fixedString)",
new ClickHouseStreamCallback() {
@Override
public void writeTo(ClickHouseRowBinaryStream stream) throws IOException {
Expand Down Expand Up @@ -373,6 +377,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException {
stream.writeFloat64Array(float64s1);
stream.writeUUID(uuid1);
stream.writeString("lowCardinality\n1");
stream.writeFixedString("fixedString1\0\0\0");

stream.writeDate(date2);
stream.writeDateTime(date2);
Expand Down Expand Up @@ -402,6 +407,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException {
stream.writeFloat64Array(new double[]{});
stream.writeUUID(uuid2);
stream.writeString("lowCardinality\n2");
stream.writeFixedString("fixedString2", 15);
}
}
);
Expand All @@ -427,6 +433,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException {
assertEquals(rs.getDouble("float64"), 42.21);
assertEquals(rs.getObject("uuid").toString(), "123e4567-e89b-12d3-a456-426655440000");
assertEquals(rs.getString("lowCardinality"), "lowCardinality\n1");
assertEquals(rs.getString("fixedString"), "fixedString1\0\0\0");

Date[] expectedDates1 = new Date[dates1.length];
for (int i = 0; i < dates1.length; i++) {
Expand Down Expand Up @@ -470,6 +477,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException {
assertEquals(rs.getDouble("float64"), 77.77);
assertEquals(rs.getString("uuid"), "789e0123-e89b-12d3-a456-426655444444");
assertEquals(rs.getString("lowCardinality"), "lowCardinality\n2");
assertEquals(rs.getString("fixedString"), "fixedString2\0\0\0");

Assert.assertFalse(rs.next());
} else {
Expand Down Expand Up @@ -518,6 +526,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException {

assertEquals(is.readUUID(), uuid1);
assertEquals(is.readString(), "lowCardinality\n1");
assertEquals(is.readFixedString(15), "fixedString1\0\0\0");

assertEquals(is.readDate(), withTimeAtStartOfDay(date2));
assertEquals(is.readDateTime().getTime(), date2.getTime());
Expand Down Expand Up @@ -550,6 +559,7 @@ public void writeTo(ClickHouseRowBinaryStream stream) throws IOException {

assertEquals(is.readUUID(), uuid2);
assertEquals(is.readString(), "lowCardinality\n2");
assertEquals(is.readFixedString(15), "fixedString2\0\0\0");

// check EOF
try {
Expand Down
74 changes: 74 additions & 0 deletions clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/util/ClickHouseRowBinaryStreamTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,80 @@ public void write(ClickHouseRowBinaryStream stream) throws Exception {
);
}

@Test
public void testFixedString() throws Exception {
check(
new StreamWriter() {
@Override
public void write(ClickHouseRowBinaryStream stream) throws Exception {
stream.writeFixedString(
"aaaa~����%20�&zzzzz"
);
}
},
new byte[]{
97, 97, 97, 97, 126, -17, -65, -67, -17, -65, -67, -17, -65, -67, -17,
-65, -67, 37, 50, 48, -17, -65, -67, 38, 122, 122, 122, 122, 122
}
//clickhouse-client -q "select toFixedString('aaaa~����%20�&zzzzz', 29) format RowBinary"| od -vAn -td1
);
}

@Test
public void testFixedStringLen() throws Exception {
check(
new StreamWriter() {
@Override
public void write(ClickHouseRowBinaryStream stream) throws Exception {
stream.writeFixedString(
"aaaa~����%20�&zzzzz", 32
);
}
},
new byte[]{
97, 97, 97, 97, 126, -17, -65, -67, -17, -65, -67, -17, -65, -67, -17,
-65, -67, 37, 50, 48, -17, -65, -67, 38, 122, 122, 122, 122, 122, 0, 0, 0
}
//clickhouse-client -q "select toFixedString('aaaa~����%20�&zzzzz', 32) format RowBinary"| od -vAn -td1
);
}

@Test
public void testFixedStringLen1() throws Exception {
check(
new StreamWriter() {
@Override
public void write(ClickHouseRowBinaryStream stream) throws Exception {
stream.writeFixedString(
"", 5
);
}
},
new byte[]{
0, 0, 0, 0, 0
}
//clickhouse-client -q "select toFixedString('', 5) format RowBinary"| od -vAn -td1
);
}

@Test
public void testFixedStringLen2() throws Exception {
check(
new StreamWriter() {
@Override
public void write(ClickHouseRowBinaryStream stream) throws Exception {
stream.writeFixedString(
"1234567890", 5
);
}
},
new byte[]{
49, 50, 51, 52, 53
}
//clickhouse-client -q "select toFixedString('1234567890', 5) format RowBinary"| od -vAn -td1
);
}

@Test
public void testUUID() throws Exception {
check(
Expand Down