From 631e736c378290dd8511b5383e2811c66c11159e Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Thu, 27 Feb 2025 22:49:15 -0500 Subject: [PATCH 01/15] Some initial tests for validating the results --- .../data_formats/RowBinaryFormatWriter.java | 6 +- .../internal/SerializerUtils.java | 2 + .../datatypes/RowBinaryFormatWriterTest.java | 284 ++++++++++++++++++ 3 files changed, 290 insertions(+), 2 deletions(-) create mode 100644 client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatWriter.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatWriter.java index 2f6aa7c95..b498abe06 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatWriter.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatWriter.java @@ -29,6 +29,8 @@ public class RowBinaryFormatWriter { private final Object[] row; + private final boolean defaultSupport; + public RowBinaryFormatWriter(OutputStream out, TableSchema tableSchema, ClickHouseFormat format) { if (format != ClickHouseFormat.RowBinary && format != ClickHouseFormat.RowBinaryWithDefaults) { throw new IllegalArgumentException("Only RowBinary and RowBinaryWithDefaults are supported"); @@ -37,6 +39,7 @@ public RowBinaryFormatWriter(OutputStream out, TableSchema tableSchema, ClickHou this.out = out; this.tableSchema = tableSchema; this.row = new Object[tableSchema.getColumns().size()]; + this.defaultSupport = format == ClickHouseFormat.RowBinaryWithDefaults; } public void setValue(String column, Object value) { @@ -48,12 +51,11 @@ public void setValue(int colIndex, Object value) { } public void commitRow() throws IOException { - List columnList = tableSchema.getColumns(); for (int i = 0; i < row.length; i++) { ClickHouseColumn column = columnList.get(i); - if (RowBinaryFormatSerializer.writeValuePreamble(out, true, column, row[i])) { + if (RowBinaryFormatSerializer.writeValuePreamble(out, defaultSupport, column, row[i])) { SerializerUtils.serializeData(out, row[i], column); } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index 076e2c73a..1e5ae3473 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -741,6 +741,8 @@ public static BigDecimal convertToBigDecimal(Object value) { return (BigDecimal) value; } else if (value instanceof BigInteger) { return new BigDecimal((BigInteger) value); + } else if (value instanceof BigDecimal) { + return (BigDecimal) value; } else if (value instanceof Number) { return BigDecimal.valueOf(((Number) value).doubleValue()); } else if (value instanceof String) { diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java new file mode 100644 index 000000000..0449dd3a1 --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -0,0 +1,284 @@ +package com.clickhouse.client.datatypes; + +import com.clickhouse.client.BaseIntegrationTest; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseServerForTest; +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.command.CommandSettings; +import com.clickhouse.client.api.data_formats.RowBinaryFormatWriter; +import com.clickhouse.client.api.enums.Protocol; +import com.clickhouse.client.api.insert.InsertResponse; +import com.clickhouse.client.api.insert.InsertSettings; +import com.clickhouse.client.api.internal.ServerSettings; +import com.clickhouse.client.api.metadata.TableSchema; +import com.clickhouse.client.api.query.GenericRecord; +import com.clickhouse.data.ClickHouseFormat; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +public class RowBinaryFormatWriterTest extends BaseIntegrationTest { + private Client client; + private InsertSettings settings; + private static final int EXECUTE_CMD_TIMEOUT = 30; + + @BeforeMethod(groups = { "integration" }) + public void setUp() throws IOException { + int bufferSize = (7 * 65500); + client = newClient() + .setSocketSndbuf(bufferSize) + .setSocketRcvbuf(bufferSize) + .setClientNetworkBufferSize(bufferSize) + .build(); + + settings = new InsertSettings() + .setDeduplicationToken(RandomStringUtils.randomAlphabetic(36)) + .setQueryId(String.valueOf(UUID.randomUUID())); + } + + protected Client.Builder newClient() { + ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); + boolean isSecure = isCloud(); + return new Client.Builder() + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isSecure) + .setUsername("default") + .setPassword(ClickHouseServerForTest.getPassword()) + .compressClientRequest(true) + .useHttpCompression(true) + .setDefaultDatabase(ClickHouseServerForTest.getDatabase()) + .serverSetting(ServerSettings.ASYNC_INSERT, "0") + .serverSetting(ServerSettings.WAIT_END_OF_QUERY, "1"); + } + + protected void initTable(String tableName, String createTableSQL, CommandSettings settings) throws Exception { + if (settings == null) { + settings = new CommandSettings(); + } + + client.execute("DROP TABLE IF EXISTS " + tableName, settings).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS); + client.execute(createTableSQL, settings).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS); + } + + private static void assertEqualsKinda(Object actual, Object expected) { + assertEquals(String.valueOf(actual), String.valueOf(expected)); + } + + + + @Test (groups = { "integration" }) + public void writeNumbersTest() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeNumbersTest_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " int8 Int8, int8_nullable Nullable(Int8), int8_default Int8 DEFAULT 3, " + + " int16 Int16, int16_nullable Nullable(Int16), int16_default Int16 DEFAULT 3, " + + " int32 Int32, int32_nullable Nullable(Int32), int32_default Int32 DEFAULT 3, " + + " int64 Int64, int64_nullable Nullable(Int64), int64_default Int64 DEFAULT 3, " + + " int128 Int128, int128_nullable Nullable(Int128), int128_default Int128 DEFAULT 3, " + + " int256 Int256, int256_nullable Nullable(Int256), int256_default Int256 DEFAULT 3, " + + " uint8 UInt8, uint8_nullable Nullable(UInt8), uint8_default UInt8 DEFAULT 3, " + + " uint16 UInt16, uint16_nullable Nullable(UInt16), uint16_default UInt16 DEFAULT 3, " + + " uint32 UInt32, uint32_nullable Nullable(UInt32), uint32_default UInt32 DEFAULT 3, " + + " uint64 UInt64, uint64_nullable Nullable(UInt64), uint64_default UInt64 DEFAULT 3, " + + " uint128 UInt128, uint128_nullable Nullable(UInt128), uint128_default UInt128 DEFAULT 3, " + + " uint256 UInt256, uint256_nullable Nullable(UInt256), uint256_default UInt256 DEFAULT 3, " + + " float32 Float32, float32_nullable Nullable(Float32), float32_default Float32 DEFAULT 3, " + + " float64 Float64, float64_nullable Nullable(Float64), float64_default Float64 DEFAULT 3, " + + " decimal Decimal(4, 2), decimal_nullable Nullable(Decimal(4, 2)), decimal_default Decimal(4, 2) DEFAULT 3, " + + " decimal32 Decimal(8, 4), decimal32_nullable Nullable(Decimal(8, 4)), decimal32_default Decimal(8, 4) DEFAULT 3, " + + " decimal64 Decimal(18, 6), decimal64_nullable Nullable(Decimal(18, 6)), decimal64_default Decimal(18, 6) DEFAULT 3, " + + " decimal128 Decimal(36, 8), decimal128_nullable Nullable(Decimal(36, 8)), decimal128_default Decimal(36, 8) DEFAULT 3, " + + " decimal256 Decimal(74, 10), decimal256_nullable Nullable(Decimal(74, 10)), decimal256_default Decimal(74, 10) DEFAULT 3" + + " ) Engine = MergeTree ORDER BY id"; + initTable(tableName, tableCreate, new CommandSettings()); + + // Insert random (valid) values + long seed = System.currentTimeMillis(); + Random rand = new Random(seed); + System.out.println("Random seed: " + seed); + + Object[][] rows = new Object[][] { + {1, + rand.nextInt(256) - 128, null, null, //Int8 + rand.nextInt(65536) - 32768, null, null, //Int16 + rand.nextInt(), null, null, //Int32 + rand.nextLong(), null, null, //Int64 + new BigInteger(127, rand), null, null, //Int128 + new BigInteger(255, rand), null, null, //Int256 + rand.nextInt(256), null, null, //UInt8 + rand.nextInt(65536), null, null, //UInt16 + rand.nextInt() & 0xFFFFFFFFL, null, null, //UInt32 + BigInteger.valueOf(rand.nextLong(Long.MAX_VALUE)), null, null, //UInt64 + new BigInteger(128, rand), null, null, //UInt128 + new BigInteger(256, rand), null, null, //UInt256 + rand.nextFloat(), null, null, //Float32 + rand.nextDouble(), null, null, //Float64 + new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(10,100)), null, null, //Decimal(4) + new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(1000, 10000)), null, null, //Decimal32 + new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000)), null, null, //Decimal64 + new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(10000000, 100000000)), null, null, //Decimal128 + new BigDecimal(new BigInteger(57, rand) + "." + rand.nextLong(1000000000, 10000000000L)), null, null //Decimal256 + }, + {2, + rand.nextInt(256) - 128, null, null, //Int8 + rand.nextInt(65536) - 32768, null, null, //Int16 + rand.nextInt(), null, null, //Int32 + rand.nextLong(), null, null, //Int64 + new BigInteger(127, rand), null, null, //Int128 + new BigInteger(255, rand), null, null, //Int256 + rand.nextInt(256), null, null, //UInt8 + rand.nextInt(65536), null, null, //UInt16 + rand.nextInt() & 0xFFFFFFFFL, null, null, //UInt32 + BigInteger.valueOf(rand.nextLong(Long.MAX_VALUE)), null, null, //UInt64 + new BigInteger(128, rand), null, null, //UInt128 + new BigInteger(256, rand), null, null, //UInt256 + rand.nextFloat(), null, null, //Float32 + rand.nextDouble(), null, null, //Float64 + new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(10,100)), null, null, //Decimal(4) + new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(1000, 10000)), null, null, //Decimal32 + new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000)), null, null, //Decimal64 + new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(10000000, 100000000)), null, null, //Decimal128 + new BigDecimal(new BigInteger(57, rand) + "." + rand.nextLong(1000000000, 10000000000L)), null, null //Decimal256 + }, + }; + + TableSchema schema = client.getTableSchema(tableName); + + ClickHouseFormat format = ClickHouseFormat.RowBinaryWithDefaults; + try (InsertResponse response = client.insert(tableName, out -> { + RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, schema, format); + for (Object[] row : rows) { + for (int i = 0; i < row.length; i++) { + w.setValue(i + 1, row[i]); + } + w.commitRow(); + } + }, format, settings).get()) { + System.out.println("Rows written: " + response.getWrittenRows()); + } + + List records = client.queryAll("SELECT * FROM \"" + tableName + "\"" ); + + int id = 1; + for (GenericRecord record : records) { + Map r = record.getValues(); + assertEquals(r.get("id"), id); + assertEqualsKinda(r.get("int8"), String.valueOf(rows[id - 1][1])); + assertEqualsKinda(r.get("int8_nullable"), rows[id - 1][2]); + assertEqualsKinda(r.get("int8_default"), 3); + assertEqualsKinda(r.get("int16"), rows[id - 1][4]); + assertEqualsKinda(r.get("int16_nullable"), rows[id - 1][5]); + assertEqualsKinda(r.get("int16_default"), 3); + assertEqualsKinda(r.get("int32"), rows[id - 1][7]); + assertEqualsKinda(r.get("int32_nullable"), rows[id - 1][8]); + assertEqualsKinda(r.get("int32_default"), 3); + assertEqualsKinda(r.get("int64"), rows[id - 1][10]); + assertEqualsKinda(r.get("int64_nullable"), rows[id - 1][11]); + assertEqualsKinda(r.get("int64_default"), 3); + assertEqualsKinda(r.get("int128"), rows[id - 1][13]); + assertEqualsKinda(r.get("int128_nullable"), rows[id - 1][14]); + assertEqualsKinda(r.get("int128_default"), 3); + assertEqualsKinda(r.get("int256"), rows[id - 1][16]); + assertEqualsKinda(r.get("int256_nullable"), rows[id - 1][17]); + assertEqualsKinda(r.get("int256_default"), 3); + assertEqualsKinda(r.get("uint8"), rows[id - 1][19]); + assertEqualsKinda(r.get("uint8_nullable"), rows[id - 1][20]); + assertEqualsKinda(r.get("uint8_default"), 3); + assertEqualsKinda(r.get("uint16"), rows[id - 1][22]); + assertEqualsKinda(r.get("uint16_nullable"), rows[id - 1][23]); + assertEqualsKinda(r.get("uint16_default"), 3); + assertEqualsKinda(r.get("uint32"), rows[id - 1][25]); + assertEqualsKinda(r.get("uint32_nullable"), rows[id - 1][26]); + assertEqualsKinda(r.get("uint32_default"), 3); + assertEqualsKinda(r.get("uint64"), rows[id - 1][28]); + assertEqualsKinda(r.get("uint64_nullable"), rows[id - 1][29]); + assertEqualsKinda(r.get("uint64_default"), 3); + assertEqualsKinda(r.get("uint128"), rows[id - 1][31]); + assertEqualsKinda(r.get("uint128_nullable"), rows[id - 1][32]); + assertEqualsKinda(r.get("uint128_default"), 3); + assertEqualsKinda(r.get("uint256"), rows[id - 1][34]); + assertEqualsKinda(r.get("uint256_nullable"), rows[id - 1][35]); + assertEqualsKinda(r.get("uint256_default"), 3); + assertEqualsKinda(r.get("float32"), rows[id - 1][37]); + assertEqualsKinda(r.get("float32_nullable"), rows[id - 1][38]); + assertEqualsKinda(r.get("float32_default"), 3.0); + assertEqualsKinda(r.get("float64"), rows[id - 1][40]); + assertEqualsKinda(r.get("float64_nullable"), rows[id - 1][41]); + assertEqualsKinda(r.get("float64_default"), 3.0); + assertEqualsKinda(r.get("decimal"), rows[id - 1][43]); + assertEqualsKinda(r.get("decimal_nullable"), rows[id - 1][44]); + assertEqualsKinda(r.get("decimal_default"), "3.00"); + assertEqualsKinda(r.get("decimal32"), rows[id - 1][46]); + assertEqualsKinda(r.get("decimal32_nullable"), rows[id - 1][47]); + assertEqualsKinda(r.get("decimal32_default"), "3.0000"); + assertEqualsKinda(r.get("decimal64"), rows[id - 1][49]); + assertEqualsKinda(r.get("decimal64_nullable"), rows[id - 1][50]); + assertEqualsKinda(r.get("decimal64_default"), "3.000000"); + assertEqualsKinda(r.get("decimal128"), rows[id - 1][52]); + assertEqualsKinda(r.get("decimal128_nullable"), rows[id - 1][53]); + assertEqualsKinda(r.get("decimal128_default"), "3.00000000"); + assertEqualsKinda(r.get("decimal256"), rows[id - 1][55]); + assertEqualsKinda(r.get("decimal256_nullable"), rows[id - 1][56]); + assertEqualsKinda(r.get("decimal256_default"), "3.0000000000"); + id++; + } + } + +// @Test +// public void testAdvancedWriter() throws Exception { +// String tableName = "very_long_table_name_with_uuid_" + UUID.randomUUID().toString().replace('-', '_'); +// String tableCreate = "CREATE TABLE \"" + tableName + "\" " + +// " (name String, " + +// " v1 Float32, " + +// " v2 Float32, " + +// " attrs Nullable(String), " + +// " corrected_time DateTime('UTC') DEFAULT now()," + +// " special_attr Nullable(Int8) DEFAULT -1)" + +// " Engine = MergeTree ORDER by ()"; +// +// initTable(tableName, tableCreate); +// +// ZonedDateTime correctedTime = Instant.now().atZone(ZoneId.of("UTC")); +// Object[][] rows = new Object[][] { +// {"foo1", 0.3f, 0.6f, "a=1,b=2,c=5", correctedTime, 10}, +// {"foo2", 0.6f, 0.1f, "a=1,b=2,c=5", correctedTime, null}, +// {"foo3", 0.7f, 0.4f, "a=1,b=2,c=5", null, null}, +// {"foo4", 0.8f, 0.5f, null, null, null}, +// }; +// +// TableSchema schema = client.getTableSchema(tableName); +// +// ClickHouseFormat format = ClickHouseFormat.RowBinaryWithDefaults; +// try (InsertResponse response = client.insert(tableName, out -> { +// RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, schema, format); +// for (Object[] row : rows) { +// for (int i = 0; i < row.length; i++) { +// w.setValue(i + 1, row[i]); +// } +// w.commitRow(); +// } +// }, format, new InsertSettings()).get()) { +// System.out.println("Rows written: " + response.getWrittenRows()); +// } +// +// List records = client.queryAll("SELECT * FROM \"" + tableName + "\"" ); +// +// for (GenericRecord record : records) { +// System.out.println("> " + record.getString(1) + ", " + record.getFloat(2) + ", " + record.getFloat(3)); +// } +// } +} From 07c2946b1ec026c7f0a6444c12329b06a9851a82 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Thu, 27 Feb 2025 23:32:10 -0500 Subject: [PATCH 02/15] Update RowBinaryFormatWriterTest.java --- .../datatypes/RowBinaryFormatWriterTest.java | 242 ++++++------------ 1 file changed, 79 insertions(+), 163 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index 0449dd3a1..ad21a0405 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -77,6 +77,36 @@ private static void assertEqualsKinda(Object actual, Object expected) { assertEquals(String.valueOf(actual), String.valueOf(expected)); } + private void writeTest(String tableName, String tableCreate, Field[][] rows) throws Exception { + initTable(tableName, tableCreate, new CommandSettings()); + TableSchema schema = client.getTableSchema(tableName); + + ClickHouseFormat format = ClickHouseFormat.RowBinaryWithDefaults; + try (InsertResponse response = client.insert(tableName, out -> { + RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, schema, format); + for (Field[] row : rows) { + for (Field field : row) { + w.setValue(schema.nameToColumnIndex(field.name), field.value); + } + w.commitRow(); + } + }, format, settings).get()) { + System.out.println("Rows written: " + response.getWrittenRows()); + } + + List records = client.queryAll("SELECT * FROM \"" + tableName + "\" ORDER BY id" ); + + int id = 1; + for (GenericRecord record : records) { + Map row = record.getValues(); + //Validate data + for (Field field : rows[id - 1]) { + assertEqualsKinda(row.get(field.name), field.getValue()); + } + id++; + } + } + @Test (groups = { "integration" }) @@ -104,181 +134,67 @@ public void writeNumbersTest() throws Exception { " decimal128 Decimal(36, 8), decimal128_nullable Nullable(Decimal(36, 8)), decimal128_default Decimal(36, 8) DEFAULT 3, " + " decimal256 Decimal(74, 10), decimal256_nullable Nullable(Decimal(74, 10)), decimal256_default Decimal(74, 10) DEFAULT 3" + " ) Engine = MergeTree ORDER BY id"; - initTable(tableName, tableCreate, new CommandSettings()); // Insert random (valid) values long seed = System.currentTimeMillis(); Random rand = new Random(seed); System.out.println("Random seed: " + seed); - Object[][] rows = new Object[][] { - {1, - rand.nextInt(256) - 128, null, null, //Int8 - rand.nextInt(65536) - 32768, null, null, //Int16 - rand.nextInt(), null, null, //Int32 - rand.nextLong(), null, null, //Int64 - new BigInteger(127, rand), null, null, //Int128 - new BigInteger(255, rand), null, null, //Int256 - rand.nextInt(256), null, null, //UInt8 - rand.nextInt(65536), null, null, //UInt16 - rand.nextInt() & 0xFFFFFFFFL, null, null, //UInt32 - BigInteger.valueOf(rand.nextLong(Long.MAX_VALUE)), null, null, //UInt64 - new BigInteger(128, rand), null, null, //UInt128 - new BigInteger(256, rand), null, null, //UInt256 - rand.nextFloat(), null, null, //Float32 - rand.nextDouble(), null, null, //Float64 - new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(10,100)), null, null, //Decimal(4) - new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(1000, 10000)), null, null, //Decimal32 - new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000)), null, null, //Decimal64 - new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(10000000, 100000000)), null, null, //Decimal128 - new BigDecimal(new BigInteger(57, rand) + "." + rand.nextLong(1000000000, 10000000000L)), null, null //Decimal256 - }, - {2, - rand.nextInt(256) - 128, null, null, //Int8 - rand.nextInt(65536) - 32768, null, null, //Int16 - rand.nextInt(), null, null, //Int32 - rand.nextLong(), null, null, //Int64 - new BigInteger(127, rand), null, null, //Int128 - new BigInteger(255, rand), null, null, //Int256 - rand.nextInt(256), null, null, //UInt8 - rand.nextInt(65536), null, null, //UInt16 - rand.nextInt() & 0xFFFFFFFFL, null, null, //UInt32 - BigInteger.valueOf(rand.nextLong(Long.MAX_VALUE)), null, null, //UInt64 - new BigInteger(128, rand), null, null, //UInt128 - new BigInteger(256, rand), null, null, //UInt256 - rand.nextFloat(), null, null, //Float32 - rand.nextDouble(), null, null, //Float64 - new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(10,100)), null, null, //Decimal(4) - new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(1000, 10000)), null, null, //Decimal32 - new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000)), null, null, //Decimal64 - new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(10000000, 100000000)), null, null, //Decimal128 - new BigDecimal(new BigInteger(57, rand) + "." + rand.nextLong(1000000000, 10000000000L)), null, null //Decimal256 - }, + Field[][] rows = new Field[][] {{ + new Field("id", 1), + new Field("int8", rand.nextInt(256) - 128), new Field("int8_nullable"), new Field("int8_default").set(3), //Int8 + new Field("int16", rand.nextInt(65536) - 32768), new Field("int16_nullable"), new Field("int16_default").set(3), //Int16 + new Field("int32", rand.nextInt()), new Field("int32_nullable"), new Field("int32_default").set(3), //Int32 + new Field("int64", rand.nextLong()), new Field("int64_nullable"), new Field("int64_default").set(3), //Int64 + new Field("int128", new BigInteger(127, rand)), new Field("int128_nullable"), new Field("int128_default").set(3), //Int128 + new Field("int256", new BigInteger(255, rand)), new Field("int256_nullable"), new Field("int256_default").set(3), //Int256 + new Field("uint8", rand.nextInt(256)), new Field("uint8_nullable"), new Field("uint8_default").set(3), //UInt8 + new Field("uint16", rand.nextInt(65536)), new Field("uint16_nullable"), new Field("uint16_default").set(3), //UInt16 + new Field("uint32", rand.nextInt() & 0xFFFFFFFFL), new Field("uint32_nullable"), new Field("uint32_default").set(3), //UInt32 + new Field("uint64", BigInteger.valueOf(rand.nextLong(Long.MAX_VALUE))), new Field("uint64_nullable"), new Field("uint64_default").set(3), //UInt64 + new Field("uint128", new BigInteger(128, rand)), new Field("uint128_nullable"), new Field("uint128_default").set(3), //UInt128 + new Field("uint256", new BigInteger(256, rand)), new Field("uint256_nullable"), new Field("uint256_default").set(3), //UInt256 + new Field("float32", rand.nextFloat()), new Field("float32_nullable"), new Field("float32_default").set("3.0"), //Float32 + new Field("float64", rand.nextDouble()), new Field("float64_nullable"), new Field("float64_default").set("3.0"), //Float64 + new Field("decimal", new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(10,100))), new Field("decimal_nullable"), new Field("decimal_default").set("3.00"), //Decimal(4) + new Field("decimal32", new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(1000, 10000))), new Field("decimal32_nullable"), new Field("decimal32_default").set("3.0000"), //Decimal32 + new Field("decimal64", new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000))), new Field("decimal64_nullable"), new Field("decimal64_default").set("3.000000"), //Decimal64 + new Field("decimal128", new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(10000000, 100000000))), new Field("decimal128_nullable"), new Field("decimal128_default").set("3.00000000"), //Decimal128 + new Field("decimal256", new BigDecimal(new BigInteger(57, rand) + "." + rand.nextLong(1000000000, 10000000000L))), new Field("decimal256_nullable"), new Field("decimal256_default").set("3.0000000000") //Decimal256 + } }; - TableSchema schema = client.getTableSchema(tableName); + writeTest(tableName, tableCreate, rows); + } - ClickHouseFormat format = ClickHouseFormat.RowBinaryWithDefaults; - try (InsertResponse response = client.insert(tableName, out -> { - RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, schema, format); - for (Object[] row : rows) { - for (int i = 0; i < row.length; i++) { - w.setValue(i + 1, row[i]); - } - w.commitRow(); - } - }, format, settings).get()) { - System.out.println("Rows written: " + response.getWrittenRows()); + + private static class Field { + String name; + Object value; + Object defaultValue; + + Field(String name) { + this.name = name; + this.value = null; + this.defaultValue = null; } - List records = client.queryAll("SELECT * FROM \"" + tableName + "\"" ); + Field(String name, Object value) { + this.name = name; + this.value = value; + } - int id = 1; - for (GenericRecord record : records) { - Map r = record.getValues(); - assertEquals(r.get("id"), id); - assertEqualsKinda(r.get("int8"), String.valueOf(rows[id - 1][1])); - assertEqualsKinda(r.get("int8_nullable"), rows[id - 1][2]); - assertEqualsKinda(r.get("int8_default"), 3); - assertEqualsKinda(r.get("int16"), rows[id - 1][4]); - assertEqualsKinda(r.get("int16_nullable"), rows[id - 1][5]); - assertEqualsKinda(r.get("int16_default"), 3); - assertEqualsKinda(r.get("int32"), rows[id - 1][7]); - assertEqualsKinda(r.get("int32_nullable"), rows[id - 1][8]); - assertEqualsKinda(r.get("int32_default"), 3); - assertEqualsKinda(r.get("int64"), rows[id - 1][10]); - assertEqualsKinda(r.get("int64_nullable"), rows[id - 1][11]); - assertEqualsKinda(r.get("int64_default"), 3); - assertEqualsKinda(r.get("int128"), rows[id - 1][13]); - assertEqualsKinda(r.get("int128_nullable"), rows[id - 1][14]); - assertEqualsKinda(r.get("int128_default"), 3); - assertEqualsKinda(r.get("int256"), rows[id - 1][16]); - assertEqualsKinda(r.get("int256_nullable"), rows[id - 1][17]); - assertEqualsKinda(r.get("int256_default"), 3); - assertEqualsKinda(r.get("uint8"), rows[id - 1][19]); - assertEqualsKinda(r.get("uint8_nullable"), rows[id - 1][20]); - assertEqualsKinda(r.get("uint8_default"), 3); - assertEqualsKinda(r.get("uint16"), rows[id - 1][22]); - assertEqualsKinda(r.get("uint16_nullable"), rows[id - 1][23]); - assertEqualsKinda(r.get("uint16_default"), 3); - assertEqualsKinda(r.get("uint32"), rows[id - 1][25]); - assertEqualsKinda(r.get("uint32_nullable"), rows[id - 1][26]); - assertEqualsKinda(r.get("uint32_default"), 3); - assertEqualsKinda(r.get("uint64"), rows[id - 1][28]); - assertEqualsKinda(r.get("uint64_nullable"), rows[id - 1][29]); - assertEqualsKinda(r.get("uint64_default"), 3); - assertEqualsKinda(r.get("uint128"), rows[id - 1][31]); - assertEqualsKinda(r.get("uint128_nullable"), rows[id - 1][32]); - assertEqualsKinda(r.get("uint128_default"), 3); - assertEqualsKinda(r.get("uint256"), rows[id - 1][34]); - assertEqualsKinda(r.get("uint256_nullable"), rows[id - 1][35]); - assertEqualsKinda(r.get("uint256_default"), 3); - assertEqualsKinda(r.get("float32"), rows[id - 1][37]); - assertEqualsKinda(r.get("float32_nullable"), rows[id - 1][38]); - assertEqualsKinda(r.get("float32_default"), 3.0); - assertEqualsKinda(r.get("float64"), rows[id - 1][40]); - assertEqualsKinda(r.get("float64_nullable"), rows[id - 1][41]); - assertEqualsKinda(r.get("float64_default"), 3.0); - assertEqualsKinda(r.get("decimal"), rows[id - 1][43]); - assertEqualsKinda(r.get("decimal_nullable"), rows[id - 1][44]); - assertEqualsKinda(r.get("decimal_default"), "3.00"); - assertEqualsKinda(r.get("decimal32"), rows[id - 1][46]); - assertEqualsKinda(r.get("decimal32_nullable"), rows[id - 1][47]); - assertEqualsKinda(r.get("decimal32_default"), "3.0000"); - assertEqualsKinda(r.get("decimal64"), rows[id - 1][49]); - assertEqualsKinda(r.get("decimal64_nullable"), rows[id - 1][50]); - assertEqualsKinda(r.get("decimal64_default"), "3.000000"); - assertEqualsKinda(r.get("decimal128"), rows[id - 1][52]); - assertEqualsKinda(r.get("decimal128_nullable"), rows[id - 1][53]); - assertEqualsKinda(r.get("decimal128_default"), "3.00000000"); - assertEqualsKinda(r.get("decimal256"), rows[id - 1][55]); - assertEqualsKinda(r.get("decimal256_nullable"), rows[id - 1][56]); - assertEqualsKinda(r.get("decimal256_default"), "3.0000000000"); - id++; + public Field set(Object defaultValue) {//For default value for comparison purposes + this.defaultValue = defaultValue; + return this; } - } -// @Test -// public void testAdvancedWriter() throws Exception { -// String tableName = "very_long_table_name_with_uuid_" + UUID.randomUUID().toString().replace('-', '_'); -// String tableCreate = "CREATE TABLE \"" + tableName + "\" " + -// " (name String, " + -// " v1 Float32, " + -// " v2 Float32, " + -// " attrs Nullable(String), " + -// " corrected_time DateTime('UTC') DEFAULT now()," + -// " special_attr Nullable(Int8) DEFAULT -1)" + -// " Engine = MergeTree ORDER by ()"; -// -// initTable(tableName, tableCreate); -// -// ZonedDateTime correctedTime = Instant.now().atZone(ZoneId.of("UTC")); -// Object[][] rows = new Object[][] { -// {"foo1", 0.3f, 0.6f, "a=1,b=2,c=5", correctedTime, 10}, -// {"foo2", 0.6f, 0.1f, "a=1,b=2,c=5", correctedTime, null}, -// {"foo3", 0.7f, 0.4f, "a=1,b=2,c=5", null, null}, -// {"foo4", 0.8f, 0.5f, null, null, null}, -// }; -// -// TableSchema schema = client.getTableSchema(tableName); -// -// ClickHouseFormat format = ClickHouseFormat.RowBinaryWithDefaults; -// try (InsertResponse response = client.insert(tableName, out -> { -// RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, schema, format); -// for (Object[] row : rows) { -// for (int i = 0; i < row.length; i++) { -// w.setValue(i + 1, row[i]); -// } -// w.commitRow(); -// } -// }, format, new InsertSettings()).get()) { -// System.out.println("Rows written: " + response.getWrittenRows()); -// } -// -// List records = client.queryAll("SELECT * FROM \"" + tableName + "\"" ); -// -// for (GenericRecord record : records) { -// System.out.println("> " + record.getString(1) + ", " + record.getFloat(2) + ", " + record.getFloat(3)); -// } -// } + public Object getValue() { + if (value == null && defaultValue != null) { + return defaultValue; + } + + return value; + } + } } From 752c3e356bc1c2cef89a7029bede681bde99b9fb Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Thu, 27 Feb 2025 23:45:58 -0500 Subject: [PATCH 03/15] Adjusting toString method and more test cases --- .../datatypes/RowBinaryFormatWriterTest.java | 59 ++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index ad21a0405..1c914c0b0 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -7,6 +7,7 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.command.CommandSettings; import com.clickhouse.client.api.data_formats.RowBinaryFormatWriter; +import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader; import com.clickhouse.client.api.enums.Protocol; import com.clickhouse.client.api.insert.InsertResponse; import com.clickhouse.client.api.insert.InsertSettings; @@ -141,7 +142,7 @@ public void writeNumbersTest() throws Exception { System.out.println("Random seed: " + seed); Field[][] rows = new Field[][] {{ - new Field("id", 1), + new Field("id", 1), //Row ID new Field("int8", rand.nextInt(256) - 128), new Field("int8_nullable"), new Field("int8_default").set(3), //Int8 new Field("int16", rand.nextInt(65536) - 32768), new Field("int16_nullable"), new Field("int16_default").set(3), //Int16 new Field("int32", rand.nextInt()), new Field("int32_nullable"), new Field("int32_default").set(3), //Int32 @@ -161,7 +162,61 @@ public void writeNumbersTest() throws Exception { new Field("decimal64", new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000))), new Field("decimal64_nullable"), new Field("decimal64_default").set("3.000000"), //Decimal64 new Field("decimal128", new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(10000000, 100000000))), new Field("decimal128_nullable"), new Field("decimal128_default").set("3.00000000"), //Decimal128 new Field("decimal256", new BigDecimal(new BigInteger(57, rand) + "." + rand.nextLong(1000000000, 10000000000L))), new Field("decimal256_nullable"), new Field("decimal256_default").set("3.0000000000") //Decimal256 - } + }, { + new Field("id", 2), //Row ID + new Field("int8", rand.nextInt(256) - 128), new Field("int8_nullable"), new Field("int8_default").set(3), //Int8 + new Field("int16", rand.nextInt(65536) - 32768), new Field("int16_nullable"), new Field("int16_default").set(3), //Int16 + new Field("int32", rand.nextInt()), new Field("int32_nullable"), new Field("int32_default").set(3), //Int32 + new Field("int64", rand.nextLong()), new Field("int64_nullable"), new Field("int64_default").set(3), //Int64 + new Field("int128", new BigInteger(127, rand)), new Field("int128_nullable"), new Field("int128_default").set(3), //Int128 + new Field("int256", new BigInteger(255, rand)), new Field("int256_nullable"), new Field("int256_default").set(3), //Int256 + new Field("uint8", rand.nextInt(256)), new Field("uint8_nullable"), new Field("uint8_default").set(3), //UInt8 + new Field("uint16", rand.nextInt(65536)), new Field("uint16_nullable"), new Field("uint16_default").set(3), //UInt16 + new Field("uint32", rand.nextInt() & 0xFFFFFFFFL), new Field("uint32_nullable"), new Field("uint32_default").set(3), //UInt32 + new Field("uint64", BigInteger.valueOf(rand.nextLong(Long.MAX_VALUE))), new Field("uint64_nullable"), new Field("uint64_default").set(3), //UInt64 + new Field("uint128", new BigInteger(128, rand)), new Field("uint128_nullable"), new Field("uint128_default").set(3), //UInt128 + new Field("uint256", new BigInteger(256, rand)), new Field("uint256_nullable"), new Field("uint256_default").set(3), //UInt256 + new Field("float32", rand.nextFloat()), new Field("float32_nullable"), new Field("float32_default").set("3.0"), //Float32 + new Field("float64", rand.nextDouble()), new Field("float64_nullable"), new Field("float64_default").set("3.0"), //Float64 + new Field("decimal", new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(10,100))), new Field("decimal_nullable"), new Field("decimal_default").set("3.00"), //Decimal(4) + new Field("decimal32", new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(1000, 10000))), new Field("decimal32_nullable"), new Field("decimal32_default").set("3.0000"), //Decimal32 + new Field("decimal64", new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000))), new Field("decimal64_nullable"), new Field("decimal64_default").set("3.000000"), //Decimal64 + new Field("decimal128", new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(10000000, 100000000))), new Field("decimal128_nullable"), new Field("decimal128_default").set("3.00000000"), //Decimal128 + new Field("decimal256", new BigDecimal(new BigInteger(57, rand) + "." + rand.nextLong(1000000000, 10000000000L))), new Field("decimal256_nullable"), new Field("decimal256_default").set("3.0000000000") //Decimal256 + }}; + + writeTest(tableName, tableCreate, rows); + } + + + @Test (groups = { "integration" }) + public void writeStringsTest() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeNumbersTest_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " string String, string_nullable Nullable(String), string_default String DEFAULT '3', " + + " fixed_string FixedString(10), fixed_string_nullable Nullable(FixedString(10)), fixed_string_default FixedString(10) DEFAULT 'tenletters', " + + " uuid UUID, uuid_nullable Nullable(UUID), uuid_default UUID DEFAULT '61f0c404-5cb3-11e7-907b-a6006ad3dba0', " + + " enum8 Enum8('a' = 1, 'b' = 2), enum8_nullable Nullable(Enum8('a' = 1, 'b' = 2)), enum8_default Enum8('a' = 1, 'b' = 2) DEFAULT 'a', " + + " enum16 Enum16('a' = 1, 'b' = 2), enum16_nullable Nullable(Enum16('a' = 1, 'b' = 2)), enum16_default Enum16('a' = 1, 'b' = 2) DEFAULT 'a', " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("string", RandomStringUtils.randomAlphabetic(1024)), new Field("string_nullable"), new Field("string_default").set("3"), //String + new Field("fixed_string", RandomStringUtils.randomAlphabetic(10)), new Field("fixed_string_nullable"), new Field("fixed_string_default").set("tenletters"), //FixedString + new Field("uuid", UUID.randomUUID()), new Field("uuid_nullable"), new Field("uuid_default").set("61f0c404-5cb3-11e7-907b-a6006ad3dba0"), //UUID + new Field("enum8", "a"), new Field("enum8_nullable"), new Field("enum8_default").set("a"), //Enum8 + new Field("enum16", "b"), new Field("enum16_nullable"), new Field("enum16_default").set("a") //Enum16 + }, { + new Field("id", 2), //Row ID + new Field("string", RandomStringUtils.randomAlphabetic(1024)), new Field("string_nullable"), new Field("string_default").set("3"), //String + new Field("fixed_string", RandomStringUtils.randomAlphabetic(10)), new Field("fixed_string_nullable"), new Field("fixed_string_default").set("tenletters"), //FixedString + new Field("uuid", UUID.randomUUID()), new Field("uuid_nullable"), new Field("uuid_default").set("61f0c404-5cb3-11e7-907b-a6006ad3dba0"), //UUID + new Field("enum8", "a"), new Field("enum8_nullable"), new Field("enum8_default").set("a"), //Enum8 + new Field("enum16", "b"), new Field("enum16_nullable"), new Field("enum16_default").set("a") //Enum16 + } }; writeTest(tableName, tableCreate, rows); From 5280c6f4ea7a7dd485f9be72ce8cffc93db938f3 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Fri, 28 Feb 2025 00:14:09 -0500 Subject: [PATCH 04/15] Update RowBinaryFormatWriterTest.java --- .../datatypes/RowBinaryFormatWriterTest.java | 82 +++++++++++++------ 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index 1c914c0b0..c7f2175e0 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import java.time.ZonedDateTime; import java.util.List; import java.util.Map; import java.util.Random; @@ -75,6 +76,14 @@ protected void initTable(String tableName, String createTableSQL, CommandSetting } private static void assertEqualsKinda(Object actual, Object expected) { + if (actual instanceof ZonedDateTime) { + actual = ((ZonedDateTime) actual).toInstant().getEpochSecond(); + } + + if (expected instanceof ZonedDateTime) { + expected = ((ZonedDateTime) expected).toInstant().getEpochSecond(); + } + assertEquals(String.valueOf(actual), String.valueOf(expected)); } @@ -102,12 +111,37 @@ private void writeTest(String tableName, String tableCreate, Field[][] rows) thr Map row = record.getValues(); //Validate data for (Field field : rows[id - 1]) { - assertEqualsKinda(row.get(field.name), field.getValue()); + assertEqualsKinda(row.get(field.name), field.comparisonValue); } id++; } } + private static class Field { + String name; + Object value; + Object comparisonValue; + + Field(String name) { + this.name = name; + this.value = null; + this.comparisonValue = null; + } + + Field(String name, Object value) { + this.name = name; + this.value = value; + this.comparisonValue = value; + } + + public Field set(Object comparisonValue) {//For comparison purposes + this.comparisonValue = comparisonValue; + return this; + } + } + + + @Test (groups = { "integration" }) @@ -223,33 +257,27 @@ public void writeStringsTest() throws Exception { } - private static class Field { - String name; - Object value; - Object defaultValue; - - Field(String name) { - this.name = name; - this.value = null; - this.defaultValue = null; - } - - Field(String name, Object value) { - this.name = name; - this.value = value; - } - - public Field set(Object defaultValue) {//For default value for comparison purposes - this.defaultValue = defaultValue; - return this; - } + @Test (groups = { "integration" }) + public void writeDatetimeTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeNumbersTest_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " datetime DateTime, datetime_nullable Nullable(DateTime), datetime_default DateTime DEFAULT '2020-01-01 00:00:00', " + + " datetime64 DateTime64, datetime64_nullable Nullable(DateTime64), datetime64_default DateTime64 DEFAULT '2025-01-01 00:00:00', " + + " date Date, date_nullable Nullable(Date), date_default Date DEFAULT '2020-01-01', " + + " date32 Date32, date32_nullable Nullable(Date32), date32_default Date32 DEFAULT '2025-01-01', " + + " ) Engine = MergeTree ORDER BY id"; - public Object getValue() { - if (value == null && defaultValue != null) { - return defaultValue; - } + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("datetime", ZonedDateTime.now()), new Field("datetime_nullable"), new Field("datetime_default").set(ZonedDateTime.parse("2020-01-01T00:00:00+00:00[UTC]")), //DateTime + new Field("datetime64", ZonedDateTime.now()), new Field("datetime64_nullable"), new Field("datetime64_default").set(ZonedDateTime.parse("2025-01-01T00:00:00+00:00[UTC]")), //DateTime64 + new Field("date", ZonedDateTime.parse("2021-01-01T00:00:00+00:00[UTC]")), new Field("date_nullable"), new Field("date_default").set(ZonedDateTime.parse("2020-01-01T00:00:00+00:00[UTC]").toEpochSecond()), //Date + new Field("date32", ZonedDateTime.parse("2021-01-01T00:00:00+00:00[UTC]")), new Field("date32_nullable"), new Field("date32_default").set(ZonedDateTime.parse("2025-01-01T00:00:00+00:00[UTC]").toEpochSecond()) //Date + } + }; - return value; - } + writeTest(tableName, tableCreate, rows); } } From b69d9e79bde880dc72a472ea0e2b74230c5251bc Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Fri, 28 Feb 2025 00:23:05 -0500 Subject: [PATCH 05/15] Update RowBinaryFormatWriterTest.java --- .../datatypes/RowBinaryFormatWriterTest.java | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index c7f2175e0..054b23158 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -84,6 +84,14 @@ private static void assertEqualsKinda(Object actual, Object expected) { expected = ((ZonedDateTime) expected).toInstant().getEpochSecond(); } + if (actual instanceof Object[]) { + actual = List.of((Object[]) actual); + } + + if (expected instanceof Object[]) { + expected = List.of((Object[]) expected); + } + assertEquals(String.valueOf(actual), String.valueOf(expected)); } @@ -225,7 +233,7 @@ public void writeNumbersTest() throws Exception { @Test (groups = { "integration" }) public void writeStringsTest() throws Exception { - String tableName = "rowBinaryFormatWriterTest_writeNumbersTest_" + UUID.randomUUID().toString().replace('-', '_'); + String tableName = "rowBinaryFormatWriterTest_writeStringsTests_" + UUID.randomUUID().toString().replace('-', '_'); String tableCreate = "CREATE TABLE \"" + tableName + "\" " + " (id Int32, " + " string String, string_nullable Nullable(String), string_default String DEFAULT '3', " + @@ -259,7 +267,7 @@ public void writeStringsTest() throws Exception { @Test (groups = { "integration" }) public void writeDatetimeTests() throws Exception { - String tableName = "rowBinaryFormatWriterTest_writeNumbersTest_" + UUID.randomUUID().toString().replace('-', '_'); + String tableName = "rowBinaryFormatWriterTest_writeDatetimeTests_" + UUID.randomUUID().toString().replace('-', '_'); String tableCreate = "CREATE TABLE \"" + tableName + "\" " + " (id Int32, " + " datetime DateTime, datetime_nullable Nullable(DateTime), datetime_default DateTime DEFAULT '2020-01-01 00:00:00', " + @@ -280,4 +288,22 @@ public void writeDatetimeTests() throws Exception { writeTest(tableName, tableCreate, rows); } + + @Test (groups = { "integration" }) + public void writeTupleTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeTuplesTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " tuple Tuple(Int8, Int16), tuple_default Tuple(Int8, Int16) DEFAULT (3, 4), " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("tuple", List.of((byte) 1, (short) 2)).set(new Object[]{(byte) 1, (short) 2}), new Field("tuple_default").set(List.of((byte) 3, (short) 4)) //Tuple + } + }; + + writeTest(tableName, tableCreate, rows); + } } From 58de2b199ad89dd01a57baa1afead31c16ecd8a5 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Fri, 28 Feb 2025 14:52:37 -0500 Subject: [PATCH 06/15] Update RowBinaryFormatWriterTest.java --- .../datatypes/RowBinaryFormatWriterTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index 054b23158..0bc11c4ef 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -152,6 +152,28 @@ public Field set(Object comparisonValue) {//For comparison purposes + @Test (groups = { "integration" }) + public void writeMissingFieldsTest() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeNumbersTest_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " int8 Int8, int8_nullable Nullable(Int8), int8_default Int8 DEFAULT 3 " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + long seed = System.currentTimeMillis(); + Random rand = new Random(seed); + System.out.println("Random seed: " + seed); + + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("int8", rand.nextInt(256) - 128)//Missing the nullable and default fields + }}; + + writeTest(tableName, tableCreate, rows); + } + + @Test (groups = { "integration" }) public void writeNumbersTest() throws Exception { String tableName = "rowBinaryFormatWriterTest_writeNumbersTest_" + UUID.randomUUID().toString().replace('-', '_'); From da0387aa2ec902d1f54147aec773dfb6d9b6b2f2 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Tue, 8 Apr 2025 16:36:19 -0400 Subject: [PATCH 07/15] Update SerializerUtils.java --- .../client/api/data_formats/internal/SerializerUtils.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index 1e5ae3473..076e2c73a 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -741,8 +741,6 @@ public static BigDecimal convertToBigDecimal(Object value) { return (BigDecimal) value; } else if (value instanceof BigInteger) { return new BigDecimal((BigInteger) value); - } else if (value instanceof BigDecimal) { - return (BigDecimal) value; } else if (value instanceof Number) { return BigDecimal.valueOf(((Number) value).doubleValue()); } else if (value instanceof String) { From 94613eb0362ca185a3edb4ee4db560665d94737d Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Thu, 10 Apr 2025 03:19:08 -0400 Subject: [PATCH 08/15] Adding additional tests and added geo support --- .../internal/BinaryStreamReader.java | 5 +- .../internal/SerializerUtils.java | 10 + .../datatypes/RowBinaryFormatWriterTest.java | 321 ++++++++++++++++++ 3 files changed, 335 insertions(+), 1 deletion(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 5714eb9a1..662dc8eb6 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -205,9 +205,12 @@ public T readValue(ClickHouseColumn column, Class typeHint) throws IOExce return (T) readGeoPolygon(); case MultiPolygon: return (T) readGeoMultiPolygon(); + case MultiLineString: + return (T) readGeoPolygon(); case Ring: return (T) readGeoRing(); - + case LineString: + return (T) readGeoRing(); case JSON: // experimental https://clickhouse.com/docs/en/sql-reference/data-types/newjson if (jsonAsString) { return (T) readString(input); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index 076e2c73a..ce245b654 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -83,6 +83,10 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo value = value instanceof ClickHouseGeoRingValue ? ((ClickHouseGeoRingValue)value).getValue() : value; serializeArrayData(stream, value, GEO_RING_ARRAY); break; + case LineString: + value = value instanceof ClickHouseGeoRingValue ? ((ClickHouseGeoRingValue)value).getValue() : value; + serializeArrayData(stream, value, GEO_LINE_ARRAY); + break; case Polygon: value = value instanceof ClickHouseGeoPolygonValue ? ((ClickHouseGeoPolygonValue)value).getValue() : value; serializeArrayData(stream, value, GEO_POLYGON_ARRAY); @@ -91,6 +95,10 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo value = value instanceof ClickHouseGeoMultiPolygonValue ? ((ClickHouseGeoMultiPolygonValue)value).getValue() : value; serializeArrayData(stream, value, GEO_MULTI_POLYGON_ARRAY); break; + case MultiLineString: + value = value instanceof ClickHouseGeoMultiPolygonValue ? ((ClickHouseGeoMultiPolygonValue)value).getValue() : value; + serializeArrayData(stream, value, GEO_MULTI_LINE_ARRAY); + break; case Dynamic: ClickHouseColumn typeColumn = valueToColumnForDynamicType(value); writeDynamicTypeTag(stream, typeColumn); @@ -678,7 +686,9 @@ private static void serializerVariant(OutputStream out, ClickHouseColumn column, } private static final ClickHouseColumn GEO_POINT_TUPLE = ClickHouseColumn.parse("geopoint Tuple(Float64, Float64)").get(0); + private static final ClickHouseColumn GEO_LINE_ARRAY = ClickHouseColumn.parse("geoline Array(Tuple(Float64, Float64))").get(0); private static final ClickHouseColumn GEO_RING_ARRAY = ClickHouseColumn.parse("georing Array(Tuple(Float64, Float64))").get(0); + private static final ClickHouseColumn GEO_MULTI_LINE_ARRAY = ClickHouseColumn.parse("geomultiline Array(Array(Tuple(Float64, Float64)))").get(0); private static final ClickHouseColumn GEO_POLYGON_ARRAY = ClickHouseColumn.parse("geopolygin Array(Array(Tuple(Float64, Float64)))").get(0); private static final ClickHouseColumn GEO_MULTI_POLYGON_ARRAY = ClickHouseColumn.parse("geomultipolygin Array(Array(Array(Tuple(Float64, Float64))))").get(0); diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index 0bc11c4ef..95c9c673e 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -22,9 +22,16 @@ import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.time.Duration; import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Random; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -92,9 +99,52 @@ private static void assertEqualsKinda(Object actual, Object expected) { expected = List.of((Object[]) expected); } + if (actual instanceof BinaryStreamReader.ArrayValue) { + actual = ((BinaryStreamReader.ArrayValue) actual).asList(); + } + + if (expected instanceof BinaryStreamReader.ArrayValue) { + expected = ((BinaryStreamReader.ArrayValue) expected).asList(); + } + + if (actual instanceof double[]) { + List tempList = new ArrayList<>(); + for (double d : (double[]) actual) { + tempList.add(d); + } + actual = tempList; + } + + if (expected instanceof double[]) { + List tempList = new ArrayList<>(); + for (double d : (double[]) expected) { + tempList.add(d); + } + expected = tempList; + } + + if (actual instanceof List && expected instanceof List) { + compareLists((List) actual, (List) expected); + return; + } + assertEquals(String.valueOf(actual), String.valueOf(expected)); } + private static void compareLists(List list1, List list2) { + // Iterate and compare each element + for (int i = 0; i < list1.size(); i++) { + Object item1 = list1.get(i); + Object item2 = list2.get(i); + if (item1 instanceof List && item2 instanceof List) { + compareLists((List) item1, (List) item2); + } else { + assertEqualsKinda(item1, item2); + } + } + } + + private void writeTest(String tableName, String tableCreate, Field[][] rows) throws Exception { initTable(tableName, tableCreate, new CommandSettings()); TableSchema schema = client.getTableSchema(tableName); @@ -193,6 +243,7 @@ public void writeNumbersTest() throws Exception { " uint256 UInt256, uint256_nullable Nullable(UInt256), uint256_default UInt256 DEFAULT 3, " + " float32 Float32, float32_nullable Nullable(Float32), float32_default Float32 DEFAULT 3, " + " float64 Float64, float64_nullable Nullable(Float64), float64_default Float64 DEFAULT 3, " + +// " bfloat16 BFloat16, bfloat16_nullable Nullable(BFloat16), bfloat16_default BFloat16 DEFAULT 3, " + " decimal Decimal(4, 2), decimal_nullable Nullable(Decimal(4, 2)), decimal_default Decimal(4, 2) DEFAULT 3, " + " decimal32 Decimal(8, 4), decimal32_nullable Nullable(Decimal(8, 4)), decimal32_default Decimal(8, 4) DEFAULT 3, " + " decimal64 Decimal(18, 6), decimal64_nullable Nullable(Decimal(18, 6)), decimal64_default Decimal(18, 6) DEFAULT 3, " + @@ -221,6 +272,7 @@ public void writeNumbersTest() throws Exception { new Field("uint256", new BigInteger(256, rand)), new Field("uint256_nullable"), new Field("uint256_default").set(3), //UInt256 new Field("float32", rand.nextFloat()), new Field("float32_nullable"), new Field("float32_default").set("3.0"), //Float32 new Field("float64", rand.nextDouble()), new Field("float64_nullable"), new Field("float64_default").set("3.0"), //Float64 +// new Field("bfloat16", rand.nextDouble()), new Field("bfloat16_nullable"), new Field("bfloat16_default").set("3.0"), //BFloat16 new Field("decimal", new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(10,100))), new Field("decimal_nullable"), new Field("decimal_default").set("3.00"), //Decimal(4) new Field("decimal32", new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(1000, 10000))), new Field("decimal32_nullable"), new Field("decimal32_default").set("3.0000"), //Decimal32 new Field("decimal64", new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000))), new Field("decimal64_nullable"), new Field("decimal64_default").set("3.000000"), //Decimal64 @@ -242,6 +294,7 @@ public void writeNumbersTest() throws Exception { new Field("uint256", new BigInteger(256, rand)), new Field("uint256_nullable"), new Field("uint256_default").set(3), //UInt256 new Field("float32", rand.nextFloat()), new Field("float32_nullable"), new Field("float32_default").set("3.0"), //Float32 new Field("float64", rand.nextDouble()), new Field("float64_nullable"), new Field("float64_default").set("3.0"), //Float64 +// new Field("bfloat16", rand.nextDouble()), new Field("bfloat16_nullable"), new Field("bfloat16_default").set("3.0"), //BFloat16 new Field("decimal", new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(10,100))), new Field("decimal_nullable"), new Field("decimal_default").set("3.00"), //Decimal(4) new Field("decimal32", new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(1000, 10000))), new Field("decimal32_nullable"), new Field("decimal32_default").set("3.0000"), //Decimal32 new Field("decimal64", new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000))), new Field("decimal64_nullable"), new Field("decimal64_default").set("3.000000"), //Decimal64 @@ -261,6 +314,7 @@ public void writeStringsTest() throws Exception { " string String, string_nullable Nullable(String), string_default String DEFAULT '3', " + " fixed_string FixedString(10), fixed_string_nullable Nullable(FixedString(10)), fixed_string_default FixedString(10) DEFAULT 'tenletters', " + " uuid UUID, uuid_nullable Nullable(UUID), uuid_default UUID DEFAULT '61f0c404-5cb3-11e7-907b-a6006ad3dba0', " + + " enum Enum('a' = 1, 'b' = 2), enum_nullable Nullable(Enum('a' = 1, 'b' = 2)), enum_default Enum('a' = 1, 'b' = 2) DEFAULT 'a', " + " enum8 Enum8('a' = 1, 'b' = 2), enum8_nullable Nullable(Enum8('a' = 1, 'b' = 2)), enum8_default Enum8('a' = 1, 'b' = 2) DEFAULT 'a', " + " enum16 Enum16('a' = 1, 'b' = 2), enum16_nullable Nullable(Enum16('a' = 1, 'b' = 2)), enum16_default Enum16('a' = 1, 'b' = 2) DEFAULT 'a', " + " ) Engine = MergeTree ORDER BY id"; @@ -271,6 +325,7 @@ public void writeStringsTest() throws Exception { new Field("string", RandomStringUtils.randomAlphabetic(1024)), new Field("string_nullable"), new Field("string_default").set("3"), //String new Field("fixed_string", RandomStringUtils.randomAlphabetic(10)), new Field("fixed_string_nullable"), new Field("fixed_string_default").set("tenletters"), //FixedString new Field("uuid", UUID.randomUUID()), new Field("uuid_nullable"), new Field("uuid_default").set("61f0c404-5cb3-11e7-907b-a6006ad3dba0"), //UUID + new Field("enum", "a"), new Field("enum_nullable"), new Field("enum_default").set("a"), //Enum8 new Field("enum8", "a"), new Field("enum8_nullable"), new Field("enum8_default").set("a"), //Enum8 new Field("enum16", "b"), new Field("enum16_nullable"), new Field("enum16_default").set("a") //Enum16 }, { @@ -278,6 +333,7 @@ public void writeStringsTest() throws Exception { new Field("string", RandomStringUtils.randomAlphabetic(1024)), new Field("string_nullable"), new Field("string_default").set("3"), //String new Field("fixed_string", RandomStringUtils.randomAlphabetic(10)), new Field("fixed_string_nullable"), new Field("fixed_string_default").set("tenletters"), //FixedString new Field("uuid", UUID.randomUUID()), new Field("uuid_nullable"), new Field("uuid_default").set("61f0c404-5cb3-11e7-907b-a6006ad3dba0"), //UUID + new Field("enum", "a"), new Field("enum_nullable"), new Field("enum_default").set("a"), //Enum8 new Field("enum8", "a"), new Field("enum8_nullable"), new Field("enum8_default").set("a"), //Enum8 new Field("enum16", "b"), new Field("enum16_nullable"), new Field("enum16_default").set("a") //Enum16 } @@ -293,6 +349,7 @@ public void writeDatetimeTests() throws Exception { String tableCreate = "CREATE TABLE \"" + tableName + "\" " + " (id Int32, " + " datetime DateTime, datetime_nullable Nullable(DateTime), datetime_default DateTime DEFAULT '2020-01-01 00:00:00', " + + " datetime32 DateTime32, datetime32_nullable Nullable(DateTime32), datetime32_default DateTime32 DEFAULT '2020-01-01 00:00:00', " + " datetime64 DateTime64, datetime64_nullable Nullable(DateTime64), datetime64_default DateTime64 DEFAULT '2025-01-01 00:00:00', " + " date Date, date_nullable Nullable(Date), date_default Date DEFAULT '2020-01-01', " + " date32 Date32, date32_nullable Nullable(Date32), date32_default Date32 DEFAULT '2025-01-01', " + @@ -302,6 +359,7 @@ public void writeDatetimeTests() throws Exception { Field[][] rows = new Field[][] {{ new Field("id", 1), //Row ID new Field("datetime", ZonedDateTime.now()), new Field("datetime_nullable"), new Field("datetime_default").set(ZonedDateTime.parse("2020-01-01T00:00:00+00:00[UTC]")), //DateTime + new Field("datetime32", ZonedDateTime.now()), new Field("datetime32_nullable"), new Field("datetime32_default").set(ZonedDateTime.parse("2020-01-01T00:00:00+00:00[UTC]")), //DateTime new Field("datetime64", ZonedDateTime.now()), new Field("datetime64_nullable"), new Field("datetime64_default").set(ZonedDateTime.parse("2025-01-01T00:00:00+00:00[UTC]")), //DateTime64 new Field("date", ZonedDateTime.parse("2021-01-01T00:00:00+00:00[UTC]")), new Field("date_nullable"), new Field("date_default").set(ZonedDateTime.parse("2020-01-01T00:00:00+00:00[UTC]").toEpochSecond()), //Date new Field("date32", ZonedDateTime.parse("2021-01-01T00:00:00+00:00[UTC]")), new Field("date32_nullable"), new Field("date32_default").set(ZonedDateTime.parse("2025-01-01T00:00:00+00:00[UTC]").toEpochSecond()) //Date @@ -328,4 +386,267 @@ public void writeTupleTests() throws Exception { writeTest(tableName, tableCreate, rows); } + + @Test (groups = { "integration" }) + public void writeIpAddressTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeIpAddressTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " ipv4 IPv4, ipv4_nullable Nullable(IPv4), ipv4_default IPv4 DEFAULT '127.0.0.1', " + + " ipv6 IPv6, ipv6_nullable Nullable(IPv6), ipv6_default IPv6 DEFAULT '::1', " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("ipv4", Inet4Address.getByName("127.0.0.1")).set(Inet4Address.getByName("127.0.0.1")), new Field("ipv4_nullable"), new Field("ipv4_default").set(Inet4Address.getByName("127.0.0.1")), + new Field("ipv6", Inet6Address.getByName("::1")).set(Inet6Address.getByName("::1")), new Field("ipv6_nullable"), new Field("ipv6_default").set(Inet6Address.getByName("::1")) + }}; + + writeTest(tableName, tableCreate, rows); + } + + @Test (groups = { "integration" }) + public void writeArrayTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeArrayTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " array Array(Int8), array_default Array(Int8) DEFAULT [3], " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("array", List.of((byte) 1, (byte) 2)).set(new Object[]{(byte) 1, (byte) 2}), new Field("array_default").set(List.of((byte) 3)) //Array + } + }; + + writeTest(tableName, tableCreate, rows); + } + + @Test (groups = { "integration" }) + public void writeGeometryTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeGeometryTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " point Point, point_default Point DEFAULT (0,0), " + + " ring Ring, ring_default Ring DEFAULT [(0, 0), (10, 0), (10, 10), (0, 10)], " + + " linestring LineString, linestring_default LineString DEFAULT [(0, 0), (10, 0), (10, 10), (0, 10)], " + + " multilinestring MultiLineString, multilinestring_default MultiLineString DEFAULT [[(0, 0), (10, 0), (10, 10), (0, 10)]], " + + " polygon Polygon, polygon_default Polygon DEFAULT [[(0, 0), (10, 0), (10, 10), (0, 10)]], " + + " multipolygon MultiPolygon, multipolygon_default MultiPolygon DEFAULT [[[(0, 0), (10, 0), (10, 10), (0, 10)]]], " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("point", List.of(1.0, 2.0)).set(List.of(1.0, 2.0)), new Field("point_default").set(List.of(0.0, 0.0)), + new Field("ring", List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))).set(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))), new Field("ring_default").set(List.of(List.of(0.0, 0.0), List.of(10.0, 0.0), List.of(10.0, 10.0), List.of(0.0, 10.0))), + new Field("linestring", List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))).set(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))), new Field("linestring_default").set(List.of(List.of(0.0, 0.0), List.of(10.0, 0.0), List.of(10.0, 10.0), List.of(0.0, 10.0))), + new Field("polygon", List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0)))).set(List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0)))) , new Field("polygon_default").set(List.of(List.of(List.of(0.0, 0.0), List.of(10.0, 0.0), List.of(10.0, 10.0), List.of(0.0, 10.0)))) , + new Field("multilinestring", List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0)))).set(List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0)))) , new Field("multilinestring_default").set(List.of(List.of(List.of(0.0, 0.0), List.of(10.0, 0.0), List.of(10.0, 10.0), List.of(0.0, 10.0)))) , + new Field("multipolygon", List.of(List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))))).set(List.of(List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))))) , new Field("multipolygon_default").set(List.of(List.of(List.of(List.of(0.0, 0.0), List.of(10.0, 0.0), List.of(10.0, 10.0), List.of(0.0, 10.0))))), + }}; + + writeTest(tableName, tableCreate, rows); + } + + + + + @Test (groups = { "integration" }) + public void writeMapTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeMapTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " map Map(String, Int16) " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("map", Map.of((byte) 1, (short) 2)).set(Map.of((byte) 1, (short) 2)), //Map + } + }; + + writeTest(tableName, tableCreate, rows); + } + + @Test (groups = { "integration" }) + public void writeNestedTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeNestedTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " nested Nested(n1 Int8, n2 Int16) " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("nested.n1", List.of(1)).set(List.of(1)), //Nested + new Field("nested.n2", List.of(2)).set(List.of(2)), //Nested + } + }; + + writeTest(tableName, tableCreate, rows); + } + + @Test (groups = { "integration" }) + public void writeNullableTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeNullableTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " nullable Nullable(Int8), nullable_default Nullable(Int8) DEFAULT 3, " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("nullable", null).set(null), new Field("nullable_default").set(3) //Nullable + } + }; + + writeTest(tableName, tableCreate, rows); + } + + @Test (groups = { "integration" }) + public void writeLowCardinalityTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeLowCardinalityTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " lowcardinality LowCardinality(String), lowcardinality_default LowCardinality(String) DEFAULT '3', " + + " ) Engine = MergeTree ORDER BY id"; + + String randomString = RandomStringUtils.randomAlphabetic(1024); + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("lowcardinality", randomString).set(randomString), new Field("lowcardinality_default").set("3") //LowCardinality + } + }; + + writeTest(tableName, tableCreate, rows); + } + + @Test (groups = { "integration" }) + public void writeBooleanTypeTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeBooleanTypeTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " boolean Bool, boolean_default Bool DEFAULT 1, " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("boolean", false).set(false), new Field("boolean_default").set(true) //Boolean + } + }; + + writeTest(tableName, tableCreate, rows); + } + + + //TODO: Do we support this? + @Test (groups = { "integration" }, enabled = false) + public void writeAggregateFunctionTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeAggregateFunctionTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " aggregate_function AggregateFunction(count, Int8), " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("aggregate_function", List.of((byte) 1)).set(List.of((byte) 1)), //AggregateFunction + } + }; + + writeTest(tableName, tableCreate, rows); + } + + + //TODO: Do we support this? + @Test (groups = { "integration" }, enabled = false) + public void writeSimpleAggregateFunctionTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeSimpleAggregateFunctionTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " simple_aggregate_function SimpleAggregateFunction(count, Int8), " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("simple_aggregate_function", List.of((byte) 1)).set(List.of((byte) 1)), //SimpleAggregateFunction + } + }; + + writeTest(tableName, tableCreate, rows); + } + + + //TODO: Currently experimental + @Test (groups = { "integration" }, enabled = false) + public void writeDynamicTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeDynamicTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " dynamic Dynamic " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("dynamic", List.of((byte) 1, (short) 2)).set(List.of((byte) 1, (short) 2)), //Dynamic + } + }; + + writeTest(tableName, tableCreate, rows); + } + + + + //TODO: Currently experimental + @Test (groups = { "integration" }, enabled = false) + public void writeJsonTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeJsonTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " json JSON, json_default JSON DEFAULT '{\"a\": 1}', " + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("json", "{\"a\": 1}").set("{\"a\": 1}"), new Field("json_default").set("{\"a\": 1}") //Json + } + }; + + writeTest(tableName, tableCreate, rows); + } + + + //TODO: Currently experimental + @Test (groups = { "integration" }, enabled = false) + public void writeVariantTests() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeVariantTests_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " variant Variant(String, Int8)," + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + Field[][] rows = new Field[][] {{ + new Field("id", 1), //Row ID + new Field("variant", (byte) 1).set((byte) 1), //Variant + }, { + new Field("id", 2), //Row ID + new Field("variant", "hello").set("hello"), //Variant + }}; + + writeTest(tableName, tableCreate, rows); + } } From 5e89949a5277238b33b062bf4c1337fa379e89c4 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Thu, 10 Apr 2025 12:33:31 -0400 Subject: [PATCH 09/15] Update RowBinaryFormatWriterTest.java --- .../datatypes/RowBinaryFormatWriterTest.java | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index 95c9c673e..671e8a3d6 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -29,6 +29,8 @@ import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalUnit; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -78,6 +80,10 @@ protected void initTable(String tableName, String createTableSQL, CommandSetting settings = new CommandSettings(); } + settings.serverSetting("allow_experimental_dynamic_type", "1"); + settings.serverSetting("allow_experimental_json_type", "1"); + settings.serverSetting("allow_experimental_variant_type", "1"); + client.execute("DROP TABLE IF EXISTS " + tableName, settings).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS); client.execute(createTableSQL, settings).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS); } @@ -92,11 +98,11 @@ private static void assertEqualsKinda(Object actual, Object expected) { } if (actual instanceof Object[]) { - actual = List.of((Object[]) actual); + actual = Arrays.asList((Object[]) actual); } if (expected instanceof Object[]) { - expected = List.of((Object[]) expected); + expected = Arrays.asList((Object[]) expected); } if (actual instanceof BinaryStreamReader.ArrayValue) { @@ -380,7 +386,7 @@ public void writeTupleTests() throws Exception { // Insert random (valid) values Field[][] rows = new Field[][] {{ new Field("id", 1), //Row ID - new Field("tuple", List.of((byte) 1, (short) 2)).set(new Object[]{(byte) 1, (short) 2}), new Field("tuple_default").set(List.of((byte) 3, (short) 4)) //Tuple + new Field("tuple", Arrays.asList((byte) 1, (short) 2)).set(new Object[]{(byte) 1, (short) 2}), new Field("tuple_default").set(Arrays.asList((byte) 3, (short) 4)) //Tuple } }; @@ -417,7 +423,7 @@ public void writeArrayTests() throws Exception { // Insert random (valid) values Field[][] rows = new Field[][] {{ new Field("id", 1), //Row ID - new Field("array", List.of((byte) 1, (byte) 2)).set(new Object[]{(byte) 1, (byte) 2}), new Field("array_default").set(List.of((byte) 3)) //Array + new Field("array", Arrays.asList((byte) 1, (byte) 2)).set(new Object[]{(byte) 1, (byte) 2}), new Field("array_default").set(Arrays.asList((byte) 3)) //Array } }; @@ -440,12 +446,12 @@ public void writeGeometryTests() throws Exception { // Insert random (valid) values Field[][] rows = new Field[][] {{ new Field("id", 1), //Row ID - new Field("point", List.of(1.0, 2.0)).set(List.of(1.0, 2.0)), new Field("point_default").set(List.of(0.0, 0.0)), - new Field("ring", List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))).set(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))), new Field("ring_default").set(List.of(List.of(0.0, 0.0), List.of(10.0, 0.0), List.of(10.0, 10.0), List.of(0.0, 10.0))), - new Field("linestring", List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))).set(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))), new Field("linestring_default").set(List.of(List.of(0.0, 0.0), List.of(10.0, 0.0), List.of(10.0, 10.0), List.of(0.0, 10.0))), - new Field("polygon", List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0)))).set(List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0)))) , new Field("polygon_default").set(List.of(List.of(List.of(0.0, 0.0), List.of(10.0, 0.0), List.of(10.0, 10.0), List.of(0.0, 10.0)))) , - new Field("multilinestring", List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0)))).set(List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0)))) , new Field("multilinestring_default").set(List.of(List.of(List.of(0.0, 0.0), List.of(10.0, 0.0), List.of(10.0, 10.0), List.of(0.0, 10.0)))) , - new Field("multipolygon", List.of(List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))))).set(List.of(List.of(List.of(List.of(1.0, 2.0), List.of(3.0, 4.0))))) , new Field("multipolygon_default").set(List.of(List.of(List.of(List.of(0.0, 0.0), List.of(10.0, 0.0), List.of(10.0, 10.0), List.of(0.0, 10.0))))), + new Field("point", Arrays.asList(1.0, 2.0)).set(Arrays.asList(1.0, 2.0)), new Field("point_default").set(Arrays.asList(0.0, 0.0)), + new Field("ring", Arrays.asList(Arrays.asList(1.0, 2.0), Arrays.asList(3.0, 4.0))).set(Arrays.asList(Arrays.asList(1.0, 2.0), Arrays.asList(3.0, 4.0))), new Field("ring_default").set(Arrays.asList(Arrays.asList(0.0, 0.0), Arrays.asList(10.0, 0.0), Arrays.asList(10.0, 10.0), Arrays.asList(0.0, 10.0))), + new Field("linestring", Arrays.asList(Arrays.asList(1.0, 2.0), Arrays.asList(3.0, 4.0))).set(Arrays.asList(Arrays.asList(1.0, 2.0), Arrays.asList(3.0, 4.0))), new Field("linestring_default").set(Arrays.asList(Arrays.asList(0.0, 0.0), Arrays.asList(10.0, 0.0), Arrays.asList(10.0, 10.0), Arrays.asList(0.0, 10.0))), + new Field("polygon", Arrays.asList(Arrays.asList(Arrays.asList(1.0, 2.0), Arrays.asList(3.0, 4.0)))).set(Arrays.asList(Arrays.asList(Arrays.asList(1.0, 2.0), Arrays.asList(3.0, 4.0)))) , new Field("polygon_default").set(Arrays.asList(Arrays.asList(Arrays.asList(0.0, 0.0), Arrays.asList(10.0, 0.0), Arrays.asList(10.0, 10.0), Arrays.asList(0.0, 10.0)))) , + new Field("multilinestring", Arrays.asList(Arrays.asList(Arrays.asList(1.0, 2.0), Arrays.asList(3.0, 4.0)))).set(Arrays.asList(Arrays.asList(Arrays.asList(1.0, 2.0), Arrays.asList(3.0, 4.0)))) , new Field("multilinestring_default").set(Arrays.asList(Arrays.asList(Arrays.asList(0.0, 0.0), Arrays.asList(10.0, 0.0), Arrays.asList(10.0, 10.0), Arrays.asList(0.0, 10.0)))) , + new Field("multipolygon", Arrays.asList(Arrays.asList(Arrays.asList(Arrays.asList(1.0, 2.0), Arrays.asList(3.0, 4.0))))).set(Arrays.asList(Arrays.asList(Arrays.asList(Arrays.asList(1.0, 2.0), Arrays.asList(3.0, 4.0))))) , new Field("multipolygon_default").set(Arrays.asList(Arrays.asList(Arrays.asList(Arrays.asList(0.0, 0.0), Arrays.asList(10.0, 0.0), Arrays.asList(10.0, 10.0), Arrays.asList(0.0, 10.0))))), }}; writeTest(tableName, tableCreate, rows); @@ -462,10 +468,14 @@ public void writeMapTests() throws Exception { " map Map(String, Int16) " + " ) Engine = MergeTree ORDER BY id"; + Map tmpMap = new HashMap<>(); + tmpMap.put("a", 1); + tmpMap.put("b", 2); + // Insert random (valid) values Field[][] rows = new Field[][] {{ new Field("id", 1), //Row ID - new Field("map", Map.of((byte) 1, (short) 2)).set(Map.of((byte) 1, (short) 2)), //Map + new Field("map", tmpMap).set(tmpMap), //Map } }; @@ -483,8 +493,8 @@ public void writeNestedTests() throws Exception { // Insert random (valid) values Field[][] rows = new Field[][] {{ new Field("id", 1), //Row ID - new Field("nested.n1", List.of(1)).set(List.of(1)), //Nested - new Field("nested.n2", List.of(2)).set(List.of(2)), //Nested + new Field("nested.n1", Arrays.asList(1)).set(Arrays.asList(1)), //Nested + new Field("nested.n2", Arrays.asList(2)).set(Arrays.asList(2)), //Nested } }; @@ -560,7 +570,7 @@ public void writeAggregateFunctionTests() throws Exception { // Insert random (valid) values Field[][] rows = new Field[][] {{ new Field("id", 1), //Row ID - new Field("aggregate_function", List.of((byte) 1)).set(List.of((byte) 1)), //AggregateFunction + new Field("aggregate_function", Arrays.asList((byte) 1)).set(Arrays.asList((byte) 1)), //AggregateFunction } }; @@ -580,7 +590,7 @@ public void writeSimpleAggregateFunctionTests() throws Exception { // Insert random (valid) values Field[][] rows = new Field[][] {{ new Field("id", 1), //Row ID - new Field("simple_aggregate_function", List.of((byte) 1)).set(List.of((byte) 1)), //SimpleAggregateFunction + new Field("simple_aggregate_function", Arrays.asList((byte) 1)).set(Arrays.asList((byte) 1)), //SimpleAggregateFunction } }; @@ -589,7 +599,7 @@ public void writeSimpleAggregateFunctionTests() throws Exception { //TODO: Currently experimental - @Test (groups = { "integration" }, enabled = false) + @Test (groups = { "integration" }) public void writeDynamicTests() throws Exception { String tableName = "rowBinaryFormatWriterTest_writeDynamicTests_" + UUID.randomUUID().toString().replace('-', '_'); String tableCreate = "CREATE TABLE \"" + tableName + "\" " + @@ -600,7 +610,7 @@ public void writeDynamicTests() throws Exception { // Insert random (valid) values Field[][] rows = new Field[][] {{ new Field("id", 1), //Row ID - new Field("dynamic", List.of((byte) 1, (short) 2)).set(List.of((byte) 1, (short) 2)), //Dynamic + new Field("dynamic", Arrays.asList((byte) 1, (short) 2)).set(Arrays.asList((byte) 1, (short) 2)), //Dynamic } }; @@ -610,7 +620,7 @@ public void writeDynamicTests() throws Exception { //TODO: Currently experimental - @Test (groups = { "integration" }, enabled = false) + @Test (groups = { "integration" }) public void writeJsonTests() throws Exception { String tableName = "rowBinaryFormatWriterTest_writeJsonTests_" + UUID.randomUUID().toString().replace('-', '_'); String tableCreate = "CREATE TABLE \"" + tableName + "\" " + @@ -622,15 +632,14 @@ public void writeJsonTests() throws Exception { Field[][] rows = new Field[][] {{ new Field("id", 1), //Row ID new Field("json", "{\"a\": 1}").set("{\"a\": 1}"), new Field("json_default").set("{\"a\": 1}") //Json - } - }; + }}; writeTest(tableName, tableCreate, rows); } //TODO: Currently experimental - @Test (groups = { "integration" }, enabled = false) + @Test (groups = { "integration" }) public void writeVariantTests() throws Exception { String tableName = "rowBinaryFormatWriterTest_writeVariantTests_" + UUID.randomUUID().toString().replace('-', '_'); String tableCreate = "CREATE TABLE \"" + tableName + "\" " + From 0d2d67ec1627f14e9fd5d101214232a80d600f17 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Thu, 10 Apr 2025 15:42:51 -0400 Subject: [PATCH 10/15] Update RowBinaryFormatWriterTest.java --- .../client/datatypes/RowBinaryFormatWriterTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index 671e8a3d6..4becab9fb 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -68,9 +68,8 @@ protected Client.Builder newClient() { .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isSecure) .setUsername("default") .setPassword(ClickHouseServerForTest.getPassword()) - .compressClientRequest(true) - .useHttpCompression(true) .setDefaultDatabase(ClickHouseServerForTest.getDatabase()) + .serverSetting(ServerSettings.INPUT_FORMAT_BINARY_READ_JSON_AS_STRING, "1") .serverSetting(ServerSettings.ASYNC_INSERT, "0") .serverSetting(ServerSettings.WAIT_END_OF_QUERY, "1"); } @@ -625,13 +624,16 @@ public void writeJsonTests() throws Exception { String tableName = "rowBinaryFormatWriterTest_writeJsonTests_" + UUID.randomUUID().toString().replace('-', '_'); String tableCreate = "CREATE TABLE \"" + tableName + "\" " + " (id Int32, " + - " json JSON, json_default JSON DEFAULT '{\"a\": 1}', " + + " json JSON, json_default JSON DEFAULT '{\"a\": 1}' " + " ) Engine = MergeTree ORDER BY id"; + Map tmpMap = new HashMap<>(); + tmpMap.put("a", 1); + // Insert random (valid) values Field[][] rows = new Field[][] {{ new Field("id", 1), //Row ID - new Field("json", "{\"a\": 1}").set("{\"a\": 1}"), new Field("json_default").set("{\"a\": 1}") //Json + new Field("json", "{\"a\": 1}").set(tmpMap), new Field("json_default").set(tmpMap)//Json }}; writeTest(tableName, tableCreate, rows); From 954c3eac66de4c794a416306f2a4f8dd93445a52 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Thu, 10 Apr 2025 15:58:16 -0400 Subject: [PATCH 11/15] Update RowBinaryFormatWriterTest.java --- .../datatypes/RowBinaryFormatWriterTest.java | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index 4becab9fb..112869d08 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -38,6 +38,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; +import static com.clickhouse.data.ClickHouseDataType.Decimal; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; @@ -133,6 +134,14 @@ private static void assertEqualsKinda(Object actual, Object expected) { return; } + if (actual instanceof BigDecimal) { + actual = ((BigDecimal) actual).stripTrailingZeros(); + } + + if (expected instanceof BigDecimal) { + expected = ((BigDecimal) expected).stripTrailingZeros(); + } + assertEquals(String.valueOf(actual), String.valueOf(expected)); } @@ -249,11 +258,6 @@ public void writeNumbersTest() throws Exception { " float32 Float32, float32_nullable Nullable(Float32), float32_default Float32 DEFAULT 3, " + " float64 Float64, float64_nullable Nullable(Float64), float64_default Float64 DEFAULT 3, " + // " bfloat16 BFloat16, bfloat16_nullable Nullable(BFloat16), bfloat16_default BFloat16 DEFAULT 3, " + - " decimal Decimal(4, 2), decimal_nullable Nullable(Decimal(4, 2)), decimal_default Decimal(4, 2) DEFAULT 3, " + - " decimal32 Decimal(8, 4), decimal32_nullable Nullable(Decimal(8, 4)), decimal32_default Decimal(8, 4) DEFAULT 3, " + - " decimal64 Decimal(18, 6), decimal64_nullable Nullable(Decimal(18, 6)), decimal64_default Decimal(18, 6) DEFAULT 3, " + - " decimal128 Decimal(36, 8), decimal128_nullable Nullable(Decimal(36, 8)), decimal128_default Decimal(36, 8) DEFAULT 3, " + - " decimal256 Decimal(74, 10), decimal256_nullable Nullable(Decimal(74, 10)), decimal256_default Decimal(74, 10) DEFAULT 3" + " ) Engine = MergeTree ORDER BY id"; // Insert random (valid) values @@ -272,17 +276,12 @@ public void writeNumbersTest() throws Exception { new Field("uint8", rand.nextInt(256)), new Field("uint8_nullable"), new Field("uint8_default").set(3), //UInt8 new Field("uint16", rand.nextInt(65536)), new Field("uint16_nullable"), new Field("uint16_default").set(3), //UInt16 new Field("uint32", rand.nextInt() & 0xFFFFFFFFL), new Field("uint32_nullable"), new Field("uint32_default").set(3), //UInt32 - new Field("uint64", BigInteger.valueOf(rand.nextLong(Long.MAX_VALUE))), new Field("uint64_nullable"), new Field("uint64_default").set(3), //UInt64 + new Field("uint64", BigInteger.valueOf(rand.nextLong())), new Field("uint64_nullable"), new Field("uint64_default").set(3), //UInt64 new Field("uint128", new BigInteger(128, rand)), new Field("uint128_nullable"), new Field("uint128_default").set(3), //UInt128 new Field("uint256", new BigInteger(256, rand)), new Field("uint256_nullable"), new Field("uint256_default").set(3), //UInt256 new Field("float32", rand.nextFloat()), new Field("float32_nullable"), new Field("float32_default").set("3.0"), //Float32 new Field("float64", rand.nextDouble()), new Field("float64_nullable"), new Field("float64_default").set("3.0"), //Float64 // new Field("bfloat16", rand.nextDouble()), new Field("bfloat16_nullable"), new Field("bfloat16_default").set("3.0"), //BFloat16 - new Field("decimal", new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(10,100))), new Field("decimal_nullable"), new Field("decimal_default").set("3.00"), //Decimal(4) - new Field("decimal32", new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(1000, 10000))), new Field("decimal32_nullable"), new Field("decimal32_default").set("3.0000"), //Decimal32 - new Field("decimal64", new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000))), new Field("decimal64_nullable"), new Field("decimal64_default").set("3.000000"), //Decimal64 - new Field("decimal128", new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(10000000, 100000000))), new Field("decimal128_nullable"), new Field("decimal128_default").set("3.00000000"), //Decimal128 - new Field("decimal256", new BigDecimal(new BigInteger(57, rand) + "." + rand.nextLong(1000000000, 10000000000L))), new Field("decimal256_nullable"), new Field("decimal256_default").set("3.0000000000") //Decimal256 }, { new Field("id", 2), //Row ID new Field("int8", rand.nextInt(256) - 128), new Field("int8_nullable"), new Field("int8_default").set(3), //Int8 @@ -300,11 +299,42 @@ public void writeNumbersTest() throws Exception { new Field("float32", rand.nextFloat()), new Field("float32_nullable"), new Field("float32_default").set("3.0"), //Float32 new Field("float64", rand.nextDouble()), new Field("float64_nullable"), new Field("float64_default").set("3.0"), //Float64 // new Field("bfloat16", rand.nextDouble()), new Field("bfloat16_nullable"), new Field("bfloat16_default").set("3.0"), //BFloat16 - new Field("decimal", new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(10,100))), new Field("decimal_nullable"), new Field("decimal_default").set("3.00"), //Decimal(4) - new Field("decimal32", new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(1000, 10000))), new Field("decimal32_nullable"), new Field("decimal32_default").set("3.0000"), //Decimal32 - new Field("decimal64", new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000))), new Field("decimal64_nullable"), new Field("decimal64_default").set("3.000000"), //Decimal64 - new Field("decimal128", new BigDecimal(new BigInteger(20, rand) + "." + rand.nextLong(10000000, 100000000))), new Field("decimal128_nullable"), new Field("decimal128_default").set("3.00000000"), //Decimal128 - new Field("decimal256", new BigDecimal(new BigInteger(57, rand) + "." + rand.nextLong(1000000000, 10000000000L))), new Field("decimal256_nullable"), new Field("decimal256_default").set("3.0000000000") //Decimal256 + }}; + + writeTest(tableName, tableCreate, rows); + } + + + @Test (groups = { "integration" }) + public void writeDecimalsTest() throws Exception { + String tableName = "rowBinaryFormatWriterTest_writeNumbersTest_" + UUID.randomUUID().toString().replace('-', '_'); + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (id Int32, " + + " decimal Decimal(4, 2), decimal_nullable Nullable(Decimal(4, 2)), decimal_default Decimal(4, 2) DEFAULT 3, " + + " decimal32 Decimal(8, 4), decimal32_nullable Nullable(Decimal(8, 4)), decimal32_default Decimal(8, 4) DEFAULT 3, " + + " decimal64 Decimal(18, 6), decimal64_nullable Nullable(Decimal(18, 6)), decimal64_default Decimal(18, 6) DEFAULT 3, " + + " decimal128 Decimal(36, 8), decimal128_nullable Nullable(Decimal(36, 8)), decimal128_default Decimal(36, 8) DEFAULT 3, " + + " decimal256 Decimal(74, 10), decimal256_nullable Nullable(Decimal(74, 10)), decimal256_default Decimal(74, 10) DEFAULT 3" + + " ) Engine = MergeTree ORDER BY id"; + + // Insert random (valid) values + long seed = System.currentTimeMillis(); + Random rand = new Random(seed); + System.out.println("Random seed: " + seed); + + BigDecimal decimal = new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(10,100)); + BigDecimal decimal32 = new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(1000, 10000)); + BigDecimal decimal64 = new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000)); + BigDecimal decimal128 = new BigDecimal(new BigInteger(20, rand) + "." + rand.nextInt(100000, 1000000)); + BigDecimal decimal256 = new BigDecimal(new BigInteger(57, rand) + "." + rand.nextInt(100000, 1000000)); + + Field[][] rows = new Field[][] {{ + new Field("id", 1), + new Field("decimal", decimal).set(decimal), new Field("decimal_nullable"), new Field("decimal_default").set("3"), //Decimal(4) + new Field("decimal32", decimal32).set(decimal32), new Field("decimal32_nullable"), new Field("decimal32_default").set("3"), //Decimal32 + new Field("decimal64", decimal64).set(decimal64), new Field("decimal64_nullable"), new Field("decimal64_default").set("3"), //Decimal64 + new Field("decimal128", decimal128).set(decimal128), new Field("decimal128_nullable"), new Field("decimal128_default").set("3"), //Decimal128 + new Field("decimal256", decimal256).set(decimal256), new Field("decimal256_nullable"), new Field("decimal256_default").set("3") //Decimal256 }}; writeTest(tableName, tableCreate, rows); From 4a405f817a4d7c6c1d74c6451724575b9248e5ee Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Thu, 10 Apr 2025 17:35:45 -0400 Subject: [PATCH 12/15] Update RowBinaryFormatWriterTest.java --- .../datatypes/RowBinaryFormatWriterTest.java | 29 ++++--------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index 112869d08..9ab746167 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -276,24 +276,7 @@ public void writeNumbersTest() throws Exception { new Field("uint8", rand.nextInt(256)), new Field("uint8_nullable"), new Field("uint8_default").set(3), //UInt8 new Field("uint16", rand.nextInt(65536)), new Field("uint16_nullable"), new Field("uint16_default").set(3), //UInt16 new Field("uint32", rand.nextInt() & 0xFFFFFFFFL), new Field("uint32_nullable"), new Field("uint32_default").set(3), //UInt32 - new Field("uint64", BigInteger.valueOf(rand.nextLong())), new Field("uint64_nullable"), new Field("uint64_default").set(3), //UInt64 - new Field("uint128", new BigInteger(128, rand)), new Field("uint128_nullable"), new Field("uint128_default").set(3), //UInt128 - new Field("uint256", new BigInteger(256, rand)), new Field("uint256_nullable"), new Field("uint256_default").set(3), //UInt256 - new Field("float32", rand.nextFloat()), new Field("float32_nullable"), new Field("float32_default").set("3.0"), //Float32 - new Field("float64", rand.nextDouble()), new Field("float64_nullable"), new Field("float64_default").set("3.0"), //Float64 -// new Field("bfloat16", rand.nextDouble()), new Field("bfloat16_nullable"), new Field("bfloat16_default").set("3.0"), //BFloat16 - }, { - new Field("id", 2), //Row ID - new Field("int8", rand.nextInt(256) - 128), new Field("int8_nullable"), new Field("int8_default").set(3), //Int8 - new Field("int16", rand.nextInt(65536) - 32768), new Field("int16_nullable"), new Field("int16_default").set(3), //Int16 - new Field("int32", rand.nextInt()), new Field("int32_nullable"), new Field("int32_default").set(3), //Int32 - new Field("int64", rand.nextLong()), new Field("int64_nullable"), new Field("int64_default").set(3), //Int64 - new Field("int128", new BigInteger(127, rand)), new Field("int128_nullable"), new Field("int128_default").set(3), //Int128 - new Field("int256", new BigInteger(255, rand)), new Field("int256_nullable"), new Field("int256_default").set(3), //Int256 - new Field("uint8", rand.nextInt(256)), new Field("uint8_nullable"), new Field("uint8_default").set(3), //UInt8 - new Field("uint16", rand.nextInt(65536)), new Field("uint16_nullable"), new Field("uint16_default").set(3), //UInt16 - new Field("uint32", rand.nextInt() & 0xFFFFFFFFL), new Field("uint32_nullable"), new Field("uint32_default").set(3), //UInt32 - new Field("uint64", BigInteger.valueOf(rand.nextLong(Long.MAX_VALUE))), new Field("uint64_nullable"), new Field("uint64_default").set(3), //UInt64 + new Field("uint64", new BigInteger(64, rand)), new Field("uint64_nullable"), new Field("uint64_default").set(3), //UInt64 new Field("uint128", new BigInteger(128, rand)), new Field("uint128_nullable"), new Field("uint128_default").set(3), //UInt128 new Field("uint256", new BigInteger(256, rand)), new Field("uint256_nullable"), new Field("uint256_default").set(3), //UInt256 new Field("float32", rand.nextFloat()), new Field("float32_nullable"), new Field("float32_default").set("3.0"), //Float32 @@ -322,11 +305,11 @@ public void writeDecimalsTest() throws Exception { Random rand = new Random(seed); System.out.println("Random seed: " + seed); - BigDecimal decimal = new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(10,100)); - BigDecimal decimal32 = new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(1000, 10000)); - BigDecimal decimal64 = new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(100000, 1000000)); - BigDecimal decimal128 = new BigDecimal(new BigInteger(20, rand) + "." + rand.nextInt(100000, 1000000)); - BigDecimal decimal256 = new BigDecimal(new BigInteger(57, rand) + "." + rand.nextInt(100000, 1000000)); + BigDecimal decimal = new BigDecimal(new BigInteger(5, rand) + "." + rand.nextInt(100)); + BigDecimal decimal32 = new BigDecimal(new BigInteger(7, rand) + "." + rand.nextInt(10000)); + BigDecimal decimal64 = new BigDecimal(new BigInteger(18, rand) + "." + rand.nextInt(1000000)); + BigDecimal decimal128 = new BigDecimal(new BigInteger(20, rand) + "." + rand.nextInt(1000000)); + BigDecimal decimal256 = new BigDecimal(new BigInteger(57, rand) + "." + rand.nextInt(1000000)); Field[][] rows = new Field[][] {{ new Field("id", 1), From 0c8f485e1ad90568642f63929faf991f3c0ead88 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Fri, 11 Apr 2025 02:07:03 -0400 Subject: [PATCH 13/15] Adjusting to add version checking --- .../datatypes/RowBinaryFormatWriterTest.java | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index 9ab746167..1d3951c0e 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -15,6 +15,7 @@ import com.clickhouse.client.api.metadata.TableSchema; import com.clickhouse.client.api.query.GenericRecord; import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseVersion; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -65,14 +66,33 @@ public void setUp() throws IOException { protected Client.Builder newClient() { ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); boolean isSecure = isCloud(); - return new Client.Builder() + Client.Builder builder = new Client.Builder() .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isSecure) .setUsername("default") .setPassword(ClickHouseServerForTest.getPassword()) .setDefaultDatabase(ClickHouseServerForTest.getDatabase()) - .serverSetting(ServerSettings.INPUT_FORMAT_BINARY_READ_JSON_AS_STRING, "1") .serverSetting(ServerSettings.ASYNC_INSERT, "0") .serverSetting(ServerSettings.WAIT_END_OF_QUERY, "1"); + + if (isVersionMatch("[24.10,)")) { + builder.serverSetting(ServerSettings.INPUT_FORMAT_BINARY_READ_JSON_AS_STRING, "1"); + } + + return builder; + } + + protected boolean isVersionMatch(String versionExpression) { + ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); + boolean isSecure = isCloud(); + try(Client client = new Client.Builder() + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isSecure) + .setUsername("default") + .setPassword(ClickHouseServerForTest.getPassword()) + .setDefaultDatabase(ClickHouseServerForTest.getDatabase()) + .build()) { + List serverVersion = client.queryAll("SELECT version()"); + return ClickHouseVersion.of(serverVersion.get(0).getString(1)).check(versionExpression); + } } protected void initTable(String tableName, String createTableSQL, CommandSettings settings) throws Exception { @@ -80,9 +100,11 @@ protected void initTable(String tableName, String createTableSQL, CommandSetting settings = new CommandSettings(); } - settings.serverSetting("allow_experimental_dynamic_type", "1"); - settings.serverSetting("allow_experimental_json_type", "1"); - settings.serverSetting("allow_experimental_variant_type", "1"); + if (isVersionMatch("[24.8,)")) { + settings.serverSetting("allow_experimental_variant_type", "1") + .serverSetting("allow_experimental_dynamic_type", "1") + .serverSetting("allow_experimental_json_type", "1"); + } client.execute("DROP TABLE IF EXISTS " + tableName, settings).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS); client.execute(createTableSQL, settings).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS); @@ -634,6 +656,11 @@ public void writeDynamicTests() throws Exception { //TODO: Currently experimental @Test (groups = { "integration" }) public void writeJsonTests() throws Exception { + if (!isVersionMatch("[24.10,)")) { + System.out.println("Skipping test: ClickHouse version is not compatible with JSON type"); + return; + } + String tableName = "rowBinaryFormatWriterTest_writeJsonTests_" + UUID.randomUUID().toString().replace('-', '_'); String tableCreate = "CREATE TABLE \"" + tableName + "\" " + " (id Int32, " + From 7fa62c8eeac88cfadd00b38f7dc5c19467e27a54 Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Fri, 11 Apr 2025 02:26:04 -0400 Subject: [PATCH 14/15] Update RowBinaryFormatWriterTest.java --- .../datatypes/RowBinaryFormatWriterTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index 1d3951c0e..d1c2f489b 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -466,6 +466,11 @@ public void writeArrayTests() throws Exception { @Test (groups = { "integration" }) public void writeGeometryTests() throws Exception { + if (!isVersionMatch("[24.6,)")) { + System.out.println("Skipping test: ClickHouse version is not compatible with LINESTRING type"); + return; + } + String tableName = "rowBinaryFormatWriterTest_writeGeometryTests_" + UUID.randomUUID().toString().replace('-', '_'); String tableCreate = "CREATE TABLE \"" + tableName + "\" " + " (id Int32, " + @@ -635,6 +640,11 @@ public void writeSimpleAggregateFunctionTests() throws Exception { //TODO: Currently experimental @Test (groups = { "integration" }) public void writeDynamicTests() throws Exception { + if (!isVersionMatch("[24.8,)")) { + System.out.println("Skipping test: ClickHouse version is not compatible with DYNAMIC type"); + return; + } + String tableName = "rowBinaryFormatWriterTest_writeDynamicTests_" + UUID.randomUUID().toString().replace('-', '_'); String tableCreate = "CREATE TABLE \"" + tableName + "\" " + " (id Int32, " + @@ -683,6 +693,11 @@ public void writeJsonTests() throws Exception { //TODO: Currently experimental @Test (groups = { "integration" }) public void writeVariantTests() throws Exception { + if (!isVersionMatch("[24.8,)")) { + System.out.println("Skipping test: ClickHouse version is not compatible with VARIANT type"); + return; + } + String tableName = "rowBinaryFormatWriterTest_writeVariantTests_" + UUID.randomUUID().toString().replace('-', '_'); String tableCreate = "CREATE TABLE \"" + tableName + "\" " + " (id Int32, " + From 8136f64974ca7fa8ae9ef46d86a579e69a342ead Mon Sep 17 00:00:00 2001 From: Paultagoras Date: Wed, 16 Apr 2025 00:15:30 -0400 Subject: [PATCH 15/15] Adjusting to reduce lines since we reuse --- .../api/data_formats/internal/SerializerUtils.java | 12 ++---------- .../client/datatypes/RowBinaryFormatWriterTest.java | 7 ------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index ce245b654..4fdade64c 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -80,14 +80,12 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo serializeTupleData(stream, value, GEO_POINT_TUPLE); break; case Ring: - value = value instanceof ClickHouseGeoRingValue ? ((ClickHouseGeoRingValue)value).getValue() : value; - serializeArrayData(stream, value, GEO_RING_ARRAY); - break; case LineString: value = value instanceof ClickHouseGeoRingValue ? ((ClickHouseGeoRingValue)value).getValue() : value; - serializeArrayData(stream, value, GEO_LINE_ARRAY); + serializeArrayData(stream, value, GEO_RING_ARRAY); break; case Polygon: + case MultiLineString: value = value instanceof ClickHouseGeoPolygonValue ? ((ClickHouseGeoPolygonValue)value).getValue() : value; serializeArrayData(stream, value, GEO_POLYGON_ARRAY); break; @@ -95,10 +93,6 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo value = value instanceof ClickHouseGeoMultiPolygonValue ? ((ClickHouseGeoMultiPolygonValue)value).getValue() : value; serializeArrayData(stream, value, GEO_MULTI_POLYGON_ARRAY); break; - case MultiLineString: - value = value instanceof ClickHouseGeoMultiPolygonValue ? ((ClickHouseGeoMultiPolygonValue)value).getValue() : value; - serializeArrayData(stream, value, GEO_MULTI_LINE_ARRAY); - break; case Dynamic: ClickHouseColumn typeColumn = valueToColumnForDynamicType(value); writeDynamicTypeTag(stream, typeColumn); @@ -686,9 +680,7 @@ private static void serializerVariant(OutputStream out, ClickHouseColumn column, } private static final ClickHouseColumn GEO_POINT_TUPLE = ClickHouseColumn.parse("geopoint Tuple(Float64, Float64)").get(0); - private static final ClickHouseColumn GEO_LINE_ARRAY = ClickHouseColumn.parse("geoline Array(Tuple(Float64, Float64))").get(0); private static final ClickHouseColumn GEO_RING_ARRAY = ClickHouseColumn.parse("georing Array(Tuple(Float64, Float64))").get(0); - private static final ClickHouseColumn GEO_MULTI_LINE_ARRAY = ClickHouseColumn.parse("geomultiline Array(Array(Tuple(Float64, Float64)))").get(0); private static final ClickHouseColumn GEO_POLYGON_ARRAY = ClickHouseColumn.parse("geopolygin Array(Array(Tuple(Float64, Float64)))").get(0); private static final ClickHouseColumn GEO_MULTI_POLYGON_ARRAY = ClickHouseColumn.parse("geomultipolygin Array(Array(Array(Tuple(Float64, Float64))))").get(0); diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java index d1c2f489b..ff8ff7532 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java @@ -25,24 +25,17 @@ import java.math.BigInteger; import java.net.Inet4Address; import java.net.Inet6Address; -import java.time.Duration; import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalUnit; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Random; import java.util.UUID; import java.util.concurrent.TimeUnit; -import static com.clickhouse.data.ClickHouseDataType.Decimal; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNull; -import static org.testng.Assert.assertTrue; public class RowBinaryFormatWriterTest extends BaseIntegrationTest { private Client client;