From 87839185a480beb472821a3d76352d4e25ebbed5 Mon Sep 17 00:00:00 2001 From: Felix Mueller Date: Mon, 21 Jul 2025 13:32:20 +0200 Subject: [PATCH 1/5] Instant / DateTime64 via parameter --- .../com/clickhouse/client/api/Client.java | 14 +- .../clickhouse/client/api/DataTypeUtils.java | 118 +++++++ .../RowBinaryFormatSerializer.java | 3 +- .../internal/BinaryStreamReader.java | 6 +- .../internal/SerializerUtils.java | 17 +- .../api/internal/HttpAPIClientHelper.java | 13 +- .../client/api/DataTypeUtilsTests.java | 113 +++++- .../ClickHouseBinaryFormatReaderTest.java | 9 +- .../internal/BinaryStreamReaderTests.java | 156 ++++++++- .../client/e2e/ParameterizedQueryTest.java | 322 ++++++++++++++++++ 10 files changed, 734 insertions(+), 37 deletions(-) create mode 100644 client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 44777c51a..2a1eff3ae 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -7,7 +7,6 @@ import com.clickhouse.client.api.data_formats.RowBinaryFormatReader; import com.clickhouse.client.api.data_formats.RowBinaryWithNamesAndTypesFormatReader; import com.clickhouse.client.api.data_formats.RowBinaryWithNamesFormatReader; -import com.clickhouse.client.api.data_formats.internal.AbstractBinaryFormatReader; import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader; import com.clickhouse.client.api.data_formats.internal.MapBackedRecord; import com.clickhouse.client.api.data_formats.internal.ProcessParser; @@ -37,7 +36,6 @@ import com.clickhouse.client.api.transport.Endpoint; import com.clickhouse.client.api.transport.HttpEndpoint; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.config.ClickHouseOption; import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseFormat; @@ -1575,7 +1573,9 @@ public CompletableFuture query(String sqlQuery, Map responseSupplier; if (queryParams != null) { - settings.setOption("statement_params", queryParams); + settings.setOption( + HttpAPIClientHelper.KEY_STATEMENT_PARAMS, + formatQueryParameters(queryParams)); } final QuerySettings finalSettings = new QuerySettings(buildRequestSettings(settings.getAllSettings())); responseSupplier = () -> { @@ -2100,4 +2100,12 @@ private Map buildRequestSettings(Map opSettings) requestSettings.putAll(opSettings); return requestSettings; } + + private Map formatQueryParameters(Map queryParams) { + HashMap newMap = new HashMap<>(queryParams.size()); + for (String key : queryParams.keySet()) { + newMap.put(key, DataTypeUtils.format(queryParams.get(key))); + } + return newMap; + } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java index 49e8b30a5..ae82493c3 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java @@ -1,6 +1,12 @@ package com.clickhouse.client.api; +import java.time.Instant; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.Objects; + +import com.clickhouse.data.ClickHouseDataType; public class DataTypeUtils { @@ -19,4 +25,116 @@ public class DataTypeUtils { */ public static DateTimeFormatter DATETIME_WITH_NANOS_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.nnnnnnnnn"); + /** + * Formats a Java object for use in SQL statements or as query parameter. + * + * Note, that this method returns the empty {@link java.lang.String} + * "" for {@code null} objects. + * + * @param object + * the Java object to format + * @return a suitable String representation of {@code object}, or the empty + * String for {@code null} objects + */ + public static String format(Object object) { + return format(object, null); + } + + /** + * Formats a Java object for use in SQL statements or as query parameter. + * + * This method uses the {@code dataTypeHint} parameter to find be best + * suitable format for the object. + * + * Note, that this method returns the empty {@link java.lang.String} + * "" for {@code null} objects. This might not be the correct + * default value for the {@code dataTypeHint} + * + * @param object + * the Java object to format + * @param dataTypeHint + * the ClickHouse data type {@code object} should be used for + * @return a suitable String representation of {@code object}, or the empty + * String for {@code null} objects + */ + public static String format(Object object, ClickHouseDataType dataTypeHint) { + return format(object, dataTypeHint, null); + } + + /** + * Formats a Java object for use in SQL statements or as query parameter. + * + * This method uses the {@code dataTypeHint} parameter to find be best + * suitable format for the object. + * + * For some formatting operations, providing a {@code timeZone} is + * mandatory: When formatting time-zone-based values (e.g. + * {@link java.time.OffsetDateTime}, {@link java.time.ZonedDateTime}, etc.) + * for use as ClickHouse data types which are not time-zone-based, e.g. + * {@link ClickHouseDataType#Date}, or vice-versa when formatting + * non-time-zone-based Java objects (e.g. {@link java.time.LocalDateTime} or + * any {@link java.util.Calendar} based objects without time-zone) for use + * as time-zone-based ClickHouse data types, e.g. + * {@link ClickHouseDataType#DateTime64}. Although the ClickHouse server + * might understand simple wall-time Strings ("2025-08-20 13:37:42") + * even for those data types, it is preferable to use timestamp values. + * + * Note, that this method returns the empty {@link java.lang.String} + * "" for {@code null} objects. This might not be the correct + * default value for {@code dataTypeHint}. + * + * @param object + * the Java object to format + * @param dataTypeHint + * the ClickHouse data type {@code object} should be used for + * @param timeZone + * the time zone to be used when formatting time-zone-based Java + * objects for use in non-time-zone-based ClickHouse data types + * and vice versa + * @return a suitable String representation of {@code object}, or the empty + * String for {@code null} objects + */ + public static String format(Object object, ClickHouseDataType dataTypeHint, + ZoneId timeZone) + { + if (object == null) { + return ""; + } + if (object instanceof Instant) { + return formatInstant((Instant) object, dataTypeHint, timeZone); + } + return String.valueOf(object); + } + + private static String formatInstant(Instant instant, ClickHouseDataType dataTypeHint, + ZoneId timeZone) + { + if (dataTypeHint == null) { + return formatInstantDefault(instant); + } + switch (dataTypeHint) { + case Date: + case Date32: + Objects.requireNonNull( + timeZone, + "TimeZone required for formatting Instant for '" + dataTypeHint + "' use"); + return DATE_FORMATTER.format( + instant.atZone(timeZone).toLocalDate()); + case DateTime: + case DateTime32: + return String.valueOf(instant.getEpochSecond()); + default: + return formatInstantDefault(instant); + } + } + + private static String formatInstantDefault(Instant instant) { + String nanos = String.valueOf(instant.getNano()); + char[] n = new char[9]; + Arrays.fill(n, '0'); + int nanosLength = Math.min(9, nanos.length()); + nanos.getChars(0, nanosLength, n, 9 - nanosLength); + return String.valueOf(instant.getEpochSecond()) + "." + new String(n); + } + } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatSerializer.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatSerializer.java index 25b170165..59205c3d0 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatSerializer.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatSerializer.java @@ -3,7 +3,6 @@ import com.clickhouse.client.api.data_formats.internal.SerializerUtils; import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; -import com.clickhouse.data.ClickHouseFormat; import com.clickhouse.data.format.BinaryStreamUtils; import java.io.IOException; @@ -132,7 +131,7 @@ public void writeFixedString(String value, int len) throws IOException { } public void writeDate(ZonedDateTime value) throws IOException { - SerializerUtils.writeDate(out, value, ZoneId.of("UTC")); + SerializerUtils.writeDate(out, value, value.getZone()); } public void writeDate32(ZonedDateTime value, ZoneId targetTz) throws IOException { diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 4db8b2b16..6c55808eb 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -104,7 +104,7 @@ public T readValue(ClickHouseColumn column, Class typeHint) throws IOExce if (column.isNullable()) { int isNull = readByteOrEOF(input); if (isNull == 1) { // is Null? - return (T) null; + return null; } } @@ -588,7 +588,7 @@ public static byte[] readNBytesLE(InputStream input, byte[] buffer, int offset, return bytes; } - + /** * Reads a array into an ArrayValue object. * @param column - column information @@ -964,7 +964,7 @@ private ZonedDateTime readDateTime32(TimeZone tz) throws IOException { */ public static ZonedDateTime readDateTime32(InputStream input, byte[] buff, TimeZone tz) throws IOException { long time = readUnsignedIntLE(input, buff); - return LocalDateTime.ofInstant(Instant.ofEpochSecond(Math.max(time, 0L)), tz.toZoneId()).atZone(tz.toZoneId()); + return Instant.ofEpochSecond(Math.max(time, 0L)).atZone(tz.toZoneId()); } /** diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index 916e1294b..e53a0617f 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -17,9 +17,6 @@ import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; import org.objectweb.asm.Type; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.OutputStream; import java.lang.reflect.Array; @@ -29,7 +26,14 @@ import java.net.Inet4Address; import java.net.Inet6Address; import java.sql.Timestamp; -import java.time.*; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.Period; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -54,8 +58,6 @@ public class SerializerUtils { - private static final Logger LOG = LoggerFactory.getLogger(SerializerUtils.class); - public static void serializeData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException { //Serialize the value to the stream based on the data type switch (column.getDataType()) { @@ -1070,6 +1072,9 @@ public static void writeDate(OutputStream output, Object value, ZoneId targetTz) } else if (value instanceof ZonedDateTime) { ZonedDateTime dt = (ZonedDateTime) value; epochDays = (int)dt.withZoneSameInstant(targetTz).toLocalDate().toEpochDay(); + } else if (value instanceof OffsetDateTime) { + OffsetDateTime dt = (OffsetDateTime) value; + epochDays = (int) dt.atZoneSameInstant(targetTz).toLocalDate().toEpochDay(); } else { throw new IllegalArgumentException("Cannot convert " + value + " to Long"); } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 0e5bc8e25..189070bdf 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -98,7 +98,10 @@ import java.util.regex.Pattern; public class HttpAPIClientHelper { - private static final Logger LOG = LoggerFactory.getLogger(Client.class); + + public static final String KEY_STATEMENT_PARAMS = "statement_params"; + + private static final Logger LOG = LoggerFactory.getLogger(HttpAPIClientHelper.class); private static final int ERROR_BODY_BUFFER_SIZE = 1024; // Error messages are usually small @@ -567,11 +570,9 @@ private void addQueryParams(URIBuilder req, Map requestConfig) { if (requestConfig.containsKey(ClientConfigProperties.QUERY_ID.getKey())) { req.addParameter(ClickHouseHttpProto.QPARAM_QUERY_ID, requestConfig.get(ClientConfigProperties.QUERY_ID.getKey()).toString()); } - if (requestConfig.containsKey("statement_params")) { - Map params = (Map) requestConfig.get("statement_params"); - for (Map.Entry entry : params.entrySet()) { - req.addParameter("param_" + entry.getKey(), String.valueOf(entry.getValue())); - } + if (requestConfig.containsKey(KEY_STATEMENT_PARAMS)) { + Map params = (Map) requestConfig.get(KEY_STATEMENT_PARAMS); + params.forEach((k, v) -> req.addParameter("param_" + k, (String.valueOf(v)))); } boolean clientCompression = ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getOrDefault(requestConfig); diff --git a/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java b/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java index 4c5a9e70c..433ab228b 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java @@ -2,31 +2,124 @@ import org.testng.annotations.Test; -import java.time.LocalDateTime; - -import static org.testng.AssertJUnit.assertEquals; +import com.clickhouse.data.ClickHouseDataType; -public class DataTypeUtilsTests { +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.TimeZone; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; +class DataTypeUtilsTests { @Test - public void testDateTimeFormatter() { + void testDateTimeFormatter() { LocalDateTime dateTime = LocalDateTime.of(2021, 12, 31, 23, 59, 59); String formattedDateTime = dateTime.format(DataTypeUtils.DATETIME_FORMATTER); - assertEquals("2021-12-31 23:59:59", formattedDateTime); + assertEquals(formattedDateTime, "2021-12-31 23:59:59"); } @Test - public void testDateFormatter() { + void testDateFormatter() { LocalDateTime date = LocalDateTime.of(2021, 12, 31, 10, 20); String formattedDate = date.toLocalDate().format(DataTypeUtils.DATE_FORMATTER); - assertEquals("2021-12-31", formattedDate); + assertEquals(formattedDate, "2021-12-31"); } @Test - public void testDateTimeWithNanosFormatter() { + void testDateTimeWithNanosFormatter() { LocalDateTime dateTime = LocalDateTime.of(2021, 12, 31, 23, 59, 59, 123456789); String formattedDateTimeWithNanos = dateTime.format(DataTypeUtils.DATETIME_WITH_NANOS_FORMATTER); - assertEquals("2021-12-31 23:59:59.123456789", formattedDateTimeWithNanos); + assertEquals(formattedDateTimeWithNanos, "2021-12-31 23:59:59.123456789"); + } + + @Test + void formatInstantForDateNullInstant() { + assertEquals("", DataTypeUtils.format( + null, ClickHouseDataType.Date, ZoneId.systemDefault())); + } + + @Test + void formatInstantForDateNullTimeZone() { + assertThrows( + NullPointerException.class, + () -> DataTypeUtils.format(Instant.now(), ClickHouseDataType.Date, null)); + } + + @Test + void formatInstantForDate() { + ZoneId tzBER = ZoneId.of("Europe/Berlin"); + ZoneId tzLAX = ZoneId.of("America/Los_Angeles"); + Instant instant = ZonedDateTime.of( + 2025, 7, 20, 5, 5, 42, 0, tzBER).toInstant(); + assertEquals(DataTypeUtils.format(instant, ClickHouseDataType.Date, tzBER), "2025-07-20"); + assertEquals(DataTypeUtils.format(instant, ClickHouseDataType.Date, tzLAX), "2025-07-19"); + } + + @Test + void formatNullValue() { + assertEquals("", DataTypeUtils.format(null)); + } + + @Test + void formatInstantForDateTime() { + TimeZone tzBER = TimeZone.getTimeZone("Europe/Berlin"); + Instant instant = ZonedDateTime.of( + 2025, 7, 20, 5, 5, 42, 232323, tzBER.toZoneId()).toInstant(); + String formatted = DataTypeUtils.format(instant, ClickHouseDataType.DateTime); + assertEquals(formatted, "1752980742"); + assertEquals( + Instant.ofEpochSecond(Long.parseLong(formatted)), + instant.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test + void formatInstantForDateTime64() { + TimeZone tzBER = TimeZone.getTimeZone("Europe/Berlin"); + Instant instant = ZonedDateTime.of( + 2025, 7, 20, 5, 5, 42, 232323232, tzBER.toZoneId()).toInstant(); + String formatted = DataTypeUtils.format(instant); + assertEquals(formatted, "1752980742.232323232"); + String[] formattedParts = formatted.split("\\."); + assertEquals( + Instant + .ofEpochSecond(Long.parseLong(formattedParts[0])) + .plusNanos(Long.parseLong(formattedParts[1])), + instant); } + + @Test + void formatInstantForDateTime64SmallerNanos() { + TimeZone tzBER = TimeZone.getTimeZone("Europe/Berlin"); + Instant instant = ZonedDateTime.of( + 2025, 7, 20, 5, 5, 42, 23, tzBER.toZoneId()).toInstant(); + String formatted = DataTypeUtils.format(instant); + assertEquals(formatted, "1752980742.000000023"); + String[] formattedParts = formatted.split("\\."); + assertEquals( + Instant + .ofEpochSecond(Long.parseLong(formattedParts[0])) + .plusNanos(Long.parseLong(formattedParts[1])), + instant); + } + + @Test + void formatInstantForDateTime64Truncated() { + // precision is constant for Instant + TimeZone tzBER = TimeZone.getTimeZone("Europe/Berlin"); + Instant instant = ZonedDateTime.of( + 2025, 7, 20, 5, 5, 42, 232323232, tzBER.toZoneId()).toInstant(); + assertEquals( + DataTypeUtils.format( + instant.truncatedTo(ChronoUnit.SECONDS)), + "1752980742.000000000"); + assertEquals( + DataTypeUtils.format( + instant.truncatedTo(ChronoUnit.MILLIS)), + "1752980742.232000000"); + } + } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReaderTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReaderTest.java index ccf869358..df87d0325 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReaderTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReaderTest.java @@ -14,7 +14,6 @@ import java.math.BigInteger; import java.util.Arrays; import java.util.TimeZone; -import java.util.function.BiConsumer; import java.util.function.Consumer; public class ClickHouseBinaryFormatReaderTest { @@ -70,11 +69,11 @@ public void testReadingNumbers() throws IOException { Assert.assertEquals(reader.getBoolean(name), Boolean.TRUE); Assert.assertEquals(reader.getByte(name), (byte)testValue); Assert.assertEquals(reader.getShort(name), (short)testValue); - Assert.assertEquals(reader.getInteger(name), (int)testValue); - Assert.assertEquals(reader.getLong(name), (long)testValue); + Assert.assertEquals(reader.getInteger(name), testValue); + Assert.assertEquals(reader.getLong(name), testValue); - Assert.assertEquals(reader.getFloat(name), (float) testValue); - Assert.assertEquals(reader.getDouble(name), (double) testValue); + Assert.assertEquals(reader.getFloat(name), testValue); + Assert.assertEquals(reader.getDouble(name), testValue); Assert.assertEquals(reader.getBigInteger(name), BigInteger.valueOf((testValue))); Assert.assertTrue(reader.getBigDecimal(name).compareTo(BigDecimal.valueOf((testValue))) == 0); diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java index 4fbf1d314..81c1f32c5 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java @@ -1,10 +1,28 @@ package com.clickhouse.client.api.data_formats.internal; -import org.junit.Assert; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.TimeZone; + +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class BinaryStreamReaderTests { + private ZoneId tzLAX; + private ZoneId tzBER; + + @BeforeClass + void beforeClass() { + tzLAX = ZoneId.of("America/Los_Angeles"); + tzBER = ZoneId.of("Europe/Berlin"); + } @Test public void testCachedByteAllocator() { @@ -14,7 +32,7 @@ public void testCachedByteAllocator() { int size = (int) Math.pow(2, i); byte[] firstAllocation = allocator.allocate(size); byte[] nextAllocation = allocator.allocate(size); - Assert.assertSame( "Should be the same buffer for size " + size, firstAllocation, nextAllocation); + Assert.assertTrue(firstAllocation == nextAllocation, "Should be the same buffer for size " + size); } for (int i = 6; i < 16; i++) { @@ -24,4 +42,138 @@ public void testCachedByteAllocator() { Assert.assertNotSame(firstAllocation, nextAllocation); } } + + @Test(dataProvider = "dateTestData") + void readDateZonedDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + /* + * Date is number of days since 1970-01-01 (unsigned) + * ... The date value is stored without the time zone. + */ + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDate(baos, zdt, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDate( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT); + } + + @Test(dataProvider = "dateTestData") + void readDateOffsetDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDate(baos, zdt.toOffsetDateTime(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDate( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), + expectedZDT.toOffsetDateTime()); + } + + @DataProvider(name = "dateTestData") + private Object[][] provideDateTestData() { + ZonedDateTime zdtLAX = ZonedDateTime.of( + 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); + ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); + return new Object[][] { + // no conversion at all + { zdtLAX, tzLAX, tzLAX, zdtLAX.truncatedTo(ChronoUnit.DAYS) }, + + // write using Berlin local date -> next day + { zdtLAX, tzBER, tzBER, zdtLAX.plusDays(1L).withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) }, + + // read using different time zone: local date same as original + { zdtLAX, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) }, + + // write using different time zone: local date same as original + { zdtBER, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) } + }; + + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32ZonedDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32OffsetDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt.toOffsetDateTime(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), + expectedZDT.toOffsetDateTime().truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt.toInstant(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime64Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime64(baos, zdt.toInstant(), 9, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime64( + new ByteArrayInputStream(bytes), + bytes, + 9, + TimeZone.getTimeZone(readTZ)), + expectedZDT); + } + + @DataProvider(name = "dateTimeTestData") + private Object[][] provideDateTimeTestData() { + ZonedDateTime zdtLAX = ZonedDateTime.of( + 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); + ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); + return new Object[][] { + { zdtLAX, tzLAX, tzLAX, zdtLAX }, + { zdtLAX, tzBER, tzLAX, zdtLAX }, + { zdtLAX, tzLAX, tzBER, zdtBER }, + { zdtBER, tzLAX, tzBER, zdtBER } + }; + } + } diff --git a/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java b/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java new file mode 100644 index 000000000..7a5953015 --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2025 Riege Software. All rights reserved. + * Use is subject to license terms. + */ +package com.clickhouse.client.e2e; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import com.clickhouse.client.BaseIntegrationTest; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseServerForTest; +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.DataTypeUtils; +import com.clickhouse.client.api.command.CommandSettings; +import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader; +import com.clickhouse.client.api.enums.Protocol; +import com.clickhouse.client.api.internal.ServerSettings; +import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.data.ClickHouseDataType; + +public class ParameterizedQueryTest extends BaseIntegrationTest { + + private Client client; + + @BeforeMethod(groups = {"integration"}) + public void setUp() { + ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); + client = newClient().build(); + } + + @AfterMethod(groups = {"integration"}) + public void tearDown() { + client.close(); + } + + @Test(groups = {"integration"}) + void testParamsWithInstant() throws Exception { + String tableName = "test_dt64_instant"; + ZoneId tzBER = ZoneId.of("Europe/Berlin"); + ZoneId tzLAX = ZoneId.of("America/Los_Angeles"); + LocalDateTime localDateTime = LocalDateTime.of(2025, 7, 20, 5, 5, 42, 232323232); + List testValues = Arrays.asList( + localDateTime.atZone(tzBER), + localDateTime.atZone(tzLAX)); + + AtomicInteger rowId = new AtomicInteger(-1); + + // Insert two rows via helper method + + prepareDataSet( + tableName, + Arrays.asList( + "id UInt16", + "d Date", + "dt DateTime", + "dt64_3 DateTime64(3)", + "dt64_9 DateTime64(9)", + "dt64_3_lax DateTime64(3, 'America/Los_Angeles')"), + Arrays.asList( + () -> Integer.valueOf(rowId.incrementAndGet() % 2), + () -> testValues.get(rowId.get() % 2).toLocalDate() + .toString(), + () -> DataTypeUtils.format( + testValues.get(rowId.get() % 2).toInstant(), + ClickHouseDataType.DateTime), + () -> DataTypeUtils.format( + testValues.get(rowId.get() % 2).toInstant()), + () -> DataTypeUtils.format( + testValues.get(rowId.get() % 2).toInstant()), + () -> DataTypeUtils.format( + testValues.get(rowId.get() % 2).toInstant())), + 2); + + // Insert one row using query parameters + + ZoneId tzUTC = ZoneId.of("UTC"); + ZoneId tzServer = ZoneId.of(client.getServerTimeZone()); + Instant manualTestValue = localDateTime.atZone(tzUTC).toInstant(); + client.query( + "INSERT INTO " + tableName + " (id, d, dt, dt64_3, dt64_9, dt64_3_lax) " + + "VALUES (" + + rowId.incrementAndGet() + ", " + + "'" + DataTypeUtils.format(manualTestValue, ClickHouseDataType.Date, tzUTC) + "', " + + "'" + DataTypeUtils.format(manualTestValue, ClickHouseDataType.DateTime) + "', " + + "{manualTestValue:DateTime64}, " + + "{manualTestValue:DateTime64(9)}, " + + "{manualTestValue:DateTime64})", + Collections.singletonMap("manualTestValue", manualTestValue)); + + try (QueryResponse response = client.query( + "SELECT * FROM " + tableName + " ORDER by id ASC").get(); + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response)) + { + reader.next(); + Assert.assertEquals( + reader.getLocalDate(2), + localDateTime.toLocalDate()); + Assert.assertEquals( + reader.getZonedDateTime(3), + localDateTime.atZone(tzBER).withZoneSameInstant(tzServer) + .truncatedTo(ChronoUnit.SECONDS)); + Assert.assertEquals( + reader.getZonedDateTime(4), + localDateTime.atZone(tzBER).withZoneSameInstant(tzServer) + .truncatedTo(ChronoUnit.MILLIS)); + Assert.assertEquals( + reader.getZonedDateTime(5), + localDateTime.atZone(tzBER).withZoneSameInstant(tzServer)); + Assert.assertEquals( + reader.getZonedDateTime(6), + localDateTime.atZone(tzBER).withZoneSameInstant(tzLAX) + .truncatedTo(ChronoUnit.MILLIS)); + + reader.next(); + Assert.assertEquals( + reader.getLocalDate(2), + localDateTime.toLocalDate()); + Assert.assertEquals( + reader.getZonedDateTime(3), + localDateTime.atZone(tzLAX).withZoneSameInstant(tzServer) + .truncatedTo(ChronoUnit.SECONDS)); + Assert.assertEquals( + reader.getZonedDateTime(4), + localDateTime.atZone(tzLAX).withZoneSameInstant(tzServer) + .truncatedTo(ChronoUnit.MILLIS)); + Assert.assertEquals( + reader.getZonedDateTime(5), + localDateTime.atZone(tzLAX).withZoneSameInstant(tzServer)); + Assert.assertEquals( + reader.getZonedDateTime(6), + localDateTime.atZone(tzLAX).truncatedTo(ChronoUnit.MILLIS)); + + reader.next(); + Assert.assertEquals( + reader.getLocalDate(2), + localDateTime.toLocalDate()); + Assert.assertEquals( + reader.getZonedDateTime(3), + localDateTime.atZone(tzUTC).truncatedTo(ChronoUnit.SECONDS)); + Assert.assertEquals( + reader.getZonedDateTime(4), + localDateTime.atZone(tzServer).truncatedTo(ChronoUnit.MILLIS)); + Assert.assertEquals( + reader.getZonedDateTime(5), + localDateTime.atZone(tzServer)); + Assert.assertEquals( + reader.getZonedDateTime(6), + localDateTime.atZone(tzServer).withZoneSameInstant(tzLAX) + .truncatedTo(ChronoUnit.MILLIS)); + } + + // test some queries with parameters + + List queryTests = Arrays.asList(new Object[][]{ + // date column + { "d", "=", Instant.now(), -1, new int[0] }, + { "d", "=", testValues.get(1).toInstant(), -1, new int[0] }, + { "d", "=", testValues.get(1).toInstant().truncatedTo(ChronoUnit.DAYS), + -1, new int[] { 0, 1, 2 } }, + { "d", "<", testValues.get(1).toInstant(), + -1, new int[] { 0, 1, 2 } }, + { "d", ">", testValues.get(1).toInstant(), -1, new int[0] }, + + // datetime column + { "dt", "=", Instant.now(), -1, new int[0] }, + { "dt", "=", testValues.get(1).toInstant(), -1, new int[0] }, + { "dt", "=", testValues.get(1).toInstant().truncatedTo(ChronoUnit.SECONDS), + -1, new int[] { 1 } }, + { "dt", "<", testValues.get(1).toInstant(), -1, new int[] { 0, 1, 2 } }, + { "dt", ">", testValues.get(1).toInstant(), -1, new int[0] }, + + // dt63_3 column + { "dt64_3", "=", Instant.now(), -1, new int[0] }, + { "dt64_3", "=", testValues.get(1).toInstant(), -1, new int[]{ 1 } }, + { "dt64_3", "=", testValues.get(1).toInstant().truncatedTo(ChronoUnit.MILLIS), + -1, new int[]{ 1 } }, + { "dt64_3", "<", testValues.get(1).toInstant(), + -1, new int[] { 0, 2 } }, + { "dt64_3", ">", testValues.get(1).toInstant(), -1, new int[0] }, + + // dt63_9 column + { "dt64_9", "=", Instant.now(), 9, new int[0] }, + { "dt64_9", "=", testValues.get(1).toInstant(), 9, + new int[]{ 1 } }, + { "dt64_9", "=", testValues.get(1).toInstant().truncatedTo(ChronoUnit.MILLIS), + 9, new int[0] }, + { "dt64_9", "<", testValues.get(1).toInstant(), 9, new int[] { 0, 2 } }, + { "dt64_9", ">", testValues.get(1).toInstant(), 9, new int[0] }, + + // dt63_3_lax column + { "dt64_3_lax", "=", Instant.now(), -1, new int[0] }, + { "dt64_3_lax", "=", testValues.get(1).toInstant(), -1, + new int[]{ 1 } }, + { "dt64_3_lax", "=", testValues.get(1).toInstant().truncatedTo(ChronoUnit.MILLIS), + -1, new int[]{ 1 } }, + { "dt64_3_lax", "<", testValues.get(1).toInstant(), + -1, new int[] { 0, 2 } }, + { "dt64_3_lax", ">", testValues.get(1).toInstant(), -1, new int[0] }, + + }); + for (Object[] queryTest : queryTests) { + Assert.assertEquals( + queryInstant( + tableName, + (String) queryTest[0], + (String) queryTest[1], + (Instant) queryTest[2], + ((Integer) queryTest[3]).intValue()), + queryTest[4], + "Test: " + (String) queryTest[0] + " " + (String) queryTest[1] + " " + + queryTest[2]); + } + } + + private int[] queryInstant(String tableName, String fieldName, String operator, + Instant param, int scale) throws InterruptedException, ExecutionException, Exception + { + try (QueryResponse response = client.query( + "SELECT id FROM " + tableName + " WHERE " + fieldName + " " + + operator + " {x:DateTime64" + (scale > 0 ? "(" + scale + ")" : "") + "} " + + "ORDER by id ASC", + Collections.singletonMap("x", param)).get(); + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response)) + { + List ints = new ArrayList<>(3); + while (reader.hasNext()) { + reader.next(); + ints.add(Integer.valueOf(reader.getInteger(1))); + } + return ints.stream().mapToInt(Integer::intValue).toArray(); + } + } + + private Client.Builder newClient() { + ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); + boolean isSecure = isCloud(); + return new Client.Builder() + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isSecure) + .setUsername("default") + .setPassword(ClickHouseServerForTest.getPassword()) + .compressClientRequest(false) + .setDefaultDatabase(ClickHouseServerForTest.getDatabase()) + .serverSetting(ServerSettings.WAIT_ASYNC_INSERT, "1") + .serverSetting(ServerSettings.ASYNC_INSERT, "0"); + } + + private void prepareDataSet(String table, List columns, List> valueGenerators, + int rows) + { + List> data = new ArrayList<>(rows); + try { + // Drop table + client.execute("DROP TABLE IF EXISTS " + table).get(10, TimeUnit.SECONDS); + + // Create table + CommandSettings settings = new CommandSettings(); + StringBuilder createStmtBuilder = new StringBuilder(); + createStmtBuilder.append("CREATE TABLE IF NOT EXISTS ").append(table).append(" ("); + for (String column : columns) { + createStmtBuilder.append(column).append(", "); + } + createStmtBuilder.setLength(createStmtBuilder.length() - 2); + createStmtBuilder.append(") ENGINE = MergeTree ORDER BY tuple()"); + client.execute(createStmtBuilder.toString(), settings).get(10, TimeUnit.SECONDS); + + // Insert data + StringBuilder insertStmtBuilder = new StringBuilder(); + insertStmtBuilder.append("INSERT INTO ").append(table).append(" VALUES "); + for (int i = 0; i < rows; i++) { + insertStmtBuilder.append("("); + Map values = writeValuesRow(insertStmtBuilder, columns, valueGenerators); + insertStmtBuilder.setLength(insertStmtBuilder.length() - 2); + insertStmtBuilder.append("), "); + data.add(values); + } + insertStmtBuilder.setLength(insertStmtBuilder.length() - 2); + String s = insertStmtBuilder.toString(); + client.execute(s).get(10, TimeUnit.SECONDS); + } catch (Exception e) { + Assert.fail("failed to prepare data set", e); + } + } + + private Map writeValuesRow(StringBuilder insertStmtBuilder, List columns, + List> valueGenerators) + { + Map values = new HashMap<>(); + Iterator columnIterator = columns.iterator(); + for (Supplier valueGenerator : valueGenerators) { + Object value = valueGenerator.get(); + if (value instanceof String) { + insertStmtBuilder.append('\'').append(value).append('\'').append(", "); + } else { + insertStmtBuilder.append(value).append(", "); + } + values.put(columnIterator.next().split(" ")[0], value); + + } + return values; + } + +} From 038c20285e09d6e4b0f2f7d1bd6c1a47a8815ad0 Mon Sep 17 00:00:00 2001 From: Felix Mueller Date: Tue, 22 Jul 2025 19:11:45 +0200 Subject: [PATCH 2/5] Add tests for unusual String parameters --- .../com/clickhouse/client/api/Client.java | 2 + .../clickhouse/client/api/DataTypeUtils.java | 6 ++- .../client/e2e/ParameterizedQueryTest.java | 54 ++++++++++++++++--- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 2a1eff3ae..0c06dcf64 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -2027,6 +2027,7 @@ protected int getOperationTimeout() { * @return - set of endpoints * @deprecated */ + @Deprecated public Set getEndpoints() { return endpoints.stream().map(Endpoint::getBaseURL).collect(Collectors.toSet()); } @@ -2108,4 +2109,5 @@ private Map formatQueryParameters(Map queryParam } return newMap; } + } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java index ae82493c3..c122ac5b9 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java @@ -103,7 +103,7 @@ public static String format(Object object, ClickHouseDataType dataTypeHint, if (object instanceof Instant) { return formatInstant((Instant) object, dataTypeHint, timeZone); } - return String.valueOf(object); + return escapeString(String.valueOf(object)); } private static String formatInstant(Instant instant, ClickHouseDataType dataTypeHint, @@ -137,4 +137,8 @@ private static String formatInstantDefault(Instant instant) { return String.valueOf(instant.getEpochSecond()) + "." + new String(n); } + private static String escapeString(String x) { + return x.replace("\\", "\\\\").replace("'", "\\'"); + } + } diff --git a/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java b/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java index 7a5953015..faa91dda1 100644 --- a/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java @@ -23,6 +23,7 @@ import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import com.clickhouse.client.BaseIntegrationTest; @@ -232,6 +233,31 @@ void testParamsWithInstant() throws Exception { } } + @Test(dataProvider = "stringParameters") + void testStringParams(String paramValue) throws Exception { + String table = "test_params_unicode"; + String column = "val"; + client.execute("DROP TABLE IF EXISTS " + table).get(); + client.execute("CREATE TABLE " + table + "(" + column + " String) Engine = Memory").get(); + client.query( + "INSERT INTO " + table + "(" + column + ") VALUES ('" + paramValue + "')").get(); + try (QueryResponse r = client.query( + "SELECT " + column + " FROM " + table + " WHERE " + column + "='" + paramValue + "'").get(); + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(r)) + { + reader.next(); + Assert.assertEquals(reader.getString(1), paramValue); + } + try (QueryResponse r = client.query( + "SELECT " + column + " FROM " + table + " WHERE " + column + "={x:String}", + Collections.singletonMap("x", paramValue)).get(); + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(r)) + { + reader.next(); + Assert.assertEquals(reader.getString(1), paramValue); + } + } + private int[] queryInstant(String tableName, String fieldName, String operator, Instant param, int scale) throws InterruptedException, ExecutionException, Exception { @@ -255,13 +281,13 @@ private Client.Builder newClient() { ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); boolean isSecure = isCloud(); return new Client.Builder() - .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isSecure) - .setUsername("default") - .setPassword(ClickHouseServerForTest.getPassword()) - .compressClientRequest(false) - .setDefaultDatabase(ClickHouseServerForTest.getDatabase()) - .serverSetting(ServerSettings.WAIT_ASYNC_INSERT, "1") - .serverSetting(ServerSettings.ASYNC_INSERT, "0"); + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isSecure) + .setUsername("default") + .setPassword(ClickHouseServerForTest.getPassword()) + .compressClientRequest(false) + .setDefaultDatabase(ClickHouseServerForTest.getDatabase()) + .serverSetting(ServerSettings.WAIT_ASYNC_INSERT, "1") + .serverSetting(ServerSettings.ASYNC_INSERT, "0"); } private void prepareDataSet(String table, List columns, List> valueGenerators, @@ -319,4 +345,18 @@ private Map writeValuesRow(StringBuilder insertStmtBuilder, List return values; } + @DataProvider(name = "stringParameters") + private static Object[][] createStringParameterValues() { + return new Object[][] { + { "foo" }, + { "with-dashes" }, + { "☺" }, + { "foo/bar" }, + { "foobar 20" }, + { " leading_and_trailing_spaces " }, + { "multi\nline\r\ndos" }, + { "nicely\"quoted\'string\'" }, + }; + } + } From 96f988391e8f3ef72e699ab552b9f4e016c31e3f Mon Sep 17 00:00:00 2001 From: Felix Mueller Date: Tue, 22 Jul 2025 19:40:13 +0200 Subject: [PATCH 3/5] Disable tests for escaped String parameter values --- .../main/java/com/clickhouse/client/api/DataTypeUtils.java | 6 +----- .../com/clickhouse/client/e2e/ParameterizedQueryTest.java | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java index c122ac5b9..ae82493c3 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java @@ -103,7 +103,7 @@ public static String format(Object object, ClickHouseDataType dataTypeHint, if (object instanceof Instant) { return formatInstant((Instant) object, dataTypeHint, timeZone); } - return escapeString(String.valueOf(object)); + return String.valueOf(object); } private static String formatInstant(Instant instant, ClickHouseDataType dataTypeHint, @@ -137,8 +137,4 @@ private static String formatInstantDefault(Instant instant) { return String.valueOf(instant.getEpochSecond()) + "." + new String(n); } - private static String escapeString(String x) { - return x.replace("\\", "\\\\").replace("'", "\\'"); - } - } diff --git a/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java b/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java index faa91dda1..8ac844f33 100644 --- a/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java @@ -354,8 +354,8 @@ private static Object[][] createStringParameterValues() { { "foo/bar" }, { "foobar 20" }, { " leading_and_trailing_spaces " }, - { "multi\nline\r\ndos" }, - { "nicely\"quoted\'string\'" }, + // { "multi\nline\r\ndos" }, + // { "nicely\"quoted\'string\'" }, }; } From 94a00cc7f55b045ae5106eb42989b8f263d6d714 Mon Sep 17 00:00:00 2001 From: Felix Mueller Date: Wed, 23 Jul 2025 07:09:25 +0200 Subject: [PATCH 4/5] Incorporate review feedback --- .../com/clickhouse/client/api/Client.java | 12 +- .../clickhouse/client/api/DataTypeUtils.java | 104 +++++++----------- .../api/internal/HttpAPIClientHelper.java | 2 +- .../{e2e => }/ParameterizedQueryTest.java | 38 +++---- .../client/api/DataTypeUtilsTests.java | 32 ++++-- 5 files changed, 79 insertions(+), 109 deletions(-) rename client-v2/src/test/java/com/clickhouse/client/{e2e => }/ParameterizedQueryTest.java (93%) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 0c06dcf64..b290c478a 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -1573,9 +1573,7 @@ public CompletableFuture query(String sqlQuery, Map responseSupplier; if (queryParams != null) { - settings.setOption( - HttpAPIClientHelper.KEY_STATEMENT_PARAMS, - formatQueryParameters(queryParams)); + settings.setOption(HttpAPIClientHelper.KEY_STATEMENT_PARAMS, queryParams); } final QuerySettings finalSettings = new QuerySettings(buildRequestSettings(settings.getAllSettings())); responseSupplier = () -> { @@ -2102,12 +2100,4 @@ private Map buildRequestSettings(Map opSettings) return requestSettings; } - private Map formatQueryParameters(Map queryParams) { - HashMap newMap = new HashMap<>(queryParams.size()); - for (String key : queryParams.keySet()) { - newMap.put(key, DataTypeUtils.format(queryParams.get(key))); - } - return newMap; - } - } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java index ae82493c3..5fbf7ca08 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java @@ -3,11 +3,14 @@ import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.Arrays; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; import java.util.Objects; import com.clickhouse.data.ClickHouseDataType; +import static java.time.temporal.ChronoField.NANO_OF_SECOND; + public class DataTypeUtils { /** @@ -25,90 +28,70 @@ public class DataTypeUtils { */ public static DateTimeFormatter DATETIME_WITH_NANOS_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.nnnnnnnnn"); + private static final DateTimeFormatter INSTANT_FORMATTER = new DateTimeFormatterBuilder() + .appendValue(ChronoField.INSTANT_SECONDS) + .appendFraction(NANO_OF_SECOND, 9, 9, true) + .toFormatter(); + /** - * Formats a Java object for use in SQL statements or as query parameter. + * Formats an {@link Instant} object for use in SQL statements or as query + * parameter. * - * Note, that this method returns the empty {@link java.lang.String} - * "" for {@code null} objects. - * - * @param object + * @param instant * the Java object to format - * @return a suitable String representation of {@code object}, or the empty - * String for {@code null} objects + * @return a suitable String representation of {@code instant} + * @throws NullPointerException + * if {@code instant} is null */ - public static String format(Object object) { - return format(object, null); + public static String formatInstant(Instant instant) { + return formatInstant(instant, null); } /** - * Formats a Java object for use in SQL statements or as query parameter. - * - * This method uses the {@code dataTypeHint} parameter to find be best - * suitable format for the object. + * Formats an {@link Instant} object for use in SQL statements or as query + * parameter. * - * Note, that this method returns the empty {@link java.lang.String} - * "" for {@code null} objects. This might not be the correct - * default value for the {@code dataTypeHint} + * This method uses the {@code dataTypeHint} parameter to find the best + * suitable format for the instant. * - * @param object + * @param instant * the Java object to format * @param dataTypeHint - * the ClickHouse data type {@code object} should be used for - * @return a suitable String representation of {@code object}, or the empty - * String for {@code null} objects + * the ClickHouse data type {@code instant} should be used for + * @return a suitable String representation of {@code instant} + * @throws NullPointerException + * if {@code instant} is null */ - public static String format(Object object, ClickHouseDataType dataTypeHint) { - return format(object, dataTypeHint, null); + public static String formatInstant(Instant instant, ClickHouseDataType dataTypeHint) { + return formatInstant(instant, dataTypeHint, null); } /** - * Formats a Java object for use in SQL statements or as query parameter. + * Formats an {@link Instant} object for use in SQL statements or as query + * parameter. * - * This method uses the {@code dataTypeHint} parameter to find be best - * suitable format for the object. + * This method uses the {@code dataTypeHint} parameter to find the best + * suitable format for the instant. * * For some formatting operations, providing a {@code timeZone} is - * mandatory: When formatting time-zone-based values (e.g. - * {@link java.time.OffsetDateTime}, {@link java.time.ZonedDateTime}, etc.) - * for use as ClickHouse data types which are not time-zone-based, e.g. - * {@link ClickHouseDataType#Date}, or vice-versa when formatting - * non-time-zone-based Java objects (e.g. {@link java.time.LocalDateTime} or - * any {@link java.util.Calendar} based objects without time-zone) for use - * as time-zone-based ClickHouse data types, e.g. - * {@link ClickHouseDataType#DateTime64}. Although the ClickHouse server - * might understand simple wall-time Strings ("2025-08-20 13:37:42") - * even for those data types, it is preferable to use timestamp values. - * - * Note, that this method returns the empty {@link java.lang.String} - * "" for {@code null} objects. This might not be the correct - * default value for {@code dataTypeHint}. + * mandatory, e.g. for {@link ClickHouseDataType#Date}. * - * @param object + * @param instant * the Java object to format * @param dataTypeHint * the ClickHouse data type {@code object} should be used for * @param timeZone - * the time zone to be used when formatting time-zone-based Java - * objects for use in non-time-zone-based ClickHouse data types - * and vice versa + * the time zone to be used when formatting the instant for use + * in non-time-zone-based ClickHouse data types * @return a suitable String representation of {@code object}, or the empty * String for {@code null} objects + * @throws NullPointerException + * if {@code instant} is null */ - public static String format(Object object, ClickHouseDataType dataTypeHint, - ZoneId timeZone) - { - if (object == null) { - return ""; - } - if (object instanceof Instant) { - return formatInstant((Instant) object, dataTypeHint, timeZone); - } - return String.valueOf(object); - } - - private static String formatInstant(Instant instant, ClickHouseDataType dataTypeHint, + public static String formatInstant(Instant instant, ClickHouseDataType dataTypeHint, ZoneId timeZone) { + Objects.requireNonNull(instant, "Instant required for formatInstant"); if (dataTypeHint == null) { return formatInstantDefault(instant); } @@ -129,12 +112,7 @@ private static String formatInstant(Instant instant, ClickHouseDataType dataType } private static String formatInstantDefault(Instant instant) { - String nanos = String.valueOf(instant.getNano()); - char[] n = new char[9]; - Arrays.fill(n, '0'); - int nanosLength = Math.min(9, nanos.length()); - nanos.getChars(0, nanosLength, n, 9 - nanosLength); - return String.valueOf(instant.getEpochSecond()) + "." + new String(n); + return INSTANT_FORMATTER.format(instant); } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 189070bdf..f16e514d0 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -572,7 +572,7 @@ private void addQueryParams(URIBuilder req, Map requestConfig) { } if (requestConfig.containsKey(KEY_STATEMENT_PARAMS)) { Map params = (Map) requestConfig.get(KEY_STATEMENT_PARAMS); - params.forEach((k, v) -> req.addParameter("param_" + k, (String.valueOf(v)))); + params.forEach((k, v) -> req.addParameter("param_" + k, String.valueOf(v))); } boolean clientCompression = ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getOrDefault(requestConfig); diff --git a/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java b/client-v2/src/test/java/com/clickhouse/client/ParameterizedQueryTest.java similarity index 93% rename from client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java rename to client-v2/src/test/java/com/clickhouse/client/ParameterizedQueryTest.java index 8ac844f33..29da9c96e 100644 --- a/client-v2/src/test/java/com/clickhouse/client/e2e/ParameterizedQueryTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/ParameterizedQueryTest.java @@ -1,8 +1,4 @@ -/* - * Copyright (c) 2025 Riege Software. All rights reserved. - * Use is subject to license terms. - */ -package com.clickhouse.client.e2e; +package com.clickhouse.client; import java.time.Instant; import java.time.LocalDateTime; @@ -26,10 +22,6 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.clickhouse.client.BaseIntegrationTest; -import com.clickhouse.client.ClickHouseNode; -import com.clickhouse.client.ClickHouseProtocol; -import com.clickhouse.client.ClickHouseServerForTest; import com.clickhouse.client.api.Client; import com.clickhouse.client.api.DataTypeUtils; import com.clickhouse.client.api.command.CommandSettings; @@ -45,7 +37,6 @@ public class ParameterizedQueryTest extends BaseIntegrationTest { @BeforeMethod(groups = {"integration"}) public void setUp() { - ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); client = newClient().build(); } @@ -81,14 +72,14 @@ void testParamsWithInstant() throws Exception { () -> Integer.valueOf(rowId.incrementAndGet() % 2), () -> testValues.get(rowId.get() % 2).toLocalDate() .toString(), - () -> DataTypeUtils.format( + () -> DataTypeUtils.formatInstant( testValues.get(rowId.get() % 2).toInstant(), ClickHouseDataType.DateTime), - () -> DataTypeUtils.format( + () -> DataTypeUtils.formatInstant( testValues.get(rowId.get() % 2).toInstant()), - () -> DataTypeUtils.format( + () -> DataTypeUtils.formatInstant( testValues.get(rowId.get() % 2).toInstant()), - () -> DataTypeUtils.format( + () -> DataTypeUtils.formatInstant( testValues.get(rowId.get() % 2).toInstant())), 2); @@ -101,12 +92,14 @@ void testParamsWithInstant() throws Exception { "INSERT INTO " + tableName + " (id, d, dt, dt64_3, dt64_9, dt64_3_lax) " + "VALUES (" + rowId.incrementAndGet() + ", " - + "'" + DataTypeUtils.format(manualTestValue, ClickHouseDataType.Date, tzUTC) + "', " - + "'" + DataTypeUtils.format(manualTestValue, ClickHouseDataType.DateTime) + "', " + + "'" + DataTypeUtils.formatInstant(manualTestValue, ClickHouseDataType.Date, tzUTC) + "', " + + "'" + DataTypeUtils.formatInstant(manualTestValue, ClickHouseDataType.DateTime) + "', " + "{manualTestValue:DateTime64}, " + "{manualTestValue:DateTime64(9)}, " + "{manualTestValue:DateTime64})", - Collections.singletonMap("manualTestValue", manualTestValue)); + Collections.singletonMap( + "manualTestValue", + DataTypeUtils.formatInstant(manualTestValue))); try (QueryResponse response = client.query( "SELECT * FROM " + tableName + " ORDER by id ASC").get(); @@ -190,7 +183,7 @@ void testParamsWithInstant() throws Exception { { "dt", "<", testValues.get(1).toInstant(), -1, new int[] { 0, 1, 2 } }, { "dt", ">", testValues.get(1).toInstant(), -1, new int[0] }, - // dt63_3 column + // dt64_3 column { "dt64_3", "=", Instant.now(), -1, new int[0] }, { "dt64_3", "=", testValues.get(1).toInstant(), -1, new int[]{ 1 } }, { "dt64_3", "=", testValues.get(1).toInstant().truncatedTo(ChronoUnit.MILLIS), @@ -199,7 +192,7 @@ void testParamsWithInstant() throws Exception { -1, new int[] { 0, 2 } }, { "dt64_3", ">", testValues.get(1).toInstant(), -1, new int[0] }, - // dt63_9 column + // dt64_9 column { "dt64_9", "=", Instant.now(), 9, new int[0] }, { "dt64_9", "=", testValues.get(1).toInstant(), 9, new int[]{ 1 } }, @@ -208,7 +201,7 @@ void testParamsWithInstant() throws Exception { { "dt64_9", "<", testValues.get(1).toInstant(), 9, new int[] { 0, 2 } }, { "dt64_9", ">", testValues.get(1).toInstant(), 9, new int[0] }, - // dt63_3_lax column + // dt64_3_lax column { "dt64_3_lax", "=", Instant.now(), -1, new int[0] }, { "dt64_3_lax", "=", testValues.get(1).toInstant(), -1, new int[]{ 1 } }, @@ -233,7 +226,7 @@ void testParamsWithInstant() throws Exception { } } - @Test(dataProvider = "stringParameters") + @Test(groups = {"integration"}, dataProvider = "stringParameters") void testStringParams(String paramValue) throws Exception { String table = "test_params_unicode"; String column = "val"; @@ -265,7 +258,7 @@ private int[] queryInstant(String tableName, String fieldName, String operator, "SELECT id FROM " + tableName + " WHERE " + fieldName + " " + operator + " {x:DateTime64" + (scale > 0 ? "(" + scale + ")" : "") + "} " + "ORDER by id ASC", - Collections.singletonMap("x", param)).get(); + Collections.singletonMap("x", DataTypeUtils.formatInstant(param))).get(); ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response)) { List ints = new ArrayList<>(3); @@ -354,6 +347,7 @@ private static Object[][] createStringParameterValues() { { "foo/bar" }, { "foobar 20" }, { " leading_and_trailing_spaces " }, + // https://github.com/ClickHouse/ClickHouse/issues/70240 // { "multi\nline\r\ndos" }, // { "nicely\"quoted\'string\'" }, }; diff --git a/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java b/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java index 433ab228b..cfc3efafd 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java @@ -38,15 +38,17 @@ void testDateTimeWithNanosFormatter() { @Test void formatInstantForDateNullInstant() { - assertEquals("", DataTypeUtils.format( - null, ClickHouseDataType.Date, ZoneId.systemDefault())); + assertThrows( + NullPointerException.class, + () -> DataTypeUtils.formatInstant(null, ClickHouseDataType.Date, + ZoneId.systemDefault())); } @Test void formatInstantForDateNullTimeZone() { assertThrows( NullPointerException.class, - () -> DataTypeUtils.format(Instant.now(), ClickHouseDataType.Date, null)); + () -> DataTypeUtils.formatInstant(Instant.now(), ClickHouseDataType.Date, null)); } @Test @@ -55,13 +57,19 @@ void formatInstantForDate() { ZoneId tzLAX = ZoneId.of("America/Los_Angeles"); Instant instant = ZonedDateTime.of( 2025, 7, 20, 5, 5, 42, 0, tzBER).toInstant(); - assertEquals(DataTypeUtils.format(instant, ClickHouseDataType.Date, tzBER), "2025-07-20"); - assertEquals(DataTypeUtils.format(instant, ClickHouseDataType.Date, tzLAX), "2025-07-19"); + assertEquals( + DataTypeUtils.formatInstant(instant, ClickHouseDataType.Date, tzBER), + "2025-07-20"); + assertEquals( + DataTypeUtils.formatInstant(instant, ClickHouseDataType.Date, tzLAX), + "2025-07-19"); } @Test - void formatNullValue() { - assertEquals("", DataTypeUtils.format(null)); + void formatInstantNullValue() { + assertThrows( + NullPointerException.class, + () -> DataTypeUtils.formatInstant(null)); } @Test @@ -69,7 +77,7 @@ void formatInstantForDateTime() { TimeZone tzBER = TimeZone.getTimeZone("Europe/Berlin"); Instant instant = ZonedDateTime.of( 2025, 7, 20, 5, 5, 42, 232323, tzBER.toZoneId()).toInstant(); - String formatted = DataTypeUtils.format(instant, ClickHouseDataType.DateTime); + String formatted = DataTypeUtils.formatInstant(instant, ClickHouseDataType.DateTime); assertEquals(formatted, "1752980742"); assertEquals( Instant.ofEpochSecond(Long.parseLong(formatted)), @@ -81,7 +89,7 @@ void formatInstantForDateTime64() { TimeZone tzBER = TimeZone.getTimeZone("Europe/Berlin"); Instant instant = ZonedDateTime.of( 2025, 7, 20, 5, 5, 42, 232323232, tzBER.toZoneId()).toInstant(); - String formatted = DataTypeUtils.format(instant); + String formatted = DataTypeUtils.formatInstant(instant); assertEquals(formatted, "1752980742.232323232"); String[] formattedParts = formatted.split("\\."); assertEquals( @@ -96,7 +104,7 @@ void formatInstantForDateTime64SmallerNanos() { TimeZone tzBER = TimeZone.getTimeZone("Europe/Berlin"); Instant instant = ZonedDateTime.of( 2025, 7, 20, 5, 5, 42, 23, tzBER.toZoneId()).toInstant(); - String formatted = DataTypeUtils.format(instant); + String formatted = DataTypeUtils.formatInstant(instant); assertEquals(formatted, "1752980742.000000023"); String[] formattedParts = formatted.split("\\."); assertEquals( @@ -113,11 +121,11 @@ void formatInstantForDateTime64Truncated() { Instant instant = ZonedDateTime.of( 2025, 7, 20, 5, 5, 42, 232323232, tzBER.toZoneId()).toInstant(); assertEquals( - DataTypeUtils.format( + DataTypeUtils.formatInstant( instant.truncatedTo(ChronoUnit.SECONDS)), "1752980742.000000000"); assertEquals( - DataTypeUtils.format( + DataTypeUtils.formatInstant( instant.truncatedTo(ChronoUnit.MILLIS)), "1752980742.232000000"); } From 516ffb3d405f8f1c2e768fa7c7aab8104d4c2a0e Mon Sep 17 00:00:00 2001 From: Felix Mueller Date: Wed, 23 Jul 2025 07:35:18 +0200 Subject: [PATCH 5/5] consistency --- .../main/java/com/clickhouse/client/api/DataTypeUtils.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java index 5fbf7ca08..d0a49bc7a 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java @@ -9,8 +9,6 @@ import com.clickhouse.data.ClickHouseDataType; -import static java.time.temporal.ChronoField.NANO_OF_SECOND; - public class DataTypeUtils { /** @@ -30,7 +28,7 @@ public class DataTypeUtils { private static final DateTimeFormatter INSTANT_FORMATTER = new DateTimeFormatterBuilder() .appendValue(ChronoField.INSTANT_SECONDS) - .appendFraction(NANO_OF_SECOND, 9, 9, true) + .appendFraction(ChronoField.NANO_OF_SECOND, 9, 9, true) .toFormatter(); /**