diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 44777c51a..b290c478a 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -7,7 +7,6 @@ import com.clickhouse.client.api.data_formats.RowBinaryFormatReader; import com.clickhouse.client.api.data_formats.RowBinaryWithNamesAndTypesFormatReader; import com.clickhouse.client.api.data_formats.RowBinaryWithNamesFormatReader; -import com.clickhouse.client.api.data_formats.internal.AbstractBinaryFormatReader; import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader; import com.clickhouse.client.api.data_formats.internal.MapBackedRecord; import com.clickhouse.client.api.data_formats.internal.ProcessParser; @@ -37,7 +36,6 @@ import com.clickhouse.client.api.transport.Endpoint; import com.clickhouse.client.api.transport.HttpEndpoint; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.config.ClickHouseOption; import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseFormat; @@ -1575,7 +1573,7 @@ public CompletableFuture query(String sqlQuery, Map responseSupplier; if (queryParams != null) { - settings.setOption("statement_params", queryParams); + settings.setOption(HttpAPIClientHelper.KEY_STATEMENT_PARAMS, queryParams); } final QuerySettings finalSettings = new QuerySettings(buildRequestSettings(settings.getAllSettings())); responseSupplier = () -> { @@ -2027,6 +2025,7 @@ protected int getOperationTimeout() { * @return - set of endpoints * @deprecated */ + @Deprecated public Set getEndpoints() { return endpoints.stream().map(Endpoint::getBaseURL).collect(Collectors.toSet()); } @@ -2100,4 +2099,5 @@ private Map buildRequestSettings(Map opSettings) requestSettings.putAll(opSettings); return requestSettings; } + } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java index 49e8b30a5..d0a49bc7a 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/DataTypeUtils.java @@ -1,6 +1,13 @@ package com.clickhouse.client.api; +import java.time.Instant; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.util.Objects; + +import com.clickhouse.data.ClickHouseDataType; public class DataTypeUtils { @@ -19,4 +26,91 @@ public class DataTypeUtils { */ public static DateTimeFormatter DATETIME_WITH_NANOS_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.nnnnnnnnn"); + private static final DateTimeFormatter INSTANT_FORMATTER = new DateTimeFormatterBuilder() + .appendValue(ChronoField.INSTANT_SECONDS) + .appendFraction(ChronoField.NANO_OF_SECOND, 9, 9, true) + .toFormatter(); + + /** + * Formats an {@link Instant} object for use in SQL statements or as query + * parameter. + * + * @param instant + * the Java object to format + * @return a suitable String representation of {@code instant} + * @throws NullPointerException + * if {@code instant} is null + */ + public static String formatInstant(Instant instant) { + return formatInstant(instant, null); + } + + /** + * Formats an {@link Instant} object for use in SQL statements or as query + * parameter. + * + * This method uses the {@code dataTypeHint} parameter to find the best + * suitable format for the instant. + * + * @param instant + * the Java object to format + * @param dataTypeHint + * the ClickHouse data type {@code instant} should be used for + * @return a suitable String representation of {@code instant} + * @throws NullPointerException + * if {@code instant} is null + */ + public static String formatInstant(Instant instant, ClickHouseDataType dataTypeHint) { + return formatInstant(instant, dataTypeHint, null); + } + + /** + * Formats an {@link Instant} object for use in SQL statements or as query + * parameter. + * + * This method uses the {@code dataTypeHint} parameter to find the best + * suitable format for the instant. + * + * For some formatting operations, providing a {@code timeZone} is + * mandatory, e.g. for {@link ClickHouseDataType#Date}. + * + * @param instant + * the Java object to format + * @param dataTypeHint + * the ClickHouse data type {@code object} should be used for + * @param timeZone + * the time zone to be used when formatting the instant for use + * in non-time-zone-based ClickHouse data types + * @return a suitable String representation of {@code object}, or the empty + * String for {@code null} objects + * @throws NullPointerException + * if {@code instant} is null + */ + public static String formatInstant(Instant instant, ClickHouseDataType dataTypeHint, + ZoneId timeZone) + { + Objects.requireNonNull(instant, "Instant required for formatInstant"); + if (dataTypeHint == null) { + return formatInstantDefault(instant); + } + switch (dataTypeHint) { + case Date: + case Date32: + Objects.requireNonNull( + timeZone, + "TimeZone required for formatting Instant for '" + dataTypeHint + "' use"); + return DATE_FORMATTER.format( + instant.atZone(timeZone).toLocalDate()); + case DateTime: + case DateTime32: + return String.valueOf(instant.getEpochSecond()); + default: + return formatInstantDefault(instant); + } + } + + private static String formatInstantDefault(Instant instant) { + return INSTANT_FORMATTER.format(instant); + } + } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatSerializer.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatSerializer.java index 25b170165..59205c3d0 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatSerializer.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatSerializer.java @@ -3,7 +3,6 @@ import com.clickhouse.client.api.data_formats.internal.SerializerUtils; import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; -import com.clickhouse.data.ClickHouseFormat; import com.clickhouse.data.format.BinaryStreamUtils; import java.io.IOException; @@ -132,7 +131,7 @@ public void writeFixedString(String value, int len) throws IOException { } public void writeDate(ZonedDateTime value) throws IOException { - SerializerUtils.writeDate(out, value, ZoneId.of("UTC")); + SerializerUtils.writeDate(out, value, value.getZone()); } public void writeDate32(ZonedDateTime value, ZoneId targetTz) throws IOException { diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 4db8b2b16..6c55808eb 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -104,7 +104,7 @@ public T readValue(ClickHouseColumn column, Class typeHint) throws IOExce if (column.isNullable()) { int isNull = readByteOrEOF(input); if (isNull == 1) { // is Null? - return (T) null; + return null; } } @@ -588,7 +588,7 @@ public static byte[] readNBytesLE(InputStream input, byte[] buffer, int offset, return bytes; } - + /** * Reads a array into an ArrayValue object. * @param column - column information @@ -964,7 +964,7 @@ private ZonedDateTime readDateTime32(TimeZone tz) throws IOException { */ public static ZonedDateTime readDateTime32(InputStream input, byte[] buff, TimeZone tz) throws IOException { long time = readUnsignedIntLE(input, buff); - return LocalDateTime.ofInstant(Instant.ofEpochSecond(Math.max(time, 0L)), tz.toZoneId()).atZone(tz.toZoneId()); + return Instant.ofEpochSecond(Math.max(time, 0L)).atZone(tz.toZoneId()); } /** diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java index 916e1294b..e53a0617f 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/SerializerUtils.java @@ -17,9 +17,6 @@ import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; import org.objectweb.asm.Type; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.OutputStream; import java.lang.reflect.Array; @@ -29,7 +26,14 @@ import java.net.Inet4Address; import java.net.Inet6Address; import java.sql.Timestamp; -import java.time.*; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.Period; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -54,8 +58,6 @@ public class SerializerUtils { - private static final Logger LOG = LoggerFactory.getLogger(SerializerUtils.class); - public static void serializeData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException { //Serialize the value to the stream based on the data type switch (column.getDataType()) { @@ -1070,6 +1072,9 @@ public static void writeDate(OutputStream output, Object value, ZoneId targetTz) } else if (value instanceof ZonedDateTime) { ZonedDateTime dt = (ZonedDateTime) value; epochDays = (int)dt.withZoneSameInstant(targetTz).toLocalDate().toEpochDay(); + } else if (value instanceof OffsetDateTime) { + OffsetDateTime dt = (OffsetDateTime) value; + epochDays = (int) dt.atZoneSameInstant(targetTz).toLocalDate().toEpochDay(); } else { throw new IllegalArgumentException("Cannot convert " + value + " to Long"); } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 0e5bc8e25..f16e514d0 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -98,7 +98,10 @@ import java.util.regex.Pattern; public class HttpAPIClientHelper { - private static final Logger LOG = LoggerFactory.getLogger(Client.class); + + public static final String KEY_STATEMENT_PARAMS = "statement_params"; + + private static final Logger LOG = LoggerFactory.getLogger(HttpAPIClientHelper.class); private static final int ERROR_BODY_BUFFER_SIZE = 1024; // Error messages are usually small @@ -567,11 +570,9 @@ private void addQueryParams(URIBuilder req, Map requestConfig) { if (requestConfig.containsKey(ClientConfigProperties.QUERY_ID.getKey())) { req.addParameter(ClickHouseHttpProto.QPARAM_QUERY_ID, requestConfig.get(ClientConfigProperties.QUERY_ID.getKey()).toString()); } - if (requestConfig.containsKey("statement_params")) { - Map params = (Map) requestConfig.get("statement_params"); - for (Map.Entry entry : params.entrySet()) { - req.addParameter("param_" + entry.getKey(), String.valueOf(entry.getValue())); - } + if (requestConfig.containsKey(KEY_STATEMENT_PARAMS)) { + Map params = (Map) requestConfig.get(KEY_STATEMENT_PARAMS); + params.forEach((k, v) -> req.addParameter("param_" + k, String.valueOf(v))); } boolean clientCompression = ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getOrDefault(requestConfig); diff --git a/client-v2/src/test/java/com/clickhouse/client/ParameterizedQueryTest.java b/client-v2/src/test/java/com/clickhouse/client/ParameterizedQueryTest.java new file mode 100644 index 000000000..29da9c96e --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/ParameterizedQueryTest.java @@ -0,0 +1,356 @@ +package com.clickhouse.client; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.DataTypeUtils; +import com.clickhouse.client.api.command.CommandSettings; +import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader; +import com.clickhouse.client.api.enums.Protocol; +import com.clickhouse.client.api.internal.ServerSettings; +import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.data.ClickHouseDataType; + +public class ParameterizedQueryTest extends BaseIntegrationTest { + + private Client client; + + @BeforeMethod(groups = {"integration"}) + public void setUp() { + client = newClient().build(); + } + + @AfterMethod(groups = {"integration"}) + public void tearDown() { + client.close(); + } + + @Test(groups = {"integration"}) + void testParamsWithInstant() throws Exception { + String tableName = "test_dt64_instant"; + ZoneId tzBER = ZoneId.of("Europe/Berlin"); + ZoneId tzLAX = ZoneId.of("America/Los_Angeles"); + LocalDateTime localDateTime = LocalDateTime.of(2025, 7, 20, 5, 5, 42, 232323232); + List testValues = Arrays.asList( + localDateTime.atZone(tzBER), + localDateTime.atZone(tzLAX)); + + AtomicInteger rowId = new AtomicInteger(-1); + + // Insert two rows via helper method + + prepareDataSet( + tableName, + Arrays.asList( + "id UInt16", + "d Date", + "dt DateTime", + "dt64_3 DateTime64(3)", + "dt64_9 DateTime64(9)", + "dt64_3_lax DateTime64(3, 'America/Los_Angeles')"), + Arrays.asList( + () -> Integer.valueOf(rowId.incrementAndGet() % 2), + () -> testValues.get(rowId.get() % 2).toLocalDate() + .toString(), + () -> DataTypeUtils.formatInstant( + testValues.get(rowId.get() % 2).toInstant(), + ClickHouseDataType.DateTime), + () -> DataTypeUtils.formatInstant( + testValues.get(rowId.get() % 2).toInstant()), + () -> DataTypeUtils.formatInstant( + testValues.get(rowId.get() % 2).toInstant()), + () -> DataTypeUtils.formatInstant( + testValues.get(rowId.get() % 2).toInstant())), + 2); + + // Insert one row using query parameters + + ZoneId tzUTC = ZoneId.of("UTC"); + ZoneId tzServer = ZoneId.of(client.getServerTimeZone()); + Instant manualTestValue = localDateTime.atZone(tzUTC).toInstant(); + client.query( + "INSERT INTO " + tableName + " (id, d, dt, dt64_3, dt64_9, dt64_3_lax) " + + "VALUES (" + + rowId.incrementAndGet() + ", " + + "'" + DataTypeUtils.formatInstant(manualTestValue, ClickHouseDataType.Date, tzUTC) + "', " + + "'" + DataTypeUtils.formatInstant(manualTestValue, ClickHouseDataType.DateTime) + "', " + + "{manualTestValue:DateTime64}, " + + "{manualTestValue:DateTime64(9)}, " + + "{manualTestValue:DateTime64})", + Collections.singletonMap( + "manualTestValue", + DataTypeUtils.formatInstant(manualTestValue))); + + try (QueryResponse response = client.query( + "SELECT * FROM " + tableName + " ORDER by id ASC").get(); + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response)) + { + reader.next(); + Assert.assertEquals( + reader.getLocalDate(2), + localDateTime.toLocalDate()); + Assert.assertEquals( + reader.getZonedDateTime(3), + localDateTime.atZone(tzBER).withZoneSameInstant(tzServer) + .truncatedTo(ChronoUnit.SECONDS)); + Assert.assertEquals( + reader.getZonedDateTime(4), + localDateTime.atZone(tzBER).withZoneSameInstant(tzServer) + .truncatedTo(ChronoUnit.MILLIS)); + Assert.assertEquals( + reader.getZonedDateTime(5), + localDateTime.atZone(tzBER).withZoneSameInstant(tzServer)); + Assert.assertEquals( + reader.getZonedDateTime(6), + localDateTime.atZone(tzBER).withZoneSameInstant(tzLAX) + .truncatedTo(ChronoUnit.MILLIS)); + + reader.next(); + Assert.assertEquals( + reader.getLocalDate(2), + localDateTime.toLocalDate()); + Assert.assertEquals( + reader.getZonedDateTime(3), + localDateTime.atZone(tzLAX).withZoneSameInstant(tzServer) + .truncatedTo(ChronoUnit.SECONDS)); + Assert.assertEquals( + reader.getZonedDateTime(4), + localDateTime.atZone(tzLAX).withZoneSameInstant(tzServer) + .truncatedTo(ChronoUnit.MILLIS)); + Assert.assertEquals( + reader.getZonedDateTime(5), + localDateTime.atZone(tzLAX).withZoneSameInstant(tzServer)); + Assert.assertEquals( + reader.getZonedDateTime(6), + localDateTime.atZone(tzLAX).truncatedTo(ChronoUnit.MILLIS)); + + reader.next(); + Assert.assertEquals( + reader.getLocalDate(2), + localDateTime.toLocalDate()); + Assert.assertEquals( + reader.getZonedDateTime(3), + localDateTime.atZone(tzUTC).truncatedTo(ChronoUnit.SECONDS)); + Assert.assertEquals( + reader.getZonedDateTime(4), + localDateTime.atZone(tzServer).truncatedTo(ChronoUnit.MILLIS)); + Assert.assertEquals( + reader.getZonedDateTime(5), + localDateTime.atZone(tzServer)); + Assert.assertEquals( + reader.getZonedDateTime(6), + localDateTime.atZone(tzServer).withZoneSameInstant(tzLAX) + .truncatedTo(ChronoUnit.MILLIS)); + } + + // test some queries with parameters + + List queryTests = Arrays.asList(new Object[][]{ + // date column + { "d", "=", Instant.now(), -1, new int[0] }, + { "d", "=", testValues.get(1).toInstant(), -1, new int[0] }, + { "d", "=", testValues.get(1).toInstant().truncatedTo(ChronoUnit.DAYS), + -1, new int[] { 0, 1, 2 } }, + { "d", "<", testValues.get(1).toInstant(), + -1, new int[] { 0, 1, 2 } }, + { "d", ">", testValues.get(1).toInstant(), -1, new int[0] }, + + // datetime column + { "dt", "=", Instant.now(), -1, new int[0] }, + { "dt", "=", testValues.get(1).toInstant(), -1, new int[0] }, + { "dt", "=", testValues.get(1).toInstant().truncatedTo(ChronoUnit.SECONDS), + -1, new int[] { 1 } }, + { "dt", "<", testValues.get(1).toInstant(), -1, new int[] { 0, 1, 2 } }, + { "dt", ">", testValues.get(1).toInstant(), -1, new int[0] }, + + // dt64_3 column + { "dt64_3", "=", Instant.now(), -1, new int[0] }, + { "dt64_3", "=", testValues.get(1).toInstant(), -1, new int[]{ 1 } }, + { "dt64_3", "=", testValues.get(1).toInstant().truncatedTo(ChronoUnit.MILLIS), + -1, new int[]{ 1 } }, + { "dt64_3", "<", testValues.get(1).toInstant(), + -1, new int[] { 0, 2 } }, + { "dt64_3", ">", testValues.get(1).toInstant(), -1, new int[0] }, + + // dt64_9 column + { "dt64_9", "=", Instant.now(), 9, new int[0] }, + { "dt64_9", "=", testValues.get(1).toInstant(), 9, + new int[]{ 1 } }, + { "dt64_9", "=", testValues.get(1).toInstant().truncatedTo(ChronoUnit.MILLIS), + 9, new int[0] }, + { "dt64_9", "<", testValues.get(1).toInstant(), 9, new int[] { 0, 2 } }, + { "dt64_9", ">", testValues.get(1).toInstant(), 9, new int[0] }, + + // dt64_3_lax column + { "dt64_3_lax", "=", Instant.now(), -1, new int[0] }, + { "dt64_3_lax", "=", testValues.get(1).toInstant(), -1, + new int[]{ 1 } }, + { "dt64_3_lax", "=", testValues.get(1).toInstant().truncatedTo(ChronoUnit.MILLIS), + -1, new int[]{ 1 } }, + { "dt64_3_lax", "<", testValues.get(1).toInstant(), + -1, new int[] { 0, 2 } }, + { "dt64_3_lax", ">", testValues.get(1).toInstant(), -1, new int[0] }, + + }); + for (Object[] queryTest : queryTests) { + Assert.assertEquals( + queryInstant( + tableName, + (String) queryTest[0], + (String) queryTest[1], + (Instant) queryTest[2], + ((Integer) queryTest[3]).intValue()), + queryTest[4], + "Test: " + (String) queryTest[0] + " " + (String) queryTest[1] + " " + + queryTest[2]); + } + } + + @Test(groups = {"integration"}, dataProvider = "stringParameters") + void testStringParams(String paramValue) throws Exception { + String table = "test_params_unicode"; + String column = "val"; + client.execute("DROP TABLE IF EXISTS " + table).get(); + client.execute("CREATE TABLE " + table + "(" + column + " String) Engine = Memory").get(); + client.query( + "INSERT INTO " + table + "(" + column + ") VALUES ('" + paramValue + "')").get(); + try (QueryResponse r = client.query( + "SELECT " + column + " FROM " + table + " WHERE " + column + "='" + paramValue + "'").get(); + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(r)) + { + reader.next(); + Assert.assertEquals(reader.getString(1), paramValue); + } + try (QueryResponse r = client.query( + "SELECT " + column + " FROM " + table + " WHERE " + column + "={x:String}", + Collections.singletonMap("x", paramValue)).get(); + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(r)) + { + reader.next(); + Assert.assertEquals(reader.getString(1), paramValue); + } + } + + private int[] queryInstant(String tableName, String fieldName, String operator, + Instant param, int scale) throws InterruptedException, ExecutionException, Exception + { + try (QueryResponse response = client.query( + "SELECT id FROM " + tableName + " WHERE " + fieldName + " " + + operator + " {x:DateTime64" + (scale > 0 ? "(" + scale + ")" : "") + "} " + + "ORDER by id ASC", + Collections.singletonMap("x", DataTypeUtils.formatInstant(param))).get(); + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(response)) + { + List ints = new ArrayList<>(3); + while (reader.hasNext()) { + reader.next(); + ints.add(Integer.valueOf(reader.getInteger(1))); + } + return ints.stream().mapToInt(Integer::intValue).toArray(); + } + } + + private Client.Builder newClient() { + ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); + boolean isSecure = isCloud(); + return new Client.Builder() + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isSecure) + .setUsername("default") + .setPassword(ClickHouseServerForTest.getPassword()) + .compressClientRequest(false) + .setDefaultDatabase(ClickHouseServerForTest.getDatabase()) + .serverSetting(ServerSettings.WAIT_ASYNC_INSERT, "1") + .serverSetting(ServerSettings.ASYNC_INSERT, "0"); + } + + private void prepareDataSet(String table, List columns, List> valueGenerators, + int rows) + { + List> data = new ArrayList<>(rows); + try { + // Drop table + client.execute("DROP TABLE IF EXISTS " + table).get(10, TimeUnit.SECONDS); + + // Create table + CommandSettings settings = new CommandSettings(); + StringBuilder createStmtBuilder = new StringBuilder(); + createStmtBuilder.append("CREATE TABLE IF NOT EXISTS ").append(table).append(" ("); + for (String column : columns) { + createStmtBuilder.append(column).append(", "); + } + createStmtBuilder.setLength(createStmtBuilder.length() - 2); + createStmtBuilder.append(") ENGINE = MergeTree ORDER BY tuple()"); + client.execute(createStmtBuilder.toString(), settings).get(10, TimeUnit.SECONDS); + + // Insert data + StringBuilder insertStmtBuilder = new StringBuilder(); + insertStmtBuilder.append("INSERT INTO ").append(table).append(" VALUES "); + for (int i = 0; i < rows; i++) { + insertStmtBuilder.append("("); + Map values = writeValuesRow(insertStmtBuilder, columns, valueGenerators); + insertStmtBuilder.setLength(insertStmtBuilder.length() - 2); + insertStmtBuilder.append("), "); + data.add(values); + } + insertStmtBuilder.setLength(insertStmtBuilder.length() - 2); + String s = insertStmtBuilder.toString(); + client.execute(s).get(10, TimeUnit.SECONDS); + } catch (Exception e) { + Assert.fail("failed to prepare data set", e); + } + } + + private Map writeValuesRow(StringBuilder insertStmtBuilder, List columns, + List> valueGenerators) + { + Map values = new HashMap<>(); + Iterator columnIterator = columns.iterator(); + for (Supplier valueGenerator : valueGenerators) { + Object value = valueGenerator.get(); + if (value instanceof String) { + insertStmtBuilder.append('\'').append(value).append('\'').append(", "); + } else { + insertStmtBuilder.append(value).append(", "); + } + values.put(columnIterator.next().split(" ")[0], value); + + } + return values; + } + + @DataProvider(name = "stringParameters") + private static Object[][] createStringParameterValues() { + return new Object[][] { + { "foo" }, + { "with-dashes" }, + { "☺" }, + { "foo/bar" }, + { "foobar 20" }, + { " leading_and_trailing_spaces " }, + // https://github.com/ClickHouse/ClickHouse/issues/70240 + // { "multi\nline\r\ndos" }, + // { "nicely\"quoted\'string\'" }, + }; + } + +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java b/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java index 4c5a9e70c..cfc3efafd 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/DataTypeUtilsTests.java @@ -2,31 +2,132 @@ import org.testng.annotations.Test; -import java.time.LocalDateTime; - -import static org.testng.AssertJUnit.assertEquals; +import com.clickhouse.data.ClickHouseDataType; -public class DataTypeUtilsTests { +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.TimeZone; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; +class DataTypeUtilsTests { @Test - public void testDateTimeFormatter() { + void testDateTimeFormatter() { LocalDateTime dateTime = LocalDateTime.of(2021, 12, 31, 23, 59, 59); String formattedDateTime = dateTime.format(DataTypeUtils.DATETIME_FORMATTER); - assertEquals("2021-12-31 23:59:59", formattedDateTime); + assertEquals(formattedDateTime, "2021-12-31 23:59:59"); } @Test - public void testDateFormatter() { + void testDateFormatter() { LocalDateTime date = LocalDateTime.of(2021, 12, 31, 10, 20); String formattedDate = date.toLocalDate().format(DataTypeUtils.DATE_FORMATTER); - assertEquals("2021-12-31", formattedDate); + assertEquals(formattedDate, "2021-12-31"); } @Test - public void testDateTimeWithNanosFormatter() { + void testDateTimeWithNanosFormatter() { LocalDateTime dateTime = LocalDateTime.of(2021, 12, 31, 23, 59, 59, 123456789); String formattedDateTimeWithNanos = dateTime.format(DataTypeUtils.DATETIME_WITH_NANOS_FORMATTER); - assertEquals("2021-12-31 23:59:59.123456789", formattedDateTimeWithNanos); + assertEquals(formattedDateTimeWithNanos, "2021-12-31 23:59:59.123456789"); + } + + @Test + void formatInstantForDateNullInstant() { + assertThrows( + NullPointerException.class, + () -> DataTypeUtils.formatInstant(null, ClickHouseDataType.Date, + ZoneId.systemDefault())); + } + + @Test + void formatInstantForDateNullTimeZone() { + assertThrows( + NullPointerException.class, + () -> DataTypeUtils.formatInstant(Instant.now(), ClickHouseDataType.Date, null)); + } + + @Test + void formatInstantForDate() { + ZoneId tzBER = ZoneId.of("Europe/Berlin"); + ZoneId tzLAX = ZoneId.of("America/Los_Angeles"); + Instant instant = ZonedDateTime.of( + 2025, 7, 20, 5, 5, 42, 0, tzBER).toInstant(); + assertEquals( + DataTypeUtils.formatInstant(instant, ClickHouseDataType.Date, tzBER), + "2025-07-20"); + assertEquals( + DataTypeUtils.formatInstant(instant, ClickHouseDataType.Date, tzLAX), + "2025-07-19"); + } + + @Test + void formatInstantNullValue() { + assertThrows( + NullPointerException.class, + () -> DataTypeUtils.formatInstant(null)); + } + + @Test + void formatInstantForDateTime() { + TimeZone tzBER = TimeZone.getTimeZone("Europe/Berlin"); + Instant instant = ZonedDateTime.of( + 2025, 7, 20, 5, 5, 42, 232323, tzBER.toZoneId()).toInstant(); + String formatted = DataTypeUtils.formatInstant(instant, ClickHouseDataType.DateTime); + assertEquals(formatted, "1752980742"); + assertEquals( + Instant.ofEpochSecond(Long.parseLong(formatted)), + instant.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test + void formatInstantForDateTime64() { + TimeZone tzBER = TimeZone.getTimeZone("Europe/Berlin"); + Instant instant = ZonedDateTime.of( + 2025, 7, 20, 5, 5, 42, 232323232, tzBER.toZoneId()).toInstant(); + String formatted = DataTypeUtils.formatInstant(instant); + assertEquals(formatted, "1752980742.232323232"); + String[] formattedParts = formatted.split("\\."); + assertEquals( + Instant + .ofEpochSecond(Long.parseLong(formattedParts[0])) + .plusNanos(Long.parseLong(formattedParts[1])), + instant); } + + @Test + void formatInstantForDateTime64SmallerNanos() { + TimeZone tzBER = TimeZone.getTimeZone("Europe/Berlin"); + Instant instant = ZonedDateTime.of( + 2025, 7, 20, 5, 5, 42, 23, tzBER.toZoneId()).toInstant(); + String formatted = DataTypeUtils.formatInstant(instant); + assertEquals(formatted, "1752980742.000000023"); + String[] formattedParts = formatted.split("\\."); + assertEquals( + Instant + .ofEpochSecond(Long.parseLong(formattedParts[0])) + .plusNanos(Long.parseLong(formattedParts[1])), + instant); + } + + @Test + void formatInstantForDateTime64Truncated() { + // precision is constant for Instant + TimeZone tzBER = TimeZone.getTimeZone("Europe/Berlin"); + Instant instant = ZonedDateTime.of( + 2025, 7, 20, 5, 5, 42, 232323232, tzBER.toZoneId()).toInstant(); + assertEquals( + DataTypeUtils.formatInstant( + instant.truncatedTo(ChronoUnit.SECONDS)), + "1752980742.000000000"); + assertEquals( + DataTypeUtils.formatInstant( + instant.truncatedTo(ChronoUnit.MILLIS)), + "1752980742.232000000"); + } + } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReaderTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReaderTest.java index ccf869358..df87d0325 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReaderTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReaderTest.java @@ -14,7 +14,6 @@ import java.math.BigInteger; import java.util.Arrays; import java.util.TimeZone; -import java.util.function.BiConsumer; import java.util.function.Consumer; public class ClickHouseBinaryFormatReaderTest { @@ -70,11 +69,11 @@ public void testReadingNumbers() throws IOException { Assert.assertEquals(reader.getBoolean(name), Boolean.TRUE); Assert.assertEquals(reader.getByte(name), (byte)testValue); Assert.assertEquals(reader.getShort(name), (short)testValue); - Assert.assertEquals(reader.getInteger(name), (int)testValue); - Assert.assertEquals(reader.getLong(name), (long)testValue); + Assert.assertEquals(reader.getInteger(name), testValue); + Assert.assertEquals(reader.getLong(name), testValue); - Assert.assertEquals(reader.getFloat(name), (float) testValue); - Assert.assertEquals(reader.getDouble(name), (double) testValue); + Assert.assertEquals(reader.getFloat(name), testValue); + Assert.assertEquals(reader.getDouble(name), testValue); Assert.assertEquals(reader.getBigInteger(name), BigInteger.valueOf((testValue))); Assert.assertTrue(reader.getBigDecimal(name).compareTo(BigDecimal.valueOf((testValue))) == 0); diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java index 4fbf1d314..81c1f32c5 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java @@ -1,10 +1,28 @@ package com.clickhouse.client.api.data_formats.internal; -import org.junit.Assert; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.TimeZone; + +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class BinaryStreamReaderTests { + private ZoneId tzLAX; + private ZoneId tzBER; + + @BeforeClass + void beforeClass() { + tzLAX = ZoneId.of("America/Los_Angeles"); + tzBER = ZoneId.of("Europe/Berlin"); + } @Test public void testCachedByteAllocator() { @@ -14,7 +32,7 @@ public void testCachedByteAllocator() { int size = (int) Math.pow(2, i); byte[] firstAllocation = allocator.allocate(size); byte[] nextAllocation = allocator.allocate(size); - Assert.assertSame( "Should be the same buffer for size " + size, firstAllocation, nextAllocation); + Assert.assertTrue(firstAllocation == nextAllocation, "Should be the same buffer for size " + size); } for (int i = 6; i < 16; i++) { @@ -24,4 +42,138 @@ public void testCachedByteAllocator() { Assert.assertNotSame(firstAllocation, nextAllocation); } } + + @Test(dataProvider = "dateTestData") + void readDateZonedDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + /* + * Date is number of days since 1970-01-01 (unsigned) + * ... The date value is stored without the time zone. + */ + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDate(baos, zdt, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDate( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT); + } + + @Test(dataProvider = "dateTestData") + void readDateOffsetDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDate(baos, zdt.toOffsetDateTime(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDate( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), + expectedZDT.toOffsetDateTime()); + } + + @DataProvider(name = "dateTestData") + private Object[][] provideDateTestData() { + ZonedDateTime zdtLAX = ZonedDateTime.of( + 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); + ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); + return new Object[][] { + // no conversion at all + { zdtLAX, tzLAX, tzLAX, zdtLAX.truncatedTo(ChronoUnit.DAYS) }, + + // write using Berlin local date -> next day + { zdtLAX, tzBER, tzBER, zdtLAX.plusDays(1L).withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) }, + + // read using different time zone: local date same as original + { zdtLAX, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) }, + + // write using different time zone: local date same as original + { zdtBER, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) } + }; + + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32ZonedDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32OffsetDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt.toOffsetDateTime(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), + expectedZDT.toOffsetDateTime().truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt.toInstant(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime64Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime64(baos, zdt.toInstant(), 9, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime64( + new ByteArrayInputStream(bytes), + bytes, + 9, + TimeZone.getTimeZone(readTZ)), + expectedZDT); + } + + @DataProvider(name = "dateTimeTestData") + private Object[][] provideDateTimeTestData() { + ZonedDateTime zdtLAX = ZonedDateTime.of( + 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); + ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); + return new Object[][] { + { zdtLAX, tzLAX, tzLAX, zdtLAX }, + { zdtLAX, tzBER, tzLAX, zdtLAX }, + { zdtLAX, tzLAX, tzBER, zdtBER }, + { zdtBER, tzLAX, tzBER, zdtBER } + }; + } + }