From 4fca68289f0456be8d25ce1a9d46d72cbc2a80b7 Mon Sep 17 00:00:00 2001 From: gfunc Date: Thu, 24 Apr 2025 05:39:19 +0000 Subject: [PATCH 1/5] debug UInt32 and UInt64 conversion error and add support for not flattened Nested type --- .../internal/BinaryStreamReader.java | 29 +++ .../clickhouse/jdbc/internal/JdbcUtils.java | 3 +- .../com/clickhouse/jdbc/DataTypeTests.java | 225 +++++++++++++++++- 3 files changed, 253 insertions(+), 4 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 662dc8eb6..62298e572 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -234,6 +234,8 @@ public T readValue(ClickHouseColumn column, Class typeHint) throws IOExce return (T) readVariant(actualColumn); case Dynamic: return (T) readValue(actualColumn, typeHint); + case Nested: + return convertArray(readNested(actualColumn), typeHint); default: throw new IllegalArgumentException("Unsupported data type: " + actualColumn.getDataType()); } @@ -785,6 +787,33 @@ public Object[] readTuple(ClickHouseColumn column) throws IOException { return tuple; } + /** + * Reads a nested into an ArrayValue object. + * @param column - column information + * @return array value + * @throws IOException when IO error occurs + */ + public ArrayValue readNested(ClickHouseColumn column) throws IOException { + int len = readVarInt(input); + if (len == 0) { + return new ArrayValue(Object[].class, 0); + } + + ArrayValue array; + array = new ArrayValue(Object[].class, len); + for (int i = 0; i < len; i++) { + int tupleLen = column.getNestedColumns().size(); + Object[] tuple = new Object[tupleLen]; + for (int j = 0; j < tupleLen; j++) { + tuple[j] = readValue(column.getNestedColumns().get(j)); + } + + array.set(i, tuple); + } + + return array; + } + public Object readVariant(ClickHouseColumn column) throws IOException { int ordNum = readByte(); return readValue(column.getNestedColumns().get(ordNum)); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java index 754803fc1..7f57a55b3 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java @@ -44,9 +44,8 @@ private static Map generateTypeMap() { map.put(ClickHouseDataType.Int16, JDBCType.SMALLINT); map.put(ClickHouseDataType.UInt16, JDBCType.SMALLINT); map.put(ClickHouseDataType.Int32, JDBCType.INTEGER); - map.put(ClickHouseDataType.UInt32, JDBCType.INTEGER); + map.put(ClickHouseDataType.UInt32, JDBCType.BIGINT); map.put(ClickHouseDataType.Int64, JDBCType.BIGINT); - map.put(ClickHouseDataType.UInt64, JDBCType.BIGINT); map.put(ClickHouseDataType.Float32, JDBCType.FLOAT); map.put(ClickHouseDataType.Float64, JDBCType.DOUBLE); map.put(ClickHouseDataType.Bool, JDBCType.BOOLEAN); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java index b19fd02e4..c1b71bfe7 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java @@ -13,6 +13,7 @@ import java.net.Inet6Address; import java.net.InetAddress; import java.net.UnknownHostException; +import java.sql.Array; import java.sql.Connection; import java.sql.Date; import java.sql.JDBCType; @@ -21,6 +22,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; +import java.text.DecimalFormat; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.OffsetDateTime; @@ -165,6 +167,57 @@ public void testIntegerTypes() throws SQLException { } } } + + // Check the with getObject + try (Connection conn = getConnection()) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_integers ORDER BY order")) { + assertTrue(rs.next()); + assertEquals(rs.getObject("int8"), -128); + assertEquals(rs.getObject("int16"), -32768); + assertEquals(rs.getObject("int32"), -2147483648); + assertEquals(rs.getObject("int64"), -9223372036854775808L); + assertEquals(rs.getObject("int128"), new BigInteger("-170141183460469231731687303715884105728")); + assertEquals(rs.getObject("int256"), new BigInteger("-57896044618658097711785492504343953926634992332820282019728792003956564819968")); + assertEquals(rs.getObject("uint8"), 0); + assertEquals(rs.getObject("uint16"), 0); + assertEquals(rs.getObject("uint32"), 0L); + assertEquals(rs.getObject("uint64"), new BigInteger("0")); + assertEquals(rs.getObject("uint128"), new BigInteger("0")); + assertEquals(rs.getObject("uint256"), new BigInteger("0")); + + assertTrue(rs.next()); + assertEquals(rs.getObject("int8"), 127); + assertEquals(rs.getObject("int16"), 32767); + assertEquals(rs.getObject("int32"), 2147483647); + assertEquals(rs.getObject("int64"), 9223372036854775807L); + assertEquals(rs.getObject("int128"), new BigInteger("170141183460469231731687303715884105727")); + assertEquals(rs.getObject("int256"), new BigInteger("57896044618658097711785492504343953926634992332820282019728792003956564819967")); + assertEquals(rs.getObject("uint8"), 255); + assertEquals(rs.getObject("uint16"), 65535); + assertEquals(rs.getObject("uint32"), 4294967295L); + assertEquals(rs.getObject("uint64"), new BigInteger("18446744073709551615")); + assertEquals(rs.getObject("uint128"), new BigInteger("340282366920938463463374607431768211455")); + assertEquals(rs.getObject("uint256"), new BigInteger("115792089237316195423570985008687907853269984665640564039457584007913129639935")); + + assertTrue(rs.next()); + assertEquals(rs.getObject("int8"), int8); + assertEquals(rs.getObject("int16"), int16); + assertEquals(rs.getObject("int32"), int32); + assertEquals(rs.getObject("int64"), int64); + assertEquals(rs.getObject("int128"), int128); + assertEquals(rs.getObject("int256"), int256); + assertEquals(rs.getObject("uint8"), uint8); + assertEquals(rs.getObject("uint16"), uint16); + assertEquals(rs.getObject("uint32"), uint32); + assertEquals(rs.getObject("uint64"), uint64); + assertEquals(rs.getObject("uint128"), uint128); + assertEquals(rs.getObject("uint256"), uint256); + + assertFalse(rs.next()); + } + } + } } @Test(groups = { "integration" }) @@ -233,6 +286,36 @@ public void testDecimalTypes() throws SQLException { } } } + + // Check the results with getObject + try (Connection conn = getConnection()) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_decimals ORDER BY order")) { + assertTrue(rs.next()); + assertEquals(rs.getObject("dec"), new BigDecimal("-9999999.99")); + assertEquals(rs.getObject("dec32"), new BigDecimal("-99999.9999")); + assertEquals(rs.getObject("dec64"), new BigDecimal("-9999999999.99999999")); + assertEquals(rs.getObject("dec128"), new BigDecimal("-99999999999999999999.999999999999999999")); + assertEquals(rs.getObject("dec256"), new BigDecimal("-9999999999999999999999999999999999999999999999999999999999.999999999999999999")); + + assertTrue(rs.next()); + assertEquals(rs.getObject("dec"), new BigDecimal("9999999.99")); + assertEquals(rs.getObject("dec32"), new BigDecimal("99999.9999")); + assertEquals(rs.getObject("dec64"), new BigDecimal("9999999999.99999999")); + assertEquals(rs.getObject("dec128"), new BigDecimal("99999999999999999999.999999999999999999")); + assertEquals(rs.getObject("dec256"), new BigDecimal("9999999999999999999999999999999999999999999999999999999999.999999999999999999")); + + assertTrue(rs.next()); + assertEquals(rs.getObject("dec"), dec); + assertEquals(rs.getObject("dec32"), dec32); + assertEquals(rs.getObject("dec64"), dec64); + assertEquals(rs.getObject("dec128"), dec128); + assertEquals(rs.getObject("dec256"), dec256); + + assertFalse(rs.next()); + } + } + } } @Test(groups = { "integration" }) @@ -315,6 +398,42 @@ public void testDateTypes() throws SQLException { } } } + + // Check the results + try (Connection conn = getConnection()) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_dates ORDER BY order")) { + assertTrue(rs.next()); + assertEquals(rs.getObject("date"), Date.valueOf("1970-01-01")); + assertEquals(rs.getObject("date32"), Date.valueOf("1970-01-01")); + assertEquals(rs.getObject("dateTime").toString(), "1970-01-01 00:00:00.0"); + assertEquals(rs.getObject("dateTime32").toString(), "1970-01-01 00:00:00.0"); + assertEquals(rs.getObject("dateTime643").toString(), "1970-01-01 00:00:00.0"); + assertEquals(rs.getObject("dateTime646").toString(), "1970-01-01 00:00:00.0"); + assertEquals(rs.getObject("dateTime649").toString(), "1970-01-01 00:00:00.0"); + + assertTrue(rs.next()); + assertEquals(rs.getObject("date"), Date.valueOf("2149-06-06")); + assertEquals(rs.getObject("date32"), Date.valueOf("2299-12-31")); + assertEquals(rs.getObject("dateTime").toString(), "2106-02-07 06:28:15.0"); + assertEquals(rs.getObject("dateTime32").toString(), "2106-02-07 06:28:15.0"); + assertEquals(rs.getObject("dateTime643").toString(), "2261-12-31 23:59:59.999"); + assertEquals(rs.getObject("dateTime646").toString(), "2261-12-31 23:59:59.999999"); + assertEquals(rs.getObject("dateTime649").toString(), "2261-12-31 23:59:59.999999999"); + + assertTrue(rs.next()); + assertEquals(rs.getObject("date").toString(), date.toString()); + assertEquals(rs.getObject("date32").toString(), date32.toString()); + assertEquals(rs.getObject("dateTime").toString(), dateTime.toString()); + assertEquals(rs.getObject("dateTime32").toString(), dateTime32.toString()); + assertEquals(rs.getObject("dateTime643").toString(), dateTime643.toString()); + assertEquals(rs.getObject("dateTime646").toString(), dateTime646.toString()); + assertEquals(rs.getObject("dateTime649").toString(), dateTime649.toString()); + + assertFalse(rs.next()); + } + } + } } @Test(groups = { "integration" }) @@ -368,6 +487,23 @@ public void testStringTypes() throws SQLException { } } } + + // Check the results with getObject + try (Connection conn = getConnection()) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_strings ORDER BY order")) { + assertTrue(rs.next()); + assertEquals(rs.getObject("str"), str); + assertEquals(rs.getObject("fixed"), fixed); + assertEquals(rs.getObject("enum"), "a"); + assertEquals(rs.getObject("enum8"), "a"); + assertEquals(rs.getObject("enum16"), "b"); + assertEquals(rs.getObject("uuid"), UUID.fromString(uuid)); + assertEquals(rs.getObject("escaped"), escaped); + assertFalse(rs.next()); + } + } + } } @Test(groups = { "integration" }) @@ -457,6 +593,29 @@ public void testFloatTypes() throws SQLException { } } } + + // Check the results with getObject + try (Connection conn = getConnection()) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_floats ORDER BY order")) { + assertTrue(rs.next()); + assertEquals(rs.getObject("float32"), -3.402823E38d); + assertEquals(rs.getObject("float64"), Double.valueOf(-1.7976931348623157E308)); + + assertTrue(rs.next()); + assertEquals(rs.getObject("float32"), 3.402823E38d); + assertEquals(rs.getObject("float64"), Double.valueOf(1.7976931348623157E308)); + + assertTrue(rs.next()); + + DecimalFormat df = new DecimalFormat("#.########"); + assertEquals(df.format(rs.getObject("float32")), df.format(float32)); + assertEquals(rs.getObject("float64"), float64); + + assertFalse(rs.next()); + } + } + } } @Test(groups = { "integration" }) @@ -490,6 +649,18 @@ public void testBooleanTypes() throws SQLException { } } } + + // Check the results with getObject + try (Connection conn = getConnection()) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_booleans ORDER BY order")) { + assertTrue(rs.next()); + assertEquals(rs.getObject("bool"), bool); + + assertFalse(rs.next()); + } + } + } } @Test(groups = { "integration" }) @@ -556,6 +727,35 @@ public void testArrayTypes() throws SQLException { } } } + + // Check the results with getObject + try (Connection conn = getConnection()) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_arrays ORDER BY order")) { + assertTrue(rs.next()); + Object[] arrayResult = (Object[]) ((Array) rs.getObject("array")).getArray(); + assertEquals(arrayResult.length, array.length); + for (int i = 0; i < array.length; i++) { + assertEquals(String.valueOf(arrayResult[i]), String.valueOf(array[i])); + } + + Object[] arraystrResult = (Object[]) ((Array) rs.getObject("arraystr")).getArray(); + assertEquals(arraystrResult.length, arraystr.length); + for (int i = 0; i < arraystr.length; i++) { + assertEquals(arraystrResult[i], arraystr[i]); + } + Object[] arraytupleResult = (Object[]) ((Array) rs.getObject("arraytuple")).getArray(); + assertEquals(arraytupleResult.length, arraytuple.length); + for (int i = 0; i < arraytuple.length; i++) { + Tuple tuple = arraytuple[i]; + Tuple tupleResult = new Tuple(((Object[]) arraytupleResult[i])); + assertEquals(String.valueOf(tupleResult.getValue(0)), String.valueOf(tuple.getValue(0))); + assertEquals(String.valueOf(tupleResult.getValue(1)), String.valueOf(tuple.getValue(1))); + } + assertFalse(rs.next()); + } + } + } } @Test(groups = { "integration" }) @@ -669,7 +869,7 @@ public void testLowCardinalityTypeSimpleStatement() throws SQLException { try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_low_cardinality ORDER BY order")) { assertTrue(rs.next()); assertEquals(rs.getString("lowcardinality"), lowcardinality); - + assertEquals(rs.getObject("lowcardinality"), lowcardinality); assertFalse(rs.next()); } } @@ -700,6 +900,7 @@ public void testSimpleAggregateFunction() throws SQLException { try (ResultSet rs = stmt.executeQuery("SELECT sum(int8) FROM test_aggregate")) { assertTrue(rs.next()); assertEquals(rs.getInt(1), int8 * 3); + assertEquals(rs.getObject(1), (long) (int8 * 3)); } try (ResultSet rs = stmt.executeQuery("SELECT any(val) FROM test_aggregate")) { assertTrue(rs.next()); @@ -715,7 +916,9 @@ public void testNestedTypeSimpleStatement() throws SQLException { runQuery("CREATE TABLE test_nested (order Int8, " + "nested Nested (int8 Int8, int16 Int16, int32 Int32, int64 Int64, int128 Int128, int256 Int256)" + ") ENGINE = MergeTree ORDER BY ()"); - + runQuery("CREATE TABLE test_nested_not_flatten (order Int8, " + + "nested Nested (int8 Int8, int16 Int16, int32 Int32, int64 Int64, int128 Int128, int256 Int256)" + + ") ENGINE = MergeTree ORDER BY () SETTINGS flatten_nested = 0"); // Insert random (valid) values long seed = System.currentTimeMillis(); Random rand = new Random(seed); @@ -733,6 +936,11 @@ public void testNestedTypeSimpleStatement() throws SQLException { log.info("SQL: {}", sql); insertData(sql); + String nsql = String.format("INSERT INTO test_nested_not_flatten VALUES ( 1, [(%s,%s,%s,%s,%s,%s)])", + int8, int16, int32, int64, int128, int256); + log.info("SQL: {}", nsql); + insertData(nsql); + // Check the results try (Connection conn = getConnection()) { try (Statement stmt = conn.createStatement()) { @@ -749,6 +957,19 @@ public void testNestedTypeSimpleStatement() throws SQLException { } } } + + // Check the results + try (Connection conn = getConnection()) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_nested_not_flatten ORDER BY order")) { + assertTrue(rs.next()); + assertEquals((Object[])((Object[])((java.sql.Array) rs.getObject("nested")).getArray())[0], + new Object[] {(byte) int8, (short) int16, int32, int64, int128, int256}); + + assertFalse(rs.next()); + } + } + } } @Test(groups = { "integration" }) From 983a527c4a5f7afc517505093fb282725980088a Mon Sep 17 00:00:00 2001 From: gfunc Date: Thu, 24 Apr 2025 14:42:59 +0000 Subject: [PATCH 2/5] add support for SET stmt and debug --- .../com/clickhouse/client/api/Client.java | 4 ++ .../client/api/ClientConfigProperties.java | 2 + .../api/internal/HttpAPIClientHelper.java | 7 +++- .../com/clickhouse/jdbc/StatementImpl.java | 14 +++++++ .../com/clickhouse/jdbc/DataTypeTests.java | 40 ++++++++++++++----- .../com/clickhouse/jdbc/StatementTest.java | 27 +++++++++++++ .../jdbc/metadata/DatabaseMetaDataTest.java | 16 ++++---- 7 files changed, 91 insertions(+), 19 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 8c6e58803..0a8a3b8b8 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -2190,6 +2190,10 @@ public void setDBRoles(Collection dbRoles) { this.configuration.get(ClientConfigProperties.SESSION_DB_ROLES.getKey()))); } + public void setProfile(String name) { + this.configuration.put(ClientConfigProperties.PROFILE.getKey(), name); + } + public void updateClientName(String name) { this.configuration.put(ClientConfigProperties.CLIENT_NAME.getKey(), name); } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index c2137b736..ab9c76986 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -22,6 +22,8 @@ public enum ClientConfigProperties { PASSWORD("password", ""), + PROFILE("profile"), + /** * Maximum number of active connection in internal connection pool. */ diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 3cb454e6b..3dfd05767 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -556,10 +556,15 @@ private void addQueryParams(URIBuilder req, Map chConfig, Map sessionRoles = (Collection) requestConfig.getOrDefault(ClientConfigProperties.SESSION_DB_ROLES.getKey(), ClientConfigProperties.valuesFromCommaSeparated(chConfiguration.getOrDefault(ClientConfigProperties.SESSION_DB_ROLES.getKey(), ""))); if (!sessionRoles.isEmpty()) { - sessionRoles.forEach(r -> req.addParameter(ClickHouseHttpProto.QPARAM_ROLE, r)); } + String profile = (String) requestConfig.getOrDefault(ClientConfigProperties.PROFILE.getKey(), + chConfiguration.getOrDefault(ClientConfigProperties.PROFILE.getKey(), "")); + if (!profile.isEmpty()) { + req.addParameter(ClientConfigProperties.PROFILE.getKey(), profile); + } + for (String key : requestConfig.keySet()) { if (key.startsWith(ClientConfigProperties.SERVER_SETTING_PREFIX)) { Object val = requestConfig.get(key); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java index ce4bcf013..4285cb4ae 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java @@ -376,6 +376,19 @@ public boolean execute(String sql, QuerySettings settings) throws SQLException { connection.client.setDBRoles(roles); } } + + } else if (JdbcUtils.containsIgnoresCase(tokens, "=")){ + String key = tokens.get(1); + String value = tokens.get(3).replace("'", ""); + //SET profile + if (key.equals("profile")) { + connection.client.setProfile(value); + LOG.debug("Set profile to {}", value); + } else { + //SET session settings + connection.getDefaultQuerySettings().serverSetting(key, value); + LOG.debug("Set session server setting {} to {}", key, value); + } } return false; } else if (type == StatementType.USE) { @@ -383,6 +396,7 @@ public boolean execute(String sql, QuerySettings settings) throws SQLException { //USE Database List tokens = JdbcUtils.tokenizeSQL(sql); this.schema = tokens.get(1).replace("\"", ""); + connection.getDefaultQuerySettings().setDatabase(schema); LOG.debug("Changed statement schema {}", schema); return false; } else { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java index c1b71bfe7..208995c9d 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java @@ -608,7 +608,7 @@ public void testFloatTypes() throws SQLException { assertTrue(rs.next()); - DecimalFormat df = new DecimalFormat("#.########"); + DecimalFormat df = new DecimalFormat("#.######"); assertEquals(df.format(rs.getObject("float32")), df.format(float32)); assertEquals(rs.getObject("float64"), float64); @@ -916,9 +916,7 @@ public void testNestedTypeSimpleStatement() throws SQLException { runQuery("CREATE TABLE test_nested (order Int8, " + "nested Nested (int8 Int8, int16 Int16, int32 Int32, int64 Int64, int128 Int128, int256 Int256)" + ") ENGINE = MergeTree ORDER BY ()"); - runQuery("CREATE TABLE test_nested_not_flatten (order Int8, " - + "nested Nested (int8 Int8, int16 Int16, int32 Int32, int64 Int64, int128 Int128, int256 Int256)" - + ") ENGINE = MergeTree ORDER BY () SETTINGS flatten_nested = 0"); + // Insert random (valid) values long seed = System.currentTimeMillis(); Random rand = new Random(seed); @@ -936,11 +934,6 @@ public void testNestedTypeSimpleStatement() throws SQLException { log.info("SQL: {}", sql); insertData(sql); - String nsql = String.format("INSERT INTO test_nested_not_flatten VALUES ( 1, [(%s,%s,%s,%s,%s,%s)])", - int8, int16, int32, int64, int128, int256); - log.info("SQL: {}", nsql); - insertData(nsql); - // Check the results try (Connection conn = getConnection()) { try (Statement stmt = conn.createStatement()) { @@ -958,9 +951,36 @@ public void testNestedTypeSimpleStatement() throws SQLException { } } - // Check the results + } + + @Test(groups = { "integration" }) + public void testNestedTypeNonFlatten() throws SQLException { try (Connection conn = getConnection()) { try (Statement stmt = conn.createStatement()) { + stmt.execute("SET flatten_nested = 0"); + stmt.execute("CREATE TABLE test_nested_not_flatten (order Int8, " + + "nested Nested (int8 Int8, int16 Int16, int32 Int32, int64 Int64, int128 Int128, int256 Int256)" + + ") ENGINE = MergeTree ORDER BY ()"); + // Insert random (valid) values + long seed = System.currentTimeMillis(); + Random rand = new Random(seed); + log.info("Random seed was: {}", seed); + + int int8 = rand.nextInt(256) - 128; + int int16 = rand.nextInt(65536) - 32768; + int int32 = rand.nextInt(); + long int64 = rand.nextLong(); + BigInteger int128 = new BigInteger(127, rand); + BigInteger int256 = new BigInteger(255, rand); + + + String nsql = String.format("INSERT INTO test_nested_not_flatten VALUES ( 1, [(%s,%s,%s,%s,%s,%s)])", + int8, int16, int32, int64, int128, int256); + log.info("SQL: {}", nsql); + stmt.executeUpdate(nsql); + + // Check the results + try (ResultSet rs = stmt.executeQuery("SELECT * FROM test_nested_not_flatten ORDER BY order")) { assertTrue(rs.next()); assertEquals((Object[])((Object[])((java.sql.Array) rs.getObject("nested")).getArray())[0], diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index 7396f037c..3a539c736 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -412,6 +412,33 @@ record = conn.client.queryAll("SELECT currentRoles()").get(0); } } + @Test(groups = { "integration" }) + public void testSetStatement() throws SQLException { + try (ConnectionImpl conn = (ConnectionImpl) getJdbcConnection()) { + try (Statement stmt = conn.createStatement()) { + + stmt.execute("SET session_timezone = 'Europe/Berlin'"); + try (ResultSet rs = stmt.executeQuery("SELECT timezone()")){ + assertTrue(rs.next()); + assertEquals(rs.getString(1),"Europe/Berlin"); + assertFalse(rs.next()); + } + stmt.execute("SET network_compression_method = 'GZIP'"); + try (ResultSet rs = stmt.executeQuery("SELECT getSetting('network_compression_method')")){ + assertTrue(rs.next()); + assertEquals(rs.getString(1),"GZIP"); + assertFalse(rs.next()); + } + stmt.execute("SET profile = 'readonly'"); + try (ResultSet rs = stmt.executeQuery("SELECT currentProfiles()")){ + assertTrue(rs.next()); + assertEquals(((Object[]) rs.getArray(1).getArray())[0],"readonly"); + assertFalse(rs.next()); + } + } + } + } + @Test public void testGettingArrays() throws Exception { try (ConnectionImpl conn = (ConnectionImpl) getJdbcConnection(); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java index b9048f04e..bffd1b919 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataTest.java @@ -32,14 +32,14 @@ public void testGetColumns() throws Exception { conn.createStatement().execute("DROP TABLE IF EXISTS " + tableName); StringBuilder createTableStmt = new StringBuilder("CREATE TABLE " + tableName + " ("); - List columnNames = Arrays.asList("id", "name", "float1", "fixed_string1", "decimal_1", "nullable_column", "date", "datetime"); - List columnTypes = Arrays.asList("UInt64", "String", "Float32", "FixedString(10)", "Decimal(10, 2)", "Nullable(Decimal(5, 4))", "Date", "DateTime"); - List columnSizes = Arrays.asList(8, 0, 4, 10, 10, 5, 2, 0); - List columnJDBCDataTypes = Arrays.asList(Types.BIGINT, Types.VARCHAR, Types.FLOAT, Types.VARCHAR, Types.DECIMAL, Types.DECIMAL, Types.DATE, Types.TIMESTAMP); - List columnTypeNames = Arrays.asList("UInt64", "String", "Float32", "FixedString(10)", "Decimal(10, 2)", "Nullable(Decimal(5, 4))", "Date", "DateTime"); - List columnNullable = Arrays.asList(false, false, false, false, false, true, false, false); - List columnDecimalDigits = Arrays.asList(null, null, null, null, 2, 4, null, null); - List columnRadix = Arrays.asList(2, null, null, null, 10, 10, null, null); + List columnNames = Arrays.asList("id", "huge_integer", "name", "float1", "fixed_string1", "decimal_1", "nullable_column", "date", "datetime"); + List columnTypes = Arrays.asList("Int64", "UInt128", "String", "Float32", "FixedString(10)", "Decimal(10, 2)", "Nullable(Decimal(5, 4))", "Date", "DateTime"); + List columnSizes = Arrays.asList(8, 16, 0, 4, 10, 10, 5, 2, 0); + List columnJDBCDataTypes = Arrays.asList(Types.BIGINT, Types.OTHER, Types.VARCHAR, Types.FLOAT, Types.VARCHAR, Types.DECIMAL, Types.DECIMAL, Types.DATE, Types.TIMESTAMP); + List columnTypeNames = Arrays.asList("Int64", "UInt128", "String", "Float32", "FixedString(10)", "Decimal(10, 2)", "Nullable(Decimal(5, 4))", "Date", "DateTime"); + List columnNullable = Arrays.asList(false, false, false, false, false, false, true, false, false); + List columnDecimalDigits = Arrays.asList(null, null, null, null, null, 2, 4, null, null); + List columnRadix = Arrays.asList(2, 2, null, null, null, 10, 10, null, null); for (int i = 0; i < columnNames.size(); i++) { createTableStmt.append(columnNames.get(i)).append(" ").append(columnTypes.get(i)).append(','); From 8d77e8f2ffa492e6987960b3d7e2f7af9d44b2b9 Mon Sep 17 00:00:00 2001 From: gfunc Date: Thu, 24 Apr 2025 16:18:05 +0000 Subject: [PATCH 3/5] fix USE statement --- .../main/java/com/clickhouse/jdbc/StatementImpl.java | 4 ++-- .../test/java/com/clickhouse/jdbc/StatementTest.java | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java index 4285cb4ae..63cfe7102 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java @@ -396,8 +396,8 @@ public boolean execute(String sql, QuerySettings settings) throws SQLException { //USE Database List tokens = JdbcUtils.tokenizeSQL(sql); this.schema = tokens.get(1).replace("\"", ""); - connection.getDefaultQuerySettings().setDatabase(schema); - LOG.debug("Changed statement schema {}", schema); + connection.setSchema(schema); + LOG.debug("Changed statement schema to {}", schema); return false; } else { executeUpdate(sql, settings); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index 3a539c736..abfc7c986 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -591,6 +591,16 @@ public void testSwitchDatabase() throws Exception { assertFalse(stmt.execute("USE \"" + databaseName + "\"")); assertEquals(stmt.executeUpdate(createSql), 0); } + conn.createStatement().execute("USE system"); + ResultSet rs = conn.createStatement().executeQuery("SELECT name FROM settings LIMIT 1;"); + assertTrue(rs.next()); + assertNotNull(rs.getString(1)); + assertFalse(rs.next()); + conn.createStatement().execute("USE \"" + databaseName + "\""); + rs = conn.createStatement().executeQuery("SHOW TABLES LIMIT 1"); + assertTrue(rs.next()); + assertEquals(rs.getString(1), "switchDatabaseWithUse"); + assertFalse(rs.next()); } } From c3bec5f142eaf86fd3aac9b0cb003718ab988d4e Mon Sep 17 00:00:00 2001 From: gfunc Date: Fri, 25 Apr 2025 04:22:55 +0000 Subject: [PATCH 4/5] remove SET statement support --- .../com/clickhouse/client/api/Client.java | 4 --- .../client/api/ClientConfigProperties.java | 2 -- .../api/internal/HttpAPIClientHelper.java | 6 ----- .../com/clickhouse/jdbc/StatementImpl.java | 13 --------- .../com/clickhouse/jdbc/DataTypeTests.java | 5 +++- .../com/clickhouse/jdbc/StatementTest.java | 27 ------------------- 6 files changed, 4 insertions(+), 53 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index b79f4fb5b..06f7935c6 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -2213,10 +2213,6 @@ public void setDBRoles(Collection dbRoles) { this.configuration.get(ClientConfigProperties.SESSION_DB_ROLES.getKey()))); } - public void setProfile(String name) { - this.configuration.put(ClientConfigProperties.PROFILE.getKey(), name); - } - public void updateClientName(String name) { this.configuration.put(ClientConfigProperties.CLIENT_NAME.getKey(), name); } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index f428f27a2..0327e1e80 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -22,8 +22,6 @@ public enum ClientConfigProperties { PASSWORD("password", ""), - PROFILE("profile"), - /** * Maximum number of active connection in internal connection pool. */ diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 3dfd05767..17b6e7e5f 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -559,12 +559,6 @@ private void addQueryParams(URIBuilder req, Map chConfig, Map req.addParameter(ClickHouseHttpProto.QPARAM_ROLE, r)); } - String profile = (String) requestConfig.getOrDefault(ClientConfigProperties.PROFILE.getKey(), - chConfiguration.getOrDefault(ClientConfigProperties.PROFILE.getKey(), "")); - if (!profile.isEmpty()) { - req.addParameter(ClientConfigProperties.PROFILE.getKey(), profile); - } - for (String key : requestConfig.keySet()) { if (key.startsWith(ClientConfigProperties.SERVER_SETTING_PREFIX)) { Object val = requestConfig.get(key); diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java index 8aea3ce29..4163c07db 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java @@ -373,19 +373,6 @@ public boolean executeImpl(String sql, StatementType type, QuerySettings setting connection.client.setDBRoles(roles); } } - - } else if (JdbcUtils.containsIgnoresCase(tokens, "=")){ - String key = tokens.get(1); - String value = tokens.get(3).replace("'", ""); - //SET profile - if (key.equals("profile")) { - connection.client.setProfile(value); - LOG.debug("Set profile to {}", value); - } else { - //SET session settings - connection.getDefaultQuerySettings().serverSetting(key, value); - LOG.debug("Set session server setting {} to {}", key, value); - } } return false; } else if (type == StatementType.USE) { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java index 208995c9d..fbf6f16b4 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java @@ -955,12 +955,15 @@ public void testNestedTypeSimpleStatement() throws SQLException { @Test(groups = { "integration" }) public void testNestedTypeNonFlatten() throws SQLException { + if (earlierThan(25,1)){ + return; + } try (Connection conn = getConnection()) { try (Statement stmt = conn.createStatement()) { stmt.execute("SET flatten_nested = 0"); stmt.execute("CREATE TABLE test_nested_not_flatten (order Int8, " + "nested Nested (int8 Int8, int16 Int16, int32 Int32, int64 Int64, int128 Int128, int256 Int256)" - + ") ENGINE = MergeTree ORDER BY ()"); + + ") ENGINE = MergeTree ORDER BY () SETTINGS flatten_nested = 0"); // Insert random (valid) values long seed = System.currentTimeMillis(); Random rand = new Random(seed); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index abfc7c986..0fe8eeb8e 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -412,33 +412,6 @@ record = conn.client.queryAll("SELECT currentRoles()").get(0); } } - @Test(groups = { "integration" }) - public void testSetStatement() throws SQLException { - try (ConnectionImpl conn = (ConnectionImpl) getJdbcConnection()) { - try (Statement stmt = conn.createStatement()) { - - stmt.execute("SET session_timezone = 'Europe/Berlin'"); - try (ResultSet rs = stmt.executeQuery("SELECT timezone()")){ - assertTrue(rs.next()); - assertEquals(rs.getString(1),"Europe/Berlin"); - assertFalse(rs.next()); - } - stmt.execute("SET network_compression_method = 'GZIP'"); - try (ResultSet rs = stmt.executeQuery("SELECT getSetting('network_compression_method')")){ - assertTrue(rs.next()); - assertEquals(rs.getString(1),"GZIP"); - assertFalse(rs.next()); - } - stmt.execute("SET profile = 'readonly'"); - try (ResultSet rs = stmt.executeQuery("SELECT currentProfiles()")){ - assertTrue(rs.next()); - assertEquals(((Object[]) rs.getArray(1).getArray())[0],"readonly"); - assertFalse(rs.next()); - } - } - } - } - @Test public void testGettingArrays() throws Exception { try (ConnectionImpl conn = (ConnectionImpl) getJdbcConnection(); From 9f09e7d14790b6680f7a7a28c11595ae686f17d2 Mon Sep 17 00:00:00 2001 From: gfunc Date: Fri, 25 Apr 2025 07:19:06 +0000 Subject: [PATCH 5/5] remove UInt from JDBC type mapping --- .../main/java/com/clickhouse/jdbc/internal/JdbcUtils.java | 3 --- .../src/test/java/com/clickhouse/jdbc/DataTypeTests.java | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java index 7f57a55b3..1a48260f1 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcUtils.java @@ -40,11 +40,8 @@ public class JdbcUtils { private static Map generateTypeMap() { Map map = new TreeMap<>(); // TreeMap is used to sort the keys in natural order so FixedString will be before String :-) (type match should be more accurate) map.put(ClickHouseDataType.Int8, JDBCType.TINYINT); - map.put(ClickHouseDataType.UInt8, JDBCType.TINYINT); map.put(ClickHouseDataType.Int16, JDBCType.SMALLINT); - map.put(ClickHouseDataType.UInt16, JDBCType.SMALLINT); map.put(ClickHouseDataType.Int32, JDBCType.INTEGER); - map.put(ClickHouseDataType.UInt32, JDBCType.BIGINT); map.put(ClickHouseDataType.Int64, JDBCType.BIGINT); map.put(ClickHouseDataType.Float32, JDBCType.FLOAT); map.put(ClickHouseDataType.Float64, JDBCType.DOUBLE); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java index fbf6f16b4..cf1c8e093 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java @@ -92,7 +92,7 @@ public void testIntegerTypes() throws SQLException { long int64 = rand.nextLong(); BigInteger int128 = new BigInteger(127, rand); BigInteger int256 = new BigInteger(255, rand); - int uint8 = rand.nextInt(256); + Short uint8 = Integer.valueOf(rand.nextInt(256)).shortValue(); int uint16 = rand.nextInt(65536); long uint32 = rand.nextInt() & 0xFFFFFFFFL; BigInteger uint64 = BigInteger.valueOf(rand.nextLong(Long.MAX_VALUE)); @@ -179,7 +179,7 @@ public void testIntegerTypes() throws SQLException { assertEquals(rs.getObject("int64"), -9223372036854775808L); assertEquals(rs.getObject("int128"), new BigInteger("-170141183460469231731687303715884105728")); assertEquals(rs.getObject("int256"), new BigInteger("-57896044618658097711785492504343953926634992332820282019728792003956564819968")); - assertEquals(rs.getObject("uint8"), 0); + assertEquals(rs.getObject("uint8"), Short.valueOf("0")); assertEquals(rs.getObject("uint16"), 0); assertEquals(rs.getObject("uint32"), 0L); assertEquals(rs.getObject("uint64"), new BigInteger("0")); @@ -193,7 +193,7 @@ public void testIntegerTypes() throws SQLException { assertEquals(rs.getObject("int64"), 9223372036854775807L); assertEquals(rs.getObject("int128"), new BigInteger("170141183460469231731687303715884105727")); assertEquals(rs.getObject("int256"), new BigInteger("57896044618658097711785492504343953926634992332820282019728792003956564819967")); - assertEquals(rs.getObject("uint8"), 255); + assertEquals(rs.getObject("uint8"), Short.valueOf("255")); assertEquals(rs.getObject("uint16"), 65535); assertEquals(rs.getObject("uint32"), 4294967295L); assertEquals(rs.getObject("uint64"), new BigInteger("18446744073709551615"));