From 771441ebc98eb86cdbfeba2fd2001eb5225fc51a Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 11 Aug 2025 21:16:57 -0700 Subject: [PATCH 1/8] added sub-column logic to JSON columns. added test for JSON with different definition --- .../com/clickhouse/data/ClickHouseColumn.java | 16 +++++++++ .../client/datatypes/DataTypeTests.java | 35 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java index 2ae64bf99..b642c1110 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java @@ -72,6 +72,7 @@ public final class ClickHouseColumn implements Serializable { private static final String KEYWORD_MAP = ClickHouseDataType.Map.name(); private static final String KEYWORD_NESTED = ClickHouseDataType.Nested.name(); private static final String KEYWORD_VARIANT = ClickHouseDataType.Variant.name(); + private static final String KEYWORD_JSON = ClickHouseDataType.JSON.name(); private int columnCount; private int columnIndex; @@ -504,6 +505,21 @@ protected static int readColumn(String args, int startIndex, int len, String nam } } } + } else if (args.startsWith(KEYWORD_JSON, i)) { + int index = args.indexOf('(', i + KEYWORD_JSON.length()); + if (index < i) { + throw new IllegalArgumentException(ERROR_MISSING_NESTED_TYPE); + } + i = ClickHouseUtils.skipBrackets(args, index, len, '('); + String originalTypeName = args.substring(startIndex, i); + List nestedColumns = parse(args.substring(index + 1, i - 1)); + if (nestedColumns.isEmpty()) { + throw new IllegalArgumentException("Nested should have at least one nested column"); + } + column = new ClickHouseColumn(ClickHouseDataType.JSON, name, originalTypeName, nullable, lowCardinality, + null, nestedColumns); + fixedLength = false; + estimatedLength++; } if (column == null) { diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index b829c63a8..1836f4912 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -7,11 +7,14 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.DataTypeUtils; import com.clickhouse.client.api.command.CommandSettings; +import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader; +import com.clickhouse.client.api.data_formats.internal.SerializerUtils; import com.clickhouse.client.api.enums.Protocol; import com.clickhouse.client.api.insert.InsertSettings; import com.clickhouse.client.api.metadata.TableSchema; import com.clickhouse.client.api.query.GenericRecord; import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.client.api.sql.SQLUtils; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseVersion; import lombok.AllArgsConstructor; @@ -874,6 +877,38 @@ private void testVariantWith(String withWhat, String[] fields, Object[] values, } } + @Test(groups = {"integration"}, dataProvider = "testJSONBinaryFormat_dp") + public void testJSONBinaryFormat(String jsonDef) throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + final String table = "test_json_binary_format"; + final String jsonCol = "value " + jsonDef; + final String jsonValue = "{\"count\": 1000, \"stat\": {\"float\": 0.999, \"name\": \"temp\" }}"; + + client.execute("DROP TABLE IF EXISTS " + table).get().close(); + client.execute(tableDefinition(table, jsonCol)).get().close(); + client.execute("INSERT INTO " + table + " VALUES (" + SQLUtils.enquoteLiteral(jsonValue) + ")").get().close(); + + try (QueryResponse queryResponse = client.query("SELECT * FROM " + table + " LIMIT 1").get()) { + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(queryResponse); + Map row = reader.next(); + Object value = row.get("value"); + System.out.println(value); + } + } + + @DataProvider + public Object[][] testJSONBinaryFormat_dp() { + + return new Object[][] { + {"JSON"}, +// {"JSON(a Int32, d String)"}, + {"JSON(stat.name String, count Int32)"}, + }; + } + public static String tableDefinition(String table, String... columns) { StringBuilder sb = new StringBuilder(); sb.append("CREATE TABLE " + table + " ( "); From df769ccaac63fd058376bc0bb0177a4629b76df0 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 13 Aug 2025 14:17:41 -0700 Subject: [PATCH 2/8] implemented reading JSON columns with different definitions --- .../com/clickhouse/data/ClickHouseColumn.java | 68 +++++++++++++++---- .../internal/BinaryStreamReader.java | 12 +++- .../client/datatypes/DataTypeTests.java | 6 +- 3 files changed, 68 insertions(+), 18 deletions(-) diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java index b642c1110..0a744b86d 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java @@ -37,11 +37,9 @@ import java.io.Serializable; import java.lang.reflect.Array; -import java.math.BigInteger; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -507,19 +505,19 @@ protected static int readColumn(String args, int startIndex, int len, String nam } } else if (args.startsWith(KEYWORD_JSON, i)) { int index = args.indexOf('(', i + KEYWORD_JSON.length()); - if (index < i) { - throw new IllegalArgumentException(ERROR_MISSING_NESTED_TYPE); - } - i = ClickHouseUtils.skipBrackets(args, index, len, '('); - String originalTypeName = args.substring(startIndex, i); - List nestedColumns = parse(args.substring(index + 1, i - 1)); - if (nestedColumns.isEmpty()) { - throw new IllegalArgumentException("Nested should have at least one nested column"); + if (index > i) { + i = ClickHouseUtils.skipBrackets(args, index, len, '('); + String originalTypeName = args.substring(startIndex, i); + List params = new ArrayList<>(); + List nestedColumns = new ArrayList<>(); + List parameters = new ArrayList<>(); + parseJSONColumn(args.substring(index + 1, i - 1), nestedColumns, parameters); + nestedColumns.sort(Comparator.comparing(o -> o.getDataType().name())); + column = new ClickHouseColumn(ClickHouseDataType.JSON, name, originalTypeName, nullable, lowCardinality, + parameters, nestedColumns); + fixedLength = false; + estimatedLength++; } - column = new ClickHouseColumn(ClickHouseDataType.JSON, name, originalTypeName, nullable, lowCardinality, - null, nestedColumns); - fixedLength = false; - estimatedLength++; } if (column == null) { @@ -674,6 +672,48 @@ public static List parse(String args) { return Collections.unmodifiableList(c); } + public static final String JSON_MAX_PATHS_PARAM = "max_dynamic_paths"; + public static final String JSON_MAX_DYN_TYPES_PARAM = "max_dynamic_types"; + public static final String JSON_SKIP_MARKER = "SKIP"; + + public static void parseJSONColumn(String args, List nestedColumns, List parameters) { + if (args == null || args.isEmpty()) { + return; + } + + String name = null; + ClickHouseColumn column = null; + StringBuilder builder = new StringBuilder(); + for (int i = 0, len = args.length(); i < len; i++) { + char ch = args.charAt(i); + if (Character.isWhitespace(ch)) { + continue; + } + + if (name == null) { // column name + i = ClickHouseUtils.readNameOrQuotedString(args, i, len, builder) - 1; + name = builder.toString(); + if (name.startsWith(JSON_SKIP_MARKER)) { + name = null; // skip parameters + i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1; + } else if ( name.startsWith(JSON_MAX_PATHS_PARAM) || name.startsWith(JSON_MAX_DYN_TYPES_PARAM)) { + parameters.add(name); + name = null; + i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1; + } + builder.setLength(0); + } else if (column == null) { // now type + LinkedList colList = new LinkedList<>(); + i = readColumn(args, i, len, name, colList) - 1; + nestedColumns.add(column = colList.getFirst()); + } else { // prepare for next column + i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1; + name = null; + column = null; + } + } + } + public ClickHouseColumn(ClickHouseDataType dataType, String columnName, String originalTypeName, boolean nullable, boolean lowCardinality, List parameters, List nestedColumns) { this(dataType, columnName, originalTypeName, nullable, lowCardinality, parameters, nestedColumns, ClickHouseEnum.EMPTY); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 6ee472456..2000089e5 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -35,6 +35,8 @@ import java.util.Set; import java.util.TimeZone; import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; /** * This class is not thread safe and should not be shared between multiple threads. @@ -226,7 +228,7 @@ public T readValue(ClickHouseColumn column, Class typeHint) throws IOExce if (jsonAsString) { return (T) readString(input); } else { - return (T) readJsonData(input); + return (T) readJsonData(input, actualColumn); } // case Object: // deprecated https://clickhouse.com/docs/en/sql-reference/data-types/object-data-type case Array: @@ -1192,16 +1194,20 @@ private ClickHouseColumn readDynamicData() throws IOException { private static final ClickHouseColumn JSON_PLACEHOLDER_COL = ClickHouseColumn.parse("v Dynamic").get(0); - private Map readJsonData(InputStream input) throws IOException { + private Map readJsonData(InputStream input, ClickHouseColumn column) throws IOException { int numOfPaths = readVarInt(input); if (numOfPaths == 0) { return Collections.emptyMap(); } Map obj = new HashMap<>(); + + final int predefinedPaths = column.getNestedColumns().size(); + final List predefinedColumns = column.getNestedColumns(); for (int i = 0; i < numOfPaths; i++) { String path = readString(input); - Object value = readValue(JSON_PLACEHOLDER_COL); + ClickHouseColumn dataColumn = i < predefinedPaths ? predefinedColumns.get(i) : JSON_PLACEHOLDER_COL; + Object value = readValue(dataColumn); obj.put(path, value); } return obj; diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index 1836f4912..b524db488 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -904,8 +904,12 @@ public Object[][] testJSONBinaryFormat_dp() { return new Object[][] { {"JSON"}, -// {"JSON(a Int32, d String)"}, + {"JSON()"}, {"JSON(stat.name String, count Int32)"}, + {"JSON(stat.name String, `comments` String)"}, + {"JSON(max_dynamic_paths=3, stat.name String, SKIP alt_count)"}, + {"JSON(max_dynamic_paths=3, stat.name String, SKIP REGEXP '^-.*')"}, + {"JSON(max_dynamic_paths=3,SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)"}, }; } From 97c551759d00d8139199b3c8b23042555572672a Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 13 Aug 2025 15:22:15 -0700 Subject: [PATCH 3/8] fix cloud tests --- .../java/com/clickhouse/client/datatypes/DataTypeTests.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index b524db488..19e64f316 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -888,7 +888,10 @@ public void testJSONBinaryFormat(String jsonDef) throws Exception { final String jsonValue = "{\"count\": 1000, \"stat\": {\"float\": 0.999, \"name\": \"temp\" }}"; client.execute("DROP TABLE IF EXISTS " + table).get().close(); - client.execute(tableDefinition(table, jsonCol)).get().close(); + client.execute(tableDefinition(table, jsonCol), + (CommandSettings) new CommandSettings() + .serverSetting("enable_json_type", "1") + .serverSetting("allow_experimental_json_type", "1")).get().close(); client.execute("INSERT INTO " + table + " VALUES (" + SQLUtils.enquoteLiteral(jsonValue) + ")").get().close(); try (QueryResponse queryResponse = client.query("SELECT * FROM " + table + " LIMIT 1").get()) { From 37ec609296649d3905c799e7c5c94e1782727ac3 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 13 Aug 2025 15:44:37 -0700 Subject: [PATCH 4/8] adding more unit test to make sonar happy --- .../com/clickhouse/data/ClickHouseColumn.java | 1 - .../clickhouse/data/ClickHouseColumnTest.java | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java index 0a744b86d..ae8149b9c 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java @@ -508,7 +508,6 @@ protected static int readColumn(String args, int startIndex, int len, String nam if (index > i) { i = ClickHouseUtils.skipBrackets(args, index, len, '('); String originalTypeName = args.substring(startIndex, i); - List params = new ArrayList<>(); List nestedColumns = new ArrayList<>(); List parameters = new ArrayList<>(); parseJSONColumn(args.substring(index + 1, i - 1), nestedColumns, parameters); diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java index a04f42d32..886ce9027 100644 --- a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java @@ -1,9 +1,11 @@ package com.clickhouse.data; import java.math.BigInteger; +import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Map; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -441,4 +443,26 @@ public boolean isWidenUnsignedTypes() { } } } + + @Test(groups = {"unit"}, dataProvider = "testJSONBinaryFormat_dp") + public void testJSONBinaryFormat(String jsonDef, int params, List predefinedPaths) throws Exception { + ClickHouseColumn column = ClickHouseColumn.of("v", jsonDef); + Assert.assertEquals(column.getNestedColumns().size(), predefinedPaths.size(), "predefined paths count mismatch"); + Assert.assertEquals(column.getParameters().size(), params, "parameters count mismatch"); + } + + @DataProvider + public Object[][] testJSONBinaryFormat_dp() { + + return new Object[][] { + {"JSON", 0, Collections.emptyList()}, + {"JSON()", 0, Collections.emptyList()}, + {"JSON(stat.name String, count Int32)", 0, Arrays.asList("stat.name", "count")}, + {"JSON(stat.name String, `comments` String)", 0, Arrays.asList("stat.name", "comments")}, + {"JSON(max_dynamic_paths=3, stat.name String, count Int8, SKIP alt_count)", 1, Arrays.asList("stat.name", "count")}, + {"JSON(max_dynamic_paths=3, stat.name String, SKIP REGEXP '^-.*')", 1, Arrays.asList("stat.name")}, + {"JSON(max_dynamic_paths=3,SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)", 1, Arrays.asList("flags")}, + {"JSON(max_dynamic_types=3,max_dynamic_paths=3, SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)", 2, Arrays.asList("flags")}, + }; + } } From 6ace7723bbf99e455f65094ee754501044e02ea1 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 13 Aug 2025 17:08:13 -0700 Subject: [PATCH 5/8] added temp check to debug cloud tests --- .../client/api/data_formats/internal/BinaryStreamReader.java | 4 ++++ .../java/com/clickhouse/client/datatypes/DataTypeTests.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 2000089e5..5a3caf3bf 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -1,6 +1,7 @@ package com.clickhouse.client.api.data_formats.internal; import com.clickhouse.client.api.ClientException; +import com.clickhouse.client.api.DataTransferException; import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseEnum; @@ -1207,6 +1208,9 @@ private Map readJsonData(InputStream input, ClickHouseColumn col for (int i = 0; i < numOfPaths; i++) { String path = readString(input); ClickHouseColumn dataColumn = i < predefinedPaths ? predefinedColumns.get(i) : JSON_PLACEHOLDER_COL; + if (dataColumn != JSON_PLACEHOLDER_COL && !dataColumn.getColumnName().equals(path)) { + throw new DataTransferException("Wrong column at position " + i + " where path " + path + " expected"); + } Object value = readValue(dataColumn); obj.put(path, value); } diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index 19e64f316..3b66d4c7c 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -898,7 +898,7 @@ public void testJSONBinaryFormat(String jsonDef) throws Exception { ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(queryResponse); Map row = reader.next(); Object value = row.get("value"); - System.out.println(value); + Assert.assertNotNull(value); } } From 74bb966461b617872658b8c596ea591eb5b497b9 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 13 Aug 2025 17:31:05 -0700 Subject: [PATCH 6/8] removed optimization :-( --- .../main/java/com/clickhouse/data/ClickHouseColumn.java | 9 +++++++++ .../api/data_formats/internal/BinaryStreamReader.java | 9 +++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java index ae8149b9c..8c64309d4 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java @@ -49,6 +49,7 @@ import java.util.Objects; import java.util.Set; import java.util.TimeZone; +import java.util.stream.Collectors; /** * This class represents a column defined in database. @@ -89,6 +90,7 @@ public final class ClickHouseColumn implements Serializable { private List nested; private List parameters; private ClickHouseEnum enumConstants; + private Map jsonPredefinedPaths; private int arrayLevel; private ClickHouseColumn arrayBaseColumn; @@ -509,11 +511,14 @@ protected static int readColumn(String args, int startIndex, int len, String nam i = ClickHouseUtils.skipBrackets(args, index, len, '('); String originalTypeName = args.substring(startIndex, i); List nestedColumns = new ArrayList<>(); + List parameters = new ArrayList<>(); parseJSONColumn(args.substring(index + 1, i - 1), nestedColumns, parameters); nestedColumns.sort(Comparator.comparing(o -> o.getDataType().name())); column = new ClickHouseColumn(ClickHouseDataType.JSON, name, originalTypeName, nullable, lowCardinality, parameters, nestedColumns); + column.jsonPredefinedPaths = nestedColumns.stream().collect(Collectors.toMap(ClickHouseColumn::getColumnName, + c -> c)); fixedLength = false; estimatedLength++; } @@ -1009,6 +1014,10 @@ public ClickHouseAggregateFunction getAggregateFunction() { return aggFuncType; } + public Map getJsonPredefinedPaths() { + return jsonPredefinedPaths; + } + public ClickHouseArraySequence newArrayValue(ClickHouseDataConfig config) { int level = arrayLevel; ClickHouseArraySequence value; diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 5a3caf3bf..2309950e1 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -1203,14 +1203,11 @@ private Map readJsonData(InputStream input, ClickHouseColumn col Map obj = new HashMap<>(); - final int predefinedPaths = column.getNestedColumns().size(); - final List predefinedColumns = column.getNestedColumns(); + final Map predefinedColumns = column.getJsonPredefinedPaths(); for (int i = 0; i < numOfPaths; i++) { String path = readString(input); - ClickHouseColumn dataColumn = i < predefinedPaths ? predefinedColumns.get(i) : JSON_PLACEHOLDER_COL; - if (dataColumn != JSON_PLACEHOLDER_COL && !dataColumn.getColumnName().equals(path)) { - throw new DataTransferException("Wrong column at position " + i + " where path " + path + " expected"); - } + ClickHouseColumn dataColumn = predefinedColumns == null? JSON_PLACEHOLDER_COL : + predefinedColumns.getOrDefault(path, JSON_PLACEHOLDER_COL); Object value = readValue(dataColumn); obj.put(path, value); } From 70a905a35887d23df7a35d26b70209d3c971ff30 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 13 Aug 2025 20:12:04 -0700 Subject: [PATCH 7/8] changed for loop to while (just to make sonar happy). removed unused imports and undeprecated ClickHouseUtils as we use it --- .../java/com/clickhouse/data/ClickHouseColumn.java | 10 ++++++++-- .../main/java/com/clickhouse/data/ClickHouseUtils.java | 1 - .../api/data_formats/internal/BinaryStreamReader.java | 3 --- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java index 8c64309d4..0b5f4e28a 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java @@ -688,9 +688,12 @@ public static void parseJSONColumn(String args, List nestedCol String name = null; ClickHouseColumn column = null; StringBuilder builder = new StringBuilder(); - for (int i = 0, len = args.length(); i < len; i++) { + int i =0; + int len = args.length(); + while (i < len) { char ch = args.charAt(i); if (Character.isWhitespace(ch)) { + i++; continue; } @@ -709,12 +712,15 @@ public static void parseJSONColumn(String args, List nestedCol } else if (column == null) { // now type LinkedList colList = new LinkedList<>(); i = readColumn(args, i, len, name, colList) - 1; - nestedColumns.add(column = colList.getFirst()); + column = colList.getFirst(); + nestedColumns.add(column); } else { // prepare for next column i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1; name = null; column = null; } + + i++; } } diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java index d47824eb0..d3f08d4a8 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java @@ -51,7 +51,6 @@ import java.util.function.Supplier; import java.util.function.UnaryOperator; -@Deprecated public final class ClickHouseUtils { private static final boolean IS_UNIX; private static final boolean IS_WINDOWS; diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 2309950e1..c2a8a1ef0 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -1,7 +1,6 @@ package com.clickhouse.client.api.data_formats.internal; import com.clickhouse.client.api.ClientException; -import com.clickhouse.client.api.DataTransferException; import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseEnum; @@ -36,8 +35,6 @@ import java.util.Set; import java.util.TimeZone; import java.util.UUID; -import java.util.function.Function; -import java.util.stream.Collectors; /** * This class is not thread safe and should not be shared between multiple threads. From 1bc5b01a0689e70d7f6f2da70d584faa949e4bfa Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 13 Aug 2025 21:07:48 -0700 Subject: [PATCH 8/8] added JDBC test --- .../com/clickhouse/jdbc/DataTypeTests.java | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java index 41e5b5aa5..ad9400028 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java @@ -3,6 +3,7 @@ import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.DataTypeUtils; import com.clickhouse.client.api.internal.ServerSettings; +import com.clickhouse.client.api.sql.SQLUtils; import com.clickhouse.data.ClickHouseVersion; import com.clickhouse.data.Tuple; import org.slf4j.Logger; @@ -47,6 +48,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; @@ -1372,7 +1374,32 @@ public void testJSONWritingAsString() throws SQLException { } } - @Test(groups = { "integration" }, enabled = false) + @Test(groups = { "integration" }) + public void testReadingJSONBinary() throws SQLException { + if (ClickHouseVersion.of(getServerVersion()).check("(,24.8]")) { + return; // JSON was introduced in 24.10 + } + + Properties properties = new Properties(); + properties.put(ClientConfigProperties.serverSetting("allow_experimental_json_type"), "1"); + try (Connection conn = getJdbcConnection(properties); + Statement stmt = conn.createStatement()) { + + final String json = "{\"count\": 1000, \"event\": { \"name\": \"start\", \"value\": 0.10} }"; + String sql = String.format("SELECT %1$s::JSON(), %1$s::JSON(count Int16)", SQLUtils.enquoteLiteral(json)); + try (ResultSet rs = stmt.executeQuery(sql)) { + rs.next(); + + Map val1 = (Map) rs.getObject(1); + assertEquals(val1.get("count"), 1000L); + Map val2 = (Map) rs.getObject(2); + assertEquals(val2.get("count"), (short)1000); + } + } + } + + + @Test(groups = { "integration" }, enabled = false) public void testGeometricTypesSimpleStatement() throws SQLException { // TODO: add LineString and MultiLineString support runQuery("CREATE TABLE test_geometric (order Int8, "