[client-v2] Adding JSON support with predefined paths

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java

@@ -37,11 +37,9 @@
 
 import java.io.Serializable;
 import java.lang.reflect.Array;
-import java.math.BigInteger;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -51,6 +49,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.TimeZone;
+import java.util.stream.Collectors;
 
 /**
  * This class represents a column defined in database.
@@ -72,6 +71,7 @@ public final class ClickHouseColumn implements Serializable {
     private static final String KEYWORD_MAP = ClickHouseDataType.Map.name();
     private static final String KEYWORD_NESTED = ClickHouseDataType.Nested.name();
     private static final String KEYWORD_VARIANT = ClickHouseDataType.Variant.name();
+    private static final String KEYWORD_JSON = ClickHouseDataType.JSON.name();
 
     private int columnCount;
     private int columnIndex;
@@ -90,6 +90,7 @@ public final class ClickHouseColumn implements Serializable {
     private List<ClickHouseColumn> nested;
     private List<String> parameters;
     private ClickHouseEnum enumConstants;
+    private Map<String, ClickHouseColumn> jsonPredefinedPaths;
 
     private int arrayLevel;
     private ClickHouseColumn arrayBaseColumn;
@@ -504,6 +505,23 @@ protected static int readColumn(String args, int startIndex, int len, String nam
                     }
                 }
             }
+        } else if (args.startsWith(KEYWORD_JSON, i)) {
+            int index = args.indexOf('(', i + KEYWORD_JSON.length());
+            if (index > i) {
+                i = ClickHouseUtils.skipBrackets(args, index, len, '(');
+                String originalTypeName = args.substring(startIndex, i);
+                List<ClickHouseColumn> nestedColumns = new ArrayList<>();
+
+                List<String> parameters = new ArrayList<>();
+                parseJSONColumn(args.substring(index + 1, i - 1), nestedColumns, parameters);
+                nestedColumns.sort(Comparator.comparing(o -> o.getDataType().name()));
+                column = new ClickHouseColumn(ClickHouseDataType.JSON, name, originalTypeName, nullable, lowCardinality,
+                        parameters, nestedColumns);
+                column.jsonPredefinedPaths = nestedColumns.stream().collect(Collectors.toMap(ClickHouseColumn::getColumnName,
+                        c -> c));
+                fixedLength = false;
+                estimatedLength++;
+            }
         }
 
         if (column == null) {
@@ -658,6 +676,54 @@ public static List<ClickHouseColumn> parse(String args) {
         return Collections.unmodifiableList(c);
     }
 
+    public static final String JSON_MAX_PATHS_PARAM = "max_dynamic_paths";
+    public static final String JSON_MAX_DYN_TYPES_PARAM = "max_dynamic_types";
+    public static final String JSON_SKIP_MARKER = "SKIP";
+
+    public static void parseJSONColumn(String args, List<ClickHouseColumn> nestedColumns, List<String> parameters) {
+        if (args == null || args.isEmpty()) {
+            return;
+        }
+
+        String name = null;
+        ClickHouseColumn column = null;
+        StringBuilder builder = new StringBuilder();
+        int i =0;
+        int len = args.length();
+        while (i < len) {
+            char ch = args.charAt(i);
+            if (Character.isWhitespace(ch)) {
+                i++;
+                continue;
+            }
+
+            if (name == null) { // column name
+                i = ClickHouseUtils.readNameOrQuotedString(args, i, len, builder) - 1;
+                name = builder.toString();
+                if (name.startsWith(JSON_SKIP_MARKER)) {
+                    name = null; // skip parameters
+                    i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
+                } else if ( name.startsWith(JSON_MAX_PATHS_PARAM) || name.startsWith(JSON_MAX_DYN_TYPES_PARAM)) {
+                    parameters.add(name);
+                    name = null;
+                    i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
+                }
+                builder.setLength(0);
+            } else if (column == null) { // now type
+                LinkedList<ClickHouseColumn> colList = new LinkedList<>();
+                i = readColumn(args, i, len, name, colList) - 1;
+                column = colList.getFirst();
+                nestedColumns.add(column);
+            } else { // prepare for next column
+                i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
+                name = null;
+                column = null;
+            }
+
+            i++;
+        }
+    }
+
     public ClickHouseColumn(ClickHouseDataType dataType, String columnName, String originalTypeName, boolean nullable,
             boolean lowCardinality, List<String> parameters, List<ClickHouseColumn> nestedColumns) {
         this(dataType, columnName, originalTypeName, nullable, lowCardinality, parameters, nestedColumns, ClickHouseEnum.EMPTY);
@@ -954,6 +1020,10 @@ public ClickHouseAggregateFunction getAggregateFunction() {
         return aggFuncType;
     }
 
+    public Map<String, ClickHouseColumn> getJsonPredefinedPaths() {
+        return jsonPredefinedPaths;
+    }
+
     public ClickHouseArraySequence newArrayValue(ClickHouseDataConfig config) {
         int level = arrayLevel;
         ClickHouseArraySequence value;

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java

@@ -51,7 +51,6 @@
 import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 
-@Deprecated
 public final class ClickHouseUtils {
     private static final boolean IS_UNIX;
     private static final boolean IS_WINDOWS;

clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java

@@ -1,9 +1,11 @@
 package com.clickhouse.data;
 
 import java.math.BigInteger;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 
 import org.testng.Assert;
 import org.testng.annotations.DataProvider;
@@ -441,4 +443,26 @@ public boolean isWidenUnsignedTypes() {
             }
         }
     }
+
+    @Test(groups = {"unit"}, dataProvider = "testJSONBinaryFormat_dp")
+    public void testJSONBinaryFormat(String jsonDef, int params, List<String> predefinedPaths) throws Exception {
+        ClickHouseColumn column = ClickHouseColumn.of("v", jsonDef);
+        Assert.assertEquals(column.getNestedColumns().size(), predefinedPaths.size(), "predefined paths count mismatch");
+        Assert.assertEquals(column.getParameters().size(), params, "parameters count mismatch");
+    }
+
+    @DataProvider
+    public Object[][] testJSONBinaryFormat_dp() {
+
+        return new Object[][] {
+                {"JSON", 0, Collections.emptyList()},
+                {"JSON()", 0, Collections.emptyList()},
+                {"JSON(stat.name String, count Int32)", 0, Arrays.asList("stat.name", "count")},
+                {"JSON(stat.name String, `comments` String)", 0,  Arrays.asList("stat.name", "comments")},
+                {"JSON(max_dynamic_paths=3, stat.name String, count Int8, SKIP alt_count)", 1,  Arrays.asList("stat.name", "count")},
+                {"JSON(max_dynamic_paths=3, stat.name String, SKIP REGEXP '^-.*')", 1,  Arrays.asList("stat.name")},
+                {"JSON(max_dynamic_paths=3,SKIP REGEXP '^-.*',SKIP ff,   flags Array(Array(Array(Int8))), SKIP alt_count)", 1, Arrays.asList("flags")},
+                {"JSON(max_dynamic_types=3,max_dynamic_paths=3, SKIP REGEXP '^-.*',SKIP ff,   flags Array(Array(Array(Int8))), SKIP alt_count)", 2, Arrays.asList("flags")},
+        };
+    }
 }

client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java

@@ -226,7 +226,7 @@ public <T> T readValue(ClickHouseColumn column, Class<?> typeHint) throws IOExce
                     if (jsonAsString) {
                         return (T) readString(input);
                     } else {
-                        return (T) readJsonData(input);
+                        return (T) readJsonData(input, actualColumn);
                     }
 //                case Object: // deprecated https://clickhouse.com/docs/en/sql-reference/data-types/object-data-type
                 case Array:
@@ -1192,16 +1192,20 @@ private ClickHouseColumn readDynamicData() throws IOException {
 
     private static final ClickHouseColumn JSON_PLACEHOLDER_COL = ClickHouseColumn.parse("v Dynamic").get(0);
 
-    private Map<String, Object> readJsonData(InputStream input) throws IOException {
+    private Map<String, Object> readJsonData(InputStream input, ClickHouseColumn column) throws IOException {
         int numOfPaths = readVarInt(input);
         if (numOfPaths == 0) {
             return Collections.emptyMap();
         }
 
         Map<String, Object> obj = new HashMap<>();
+
+        final Map<String, ClickHouseColumn> predefinedColumns = column.getJsonPredefinedPaths();
         for (int i = 0; i < numOfPaths; i++) {
             String path = readString(input);
-            Object value = readValue(JSON_PLACEHOLDER_COL);
+            ClickHouseColumn dataColumn = predefinedColumns == null? JSON_PLACEHOLDER_COL :
+                    predefinedColumns.getOrDefault(path, JSON_PLACEHOLDER_COL);
+            Object value = readValue(dataColumn);
             obj.put(path, value);
         }
         return obj;

client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java

@@ -7,11 +7,14 @@
 import com.clickhouse.client.api.Client;
 import com.clickhouse.client.api.DataTypeUtils;
 import com.clickhouse.client.api.command.CommandSettings;
+import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
+import com.clickhouse.client.api.data_formats.internal.SerializerUtils;
 import com.clickhouse.client.api.enums.Protocol;
 import com.clickhouse.client.api.insert.InsertSettings;
 import com.clickhouse.client.api.metadata.TableSchema;
 import com.clickhouse.client.api.query.GenericRecord;
 import com.clickhouse.client.api.query.QueryResponse;
+import com.clickhouse.client.api.sql.SQLUtils;
 import com.clickhouse.data.ClickHouseDataType;
 import com.clickhouse.data.ClickHouseVersion;
 import lombok.AllArgsConstructor;
@@ -874,6 +877,45 @@ private void testVariantWith(String withWhat, String[] fields, Object[] values,
         }
     }
 
+    @Test(groups = {"integration"}, dataProvider = "testJSONBinaryFormat_dp")
+    public void testJSONBinaryFormat(String jsonDef) throws Exception {
+        if (isVersionMatch("(,24.8]")) {
+            return;
+        }
+
+        final String table = "test_json_binary_format";
+        final String jsonCol = "value " + jsonDef;
+        final String jsonValue = "{\"count\": 1000, \"stat\": {\"float\": 0.999, \"name\": \"temp\" }}";
+
+        client.execute("DROP TABLE IF EXISTS " + table).get().close();
+        client.execute(tableDefinition(table, jsonCol),
+                (CommandSettings) new CommandSettings()
+                        .serverSetting("enable_json_type", "1")
+                        .serverSetting("allow_experimental_json_type", "1")).get().close();
+        client.execute("INSERT INTO " + table + " VALUES (" + SQLUtils.enquoteLiteral(jsonValue) + ")").get().close();
+
+        try (QueryResponse  queryResponse = client.query("SELECT * FROM " + table + " LIMIT 1").get()) {
+            ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(queryResponse);
+            Map<String, Object> row = reader.next();
+            Object value = row.get("value");
+            Assert.assertNotNull(value);
+        }
+    }
+
+    @DataProvider
+    public Object[][] testJSONBinaryFormat_dp() {
+
+        return new Object[][] {
+                {"JSON"},
+                {"JSON()"},
+                {"JSON(stat.name String, count Int32)"},
+                {"JSON(stat.name String, `comments` String)"},
+                {"JSON(max_dynamic_paths=3, stat.name String, SKIP alt_count)"},
+                {"JSON(max_dynamic_paths=3, stat.name String, SKIP REGEXP '^-.*')"},
+                {"JSON(max_dynamic_paths=3,SKIP REGEXP '^-.*',SKIP ff,   flags Array(Array(Array(Int8))), SKIP alt_count)"},
+        };
+    }
+
     public static String tableDefinition(String table, String... columns) {
         StringBuilder sb = new StringBuilder();
         sb.append("CREATE TABLE " + table + " ( ");

jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java

@@ -3,6 +3,7 @@
 import com.clickhouse.client.api.ClientConfigProperties;
 import com.clickhouse.client.api.DataTypeUtils;
 import com.clickhouse.client.api.internal.ServerSettings;
+import com.clickhouse.client.api.sql.SQLUtils;
 import com.clickhouse.data.ClickHouseVersion;
 import com.clickhouse.data.Tuple;
 import org.slf4j.Logger;
@@ -47,6 +48,7 @@
 
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotEquals;
 import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertThrows;
 import static org.testng.Assert.assertTrue;
@@ -1372,7 +1374,32 @@ public void testJSONWritingAsString() throws SQLException {
         }
     }
 
-    @Test(groups = { "integration" }, enabled = false)
+    @Test(groups = { "integration" })
+    public void testReadingJSONBinary() throws SQLException {
+        if (ClickHouseVersion.of(getServerVersion()).check("(,24.8]")) {
+            return; // JSON was introduced in 24.10
+        }
+
+        Properties properties = new Properties();
+        properties.put(ClientConfigProperties.serverSetting("allow_experimental_json_type"), "1");
+        try (Connection conn = getJdbcConnection(properties);
+             Statement stmt = conn.createStatement()) {
+
+            final String json = "{\"count\": 1000, \"event\": { \"name\": \"start\", \"value\": 0.10} }";
+            String sql = String.format("SELECT %1$s::JSON(), %1$s::JSON(count Int16)", SQLUtils.enquoteLiteral(json));
+            try (ResultSet rs = stmt.executeQuery(sql)) {
+                rs.next();
+
+                Map<String, Object> val1 = (Map<String, Object>) rs.getObject(1);
+                assertEquals(val1.get("count"), 1000L);
+                Map<String, Object> val2 = (Map<String, Object>) rs.getObject(2);
+                assertEquals(val2.get("count"), (short)1000);
+            }
+        }
+    }
+
+
+        @Test(groups = { "integration" }, enabled = false)
     public void testGeometricTypesSimpleStatement() throws SQLException {
         // TODO: add LineString and MultiLineString support
         runQuery("CREATE TABLE test_geometric (order Int8, "