diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java index 2ae64bf99..0b5f4e28a 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java @@ -37,11 +37,9 @@ import java.io.Serializable; import java.lang.reflect.Array; -import java.math.BigInteger; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -51,6 +49,7 @@ import java.util.Objects; import java.util.Set; import java.util.TimeZone; +import java.util.stream.Collectors; /** * This class represents a column defined in database. @@ -72,6 +71,7 @@ public final class ClickHouseColumn implements Serializable { private static final String KEYWORD_MAP = ClickHouseDataType.Map.name(); private static final String KEYWORD_NESTED = ClickHouseDataType.Nested.name(); private static final String KEYWORD_VARIANT = ClickHouseDataType.Variant.name(); + private static final String KEYWORD_JSON = ClickHouseDataType.JSON.name(); private int columnCount; private int columnIndex; @@ -90,6 +90,7 @@ public final class ClickHouseColumn implements Serializable { private List nested; private List parameters; private ClickHouseEnum enumConstants; + private Map jsonPredefinedPaths; private int arrayLevel; private ClickHouseColumn arrayBaseColumn; @@ -504,6 +505,23 @@ protected static int readColumn(String args, int startIndex, int len, String nam } } } + } else if (args.startsWith(KEYWORD_JSON, i)) { + int index = args.indexOf('(', i + KEYWORD_JSON.length()); + if (index > i) { + i = ClickHouseUtils.skipBrackets(args, index, len, '('); + String originalTypeName = args.substring(startIndex, i); + List nestedColumns = new ArrayList<>(); + + List parameters = new ArrayList<>(); + parseJSONColumn(args.substring(index + 1, i - 1), nestedColumns, parameters); + nestedColumns.sort(Comparator.comparing(o -> o.getDataType().name())); + column = new ClickHouseColumn(ClickHouseDataType.JSON, name, originalTypeName, nullable, lowCardinality, + parameters, nestedColumns); + column.jsonPredefinedPaths = nestedColumns.stream().collect(Collectors.toMap(ClickHouseColumn::getColumnName, + c -> c)); + fixedLength = false; + estimatedLength++; + } } if (column == null) { @@ -658,6 +676,54 @@ public static List parse(String args) { return Collections.unmodifiableList(c); } + public static final String JSON_MAX_PATHS_PARAM = "max_dynamic_paths"; + public static final String JSON_MAX_DYN_TYPES_PARAM = "max_dynamic_types"; + public static final String JSON_SKIP_MARKER = "SKIP"; + + public static void parseJSONColumn(String args, List nestedColumns, List parameters) { + if (args == null || args.isEmpty()) { + return; + } + + String name = null; + ClickHouseColumn column = null; + StringBuilder builder = new StringBuilder(); + int i =0; + int len = args.length(); + while (i < len) { + char ch = args.charAt(i); + if (Character.isWhitespace(ch)) { + i++; + continue; + } + + if (name == null) { // column name + i = ClickHouseUtils.readNameOrQuotedString(args, i, len, builder) - 1; + name = builder.toString(); + if (name.startsWith(JSON_SKIP_MARKER)) { + name = null; // skip parameters + i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1; + } else if ( name.startsWith(JSON_MAX_PATHS_PARAM) || name.startsWith(JSON_MAX_DYN_TYPES_PARAM)) { + parameters.add(name); + name = null; + i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1; + } + builder.setLength(0); + } else if (column == null) { // now type + LinkedList colList = new LinkedList<>(); + i = readColumn(args, i, len, name, colList) - 1; + column = colList.getFirst(); + nestedColumns.add(column); + } else { // prepare for next column + i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1; + name = null; + column = null; + } + + i++; + } + } + public ClickHouseColumn(ClickHouseDataType dataType, String columnName, String originalTypeName, boolean nullable, boolean lowCardinality, List parameters, List nestedColumns) { this(dataType, columnName, originalTypeName, nullable, lowCardinality, parameters, nestedColumns, ClickHouseEnum.EMPTY); @@ -954,6 +1020,10 @@ public ClickHouseAggregateFunction getAggregateFunction() { return aggFuncType; } + public Map getJsonPredefinedPaths() { + return jsonPredefinedPaths; + } + public ClickHouseArraySequence newArrayValue(ClickHouseDataConfig config) { int level = arrayLevel; ClickHouseArraySequence value; diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java index d47824eb0..d3f08d4a8 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java @@ -51,7 +51,6 @@ import java.util.function.Supplier; import java.util.function.UnaryOperator; -@Deprecated public final class ClickHouseUtils { private static final boolean IS_UNIX; private static final boolean IS_WINDOWS; diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java index a04f42d32..886ce9027 100644 --- a/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java @@ -1,9 +1,11 @@ package com.clickhouse.data; import java.math.BigInteger; +import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Map; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -441,4 +443,26 @@ public boolean isWidenUnsignedTypes() { } } } + + @Test(groups = {"unit"}, dataProvider = "testJSONBinaryFormat_dp") + public void testJSONBinaryFormat(String jsonDef, int params, List predefinedPaths) throws Exception { + ClickHouseColumn column = ClickHouseColumn.of("v", jsonDef); + Assert.assertEquals(column.getNestedColumns().size(), predefinedPaths.size(), "predefined paths count mismatch"); + Assert.assertEquals(column.getParameters().size(), params, "parameters count mismatch"); + } + + @DataProvider + public Object[][] testJSONBinaryFormat_dp() { + + return new Object[][] { + {"JSON", 0, Collections.emptyList()}, + {"JSON()", 0, Collections.emptyList()}, + {"JSON(stat.name String, count Int32)", 0, Arrays.asList("stat.name", "count")}, + {"JSON(stat.name String, `comments` String)", 0, Arrays.asList("stat.name", "comments")}, + {"JSON(max_dynamic_paths=3, stat.name String, count Int8, SKIP alt_count)", 1, Arrays.asList("stat.name", "count")}, + {"JSON(max_dynamic_paths=3, stat.name String, SKIP REGEXP '^-.*')", 1, Arrays.asList("stat.name")}, + {"JSON(max_dynamic_paths=3,SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)", 1, Arrays.asList("flags")}, + {"JSON(max_dynamic_types=3,max_dynamic_paths=3, SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)", 2, Arrays.asList("flags")}, + }; + } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 6ee472456..c2a8a1ef0 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -226,7 +226,7 @@ public T readValue(ClickHouseColumn column, Class typeHint) throws IOExce if (jsonAsString) { return (T) readString(input); } else { - return (T) readJsonData(input); + return (T) readJsonData(input, actualColumn); } // case Object: // deprecated https://clickhouse.com/docs/en/sql-reference/data-types/object-data-type case Array: @@ -1192,16 +1192,20 @@ private ClickHouseColumn readDynamicData() throws IOException { private static final ClickHouseColumn JSON_PLACEHOLDER_COL = ClickHouseColumn.parse("v Dynamic").get(0); - private Map readJsonData(InputStream input) throws IOException { + private Map readJsonData(InputStream input, ClickHouseColumn column) throws IOException { int numOfPaths = readVarInt(input); if (numOfPaths == 0) { return Collections.emptyMap(); } Map obj = new HashMap<>(); + + final Map predefinedColumns = column.getJsonPredefinedPaths(); for (int i = 0; i < numOfPaths; i++) { String path = readString(input); - Object value = readValue(JSON_PLACEHOLDER_COL); + ClickHouseColumn dataColumn = predefinedColumns == null? JSON_PLACEHOLDER_COL : + predefinedColumns.getOrDefault(path, JSON_PLACEHOLDER_COL); + Object value = readValue(dataColumn); obj.put(path, value); } return obj; diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index b829c63a8..3b66d4c7c 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -7,11 +7,14 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.DataTypeUtils; import com.clickhouse.client.api.command.CommandSettings; +import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader; +import com.clickhouse.client.api.data_formats.internal.SerializerUtils; import com.clickhouse.client.api.enums.Protocol; import com.clickhouse.client.api.insert.InsertSettings; import com.clickhouse.client.api.metadata.TableSchema; import com.clickhouse.client.api.query.GenericRecord; import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.client.api.sql.SQLUtils; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseVersion; import lombok.AllArgsConstructor; @@ -874,6 +877,45 @@ private void testVariantWith(String withWhat, String[] fields, Object[] values, } } + @Test(groups = {"integration"}, dataProvider = "testJSONBinaryFormat_dp") + public void testJSONBinaryFormat(String jsonDef) throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + final String table = "test_json_binary_format"; + final String jsonCol = "value " + jsonDef; + final String jsonValue = "{\"count\": 1000, \"stat\": {\"float\": 0.999, \"name\": \"temp\" }}"; + + client.execute("DROP TABLE IF EXISTS " + table).get().close(); + client.execute(tableDefinition(table, jsonCol), + (CommandSettings) new CommandSettings() + .serverSetting("enable_json_type", "1") + .serverSetting("allow_experimental_json_type", "1")).get().close(); + client.execute("INSERT INTO " + table + " VALUES (" + SQLUtils.enquoteLiteral(jsonValue) + ")").get().close(); + + try (QueryResponse queryResponse = client.query("SELECT * FROM " + table + " LIMIT 1").get()) { + ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(queryResponse); + Map row = reader.next(); + Object value = row.get("value"); + Assert.assertNotNull(value); + } + } + + @DataProvider + public Object[][] testJSONBinaryFormat_dp() { + + return new Object[][] { + {"JSON"}, + {"JSON()"}, + {"JSON(stat.name String, count Int32)"}, + {"JSON(stat.name String, `comments` String)"}, + {"JSON(max_dynamic_paths=3, stat.name String, SKIP alt_count)"}, + {"JSON(max_dynamic_paths=3, stat.name String, SKIP REGEXP '^-.*')"}, + {"JSON(max_dynamic_paths=3,SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)"}, + }; + } + public static String tableDefinition(String table, String... columns) { StringBuilder sb = new StringBuilder(); sb.append("CREATE TABLE " + table + " ( "); diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java index 41e5b5aa5..ad9400028 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java @@ -3,6 +3,7 @@ import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.DataTypeUtils; import com.clickhouse.client.api.internal.ServerSettings; +import com.clickhouse.client.api.sql.SQLUtils; import com.clickhouse.data.ClickHouseVersion; import com.clickhouse.data.Tuple; import org.slf4j.Logger; @@ -47,6 +48,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; @@ -1372,7 +1374,32 @@ public void testJSONWritingAsString() throws SQLException { } } - @Test(groups = { "integration" }, enabled = false) + @Test(groups = { "integration" }) + public void testReadingJSONBinary() throws SQLException { + if (ClickHouseVersion.of(getServerVersion()).check("(,24.8]")) { + return; // JSON was introduced in 24.10 + } + + Properties properties = new Properties(); + properties.put(ClientConfigProperties.serverSetting("allow_experimental_json_type"), "1"); + try (Connection conn = getJdbcConnection(properties); + Statement stmt = conn.createStatement()) { + + final String json = "{\"count\": 1000, \"event\": { \"name\": \"start\", \"value\": 0.10} }"; + String sql = String.format("SELECT %1$s::JSON(), %1$s::JSON(count Int16)", SQLUtils.enquoteLiteral(json)); + try (ResultSet rs = stmt.executeQuery(sql)) { + rs.next(); + + Map val1 = (Map) rs.getObject(1); + assertEquals(val1.get("count"), 1000L); + Map val2 = (Map) rs.getObject(2); + assertEquals(val2.get("count"), (short)1000); + } + } + } + + + @Test(groups = { "integration" }, enabled = false) public void testGeometricTypesSimpleStatement() throws SQLException { // TODO: add LineString and MultiLineString support runQuery("CREATE TABLE test_geometric (order Int8, "