diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java index 13995c1b7..0e88403f1 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java @@ -106,6 +106,21 @@ public final class ClickHouseColumn implements Serializable { private Map, Integer> mapKeyToVariantOrdNumMap; private Map, Integer> mapValueToVariantOrdNumMap; + public enum DefaultValue { + DEFAULT("Default"), + MATERIALIZED("MATERIALIZED"), + EPHEMERAL("EPHEMERAL"), + ALIAS("ALIAS"); + + public final String defaultValue; + + DefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + } + + private DefaultValue defaultValue; + private String defaultExpression; private static ClickHouseColumn update(ClickHouseColumn column) { column.enumConstants = ClickHouseEnum.EMPTY; @@ -853,6 +868,14 @@ public void setHasDefault(boolean hasDefault) { this.hasDefault = hasDefault; } + public void setDefaultValue(DefaultValue defaultValue) { this.defaultValue = defaultValue; } + + public DefaultValue getDefaultValue() { return defaultValue; } + + public void setDefaultExpression(String defaultExpression) { this.defaultExpression = defaultExpression; } + + public String getDefaultExpression() { return defaultExpression; } + public boolean isLowCardinality() { return !lowCardinalityDisabled && lowCardinality; } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index b0825f9a1..8c6e58803 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -1340,6 +1340,9 @@ public CompletableFuture insert(String tableName, List data, .getOrDefault(tableName, Collections.emptyMap()); List serializersForTable = new ArrayList<>(); for (ClickHouseColumn column : tableSchema.getColumns()) { + if (column.hasDefault() && column.getDefaultValue() != ClickHouseColumn.DefaultValue.DEFAULT ) { + continue; + } POJOSerializer serializer = classSerializers.get(column.getColumnName()); if (serializer == null) { throw new IllegalArgumentException("No serializer found for column '" + column.getColumnName() + "'. Did you forget to register it?"); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatWriter.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatWriter.java index b498abe06..ef8c18914 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatWriter.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatWriter.java @@ -54,7 +54,9 @@ public void commitRow() throws IOException { List columnList = tableSchema.getColumns(); for (int i = 0; i < row.length; i++) { ClickHouseColumn column = columnList.get(i); - + // here we skip if we have a default value that is MATERIALIZED or ALIAS or ... + if (column.hasDefault() && column.getDefaultValue() != ClickHouseColumn.DefaultValue.DEFAULT) + continue; if (RowBinaryFormatSerializer.writeValuePreamble(out, defaultSupport, column, row[i])) { SerializerUtils.serializeData(out, row[i], column); } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/TableSchemaParser.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/TableSchemaParser.java index 1b7bbd627..74a1ba1b2 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/TableSchemaParser.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/TableSchemaParser.java @@ -25,7 +25,13 @@ public static TableSchema readTSKV(InputStream content, String table, String sql p.load(new StringReader(line.replaceAll("\t", "\n"))); ClickHouseColumn column = ClickHouseColumn.of(p.getProperty("name"), p.getProperty("type")); String defaultType = p.getProperty("default_type"); + String defaultExpression = p.getProperty("default_expression"); column.setHasDefault(defaultType != null && !defaultType.isEmpty()); + if ( column.hasDefault() ) { + column.setDefaultValue(ClickHouseColumn.DefaultValue.valueOf(defaultType)); + if ( defaultExpression != null && !defaultExpression.isEmpty() ) + column.setDefaultExpression(defaultExpression); + } columns.add(column); } } diff --git a/client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java b/client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java index 64e5441c2..e4aac9888 100644 --- a/client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java @@ -554,6 +554,54 @@ public void testAdvancedWriter() throws Exception { } } + @Test + public void testWriterWithMaterialize() throws Exception { + String tableName = "table_name_with_materialize"; + String tableCreate = "CREATE TABLE \"" + tableName + "\" " + + " (name String, " + + " v1 Float32, " + + " v2 Float32, " + + " attrs Nullable(String), " + + " corrected_time DateTime('UTC') DEFAULT now()," + + " special_attr Nullable(Int8) DEFAULT -1," + + " name_lower String MATERIALIZED lower(name)," + + " name_lower_alias String ALIAS lower(name)," + + " unhexed String EPHEMERAL," + + " hexed FixedString(4) DEFAULT unhex(unhexed)" + + " ) Engine = MergeTree ORDER by (name)"; + + initTable(tableName, tableCreate); + + ZonedDateTime correctedTime = Instant.now().atZone(ZoneId.of("UTC")); + Object[][] rows = new Object[][] { + {"foo1", 0.3f, 0.6f, "a=1,b=2,c=5", correctedTime, 10, "Z��"}, + {"foo2", 0.6f, 0.1f, "a=1,b=2,c=5", correctedTime, null, "Z��"}, + {"foo3", 0.7f, 0.4f, "a=1,b=2,c=5", null, null, "Z��"}, + {"foo4", 0.8f, 0.5f, null, null, null, "Z��"}, + }; + + TableSchema schema = client.getTableSchema(tableName); + + ClickHouseFormat format = ClickHouseFormat.RowBinaryWithDefaults; + try (InsertResponse response = client.insert(tableName, out -> { + RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, schema, format); + for (Object[] row : rows) { + for (int i = 0; i < row.length; i++) { + w.setValue(i + 1, row[i]); + } + w.commitRow(); + } + }, format, new InsertSettings()).get()) { + System.out.println("Rows written: " + response.getWrittenRows()); + } + + List records = client.queryAll("SELECT * FROM \"" + tableName + "\"" ); + + for (GenericRecord record : records) { + System.out.println("> " + record.getString(1) + ", " + record.getFloat(2) + ", " + record.getFloat(3)); + } + } + @Test public void testCollectionInsert() throws Exception { String tableName = "very_long_table_name_with_uuid_" + UUID.randomUUID().toString().replace('-', '_'); @@ -705,6 +753,33 @@ public void testPOJOWithDynamicType() throws Exception { } } + @Test(groups = { "integration" }, enabled = true) + public void insertSimplePOJOsWithMaterializeColumn() throws Exception { + String tableName = "simple_pojo_table_with_materialize_column"; + String createSQL = SimplePOJO.generateTableCreateSQL(tableName); + String uuid = UUID.randomUUID().toString(); + + initTable(tableName, createSQL); + + client.register(SimplePOJO.class, client.getTableSchema(tableName)); + List simplePOJOs = new ArrayList<>(); + + for (int i = 0; i < 1000; i++) { + simplePOJOs.add(new SimplePOJO()); + } + settings.setQueryId(uuid); + InsertResponse response = client.insert(tableName, simplePOJOs, settings).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS); + + OperationMetrics metrics = response.getMetrics(); + assertEquals(simplePOJOs.size(), metrics.getMetric(ServerMetrics.NUM_ROWS_WRITTEN).getLong()); + assertEquals(simplePOJOs.size(), response.getWrittenRows()); + assertTrue(metrics.getMetric(ClientMetrics.OP_DURATION).getLong() > 0); + assertTrue(metrics.getMetric(ClientMetrics.OP_SERIALIZATION).getLong() > 0); + assertEquals(metrics.getQueryId(), uuid); + assertEquals(response.getQueryId(), uuid); + } + + protected void initTable(String tableName, String createTableSQL) throws Exception { initTable(tableName, createTableSQL, new CommandSettings()); } diff --git a/client-v2/src/test/java/com/clickhouse/client/insert/SimplePOJO.java b/client-v2/src/test/java/com/clickhouse/client/insert/SimplePOJO.java new file mode 100644 index 000000000..843ee94f5 --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/insert/SimplePOJO.java @@ -0,0 +1,41 @@ +package com.clickhouse.client.insert; + +import com.clickhouse.client.ClientTests; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Random; + +@Getter +@Setter +public class SimplePOJO { + + private static final Logger LOGGER = LoggerFactory.getLogger(SimplePOJO.class); + private int int32; + private String str; + private String hexed; + + public SimplePOJO() { + long seed = System.currentTimeMillis(); + final Random random = new Random(seed); + this.int32 = random.nextInt(); + this.str = RandomStringUtils.randomAlphabetic(1, 256); + this.hexed = RandomStringUtils.randomAlphanumeric(4); + } + + public static String generateTableCreateSQL(String tableName) { + return "CREATE TABLE " + tableName + " (" + + "int32 Int32, " + + "str String, " + + "int64 Int64 MATERIALIZED abs(toInt64(int32)), " + + "str_lower String ALIAS lower(str), " + + "unhexed String EPHEMERAL, " + + "hexed FixedString(4) DEFAULT unhex(unhexed), " + + ") ENGINE = MergeTree ORDER BY ()"; + } + +} +