From 824e335fce37c1987e1d10eb3b7fd4b91412547b Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Tue, 3 Sep 2024 19:41:18 +0800 Subject: [PATCH] 1 --- be/src/olap/partial_update_info.cpp | 9 ++ .../org/apache/doris/nereids/DorisLexer.g4 | 1 + .../org/apache/doris/nereids/DorisParser.g4 | 5 +- .../org/apache/doris/analysis/ColumnDef.java | 6 +- .../nereids/parser/LogicalPlanBuilder.java | 2 + .../plans/commands/info/ColumnDefinition.java | 8 +- .../plans/commands/info/DefaultValue.java | 4 +- .../test_default_bitmap_empty.out | 43 ++++++ .../test_default_bitmap_empty_streamload.csv | 2 + .../test_default_bitmap_empty.groovy | 137 ++++++++++++++++++ 10 files changed, 209 insertions(+), 8 deletions(-) create mode 100644 regression-test/data/correctness_p0/test_default_bitmap_empty.out create mode 100644 regression-test/data/correctness_p0/test_default_bitmap_empty_streamload.csv create mode 100644 regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy diff --git a/be/src/olap/partial_update_info.cpp b/be/src/olap/partial_update_info.cpp index 5867a77559b36d..56024781e571f4 100644 --- a/be/src/olap/partial_update_info.cpp +++ b/be/src/olap/partial_update_info.cpp @@ -20,6 +20,10 @@ #include #include "olap/tablet_schema.h" +#include "olap/utils.h" +#include "util/bitmap_value.h" +#include "vec/common/assert_cast.h" +#include "vec/core/block.h" namespace doris { @@ -141,6 +145,11 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids( DateV2Value dv; dv.from_unixtime(timestamp_ms / 1000, timezone); default_value = dv.debug_string(); + } else if (UNLIKELY(column.type() == FieldType::OLAP_FIELD_TYPE_OBJECT && + to_lower(column.default_value()).find(to_lower("BITMAP_EMPTY")) != + std::string::npos)) { + BitmapValue v = BitmapValue {}; + default_value = v.to_string(); } else { default_value = tablet_schema.column(cur_cid).default_value(); } diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 3ed51ca8fbf646..fb6190ec1f8ff9 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -111,6 +111,7 @@ BINARY: 'BINARY'; BINLOG: 'BINLOG'; BITAND: 'BITAND'; BITMAP: 'BITMAP'; +BITMAP_EMPTY: 'BITMAP_EMPTY'; BITMAP_UNION: 'BITMAP_UNION'; BITOR: 'BITOR'; BITXOR: 'BITXOR'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index d94a0f4f680b39..275c934f7d56a0 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -589,8 +589,8 @@ columnDef (aggType=aggTypeDef)? ((NOT)? NULL)? (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)? - (DEFAULT (nullValue=NULL | INTEGER_VALUE | stringValue=STRING_LITERAL| CURRENT_DATE - | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))? + (DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | BITMAP_EMPTY | stringValue=STRING_LITERAL + | CURRENT_DATE | defaultTimestamp=CURRENT_TIMESTAMP (LEFT_PAREN defaultValuePrecision=number RIGHT_PAREN)?))? (ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)? (COMMENT comment=STRING_LITERAL)? ; @@ -1038,6 +1038,7 @@ nonReserved | BIN | BITAND | BITMAP + | BITMAP_EMPTY | BITMAP_UNION | BITOR | BITXOR diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index 6207a5088af8c6..bc722d3a1d69af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -367,8 +367,10 @@ public void analyze(boolean isOlap) throws AnalysisException { } if (type.getPrimitiveType() == PrimitiveType.BITMAP) { - if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE) { - throw new AnalysisException("Bitmap type column can not set default value"); + if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE + && !defaultValue.value.equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value)) { + throw new AnalysisException("Bitmap type column default value only support null or " + + DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.value); } defaultValue = DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c4403a98bd00e4..10523f0c76e831 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -2695,6 +2695,8 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { } } else if (ctx.CURRENT_DATE() != null) { defaultValue = Optional.of(DefaultValue.CURRENT_DATE_DEFAULT_VALUE); + } else if (ctx.BITMAP_EMPTY() != null) { + defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); } } if (ctx.UPDATE() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 049a04260730f6..36140b3b8212c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -286,13 +286,15 @@ public void validate(boolean isOlap, Set keysSet, boolean isEnableMergeO } defaultValue = Optional.of(DefaultValue.HLL_EMPTY_DEFAULT_VALUE); } else if (type.isBitmapType()) { - if (defaultValue.isPresent() && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE) { - throw new AnalysisException("Bitmap type column can not set default value"); + if (defaultValue.isPresent() && isOlap && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE + && !defaultValue.get().getValue().equals(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE.getValue())) { + throw new AnalysisException("Bitmap type column default value only support " + + DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); } defaultValue = Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE); } else if (type.isArrayType() && defaultValue.isPresent() && isOlap && defaultValue.get() != DefaultValue.NULL_DEFAULT_VALUE && !defaultValue.get() - .getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) { + .getValue().equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.getValue())) { throw new AnalysisException("Array type column default value only support null or " + DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE); } else if (type.isMapType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java index 8e7228705ac2a5..2bf5934425ac29 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DefaultValue.java @@ -27,6 +27,8 @@ public class DefaultValue { public static String CURRENT_DATE = "CURRENT_DATE"; public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP"; public static String NOW = "now"; + public static String HLL_EMPTY = "HLL_EMPTY"; + public static String BITMAP_EMPTY = "BITMAP_EMPTY"; public static DefaultValue CURRENT_DATE_DEFAULT_VALUE = new DefaultValue(CURRENT_DATE, CURRENT_DATE.toLowerCase()); public static DefaultValue CURRENT_TIMESTAMP_DEFAULT_VALUE = new DefaultValue(CURRENT_TIMESTAMP, NOW); // default null @@ -36,7 +38,7 @@ public class DefaultValue { // default "value", "0" means empty hll public static DefaultValue HLL_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO); // default "value", "0" means empty bitmap - public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO); + public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new DefaultValue(ZERO, BITMAP_EMPTY); // default "value", "[]" means empty array public static DefaultValue ARRAY_EMPTY_DEFAULT_VALUE = new DefaultValue("[]"); diff --git a/regression-test/data/correctness_p0/test_default_bitmap_empty.out b/regression-test/data/correctness_p0/test_default_bitmap_empty.out new file mode 100644 index 00000000000000..33cc825e767f55 --- /dev/null +++ b/regression-test/data/correctness_p0/test_default_bitmap_empty.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !insert_into1 -- +0 +0 +0 +0 + +-- !stream_load_csv1 -- +0 +0 +0 +0 +0 +0 + +-- !select_2 -- +0 +0 +0 +0 + +-- !stream_load_csv2 -- +0 +0 +0 +0 +0 +0 + +-- !insert_into3 -- +0 +0 +0 +0 + +-- !stream_load_csv3 -- +0 +0 +0 +0 +0 +0 + diff --git a/regression-test/data/correctness_p0/test_default_bitmap_empty_streamload.csv b/regression-test/data/correctness_p0/test_default_bitmap_empty_streamload.csv new file mode 100644 index 00000000000000..f4ec2d7748a0a2 --- /dev/null +++ b/regression-test/data/correctness_p0/test_default_bitmap_empty_streamload.csv @@ -0,0 +1,2 @@ +5,5 +6,6 \ No newline at end of file diff --git a/regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy b/regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy new file mode 100644 index 00000000000000..60b6ff73cb37e8 --- /dev/null +++ b/regression-test/suites/correctness_p0/test_default_bitmap_empty.groovy @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_default_bitmap_empty") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + def tableName = "test_default_bitmap_empty" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} + ( + k TINYINT, + v1 bitmap NOT NULL DEFAULT bitmap_empty, + v2 INT + ) + UNIQUE KEY(K) + DISTRIBUTED BY HASH(k) + PROPERTIES("replication_num" = "1"); + """ + + // test insert into. + sql " insert into ${tableName} (k, v2) values (1, 1); " + sql " insert into ${tableName} (k, v2) values (2, 2); " + sql " insert into ${tableName} (k, v2) values (3, 3); " + sql " insert into ${tableName} (k, v2) values (4, 4); " + sql "sync" + qt_insert_into1 """ select bitmap_count(v1) from ${tableName}; """ + + // test csv stream load. + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'columns', 'k, v1=bitmap_empty(), v2' + + file 'test_default_bitmap_empty_streamload.csv' + + time 10000 // limit inflight 10s + } + + sql "sync" + + qt_stream_load_csv1 """ select bitmap_count(v1) from ${tableName}; """ + + // test partial update + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} + ( + k TINYINT, + v1 bitmap NOT NULL DEFAULT bitmap_empty, + v2 INT + ) + UNIQUE KEY(K) + DISTRIBUTED BY HASH(k) + PROPERTIES("replication_num" = "1"); + """ + + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + + sql " insert into ${tableName} (k, v2) values (1, 1); " + sql " insert into ${tableName} (k, v2) values (2, 2); " + sql " insert into ${tableName} (k, v2) values (3, 3); " + sql " insert into ${tableName} (k, v2) values (4, 4); " + sql "sync" + + qt_select_2 "select bitmap_count(v1) from ${tableName};" + + streamLoad { + table "${tableName}" + + set 'partial_columns', 'true' + set 'column_separator', ',' + set 'columns', 'k, v2' + + file 'test_default_bitmap_empty_streamload.csv' + + time 10000 // limit inflight 10s + } + + sql "sync" + + qt_stream_load_csv2 """ select bitmap_count(v1) from ${tableName}; """ + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} + ( + k TINYINT, + v1 bitmap BITMAP_UNION default BITMAP_EMPTY, + v2 INT replace_if_not_null + ) + aggregate KEY(K) + DISTRIBUTED BY HASH(k) + PROPERTIES("replication_num" = "1"); + """ + + // test insert into. + sql " insert into ${tableName} (k, v2) values (1, 1); " + sql " insert into ${tableName} (k, v2) values (2, 2); " + sql " insert into ${tableName} (k, v2) values (3, 3); " + sql " insert into ${tableName} (k, v2) values (4, 4); " + sql "sync" + qt_insert_into3 """ select bitmap_count(v1) from ${tableName}; """ + + // test csv stream load. + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'columns', 'k, v1=bitmap_empty(), v2' + + file 'test_default_bitmap_empty_streamload.csv' + + time 10000 // limit inflight 10s + } + + sql "sync" + + qt_stream_load_csv3 """ select bitmap_count(v1) from ${tableName}; """ +} \ No newline at end of file