diff --git a/be/src/exec/union_node.cpp b/be/src/exec/union_node.cpp index a6ff9f75ae0316..a5b686cdd17090 100644 --- a/be/src/exec/union_node.cpp +++ b/be/src/exec/union_node.cpp @@ -219,6 +219,7 @@ Status UnionNode::get_next_const(RuntimeState* state, RowBatch* row_batch) { while (_const_expr_list_idx < _const_expr_lists.size() && !row_batch->at_capacity()) { materialize_exprs( _const_expr_lists[_const_expr_list_idx], nullptr, tuple_buf, row_batch); + RETURN_IF_ERROR(get_error_msg(_const_expr_lists[_const_expr_list_idx])); tuple_buf += _tuple_desc->byte_size(); ++_const_expr_list_idx; } diff --git a/be/src/exec/union_node.h b/be/src/exec/union_node.h index 70f65d1e05369d..daa4dc41951328 100644 --- a/be/src/exec/union_node.h +++ b/be/src/exec/union_node.h @@ -126,6 +126,8 @@ class UnionNode : public ExecNode { void materialize_exprs(const std::vector& exprs, TupleRow* row, uint8_t* tuple_buf, RowBatch* dst_batch); + Status get_error_msg(const std::vector& exprs); + /// Returns true if the child at 'child_idx' can be passed through. bool is_child_passthrough(int child_idx) const { DCHECK_LT(child_idx, _children.size()); diff --git a/be/src/exec/union_node_ir.cpp b/be/src/exec/union_node_ir.cpp index 1f656ac064dee0..32198201c7230e 100644 --- a/be/src/exec/union_node_ir.cpp +++ b/be/src/exec/union_node_ir.cpp @@ -16,6 +16,7 @@ // under the License. #include "exec/union_node.h" +#include "exprs/expr_context.h" #include "runtime/tuple_row.h" namespace doris { @@ -52,4 +53,14 @@ void UnionNode::materialize_batch(RowBatch* dst_batch, uint8_t** tuple_buf) { *tuple_buf = cur_tuple; } +Status UnionNode::get_error_msg(const std::vector& exprs) { + for (auto expr_ctx: exprs) { + std::string expr_error = expr_ctx->get_error_msg(); + if (!expr_error.empty()) { + return Status::RuntimeError(expr_error); + } + } + return Status::OK(); +} + } diff --git a/docs/en/administrator-guide/materialized_view.md b/docs/en/administrator-guide/materialized_view.md index e79f275d0308d0..52e35b11fb1323 100644 --- a/docs/en/administrator-guide/materialized_view.md +++ b/docs/en/administrator-guide/materialized_view.md @@ -90,6 +90,9 @@ The aggregate functions currently supported by the materialized view function ar + SUM, MIN, MAX (Version 0.12) + COUNT, BITMAP\_UNION, HLL\_UNION (Version 0.13) ++ The form of BITMAP\_UNION must be: `BITMAP_UNION(TO_BITMAP(COLUMN))` The column type can only be an integer (largeint also does not support), or `BITMAP_UNION(COLUMN)` and the base table is an AGG model. ++ The form of HLL\_UNION must be: `HLL_UNION(HLL_HASH(COLUMN))` The column type cannot be DECIMAL, or `HLL_UNION(COLUMN)` and the base table is an AGG model. + ### Update strategy In order to ensure the data consistency between the materialized view table and the Base table, Doris will import, delete and other operations on the Base table are synchronized to the materialized view table. And through incremental update to improve update efficiency. To ensure atomicity through transaction. diff --git a/docs/zh-CN/administrator-guide/materialized_view.md b/docs/zh-CN/administrator-guide/materialized_view.md index a0fe43c4d72650..d91f7177325d24 100644 --- a/docs/zh-CN/administrator-guide/materialized_view.md +++ b/docs/zh-CN/administrator-guide/materialized_view.md @@ -91,6 +91,9 @@ HELP CREATE MATERIALIZED VIEW + SUM, MIN, MAX (Version 0.12) + COUNT, BITMAP\_UNION, HLL\_UNION (Version 0.13) ++ BITMAP\_UNION 的形式必须为:`BITMAP_UNION(TO_BITMAP(COLUMN))` column 列的类型只能是整数(largeint也不支持), 或者 `BITMAP_UNION(COLUMN)` 且 base 表为 AGG 模型。 ++ HLL\_UNION 的形式必须为:`HLL_UNION(HLL_HASH(COLUMN))` column 列的类型不能是 DECIMAL , 或者 `HLL_UNION(COLUMN)` 且 base 表为 AGG 模型。 + ### 更新策略 为保证物化视图表和 Base 表的数据一致性, Doris 会将导入,删除等对 base 表的操作都同步到物化视图表中。并且通过增量更新的方式来提升更新效率。通过事务方式来保证原子性。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java index e2a26102a9c07c..68f92764f7dbef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java @@ -44,11 +44,13 @@ public boolean match(Expr expr) { if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.TO_BITMAP)) { return false; } - if (child0FnExpr.getChild(0).unwrapSlotRef() == null) { + SlotRef slotRef = child0FnExpr.getChild(0).unwrapSlotRef(); + if (slotRef == null) { return false; - } else { + } else if (slotRef.getType().isIntegerType()) { return true; } + return false; } else { return false; } @@ -56,7 +58,7 @@ public boolean match(Expr expr) { @Override public String toString() { - return FunctionSet.BITMAP_UNION + "(" + FunctionSet.TO_BITMAP + "(column)) " - + "or " + FunctionSet.BITMAP_UNION + "(bitmap_column) in agg table"; + return FunctionSet.BITMAP_UNION + "(" + FunctionSet.TO_BITMAP + "(column)), type of column could not be integer. " + + "Or " + FunctionSet.BITMAP_UNION + "(bitmap_column) in agg table"; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java index 11eae282cdf542..7d58b2ce4b6271 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.Type; public class MVColumnHLLUnionPattern implements MVColumnPattern { @Override @@ -43,11 +44,13 @@ public boolean match(Expr expr) { if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.HLL_HASH)) { return false; } - if (child0FnExpr.getChild(0).unwrapSlotRef() == null) { + SlotRef slotRef = child0FnExpr.getChild(0).unwrapSlotRef(); + if (slotRef == null) { + return false; + } else if (slotRef.getType() == Type.DECIMALV2) { return false; - } else { - return true; } + return true; } else { return false; } @@ -55,7 +58,7 @@ public boolean match(Expr expr) { @Override public String toString() { - return FunctionSet.HLL_UNION + "(" + FunctionSet.HLL_HASH + "(column))" - + "or " + FunctionSet.HLL_UNION + "(hll_column) in agg table"; + return FunctionSet.HLL_UNION + "(" + FunctionSet.HLL_HASH + "(column)) column could not be decimal. " + + "Or " + FunctionSet.HLL_UNION + "(hll_column) in agg table"; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java index f5998acb4a651f..4b78903d6664b1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.Type; +import org.apache.doris.common.jmockit.Deencapsulation; import com.google.common.collect.Lists; @@ -37,6 +38,7 @@ public class MVColumnBitmapUnionPatternTest { public void testCorrectExpr1() { TableName tableName = new TableName("db", "table"); SlotRef slotRef = new SlotRef(tableName, "c1"); + Deencapsulation.setField(slotRef, "type", Type.INT); List child0Params = Lists.newArrayList(); child0Params.add(slotRef); FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.TO_BITMAP, child0Params); @@ -51,6 +53,7 @@ public void testCorrectExpr1() { public void testCorrectExpr2(@Injectable CastExpr castExpr) { TableName tableName = new TableName("db", "table"); SlotRef slotRef = new SlotRef(tableName, "c1"); + Deencapsulation.setField(slotRef, "type", Type.INT); new Expectations() { { castExpr.unwrapSlotRef(); @@ -71,6 +74,7 @@ public void testCorrectExpr2(@Injectable CastExpr castExpr) { public void testUpperCaseOfFunction() { TableName tableName = new TableName("db", "table"); SlotRef slotRef = new SlotRef(tableName, "c1"); + Deencapsulation.setField(slotRef, "type", Type.INT); List child0Params = Lists.newArrayList(); child0Params.add(slotRef); FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.TO_BITMAP.toUpperCase(), child0Params); @@ -110,6 +114,21 @@ public void testIncorrectArithmeticExpr2() { Assert.assertFalse(pattern.match(expr)); } + @Test + public void testIncorrectDecimalSlotRef() { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef1 = new SlotRef(tableName, "c1"); + Deencapsulation.setField(slotRef1, "type", Type.DECIMALV2); + List child0Params = Lists.newArrayList(); + child0Params.add(slotRef1); + FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.TO_BITMAP, child0Params); + List params = Lists.newArrayList(); + params.add(child0); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.BITMAP_UNION, params); + MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); + Assert.assertFalse(pattern.match(expr)); + } + @Test public void testAggTableBitmapColumn(@Injectable SlotDescriptor desc, @Injectable Column column) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java index 7625ce6c75b8f0..189365c8f9110c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.Type; +import org.apache.doris.common.jmockit.Deencapsulation; import com.google.common.collect.Lists; @@ -104,6 +105,21 @@ public void testIncorrectLiteralExpr2() { Assert.assertFalse(pattern.match(expr)); } + @Test + public void testIncorrectDecimalSlotRef() { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef = new SlotRef(tableName, "c1"); + Deencapsulation.setField(slotRef, "type", Type.DECIMALV2); + List child0Params = Lists.newArrayList(); + child0Params.add(slotRef); + FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.HLL_HASH, child0Params); + List params = Lists.newArrayList(); + params.add(child0); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.HLL_UNION, params); + MVColumnHLLUnionPattern pattern = new MVColumnHLLUnionPattern(); + Assert.assertFalse(pattern.match(expr)); + } + @Test public void testAggTableHLLColumn(@Injectable SlotDescriptor desc, @Injectable Column column) {