From 2c5c5a2e2913aae58f585d6981855477748df19e Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Mon, 17 Oct 2022 17:41:26 +0800 Subject: [PATCH 1/3] fix core dump on bool type runtime filter --- be/src/exprs/runtime_filter.cpp | 2 +- be/src/olap/in_list_predicate.h | 6 +++++- be/src/olap/predicate_creator.h | 5 ----- .../query/operator/test_logical_operators.out | 19 +++++++++++++++++++ .../operator/test_logical_operators.groovy | 2 ++ 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp index 24e8e41a842ed8..540e81463391a2 100644 --- a/be/src/exprs/runtime_filter.cpp +++ b/be/src/exprs/runtime_filter.cpp @@ -1413,7 +1413,7 @@ void IRuntimeFilter::to_protobuf(PInFilter* filter) { switch (column_type) { case TYPE_BOOLEAN: { - batch_copy(filter, it, [](PColumnValue* column, const int32_t* value) { + batch_copy(filter, it, [](PColumnValue* column, const bool* value) { column->set_boolval(*value); }); return; diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h index b0b9fc53301cfe..3cd60c8f7ec2c9 100644 --- a/be/src/olap/in_list_predicate.h +++ b/be/src/olap/in_list_predicate.h @@ -27,6 +27,7 @@ #include "olap/column_predicate.h" #include "olap/rowset/segment_v2/bloom_filter.h" #include "olap/wrapper_field.h" +#include "runtime/define_primitive_type.h" #include "runtime/string_value.h" #include "runtime/type_limit.h" #include "uint24.h" @@ -267,7 +268,10 @@ class InListPredicateBase : public ColumnPredicate { private: template bool _operator(const LeftT& lhs, const RightT& rhs) const { - if constexpr (PT == PredicateType::IN_LIST) { + if constexpr (Type == TYPE_BOOLEAN) { + DCHECK(_values.size() == 2); + return PT == PredicateType::IN_LIST; + } else if constexpr (PT == PredicateType::IN_LIST) { return lhs != rhs; } return lhs == rhs; diff --git a/be/src/olap/predicate_creator.h b/be/src/olap/predicate_creator.h index ad1ed0d69edb56..0b1e1a4c143278 100644 --- a/be/src/olap/predicate_creator.h +++ b/be/src/olap/predicate_creator.h @@ -24,7 +24,6 @@ #include "olap/in_list_predicate.h" #include "olap/null_predicate.h" #include "olap/tablet_schema.h" -#include "runtime/type_limit.h" #include "util/date_func.h" #include "util/string_util.h" @@ -236,10 +235,6 @@ template inline ColumnPredicate* create_list_predicate(const TabletColumn& column, int index, const std::vector& conditions, bool opposite, MemPool* pool) { - if (column.type() == OLAP_FIELD_TYPE_BOOL) { - LOG(FATAL) << "Failed to create list preacate! input column type is invalid"; - return nullptr; - } static_assert(PredicateTypeTraits::is_list(PT)); return create_predicate>(column, index, conditions, opposite, pool); diff --git a/regression-test/data/query/operator/test_logical_operators.out b/regression-test/data/query/operator/test_logical_operators.out index 17f680e079a468..8db02055121179 100644 --- a/regression-test/data/query/operator/test_logical_operators.out +++ b/regression-test/data/query/operator/test_logical_operators.out @@ -367,3 +367,22 @@ -- !logical_op10 -- false 1 1989 1001 11011902 123.123 true 1989-03-21 1989-03-21T13:00 wangjuoo4 0.1 6.333 string12345 170141183460469231731687303715884105727 +-- !logical_op11 -- +false 3 1989 1002 11011905 24453.325 false 2012-03-14 2000-01-01T00:00 yunlj8@nk 78945.0 3654.0 string12345 0 +false 7 -32767 1002 7210457 3.141 false 1988-03-21 1901-01-01T00:00 jiw3n4 0.0 6058.0 string12345 -20220101 +true 8 255 2147483647 11011920 -0.123 true 1989-03-21 9999-11-11T12:12 wangjuoo5 987456.123 12.14 string12345 -2022 +true 9 1991 -2147483647 11011902 -654.654 true 1991-08-11 1989-03-21T13:11 wangjuoo4 0.0 69.123 string12345 11011903 +true 11 1989 25699 -9223372036854775807 0.666 true 2015-04-02 1989-03-21T13:11 yunlj8@nk -987.001 4.336 string12345 1701411834604692317316873037158 +true 13 -32767 2147483647 -9223372036854775807 100.001 false 2015-04-02 2015-04-02T00:00 wenlsfnl 123.456 3.1415927 string12345 701411834604692317316873037158 +true 14 255 103 11011902 0 false 2015-04-02 2015-04-02T00:00 3.141592654 2.036 string12345 701411834604692317316873 +false 1 1989 1001 11011902 123.123 true 1989-03-21 1989-03-21T13:00 wangjuoo4 0.1 6.333 string12345 170141183460469231731687303715884105727 +false 2 1986 1001 11011903 1243.5 false 1901-12-31 1989-03-21T13:00 wangynnsf 20.268 789.25 string12345 -170141183460469231731687303715884105727 +false 4 1991 3021 -11011907 243243.325 false 3124-10-10 2015-03-13T10:30 yanvjldjlll 2.06 -0.001 string12345 20220101 +false 5 1985 5014 -11011903 243.325 true 2015-01-01 2015-03-13T12:36:38 du3lnvl -0.0 -365.0 string12345 20220102 +true 15 1992 3021 11011920 0 true 9999-12-12 2015-04-02T00:00 3.141592653 20.456 string12345 701411834604692317 +true 10 1991 5014 9223372036854775807 -258.369 false 2015-04-02 2013-04-02T15:16:52 wangynnsf -123456.54 0.235 string12345 -11011903 +true 12 32767 -2147483647 9223372036854775807 243.325 false 1991-08-11 2013-04-02T15:16:52 lifsno -564.898 3.1415927 string12345 1701604692317316873037158 +false 6 32767 3021 123456 604587 true 2014-11-11 2015-03-13T12:36:38 yanavnd 0.1 80699.0 string12345 20220104 + +-- !logical_op12 -- + diff --git a/regression-test/suites/query/operator/test_logical_operators.groovy b/regression-test/suites/query/operator/test_logical_operators.groovy index 5164b4f94f0574..6f0ec25f061d6f 100644 --- a/regression-test/suites/query/operator/test_logical_operators.groovy +++ b/regression-test/suites/query/operator/test_logical_operators.groovy @@ -37,4 +37,6 @@ suite("test_logical_operators", "query,p0") { } qt_logical_op9 "select k8, k9, k8%k9, k9%NULL, NULL%k9 from ${tableName} order by 1, 2" qt_logical_op10 'select * from baseall where (k1 = 1) or (k1 = 1 and k2 = 2)' + qt_logical_op11 'select * from baseall where k0 in (false,true)' + qt_logical_op12 'select * from baseall where k0 not in (false,true)' } From a32ae63dcd18a72435c162ca0eaff58bb0a62b37 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Mon, 17 Oct 2022 22:11:07 +0800 Subject: [PATCH 2/3] fix case --- .../data/query/operator/test_logical_operators.out | 14 +++++++------- .../query/operator/test_logical_operators.groovy | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/regression-test/data/query/operator/test_logical_operators.out b/regression-test/data/query/operator/test_logical_operators.out index 8db02055121179..55b4bfe63c648d 100644 --- a/regression-test/data/query/operator/test_logical_operators.out +++ b/regression-test/data/query/operator/test_logical_operators.out @@ -368,21 +368,21 @@ false 1 1989 1001 11011902 123.123 true 1989-03-21 1989-03-21T13:00 wangjuoo4 0.1 6.333 string12345 170141183460469231731687303715884105727 -- !logical_op11 -- +false 1 1989 1001 11011902 123.123 true 1989-03-21 1989-03-21T13:00 wangjuoo4 0.1 6.333 string12345 170141183460469231731687303715884105727 +false 2 1986 1001 11011903 1243.5 false 1901-12-31 1989-03-21T13:00 wangynnsf 20.268 789.25 string12345 -170141183460469231731687303715884105727 false 3 1989 1002 11011905 24453.325 false 2012-03-14 2000-01-01T00:00 yunlj8@nk 78945.0 3654.0 string12345 0 +false 4 1991 3021 -11011907 243243.325 false 3124-10-10 2015-03-13T10:30 yanvjldjlll 2.06 -0.001 string12345 20220101 +false 5 1985 5014 -11011903 243.325 true 2015-01-01 2015-03-13T12:36:38 du3lnvl -0.0 -365.0 string12345 20220102 +false 6 32767 3021 123456 604587 true 2014-11-11 2015-03-13T12:36:38 yanavnd 0.1 80699.0 string12345 20220104 false 7 -32767 1002 7210457 3.141 false 1988-03-21 1901-01-01T00:00 jiw3n4 0.0 6058.0 string12345 -20220101 true 8 255 2147483647 11011920 -0.123 true 1989-03-21 9999-11-11T12:12 wangjuoo5 987456.123 12.14 string12345 -2022 true 9 1991 -2147483647 11011902 -654.654 true 1991-08-11 1989-03-21T13:11 wangjuoo4 0.0 69.123 string12345 11011903 +true 10 1991 5014 9223372036854775807 -258.369 false 2015-04-02 2013-04-02T15:16:52 wangynnsf -123456.54 0.235 string12345 -11011903 true 11 1989 25699 -9223372036854775807 0.666 true 2015-04-02 1989-03-21T13:11 yunlj8@nk -987.001 4.336 string12345 1701411834604692317316873037158 +true 12 32767 -2147483647 9223372036854775807 243.325 false 1991-08-11 2013-04-02T15:16:52 lifsno -564.898 3.1415927 string12345 1701604692317316873037158 true 13 -32767 2147483647 -9223372036854775807 100.001 false 2015-04-02 2015-04-02T00:00 wenlsfnl 123.456 3.1415927 string12345 701411834604692317316873037158 true 14 255 103 11011902 0 false 2015-04-02 2015-04-02T00:00 3.141592654 2.036 string12345 701411834604692317316873 -false 1 1989 1001 11011902 123.123 true 1989-03-21 1989-03-21T13:00 wangjuoo4 0.1 6.333 string12345 170141183460469231731687303715884105727 -false 2 1986 1001 11011903 1243.5 false 1901-12-31 1989-03-21T13:00 wangynnsf 20.268 789.25 string12345 -170141183460469231731687303715884105727 -false 4 1991 3021 -11011907 243243.325 false 3124-10-10 2015-03-13T10:30 yanvjldjlll 2.06 -0.001 string12345 20220101 -false 5 1985 5014 -11011903 243.325 true 2015-01-01 2015-03-13T12:36:38 du3lnvl -0.0 -365.0 string12345 20220102 true 15 1992 3021 11011920 0 true 9999-12-12 2015-04-02T00:00 3.141592653 20.456 string12345 701411834604692317 -true 10 1991 5014 9223372036854775807 -258.369 false 2015-04-02 2013-04-02T15:16:52 wangynnsf -123456.54 0.235 string12345 -11011903 -true 12 32767 -2147483647 9223372036854775807 243.325 false 1991-08-11 2013-04-02T15:16:52 lifsno -564.898 3.1415927 string12345 1701604692317316873037158 -false 6 32767 3021 123456 604587 true 2014-11-11 2015-03-13T12:36:38 yanavnd 0.1 80699.0 string12345 20220104 -- !logical_op12 -- diff --git a/regression-test/suites/query/operator/test_logical_operators.groovy b/regression-test/suites/query/operator/test_logical_operators.groovy index 6f0ec25f061d6f..e5dfe498d72313 100644 --- a/regression-test/suites/query/operator/test_logical_operators.groovy +++ b/regression-test/suites/query/operator/test_logical_operators.groovy @@ -36,7 +36,7 @@ suite("test_logical_operators", "query,p0") { } } qt_logical_op9 "select k8, k9, k8%k9, k9%NULL, NULL%k9 from ${tableName} order by 1, 2" - qt_logical_op10 'select * from baseall where (k1 = 1) or (k1 = 1 and k2 = 2)' - qt_logical_op11 'select * from baseall where k0 in (false,true)' - qt_logical_op12 'select * from baseall where k0 not in (false,true)' + qt_logical_op10 'select * from baseall where (k1 = 1) or (k1 = 1 and k2 = 2) order by k1, k2, k3, k4' + qt_logical_op11 'select * from baseall where k0 in (false,true) order by k1, k2, k3, k4' + qt_logical_op12 'select * from baseall where k0 not in (false,true) order by k1, k2, k3, k4' } From 89448275cf02613e591d522440ec148155202710 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Tue, 18 Oct 2022 12:11:34 +0800 Subject: [PATCH 3/3] forbiden bool runtime filter --- .../main/java/org/apache/doris/planner/RuntimeFilter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java index 45a0c607248883..c3fec43021e862 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java @@ -26,8 +26,8 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.analysis.TupleIsNullPredicate; -import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; import org.apache.doris.common.FeConstants; import org.apache.doris.common.IdGenerator; import org.apache.doris.qe.ConnectContext; @@ -264,8 +264,8 @@ public static RuntimeFilter create(IdGenerator idGen, Analyzer TupleIsNullPredicate.unwrapExpr(normalizedJoinConjunct.getChild(0).clone()); Expr srcExpr = normalizedJoinConjunct.getChild(1); - if (srcExpr.getType().equals(ScalarType.createHllType()) - || srcExpr.getType().equals(ScalarType.createType(PrimitiveType.BITMAP))) { + Type srcType = srcExpr.getType(); + if (srcType.equals(ScalarType.HLL) || srcType.equals(ScalarType.BITMAP) || srcType.equals(ScalarType.BOOLEAN)) { return null; }