diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java index ad6f8980c92ed7..861aeaa043efcf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java @@ -134,7 +134,7 @@ public boolean isMinValue() { } @Override - public Object getRealValue() { + public BigInteger getRealValue() { return this.value; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java index 72f50d5b7b33d3..6076a963884b3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSortedPartitionsCacheManager.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges.PartitionItemAndId; import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges.PartitionItemAndRange; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.qe.ConnectContext; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -65,6 +66,11 @@ public NereidsSortedPartitionsCacheManager() { public Optional> get( SupportBinarySearchFilteringPartitions table, CatalogRelation scan) { + ConnectContext connectContext = ConnectContext.get(); + if (connectContext != null && !connectContext.getSessionVariable().enableBinarySearchFilteringPartitions) { + return Optional.empty(); + } + DatabaseIf database = table.getDatabase(); if (database == null) { return Optional.empty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java index a7a5af74d009ab..92b96b7bd099b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java @@ -86,7 +86,7 @@ public Void visitComparisonPredicate(ComparisonPredicate comparisonPredicate, Vo @Override public Void visitInPredicate(InPredicate inPredicate, Void context) { if (containOnlyColumnRef(inPredicate.getCompareExpr(), true) - && (ExpressionUtils.isAllLiteral(inPredicate.getOptions()))) { + && (inPredicate.optionsAreLiterals())) { rangePredicates.put(inPredicate, ExpressionInfo.EMPTY); } else { residualPredicates.put(inPredicate, ExpressionInfo.EMPTY); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ColumnRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ColumnRange.java index e932d4f2364681..70c7a519b21b99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ColumnRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ColumnRange.java @@ -30,7 +30,7 @@ /** ColumnRange */ public class ColumnRange { - private final RangeSet rangeSet; + public final RangeSet rangeSet; public ColumnRange() { rangeSet = ImmutableRangeSet.of(); @@ -45,9 +45,25 @@ public ColumnRange(RangeSet rangeSet) { } public ColumnRange intersect(ColumnRange range) { - RangeSet newSet = TreeRangeSet.create(); - range.rangeSet.asRanges().forEach(r -> newSet.addAll(rangeSet.subRangeSet(r))); - return new ColumnRange(newSet); + return new ColumnRange(intersect(rangeSet, range.rangeSet)); + } + + /** intersect */ + public static > RangeSet intersect(RangeSet r1, RangeSet r2) { + RangeSet bigRangeSet; + RangeSet smallRangeSet; + if (r1.asRanges().size() < r2.asRanges().size()) { + bigRangeSet = r2; + smallRangeSet = r1; + } else { + bigRangeSet = r1; + smallRangeSet = r2; + } + RangeSet newSet = TreeRangeSet.create(); + for (Range smallRange : smallRangeSet.asRanges()) { + newSet.addAll(bigRangeSet.subRangeSet(smallRange)); + } + return newSet; } public ColumnRange union(ColumnRange range) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java index ecf8a26724113f..e0d2df9c0f25cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneListPartitionEvaluator.java @@ -17,14 +17,20 @@ package org.apache.doris.nereids.rules.expression.rules; +import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionKey; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.types.BooleanType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -58,23 +64,74 @@ public K getPartitionIdent() { @Override public List> getOnePartitionInputs() { + if (partitionSlots.size() == 1) { + // fast path + return getInputsByOneSlot(); + } else { + // slow path + return getInputsByMultiSlots(); + } + } + + private List> getInputsByOneSlot() { + ImmutableList.Builder> inputs + = ImmutableList.builderWithExpectedSize(partitionItem.getItems().size()); + Slot slot = partitionSlots.get(0); + for (PartitionKey item : partitionItem.getItems()) { + LiteralExpr legacy = item.getKeys().get(0); + inputs.add(ImmutableMap.of( + slot, + new PartitionSlotInput(Literal.fromLegacyLiteral(legacy, legacy.getType()), ImmutableMap.of())) + ); + } + return inputs.build(); + } + + private List> getInputsByMultiSlots() { return partitionItem.getItems().stream() - .map(keys -> { - List literals = keys.getKeys() - .stream() - .map(literal -> Literal.fromLegacyLiteral(literal, literal.getType())) - .collect(ImmutableList.toImmutableList()); - - return IntStream.range(0, partitionSlots.size()) - .mapToObj(index -> { - Slot partitionSlot = partitionSlots.get(index); - // partitionSlot will be replaced to this literal - Literal literal = literals.get(index); - // list partition don't need to compute the slot's range, - // so we pass through an empty map - return Pair.of(partitionSlot, new PartitionSlotInput(literal, ImmutableMap.of())); - }).collect(ImmutableMap.toImmutableMap(Pair::key, Pair::value)); - }).collect(ImmutableList.toImmutableList()); + .map(keys -> { + List literals = keys.getKeys() + .stream() + .map(literal -> Literal.fromLegacyLiteral(literal, literal.getType())) + .collect(ImmutableList.toImmutableList()); + + return IntStream.range(0, partitionSlots.size()) + .mapToObj(index -> { + Slot partitionSlot = partitionSlots.get(index); + // partitionSlot will be replaced to this literal + Literal literal = literals.get(index); + // list partition don't need to compute the slot's range, + // so we pass through an empty map + return Pair.of(partitionSlot, new PartitionSlotInput(literal, ImmutableMap.of())); + }).collect(ImmutableMap.toImmutableMap(Pair::key, Pair::value)); + }).collect(ImmutableList.toImmutableList()); + } + + @Override + public Expression visitInPredicate(InPredicate inPredicate, Map context) { + if (!inPredicate.optionsAreLiterals()) { + return super.visitInPredicate(inPredicate, context); + } + + Expression newCompareExpr = inPredicate.getCompareExpr().accept(this, context); + if (newCompareExpr.isNullLiteral()) { + return new NullLiteral(BooleanType.INSTANCE); + } + + try { + // fast path + boolean contains = inPredicate.getLiteralOptionSet().contains(newCompareExpr); + if (contains) { + return BooleanLiteral.TRUE; + } + if (inPredicate.optionsContainsNullLiteral()) { + return new NullLiteral(BooleanType.INSTANCE); + } + return BooleanLiteral.FALSE; + } catch (Throwable t) { + // slow path + return super.visitInPredicate(inPredicate, context); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index 2bedd6db2336cb..2a58b4d7c73924 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -19,7 +19,6 @@ import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.catalog.PartitionKey; -import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.catalog.Type; import org.apache.doris.common.Pair; @@ -63,8 +62,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Range; -import com.google.common.collect.RangeSet; -import com.google.common.collect.TreeRangeSet; import java.util.ArrayList; import java.util.HashMap; @@ -356,21 +353,38 @@ public EvaluateRangeResult visitNullSafeEqual(NullSafeEqual nullSafeEqual, Evalu @Override public EvaluateRangeResult visitInPredicate(InPredicate inPredicate, EvaluateRangeInput context) { - EvaluateRangeResult result = evaluateChildrenThenThis(inPredicate, context); - if (!(result.result instanceof InPredicate)) { + if (!inPredicate.optionsAreLiterals()) { + EvaluateRangeResult result = evaluateChildrenThenThis(inPredicate, context); + if (!(result.result instanceof InPredicate)) { + return result; + } + result = result.withRejectNot(false); return result; } - inPredicate = (InPredicate) result.result; - Map exprRanges = result.childrenResult.get(0).columnRanges; - if (exprRanges.containsKey(inPredicate.getCompareExpr()) - && inPredicate.getOptions().stream().allMatch(Literal.class::isInstance)) { - Expression compareExpr = inPredicate.getCompareExpr(); - ColumnRange compareExprRange = result.childrenResult.get(0).columnRanges.get(compareExpr); - RangeSet union = TreeRangeSet.create(); - for (Expression expr : inPredicate.getOptions()) { - union.addAll(compareExprRange.intersect(ColumnRange.singleton((Literal) expr)).asRanges()); + + EvaluateRangeResult leftResult = inPredicate.getCompareExpr().accept(this, context); + Map exprRanges = leftResult.columnRanges; + if (!exprRanges.containsKey(inPredicate.getCompareExpr())) { + EvaluateRangeResult result = evaluateChildrenThenThis(inPredicate, context); + if (!(result.result instanceof InPredicate)) { + return result; } - result = intersectSlotRange(result, exprRanges, compareExpr, new ColumnRange(union)); + result = result.withRejectNot(false); + return result; + } + + Expression compareExpr = inPredicate.getCompareExpr(); + ColumnRange compareExprRange = exprRanges.get(compareExpr); + ColumnRange optionsRange = inPredicate.getLiteralOptionsRangeSet(); + + ColumnRange intersect = optionsRange.intersect(compareExprRange); + Map newColumnRanges = replaceExprRange(exprRanges, compareExpr, intersect); + + EvaluateRangeResult result; + if (intersect.isEmptyRange()) { + result = new EvaluateRangeResult(BooleanLiteral.FALSE, newColumnRanges, ImmutableList.of(leftResult)); + } else { + result = new EvaluateRangeResult(inPredicate, newColumnRanges, ImmutableList.of(leftResult)); } result = result.withRejectNot(false); return result; @@ -399,7 +413,7 @@ public EvaluateRangeResult visitAnd(And and, EvaluateRangeInput context) { EvaluateRangeResult result = evaluateChildrenThenThis(and, context); EvaluateRangeResult andResult = result.childrenResult.get(0); - for (int i = 1; i < andResult.childrenResult.size(); i++) { + for (int i = 1; i < result.childrenResult.size(); i++) { andResult = mergeRanges(result.result, andResult, result.childrenResult.get(i), context.rangeMap, (leftRange, rightRange) -> leftRange.intersect(rightRange)); @@ -616,8 +630,7 @@ private List toNereidsLiterals(PartitionKey partitionKey) { private List toSingleNereidsLiteral(PartitionKey partitionKey) { List keys = partitionKey.getKeys(); LiteralExpr literalExpr = keys.get(0); - PrimitiveType primitiveType = partitionKey.getTypes().get(0); - Type type = Type.fromPrimitiveType(primitiveType); + Type type = literalExpr.getType(); return ImmutableList.of(Literal.fromLegacyLiteral(literalExpr, type)); } @@ -626,8 +639,7 @@ private List toMultiNereidsLiterals(PartitionKey partitionKey) { List literals = Lists.newArrayListWithCapacity(keys.size()); for (int i = 0; i < keys.size(); i++) { LiteralExpr literalExpr = keys.get(i); - PrimitiveType primitiveType = partitionKey.getTypes().get(i); - Type type = Type.fromPrimitiveType(primitiveType); + Type type = literalExpr.getType(); literals.add(Literal.fromLegacyLiteral(literalExpr, type)); } return literals; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java index 16006c02080be4..c739eda4cc996a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OrToIn.java @@ -80,7 +80,6 @@ public Expression rewriteTree(Expression expr, ExpressionRewriteContext context) ExpressionBottomUpRewriter simplify = ExpressionRewrite.bottomUp(SimplifyRange.INSTANCE); expr = simplify.rewrite(expr, context); return rewriteTree(expr); - } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPredicateToRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPredicateToRange.java index 4e4fabed47c194..428d5014b0eb5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPredicateToRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPredicateToRange.java @@ -84,7 +84,7 @@ public RangeSet visitAnd(And and, Void context) { intersects = childRanges; continue; } - intersects = intersection(childRanges, intersects); + intersects = ColumnRange.intersect(childRanges, intersects); if (intersects.isEmpty()) { break; } @@ -165,12 +165,9 @@ public RangeSet visitInPredicate(InPredicate inPredicate, Void Expression compareExpr = inPredicate.getCompareExpr(); if (compareExpr instanceof SlotReference) { SlotReference slot = (SlotReference) compareExpr; - if (slotIds.contains((slot).getExprId().asInt())) { + if (slotIds.contains((slot).getExprId().asInt()) && inPredicate.optionsAreLiterals()) { RangeSet union = TreeRangeSet.create(); for (Expression option : inPredicate.getOptions()) { - if (!(option instanceof Literal)) { - return null; - } Literal literal = (Literal) option; union.addAll( toRangeSet(slot, literal, BoundType.CLOSED, literal, BoundType.CLOSED) @@ -258,12 +255,4 @@ private RangeSet toRangeSet(SlotReference slotReference, Range range = Range.range(lowerBound, lowerBoundType, upperBound, upperBoundType); return ImmutableRangeSet.of(range); } - - public static > RangeSet intersection(RangeSet rangeSet1, RangeSet rangeSet2) { - RangeSet result = TreeRangeSet.create(); - for (Range range : rangeSet1.asRanges()) { - result.addAll(rangeSet2.subRangeSet(range)); - } - return result; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java index a7b8d41766a928..c5bde0b66940b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java @@ -406,6 +406,32 @@ public boolean isKeyColumnFromTable() { && ((SlotReference) this).getColumn().get().isKey(); } + /** containsNullLiteralChildren */ + public boolean containsNullLiteralChildren() { + return getOrInitMutableState("CONTAINS_NULL_LITERAL_CHILDREN", () -> { + for (Expression child : children) { + if (child instanceof NullLiteral) { + return true; + } + } + return false; + }); + } + + /** allChildrenAreLiteral */ + public boolean allChildrenAreLiteral() { + return getOrInitMutableState("ALL_CHILDREN_ARE_LITERAL", () -> { + boolean allLiteral = true; + for (Expression child : getArguments()) { + if (!(child instanceof Literal)) { + allLiteral = false; + break; + } + } + return allLiteral; + }); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java index 5a9c9aaecc172a..03f2dd2f313925 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/InPredicate.java @@ -19,8 +19,11 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.rules.expression.rules.ColumnBound; +import org.apache.doris.nereids.rules.expression.rules.ColumnRange; import org.apache.doris.nereids.trees.expressions.literal.ComparableLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; @@ -29,10 +32,14 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; import java.util.Collection; import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; import java.util.stream.Collectors; /** @@ -41,7 +48,6 @@ * 3 in (1, null, 3, 4) => null */ public class InPredicate extends Expression { - private final Expression compareExpr; private final List options; @@ -61,6 +67,56 @@ public R accept(ExpressionVisitor visitor, C context) { return visitor.visitInPredicate(this, context); } + /** optionsAreLiterals */ + public boolean optionsAreLiterals() { + return getOrInitMutableState("ALL_CHILDREN_ARE_LITERALS", () -> { + boolean allChildrenAreLiterals = true; + for (Expression option : options) { + if (!(option instanceof Literal)) { + allChildrenAreLiterals = false; + break; + } + } + return allChildrenAreLiterals; + }); + } + + /** getLiteralOptionSet */ + public Set getLiteralOptionSet() { + return getOrInitMutableState("OPTIONS_SET", () -> { + TreeSet literals = new TreeSet<>(); + for (Expression option : options) { + literals.add((Literal) option); + } + return literals; + }); + } + + /** optionsContainsNullLiteral */ + public boolean optionsContainsNullLiteral() { + return getOrInitMutableState("OPTIONS_CONTAINS_NULL_LITERAL", () -> { + boolean containsNull = false; + for (Expression option : options) { + if (option instanceof NullLiteral) { + containsNull = true; + break; + } + } + return containsNull; + }); + } + + /** getLiteralOptionsRangeSet */ + public ColumnRange getLiteralOptionsRangeSet() { + return getOrInitMutableState("OPTIONS_RANGE_SET", () -> { + RangeSet union = TreeRangeSet.create(); + for (Expression expr : getOptions()) { + union.add(ColumnBound.singleton((Literal) expr)); + } + return new ColumnRange(union); + }); + } + @Override public DataType getDataType() throws UnboundException { return BooleanType.INSTANCE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 60ffedcedbe755..d367ccfcf9ccfa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.expressions.literal; import org.apache.doris.analysis.BoolLiteral; +import org.apache.doris.analysis.IntLiteral; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.catalog.MysqlColType; import org.apache.doris.catalog.Type; @@ -33,6 +34,7 @@ import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.LargeIntType; @@ -292,8 +294,28 @@ public static Literal fromLegacyLiteral(LiteralExpr literalExpr, Type type) { } else if (literalExpr instanceof org.apache.doris.analysis.NullLiteral) { return new NullLiteral(dataType); } - // fast path switch (type.getPrimitiveType()) { + case TINYINT: { + IntLiteral intLiteral = (IntLiteral) literalExpr; + return new TinyIntLiteral((byte) intLiteral.getValue()); + } + case SMALLINT: { + IntLiteral intLiteral = (IntLiteral) literalExpr; + return new SmallIntLiteral((short) intLiteral.getValue()); + } + case INT: { + IntLiteral intLiteral = (IntLiteral) literalExpr; + return new IntegerLiteral((int) intLiteral.getValue()); + } + case BIGINT: { + IntLiteral intLiteral = (IntLiteral) literalExpr; + return new BigIntLiteral(intLiteral.getValue()); + } + case LARGEINT: { + org.apache.doris.analysis.LargeIntLiteral intLiteral + = (org.apache.doris.analysis.LargeIntLiteral) literalExpr; + return new LargeIntLiteral(intLiteral.getRealValue()); + } case DATEV2: { org.apache.doris.analysis.DateLiteral dateLiteral = (org.apache.doris.analysis.DateLiteral) literalExpr; return new DateV2Literal(dateLiteral.getYear(), dateLiteral.getMonth(), dateLiteral.getDay()); @@ -302,42 +324,68 @@ public static Literal fromLegacyLiteral(LiteralExpr literalExpr, Type type) { org.apache.doris.analysis.DateLiteral dateLiteral = (org.apache.doris.analysis.DateLiteral) literalExpr; return new DateLiteral(dateLiteral.getYear(), dateLiteral.getMonth(), dateLiteral.getDay()); } + case DATETIME: { + org.apache.doris.analysis.DateLiteral dateLiteral = (org.apache.doris.analysis.DateLiteral) literalExpr; + return new DateTimeLiteral( + DateTimeType.INSTANCE, dateLiteral.getYear(), dateLiteral.getMonth(), dateLiteral.getDay(), + dateLiteral.getHour(), dateLiteral.getMinute(), dateLiteral.getSecond(), + dateLiteral.getMicrosecond() + ); + } + case DATETIMEV2: { + org.apache.doris.analysis.DateLiteral dateLiteral = (org.apache.doris.analysis.DateLiteral) literalExpr; + return new DateTimeV2Literal( + (DateTimeV2Type) DateType.fromCatalogType(type), + dateLiteral.getYear(), dateLiteral.getMonth(), dateLiteral.getDay(), + dateLiteral.getHour(), dateLiteral.getMinute(), dateLiteral.getSecond(), + dateLiteral.getMicrosecond() + ); + } case BOOLEAN: { return ((BoolLiteral) literalExpr).getValue() ? BooleanLiteral.TRUE : BooleanLiteral.FALSE; } - default: { + case CHAR: { + return new CharLiteral(literalExpr.getStringValue(), ((CharType) dataType).getLen()); + } + case VARCHAR: { + return new VarcharLiteral(literalExpr.getStringValue(), ((VarcharType) dataType).getLen()); + } + case STRING: { + return new StringLiteral(literalExpr.getStringValue()); + } + case FLOAT: { + org.apache.doris.analysis.FloatLiteral floatLiteral + = (org.apache.doris.analysis.FloatLiteral) literalExpr; + return new FloatLiteral((float) floatLiteral.getValue()); + } + case DOUBLE: { + org.apache.doris.analysis.FloatLiteral floatLiteral + = (org.apache.doris.analysis.FloatLiteral) literalExpr; + return new DoubleLiteral(floatLiteral.getValue()); + } + case DECIMALV2: { + org.apache.doris.analysis.DecimalLiteral decimalLiteral + = (org.apache.doris.analysis.DecimalLiteral) literalExpr; + BigDecimal clonedValue = decimalLiteral.getValue().add(BigDecimal.ZERO); + return new DecimalLiteral((DecimalV2Type) dataType, clonedValue); } - } - // slow path - String stringValue = literalExpr.getStringValue(); - switch (type.getPrimitiveType()) { - case TINYINT: return new TinyIntLiteral(Byte.parseByte(stringValue)); - case SMALLINT: return new SmallIntLiteral(Short.parseShort(stringValue)); - case INT: return new IntegerLiteral(Integer.parseInt(stringValue)); - case BIGINT: return new BigIntLiteral(Long.parseLong(stringValue)); - case LARGEINT: return new LargeIntLiteral(new BigInteger(stringValue)); - case STRING: return new StringLiteral(stringValue); - case CHAR: return new CharLiteral(stringValue, ((CharType) dataType).getLen()); - case VARCHAR: return new VarcharLiteral(stringValue, ((VarcharType) dataType).getLen()); - case FLOAT: return new FloatLiteral(Float.parseFloat(stringValue)); - case DOUBLE: return new DoubleLiteral(Double.parseDouble(stringValue)); - case DECIMALV2: return new DecimalLiteral((DecimalV2Type) dataType, new BigDecimal(stringValue)); case DECIMAL32: case DECIMAL64: case DECIMAL128: case DECIMAL256: { - return new DecimalV3Literal((DecimalV3Type) dataType, new BigDecimal(stringValue)); + org.apache.doris.analysis.DecimalLiteral decimalLiteral + = (org.apache.doris.analysis.DecimalLiteral) literalExpr; + BigDecimal clonedValue = decimalLiteral.getValue().add(BigDecimal.ZERO); + return new DecimalV3Literal((DecimalV3Type) dataType, clonedValue); } - case DATETIME: return new DateTimeLiteral(stringValue); - case DATETIMEV2: return new DateTimeV2Literal(stringValue); - case JSONB: return new JsonLiteral(stringValue); - case IPV4: return new IPv4Literal(stringValue); - case IPV6: return new IPv6Literal(stringValue); + case JSONB: return new JsonLiteral(literalExpr.getStringValue()); + case IPV4: return new IPv4Literal(literalExpr.getStringValue()); + case IPV6: return new IPv6Literal(literalExpr.getStringValue()); default: { + throw new AnalysisException("Unsupported convert the " + literalExpr.getType() + + " of legacy literal to nereids literal"); } } - throw new AnalysisException("Unsupported convert the " + literalExpr.getType() - + " of legacy literal to nereids literal"); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionColumnFilterConverter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionColumnFilterConverter.java index be575f73c9213b..d2db18d72c4863 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionColumnFilterConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionColumnFilterConverter.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.SlotRef; +import org.apache.doris.common.Config; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -99,6 +100,10 @@ public Expression visitComparisonPredicate(ComparisonPredicate predicate, Void u @Override public Expression visitInPredicate(InPredicate predicate, Void unused) { + if (predicate.getOptions().size() > Config.max_distribution_pruner_recursion_depth) { + return null; + } + List literals = predicate.getOptions().stream() .map(expr -> ((Expr) ((Literal) expr).toLegacyLiteral())) .collect(Collectors.toList()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 4f0492ec63dc8a..15c646a3533d76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -189,13 +189,13 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, groupExpression, logicalProperties); Preconditions.checkArgument(selectedPartitionIds != null, "selectedPartitionIds can not be null"); - this.selectedTabletIds = ImmutableList.copyOf(selectedTabletIds); + this.selectedTabletIds = Utils.fastToImmutableList(selectedTabletIds); this.partitionPruned = partitionPruned; this.selectedIndexId = selectedIndexId <= 0 ? getTable().getBaseIndexId() : selectedIndexId; this.indexSelected = indexSelected; this.preAggStatus = preAggStatus; - this.manuallySpecifiedPartitions = ImmutableList.copyOf(specifiedPartitions); - this.manuallySpecifiedTabletIds = ImmutableList.copyOf(specifiedTabletIds); + this.manuallySpecifiedPartitions = Utils.fastToImmutableList(specifiedPartitions); + this.manuallySpecifiedTabletIds = Utils.fastToImmutableList(specifiedTabletIds); if (selectedPartitionIds.isEmpty()) { this.selectedPartitionIds = ImmutableList.of(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index d26f73195f79c4..8ac422d3b64283 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -907,8 +907,7 @@ public static Optional checkAndMaybeCommute(Expression expression) { if (!predicate.getCompareExpr().isSlot()) { return Optional.empty(); } - return Optional.ofNullable(predicate.getOptions().stream() - .allMatch(Expression::isLiteral) ? expression : null); + return Optional.ofNullable(predicate.optionsAreLiterals() ? expression : null); } else if (expression instanceof ComparisonPredicate) { ComparisonPredicate predicate = ((ComparisonPredicate) expression); if (predicate.left() instanceof Literal) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index f3d81529513104..2de77f3740083c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -156,6 +156,7 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_REWRITE_ELEMENT_AT_TO_SLOT = "enable_rewrite_element_at_to_slot"; public static final String ENABLE_ODBC_TRANSCATION = "enable_odbc_transcation"; + public static final String ENABLE_BINARY_SEARCH_FILTERING_PARTITIONS = "enable_binary_search_filtering_partitions"; public static final String ENABLE_SQL_CACHE = "enable_sql_cache"; public static final String ENABLE_QUERY_CACHE = "enable_query_cache"; public static final String QUERY_CACHE_FORCE_REFRESH = "query_cache_force_refresh"; @@ -1041,6 +1042,16 @@ public enum IgnoreSplitType { @VariableMgr.VarAttr(name = ENABLE_ODBC_TRANSCATION) public boolean enableOdbcTransaction = false; + @VariableMgr.VarAttr( + name = ENABLE_BINARY_SEARCH_FILTERING_PARTITIONS, + fuzzy = true, + description = { + "是否允许使用二分查找算法去过滤分区。默认开。", + "Whether to allow use binary search algorithm to filter partitions. ON by default." + } + ) + public boolean enableBinarySearchFilteringPartitions = true; + @VariableMgr.VarAttr(name = ENABLE_SQL_CACHE, fuzzy = true) public boolean enableSqlCache = false;