From 9702b02ef0e5fe91253dd37ec4afed990130488f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 7 Mar 2023 17:39:28 +0800 Subject: [PATCH] [Improvement](datev2) push down datev2 predicates with date literal --- .../java/org/apache/doris/catalog/Type.java | 4 ++ .../org/apache/doris/analysis/Analyzer.java | 2 + .../doris/analysis/BinaryPredicate.java | 20 ++++++ .../apache/doris/analysis/DateLiteral.java | 12 ++++ .../doris/rewrite/EraseRedundantCastExpr.java | 70 +++++++++++++++++++ 5 files changed, 108 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/rewrite/EraseRedundantCastExpr.java diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index a4bd07d33a6d52..cd9a43f9e935e8 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -507,6 +507,10 @@ public boolean isDateV2() { return isScalarType(PrimitiveType.DATEV2); } + public boolean isDateV2OrDateTimeV2() { + return isScalarType(PrimitiveType.DATEV2) || isScalarType(PrimitiveType.DATETIMEV2); + } + /** * Returns true if Impala supports this type in the metdata. It does not mean we * can manipulate data of this type. For tables that contain columns with these diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java index dc85d48f579d96..cae02941b93fe1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -47,6 +47,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.rewrite.BetweenToCompoundRule; import org.apache.doris.rewrite.CompoundPredicateWriteRule; +import org.apache.doris.rewrite.EraseRedundantCastExpr; import org.apache.doris.rewrite.ExprRewriteRule; import org.apache.doris.rewrite.ExprRewriter; import org.apache.doris.rewrite.ExtractCommonFactorsRule; @@ -415,6 +416,7 @@ public GlobalState(Env env, ConnectContext context) { rules.add(RewriteImplicitCastRule.INSTANCE); rules.add(RoundLiteralInBinaryPredicatesRule.INSTANCE); rules.add(FoldConstantsRule.INSTANCE); + rules.add(EraseRedundantCastExpr.INSTANCE); rules.add(RewriteFromUnixTimeRule.INSTANCE); rules.add(CompoundPredicateWriteRule.INSTANCE); rules.add(RewriteDateLiteralRule.INSTANCE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java index ad0f9e6a2385b9..c0bff0c759ab2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java @@ -364,6 +364,26 @@ private Type getCmpType() throws AnalysisException { } else if (getChild(1).getType().isDateV2() && (getChild(0).getType().isDate() || getChild(0).getType().isDateV2())) { return getChild(1).getType(); + } else if (getChild(0).getType().isDateV2() + && (getChild(1).getType().isStringType() && getChild(1) instanceof StringLiteral)) { + if (((StringLiteral) getChild(1)).canConvertToDateV2(Type.DATEV2)) { + return Type.DATEV2; + } else { + return Type.DATETIMEV2; + } + } else if (getChild(1).getType().isDateV2() + && (getChild(0).getType().isStringType() && getChild(0) instanceof StringLiteral)) { + if (((StringLiteral) getChild(0)).canConvertToDateV2(Type.DATEV2)) { + return Type.DATEV2; + } else { + return Type.DATETIMEV2; + } + } else if (getChild(0).getType().isDatetimeV2() + && (getChild(1).getType().isStringType() && getChild(1) instanceof StringLiteral)) { + return getChild(0).getType(); + } else if (getChild(1).getType().isDatetimeV2() + && (getChild(0).getType().isStringType() && getChild(0) instanceof StringLiteral)) { + return getChild(1).getType(); } else { return Type.DATETIME; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index d1f9e8a2709b6b..969ae83877ef60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -677,6 +677,18 @@ public void castToDate() { second = 0; } + public boolean hasTimePart() { + if (this.type.isDateV2() || this.type.isDate()) { + return false; + } else { + if (hour != 0 || minute != 0 || second != 0) { + return true; + } else { + return !this.type.isDatetime() && microsecond != 0; + } + } + } + private long makePackedDatetime() { long ymd = ((year * 13 + month) << 5) | day; long hms = (hour << 12) | (minute << 6) | second; diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/EraseRedundantCastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/EraseRedundantCastExpr.java new file mode 100644 index 00000000000000..829562f783329d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/EraseRedundantCastExpr.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is copied from +// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/FoldConstantsRule.java +// and modified by Doris + +package org.apache.doris.rewrite; + + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.CastExpr; +import org.apache.doris.analysis.DateLiteral; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.common.AnalysisException; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * This rule erase redundant cast between datev2/datetimev2 and date literal. + * + * for expression with pattern: + * BinaryPredicate + * CAST (SlotRef(TYPE = DATEV2/DATETIMEV2), TYPE = DATETIME) + * DATELITERAL(TYPE = DATETIME) + * + * this rule will be applied. + */ +public class EraseRedundantCastExpr implements ExprRewriteRule { + private static final Logger LOG = LogManager.getLogger(EraseRedundantCastExpr.class); + + public static EraseRedundantCastExpr INSTANCE = new EraseRedundantCastExpr(); + + @Override + public Expr apply(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseType) throws AnalysisException { + // BinaryPredicate + // CAST (SlotRef(TYPE = DATEV2/DATETIMEV2), TYPE = DATETIME) + // DATELITERAL(TYPE = DATETIME) + if (!(expr instanceof BinaryPredicate) || !(expr.getChild(0) instanceof CastExpr) + || !expr.getChild(0).getType().isDatetime() || !(expr.getChild(1)).getType().isDatetime() + || !expr.getChild(0).getChild(0).getType().isDateV2OrDateTimeV2() + || !(expr.getChild(0).getChild(0) instanceof SlotRef) + || !(expr.getChild(1) instanceof DateLiteral)) { + return expr; + } + + if (!((DateLiteral) expr.getChild(1)).hasTimePart() + || !(expr.getChild(1).getType().isDatetime() && expr.getChild(0).getChild(0).getType().isDateV2())) { + expr.getChild(1).setType(expr.getChild(0).getChild(0).getType()); + expr.setChild(0, expr.getChild(0).getChild(0)); + } + return expr; + } +}