From 18c571a41cd0d29a498c2d9d23db1b74a6ef1977 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 12 Mar 2019 16:05:33 +0800 Subject: [PATCH 1/3] Support calculate unix_timestamp() on Frontend --- .../org/apache/doris/rewrite/FEFunctions.java | 26 ++++++++++++--- .../doris/rewrite/FoldConstantsRule.java | 32 ++++++++++++------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index f816cc1828c810..60cb5362388f1d 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -17,8 +17,6 @@ package org.apache.doris.rewrite; -import org.apache.commons.lang.time.DateFormatUtils; -import org.apache.commons.lang.time.DateUtils; import org.apache.doris.analysis.DateLiteral; import org.apache.doris.analysis.DecimalLiteral; import org.apache.doris.analysis.FloatLiteral; @@ -29,6 +27,11 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.commons.lang.time.DateFormatUtils; +import org.apache.commons.lang.time.DateUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.math.BigDecimal; import java.math.BigInteger; import java.text.ParseException; @@ -41,6 +44,7 @@ * when you add a new function, please ensure the name, argTypes , returnType and compute logic are consistent with BE's function */ public class FEFunctions { + private static final Logger LOG = LogManager.getLogger(FEFunctions.class); /** * date and time function */ @@ -72,7 +76,6 @@ public static DateLiteral dateSub(LiteralExpr date, LiteralExpr day) throws Anal return new DateLiteral(DateFormatUtils.format(d, "yyyy-MM-dd HH:mm:ss"), Type.DATETIME); } - @FEFunction(name = "year", argTypes = { "DATETIME" }, returnType = "INT") public static IntLiteral year(LiteralExpr arg) throws AnalysisException { long timestamp = getTime(arg); @@ -97,6 +100,12 @@ public static IntLiteral day(LiteralExpr arg) throws AnalysisException { return new IntLiteral(instance.get(Calendar.DAY_OF_MONTH), Type.INT); } + @FEFunction(name = "unix_timestamp", argTypes = { "DATETIME" }, returnType = "INT") + public static IntLiteral unix_timestamp(LiteralExpr arg) throws AnalysisException { + long timestamp = getTime(arg); + return new IntLiteral(timestamp / 1000, Type.INT); + } + private static long getTime(LiteralExpr expr) throws AnalysisException { try { String[] parsePatterns = { "yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss" }; @@ -165,7 +174,6 @@ public static FloatLiteral addDouble(LiteralExpr first, LiteralExpr second) thro public static DecimalLiteral addDecimal(LiteralExpr first, LiteralExpr second) throws AnalysisException { BigDecimal left = new BigDecimal(first.getStringValue()); BigDecimal right = new BigDecimal(second.getStringValue()); - BigDecimal result = left.add(right); return new DecimalLiteral(result); } @@ -174,7 +182,6 @@ public static DecimalLiteral addDecimal(LiteralExpr first, LiteralExpr second) t public static LargeIntLiteral addBigInt(LiteralExpr first, LiteralExpr second) throws AnalysisException { BigInteger left = new BigInteger(first.getStringValue()); BigInteger right = new BigInteger(second.getStringValue()); - BigInteger result = left.add(right); return new LargeIntLiteral(result.toString()); } @@ -256,4 +263,13 @@ public static DecimalLiteral divideDecimal(LiteralExpr first, LiteralExpr second BigDecimal result = left.divide(right); return new DecimalLiteral(result); } + + public static void main(String[] args) { + try { + IntLiteral timestamp = unix_timestamp(new DateLiteral("2018-01-01", Type.DATE)); + System.out.println(timestamp); + } catch (AnalysisException e) { + e.printStackTrace(); + } + } } diff --git a/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java b/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java index 7d5a7da63aa5bd..16b06e50b9dcc6 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java @@ -18,8 +18,6 @@ package org.apache.doris.rewrite; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.ArithmeticExpr; import org.apache.doris.analysis.CastExpr; @@ -32,6 +30,14 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; @@ -39,12 +45,6 @@ import java.util.Collection; import java.util.List; import java.util.Objects; -import java.util.Set; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; /** * This rule replaces a constant Expr with its equivalent LiteralExpr by evaluating the @@ -62,6 +62,8 @@ * cast('2016-11-09' as timestamp) --> TIMESTAMP '2016-11-09 00:00:00' */ public class FoldConstantsRule implements ExprRewriteRule { + private static final Logger LOG = LogManager.getLogger(FoldConstantsRule.class); + public static ExprRewriteRule INSTANCE = new FoldConstantsRule(); private ImmutableMultimap functions; @@ -141,6 +143,7 @@ private Expr simplify(Expr constExpr) throws AnalysisException { try { return invoker.invoke(constExpr.getChildrenWithoutCast()); } catch (AnalysisException e) { + LOG.debug("failed to invoke", e); return constExpr; } } @@ -189,7 +192,7 @@ private synchronized void registerFunctions() { for (String type : annotation.argTypes()) { argTypes.add(ScalarType.createType(type)); } - FEFunctionSignature signature = new FEFunctionSignature(name, + FEFunctionSignature signature = new FEFunctionSignature(name, argTypes.toArray(new ScalarType[argTypes.size()]), returnType); mapBuilder.put(name, new FEFunctionInvoker(method, signature)); } @@ -225,10 +228,9 @@ public LiteralExpr invoke(List args) throws AnalysisException { try { return (LiteralExpr) method.invoke(null, args.toArray()); } catch (InvocationTargetException | IllegalAccessException | IllegalArgumentException e) { - throw new AnalysisException(e.getLocalizedMessage()); + throw new AnalysisException(e.getLocalizedMessage(), e); } } - } public static class FEFunctionSignature { @@ -254,6 +256,14 @@ public String getName() { return name; } + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("FEFunctionSignature. name: ").append(name).append(", return: ").append(returnType); + sb.append(", args: ").append(Joiner.on(",").join(argTypes)); + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) From 6986ece2081eb0b65a9f884c5423f23d9facec32 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 13 Mar 2019 09:29:50 +0800 Subject: [PATCH 2/3] remove main() --- .../org/apache/doris/rewrite/FEFunctions.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index 60cb5362388f1d..c3b16680eb21b5 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -202,7 +202,6 @@ public static FloatLiteral subtractDouble(LiteralExpr first, LiteralExpr second) public static DecimalLiteral subtractDecimal(LiteralExpr first, LiteralExpr second) throws AnalysisException { BigDecimal left = new BigDecimal(first.getStringValue()); BigDecimal right = new BigDecimal(second.getStringValue()); - BigDecimal result = left.subtract(right); return new DecimalLiteral(result); } @@ -211,7 +210,6 @@ public static DecimalLiteral subtractDecimal(LiteralExpr first, LiteralExpr seco public static LargeIntLiteral subtractBigInt(LiteralExpr first, LiteralExpr second) throws AnalysisException { BigInteger left = new BigInteger(first.getStringValue()); BigInteger right = new BigInteger(second.getStringValue()); - BigInteger result = left.subtract(right); return new LargeIntLiteral(result.toString()); } @@ -220,7 +218,6 @@ public static LargeIntLiteral subtractBigInt(LiteralExpr first, LiteralExpr seco public static IntLiteral multiplyInt(LiteralExpr first, LiteralExpr second) throws AnalysisException { long left = first.getLongValue(); long right = second.getLongValue(); - long result = Math.multiplyExact(left, right); return new IntLiteral(result, Type.BIGINT); } @@ -235,7 +232,6 @@ public static FloatLiteral multiplyDouble(LiteralExpr first, LiteralExpr second) public static DecimalLiteral multiplyDecimal(LiteralExpr first, LiteralExpr second) throws AnalysisException { BigDecimal left = new BigDecimal(first.getStringValue()); BigDecimal right = new BigDecimal(second.getStringValue()); - BigDecimal result = left.multiply(right); return new DecimalLiteral(result); } @@ -244,7 +240,6 @@ public static DecimalLiteral multiplyDecimal(LiteralExpr first, LiteralExpr seco public static LargeIntLiteral multiplyBigInt(LiteralExpr first, LiteralExpr second) throws AnalysisException { BigInteger left = new BigInteger(first.getStringValue()); BigInteger right = new BigInteger(second.getStringValue()); - BigInteger result = left.multiply(right); return new LargeIntLiteral(result.toString()); } @@ -259,17 +254,7 @@ public static FloatLiteral divideDouble(LiteralExpr first, LiteralExpr second) t public static DecimalLiteral divideDecimal(LiteralExpr first, LiteralExpr second) throws AnalysisException { BigDecimal left = new BigDecimal(first.getStringValue()); BigDecimal right = new BigDecimal(second.getStringValue()); - BigDecimal result = left.divide(right); return new DecimalLiteral(result); } - - public static void main(String[] args) { - try { - IntLiteral timestamp = unix_timestamp(new DateLiteral("2018-01-01", Type.DATE)); - System.out.println(timestamp); - } catch (AnalysisException e) { - e.printStackTrace(); - } - } } From ffdca48a5dc0142ea329ac06a987ed5164fbd8e0 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 13 Mar 2019 09:33:22 +0800 Subject: [PATCH 3/3] add ut --- .../apache/doris/rewrite/FEFunctionsTest.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java diff --git a/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java b/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java new file mode 100644 index 00000000000000..a72f6e8c83e04e --- /dev/null +++ b/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite; + +import org.apache.doris.analysis.DateLiteral; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; + +import org.junit.Assert; +import org.junit.Test; + +/* + * Author: Chenmingyu + * Date: Mar 13, 2019 + */ + +public class FEFunctionsTest { + + @Test + public void unixtimestampTest() { + try { + IntLiteral timestamp = FEFunctions.unix_timestamp(new DateLiteral("2018-01-01", Type.DATE)); + Assert.assertEquals(1514736000, timestamp.getValue()); + } catch (AnalysisException e) { + e.printStackTrace(); + } + } +}