From a60451cf3d0a1f0c679d4ffb5e45308bd689f209 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Tue, 13 Aug 2019 20:24:04 +0800 Subject: [PATCH 01/18] add fe time zone --- .../doris/analysis/FunctionCallExpr.java | 4 ++-- .../apache/doris/common/util/TimeUtils.java | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 6e3cc297a3cded..2b31bc89a918ca 100644 --- a/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -536,7 +536,7 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException { LOG.warn("fn {} not exists", fnName.getFunction()); throw new AnalysisException(getFunctionNotFoundError(collectChildReturnTypes())); } - + /* if (fnName.getFunction().equals("from_unixtime")) { // if has only one child, it has default time format: yyyy-MM-dd HH:mm:ss.SSSSSS if (children.size() > 1) { @@ -552,7 +552,7 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException { } } } - + */ if (fn.getFunctionName().getFunction().equals("time_diff")) { fn.getReturnType().getPrimitiveType().setTimeType(); return; diff --git a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java index 7deeaed1e336c7..fecb0e7d860a71 100644 --- a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -27,6 +27,8 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.VariableMgr; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -116,6 +118,16 @@ public static synchronized String getCurrentFormatTime() { return DATETIME_FORMAT.format(new Date()); } + public static TimeZone getTimeZone() { + String timezone; + if (ConnectContext.get() != null) { + timezone = ConnectContext.get().getSessionVariable().getTimeZone(); + } else { + timezone = VariableMgr.getGlobalSessionVariable().getTimeZone(); + } + return TimeZone.getTimeZone(ZoneId.of(timezone, timeZoneAliasMap)); + } + public static String longToTimeString(long timeStamp, SimpleDateFormat dateFormat) { if (timeStamp <= 0L) { return "N/A"; @@ -124,7 +136,11 @@ public static String longToTimeString(long timeStamp, SimpleDateFormat dateForma } public static synchronized String longToTimeString(long timeStamp) { - return longToTimeString(timeStamp, DATETIME_FORMAT); + //return longToTimeString(timeStamp, DATETIME_FORMAT); + TimeZone timeZone = getTimeZone(); + SimpleDateFormat dateFormatTimeZone = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateFormatTimeZone.setTimeZone(timeZone); + return longToTimeString(timeStamp, dateFormatTimeZone); } public static synchronized Date getTimeAsDate(String timeString) { From 701d4484ea07ded4681762fbd94b35d3e7c3743b Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Wed, 14 Aug 2019 11:18:44 +0800 Subject: [PATCH 02/18] add fe time zone --- .../org/apache/doris/rewrite/FEFunctions.java | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index 1353a0e1086344..5a384c36202561 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -30,18 +30,23 @@ import com.google.common.base.Preconditions; import org.apache.commons.lang.time.DateFormatUtils; import org.apache.commons.lang.time.DateUtils; +import org.apache.doris.common.util.TimeUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.DateTimeFormatterBuilder; import java.math.BigDecimal; import java.math.BigInteger; +import java.sql.Time; import java.text.ParseException; +import java.text.ParsePosition; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.Locale; +import java.util.TimeZone; /** * compute functions in FE. @@ -91,7 +96,7 @@ public static DateLiteral daysAdd(LiteralExpr date, LiteralExpr day) throws Anal @FEFunction(name = "date_format", argTypes = { "DATETIME", "VARCHAR" }, returnType = "VARCHAR") public static StringLiteral dateFormat(LiteralExpr date, StringLiteral fmtLiteral) throws AnalysisException { - String result = dateFormat(new Date(getTime(date)), fmtLiteral.getStringValue()); + String result = dateFormat(getTime(date), fmtLiteral.getStringValue()); return new StringLiteral(result); } @@ -283,18 +288,32 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmt } private static long getTime(LiteralExpr expr) throws AnalysisException { - try { - String[] parsePatterns = { "yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss" }; + + if (expr instanceof DateLiteral) { long time; - if (expr instanceof DateLiteral) { - time = expr.getLongValue(); - } else { - time = DateUtils.parseDate(expr.getStringValue(), parsePatterns).getTime(); - } + time = expr.getLongValue(); + // add time zone offset to get right timestamp + time += DateTimeZone.forID("Asia/Shanghai").getOffset(0) - TimeUtils.getTimeZone().getOffset(0); return time; - } catch (ParseException e) { - throw new AnalysisException(e.getLocalizedMessage()); + } else { + String[] parsePatterns = {"yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss"}; + SimpleDateFormat parser = null; + ParsePosition pos = new ParsePosition(0); + for (int i = 0; i < parsePatterns.length; ++i) { + if (i == 0) { + parser = new SimpleDateFormat(parsePatterns[0]); + parser.setTimeZone(TimeZone.getTimeZone("GMT")); + } else { + parser.applyPattern(parsePatterns[i]); + } + pos.setIndex(0); + Date date = parser.parse(expr.getStringValue(), pos); + if (date != null && pos.getIndex() == expr.getStringValue().length()) { + return date.getTime(); + } + } } + throw new AnalysisException("Unable to parse the date: " + expr.getStringValue()); } private static int calFirstWeekDay(int year, int firstWeekDay) { @@ -308,9 +327,10 @@ private static int calFirstWeekDay(int year, int firstWeekDay) { return firstDay; } - private static String dateFormat(Date date, String pattern) { + private static String dateFormat(long date, String pattern) { DateTimeFormatterBuilder formatterBuilder = new DateTimeFormatterBuilder(); Calendar calendar = Calendar.getInstance(); + calendar.setTimeZone(TimeUtils.getTimeZone()); boolean escaped = false; for (int i = 0; i < pattern.length(); i++) { char character = pattern.charAt(i); @@ -385,7 +405,7 @@ private static String dateFormat(Date date, String pattern) { case 'V': // %V Week (01..53), where Sunday is the first day of the week; used with %X { int week; - calendar.setTime(date); + calendar.setTimeInMillis(date); int firstSunday = calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.SUNDAY); if (calendar.get(Calendar.DATE) <= 7 && calendar.get(Calendar.MONTH) == Calendar.JANUARY && calendar.get(Calendar.DATE) >= firstSunday) { @@ -402,7 +422,7 @@ private static String dateFormat(Date date, String pattern) { formatterBuilder.appendWeekOfWeekyear(2); break; case 'X': // %X Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V - calendar.setTime(date); + calendar.setTimeInMillis(date); if(calendar.get(Calendar.MONTH) == Calendar.JANUARY && calendar.get(Calendar.DATE) < calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.SUNDAY)) { formatterBuilder.appendLiteral(String.valueOf(calendar.get(Calendar.YEAR) - 1)); @@ -417,7 +437,7 @@ private static String dateFormat(Date date, String pattern) { formatterBuilder.appendDayOfWeekText(); break; case 'w': // %w Day of the week (0=Sunday..6=Saturday) - calendar.setTime(date); + calendar.setTimeInMillis(date); calendar.setFirstDayOfWeek(Calendar.SUNDAY); formatterBuilder.appendLiteral(String.valueOf(calendar.get(Calendar.DAY_OF_WEEK) - 1)); break; @@ -429,7 +449,7 @@ private static String dateFormat(Date date, String pattern) { formatterBuilder.appendYear(4, 4); break; case 'D': // %D Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) - calendar.setTime(date); + calendar.setTimeInMillis(date); int day = calendar.get(Calendar.DAY_OF_MONTH); if (day >= 10 && day <= 19) { formatterBuilder.appendLiteral(String.valueOf(day) + "th"); @@ -451,7 +471,7 @@ private static String dateFormat(Date date, String pattern) { } break; case 'U': // %U Week (00..53), where Sunday is the first day of the week - calendar.setTime(date); + calendar.setTimeInMillis(date); if (calendar.get(Calendar.DATE) <= 7 && calendar.get(Calendar.MONTH) == Calendar.JANUARY) { int firstSunday = calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.SUNDAY); formatterBuilder.appendLiteral(String.format("%02d", @@ -466,7 +486,7 @@ private static String dateFormat(Date date, String pattern) { break; case 'u': // %u Week (00..53), where Monday is the first day of the week { - calendar.setTime(date); + calendar.setTimeInMillis(date); int week; int firstMonday = calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.MONDAY); if (calendar.get(Calendar.DATE) <= 7 && calendar.get(Calendar.MONTH) == Calendar.JANUARY) { @@ -497,7 +517,11 @@ else if (character == '%') { } } DateTimeFormatter formatter = formatterBuilder.toFormatter(); - return formatter.withLocale(Locale.US).print(date.getTime()); + + String ss = formatter.withZone(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())).withLocale(Locale.US).print(date); + System.out.println(ss); + return ss; + //return formatter.withLocale(Locale.US).print(date.getTime()); } /** From 194f5e9936015dd87a1fd820dfff476f35653cee Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Wed, 14 Aug 2019 20:48:31 +0800 Subject: [PATCH 03/18] add --- .../org/apache/doris/rewrite/FEFunctions.java | 71 ++++++++----------- 1 file changed, 28 insertions(+), 43 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index 5a384c36202561..926cbc336bdef7 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -30,23 +30,18 @@ import com.google.common.base.Preconditions; import org.apache.commons.lang.time.DateFormatUtils; import org.apache.commons.lang.time.DateUtils; -import org.apache.doris.common.util.TimeUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.DateTimeFormatterBuilder; import java.math.BigDecimal; import java.math.BigInteger; -import java.sql.Time; import java.text.ParseException; -import java.text.ParsePosition; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.Locale; -import java.util.TimeZone; /** * compute functions in FE. @@ -96,7 +91,7 @@ public static DateLiteral daysAdd(LiteralExpr date, LiteralExpr day) throws Anal @FEFunction(name = "date_format", argTypes = { "DATETIME", "VARCHAR" }, returnType = "VARCHAR") public static StringLiteral dateFormat(LiteralExpr date, StringLiteral fmtLiteral) throws AnalysisException { - String result = dateFormat(getTime(date), fmtLiteral.getStringValue()); + String result = dateFormat(new Date(getTime(date)), fmtLiteral.getStringValue()); return new StringLiteral(result); } @@ -288,32 +283,27 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmt } private static long getTime(LiteralExpr expr) throws AnalysisException { - - if (expr instanceof DateLiteral) { - long time; - time = expr.getLongValue(); - // add time zone offset to get right timestamp - time += DateTimeZone.forID("Asia/Shanghai").getOffset(0) - TimeUtils.getTimeZone().getOffset(0); - return time; - } else { - String[] parsePatterns = {"yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss"}; - SimpleDateFormat parser = null; - ParsePosition pos = new ParsePosition(0); - for (int i = 0; i < parsePatterns.length; ++i) { - if (i == 0) { - parser = new SimpleDateFormat(parsePatterns[0]); - parser.setTimeZone(TimeZone.getTimeZone("GMT")); - } else { - parser.applyPattern(parsePatterns[i]); - } - pos.setIndex(0); - Date date = parser.parse(expr.getStringValue(), pos); - if (date != null && pos.getIndex() == expr.getStringValue().length()) { - return date.getTime(); + if (expr instanceof DateLiteral) { + return expr.getLongValue(); + } else { + String[] parsePatterns = {"yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss"}; + SimpleDateFormat parser = null; + ParsePosition pos = new ParsePosition(0); + for (int i = 0; i < parsePatterns.length; ++i) { + if (i == 0) { + parser = new SimpleDateFormat(parsePatterns[0]); + parser.setTimeZone(TimeZone.getTimeZone("+08:00")); + } else { + parser.applyPattern(parsePatterns[i]); + } + pos.setIndex(0); + Date date = parser.parse(expr.getStringValue(), pos); + if (date != null && pos.getIndex() == expr.getStringValue().length()) { + return date.getTime(); + } } } - } - throw new AnalysisException("Unable to parse the date: " + expr.getStringValue()); + throw new AnalysisException(e.getLocalizedMessage()); } private static int calFirstWeekDay(int year, int firstWeekDay) { @@ -327,10 +317,9 @@ private static int calFirstWeekDay(int year, int firstWeekDay) { return firstDay; } - private static String dateFormat(long date, String pattern) { + private static String dateFormat(Date date, String pattern) { DateTimeFormatterBuilder formatterBuilder = new DateTimeFormatterBuilder(); Calendar calendar = Calendar.getInstance(); - calendar.setTimeZone(TimeUtils.getTimeZone()); boolean escaped = false; for (int i = 0; i < pattern.length(); i++) { char character = pattern.charAt(i); @@ -405,7 +394,7 @@ private static String dateFormat(long date, String pattern) { case 'V': // %V Week (01..53), where Sunday is the first day of the week; used with %X { int week; - calendar.setTimeInMillis(date); + calendar.setTime(date); int firstSunday = calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.SUNDAY); if (calendar.get(Calendar.DATE) <= 7 && calendar.get(Calendar.MONTH) == Calendar.JANUARY && calendar.get(Calendar.DATE) >= firstSunday) { @@ -422,7 +411,7 @@ private static String dateFormat(long date, String pattern) { formatterBuilder.appendWeekOfWeekyear(2); break; case 'X': // %X Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V - calendar.setTimeInMillis(date); + calendar.setTime(date); if(calendar.get(Calendar.MONTH) == Calendar.JANUARY && calendar.get(Calendar.DATE) < calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.SUNDAY)) { formatterBuilder.appendLiteral(String.valueOf(calendar.get(Calendar.YEAR) - 1)); @@ -437,7 +426,7 @@ private static String dateFormat(long date, String pattern) { formatterBuilder.appendDayOfWeekText(); break; case 'w': // %w Day of the week (0=Sunday..6=Saturday) - calendar.setTimeInMillis(date); + calendar.setTime(date); calendar.setFirstDayOfWeek(Calendar.SUNDAY); formatterBuilder.appendLiteral(String.valueOf(calendar.get(Calendar.DAY_OF_WEEK) - 1)); break; @@ -449,7 +438,7 @@ private static String dateFormat(long date, String pattern) { formatterBuilder.appendYear(4, 4); break; case 'D': // %D Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) - calendar.setTimeInMillis(date); + calendar.setTime(date); int day = calendar.get(Calendar.DAY_OF_MONTH); if (day >= 10 && day <= 19) { formatterBuilder.appendLiteral(String.valueOf(day) + "th"); @@ -471,7 +460,7 @@ private static String dateFormat(long date, String pattern) { } break; case 'U': // %U Week (00..53), where Sunday is the first day of the week - calendar.setTimeInMillis(date); + calendar.setTime(date); if (calendar.get(Calendar.DATE) <= 7 && calendar.get(Calendar.MONTH) == Calendar.JANUARY) { int firstSunday = calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.SUNDAY); formatterBuilder.appendLiteral(String.format("%02d", @@ -486,7 +475,7 @@ private static String dateFormat(long date, String pattern) { break; case 'u': // %u Week (00..53), where Monday is the first day of the week { - calendar.setTimeInMillis(date); + calendar.setTime(date); int week; int firstMonday = calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.MONDAY); if (calendar.get(Calendar.DATE) <= 7 && calendar.get(Calendar.MONTH) == Calendar.JANUARY) { @@ -517,11 +506,7 @@ else if (character == '%') { } } DateTimeFormatter formatter = formatterBuilder.toFormatter(); - - String ss = formatter.withZone(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())).withLocale(Locale.US).print(date); - System.out.println(ss); - return ss; - //return formatter.withLocale(Locale.US).print(date.getTime()); + return formatter.withLocale(Locale.US).print(date.getTime()); } /** From b575d4cbbe11b8309087f0ada8c3ca63c70466a2 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Wed, 14 Aug 2019 20:54:37 +0800 Subject: [PATCH 04/18] add fe time zone --- .../org/apache/doris/rewrite/FEFunctions.java | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index 926cbc336bdef7..25e8b734651de4 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -32,16 +32,19 @@ import org.apache.commons.lang.time.DateUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.DateTimeFormatterBuilder; import java.math.BigDecimal; import java.math.BigInteger; import java.text.ParseException; +import java.text.ParsePosition; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.Locale; +import java.util.TimeZone; /** * compute functions in FE. @@ -235,6 +238,7 @@ public static DateLiteral dateSub(LiteralExpr date, LiteralExpr day) throws Anal public static IntLiteral year(LiteralExpr arg) throws AnalysisException { long timestamp = getTime(arg); Calendar instance = Calendar.getInstance(); + instance.setTimeZone(TimeZone.getTimeZone("+08:00")); instance.setTimeInMillis(timestamp); return new IntLiteral(instance.get(Calendar.YEAR), Type.INT); } @@ -243,6 +247,7 @@ public static IntLiteral year(LiteralExpr arg) throws AnalysisException { public static IntLiteral month(LiteralExpr arg) throws AnalysisException { long timestamp = getTime(arg); Calendar instance = Calendar.getInstance(); + instance.setTimeZone(TimeZone.getTimeZone("+08:00")); instance.setTimeInMillis(timestamp); return new IntLiteral(instance.get(Calendar.MONTH) + 1, Type.INT); } @@ -251,6 +256,7 @@ public static IntLiteral month(LiteralExpr arg) throws AnalysisException { public static IntLiteral day(LiteralExpr arg) throws AnalysisException { long timestamp = getTime(arg); Calendar instance = Calendar.getInstance(); + instance.setTimeZone(TimeZone.getTimeZone("+08:00")); instance.setTimeInMillis(timestamp); return new IntLiteral(instance.get(Calendar.DAY_OF_MONTH), Type.INT); } @@ -283,27 +289,27 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmt } private static long getTime(LiteralExpr expr) throws AnalysisException { - if (expr instanceof DateLiteral) { - return expr.getLongValue(); - } else { - String[] parsePatterns = {"yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss"}; - SimpleDateFormat parser = null; - ParsePosition pos = new ParsePosition(0); - for (int i = 0; i < parsePatterns.length; ++i) { - if (i == 0) { - parser = new SimpleDateFormat(parsePatterns[0]); - parser.setTimeZone(TimeZone.getTimeZone("+08:00")); - } else { - parser.applyPattern(parsePatterns[i]); - } - pos.setIndex(0); - Date date = parser.parse(expr.getStringValue(), pos); - if (date != null && pos.getIndex() == expr.getStringValue().length()) { - return date.getTime(); - } + if (expr instanceof DateLiteral) { + return expr.getLongValue(); + } else { + String[] parsePatterns = {"yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss"}; + SimpleDateFormat parser = null; + ParsePosition pos = new ParsePosition(0); + for (int i = 0; i < parsePatterns.length; ++i) { + if (i == 0) { + parser = new SimpleDateFormat(parsePatterns[0]); + parser.setTimeZone(TimeZone.getTimeZone("+08:00")); + } else { + parser.applyPattern(parsePatterns[i]); + } + pos.setIndex(0); + Date date = parser.parse(expr.getStringValue(), pos); + if (date != null && pos.getIndex() == expr.getStringValue().length()) { + return date.getTime(); } } - throw new AnalysisException(e.getLocalizedMessage()); + } + throw new AnalysisException("Unable to parse the date: " + expr.getStringValue()); } private static int calFirstWeekDay(int year, int firstWeekDay) { @@ -320,6 +326,7 @@ private static int calFirstWeekDay(int year, int firstWeekDay) { private static String dateFormat(Date date, String pattern) { DateTimeFormatterBuilder formatterBuilder = new DateTimeFormatterBuilder(); Calendar calendar = Calendar.getInstance(); + calendar.setTimeZone(TimeZone.getTimeZone("+08:00")); boolean escaped = false; for (int i = 0; i < pattern.length(); i++) { char character = pattern.charAt(i); @@ -506,7 +513,7 @@ else if (character == '%') { } } DateTimeFormatter formatter = formatterBuilder.toFormatter(); - return formatter.withLocale(Locale.US).print(date.getTime()); + return formatter.withZone(DateTimeZone.forID("+08:00")).withLocale(Locale.US).print(date.getTime()); } /** From 98ab1407b820f79730ccd7686a939279c6f5dc1e Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Wed, 14 Aug 2019 20:54:37 +0800 Subject: [PATCH 05/18] add fe time zone --- .../org/apache/doris/rewrite/FEFunctions.java | 83 +++++++++++-------- .../apache/doris/rewrite/FEFunctionsTest.java | 40 ++++----- 2 files changed, 67 insertions(+), 56 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index 926cbc336bdef7..e5fcba777e3720 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -28,20 +28,23 @@ import org.apache.doris.common.AnalysisException; import com.google.common.base.Preconditions; -import org.apache.commons.lang.time.DateFormatUtils; import org.apache.commons.lang.time.DateUtils; +import org.apache.doris.common.util.TimeUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.DateTimeFormatterBuilder; import java.math.BigDecimal; import java.math.BigInteger; import java.text.ParseException; +import java.text.ParsePosition; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.Locale; +import java.util.TimeZone; /** * compute functions in FE. @@ -76,7 +79,7 @@ public static IntLiteral dateDiff(LiteralExpr first, LiteralExpr second) throws public static DateLiteral dateAdd(LiteralExpr date, LiteralExpr day) throws AnalysisException { Date d = new Date(getTime(date)); d = DateUtils.addDays(d, (int) day.getLongValue()); - return new DateLiteral(DateFormatUtils.format(d, "yyyy-MM-dd HH:mm:ss"), Type.DATETIME); + return new DateLiteral(dateFormat(d, "%Y-%m-%d %H:%i:%s", true), Type.DATETIME); } @FEFunction(name = "adddate", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") @@ -91,7 +94,7 @@ public static DateLiteral daysAdd(LiteralExpr date, LiteralExpr day) throws Anal @FEFunction(name = "date_format", argTypes = { "DATETIME", "VARCHAR" }, returnType = "VARCHAR") public static StringLiteral dateFormat(LiteralExpr date, StringLiteral fmtLiteral) throws AnalysisException { - String result = dateFormat(new Date(getTime(date)), fmtLiteral.getStringValue()); + String result = dateFormat(new Date(getTime(date)), fmtLiteral.getStringValue(), true); return new StringLiteral(result); } @@ -216,25 +219,27 @@ public static DateLiteral dateParse(StringLiteral date, StringLiteral fmtLiteral } } - Date retDate = new Date(builder.toFormatter().withLocale(Locale.ENGLISH).parseMillis(date.getStringValue())); + Date retDate = new Date(builder.toFormatter().withZone(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())).withLocale(Locale.ENGLISH).parseMillis(date.getStringValue())); + //Date retDate = new Date(date2.getTime() + + // DateTimeZone.forID("Asia/Shanghai").getOffset(0) - TimeUtils.getTimeZone().getOffset(0)); + if (hasTimePart) { - return new DateLiteral(DateFormatUtils.format(retDate, "yyyy-MM-dd HH:mm:ss"), Type.DATETIME); + return new DateLiteral(dateFormat(retDate, "%Y-%m-%d %H:%i:%s", false), Type.DATETIME); } else { - return new DateLiteral(DateFormatUtils.format(retDate, "yyyy-MM-dd"), Type.DATE); + return new DateLiteral(dateFormat(retDate, "%Y-%m-%d", false), Type.DATE); } } @FEFunction(name = "date_sub", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") public static DateLiteral dateSub(LiteralExpr date, LiteralExpr day) throws AnalysisException { - Date d = new Date(getTime(date)); - d = DateUtils.addDays(d, -(int) day.getLongValue()); - return new DateLiteral(DateFormatUtils.format(d, "yyyy-MM-dd HH:mm:ss"), Type.DATETIME); + return dateAdd(date, new IntLiteral(-(int) day.getLongValue())); } @FEFunction(name = "year", argTypes = { "DATETIME" }, returnType = "INT") public static IntLiteral year(LiteralExpr arg) throws AnalysisException { long timestamp = getTime(arg); Calendar instance = Calendar.getInstance(); + instance.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); instance.setTimeInMillis(timestamp); return new IntLiteral(instance.get(Calendar.YEAR), Type.INT); } @@ -243,6 +248,7 @@ public static IntLiteral year(LiteralExpr arg) throws AnalysisException { public static IntLiteral month(LiteralExpr arg) throws AnalysisException { long timestamp = getTime(arg); Calendar instance = Calendar.getInstance(); + instance.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); instance.setTimeInMillis(timestamp); return new IntLiteral(instance.get(Calendar.MONTH) + 1, Type.INT); } @@ -252,12 +258,14 @@ public static IntLiteral day(LiteralExpr arg) throws AnalysisException { long timestamp = getTime(arg); Calendar instance = Calendar.getInstance(); instance.setTimeInMillis(timestamp); + instance.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); return new IntLiteral(instance.get(Calendar.DAY_OF_MONTH), Type.INT); } @FEFunction(name = "unix_timestamp", argTypes = { "DATETIME" }, returnType = "INT") public static IntLiteral unix_timestamp(LiteralExpr arg) throws AnalysisException { long timestamp = getTime(arg); + timestamp += DateTimeZone.forID("Asia/Shanghai").getOffset(0) - TimeUtils.getTimeZone().getOffset(0); return new IntLiteral(timestamp / 1000, Type.INT); } @@ -268,9 +276,8 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime) throws AnalysisEx throw new AnalysisException("unixtime should larger than zero"); } Date date = new Date(unixTime.getLongValue() * 1000); - return new StringLiteral(dateFormat(date, "%Y-%m-%d %H:%i:%S")); + return new StringLiteral(dateFormat(date, "%Y-%m-%d %H:%i:%s", true)); } - @FEFunction(name = "from_unixtime", argTypes = { "INT", "VARCHAR" }, returnType = "VARCHAR") public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmtLiteral) throws AnalysisException { //if unixTime < 0, we should return null, throw a exception and let BE process @@ -279,31 +286,30 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmt } Date date = new Date(unixTime.getLongValue() * 1000); //currently, doris BE only support "yyyy-MM-dd HH:mm:ss" and "yyyy-MM-dd" format - return new StringLiteral(DateFormatUtils.format(date, fmtLiteral.getStringValue())); + return new StringLiteral(dateFormat(date, fmtLiteral.getStringValue(), false)); + //return new StringLiteral(DateFormatUtils.format(date, fmtLiteral.getStringValue())); } - private static long getTime(LiteralExpr expr) throws AnalysisException { - if (expr instanceof DateLiteral) { - return expr.getLongValue(); - } else { - String[] parsePatterns = {"yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss"}; - SimpleDateFormat parser = null; - ParsePosition pos = new ParsePosition(0); - for (int i = 0; i < parsePatterns.length; ++i) { - if (i == 0) { - parser = new SimpleDateFormat(parsePatterns[0]); - parser.setTimeZone(TimeZone.getTimeZone("+08:00")); - } else { - parser.applyPattern(parsePatterns[i]); - } - pos.setIndex(0); - Date date = parser.parse(expr.getStringValue(), pos); - if (date != null && pos.getIndex() == expr.getStringValue().length()) { - return date.getTime(); - } + if (expr instanceof DateLiteral) { + return expr.getLongValue(); + } else { + String[] parsePatterns = {"yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss"}; + + SimpleDateFormat parser = new SimpleDateFormat(parsePatterns[0]); + //parser.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); + ParsePosition pos = new ParsePosition(0); + + for (int i = 0; i < parsePatterns.length; ++i) { + parser.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); + parser.applyPattern(parsePatterns[i]); + pos.setIndex(0); + Date date = parser.parse(expr.getStringValue(), pos); + if (date != null && pos.getIndex() == expr.getStringValue().length()) { + return date.getTime(); } } - throw new AnalysisException(e.getLocalizedMessage()); + } + throw new AnalysisException("Unable to parse the date: " + expr.getStringValue()); } private static int calFirstWeekDay(int year, int firstWeekDay) { @@ -317,9 +323,14 @@ private static int calFirstWeekDay(int year, int firstWeekDay) { return firstDay; } - private static String dateFormat(Date date, String pattern) { + private static String dateFormat(Date date, String pattern, boolean isLiteral) { DateTimeFormatterBuilder formatterBuilder = new DateTimeFormatterBuilder(); Calendar calendar = Calendar.getInstance(); + if (isLiteral){ + calendar.setTimeZone(TimeZone.getTimeZone("+08:00")); + } else { + calendar.setTimeZone(TimeUtils.getTimeZone()); + } boolean escaped = false; for (int i = 0; i < pattern.length(); i++) { char character = pattern.charAt(i); @@ -506,7 +517,11 @@ else if (character == '%') { } } DateTimeFormatter formatter = formatterBuilder.toFormatter(); - return formatter.withLocale(Locale.US).print(date.getTime()); + if (isLiteral) { + return formatter.withZone(DateTimeZone.forID("+08:00")).withLocale(Locale.US).print(date.getTime()); + } else { + return formatter.withZone(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())).withLocale(Locale.US).print(date.getTime()); + } } /** diff --git a/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java b/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java index 01041a67e7b92f..1d81f6a02f685e 100644 --- a/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java +++ b/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java @@ -31,8 +31,6 @@ import org.junit.Test; import org.junit.rules.ExpectedException; -import java.util.TimeZone; - import static org.junit.Assert.fail; /* @@ -65,48 +63,48 @@ public void dateDiffTest() throws AnalysisException { expectedResult = new IntLiteral(-30); Assert.assertEquals(expectedResult, actualResult); } - + @Test public void dateAddTest() throws AnalysisException { DateLiteral actualResult = FEFunctions.dateAdd(new StringLiteral("2018-08-08"), new IntLiteral(1)); - DateLiteral expectedResult = new DateLiteral("2018-08-09", Type.DATE); + DateLiteral expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); actualResult = FEFunctions.dateAdd(new StringLiteral("2018-08-08"), new IntLiteral(-1)); - expectedResult = new DateLiteral("2018-08-07", Type.DATE); + expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); } - + @Test public void addDateTest() throws AnalysisException { - DateLiteral actualResult = FEFunctions.addDate(new StringLiteral("2018-08-08"), new IntLiteral(1)); - DateLiteral expectedResult = new DateLiteral("2018-08-09", Type.DATE); + DateLiteral actualResult = FEFunctions.addDate(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(1)); + DateLiteral expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); actualResult = FEFunctions.addDate(new StringLiteral("2018-08-08"), new IntLiteral(-1)); - expectedResult = new DateLiteral("2018-08-07", Type.DATE); + expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); } - + @Test public void daysAddTest() throws AnalysisException { DateLiteral actualResult = FEFunctions.daysAdd(new StringLiteral("2018-08-08"), new IntLiteral(1)); - DateLiteral expectedResult = new DateLiteral("2018-08-09", Type.DATE); + DateLiteral expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); actualResult = FEFunctions.daysAdd(new StringLiteral("2018-08-08"), new IntLiteral(-1)); - expectedResult = new DateLiteral("2018-08-07", Type.DATE); + expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); } - + @Test public void fromUnixTimeTest() throws AnalysisException { StringLiteral actualResult = FEFunctions.fromUnixTime(new IntLiteral(100000)); StringLiteral expectedResult = new StringLiteral("1970-01-02 11:46:40"); Assert.assertEquals(expectedResult, actualResult); - actualResult = FEFunctions.fromUnixTime(new IntLiteral(100000), new StringLiteral("yyyy-MM-dd")); + actualResult = FEFunctions.fromUnixTime(new IntLiteral(100000), new StringLiteral("%Y-%m-%d")); expectedResult = new StringLiteral("1970-01-02"); Assert.assertEquals(expectedResult, actualResult); @@ -169,8 +167,6 @@ public void dateFormatUtilTest() { @Test public void dateParseTest() { - TimeZone tz = TimeZone.getTimeZone("Asia/Shanghai"); - TimeZone.setDefault(tz); try { Assert.assertEquals("2013-05-10", FEFunctions.dateParse(new StringLiteral("2013,05,10"), new StringLiteral("%Y,%m,%d")).getStringValue()); Assert.assertEquals("2013-05-17 00:35:10", FEFunctions.dateParse(new StringLiteral("2013-05-17 12:35:10"), new StringLiteral("%Y-%m-%d %h:%i:%s")).getStringValue()); @@ -241,32 +237,32 @@ public void dateSubTest() throws AnalysisException { @Test public void yearTest() throws AnalysisException { IntLiteral actualResult = FEFunctions.year(new StringLiteral("2018-08-08")); - IntLiteral expectedResult = new IntLiteral(2018); + IntLiteral expectedResult = new IntLiteral(2018, Type.INT); Assert.assertEquals(expectedResult, actualResult); actualResult = FEFunctions.year(new StringLiteral("1970-01-02 11:46:40")); - expectedResult = new IntLiteral(1970); + expectedResult = new IntLiteral(1970, Type.INT); Assert.assertEquals(expectedResult, actualResult); } @Test public void monthTest() throws AnalysisException { IntLiteral actualResult = FEFunctions.month(new StringLiteral("2018-08-08")); - IntLiteral expectedResult = new IntLiteral(8); + IntLiteral expectedResult = new IntLiteral(8, Type.INT); Assert.assertEquals(expectedResult, actualResult); actualResult = FEFunctions.month(new StringLiteral("1970-01-02 11:46:40")); - expectedResult = new IntLiteral(1); + expectedResult = new IntLiteral(1, Type.INT); Assert.assertEquals(expectedResult, actualResult); } @Test public void dayTest() throws AnalysisException { IntLiteral actualResult = FEFunctions.day(new StringLiteral("2018-08-08")); - IntLiteral expectedResult = new IntLiteral(8); + IntLiteral expectedResult = new IntLiteral(8, Type.INT); Assert.assertEquals(expectedResult, actualResult); actualResult = FEFunctions.day(new StringLiteral("1970-01-02 11:46:40")); - expectedResult = new IntLiteral(2); + expectedResult = new IntLiteral(2, Type.INT); Assert.assertEquals(expectedResult, actualResult); } From 40c0dfe7c9a021b288251b72dcb4d1ab51aac6de Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Mon, 19 Aug 2019 14:26:54 +0800 Subject: [PATCH 06/18] add fe time zone --- .../apache/doris/analysis/DateLiteral.java | 120 ++++++++++++++---- .../org/apache/doris/common/FeConstants.java | 2 +- .../apache/doris/common/FeMetaVersion.java | 2 + 3 files changed, 98 insertions(+), 26 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java index 0a820b34a5e904..0bf71771a54cc0 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -17,9 +17,13 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.thrift.TDateLiteral; import org.apache.doris.thrift.TExprNode; @@ -29,6 +33,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.joda.time.LocalDateTime; +import org.joda.time.format.DateTimeFormatterBuilder; import java.io.DataInput; import java.io.DataOutput; @@ -38,24 +44,39 @@ public class DateLiteral extends LiteralExpr { private static final Logger LOG = LogManager.getLogger(DateLiteral.class); - private Date date; + //private Date date; + int hour; + int minute; + int second; + int year; + int month; + int day; + int microsecond; private DateLiteral() { super(); } - public DateLiteral(Type type, boolean isMax) throws AnalysisException { + public DateLiteral(Type type, boolean isMax) { super(); this.type = type; if (type == Type.DATE) { - date = isMax ? TimeUtils.MAX_DATE : TimeUtils.MIN_DATE; + if (isMax) { + init("1900-01-01", Type.DATE); + } else { + init("9999-12-31", Type.DATE); + } } else { - date = isMax ? TimeUtils.MAX_DATETIME : TimeUtils.MIN_DATETIME; + if (isMax) { + init("1900-01-01 00:00:00", Type.DATETIME); + } else { + init("9999-12-31 23:59:59", Type.DATETIME); + } } analysisDone(); } - public DateLiteral(String s, Type type) throws AnalysisException { + public DateLiteral(String s, Type type) { super(); init(s, type); analysisDone(); @@ -63,7 +84,14 @@ public DateLiteral(String s, Type type) throws AnalysisException { protected DateLiteral(DateLiteral other) { super(other); - date = other.date; + hour = other.hour; + minute = other.minute; + second = other.second; + year = other.year; + month = other.month; + day = other.day; + microsecond = other.microsecond; + //date = other.date; } @Override @@ -72,19 +100,26 @@ public Expr clone() { } public static DateLiteral createMinValue(Type type) { - DateLiteral dateLiteral = new DateLiteral(); - dateLiteral.type = type; - if (type == Type.DATE) { - dateLiteral.date = TimeUtils.MIN_DATE; - } else { - dateLiteral.date = TimeUtils.MIN_DATETIME; - } - + DateLiteral dateLiteral = new DateLiteral(type, false); return dateLiteral; } - private void init(String s, Type type) throws AnalysisException { + //private void init(String s, Type type) throws AnalysisException { + private void init(String s, Type type) { Preconditions.checkArgument(type.isDateType()); + DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); + LocalDateTime datetime = builder.toFormatter().parseLocalDateTime(s); + + year = datetime.getYear(); + month = datetime.getMonthOfYear(); + day = datetime.getDayOfMonth(); + + hour = datetime.getHourOfDay(); + minute = datetime.getMinuteOfHour(); + second = datetime.getSecondOfMinute(); + this.type = type; + + /* date = TimeUtils.parseDate(s, type); if (type.isScalarType(PrimitiveType.DATE)) { if (date.compareTo(TimeUtils.MAX_DATE) > 0 || date.compareTo(TimeUtils.MIN_DATE) < 0) { @@ -98,15 +133,16 @@ private void init(String s, Type type) throws AnalysisException { } } this.type = type; + */ } @Override public boolean isMinValue() { switch (type.getPrimitiveType()) { case DATE: - return this.date.compareTo(TimeUtils.MIN_DATE) == 0; + return this.getStringValue().compareTo("1900-01-01") == 0; case DATETIME: - return this.date.compareTo(TimeUtils.MIN_DATETIME) == 0; + return this.getStringValue().compareTo("1900-01-01 00:00:00") == 0; default: return false; } @@ -114,13 +150,15 @@ public boolean isMinValue() { @Override public Object getRealValue() { - return TimeUtils.dateTransform(date.getTime(), type); + return getStringValue(); + //return TimeUtils.dateTransform(date.getTime(), type); } // Date column and Datetime column's hash value is not same. @Override public ByteBuffer getHashValue(PrimitiveType type) { - String value = TimeUtils.format(date, type); + //String value = TimeUtils.format(date, type); + String value = getStringValue(); ByteBuffer buffer; try { buffer = ByteBuffer.wrap(value.getBytes("UTF-8")); @@ -150,17 +188,23 @@ public String toSqlImpl() { @Override public String getStringValue() { - return TimeUtils.format(date, type); + //return TimeUtils.format(date, type); + if (type == Type.DATE) { + return year + "-" + month + "-" + day; + } else { + return year + "-" + month + "-" + day + + " " + hour + ":" + minute + ":" + second; + } } @Override public long getLongValue() { - return date.getTime(); + return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; } @Override public double getDoubleValue() { - return date.getTime(); + return getLongValue(); } @Override @@ -168,10 +212,11 @@ protected void toThrift(TExprNode msg) { msg.node_type = TExprNodeType.DATE_LITERAL; msg.date_literal = new TDateLiteral(getStringValue()); } - + /* public Date getValue() { return date; } + */ @Override protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { @@ -187,13 +232,38 @@ protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { @Override public void write(DataOutput out) throws IOException { super.write(out); - out.writeLong(date.getTime()); + long ymd = ((year * 13 + month) << 5) | day; + long hms = (hour << 12) | (minute << 6) | second; + long tmp = ((ymd << 17) | hms) << 24 + microsecond; + long packed_datetime = tmp; + out.writeLong(packed_datetime); } @Override public void readFields(DataInput in) throws IOException { super.readFields(in); - date = new Date(in.readLong()); + //date = new Date(in.readLong()); + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { + long packed_time = in.readLong(); + microsecond = (int) (packed_time % (1L << 24)); + int ymdhms = (int) (packed_time >> 24); + int ymd = ymdhms >> 17; + int hms = ymdhms % (1 << 17); + + day = ymd % (1 << 5); + int ym = ymd >> 5; + month = ym % 13; + year = ym / 13; + year %= 10000; + second = hms % (1 << 6); + minute = (hms >> 6) % (1 << 6); + hour = (hms >> 12); + this.type = Type.DATETIME; + } else { + Date date = new Date(in.readLong()); + String date_str = TimeUtils.format(date, Type.DATETIME); + init(date_str, Type.DATETIME); + } } public static DateLiteral read(DataInput in) throws IOException { diff --git a/fe/src/main/java/org/apache/doris/common/FeConstants.java b/fe/src/main/java/org/apache/doris/common/FeConstants.java index 95683e20ac3693..1c87c9926f49d9 100644 --- a/fe/src/main/java/org/apache/doris/common/FeConstants.java +++ b/fe/src/main/java/org/apache/doris/common/FeConstants.java @@ -35,5 +35,5 @@ public class FeConstants { // general model // Current meta data version. Use this version to write journals and image - public static int meta_version = FeMetaVersion.VERSION_58; + public static int meta_version = FeMetaVersion.VERSION_59; } diff --git a/fe/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/src/main/java/org/apache/doris/common/FeMetaVersion.java index b5bdb2a42d9edd..2a03239d8e776b 100644 --- a/fe/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -126,4 +126,6 @@ public final class FeMetaVersion { public static final int VERSION_57 = 57; // broker load support function, persist origin stmt in broker load public static final int VERSION_58 = 58; + // date literal + public static final int VERSION_59 = 59; } From 01c855770103693a8ec2039a99a3c3b967b3203d Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Mon, 19 Aug 2019 14:34:27 +0800 Subject: [PATCH 07/18] Merge branch 'fe-time-zone-function' of https://github.com/HangyuanLiu/incubator-doris into fe-time-zone-function # Conflicts: # fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java --- .../apache/doris/analysis/DateLiteral.java | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java index 0bf71771a54cc0..7c7a710df469d1 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -19,11 +19,9 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeMetaVersion; -import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.thrift.TDateLiteral; import org.apache.doris.thrift.TExprNode; @@ -57,7 +55,7 @@ private DateLiteral() { super(); } - public DateLiteral(Type type, boolean isMax) { + public DateLiteral(Type type, boolean isMax) throws AnalysisException{ super(); this.type = type; if (type == Type.DATE) { @@ -76,7 +74,7 @@ public DateLiteral(Type type, boolean isMax) { analysisDone(); } - public DateLiteral(String s, Type type) { + public DateLiteral(String s, Type type) throws AnalysisException{ super(); init(s, type); analysisDone(); @@ -99,25 +97,29 @@ public Expr clone() { return new DateLiteral(this); } - public static DateLiteral createMinValue(Type type) { + public static DateLiteral createMinValue(Type type) throws AnalysisException{ DateLiteral dateLiteral = new DateLiteral(type, false); return dateLiteral; } //private void init(String s, Type type) throws AnalysisException { - private void init(String s, Type type) { - Preconditions.checkArgument(type.isDateType()); - DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); - LocalDateTime datetime = builder.toFormatter().parseLocalDateTime(s); + private void init(String s, Type type) throws AnalysisException { + try { + Preconditions.checkArgument(type.isDateType()); + DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); + LocalDateTime datetime = builder.toFormatter().parseLocalDateTime(s); - year = datetime.getYear(); - month = datetime.getMonthOfYear(); - day = datetime.getDayOfMonth(); + year = datetime.getYear(); + month = datetime.getMonthOfYear(); + day = datetime.getDayOfMonth(); - hour = datetime.getHourOfDay(); - minute = datetime.getMinuteOfHour(); - second = datetime.getSecondOfMinute(); - this.type = type; + hour = datetime.getHourOfDay(); + minute = datetime.getMinuteOfHour(); + second = datetime.getSecondOfMinute(); + this.type = type; + } catch (Exception ex) { + throw new AnalysisException(ex.getMessage()); + } /* date = TimeUtils.parseDate(s, type); @@ -262,7 +264,11 @@ public void readFields(DataInput in) throws IOException { } else { Date date = new Date(in.readLong()); String date_str = TimeUtils.format(date, Type.DATETIME); - init(date_str, Type.DATETIME); + try { + init(date_str, Type.DATETIME); + } catch (AnalysisException ex) { + throw new IOException(ex.getMessage()); + } } } From 58e856ba858ef09d492b6e8b9980a9bdcca89563 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Mon, 19 Aug 2019 17:30:02 +0800 Subject: [PATCH 08/18] add --- .../apache/doris/analysis/DateLiteral.java | 279 +++++++++++- .../org/apache/doris/rewrite/FEFunctions.java | 422 ++---------------- 2 files changed, 294 insertions(+), 407 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java index 7c7a710df469d1..ed90fd1116d8d7 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -38,18 +38,79 @@ import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Date; +import java.util.TimeZone; public class DateLiteral extends LiteralExpr { private static final Logger LOG = LogManager.getLogger(DateLiteral.class); + + public long getHour() { + return hour; + } + + public void setHour(long hour) { + this.hour = hour; + } + //private Date date; - int hour; - int minute; - int second; - int year; - int month; - int day; - int microsecond; + long hour; + + public long getMinute() { + return minute; + } + + public void setMinute(long minute) { + this.minute = minute; + } + + public long getSecond() { + return second; + } + + public void setSecond(long second) { + this.second = second; + } + + public long getYear() { + return year; + } + + public void setYear(long year) { + this.year = year; + } + + public long getMonth() { + return month; + } + + public void setMonth(long month) { + this.month = month; + } + + public long getDay() { + return day; + } + + public void setDay(long day) { + this.day = day; + } + + public long getMicrosecond() { + return microsecond; + } + + public void setMicrosecond(long microsecond) { + this.microsecond = microsecond; + } + + long minute; + long second; + long year; + long month; + long day; + long microsecond; private DateLiteral() { super(); @@ -80,7 +141,7 @@ public DateLiteral(String s, Type type) throws AnalysisException{ analysisDone(); } - protected DateLiteral(DateLiteral other) { + public DateLiteral(DateLiteral other) { super(other); hour = other.hour; minute = other.minute; @@ -102,20 +163,194 @@ public static DateLiteral createMinValue(Type type) throws AnalysisException{ return dateLiteral; } + public long unixTime(TimeZone timeZone) throws ParseException { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateFormat.setTimeZone(timeZone); + long timestamp = dateFormat.parse(getStringValue()).getTime(); + return timestamp; + } + + public static DateLiteral dateParser(String date, String pattern) throws AnalysisException{ + DateLiteral dateLiteral = new DateLiteral(); + + LocalDateTime dateTime = FormatBuilder(pattern).toFormatter().parseLocalDateTime(date); + dateLiteral.setYear(dateTime.getYear()); + dateLiteral.setMonth(dateTime.getMonthOfYear()); + dateLiteral.setDay(dateTime.getDayOfMonth()); + dateLiteral.setHour(dateTime.getHourOfDay()); + dateLiteral.setMinute(dateTime.getMinuteOfHour()); + dateLiteral.setSecond(dateTime.getSecondOfMinute()); + return dateLiteral; + } + + public String dateFormat(String pattern) throws AnalysisException{ + DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); + if (type == Type.DATE) { + builder.appendYear(4, 4).appendLiteral("-"). + appendMonthOfYear(2).appendLiteral("-").appendDayOfMonth(2); + } else { + builder.appendYear(4, 4).appendLiteral("-"). + appendMonthOfYear(2).appendLiteral("-") + .appendDayOfMonth(2).appendLiteral(" ") + .appendHourOfDay(2).appendLiteral(":") + .appendMinuteOfHour(2).appendLiteral(":") + .appendSecondOfMinute(2); + } + + return builder.toFormatter().parseLocalDateTime(getStringValue()) + .toString(FormatBuilder(pattern).toFormatter()); + } + + + private static DateTimeFormatterBuilder FormatBuilder(String pattern) throws AnalysisException{ + boolean hasTimePart = false; + DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); + //String formatString = fmtLiteral.getStringValue(); + boolean escaped = false; + for (int i = 0; i < pattern.length(); i++) { + char character = pattern.charAt(i); + if (escaped) { + switch (character) { + case 'a': // %a Abbreviated weekday name (Sun..Sat) + builder.appendDayOfWeekShortText(); + break; + case 'b': // %b Abbreviated month name (Jan..Dec) + builder.appendMonthOfYearShortText(); + break; + case 'c': // %c Month, numeric (0..12) + builder.appendMonthOfYear(1); + break; + case 'd': // %d Day of the month, numeric (00..31) + builder.appendDayOfMonth(2); + break; + case 'e': // %e Day of the month, numeric (0..31) + builder.appendDayOfMonth(1); + break; + case 'H': // %H Hour (00..23) + builder.appendHourOfDay(2); + hasTimePart = true; + break; + case 'h': // %h Hour (01..12) + case 'I': // %I Hour (01..12) + builder.appendClockhourOfHalfday(2); + hasTimePart = true; + break; + case 'i': // %i Minutes, numeric (00..59) + builder.appendMinuteOfHour(2); + hasTimePart = true; + break; + case 'j': // %j Day of year (001..366) + builder.appendDayOfYear(3); + break; + case 'k': // %k Hour (0..23) + builder.appendHourOfDay(1); + hasTimePart = true; + break; + case 'l': // %l Hour (1..12) + builder.appendClockhourOfHalfday(1); + hasTimePart = true; + break; + case 'M': // %M Month name (January..December) + builder.appendMonthOfYearText(); + break; + case 'm': // %m Month, numeric (00..12) + builder.appendMonthOfYear(2); + break; + case 'p': // %p AM or PM + builder.appendHalfdayOfDayText(); + break; + case 'r': // %r Time, 12-hour (hh:mm:ss followed by AM or PM) + builder.appendClockhourOfHalfday(2) + .appendLiteral(':') + .appendMinuteOfHour(2) + .appendLiteral(':') + .appendSecondOfMinute(2) + .appendLiteral(' ') + .appendHalfdayOfDayText(); + hasTimePart = true; + break; + case 'S': // %S Seconds (00..59) + case 's': // %s Seconds (00..59) + builder.appendSecondOfMinute(2); + hasTimePart = true; + break; + case 'T': // %T Time, 24-hour (hh:mm:ss) + builder.appendHourOfDay(2) + .appendLiteral(':') + .appendMinuteOfHour(2) + .appendLiteral(':') + .appendSecondOfMinute(2); + hasTimePart = true; + break; + case 'v': // %v Week (01..53), where Monday is the first day of the week; used with %x + builder.appendWeekOfWeekyear(2); + break; + case 'x': // %x Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v + builder.appendWeekyear(4, 4); + break; + case 'W': // %W Weekday name (Sunday..Saturday) + builder.appendDayOfWeekText(); + break; + case 'Y': // %Y Year, numeric, four digits + builder.appendYear(4, 4); + break; + case 'y': // %y Year, numeric (two digits) + builder.appendTwoDigitYear(2020); + break; + case 'f': // %f Microseconds (000000..999999) + case 'w': // %w Day of the week (0=Sunday..6=Saturday) + case 'U': // %U Week (00..53), where Sunday is the first day of the week + case 'u': // %u Week (00..53), where Monday is the first day of the week + case 'V': // %V Week (01..53), where Sunday is the first day of the week; used with %X + case 'X': // %X Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V + case 'D': // %D Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) + throw new AnalysisException(String.format("%%%s not supported in date format string", character)); + case '%': // %% A literal "%" character + builder.appendLiteral('%'); + break; + default: // % The literal character represented by + builder.appendLiteral(character); + break; + } + escaped = false; + } else if (character == '%') { + escaped = true; + } else { + builder.appendLiteral(character); + } + } + return builder; + } + + //private void init(String s, Type type) throws AnalysisException { private void init(String s, Type type) throws AnalysisException { try { Preconditions.checkArgument(type.isDateType()); DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); - LocalDateTime datetime = builder.toFormatter().parseLocalDateTime(s); - - year = datetime.getYear(); - month = datetime.getMonthOfYear(); - day = datetime.getDayOfMonth(); + if (type == Type.DATE) { + builder.appendYear(4, 4).appendLiteral("-"). + appendMonthOfYear(2).appendLiteral("-").appendDayOfMonth(2); + LocalDateTime datetime = builder.toFormatter().parseLocalDateTime(s); + year = datetime.getYear(); + month = datetime.getMonthOfYear(); + day = datetime.getDayOfMonth(); + } else { + builder.appendYear(4, 4).appendLiteral("-"). + appendMonthOfYear(2).appendLiteral("-") + .appendDayOfMonth(2).appendLiteral(" ") + .appendHourOfDay(2).appendLiteral(":") + .appendMinuteOfHour(2).appendLiteral(":") + .appendSecondOfMinute(2); - hour = datetime.getHourOfDay(); - minute = datetime.getMinuteOfHour(); - second = datetime.getSecondOfMinute(); + LocalDateTime datetime = builder.toFormatter().parseLocalDateTime(s); + year = datetime.getYear(); + month = datetime.getMonthOfYear(); + day = datetime.getDayOfMonth(); + hour = datetime.getHourOfDay(); + minute = datetime.getMinuteOfHour(); + second = datetime.getSecondOfMinute(); + } this.type = type; } catch (Exception ex) { throw new AnalysisException(ex.getMessage()); @@ -192,10 +427,16 @@ public String toSqlImpl() { public String getStringValue() { //return TimeUtils.format(date, type); if (type == Type.DATE) { - return year + "-" + month + "-" + day; + return String.format("%04d", year) + "-" + + String.format("%02d", month) + "-" + + String.format("%02d", day); } else { - return year + "-" + month + "-" + day + - " " + hour + ":" + minute + ":" + second; + return String.format("%04d", year) + "-" + + String.format("%02d", month) + "-" + + String.format("%02d", day) + " " + + String.format("%02d", hour) + ":" + + String.format("%02d",minute) + ":" + + String.format("%02d",second); } } diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index e5fcba777e3720..f8e7ed3a0f88fe 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -28,23 +28,16 @@ import org.apache.doris.common.AnalysisException; import com.google.common.base.Preconditions; -import org.apache.commons.lang.time.DateUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.DateTimeFormatterBuilder; import java.math.BigDecimal; import java.math.BigInteger; import java.text.ParseException; -import java.text.ParsePosition; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; -import java.util.Locale; -import java.util.TimeZone; /** * compute functions in FE. @@ -58,8 +51,13 @@ public class FEFunctions { */ @FEFunction(name = "timediff", argTypes = { "DATETIME", "DATETIME" }, returnType = "TIME") public static FloatLiteral timeDiff(LiteralExpr first, LiteralExpr second) throws AnalysisException { - long timediff = (getTime(first) - getTime(second)) / 1000; - return new FloatLiteral((double)timediff, Type.TIME); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + try { + long timediff = sdf.parse(first.getStringValue()).getTime() - sdf.parse(second.getStringValue()).getTime(); + return new FloatLiteral((double) timediff, Type.TIME); + } catch (ParseException e) { + throw new AnalysisException(e.getLocalizedMessage()); + } } @FEFunction(name = "datediff", argTypes = { "DATETIME", "DATETIME" }, returnType = "INT") @@ -77,9 +75,11 @@ public static IntLiteral dateDiff(LiteralExpr first, LiteralExpr second) throws @FEFunction(name = "date_add", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") public static DateLiteral dateAdd(LiteralExpr date, LiteralExpr day) throws AnalysisException { - Date d = new Date(getTime(date)); - d = DateUtils.addDays(d, (int) day.getLongValue()); - return new DateLiteral(dateFormat(d, "%Y-%m-%d %H:%i:%s", true), Type.DATETIME); + DateLiteral dateLiteral = (DateLiteral) date; + + DateLiteral result = new DateLiteral(dateLiteral); + result.setDay(dateLiteral.getDay() - 1); + return result; } @FEFunction(name = "adddate", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") @@ -94,140 +94,14 @@ public static DateLiteral daysAdd(LiteralExpr date, LiteralExpr day) throws Anal @FEFunction(name = "date_format", argTypes = { "DATETIME", "VARCHAR" }, returnType = "VARCHAR") public static StringLiteral dateFormat(LiteralExpr date, StringLiteral fmtLiteral) throws AnalysisException { - String result = dateFormat(new Date(getTime(date)), fmtLiteral.getStringValue(), true); + String result = ((DateLiteral) date).dateFormat(fmtLiteral.getStringValue()); return new StringLiteral(result); } @FEFunction(name = "str_to_date", argTypes = { "VARCHAR", "VARCHAR" }, returnType = "DATETIME") public static DateLiteral dateParse(StringLiteral date, StringLiteral fmtLiteral) throws AnalysisException { - boolean hasTimePart = false; - DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); - - String formatString = fmtLiteral.getStringValue(); - boolean escaped = false; - for (int i = 0; i < formatString.length(); i++) { - char character = formatString.charAt(i); - - if (escaped) { - switch (character) { - case 'a': // %a Abbreviated weekday name (Sun..Sat) - builder.appendDayOfWeekShortText(); - break; - case 'b': // %b Abbreviated month name (Jan..Dec) - builder.appendMonthOfYearShortText(); - break; - case 'c': // %c Month, numeric (0..12) - builder.appendMonthOfYear(1); - break; - case 'd': // %d Day of the month, numeric (00..31) - builder.appendDayOfMonth(2); - break; - case 'e': // %e Day of the month, numeric (0..31) - builder.appendDayOfMonth(1); - break; - case 'H': // %H Hour (00..23) - builder.appendHourOfDay(2); - hasTimePart = true; - break; - case 'h': // %h Hour (01..12) - case 'I': // %I Hour (01..12) - builder.appendClockhourOfHalfday(2); - hasTimePart = true; - break; - case 'i': // %i Minutes, numeric (00..59) - builder.appendMinuteOfHour(2); - hasTimePart = true; - break; - case 'j': // %j Day of year (001..366) - builder.appendDayOfYear(3); - break; - case 'k': // %k Hour (0..23) - builder.appendHourOfDay(1); - hasTimePart = true; - break; - case 'l': // %l Hour (1..12) - builder.appendClockhourOfHalfday(1); - hasTimePart = true; - break; - case 'M': // %M Month name (January..December) - builder.appendMonthOfYearText(); - break; - case 'm': // %m Month, numeric (00..12) - builder.appendMonthOfYear(2); - break; - case 'p': // %p AM or PM - builder.appendHalfdayOfDayText(); - break; - case 'r': // %r Time, 12-hour (hh:mm:ss followed by AM or PM) - builder.appendClockhourOfHalfday(2) - .appendLiteral(':') - .appendMinuteOfHour(2) - .appendLiteral(':') - .appendSecondOfMinute(2) - .appendLiteral(' ') - .appendHalfdayOfDayText(); - hasTimePart = true; - break; - case 'S': // %S Seconds (00..59) - case 's': // %s Seconds (00..59) - builder.appendSecondOfMinute(2); - hasTimePart = true; - break; - case 'T': // %T Time, 24-hour (hh:mm:ss) - builder.appendHourOfDay(2) - .appendLiteral(':') - .appendMinuteOfHour(2) - .appendLiteral(':') - .appendSecondOfMinute(2); - hasTimePart = true; - break; - case 'v': // %v Week (01..53), where Monday is the first day of the week; used with %x - builder.appendWeekOfWeekyear(2); - break; - case 'x': // %x Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v - builder.appendWeekyear(4, 4); - break; - case 'W': // %W Weekday name (Sunday..Saturday) - builder.appendDayOfWeekText(); - break; - case 'Y': // %Y Year, numeric, four digits - builder.appendYear(4, 4); - break; - case 'y': // %y Year, numeric (two digits) - builder.appendTwoDigitYear(2020); - break; - case 'f': // %f Microseconds (000000..999999) - case 'w': // %w Day of the week (0=Sunday..6=Saturday) - case 'U': // %U Week (00..53), where Sunday is the first day of the week - case 'u': // %u Week (00..53), where Monday is the first day of the week - case 'V': // %V Week (01..53), where Sunday is the first day of the week; used with %X - case 'X': // %X Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V - case 'D': // %D Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) - throw new AnalysisException(String.format("%%%s not supported in date format string", character)); - case '%': // %% A literal "%" character - builder.appendLiteral('%'); - break; - default: // % The literal character represented by - builder.appendLiteral(character); - break; - } - escaped = false; - } else if (character == '%') { - escaped = true; - } else { - builder.appendLiteral(character); - } - } - - Date retDate = new Date(builder.toFormatter().withZone(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())).withLocale(Locale.ENGLISH).parseMillis(date.getStringValue())); - //Date retDate = new Date(date2.getTime() + - // DateTimeZone.forID("Asia/Shanghai").getOffset(0) - TimeUtils.getTimeZone().getOffset(0)); - - if (hasTimePart) { - return new DateLiteral(dateFormat(retDate, "%Y-%m-%d %H:%i:%s", false), Type.DATETIME); - } else { - return new DateLiteral(dateFormat(retDate, "%Y-%m-%d", false), Type.DATE); - } + DateLiteral dateLiteral = DateLiteral.dateParser(date.getStringValue(), fmtLiteral.getStringValue()); + return dateLiteral; } @FEFunction(name = "date_sub", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") @@ -237,36 +111,27 @@ public static DateLiteral dateSub(LiteralExpr date, LiteralExpr day) throws Anal @FEFunction(name = "year", argTypes = { "DATETIME" }, returnType = "INT") public static IntLiteral year(LiteralExpr arg) throws AnalysisException { - long timestamp = getTime(arg); - Calendar instance = Calendar.getInstance(); - instance.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); - instance.setTimeInMillis(timestamp); - return new IntLiteral(instance.get(Calendar.YEAR), Type.INT); + return new IntLiteral(((DateLiteral) arg).getYear(), Type.INT); } @FEFunction(name = "month", argTypes = { "DATETIME" }, returnType = "INT") public static IntLiteral month(LiteralExpr arg) throws AnalysisException { - long timestamp = getTime(arg); - Calendar instance = Calendar.getInstance(); - instance.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); - instance.setTimeInMillis(timestamp); - return new IntLiteral(instance.get(Calendar.MONTH) + 1, Type.INT); + return new IntLiteral(((DateLiteral) arg).getMonth(), Type.INT); } @FEFunction(name = "day", argTypes = { "DATETIME" }, returnType = "INT") public static IntLiteral day(LiteralExpr arg) throws AnalysisException { - long timestamp = getTime(arg); - Calendar instance = Calendar.getInstance(); - instance.setTimeInMillis(timestamp); - instance.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); - return new IntLiteral(instance.get(Calendar.DAY_OF_MONTH), Type.INT); + return new IntLiteral(((DateLiteral) arg).getDay(), Type.INT); } @FEFunction(name = "unix_timestamp", argTypes = { "DATETIME" }, returnType = "INT") public static IntLiteral unix_timestamp(LiteralExpr arg) throws AnalysisException { - long timestamp = getTime(arg); - timestamp += DateTimeZone.forID("Asia/Shanghai").getOffset(0) - TimeUtils.getTimeZone().getOffset(0); - return new IntLiteral(timestamp / 1000, Type.INT); + try { + + return new IntLiteral(((DateLiteral) arg).unixTime(TimeUtils.getTimeZone()), Type.INT); + } catch (ParseException e) { + throw new AnalysisException(e.getLocalizedMessage()); + } } @FEFunction(name = "from_unixtime", argTypes = { "INT" }, returnType = "VARCHAR") @@ -275,8 +140,10 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime) throws AnalysisEx if (unixTime.getLongValue() < 0) { throw new AnalysisException("unixtime should larger than zero"); } - Date date = new Date(unixTime.getLongValue() * 1000); - return new StringLiteral(dateFormat(date, "%Y-%m-%d %H:%i:%s", true)); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateFormat.setTimeZone(TimeUtils.getTimeZone()); + String dateLiteral = dateFormat.format(new Date(unixTime.getLongValue() * 1000)); + return new StringLiteral(dateLiteral); } @FEFunction(name = "from_unixtime", argTypes = { "INT", "VARCHAR" }, returnType = "VARCHAR") public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmtLiteral) throws AnalysisException { @@ -284,32 +151,12 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmt if (unixTime.getLongValue() < 0) { throw new AnalysisException("unixtime should larger than zero"); } - Date date = new Date(unixTime.getLongValue() * 1000); - //currently, doris BE only support "yyyy-MM-dd HH:mm:ss" and "yyyy-MM-dd" format - return new StringLiteral(dateFormat(date, fmtLiteral.getStringValue(), false)); - //return new StringLiteral(DateFormatUtils.format(date, fmtLiteral.getStringValue())); - } - private static long getTime(LiteralExpr expr) throws AnalysisException { - if (expr instanceof DateLiteral) { - return expr.getLongValue(); - } else { - String[] parsePatterns = {"yyyyMMdd", "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss"}; - - SimpleDateFormat parser = new SimpleDateFormat(parsePatterns[0]); - //parser.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); - ParsePosition pos = new ParsePosition(0); - - for (int i = 0; i < parsePatterns.length; ++i) { - parser.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); - parser.applyPattern(parsePatterns[i]); - pos.setIndex(0); - Date date = parser.parse(expr.getStringValue(), pos); - if (date != null && pos.getIndex() == expr.getStringValue().length()) { - return date.getTime(); - } - } - } - throw new AnalysisException("Unable to parse the date: " + expr.getStringValue()); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + dateFormat.setTimeZone(TimeUtils.getTimeZone()); + String dateLiteral = dateFormat.format(new Date(unixTime.getLongValue() * 1000)); + + DateLiteral d = new DateLiteral(dateLiteral, Type.DATETIME); + return new StringLiteral(d.dateFormat(fmtLiteral.getStringValue())); } private static int calFirstWeekDay(int year, int firstWeekDay) { @@ -323,207 +170,6 @@ private static int calFirstWeekDay(int year, int firstWeekDay) { return firstDay; } - private static String dateFormat(Date date, String pattern, boolean isLiteral) { - DateTimeFormatterBuilder formatterBuilder = new DateTimeFormatterBuilder(); - Calendar calendar = Calendar.getInstance(); - if (isLiteral){ - calendar.setTimeZone(TimeZone.getTimeZone("+08:00")); - } else { - calendar.setTimeZone(TimeUtils.getTimeZone()); - } - boolean escaped = false; - for (int i = 0; i < pattern.length(); i++) { - char character = pattern.charAt(i); - if (escaped) { - switch (character) { - case 'a': // %a Abbreviated weekday name (Sun..Sat) - formatterBuilder.appendDayOfWeekShortText(); - break; - case 'b': // %b Abbreviated month name (Jan..Dec) - formatterBuilder.appendMonthOfYearShortText(); - break; - case 'c': // %c Month, numeric (0..12) - formatterBuilder.appendMonthOfYear(1); - break; - case 'd': // %d Day of the month, numeric (00..31) - formatterBuilder.appendDayOfMonth(2); - break; - case 'e': // %e Day of the month, numeric (0..31) - formatterBuilder.appendDayOfMonth(1); - break; - case 'f': // %f Microseconds (000000..999999) - formatterBuilder.appendFractionOfSecond(6, 9); - break; - case 'H': // %H Hour (00..23) - formatterBuilder.appendHourOfDay(2); - break; - case 'h': // %h Hour (01..12) - case 'I': // %I Hour (01..12) - formatterBuilder.appendClockhourOfHalfday(2); - break; - case 'i': // %i Minutes, numeric (00..59) - formatterBuilder.appendMinuteOfHour(2); - break; - case 'j': // %j Day of year (001..366) - formatterBuilder.appendDayOfYear(3); - break; - case 'k': // %k Hour (0..23) - formatterBuilder.appendHourOfDay(1); - break; - case 'l': // %l Hour (1..12) - formatterBuilder.appendClockhourOfHalfday(1); - break; - case 'M': // %M Month name (January..December) - formatterBuilder.appendMonthOfYearText(); - break; - case 'm': // %m Month, numeric (00..12) - formatterBuilder.appendMonthOfYear(2); - break; - case 'p': // %p AM or PM - formatterBuilder.appendHalfdayOfDayText(); - break; - case 'r': // %r Time, 12-hour (hh:mm:ss followed by AM or PM) - formatterBuilder.appendClockhourOfHalfday(2) - .appendLiteral(':') - .appendMinuteOfHour(2) - .appendLiteral(':') - .appendSecondOfMinute(2) - .appendLiteral(' ') - .appendHalfdayOfDayText(); - break; - case 'S': // %S Seconds (00..59) - case 's': // %s Seconds (00..59) - formatterBuilder.appendSecondOfMinute(2); - break; - case 'T': // %T Time, 24-hour (hh:mm:ss) - formatterBuilder.appendHourOfDay(2) - .appendLiteral(':') - .appendMinuteOfHour(2) - .appendLiteral(':') - .appendSecondOfMinute(2); - break; - case 'V': // %V Week (01..53), where Sunday is the first day of the week; used with %X - { - int week; - calendar.setTime(date); - int firstSunday = calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.SUNDAY); - if (calendar.get(Calendar.DATE) <= 7 && calendar.get(Calendar.MONTH) == Calendar.JANUARY - && calendar.get(Calendar.DATE) >= firstSunday) { - week = 1; - } else { - calendar.add(Calendar.DATE, -7); - week = calendar.get(Calendar.WEEK_OF_YEAR) + - (calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.SUNDAY) == 1 ? 1 : 0); - } - formatterBuilder.appendLiteral(String.format("%02d", week)); - break; - } - case 'v': // %v Week (01..53), where Monday is the first day of the week; used with %x - formatterBuilder.appendWeekOfWeekyear(2); - break; - case 'X': // %X Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V - calendar.setTime(date); - if(calendar.get(Calendar.MONTH) == Calendar.JANUARY && - calendar.get(Calendar.DATE) < calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.SUNDAY)) { - formatterBuilder.appendLiteral(String.valueOf(calendar.get(Calendar.YEAR) - 1)); - } else { - formatterBuilder.appendLiteral(String.valueOf(calendar.get(Calendar.YEAR))); - } - break; - case 'x': // %x Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v - formatterBuilder.appendWeekyear(4, 4); - break; - case 'W': // %W Weekday name (Sunday..Saturday) - formatterBuilder.appendDayOfWeekText(); - break; - case 'w': // %w Day of the week (0=Sunday..6=Saturday) - calendar.setTime(date); - calendar.setFirstDayOfWeek(Calendar.SUNDAY); - formatterBuilder.appendLiteral(String.valueOf(calendar.get(Calendar.DAY_OF_WEEK) - 1)); - break; - case 'y': // %y Year, numeric (two digits) - int PIVOT_YEAR = 2020; - formatterBuilder.appendTwoDigitYear(PIVOT_YEAR); - break; - case 'Y': // %Y Year, numeric, four digits - formatterBuilder.appendYear(4, 4); - break; - case 'D': // %D Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) - calendar.setTime(date); - int day = calendar.get(Calendar.DAY_OF_MONTH); - if (day >= 10 && day <= 19) { - formatterBuilder.appendLiteral(String.valueOf(day) + "th"); - } else { - switch (day % 10) { - case 1: - formatterBuilder.appendLiteral(String.valueOf(day) + "st"); - break; - case 2: - formatterBuilder.appendLiteral(String.valueOf(day) + "nd"); - break; - case 3: - formatterBuilder.appendLiteral(String.valueOf(day) + "rd"); - break; - default: - formatterBuilder.appendLiteral(String.valueOf(day) + "th"); - break; - } - } - break; - case 'U': // %U Week (00..53), where Sunday is the first day of the week - calendar.setTime(date); - if (calendar.get(Calendar.DATE) <= 7 && calendar.get(Calendar.MONTH) == Calendar.JANUARY) { - int firstSunday = calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.SUNDAY); - formatterBuilder.appendLiteral(String.format("%02d", - ((calendar.get(Calendar.DATE) < firstSunday && firstSunday != 1) ? 0 : 1))); - } else { - calendar.add(Calendar.DATE, -7); - calendar.setFirstDayOfWeek(Calendar.SUNDAY); - formatterBuilder.appendLiteral(String.format("%02d", - calendar.get(Calendar.WEEK_OF_YEAR) - + (calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.SUNDAY) == 1 ? 1 : 0))); - } - break; - case 'u': // %u Week (00..53), where Monday is the first day of the week - { - calendar.setTime(date); - int week; - int firstMonday = calFirstWeekDay(calendar.get(Calendar.YEAR), Calendar.MONDAY); - if (calendar.get(Calendar.DATE) <= 7 && calendar.get(Calendar.MONTH) == Calendar.JANUARY) { - week = (calendar.get(Calendar.DATE) >= firstMonday || firstMonday == 1) ? 1 : 0 ; - week += (firstMonday >= 5 ? 1 : 0); - } else { - calendar.add(Calendar.DATE, -7); - calendar.setFirstDayOfWeek(Calendar.MONDAY); - week = calendar.get(Calendar.WEEK_OF_YEAR) + ((firstMonday >= 5 || firstMonday == 1) ? 1 : 0); - } - formatterBuilder.appendLiteral(String.format("%02d", week)); - break; - } - case '%': // %% A literal “%” character - formatterBuilder.appendLiteral('%'); - break; - default: // % The literal character represented by - formatterBuilder.appendLiteral(character); - break; - } - escaped = false; - } - else if (character == '%') { - escaped = true; - } - else { - formatterBuilder.appendLiteral(character); - } - } - DateTimeFormatter formatter = formatterBuilder.toFormatter(); - if (isLiteral) { - return formatter.withZone(DateTimeZone.forID("+08:00")).withLocale(Locale.US).print(date.getTime()); - } else { - return formatter.withZone(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())).withLocale(Locale.US).print(date.getTime()); - } - } - /** ------------------------------------------------------------------------------ */ From 00f523d6eb2ec3a29b8a1104aa4a384bb38b2c43 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Mon, 19 Aug 2019 17:32:48 +0800 Subject: [PATCH 09/18] add --- .../java/org/apache/doris/analysis/DateLiteral.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java index ed90fd1116d8d7..b8ccb2388d7dc2 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -488,13 +488,13 @@ public void readFields(DataInput in) throws IOException { //date = new Date(in.readLong()); if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { long packed_time = in.readLong(); - microsecond = (int) (packed_time % (1L << 24)); - int ymdhms = (int) (packed_time >> 24); - int ymd = ymdhms >> 17; - int hms = ymdhms % (1 << 17); + microsecond = (packed_time % (1L << 24)); + long ymdhms = (packed_time >> 24); + long ymd = ymdhms >> 17; + long hms = ymdhms % (1 << 17); day = ymd % (1 << 5); - int ym = ymd >> 5; + long ym = ymd >> 5; month = ym % 13; year = ym / 13; year %= 10000; From 3a376c01e036e1a9d609562fabac902c1ef080f6 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Mon, 19 Aug 2019 18:41:31 +0800 Subject: [PATCH 10/18] add --- .../apache/doris/analysis/DateLiteral.java | 463 ++++++++---------- .../org/apache/doris/rewrite/FEFunctions.java | 8 +- 2 files changed, 219 insertions(+), 252 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java index b8ccb2388d7dc2..f3525c686f4aff 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -42,75 +42,12 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.TimeZone; +import java.util.regex.Pattern; public class DateLiteral extends LiteralExpr { private static final Logger LOG = LogManager.getLogger(DateLiteral.class); - public long getHour() { - return hour; - } - - public void setHour(long hour) { - this.hour = hour; - } - - //private Date date; - long hour; - - public long getMinute() { - return minute; - } - - public void setMinute(long minute) { - this.minute = minute; - } - - public long getSecond() { - return second; - } - - public void setSecond(long second) { - this.second = second; - } - - public long getYear() { - return year; - } - - public void setYear(long year) { - this.year = year; - } - - public long getMonth() { - return month; - } - - public void setMonth(long month) { - this.month = month; - } - - public long getDay() { - return day; - } - - public void setDay(long day) { - this.day = day; - } - - public long getMicrosecond() { - return microsecond; - } - - public void setMicrosecond(long microsecond) { - this.microsecond = microsecond; - } - - long minute; - long second; - long year; - long month; - long day; - long microsecond; + private static final Pattern hasTimePart = Pattern.compile("^.*[HhIiklrSsT]+.*$"); private DateLiteral() { super(); @@ -150,7 +87,8 @@ public DateLiteral(DateLiteral other) { month = other.month; day = other.day; microsecond = other.microsecond; - //date = other.date; + + type = other.type; } @Override @@ -159,8 +97,174 @@ public Expr clone() { } public static DateLiteral createMinValue(Type type) throws AnalysisException{ - DateLiteral dateLiteral = new DateLiteral(type, false); - return dateLiteral; + return new DateLiteral(type, false); + } + + private void init(String s, Type type) throws AnalysisException { + try { + Preconditions.checkArgument(type.isDateType()); + LocalDateTime dateTime; + if (type == Type.DATE) { + dateTime = FormatBuilder("%Y-%m-%d").toFormatter().parseLocalDateTime(s); + } else { + dateTime = FormatBuilder("%Y-%m-%d %H-%i-%s").toFormatter().parseLocalDateTime(s); + } + year = dateTime.getYear(); + month = dateTime.getMonthOfYear(); + day = dateTime.getDayOfMonth(); + hour = dateTime.getHourOfDay(); + minute = dateTime.getMinuteOfHour(); + second = dateTime.getSecondOfMinute(); + this.type = type; + } catch (Exception ex) { + throw new AnalysisException(ex.getMessage()); + } + } + + @Override + public boolean isMinValue() { + switch (type.getPrimitiveType()) { + case DATE: + return this.getStringValue().compareTo("1900-01-01") == 0; + case DATETIME: + return this.getStringValue().compareTo("1900-01-01 00:00:00") == 0; + default: + return false; + } + } + + @Override + public Object getRealValue() { + if (type == Type.DATE) { + return year * 16 * 32L + month * 32 + day; + } else if (type == Type.DATETIME) { + return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; + } else { + Preconditions.checkState(false, "invalid date type: " + type); + return -1L; + } + } + + // Date column and Datetime column's hash value is not same. + @Override + public ByteBuffer getHashValue(PrimitiveType type) { + //String value = TimeUtils.format(date, type); + String value = getStringValue(); + ByteBuffer buffer; + try { + buffer = ByteBuffer.wrap(value.getBytes("UTF-8")); + } catch (Exception e) { + throw new RuntimeException(e); + } + return buffer; + } + + @Override + public int compareLiteral(LiteralExpr expr) { + if (expr instanceof NullLiteral) { + return 1; + } + + if (expr == MaxLiteral.MAX_VALUE) { + return -1; + } + // date time will not overflow when doing addition and subtraction + return Long.signum(getLongValue() - expr.getLongValue()); + } + + @Override + public String toSqlImpl() { + return "'" + getStringValue() + "'"; + } + + @Override + public String getStringValue() { + //return TimeUtils.format(date, type); + if (type == Type.DATE) { + return String.format("%04d", year) + "-" + + String.format("%02d", month) + "-" + + String.format("%02d", day); + } else { + return String.format("%04d", year) + "-" + + String.format("%02d", month) + "-" + + String.format("%02d", day) + " " + + String.format("%02d", hour) + ":" + + String.format("%02d",minute) + ":" + + String.format("%02d",second); + } + } + + @Override + public long getLongValue() { + return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; + } + + @Override + public double getDoubleValue() { + return getLongValue(); + } + + @Override + protected void toThrift(TExprNode msg) { + msg.node_type = TExprNodeType.DATE_LITERAL; + msg.date_literal = new TDateLiteral(getStringValue()); + } + + @Override + protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { + if (targetType.isDateType()) { + return this; + } else if (targetType.isStringType()) { + return new StringLiteral(getStringValue()); + } + Preconditions.checkState(false); + return this; + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + long ymd = ((year * 13 + month) << 5) | day; + long hms = (hour << 12) | (minute << 6) | second; + long tmp = ((ymd << 17) | hms) << 24 + microsecond; + long packed_datetime = tmp; + out.writeLong(packed_datetime); + } + + @Override + public void readFields(DataInput in) throws IOException { + super.readFields(in); + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { + long packed_time = in.readLong(); + microsecond = (packed_time % (1L << 24)); + long ymdhms = (packed_time >> 24); + long ymd = ymdhms >> 17; + long hms = ymdhms % (1 << 17); + + day = ymd % (1 << 5); + long ym = ymd >> 5; + month = ym % 13; + year = ym / 13; + year %= 10000; + second = hms % (1 << 6); + minute = (hms >> 6) % (1 << 6); + hour = (hms >> 12); + this.type = Type.DATETIME; + } else { + Date date = new Date(in.readLong()); + String date_str = TimeUtils.format(date, Type.DATETIME); + try { + init(date_str, Type.DATETIME); + } catch (AnalysisException ex) { + throw new IOException(ex.getMessage()); + } + } + } + + public static DateLiteral read(DataInput in) throws IOException { + DateLiteral literal = new DateLiteral(); + literal.readFields(in); + return literal; } public long unixTime(TimeZone timeZone) throws ParseException { @@ -173,6 +277,12 @@ public long unixTime(TimeZone timeZone) throws ParseException { public static DateLiteral dateParser(String date, String pattern) throws AnalysisException{ DateLiteral dateLiteral = new DateLiteral(); + if(hasTimePart.matcher(pattern).matches()) { + dateLiteral.setType(Type.DATETIME); + } else { + dateLiteral.setType(Type.DATE); + } + LocalDateTime dateTime = FormatBuilder(pattern).toFormatter().parseLocalDateTime(date); dateLiteral.setYear(dateTime.getYear()); dateLiteral.setMonth(dateTime.getMonthOfYear()); @@ -198,12 +308,11 @@ public String dateFormat(String pattern) throws AnalysisException{ } return builder.toFormatter().parseLocalDateTime(getStringValue()) - .toString(FormatBuilder(pattern).toFormatter()); + .toString(FormatBuilder(pattern).toFormatter()); } private static DateTimeFormatterBuilder FormatBuilder(String pattern) throws AnalysisException{ - boolean hasTimePart = false; DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); //String formatString = fmtLiteral.getStringValue(); boolean escaped = false; @@ -228,27 +337,22 @@ private static DateTimeFormatterBuilder FormatBuilder(String pattern) throws Ana break; case 'H': // %H Hour (00..23) builder.appendHourOfDay(2); - hasTimePart = true; break; case 'h': // %h Hour (01..12) case 'I': // %I Hour (01..12) builder.appendClockhourOfHalfday(2); - hasTimePart = true; break; case 'i': // %i Minutes, numeric (00..59) builder.appendMinuteOfHour(2); - hasTimePart = true; break; case 'j': // %j Day of year (001..366) builder.appendDayOfYear(3); break; case 'k': // %k Hour (0..23) builder.appendHourOfDay(1); - hasTimePart = true; break; case 'l': // %l Hour (1..12) builder.appendClockhourOfHalfday(1); - hasTimePart = true; break; case 'M': // %M Month name (January..December) builder.appendMonthOfYearText(); @@ -267,12 +371,10 @@ private static DateTimeFormatterBuilder FormatBuilder(String pattern) throws Ana .appendSecondOfMinute(2) .appendLiteral(' ') .appendHalfdayOfDayText(); - hasTimePart = true; break; case 'S': // %S Seconds (00..59) case 's': // %s Seconds (00..59) builder.appendSecondOfMinute(2); - hasTimePart = true; break; case 'T': // %T Time, 24-hour (hh:mm:ss) builder.appendHourOfDay(2) @@ -280,7 +382,6 @@ private static DateTimeFormatterBuilder FormatBuilder(String pattern) throws Ana .appendMinuteOfHour(2) .appendLiteral(':') .appendSecondOfMinute(2); - hasTimePart = true; break; case 'v': // %v Week (01..53), where Monday is the first day of the week; used with %x builder.appendWeekOfWeekyear(2); @@ -322,200 +423,68 @@ private static DateTimeFormatterBuilder FormatBuilder(String pattern) throws Ana return builder; } - - //private void init(String s, Type type) throws AnalysisException { - private void init(String s, Type type) throws AnalysisException { - try { - Preconditions.checkArgument(type.isDateType()); - DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); - if (type == Type.DATE) { - builder.appendYear(4, 4).appendLiteral("-"). - appendMonthOfYear(2).appendLiteral("-").appendDayOfMonth(2); - LocalDateTime datetime = builder.toFormatter().parseLocalDateTime(s); - year = datetime.getYear(); - month = datetime.getMonthOfYear(); - day = datetime.getDayOfMonth(); - } else { - builder.appendYear(4, 4).appendLiteral("-"). - appendMonthOfYear(2).appendLiteral("-") - .appendDayOfMonth(2).appendLiteral(" ") - .appendHourOfDay(2).appendLiteral(":") - .appendMinuteOfHour(2).appendLiteral(":") - .appendSecondOfMinute(2); - - LocalDateTime datetime = builder.toFormatter().parseLocalDateTime(s); - year = datetime.getYear(); - month = datetime.getMonthOfYear(); - day = datetime.getDayOfMonth(); - hour = datetime.getHourOfDay(); - minute = datetime.getMinuteOfHour(); - second = datetime.getSecondOfMinute(); - } - this.type = type; - } catch (Exception ex) { - throw new AnalysisException(ex.getMessage()); - } - - /* - date = TimeUtils.parseDate(s, type); - if (type.isScalarType(PrimitiveType.DATE)) { - if (date.compareTo(TimeUtils.MAX_DATE) > 0 || date.compareTo(TimeUtils.MIN_DATE) < 0) { - throw new AnalysisException("Date type column should range from [" + TimeUtils.MIN_DATE + "] to [" - + TimeUtils.MAX_DATE + "]"); - } - } else { - if (date.compareTo(TimeUtils.MAX_DATETIME) > 0 || date.compareTo(TimeUtils.MIN_DATETIME) < 0) { - throw new AnalysisException("Datetime type column should range from [" + TimeUtils.MIN_DATETIME - + "] to [" + TimeUtils.MAX_DATETIME + "]"); - } - } - this.type = type; - */ + public long getHour() { + return hour; } - @Override - public boolean isMinValue() { - switch (type.getPrimitiveType()) { - case DATE: - return this.getStringValue().compareTo("1900-01-01") == 0; - case DATETIME: - return this.getStringValue().compareTo("1900-01-01 00:00:00") == 0; - default: - return false; - } + public void setHour(long hour) { + this.hour = hour; } - @Override - public Object getRealValue() { - return getStringValue(); - //return TimeUtils.dateTransform(date.getTime(), type); + public long getMinute() { + return minute; } - // Date column and Datetime column's hash value is not same. - @Override - public ByteBuffer getHashValue(PrimitiveType type) { - //String value = TimeUtils.format(date, type); - String value = getStringValue(); - ByteBuffer buffer; - try { - buffer = ByteBuffer.wrap(value.getBytes("UTF-8")); - } catch (Exception e) { - throw new RuntimeException(e); - } - return buffer; + public void setMinute(long minute) { + this.minute = minute; } - @Override - public int compareLiteral(LiteralExpr expr) { - if (expr instanceof NullLiteral) { - return 1; - } - - if (expr == MaxLiteral.MAX_VALUE) { - return -1; - } - // date time will not overflow when doing addition and subtraction - return Long.signum(getLongValue() - expr.getLongValue()); + public long getSecond() { + return second; } - @Override - public String toSqlImpl() { - return "'" + getStringValue() + "'"; + public void setSecond(long second) { + this.second = second; } - @Override - public String getStringValue() { - //return TimeUtils.format(date, type); - if (type == Type.DATE) { - return String.format("%04d", year) + "-" + - String.format("%02d", month) + "-" + - String.format("%02d", day); - } else { - return String.format("%04d", year) + "-" + - String.format("%02d", month) + "-" + - String.format("%02d", day) + " " + - String.format("%02d", hour) + ":" + - String.format("%02d",minute) + ":" + - String.format("%02d",second); - } + public long getYear() { + return year; } - @Override - public long getLongValue() { - return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; + public void setYear(long year) { + this.year = year; } - @Override - public double getDoubleValue() { - return getLongValue(); + public long getMonth() { + return month; } - @Override - protected void toThrift(TExprNode msg) { - msg.node_type = TExprNodeType.DATE_LITERAL; - msg.date_literal = new TDateLiteral(getStringValue()); - } - /* - public Date getValue() { - return date; + public void setMonth(long month) { + this.month = month; } - */ - @Override - protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { - if (targetType.isDateType()) { - return this; - } else if (targetType.isStringType()) { - return new StringLiteral(getStringValue()); - } - Preconditions.checkState(false); - return this; + public long getDay() { + return day; } - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - long ymd = ((year * 13 + month) << 5) | day; - long hms = (hour << 12) | (minute << 6) | second; - long tmp = ((ymd << 17) | hms) << 24 + microsecond; - long packed_datetime = tmp; - out.writeLong(packed_datetime); + public void setDay(long day) { + this.day = day; } - @Override - public void readFields(DataInput in) throws IOException { - super.readFields(in); - //date = new Date(in.readLong()); - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { - long packed_time = in.readLong(); - microsecond = (packed_time % (1L << 24)); - long ymdhms = (packed_time >> 24); - long ymd = ymdhms >> 17; - long hms = ymdhms % (1 << 17); - - day = ymd % (1 << 5); - long ym = ymd >> 5; - month = ym % 13; - year = ym / 13; - year %= 10000; - second = hms % (1 << 6); - minute = (hms >> 6) % (1 << 6); - hour = (hms >> 12); - this.type = Type.DATETIME; - } else { - Date date = new Date(in.readLong()); - String date_str = TimeUtils.format(date, Type.DATETIME); - try { - init(date_str, Type.DATETIME); - } catch (AnalysisException ex) { - throw new IOException(ex.getMessage()); - } - } + public long getMicrosecond() { + return microsecond; } - public static DateLiteral read(DataInput in) throws IOException { - DateLiteral literal = new DateLiteral(); - literal.readFields(in); - return literal; + public void setMicrosecond(long microsecond) { + this.microsecond = microsecond; } + + + private long year; + private long month; + private long day; + private long hour; + private long minute; + private long second; + private long microsecond; } diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index f8e7ed3a0f88fe..ef1ac7eb953c9b 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -100,8 +100,7 @@ public static StringLiteral dateFormat(LiteralExpr date, StringLiteral fmtLitera @FEFunction(name = "str_to_date", argTypes = { "VARCHAR", "VARCHAR" }, returnType = "DATETIME") public static DateLiteral dateParse(StringLiteral date, StringLiteral fmtLiteral) throws AnalysisException { - DateLiteral dateLiteral = DateLiteral.dateParser(date.getStringValue(), fmtLiteral.getStringValue()); - return dateLiteral; + return DateLiteral.dateParser(date.getStringValue(), fmtLiteral.getStringValue()); } @FEFunction(name = "date_sub", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") @@ -127,7 +126,6 @@ public static IntLiteral day(LiteralExpr arg) throws AnalysisException { @FEFunction(name = "unix_timestamp", argTypes = { "DATETIME" }, returnType = "INT") public static IntLiteral unix_timestamp(LiteralExpr arg) throws AnalysisException { try { - return new IntLiteral(((DateLiteral) arg).unixTime(TimeUtils.getTimeZone()), Type.INT); } catch (ParseException e) { throw new AnalysisException(e.getLocalizedMessage()); @@ -155,8 +153,8 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmt dateFormat.setTimeZone(TimeUtils.getTimeZone()); String dateLiteral = dateFormat.format(new Date(unixTime.getLongValue() * 1000)); - DateLiteral d = new DateLiteral(dateLiteral, Type.DATETIME); - return new StringLiteral(d.dateFormat(fmtLiteral.getStringValue())); + DateLiteral dl = new DateLiteral(dateLiteral, Type.DATETIME); + return new StringLiteral(dl.dateFormat(fmtLiteral.getStringValue())); } private static int calFirstWeekDay(int year, int firstWeekDay) { From 4c486b98d751c378d7fec7318ce7e7271c5d5179 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Mon, 19 Aug 2019 19:06:46 +0800 Subject: [PATCH 11/18] add --- .../apache/doris/rewrite/FEFunctionsTest.java | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java b/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java index 1d81f6a02f685e..f84333207de56b 100644 --- a/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java +++ b/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java @@ -66,11 +66,11 @@ public void dateDiffTest() throws AnalysisException { @Test public void dateAddTest() throws AnalysisException { - DateLiteral actualResult = FEFunctions.dateAdd(new StringLiteral("2018-08-08"), new IntLiteral(1)); + DateLiteral actualResult = FEFunctions.dateAdd(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(1)); DateLiteral expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); - actualResult = FEFunctions.dateAdd(new StringLiteral("2018-08-08"), new IntLiteral(-1)); + actualResult = FEFunctions.dateAdd(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(-1)); expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); } @@ -81,7 +81,7 @@ public void addDateTest() throws AnalysisException { DateLiteral expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); - actualResult = FEFunctions.addDate(new StringLiteral("2018-08-08"), new IntLiteral(-1)); + actualResult = FEFunctions.addDate(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(-1)); expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); @@ -89,11 +89,11 @@ public void addDateTest() throws AnalysisException { @Test public void daysAddTest() throws AnalysisException { - DateLiteral actualResult = FEFunctions.daysAdd(new StringLiteral("2018-08-08"), new IntLiteral(1)); + DateLiteral actualResult = FEFunctions.daysAdd(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(1)); DateLiteral expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); - actualResult = FEFunctions.daysAdd(new StringLiteral("2018-08-08"), new IntLiteral(-1)); + actualResult = FEFunctions.daysAdd(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(-1)); expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIME); Assert.assertEquals(expectedResult, actualResult); } @@ -123,11 +123,6 @@ public void fromUnixTimeTestException() throws AnalysisException { @Test public void dateFormatUtilTest() { try { - Assert.assertEquals("19670102,196701,196701,0101", FEFunctions.dateFormat(new DateLiteral("1967-01-02 13:04:05", Type.DATETIME), new StringLiteral("%Y%m%d,%X%V,%x%v,%U%u")).getStringValue()); - Assert.assertEquals("19960105,199553,199601,0001", FEFunctions.dateFormat(new DateLiteral("1996-01-05 13:04:05", Type.DATETIME), new StringLiteral("%Y%m%d,%X%V,%x%v,%U%u")).getStringValue()); - - Assert.assertEquals("2017-01-01,01,00", FEFunctions.dateFormat(new DateLiteral("2017-01-01 13:04:05", Type.DATETIME), new StringLiteral("%Y-%m-%d,%U,%u")).getStringValue()); - Assert.assertEquals("201753,201752,5352", FEFunctions.dateFormat(new DateLiteral("2017-12-31 13:04:05", Type.DATETIME),new StringLiteral("%X%V,%x%v,%U%u")).getStringValue()); DateLiteral testDate = new DateLiteral("2001-01-09 13:04:05", Type.DATETIME); Assert.assertEquals("Tue", FEFunctions.dateFormat(testDate, new StringLiteral("%a")).getStringValue()); @@ -151,7 +146,6 @@ public void dateFormatUtilTest() { Assert.assertEquals("13:04:05", FEFunctions.dateFormat(testDate, new StringLiteral("%T")).getStringValue()); Assert.assertEquals("02", FEFunctions.dateFormat(testDate, new StringLiteral("%v")).getStringValue()); Assert.assertEquals("Tuesday", FEFunctions.dateFormat(testDate, new StringLiteral("%W")).getStringValue()); - Assert.assertEquals("2", FEFunctions.dateFormat(testDate, new StringLiteral("%w")).getStringValue()); Assert.assertEquals("2001", FEFunctions.dateFormat(testDate, new StringLiteral("%Y")).getStringValue()); Assert.assertEquals("01", FEFunctions.dateFormat(testDate, new StringLiteral("%y")).getStringValue()); Assert.assertEquals("%", FEFunctions.dateFormat(testDate, new StringLiteral("%%")).getStringValue()); @@ -159,7 +153,6 @@ public void dateFormatUtilTest() { Assert.assertEquals("g", FEFunctions.dateFormat(testDate, new StringLiteral("%g")).getStringValue()); Assert.assertEquals("4", FEFunctions.dateFormat(testDate, new StringLiteral("%4")).getStringValue()); Assert.assertEquals("2001 02" ,FEFunctions.dateFormat(testDate, new StringLiteral("%x %v")).getStringValue()); - Assert.assertEquals("9th" ,FEFunctions.dateFormat(testDate, new StringLiteral("%D")).getStringValue()); } catch (AnalysisException e) { e.printStackTrace(); } @@ -225,43 +218,43 @@ public void dateParseTest() { @Test public void dateSubTest() throws AnalysisException { - DateLiteral actualResult = FEFunctions.dateSub(new StringLiteral("2018-08-08"), new IntLiteral(1)); + DateLiteral actualResult = FEFunctions.dateSub(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(1)); DateLiteral expectedResult = new DateLiteral("2018-08-07", Type.DATE); Assert.assertEquals(expectedResult, actualResult); - actualResult = FEFunctions.dateSub(new StringLiteral("2018-08-08"), new IntLiteral(-1)); + actualResult = FEFunctions.dateSub(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(-1)); expectedResult = new DateLiteral("2018-08-09", Type.DATE); Assert.assertEquals(expectedResult, actualResult); } @Test public void yearTest() throws AnalysisException { - IntLiteral actualResult = FEFunctions.year(new StringLiteral("2018-08-08")); + IntLiteral actualResult = FEFunctions.year(new DateLiteral("2018-08-08", Type.DATE)); IntLiteral expectedResult = new IntLiteral(2018, Type.INT); Assert.assertEquals(expectedResult, actualResult); - actualResult = FEFunctions.year(new StringLiteral("1970-01-02 11:46:40")); + actualResult = FEFunctions.year(new DateLiteral("1970-01-02 11:46:40", Type.DATETIME)); expectedResult = new IntLiteral(1970, Type.INT); Assert.assertEquals(expectedResult, actualResult); } @Test public void monthTest() throws AnalysisException { - IntLiteral actualResult = FEFunctions.month(new StringLiteral("2018-08-08")); + IntLiteral actualResult = FEFunctions.month(new DateLiteral("2018-08-08", Type.DATE)); IntLiteral expectedResult = new IntLiteral(8, Type.INT); Assert.assertEquals(expectedResult, actualResult); - actualResult = FEFunctions.month(new StringLiteral("1970-01-02 11:46:40")); + actualResult = FEFunctions.month(new DateLiteral("1970-01-02 11:46:40", Type.DATETIME)); expectedResult = new IntLiteral(1, Type.INT); Assert.assertEquals(expectedResult, actualResult); } @Test public void dayTest() throws AnalysisException { - IntLiteral actualResult = FEFunctions.day(new StringLiteral("2018-08-08")); + IntLiteral actualResult = FEFunctions.day(new DateLiteral("2018-08-08", Type.DATE)); IntLiteral expectedResult = new IntLiteral(8, Type.INT); Assert.assertEquals(expectedResult, actualResult); - actualResult = FEFunctions.day(new StringLiteral("1970-01-02 11:46:40")); + actualResult = FEFunctions.day(new DateLiteral("1970-01-02 11:46:40", Type.DATETIME)); expectedResult = new IntLiteral(2, Type.INT); Assert.assertEquals(expectedResult, actualResult); } From 6f123791c1c34be0ef012bd474d6d01edc69d717 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Mon, 19 Aug 2019 21:17:53 +0800 Subject: [PATCH 12/18] add --- .../org/apache/doris/analysis/DateLiteral.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java index f3525c686f4aff..3a4aa0e8d93544 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -47,7 +47,7 @@ public class DateLiteral extends LiteralExpr { private static final Logger LOG = LogManager.getLogger(DateLiteral.class); - private static final Pattern hasTimePart = Pattern.compile("^.*[HhIiklrSsT]+.*$"); + private static final Pattern HAS_TIME_PART = Pattern.compile("^.*[HhIiklrSsT]+.*$"); private DateLiteral() { super(); @@ -117,7 +117,7 @@ private void init(String s, Type type) throws AnalysisException { second = dateTime.getSecondOfMinute(); this.type = type; } catch (Exception ex) { - throw new AnalysisException(ex.getMessage()); + throw new AnalysisException("date literal [" + s + "] is valid"); } } @@ -148,7 +148,6 @@ public Object getRealValue() { // Date column and Datetime column's hash value is not same. @Override public ByteBuffer getHashValue(PrimitiveType type) { - //String value = TimeUtils.format(date, type); String value = getStringValue(); ByteBuffer buffer; try { @@ -179,7 +178,6 @@ public String toSqlImpl() { @Override public String getStringValue() { - //return TimeUtils.format(date, type); if (type == Type.DATE) { return String.format("%04d", year) + "-" + String.format("%02d", month) + "-" + @@ -226,8 +224,7 @@ public void write(DataOutput out) throws IOException { super.write(out); long ymd = ((year * 13 + month) << 5) | day; long hms = (hour << 12) | (minute << 6) | second; - long tmp = ((ymd << 17) | hms) << 24 + microsecond; - long packed_datetime = tmp; + long packed_datetime = ((ymd << 17) | hms) << 24 + microsecond; out.writeLong(packed_datetime); } @@ -270,14 +267,12 @@ public static DateLiteral read(DataInput in) throws IOException { public long unixTime(TimeZone timeZone) throws ParseException { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); dateFormat.setTimeZone(timeZone); - long timestamp = dateFormat.parse(getStringValue()).getTime(); - return timestamp; + return dateFormat.parse(getStringValue()).getTime(); } public static DateLiteral dateParser(String date, String pattern) throws AnalysisException{ DateLiteral dateLiteral = new DateLiteral(); - - if(hasTimePart.matcher(pattern).matches()) { + if(HAS_TIME_PART.matcher(pattern).matches()) { dateLiteral.setType(Type.DATETIME); } else { dateLiteral.setType(Type.DATE); @@ -311,10 +306,8 @@ public String dateFormat(String pattern) throws AnalysisException{ .toString(FormatBuilder(pattern).toFormatter()); } - private static DateTimeFormatterBuilder FormatBuilder(String pattern) throws AnalysisException{ DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); - //String formatString = fmtLiteral.getStringValue(); boolean escaped = false; for (int i = 0; i < pattern.length(); i++) { char character = pattern.charAt(i); @@ -479,7 +472,6 @@ public void setMicrosecond(long microsecond) { this.microsecond = microsecond; } - private long year; private long month; private long day; From e4917d0168446e72a12b1cc2c4a8d76168edb9c2 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Mon, 19 Aug 2019 21:18:29 +0800 Subject: [PATCH 13/18] add --- .../org/apache/doris/analysis/DateLiteral.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java index f3525c686f4aff..c04fb2592ecfab 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -58,15 +58,15 @@ public DateLiteral(Type type, boolean isMax) throws AnalysisException{ this.type = type; if (type == Type.DATE) { if (isMax) { - init("1900-01-01", Type.DATE); - } else { init("9999-12-31", Type.DATE); + } else { + init("1900-01-01", Type.DATE); } } else { if (isMax) { - init("1900-01-01 00:00:00", Type.DATETIME); - } else { init("9999-12-31 23:59:59", Type.DATETIME); + } else { + init("1900-01-01 00:00:00", Type.DATETIME); } } analysisDone(); @@ -107,7 +107,7 @@ private void init(String s, Type type) throws AnalysisException { if (type == Type.DATE) { dateTime = FormatBuilder("%Y-%m-%d").toFormatter().parseLocalDateTime(s); } else { - dateTime = FormatBuilder("%Y-%m-%d %H-%i-%s").toFormatter().parseLocalDateTime(s); + dateTime = FormatBuilder("%Y-%m-%d %H:%i:%s").toFormatter().parseLocalDateTime(s); } year = dateTime.getYear(); month = dateTime.getMonthOfYear(); @@ -179,7 +179,6 @@ public String toSqlImpl() { @Override public String getStringValue() { - //return TimeUtils.format(date, type); if (type == Type.DATE) { return String.format("%04d", year) + "-" + String.format("%02d", month) + "-" + @@ -268,7 +267,12 @@ public static DateLiteral read(DataInput in) throws IOException { } public long unixTime(TimeZone timeZone) throws ParseException { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + SimpleDateFormat dateFormat; + if (type == Type.DATE) { + dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + } else { + dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + } dateFormat.setTimeZone(timeZone); long timestamp = dateFormat.parse(getStringValue()).getTime(); return timestamp; From 05335dffefb265846466835fb7ee3163d5fbca63 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Mon, 19 Aug 2019 21:20:58 +0800 Subject: [PATCH 14/18] refactor date literal --- .../doris/catalog/RangePartitionInfo.java | 1 + .../org/apache/doris/rewrite/FEFunctions.java | 5 +++-- .../doris/catalog/PartitionKeyTest.java | 20 ++++++++++++++++++- .../doris/common/util/TimeUtilsTest.java | 3 --- .../apache/doris/rewrite/FEFunctionsTest.java | 5 +++-- 5 files changed, 26 insertions(+), 8 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java b/fe/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java index 404ed8c84e93dc..f4fbb27aa26abc 100644 --- a/fe/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java +++ b/fe/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java @@ -178,6 +178,7 @@ public Range handleNewSinglePartitionDesc(SingleRangePartitionDesc Range range = null; try { range = checkAndCreateRange(desc); + System.out.println(range.toString()); idToRange.put(partitionId, range); } catch (IllegalArgumentException e) { // Range.closedOpen may throw this if (lower > upper) diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index ef1ac7eb953c9b..85a95099834a3d 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -78,7 +78,7 @@ public static DateLiteral dateAdd(LiteralExpr date, LiteralExpr day) throws Anal DateLiteral dateLiteral = (DateLiteral) date; DateLiteral result = new DateLiteral(dateLiteral); - result.setDay(dateLiteral.getDay() - 1); + result.setDay(dateLiteral.getDay() + day.getLongValue()); return result; } @@ -126,7 +126,8 @@ public static IntLiteral day(LiteralExpr arg) throws AnalysisException { @FEFunction(name = "unix_timestamp", argTypes = { "DATETIME" }, returnType = "INT") public static IntLiteral unix_timestamp(LiteralExpr arg) throws AnalysisException { try { - return new IntLiteral(((DateLiteral) arg).unixTime(TimeUtils.getTimeZone()), Type.INT); + System.out.println("unix " + ((DateLiteral) arg).unixTime(TimeUtils.getTimeZone())); + return new IntLiteral(((DateLiteral) arg).unixTime(TimeUtils.getTimeZone()) / 1000, Type.INT); } catch (ParseException e) { throw new AnalysisException(e.getLocalizedMessage()); } diff --git a/fe/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java b/fe/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java index ad1702c4b89097..d6f2f2dc912f1a 100644 --- a/fe/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java +++ b/fe/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java @@ -29,10 +29,20 @@ import java.util.List; import java.util.TimeZone; +import org.apache.doris.common.FeConstants; +import org.easymock.EasyMock; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; - +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +@RunWith(PowerMockRunner.class) +@PowerMockIgnore({ "org.apache.log4j.*", "javax.management.*" }) +@PrepareForTest(Catalog.class) public class PartitionKeyTest { private static List allColumns; @@ -43,6 +53,8 @@ public class PartitionKeyTest { private static Column largeInt; private static Column date; private static Column datetime; + + private Catalog catalog; @BeforeClass public static void setUp() { @@ -143,6 +155,12 @@ public void compareTest() throws AnalysisException { @Test public void testSerialization() throws Exception { + catalog = EasyMock.createMock(Catalog.class); + PowerMock.mockStatic(Catalog.class); + EasyMock.expect(Catalog.getInstance()).andReturn(catalog).anyTimes(); + EasyMock.expect(Catalog.getCurrentCatalogJournalVersion()).andReturn(FeConstants.meta_version).anyTimes(); + PowerMock.replay(Catalog.class); + // 1. Write objects to file File file = new File("./keyRangePartition"); file.createNewFile(); diff --git a/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java b/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java index 49e86f36c686bd..1cbabecb8d9b38 100644 --- a/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java +++ b/fe/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java @@ -132,9 +132,6 @@ public void testDateTrans() throws AnalysisException { DateLiteral datetime = new DateLiteral("2015-03-01 12:00:00", ScalarType.DATETIME); Assert.assertEquals(20150301120000L, datetime.getRealValue()); - - Assert.assertEquals("2015-03-01", TimeUtils.format(date.getValue(), date.getType())); - Assert.assertEquals("2015-03-01 12:00:00", TimeUtils.format(datetime.getValue(), datetime.getType())); } } diff --git a/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java b/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java index f84333207de56b..d5bfde952c3512 100644 --- a/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java +++ b/fe/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java @@ -90,11 +90,12 @@ public void addDateTest() throws AnalysisException { @Test public void daysAddTest() throws AnalysisException { DateLiteral actualResult = FEFunctions.daysAdd(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(1)); - DateLiteral expectedResult = new DateLiteral("2018-08-09 00:00:00", Type.DATETIME); + DateLiteral expectedResult = new DateLiteral("2018-08-09", Type.DATE); + System.out.println(actualResult.getStringValue()); Assert.assertEquals(expectedResult, actualResult); actualResult = FEFunctions.daysAdd(new DateLiteral("2018-08-08", Type.DATE), new IntLiteral(-1)); - expectedResult = new DateLiteral("2018-08-07 00:00:00", Type.DATETIME); + expectedResult = new DateLiteral("2018-08-07", Type.DATE); Assert.assertEquals(expectedResult, actualResult); } From c587d5591d4a8ed02552b95653f08a9abba0c1f9 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Wed, 21 Aug 2019 13:31:54 +0800 Subject: [PATCH 15/18] add --- .../apache/doris/analysis/DateLiteral.java | 192 +++++++++--------- .../org/apache/doris/rewrite/FEFunctions.java | 34 ++-- 2 files changed, 117 insertions(+), 109 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java index 3a4aa0e8d93544..b03b3838af70d1 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -47,6 +47,13 @@ public class DateLiteral extends LiteralExpr { private static final Logger LOG = LogManager.getLogger(DateLiteral.class); + private static final DateLiteral MIN_DATE = new DateLiteral(1900, 1, 1); + private static final DateLiteral MAX_DATE = new DateLiteral(9999, 12, 31); + private static final DateLiteral MIN_DATETIME = + new DateLiteral(1900, 1, 1, 0, 0, 0); + private static final DateLiteral MAX_DATETIME = + new DateLiteral(9999, 12, 31, 23, 59, 59); + //Regex used to determine if the TIME field exists int date_format private static final Pattern HAS_TIME_PART = Pattern.compile("^.*[HhIiklrSsT]+.*$"); private DateLiteral() { @@ -58,15 +65,15 @@ public DateLiteral(Type type, boolean isMax) throws AnalysisException{ this.type = type; if (type == Type.DATE) { if (isMax) { - init("1900-01-01", Type.DATE); + copy(MIN_DATE); } else { - init("9999-12-31", Type.DATE); + copy(MAX_DATE); } } else { if (isMax) { - init("1900-01-01 00:00:00", Type.DATETIME); + copy(MIN_DATETIME); } else { - init("9999-12-31 23:59:59", Type.DATETIME); + copy(MAX_DATETIME); } } analysisDone(); @@ -78,6 +85,26 @@ public DateLiteral(String s, Type type) throws AnalysisException{ analysisDone(); } + public DateLiteral(long year, long month, long day) { + this.hour = 0; + this.minute = 0; + this.second = 0; + this.year = year; + this.month = month; + this.day = day; + this.type = Type.DATE; + } + + public DateLiteral(long year, long month, long day, long hour, long minute, long second) { + this.hour = hour; + this.minute = minute; + this.second = second; + this.year = year; + this.month = month; + this.day = day; + this.type = Type.DATETIME; + } + public DateLiteral(DateLiteral other) { super(other); hour = other.hour; @@ -87,15 +114,9 @@ public DateLiteral(DateLiteral other) { month = other.month; day = other.day; microsecond = other.microsecond; - type = other.type; } - @Override - public Expr clone() { - return new DateLiteral(this); - } - public static DateLiteral createMinValue(Type type) throws AnalysisException{ return new DateLiteral(type, false); } @@ -105,9 +126,9 @@ private void init(String s, Type type) throws AnalysisException { Preconditions.checkArgument(type.isDateType()); LocalDateTime dateTime; if (type == Type.DATE) { - dateTime = FormatBuilder("%Y-%m-%d").toFormatter().parseLocalDateTime(s); + dateTime = formatBuilder("%Y-%m-%d").toFormatter().parseLocalDateTime(s); } else { - dateTime = FormatBuilder("%Y-%m-%d %H-%i-%s").toFormatter().parseLocalDateTime(s); + dateTime = formatBuilder("%Y-%m-%d %H-%i-%s").toFormatter().parseLocalDateTime(s); } year = dateTime.getYear(); month = dateTime.getMonthOfYear(); @@ -121,13 +142,29 @@ private void init(String s, Type type) throws AnalysisException { } } + private void copy(DateLiteral other) { + hour = other.hour; + minute = other.minute; + second = other.second; + year = other.year; + month = other.month; + day = other.day; + microsecond = other.microsecond; + type = other.type; + } + + @Override + public Expr clone() { + return new DateLiteral(this); + } + @Override public boolean isMinValue() { switch (type.getPrimitiveType()) { case DATE: - return this.getStringValue().compareTo("1900-01-01") == 0; + return this.getStringValue().compareTo(MIN_DATE.getStringValue()) == 0; case DATETIME: - return this.getStringValue().compareTo("1900-01-01 00:00:00") == 0; + return this.getStringValue().compareTo(MIN_DATETIME.getStringValue()) == 0; default: return false; } @@ -179,16 +216,9 @@ public String toSqlImpl() { @Override public String getStringValue() { if (type == Type.DATE) { - return String.format("%04d", year) + "-" + - String.format("%02d", month) + "-" + - String.format("%02d", day); + return String.format("%04d-%02d-%02d", year, month, day); } else { - return String.format("%04d", year) + "-" + - String.format("%02d", month) + "-" + - String.format("%02d", day) + " " + - String.format("%02d", hour) + ":" + - String.format("%02d",minute) + ":" + - String.format("%02d",second); + return String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second); } } @@ -219,34 +249,41 @@ protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { return this; } - @Override - public void write(DataOutput out) throws IOException { - super.write(out); + + private long make_packed_datetime () { long ymd = ((year * 13 + month) << 5) | day; long hms = (hour << 12) | (minute << 6) | second; long packed_datetime = ((ymd << 17) | hms) << 24 + microsecond; - out.writeLong(packed_datetime); + return packed_datetime; + } + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeLong(make_packed_datetime()); + } + + private void from_packed_datetime (long packed_time) { + microsecond = (packed_time % (1L << 24)); + long ymdhms = (packed_time >> 24); + long ymd = ymdhms >> 17; + long hms = ymdhms % (1 << 17); + + day = ymd % (1 << 5); + long ym = ymd >> 5; + month = ym % 13; + year = ym / 13; + year %= 10000; + second = hms % (1 << 6); + minute = (hms >> 6) % (1 << 6); + hour = (hms >> 12); + this.type = Type.DATETIME; } @Override public void readFields(DataInput in) throws IOException { super.readFields(in); if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { - long packed_time = in.readLong(); - microsecond = (packed_time % (1L << 24)); - long ymdhms = (packed_time >> 24); - long ymd = ymdhms >> 17; - long hms = ymdhms % (1 << 17); - - day = ymd % (1 << 5); - long ym = ymd >> 5; - month = ym % 13; - year = ym / 13; - year %= 10000; - second = hms % (1 << 6); - minute = (hms >> 6) % (1 << 6); - hour = (hms >> 12); - this.type = Type.DATETIME; + from_packed_datetime(in.readLong()); } else { Date date = new Date(in.readLong()); String date_str = TimeUtils.format(date, Type.DATETIME); @@ -265,29 +302,30 @@ public static DateLiteral read(DataInput in) throws IOException { } public long unixTime(TimeZone timeZone) throws ParseException { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); dateFormat.setTimeZone(timeZone); - return dateFormat.parse(getStringValue()).getTime(); + return dateFormat.parse(String.valueOf(getLongValue())).getTime(); } public static DateLiteral dateParser(String date, String pattern) throws AnalysisException{ - DateLiteral dateLiteral = new DateLiteral(); + LocalDateTime dateTime = formatBuilder(pattern).toFormatter().parseLocalDateTime(date); + DateLiteral dateLiteral = new DateLiteral( + dateTime.getYear(), + dateTime.getMonthOfYear(), + dateTime.getDayOfMonth(), + dateTime.getHourOfDay(), + dateTime.getMinuteOfHour(), + dateTime.getSecondOfMinute()); if(HAS_TIME_PART.matcher(pattern).matches()) { dateLiteral.setType(Type.DATETIME); } else { dateLiteral.setType(Type.DATE); } - - LocalDateTime dateTime = FormatBuilder(pattern).toFormatter().parseLocalDateTime(date); - dateLiteral.setYear(dateTime.getYear()); - dateLiteral.setMonth(dateTime.getMonthOfYear()); - dateLiteral.setDay(dateTime.getDayOfMonth()); - dateLiteral.setHour(dateTime.getHourOfDay()); - dateLiteral.setMinute(dateTime.getMinuteOfHour()); - dateLiteral.setSecond(dateTime.getSecondOfMinute()); return dateLiteral; } + //Return the date stored in the dateliteral as pattern format. + //eg : "%Y-%m-%d" or "%Y-%m-%d %H:%i:%s" public String dateFormat(String pattern) throws AnalysisException{ DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); if (type == Type.DATE) { @@ -303,10 +341,10 @@ public String dateFormat(String pattern) throws AnalysisException{ } return builder.toFormatter().parseLocalDateTime(getStringValue()) - .toString(FormatBuilder(pattern).toFormatter()); + .toString(formatBuilder(pattern).toFormatter()); } - private static DateTimeFormatterBuilder FormatBuilder(String pattern) throws AnalysisException{ + private static DateTimeFormatterBuilder formatBuilder(String pattern) throws AnalysisException{ DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); boolean escaped = false; for (int i = 0; i < pattern.length(); i++) { @@ -416,60 +454,28 @@ private static DateTimeFormatterBuilder FormatBuilder(String pattern) throws Ana return builder; } - public long getHour() { - return hour; - } - - public void setHour(long hour) { - this.hour = hour; - } - - public long getMinute() { - return minute; - } - - public void setMinute(long minute) { - this.minute = minute; - } - - public long getSecond() { - return second; - } - - public void setSecond(long second) { - this.second = second; - } - public long getYear() { return year; } - public void setYear(long year) { - this.year = year; - } - public long getMonth() { return month; } - public void setMonth(long month) { - this.month = month; - } - public long getDay() { return day; } - public void setDay(long day) { - this.day = day; + public long getHour() { + return hour; } - public long getMicrosecond() { - return microsecond; + public long getMinute() { + return minute; } - public void setMicrosecond(long microsecond) { - this.microsecond = microsecond; + public long getSecond() { + return second; } private long year; diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index ef1ac7eb953c9b..ea4a7fcebdb92f 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -31,12 +31,13 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.joda.time.LocalDateTime; +import org.joda.time.format.DateTimeFormatterBuilder; import java.math.BigDecimal; import java.math.BigInteger; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.Calendar; import java.util.Date; /** @@ -76,10 +77,22 @@ public static IntLiteral dateDiff(LiteralExpr first, LiteralExpr second) throws @FEFunction(name = "date_add", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") public static DateLiteral dateAdd(LiteralExpr date, LiteralExpr day) throws AnalysisException { DateLiteral dateLiteral = (DateLiteral) date; - - DateLiteral result = new DateLiteral(dateLiteral); - result.setDay(dateLiteral.getDay() - 1); - return result; + DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); + if (dateLiteral.getType() == Type.DATE) { + builder.appendYear(4, 4).appendLiteral("-"). + appendMonthOfYear(2).appendLiteral("-").appendDayOfMonth(2); + } else { + builder.appendYear(4, 4).appendLiteral("-"). + appendMonthOfYear(2).appendLiteral("-") + .appendDayOfMonth(2).appendLiteral(" ") + .appendHourOfDay(2).appendLiteral(":") + .appendMinuteOfHour(2).appendLiteral(":") + .appendSecondOfMinute(2); + } + LocalDateTime dateTime = builder.toFormatter().parseLocalDateTime(dateLiteral.getStringValue()); + dateTime.plusDays((int) day.getLongValue()); + return new DateLiteral(dateTime.getYear(), dateTime.getMonthOfYear(), dateTime.getDayOfMonth(), + dateTime.getHourOfDay(), dateTime.getMinuteOfHour(), dateTime.getSecondOfMinute()); } @FEFunction(name = "adddate", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") @@ -157,17 +170,6 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmt return new StringLiteral(dl.dateFormat(fmtLiteral.getStringValue())); } - private static int calFirstWeekDay(int year, int firstWeekDay) { - Calendar calendar = Calendar.getInstance(); - calendar.set(year, Calendar.JANUARY,1); - int firstDay = 1; - calendar.set(Calendar.DAY_OF_MONTH, firstDay); - while (calendar.get(Calendar.DAY_OF_WEEK) != firstWeekDay) { - calendar.set(Calendar.DAY_OF_MONTH, ++firstDay); - } - return firstDay; - } - /** ------------------------------------------------------------------------------ */ From 6db4c683dcaa758405b1d5a8ae3f9ce630eec23f Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Wed, 21 Aug 2019 13:34:31 +0800 Subject: [PATCH 16/18] add --- fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index ea4a7fcebdb92f..a6189de145a937 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -26,7 +26,6 @@ import org.apache.doris.analysis.StringLiteral; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; - import com.google.common.base.Preconditions; import org.apache.doris.common.util.TimeUtils; import org.apache.logging.log4j.LogManager; From 266e74b682f1d03b90947f5e4b19145694e118a0 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Wed, 21 Aug 2019 19:49:38 +0700 Subject: [PATCH 17/18] function name to camel --- .../apache/doris/analysis/DateLiteral.java | 47 +++++++++++++++++-- .../org/apache/doris/common/FeConstants.java | 2 +- .../org/apache/doris/rewrite/FEFunctions.java | 23 +-------- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java index 8daaee9fb60acc..5be657b01dba90 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -55,6 +55,11 @@ public class DateLiteral extends LiteralExpr { new DateLiteral(9999, 12, 31, 23, 59, 59); //Regex used to determine if the TIME field exists int date_format private static final Pattern HAS_TIME_PART = Pattern.compile("^.*[HhIiklrSsT]+.*$"); + //Date Literal persist type in meta + private enum DateLiteralType { + DATETIME, + DATE + } private DateLiteral() { super(); @@ -250,7 +255,7 @@ protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { } - private long make_packed_datetime () { + private long makePackedDatetime() { long ymd = ((year * 13 + month) << 5) | day; long hms = (hour << 12) | (minute << 6) | second; long packed_datetime = ((ymd << 17) | hms) << 24 + microsecond; @@ -260,10 +265,18 @@ private long make_packed_datetime () { @Override public void write(DataOutput out) throws IOException { super.write(out); - out.writeLong(make_packed_datetime()); + //set flag bit in meta, 0 is DATETIME and 1 is DATE + if (this.type == Type.DATETIME) { + out.writeShort(DateLiteralType.DATETIME.ordinal()); + } else if (this.type == Type.DATE) { + out.writeShort(DateLiteralType.DATE.ordinal()); + } else { + throw new IOException("Error date literal type : " + type); + } + out.writeLong(makePackedDatetime()); } - private void from_packed_datetime (long packed_time) { + private void fromPackedDatetime(long packed_time) { microsecond = (packed_time % (1L << 24)); long ymdhms = (packed_time >> 24); long ymd = ymdhms >> 17; @@ -277,14 +290,24 @@ private void from_packed_datetime (long packed_time) { second = hms % (1 << 6); minute = (hms >> 6) % (1 << 6); hour = (hms >> 12); + // set default date literal type to DATETIME + // date literal read from meta will set type by flag bit; this.type = Type.DATETIME; } @Override public void readFields(DataInput in) throws IOException { super.readFields(in); - if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_59) { - from_packed_datetime(in.readLong()); + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_60) { + short date_literal_type = in.readShort(); + fromPackedDatetime(in.readLong()); + if (date_literal_type == DateLiteralType.DATETIME.ordinal()) { + this.type = Type.DATETIME; + } else if (date_literal_type == DateLiteralType.DATE.ordinal()) { + this.type = Type.DATE; + } else { + throw new IOException("Error date literal type : " + type); + } } else { Date date = new Date(in.readLong()); String date_str = TimeUtils.format(date, Type.DATETIME); @@ -455,6 +478,20 @@ private static DateTimeFormatterBuilder formatBuilder(String pattern) throws Ana return builder; } + public DateLiteral plusDays(int day) throws AnalysisException { + LocalDateTime dateTime; + DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); + if (type == Type.DATE) { + dateTime = formatBuilder("%Y-%m-%d").toFormatter().parseLocalDateTime(getStringValue()).plusDays(day); + } else { + dateTime = formatBuilder("%Y-%m-%d %H-%i-%s").toFormatter().parseLocalDateTime(getStringValue()).plusDays(day); + } + DateLiteral dateLiteral = new DateLiteral(dateTime.getYear(), dateTime.getMonthOfYear(), dateTime.getDayOfMonth(), + dateTime.getHourOfDay(), dateTime.getMinuteOfHour(), dateTime.getSecondOfMinute()); + dateLiteral.setType(type); + return dateLiteral; + } + public long getYear() { return year; } diff --git a/fe/src/main/java/org/apache/doris/common/FeConstants.java b/fe/src/main/java/org/apache/doris/common/FeConstants.java index 1c87c9926f49d9..9174ec4c56aa79 100644 --- a/fe/src/main/java/org/apache/doris/common/FeConstants.java +++ b/fe/src/main/java/org/apache/doris/common/FeConstants.java @@ -35,5 +35,5 @@ public class FeConstants { // general model // Current meta data version. Use this version to write journals and image - public static int meta_version = FeMetaVersion.VERSION_59; + public static int meta_version = FeMetaVersion.VERSION_60; } diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index 52492f9fb5d486..e7407876fdce38 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -30,8 +30,6 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.joda.time.LocalDateTime; -import org.joda.time.format.DateTimeFormatterBuilder; import java.math.BigDecimal; import java.math.BigInteger; @@ -76,26 +74,7 @@ public static IntLiteral dateDiff(LiteralExpr first, LiteralExpr second) throws @FEFunction(name = "date_add", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") public static DateLiteral dateAdd(LiteralExpr date, LiteralExpr day) throws AnalysisException { DateLiteral dateLiteral = (DateLiteral) date; - DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); - if (dateLiteral.getType() == Type.DATE) { - builder.appendYear(4, 4).appendLiteral("-"). - appendMonthOfYear(2).appendLiteral("-").appendDayOfMonth(2); - } else { - builder.appendYear(4, 4).appendLiteral("-"). - appendMonthOfYear(2).appendLiteral("-") - .appendDayOfMonth(2).appendLiteral(" ") - .appendHourOfDay(2).appendLiteral(":") - .appendMinuteOfHour(2).appendLiteral(":") - .appendSecondOfMinute(2); - } - LocalDateTime dateTime = - builder.toFormatter().parseLocalDateTime(dateLiteral.getStringValue()).plusDays((int) day.getLongValue()); - if (dateLiteral.getType() == Type.DATE) { - return new DateLiteral(dateTime.getYear(), dateTime.getMonthOfYear(), dateTime.getDayOfMonth()); - } else { - return new DateLiteral(dateTime.getYear(), dateTime.getMonthOfYear(), dateTime.getDayOfMonth(), - dateTime.getHourOfDay(), dateTime.getMinuteOfHour(), dateTime.getSecondOfMinute()); - } + return dateLiteral.plusDays((int) day.getLongValue()); } @FEFunction(name = "adddate", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") From 241253839e53566ea644fa51bd05c248d3b70c7c Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Mon, 26 Aug 2019 16:43:53 +0700 Subject: [PATCH 18/18] add master forward time zone support --- .../apache/doris/analysis/DateLiteral.java | 98 +++++++++++++------ .../org/apache/doris/qe/ConnectProcessor.java | 3 + .../org/apache/doris/qe/MasterOpExecutor.java | 1 + .../org/apache/doris/rewrite/FEFunctions.java | 47 +++------ gensrc/thrift/FrontendService.thrift | 1 + 5 files changed, 88 insertions(+), 62 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java index 5be657b01dba90..ba27d36d386f11 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -31,15 +31,16 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.joda.time.LocalDateTime; +import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.DateTimeFormatterBuilder; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.Date; import java.util.TimeZone; import java.util.regex.Pattern; @@ -53,13 +54,35 @@ public class DateLiteral extends LiteralExpr { new DateLiteral(1900, 1, 1, 0, 0, 0); private static final DateLiteral MAX_DATETIME = new DateLiteral(9999, 12, 31, 23, 59, 59); + + private static DateTimeFormatter DATE_TIME_FORMATTER = null; + private static DateTimeFormatter DATE_FORMATTER = null; + static { + try { + DATE_TIME_FORMATTER = formatBuilder("%Y-%m-%d %H:%i:%s").toFormatter(); + DATE_FORMATTER = formatBuilder("%Y-%m-%d").toFormatter(); + } catch (AnalysisException e) { + LOG.error("invalid date format", e); + System.exit(-1); + } + } + //Regex used to determine if the TIME field exists int date_format private static final Pattern HAS_TIME_PART = Pattern.compile("^.*[HhIiklrSsT]+.*$"); //Date Literal persist type in meta - private enum DateLiteralType { - DATETIME, - DATE - } + private enum DateLiteralType { + DATETIME(0), + DATE(1); + + private final int value; + private DateLiteralType(int value) { + this.value = value; + } + + public int value() { + return value; + } + } private DateLiteral() { super(); @@ -90,6 +113,24 @@ public DateLiteral(String s, Type type) throws AnalysisException { analysisDone(); } + public DateLiteral(long unixTimestamp, TimeZone timeZone, Type type) { + DateTime dt = new DateTime(unixTimestamp, DateTimeZone.forTimeZone(timeZone)); + year = dt.getYear(); + month = dt.getMonthOfYear(); + day = dt.getDayOfMonth(); + hour = dt.getHourOfDay(); + minute = dt.getMinuteOfHour(); + second = dt.getSecondOfMinute(); + if (type == Type.DATE) { + hour = 0; + minute = 0; + second = 0; + this.type = Type.DATE; + } else { + this.type = Type.DATETIME; + } + } + public DateLiteral(long year, long month, long day) { this.hour = 0; this.minute = 0; @@ -131,9 +172,9 @@ private void init(String s, Type type) throws AnalysisException { Preconditions.checkArgument(type.isDateType()); LocalDateTime dateTime; if (type == Type.DATE) { - dateTime = formatBuilder("%Y-%m-%d").toFormatter().parseLocalDateTime(s); + dateTime = DATE_FORMATTER.parseLocalDateTime(s); } else { - dateTime = formatBuilder("%Y-%m-%d %H:%i:%s").toFormatter().parseLocalDateTime(s); + dateTime = DATE_TIME_FORMATTER.parseLocalDateTime(s); } year = dateTime.getYear(); month = dateTime.getMonthOfYear(); @@ -254,6 +295,12 @@ protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { return this; } + public void castToDate() { + this.type = Type.DATE; + hour = 0; + minute = 0; + second = 0; + } private long makePackedDatetime() { long ymd = ((year * 13 + month) << 5) | day; @@ -267,9 +314,9 @@ public void write(DataOutput out) throws IOException { super.write(out); //set flag bit in meta, 0 is DATETIME and 1 is DATE if (this.type == Type.DATETIME) { - out.writeShort(DateLiteralType.DATETIME.ordinal()); + out.writeShort(DateLiteralType.DATETIME.value()); } else if (this.type == Type.DATE) { - out.writeShort(DateLiteralType.DATE.ordinal()); + out.writeShort(DateLiteralType.DATE.value()); } else { throw new IOException("Error date literal type : " + type); } @@ -301,9 +348,9 @@ public void readFields(DataInput in) throws IOException { if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_60) { short date_literal_type = in.readShort(); fromPackedDatetime(in.readLong()); - if (date_literal_type == DateLiteralType.DATETIME.ordinal()) { + if (date_literal_type == DateLiteralType.DATETIME.value()) { this.type = Type.DATETIME; - } else if (date_literal_type == DateLiteralType.DATE.ordinal()) { + } else if (date_literal_type == DateLiteralType.DATE.value()) { this.type = Type.DATE; } else { throw new IOException("Error date literal type : " + type); @@ -325,10 +372,10 @@ public static DateLiteral read(DataInput in) throws IOException { return literal; } - public long unixTime(TimeZone timeZone) throws ParseException { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); - dateFormat.setTimeZone(timeZone); - return dateFormat.parse(String.valueOf(getLongValue())).getTime(); + public long unixTimestamp(TimeZone timeZone) { + DateTime dt = new DateTime((int) year, (int) month, (int) day, (int) hour, (int) minute, (int) second, + DateTimeZone.forTimeZone(timeZone)); + return dt.getMillis(); } public static DateLiteral dateParser(String date, String pattern) throws AnalysisException { @@ -353,19 +400,12 @@ public static DateLiteral dateParser(String date, String pattern) throws Analysi public String dateFormat(String pattern) throws AnalysisException { DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); if (type == Type.DATE) { - builder.appendYear(4, 4).appendLiteral("-"). - appendMonthOfYear(2).appendLiteral("-").appendDayOfMonth(2); + return DATE_FORMATTER.parseLocalDateTime(getStringValue()) + .toString(formatBuilder(pattern).toFormatter()); } else { - builder.appendYear(4, 4).appendLiteral("-"). - appendMonthOfYear(2).appendLiteral("-") - .appendDayOfMonth(2).appendLiteral(" ") - .appendHourOfDay(2).appendLiteral(":") - .appendMinuteOfHour(2).appendLiteral(":") - .appendSecondOfMinute(2); - } - - return builder.toFormatter().parseLocalDateTime(getStringValue()) + return DATE_TIME_FORMATTER.parseLocalDateTime(getStringValue()) .toString(formatBuilder(pattern).toFormatter()); + } } private static DateTimeFormatterBuilder formatBuilder(String pattern) throws AnalysisException { @@ -482,9 +522,9 @@ public DateLiteral plusDays(int day) throws AnalysisException { LocalDateTime dateTime; DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); if (type == Type.DATE) { - dateTime = formatBuilder("%Y-%m-%d").toFormatter().parseLocalDateTime(getStringValue()).plusDays(day); + dateTime = DATE_FORMATTER.parseLocalDateTime(getStringValue()).plusDays(day); } else { - dateTime = formatBuilder("%Y-%m-%d %H-%i-%s").toFormatter().parseLocalDateTime(getStringValue()).plusDays(day); + dateTime = DATE_TIME_FORMATTER.parseLocalDateTime(getStringValue()).plusDays(day); } DateLiteral dateLiteral = new DateLiteral(dateTime.getYear(), dateTime.getMonthOfYear(), dateTime.getDayOfMonth(), dateTime.getHourOfDay(), dateTime.getMinuteOfHour(), dateTime.getSecondOfMinute()); diff --git a/fe/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/src/main/java/org/apache/doris/qe/ConnectProcessor.java index a2c5bd532d43fd..cd4e6e2eb643ab 100644 --- a/fe/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -336,6 +336,9 @@ public TMasterOpResult proxyExecute(TMasterOpRequest request) { if (request.isSetUser_ip()) { ctx.setRemoteIP(request.getUser_ip()); } + if (request.isSetTime_zone()) { + ctx.getSessionVariable().setTimeZone(request.getTime_zone()); + } ctx.setThreadLocalInfo(); diff --git a/fe/src/main/java/org/apache/doris/qe/MasterOpExecutor.java b/fe/src/main/java/org/apache/doris/qe/MasterOpExecutor.java index e2ff93f7319677..e86a8517201c00 100644 --- a/fe/src/main/java/org/apache/doris/qe/MasterOpExecutor.java +++ b/fe/src/main/java/org/apache/doris/qe/MasterOpExecutor.java @@ -80,6 +80,7 @@ private void forward() throws Exception { params.setExecMemLimit(ctx.getSessionVariable().getMaxExecMemByte()); params.setQueryTimeout(ctx.getSessionVariable().getQueryTimeoutS()); params.setUser_ip(ctx.getRemoteIP()); + params.setTime_zone(ctx.getSessionVariable().getTimeZone()); LOG.info("Forward statement {} to Master {}", ctx.getStmtId(), thriftAddress); diff --git a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java index e7407876fdce38..2a8705896eb609 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -33,9 +33,6 @@ import java.math.BigDecimal; import java.math.BigInteger; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; /** * compute functions in FE. @@ -49,26 +46,20 @@ public class FEFunctions { */ @FEFunction(name = "timediff", argTypes = { "DATETIME", "DATETIME" }, returnType = "TIME") public static FloatLiteral timeDiff(LiteralExpr first, LiteralExpr second) throws AnalysisException { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - try { - long timediff = sdf.parse(first.getStringValue()).getTime() - sdf.parse(second.getStringValue()).getTime(); - return new FloatLiteral((double) timediff, Type.TIME); - } catch (ParseException e) { - throw new AnalysisException(e.getLocalizedMessage()); - } + long firstTimestamp = ((DateLiteral) first).unixTimestamp(TimeUtils.getTimeZone()); + long secondTimestamp = ((DateLiteral) second).unixTimestamp(TimeUtils.getTimeZone()); + return new FloatLiteral((double) (firstTimestamp - secondTimestamp) / 1000, Type.TIME); } @FEFunction(name = "datediff", argTypes = { "DATETIME", "DATETIME" }, returnType = "INT") public static IntLiteral dateDiff(LiteralExpr first, LiteralExpr second) throws AnalysisException { - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - try { - // DATEDIFF function only uses the date part for calculations and ignores the time part - long diff = sdf.parse(first.getStringValue()).getTime() - sdf.parse(second.getStringValue()).getTime(); - long datediff = diff / 1000 / 60 / 60 / 24; - return new IntLiteral(datediff, Type.INT); - } catch (ParseException e) { - throw new AnalysisException(e.getLocalizedMessage()); - } + DateLiteral firstDate = ((DateLiteral) first); + DateLiteral secondDate = ((DateLiteral) second); + // DATEDIFF function only uses the date part for calculations and ignores the time part + firstDate.castToDate(); + secondDate.castToDate(); + long datediff = (firstDate.unixTimestamp(TimeUtils.getTimeZone()) - secondDate.unixTimestamp(TimeUtils.getTimeZone())) / 1000 / 60 / 60 / 24; + return new IntLiteral(datediff, Type.INT); } @FEFunction(name = "date_add", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") @@ -120,11 +111,7 @@ public static IntLiteral day(LiteralExpr arg) throws AnalysisException { @FEFunction(name = "unix_timestamp", argTypes = { "DATETIME" }, returnType = "INT") public static IntLiteral unix_timestamp(LiteralExpr arg) throws AnalysisException { - try { - return new IntLiteral(((DateLiteral) arg).unixTime(TimeUtils.getTimeZone()) / 1000, Type.INT); - } catch (ParseException e) { - throw new AnalysisException(e.getLocalizedMessage()); - } + return new IntLiteral(((DateLiteral) arg).unixTimestamp(TimeUtils.getTimeZone()) / 1000, Type.INT); } @FEFunction(name = "from_unixtime", argTypes = { "INT" }, returnType = "VARCHAR") @@ -133,10 +120,8 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime) throws AnalysisEx if (unixTime.getLongValue() < 0) { throw new AnalysisException("unixtime should larger than zero"); } - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - dateFormat.setTimeZone(TimeUtils.getTimeZone()); - String dateLiteral = dateFormat.format(new Date(unixTime.getLongValue() * 1000)); - return new StringLiteral(dateLiteral); + DateLiteral dl = new DateLiteral(unixTime.getLongValue() * 1000, TimeUtils.getTimeZone(), Type.DATETIME); + return new StringLiteral(dl.getStringValue()); } @FEFunction(name = "from_unixtime", argTypes = { "INT", "VARCHAR" }, returnType = "VARCHAR") public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmtLiteral) throws AnalysisException { @@ -144,11 +129,7 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmt if (unixTime.getLongValue() < 0) { throw new AnalysisException("unixtime should larger than zero"); } - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - dateFormat.setTimeZone(TimeUtils.getTimeZone()); - String dateLiteral = dateFormat.format(new Date(unixTime.getLongValue() * 1000)); - - DateLiteral dl = new DateLiteral(dateLiteral, Type.DATETIME); + DateLiteral dl = new DateLiteral(unixTime.getLongValue() * 1000, TimeUtils.getTimeZone(), Type.DATETIME); return new StringLiteral(dl.dateFormat(fmtLiteral.getStringValue())); } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index cc90080dddc50d..dba0d5b28d27a4 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -407,6 +407,7 @@ struct TMasterOpRequest { 6: optional i64 execMemLimit 7: optional i32 queryTimeout 8: optional string user_ip + 9: optional string time_zone } struct TColumnDefinition {