From 69a50b362ccd9907e61890dd9c2141dff0d1f19d Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 25 Oct 2020 21:25:57 +0800 Subject: [PATCH 1/2] fix ut --- be/src/exprs/timestamp_functions.cpp | 20 +++++++++++++++---- be/src/runtime/datetime_value.h | 6 +++++- .../cumulative_compaction_policy_test.cpp | 6 +++--- docs/en/installing/compilation.md | 6 ++++++ docs/zh-CN/installing/compilation.md | 6 ++++++ .../doris/analysis/FunctionCallExpr.java | 3 +-- 6 files changed, 37 insertions(+), 10 deletions(-) diff --git a/be/src/exprs/timestamp_functions.cpp b/be/src/exprs/timestamp_functions.cpp index 79faee089c8a9f..0c8e6669231fac 100644 --- a/be/src/exprs/timestamp_functions.cpp +++ b/be/src/exprs/timestamp_functions.cpp @@ -126,7 +126,10 @@ IntVal TimestampFunctions::day_of_week( return IntVal::null(); } const DateTimeValue& ts_value = DateTimeValue::from_datetime_val(ts_val); - return IntVal((ts_value.weekday() + 1 ) % 7 + 1); + if (ts_value.is_valid_date()) { + return IntVal((ts_value.weekday() + 1 ) % 7 + 1); + } + return IntVal::null(); } IntVal TimestampFunctions::day_of_month( @@ -135,7 +138,10 @@ IntVal TimestampFunctions::day_of_month( return IntVal::null(); } const DateTimeValue& ts_value = DateTimeValue::from_datetime_val(ts_val); - return IntVal(ts_value.day()); + if (ts_value.is_valid_date()) { + return IntVal(ts_value.day()); + } + return IntVal::null(); } IntVal TimestampFunctions::day_of_year( @@ -144,7 +150,10 @@ IntVal TimestampFunctions::day_of_year( return IntVal::null(); } const DateTimeValue& ts_value = DateTimeValue::from_datetime_val(ts_val); - return IntVal(ts_value.day_of_year()); + if (ts_value.is_valid_date()) { + return IntVal(ts_value.day_of_year()); + } + return IntVal::null(); } IntVal TimestampFunctions::week_of_year( @@ -153,7 +162,10 @@ IntVal TimestampFunctions::week_of_year( return IntVal::null(); } const DateTimeValue& ts_value = DateTimeValue::from_datetime_val(ts_val); - return IntVal(ts_value.week(mysql_week_mode(3))); + if (ts_value.is_valid_date()) { + return IntVal(ts_value.week(mysql_week_mode(3))); + } + return IntVal::null(); } IntVal TimestampFunctions::hour( diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h index e8e80d71b639d6..37fa461464c21d 100644 --- a/be/src/runtime/datetime_value.h +++ b/be/src/runtime/datetime_value.h @@ -462,6 +462,10 @@ class DateTimeValue { void set_type(int type); + bool is_valid_date() const { + return !check_range() && !check_date() && _month > 0 && _day > 0; + } + private: // Used to make sure sizeof DateTimeValue friend class UnusedClass; @@ -502,7 +506,7 @@ class DateTimeValue { return _neg ? -tmp : tmp; } - // Check whether value of field is valid. + // Return true if range or date is invalid bool check_range() const; bool check_date() const; diff --git a/be/test/olap/cumulative_compaction_policy_test.cpp b/be/test/olap/cumulative_compaction_policy_test.cpp index ca80100bde8c5b..4d5d0d19c6757b 100644 --- a/be/test/olap/cumulative_compaction_policy_test.cpp +++ b/be/test/olap/cumulative_compaction_policy_test.cpp @@ -1052,20 +1052,20 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy, _pick_missing_version_cumulative rowsets.push_back(_tablet->get_rowset_by_version({4, 4})); std::shared_ptr mem_tracker(new MemTracker()); CumulativeCompaction compaction(_tablet, "label", mem_tracker); - compaction.find_longest_consecutive_version(&rowsets); + compaction.find_longest_consecutive_version(&rowsets, nullptr); ASSERT_EQ(3, rowsets.size()); ASSERT_EQ(2, rowsets[2]->end_version()); // no miss version std::vector rowsets2; rowsets2.push_back(_tablet->get_rowset_by_version({0, 0})); - compaction.find_longest_consecutive_version(&rowsets2); + compaction.find_longest_consecutive_version(&rowsets2, nullptr); ASSERT_EQ(1, rowsets2.size()); ASSERT_EQ(0, rowsets[0]->end_version()); // no version std::vector rowsets3; - compaction.find_longest_consecutive_version(&rowsets3); + compaction.find_longest_consecutive_version(&rowsets3, nullptr); ASSERT_EQ(0, rowsets3.size()); } } diff --git a/docs/en/installing/compilation.md b/docs/en/installing/compilation.md index 94bd5f99de2560..3cb7c5a0b8fc26 100644 --- a/docs/en/installing/compilation.md +++ b/docs/en/installing/compilation.md @@ -107,6 +107,12 @@ You can try to compile Doris directly in your own Linux environment. $ sh build.sh ``` After compilation, the output file is in the `output/` directory. + +## FAQ + +1. `Could not transfer artifact net.sourceforge.czt.dev:cup-maven-plugin:pom:1.6-cdh from/to xxx` + + If you encounter the above error, please refer to [PR #4769](https://github.com/apache/incubator-doris/pull/4769/files) to modify the cloudera-related repo configuration in `fe/pom.xml`. ## Special statement diff --git a/docs/zh-CN/installing/compilation.md b/docs/zh-CN/installing/compilation.md index 0df6b0e6b4d2d7..bb668392406511 100644 --- a/docs/zh-CN/installing/compilation.md +++ b/docs/zh-CN/installing/compilation.md @@ -109,6 +109,12 @@ under the License. 编译完成后，产出文件在 `output/` 目录中。 +## 常见问题 + +1. `Could not transfer artifact net.sourceforge.czt.dev:cup-maven-plugin:pom:1.6-cdh from/to xxx` + + 如遇到上述错误，请参照 [PR #4769](https://github.com/apache/incubator-doris/pull/4769/files) 修改 `fe/pom.xml` 中 cloudera 相关的仓库配置。 + ## 特别声明 自 0.13 版本开始，默认的编译产出中将取消对 [1] 和 [2] 两个第三方库的依赖。这两个第三方库为 [GNU General Public License V3](https://www.gnu.org/licenses/gpl-3.0.en.html) 协议。该协议与 [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) 协议不兼容，因此默认不出现在 Apache 发布版本中。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index d43178d4146d5e..074d4f7400ad1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -604,9 +604,8 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException { } } - if (fn.getFunctionName().getFunction().equals("time_diff")) { + if (fn.getFunctionName().getFunction().equals("timediff")) { fn.getReturnType().getPrimitiveType().setTimeType(); - return; } if (isAggregateFunction()) { From a4ecb658a68b4c84d87c295cf28b48739327e8d0 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 26 Oct 2020 15:31:42 +0800 Subject: [PATCH 2/2] add ut --- be/src/exprs/timestamp_functions.cpp | 11 +- be/test/exprs/timestamp_functions_test.cpp | 127 +++++++++++++++++++++ 2 files changed, 133 insertions(+), 5 deletions(-) diff --git a/be/src/exprs/timestamp_functions.cpp b/be/src/exprs/timestamp_functions.cpp index 0c8e6669231fac..9c64e60c5c522c 100644 --- a/be/src/exprs/timestamp_functions.cpp +++ b/be/src/exprs/timestamp_functions.cpp @@ -120,6 +120,7 @@ IntVal TimestampFunctions::month( const DateTimeValue& ts_value = DateTimeValue::from_datetime_val(ts_val); return IntVal(ts_value.month()); } + IntVal TimestampFunctions::day_of_week( FunctionContext* context, const DateTimeVal& ts_val) { if (ts_val.is_null) { @@ -138,10 +139,7 @@ IntVal TimestampFunctions::day_of_month( return IntVal::null(); } const DateTimeValue& ts_value = DateTimeValue::from_datetime_val(ts_val); - if (ts_value.is_valid_date()) { - return IntVal(ts_value.day()); - } - return IntVal::null(); + return IntVal(ts_value.day()); } IntVal TimestampFunctions::day_of_year( @@ -685,7 +683,10 @@ DoubleVal TimestampFunctions::time_diff( const DateTimeValue& ts_value1 = DateTimeValue::from_datetime_val(ts_val1); const DateTimeValue& ts_value2 = DateTimeValue::from_datetime_val(ts_val2); - return DoubleVal(ts_value1.second_diff(ts_value2)); + if (ts_value1.is_valid_date() && ts_value2.is_valid_date()) { + return DoubleVal(ts_value1.second_diff(ts_value2)); + } + return DoubleVal::null(); } IntVal TimestampFunctions::date_diff( diff --git a/be/test/exprs/timestamp_functions_test.cpp b/be/test/exprs/timestamp_functions_test.cpp index 09e7637e1a5873..6a4307099d32d0 100644 --- a/be/test/exprs/timestamp_functions_test.cpp +++ b/be/test/exprs/timestamp_functions_test.cpp @@ -72,6 +72,107 @@ TEST_F(TimestampFunctionsTest, day_of_week_test) { tv.type = TIME_DATETIME; ASSERT_EQ(7, TimestampFunctions::day_of_week(context, tv).val); + + // 2020-00-01 00:00:00 + DateTimeValue dtv2(20200001000000); + dtv2.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv2; + dtv2.to_datetime_val(&tv2); + ASSERT_EQ(true, TimestampFunctions::day_of_week(context, tv2).is_null); + + // 2020-01-00 00:00:00 + DateTimeValue dtv3(20200100000000); + dtv3.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv3; + dtv3.to_datetime_val(&tv3); + ASSERT_EQ(true, TimestampFunctions::day_of_week(context, tv3).is_null); + + delete context; +} + +TEST_F(TimestampFunctionsTest, day_of_month_test) { + doris_udf::FunctionContext *context = new doris_udf::FunctionContext(); + + // 2020-00-01 00:00:00 + DateTimeValue dtv1(20200001000000); + dtv1.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv1; + dtv1.to_datetime_val(&tv1); + ASSERT_EQ(false, TimestampFunctions::day_of_month(context, tv1).is_null); + ASSERT_EQ(1, TimestampFunctions::day_of_month(context, tv1).val); + + // 2020-01-00 00:00:00 + DateTimeValue dtv2(20200100000000); + dtv2.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv2; + dtv2.to_datetime_val(&tv2); + ASSERT_EQ(false, TimestampFunctions::day_of_month(context, tv2).is_null); + ASSERT_EQ(0, TimestampFunctions::day_of_month(context, tv2).val); + + // 2020-02-29 00:00:00 + DateTimeValue dtv3(20200229000000); + dtv3.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv3; + dtv3.to_datetime_val(&tv3); + ASSERT_EQ(false, TimestampFunctions::day_of_month(context, tv3).is_null); + ASSERT_EQ(29, TimestampFunctions::day_of_month(context, tv3).val); + + delete context; +} + +TEST_F(TimestampFunctionsTest, day_of_year_test) { + doris_udf::FunctionContext *context = new doris_udf::FunctionContext(); + + // 2020-00-01 00:00:00 + DateTimeValue dtv1(20200001000000); + dtv1.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv1; + dtv1.to_datetime_val(&tv1); + ASSERT_EQ(true, TimestampFunctions::day_of_year(context, tv1).is_null); + + // 2020-01-00 00:00:00 + DateTimeValue dtv2(20200100000000); + dtv2.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv2; + dtv2.to_datetime_val(&tv2); + ASSERT_EQ(true, TimestampFunctions::day_of_year(context, tv2).is_null); + + // 2020-02-29 00:00:00 + DateTimeValue dtv3(20200229000000); + dtv3.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv3; + dtv3.to_datetime_val(&tv3); + ASSERT_EQ(false, TimestampFunctions::day_of_year(context, tv3).is_null); + ASSERT_EQ(60, TimestampFunctions::day_of_year(context, tv3).val); + + delete context; +} + +TEST_F(TimestampFunctionsTest, week_of_year_test) { + doris_udf::FunctionContext *context = new doris_udf::FunctionContext(); + + // 2020-00-01 00:00:00 + DateTimeValue dtv1(20200001000000); + dtv1.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv1; + dtv1.to_datetime_val(&tv1); + ASSERT_EQ(true, TimestampFunctions::week_of_year(context, tv1).is_null); + + // 2020-01-00 00:00:00 + DateTimeValue dtv2(20200100000000); + dtv2.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv2; + dtv2.to_datetime_val(&tv2); + ASSERT_EQ(true, TimestampFunctions::week_of_year(context, tv2).is_null); + + // 2020-02-29 00:00:00 + DateTimeValue dtv3(20200229000000); + dtv3.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv3; + dtv3.to_datetime_val(&tv3); + ASSERT_EQ(false, TimestampFunctions::week_of_year(context, tv3).is_null); + ASSERT_EQ(9, TimestampFunctions::week_of_year(context, tv3).val); + delete context; } @@ -87,6 +188,32 @@ TEST_F(TimestampFunctionsTest, time_diff_test) { dt2.to_datetime_val(&tv2); ASSERT_EQ(-3662, TimestampFunctions::time_diff(ctx, tv1, tv2).val); + + // invalid + DateTimeValue dt3(20190018120000); + dt3.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv3; + dt3.to_datetime_val(&tv3); + + DateTimeValue dt4(20190718130102); + dt4.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv4; + dt4.to_datetime_val(&tv4); + + ASSERT_EQ(true, TimestampFunctions::time_diff(ctx, tv3, tv4).is_null); + + // invalid + DateTimeValue dt5(20190718120000); + dt5.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv5; + dt5.to_datetime_val(&tv5); + + DateTimeValue dt6(20190700130102); + dt6.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv6; + dt6.to_datetime_val(&tv6); + + ASSERT_EQ(true, TimestampFunctions::time_diff(ctx, tv5, tv6).is_null); } TEST_F(TimestampFunctionsTest, now) {