From b82a1bb83c4cf33f602bfa3bec96ca7f8fad555b Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Mon, 10 Mar 2025 17:53:10 +0800 Subject: [PATCH 1/4] support year of week --- be/src/vec/functions/date_time_transforms.h | 31 ++++++- be/src/vec/functions/to_time_function.cpp | 10 +-- be/src/vec/runtime/vdatetime_value.cpp | 26 ++++++ be/src/vec/runtime/vdatetime_value.h | 1 + .../doris/catalog/BuiltinScalarFunctions.java | 2 + .../nereids/stats/ExpressionEstimation.java | 16 ++++ .../functions/scalar/YearOfWeek.java | 86 +++++++++++++++++++ .../visitor/ScalarFunctionVisitor.java | 5 ++ .../datetime_functions/test_date_function.out | 62 +++++++++++++ .../test_date_function.groovy | 24 ++++++ 10 files changed, 252 insertions(+), 11 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/YearOfWeek.java diff --git a/be/src/vec/functions/date_time_transforms.h b/be/src/vec/functions/date_time_transforms.h index 63ed635df45431..b3e51a36907309 100644 --- a/be/src/vec/functions/date_time_transforms.h +++ b/be/src/vec/functions/date_time_transforms.h @@ -58,6 +58,7 @@ namespace doris::vectorized { #define TO_TIME_FUNCTION(CLASS, UNIT) TIME_FUNCTION_IMPL(CLASS, UNIT, UNIT()) TO_TIME_FUNCTION(ToYearImpl, year); +TO_TIME_FUNCTION(ToYearOfWeekImpl, year_of_week); TO_TIME_FUNCTION(ToQuarterImpl, quarter); TO_TIME_FUNCTION(ToMonthImpl, month); TO_TIME_FUNCTION(ToDayImpl, day); @@ -377,8 +378,8 @@ struct Transformer { } }; -template -struct Transformer> { +template typename Impl> +struct TransformerYear { static void vector(const PaddedPODArray& vec_from, PaddedPODArray& vec_to, NullMap& null_map) { size_t size = vec_from.size(); @@ -390,7 +391,7 @@ struct Transformer> { auto* __restrict null_map_ptr = null_map.data(); for (size_t i = 0; i < size; ++i) { - to_ptr[i] = ToYearImpl::execute(from_ptr[i]); + to_ptr[i] = Impl::execute(from_ptr[i]); } for (size_t i = 0; i < size; ++i) { @@ -406,11 +407,33 @@ struct Transformer> { auto* __restrict from_ptr = vec_from.data(); for (size_t i = 0; i < size; ++i) { - to_ptr[i] = ToYearImpl::execute(from_ptr[i]); + to_ptr[i] = Impl::execute(from_ptr[i]); } } }; +template +struct Transformer> { + static void vector(const PaddedPODArray& vec_from, PaddedPODArray& vec_to, + NullMap& null_map) { + TransformerYear::vector(vec_from, vec_to, null_map); + } + static void vector(const PaddedPODArray& vec_from, PaddedPODArray& vec_to) { + TransformerYear::vector(vec_from, vec_to); + } +}; + +template +struct Transformer> { + static void vector(const PaddedPODArray& vec_from, PaddedPODArray& vec_to, + NullMap& null_map) { + TransformerYear::vector(vec_from, vec_to, null_map); + } + static void vector(const PaddedPODArray& vec_from, PaddedPODArray& vec_to) { + TransformerYear::vector(vec_from, vec_to); + } +}; + template struct DateTimeTransformImpl { static Status execute(Block& block, const ColumnNumbers& arguments, uint32_t result, diff --git a/be/src/vec/functions/to_time_function.cpp b/be/src/vec/functions/to_time_function.cpp index 126296fc40abe0..f160ad531a369d 100644 --- a/be/src/vec/functions/to_time_function.cpp +++ b/be/src/vec/functions/to_time_function.cpp @@ -16,15 +16,8 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include -#include - #include "vec/core/types.h" -#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_date_time.h" -#include "vec/data_types/data_type_nullable.h" #include "vec/data_types/data_type_number.h" #include "vec/data_types/data_type_time_v2.h" #include "vec/functions/date_time_transforms.h" @@ -35,6 +28,8 @@ namespace doris::vectorized { using FunctionYear = FunctionDateOrDateTimeToSomething>; using FunctionYearV2 = FunctionDateOrDateTimeToSomething>; +using FunctionYearOfWeek = + FunctionDateOrDateTimeToSomething>; using FunctionQuarter = FunctionDateOrDateTimeToSomething>; using FunctionQuarterV2 = FunctionDateOrDateTimeToSomething>; using FunctionMonth = FunctionDateOrDateTimeToSomething>; @@ -102,6 +97,7 @@ void register_function_to_time_function(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_function(); factory.register_function(); diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp index 1f2c14360e2273..b4df7813f1db0c 100644 --- a/be/src/vec/runtime/vdatetime_value.cpp +++ b/be/src/vec/runtime/vdatetime_value.cpp @@ -3735,6 +3735,32 @@ bool DateV2Value::from_date_int64(int64_t value) { } } +// An ISO week-numbering year (also called ISO year informally) has 52 or 53 full weeks. That is 364 or 371 days instead of the usual 365 or 366 days. These 53-week years occur on all years that have Thursday as 1 January and on leap years that start on Wednesday. The extra week is sometimes referred to as a leap week, although ISO 8601 does not use this term. https://en.wikipedia.org/wiki/ISO_week_date +template +uint16_t DateV2Value::year_of_week() const { + constexpr uint8_t THURSDAY = 3; + + if (date_v2_value_.month_ == 1) { + constexpr uint8_t MAX_DISTANCE_WITH_THURSDAY = 6 - THURSDAY; + if (date_v2_value_.day_ <= MAX_DISTANCE_WITH_THURSDAY) { + auto weekday = calc_weekday(daynr(), false); + // if the current day is after Thursday and Thursday is in the previous year, return the previous year + return date_v2_value_.year_ - + (weekday > THURSDAY && weekday - THURSDAY > date_v2_value_.day_ - 1); + } + } else if (date_v2_value_.month_ == 12) { + constexpr uint8_t MAX_DISTANCE_WITH_THURSDAY = THURSDAY - 0; + if (S_DAYS_IN_MONTH[12] - date_v2_value_.day_ <= MAX_DISTANCE_WITH_THURSDAY) { + auto weekday = calc_weekday(daynr(), false); + // if the current day is before Thursday and Thursday is in the next year, return the next year + return date_v2_value_.year_ + + (weekday < THURSDAY && + (THURSDAY - weekday) > S_DAYS_IN_MONTH[12] - date_v2_value_.day_); + } + } + return date_v2_value_.year_; +} + template uint8_t DateV2Value::calc_week(const uint32_t& day_nr, const uint16_t& year, const uint8_t& month, const uint8_t& day, uint8_t mode, diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index eed30466db9270..680d53d2e6960a 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -965,6 +965,7 @@ class DateV2Value { } uint16_t year() const { return date_v2_value_.year_; } + uint16_t year_of_week() const; uint8_t month() const { return date_v2_value_.month_; } int quarter() const { return (date_v2_value_.month_ - 1) / 3 + 1; } int week() const { return week(mysql_week_mode(0)); } //00-53 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index e482b419ac5613..ae638bded38315 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -479,6 +479,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Year; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearCeil; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearFloor; +import org.apache.doris.nereids.trees.expressions.functions.scalar.YearOfWeek; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearWeek; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsDiff; @@ -974,6 +975,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Year.class, "year"), scalar(YearCeil.class, "year_ceil"), scalar(YearFloor.class, "year_floor"), + scalar(YearOfWeek.class, "year_of_week", "yow"), scalar(YearWeek.class, "yearweek"), scalar(YearsAdd.class, "years_add"), scalar(YearsDiff.class, "years_diff"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java index 6056b41f99e02a..460d2b39c404d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java @@ -87,6 +87,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.WeekOfYear; import org.apache.doris.nereids.trees.expressions.functions.scalar.WeeksDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.Year; +import org.apache.doris.nereids.trees.expressions.functions.scalar.YearOfWeek; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsDiff; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsSub; @@ -405,6 +406,21 @@ public ColumnStatistic visitYear(Year year, Statistics context) { .setMaxValue(maxYear).setMinExpr(null).build(); } + @Override + public ColumnStatistic visitYearOfWeek(YearOfWeek yearOfWeek, Statistics context) { + ColumnStatistic childStat = yearOfWeek.child().accept(this, context); + double rowCount = context.getRowCount(); + long minYear = 1970; + long maxYear = 2038; + return new ColumnStatisticBuilder() + .setNdv(maxYear - minYear + 1) + .setAvgSizeByte(4) + .setNumNulls(childStat.numNulls) + .setDataSize(4 * rowCount) + .setMinValue(minYear) + .setMaxValue(maxYear).setMinExpr(null).build(); + } + @Override public ColumnStatistic visitWeekOfYear(WeekOfYear weekOfYear, Statistics context) { ColumnStatistic childStat = weekOfYear.child().accept(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/YearOfWeek.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/YearOfWeek.java new file mode 100644 index 00000000000000..43db8fe8960a1a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/YearOfWeek.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Monotonic; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullableOnDateLikeV2Args; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DateTimeType; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.nereids.types.DateV2Type; +import org.apache.doris.nereids.types.SmallIntType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'year_of_week'. This class is generated by GenerateFunction. + */ +public class YearOfWeek extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args, Monotonic { + + private static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(SmallIntType.INSTANCE).args(DateV2Type.INSTANCE)); + + /** + * constructor with 1 argument. + */ + public YearOfWeek(Expression arg) { + super("year_of_week", arg); + } + + /** + * withChildren. + */ + @Override + public YearOfWeek withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new YearOfWeek(children.get(0)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitYearOfWeek(this, context); + } + + @Override + public boolean isPositive() { + return true; + } + + @Override + public int getMonotonicFunctionChildIndex() { + return 0; + } + + @Override + public Expression withConstantArgs(Expression literal) { + return new YearOfWeek(literal); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index 4346fb71591768..9c19443419953a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -476,6 +476,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Year; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearCeil; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearFloor; +import org.apache.doris.nereids.trees.expressions.functions.scalar.YearOfWeek; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearWeek; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.YearsDiff; @@ -2261,6 +2262,10 @@ default R visitYear(Year year, C context) { return visitScalarFunction(year, context); } + default R visitYearOfWeek(YearOfWeek yearOfWeek, C context) { + return visitScalarFunction(yearOfWeek, context); + } + default R visitYearCeil(YearCeil yearCeil, C context) { return visitScalarFunction(yearCeil, context); } diff --git a/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out b/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out index b25a92795c5902..d0f386acf8eac8 100644 --- a/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out +++ b/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out @@ -415,6 +415,68 @@ February 2019-08-01T13:21:03 2019 9999-08-01T13:21:03 9999 +-- !sql -- +1987 + +-- !sql -- +2049 + +-- !sql -- +0000-08-01T13:21:03 0 +2019-08-01T13:21:03 2019 +9999-08-01T13:21:03 9999 + +-- !sql -- +1987 + +-- !sql -- +2004 + +-- !sql -- +2004 + +-- !sql -- +2005 + +-- !sql -- +2007 + +-- !sql -- +2007 + +-- !sql -- +2008 + +-- !sql -- +2008 + +-- !sql -- +2008 + +-- !sql -- +2009 + +-- !sql -- +2009 + +-- !sql -- +2009 + +-- !sql -- +2009 + +-- !sql -- +2009 + +-- !sql -- +2009 + +-- !sql -- +2009 + +-- !sql -- +2009 + -- !sql -- 202052 diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy index bf9d50427e136d..3324d6b10816f6 100644 --- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy @@ -475,6 +475,30 @@ suite("test_date_function") { qt_sql """ select year('2050-01-01') """ qt_sql """ select test_datetime, year(test_datetime) from ${tableName} order by test_datetime """ + // YEAROFWEEK + qt_sql """ select year_of_week('1987-01-01') """ + qt_sql """ select year_of_week('2050-01-01') """ + qt_sql """ select test_datetime, year_of_week(test_datetime) from ${tableName} order by test_datetime """ + + qt_sql """ select yow('1987-01-01') """ + + qt_sql "select year_of_week('2005-01-01')" // 2004-W53-6 + qt_sql "select year_of_week('2005-01-02')" // 2004-W53-7 + qt_sql "select year_of_week('2005-12-31')" // 2005-W52-6 + qt_sql "select year_of_week('2007-01-01')" // 2007-W01-1 + qt_sql "select year_of_week('2007-12-30')" // 2007-W52-7 + qt_sql "select year_of_week('2007-12-31')" // 2008-W01-1 + qt_sql "select year_of_week('2008-01-01')" // 2008-W01-2 + qt_sql "select year_of_week('2008-12-28')" // 2008-W52-7 + qt_sql "select year_of_week('2008-12-29')" // 2009-W01-1 + qt_sql "select year_of_week('2008-12-30')" // 2009-W01-2 + qt_sql "select year_of_week('2008-12-31')" // 2009-W01-3 + qt_sql "select year_of_week('2009-01-01')" // 2009-W01-4 + qt_sql "select year_of_week('2009-12-31')" // 2009-W53-4 + qt_sql "select year_of_week('2010-01-01')" // 2009-W53-5 + qt_sql "select year_of_week('2010-01-02')" // 2009-W53-6 + qt_sql "select year_of_week('2010-01-03')" // 2009-W53-7 + // YEARWEEK qt_sql """ select yearweek('2021-1-1') """ qt_sql """ select yearweek('2020-7-1') """ From 444c8a67ec4bd372351037d6b446cc59eeb0e8f7 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Mon, 10 Mar 2025 18:11:40 +0800 Subject: [PATCH 2/4] fix --- .../nereids/trees/expressions/functions/scalar/YearOfWeek.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/YearOfWeek.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/YearOfWeek.java index 43db8fe8960a1a..ff89d93227de81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/YearOfWeek.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/YearOfWeek.java @@ -24,8 +24,6 @@ import org.apache.doris.nereids.trees.expressions.functions.PropagateNullableOnDateLikeV2Args; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.DateTimeType; -import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.SmallIntType; From a04c0af29cf2f5745eaa2ac1146fc4e56b4ef02c Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Tue, 11 Mar 2025 11:44:06 +0800 Subject: [PATCH 3/4] update --- be/test/vec/function/function_time_test.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/be/test/vec/function/function_time_test.cpp b/be/test/vec/function/function_time_test.cpp index ddfc722c7ab452..88d008117085ec 100644 --- a/be/test/vec/function/function_time_test.cpp +++ b/be/test/vec/function/function_time_test.cpp @@ -1819,4 +1819,16 @@ TEST(VTimestampFunctionsTest, seconds_sub_v2_test) { } } +TEST(VTimestampFunctionsTest, year_of_week_test) { + std::string func_name = "year_of_week"; + { + InputTypeSet input_types = {TypeIndex::DateV2}; + DataSet data_set = {{{std::string("2005-01-01")}, int16_t(2004)}, + {{std::string("2008-12-30")}, int16_t(2009)}, + {{std::string("12008-12-30")}, Null()}, + {{Null()}, Null()}}; + static_cast(check_function(func_name, input_types, data_set)); + } +} + } // namespace doris::vectorized From 52f739d9ed467fc6254ea2946a552baf95851cbe Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Wed, 12 Mar 2025 11:30:16 +0800 Subject: [PATCH 4/4] update --- be/src/vec/functions/date_time_transforms.h | 22 ++++----------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/be/src/vec/functions/date_time_transforms.h b/be/src/vec/functions/date_time_transforms.h index b3e51a36907309..94d473a6ba5a71 100644 --- a/be/src/vec/functions/date_time_transforms.h +++ b/be/src/vec/functions/date_time_transforms.h @@ -413,26 +413,12 @@ struct TransformerYear { }; template -struct Transformer> { - static void vector(const PaddedPODArray& vec_from, PaddedPODArray& vec_to, - NullMap& null_map) { - TransformerYear::vector(vec_from, vec_to, null_map); - } - static void vector(const PaddedPODArray& vec_from, PaddedPODArray& vec_to) { - TransformerYear::vector(vec_from, vec_to); - } -}; +struct Transformer> + : public TransformerYear {}; template -struct Transformer> { - static void vector(const PaddedPODArray& vec_from, PaddedPODArray& vec_to, - NullMap& null_map) { - TransformerYear::vector(vec_from, vec_to, null_map); - } - static void vector(const PaddedPODArray& vec_from, PaddedPODArray& vec_to) { - TransformerYear::vector(vec_from, vec_to); - } -}; +struct Transformer> + : public TransformerYear {}; template struct DateTimeTransformImpl {