Skip to content

[fix](function) Fix from_second functions overflow and wrong result #28685

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Gabriel39 merged 2 commits into from
Dec 22, 2023
Merged

[fix](function) Fix from_second functions overflow and wrong result #28685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9d7c498
1
zclllyybb Dec 19, 2023
9354891
2
zclllyybb Dec 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions be/src/exec/es/es_scroll_parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,8 @@ Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool
if (ok) {
// The local time zone can change by session variable `time_zone`
// We should use the user specified time zone, not the actual system local time zone.
success = dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), time_zone);
success = true;
dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), time_zone);
}
} else if (str_length == 19) {
// YYYY-MM-DDTHH:MM:SS
Expand All @@ -231,8 +232,8 @@ Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool
const bool ok =
cctz::parse("%Y-%m-%dT%H:%M:%S", str_date, cctz::utc_time_zone(), &tp);
if (ok) {
success = dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp),
time_zone);
success = true;
dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), time_zone);
}
} else {
// YYYY-MM-DD HH:MM:SS
Expand All @@ -243,7 +244,8 @@ Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool
// string long like "1677895728000"
int64_t time_long = std::atol(str_date.c_str());
if (time_long > 0) {
success = dt_val.from_unixtime(time_long / 1000, time_zone);
success = true;
dt_val.from_unixtime(time_long / 1000, time_zone);
}
} else {
// YYYY-MM-DD or others
Expand All @@ -255,9 +257,7 @@ Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool
}

} else {
if (!dt_val.from_unixtime(col.GetInt64() / 1000, time_zone)) {
RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type);
}
dt_val.from_unixtime(col.GetInt64() / 1000, time_zone);
}
if constexpr (is_datetime_v1) {
if (type == TYPE_DATE) {
Expand Down
1 change: 1 addition & 0 deletions be/src/runtime/runtime_state.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ class RuntimeState {
TQueryType::type query_type() const { return _query_options.query_type; }
int64_t timestamp_ms() const { return _timestamp_ms; }
int32_t nano_seconds() const { return _nano_seconds; }
// if possible, use timezone_obj() rather than timezone()
const std::string& timezone() const { return _timezone; }
const cctz::time_zone& timezone_obj() const { return _timezone_obj; }
const std::string& user() const { return _user; }
Expand Down
4 changes: 2 additions & 2 deletions be/src/vec/functions/date_time_transforms.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,10 +222,10 @@ struct FromUnixTimeImpl {
static inline auto execute(FromType val, StringRef format, ColumnString::Chars& res_data,
size_t& offset, const cctz::time_zone& time_zone) {
DateType dt;
if (format.size > 128 || val < 0 || val > TIMESTAMP_VALID_MAX ||
!dt.from_unixtime(val, time_zone)) {
if (format.size > 128 || val < 0 || val > TIMESTAMP_VALID_MAX) {
return std::pair {offset, true};
}
dt.from_unixtime(val, time_zone);

char buf[128];
if (!dt.to_format_string(format.data, format.size, buf)) {
Expand Down
13 changes: 2 additions & 11 deletions be/src/vec/functions/function_convert_tz.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,12 +201,7 @@ class FunctionConvertTZ : public IFunction {
result_column->insert_default();
return;
}

if (!ts_value2.from_unixtime(timestamp, to_tz)) {
result_null_map[index_now] = true;
result_column->insert_default();
return;
}
ts_value2.from_unixtime(timestamp, to_tz);
} else {
int64_t timestamp;
if (!ts_value.unix_timestamp(&timestamp, from_tz)) {
Expand All @@ -215,11 +210,7 @@ class FunctionConvertTZ : public IFunction {
return;
}

if (!ts_value2.from_unixtime(timestamp, to_tz)) {
result_null_map[index_now] = true;
result_column->insert_default();
return;
}
ts_value2.from_unixtime(timestamp, to_tz);
}

result_column->insert(binary_cast<ReturnDateValueType, ReturnNativeType>(ts_value2));
Expand Down
144 changes: 64 additions & 80 deletions be/src/vec/functions/function_date_or_datetime_computation.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -918,34 +918,30 @@ struct CurrentDateTimeImpl {
DateValueType dtv;
bool use_const;
if constexpr (WithPrecision) {
if (const ColumnConst* const_column = check_and_get_column<ColumnConst>(
if (const auto* const_column = check_and_get_column<ColumnConst>(
block.get_by_position(arguments[0]).column)) {
int scale = const_column->get_int(0);
if (dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->nano_seconds(),
context->state()->timezone_obj(), scale)) {
if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
reinterpret_cast<DateValueType*>(&dtv)->set_type(TIME_DATETIME);
}
auto date_packed_int = binary_cast<DateValueType, NativeType>(dtv);
col_to->insert_data(
const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)), 0);

} else {
auto invalid_val = 0;
col_to->insert_data(
const_cast<const char*>(reinterpret_cast<char*>(&invalid_val)), 0);
dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->nano_seconds(),
context->state()->timezone_obj(), scale);
if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
reinterpret_cast<DateValueType*>(&dtv)->set_type(TIME_DATETIME);
}
auto date_packed_int = binary_cast<DateValueType, NativeType>(dtv);
col_to->insert_data(
const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)), 0);

use_const = true;
} else if (const ColumnNullable* nullable_column = check_and_get_column<ColumnNullable>(
} else if (const auto* nullable_column = check_and_get_column<ColumnNullable>(
block.get_by_position(arguments[0]).column)) {
const auto& null_map = nullable_column->get_null_map_data();
const auto& nested_column = nullable_column->get_nested_column_ptr();
for (int i = 0; i < input_rows_count; i++) {
if (!null_map[i] && dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->nano_seconds(),
context->state()->timezone_obj(),
nested_column->get64(i))) {
if (!null_map[i]) {
dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->nano_seconds(),
context->state()->timezone_obj(),
nested_column->get64(i));
if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
reinterpret_cast<DateValueType*>(&dtv)->set_type(TIME_DATETIME);
}
Expand All @@ -963,38 +959,27 @@ struct CurrentDateTimeImpl {
} else {
auto& int_column = block.get_by_position(arguments[0]).column;
for (int i = 0; i < input_rows_count; i++) {
if (dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->nano_seconds(),
context->state()->timezone_obj(), int_column->get64(i))) {
if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
reinterpret_cast<DateValueType*>(&dtv)->set_type(TIME_DATETIME);
}
auto date_packed_int = binary_cast<DateValueType, NativeType>(dtv);
col_to->insert_data(
const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)),
0);
} else {
auto invalid_val = 0;
col_to->insert_data(
const_cast<const char*>(reinterpret_cast<char*>(&invalid_val)), 0);
dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->nano_seconds(),
context->state()->timezone_obj(), int_column->get64(i));
if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
reinterpret_cast<DateValueType*>(&dtv)->set_type(TIME_DATETIME);
}
auto date_packed_int = binary_cast<DateValueType, NativeType>(dtv);
col_to->insert_data(
const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)), 0);
}
use_const = false;
}
} else {
if (dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->timezone_obj())) {
if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
reinterpret_cast<DateValueType*>(&dtv)->set_type(TIME_DATETIME);
}
auto date_packed_int = binary_cast<DateValueType, NativeType>(dtv);
col_to->insert_data(
const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)), 0);
} else {
auto invalid_val = 0;
col_to->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&invalid_val)),
0);
dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->timezone_obj());
if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
reinterpret_cast<DateValueType*>(&dtv)->set_type(TIME_DATETIME);
}
auto date_packed_int = binary_cast<DateValueType, NativeType>(dtv);
col_to->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)),
0);
use_const = true;
}

Expand All @@ -1017,31 +1002,21 @@ struct CurrentDateImpl {
auto col_to = ColumnVector<NativeType>::create();
if constexpr (std::is_same_v<DateType, DataTypeDateV2>) {
DateV2Value<DateV2ValueType> dtv;
if (dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->timezone_obj())) {
auto date_packed_int = binary_cast<DateV2Value<DateV2ValueType>, uint32_t>(
*reinterpret_cast<DateV2Value<DateV2ValueType>*>(&dtv));
col_to->insert_data(
const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)), 0);
} else {
auto invalid_val = 0;
col_to->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&invalid_val)),
0);
}
dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->timezone_obj());
auto date_packed_int = binary_cast<DateV2Value<DateV2ValueType>, uint32_t>(
*reinterpret_cast<DateV2Value<DateV2ValueType>*>(&dtv));
col_to->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)),
0);
} else {
VecDateTimeValue dtv;
if (dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->timezone_obj())) {
reinterpret_cast<VecDateTimeValue*>(&dtv)->set_type(TIME_DATE);
auto date_packed_int = binary_cast<doris::VecDateTimeValue, int64_t>(
*reinterpret_cast<VecDateTimeValue*>(&dtv));
col_to->insert_data(
const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)), 0);
} else {
auto invalid_val = 0;
col_to->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&invalid_val)),
0);
}
dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->timezone_obj());
reinterpret_cast<VecDateTimeValue*>(&dtv)->set_type(TIME_DATE);
auto date_packed_int = binary_cast<doris::VecDateTimeValue, int64_t>(
*reinterpret_cast<VecDateTimeValue*>(&dtv));
col_to->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&date_packed_int)),
0);
}
block.get_by_position(result).column =
ColumnConst::create(std::move(col_to), input_rows_count);
Expand All @@ -1057,15 +1032,11 @@ struct CurrentTimeImpl {
size_t result, size_t input_rows_count) {
auto col_to = ColumnVector<Float64>::create();
VecDateTimeValue dtv;
if (dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->timezone_obj())) {
double time = dtv.hour() * 3600l + dtv.minute() * 60l + dtv.second();
time *= (1000 * 1000);
col_to->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&time)), 0);
} else {
auto invalid_val = 0;
col_to->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&invalid_val)), 0);
}
dtv.from_unixtime(context->state()->timestamp_ms() / 1000,
context->state()->timezone_obj());
double time = dtv.hour() * 3600l + dtv.minute() * 60l + dtv.second();
time *= (1000 * 1000);
col_to->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&time)), 0);
block.get_by_position(result).column =
ColumnConst::create(std::move(col_to), input_rows_count);
return Status::OK();
Expand Down Expand Up @@ -1144,15 +1115,28 @@ struct TimestampToDateTime : IFunction {
auto null_vector = ColumnVector<UInt8>::create();
res_col->get_data().resize_fill(input_rows_count, 0);
null_vector->get_data().resize_fill(input_rows_count, false);

NullMap& null_map = null_vector->get_data();
auto& res_data = res_col->get_data();
const cctz::time_zone& time_zone = context->state()->timezone_obj();

for (int i = 0; i < input_rows_count; ++i) {
Int64 value = column_data.get_element(i);
if (value < 0) {
null_map[i] = true;
continue;
}

auto& dt = reinterpret_cast<DateV2Value<DateTimeV2ValueType>&>(res_data[i]);
null_map[i] = !dt.from_unixtime(value / Impl::ratio, time_zone);
dt.set_microsecond((value % Impl::ratio) * ratio_to_micro);
dt.from_unixtime(value / Impl::ratio, time_zone);

if (dt.is_valid_date()) [[likely]] {
dt.set_microsecond((value % Impl::ratio) * ratio_to_micro);
} else {
null_map[i] = true;
}
}

block.get_by_position(result).column =
ColumnNullable::create(std::move(res_col), std::move(null_vector));
return Status::OK();
Expand Down
30 changes: 16 additions & 14 deletions be/src/vec/runtime/vdatetime_value.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1731,17 +1731,20 @@ bool VecDateTimeValue::from_unixtime(int64_t timestamp, const std::string& timez
if (!TimezoneUtils::find_cctz_time_zone(timezone, ctz)) {
return false;
}
return from_unixtime(timestamp, ctz);
from_unixtime(timestamp, ctz);
return true;
}

bool VecDateTimeValue::from_unixtime(int64_t timestamp, const cctz::time_zone& ctz) {
void VecDateTimeValue::from_unixtime(int64_t timestamp, const cctz::time_zone& ctz) {
static const cctz::time_point<cctz::sys_seconds> epoch =
std::chrono::time_point_cast<cctz::sys_seconds>(
std::chrono::system_clock::from_time_t(0));
cctz::time_point<cctz::sys_seconds> t = epoch + cctz::seconds(timestamp);

const auto tp = cctz::convert(t, ctz);

// there's no overflow check since it's hot path

_neg = 0;
_type = TIME_DATETIME;
_year = tp.year();
Expand All @@ -1750,8 +1753,6 @@ bool VecDateTimeValue::from_unixtime(int64_t timestamp, const cctz::time_zone& c
_hour = tp.hour();
_minute = tp.minute();
_second = tp.second();

return true;
}

const char* VecDateTimeValue::month_name() const {
Expand Down Expand Up @@ -3202,20 +3203,21 @@ bool DateV2Value<T>::from_unixtime(int64_t timestamp, const std::string& timezon
if (!TimezoneUtils::find_cctz_time_zone(timezone, ctz)) {
return false;
}
return from_unixtime(timestamp, ctz);
from_unixtime(timestamp, ctz);
return true;
}

template <typename T>
bool DateV2Value<T>::from_unixtime(int64_t timestamp, const cctz::time_zone& ctz) {
void DateV2Value<T>::from_unixtime(int64_t timestamp, const cctz::time_zone& ctz) {
static const cctz::time_point<cctz::sys_seconds> epoch =
std::chrono::time_point_cast<cctz::sys_seconds>(
std::chrono::system_clock::from_time_t(0));
cctz::time_point<cctz::sys_seconds> t = epoch + cctz::seconds(timestamp);

const auto tp = cctz::convert(t, ctz);

// there's no overflow check since it's hot path

set_time(tp.year(), tp.month(), tp.day(), tp.hour(), tp.minute(), tp.second(), 0);
return true;
}

template <typename T>
Expand All @@ -3225,11 +3227,12 @@ bool DateV2Value<T>::from_unixtime(std::pair<int64_t, int64_t> timestamp,
if (!TimezoneUtils::find_cctz_time_zone(timezone, ctz)) {
return false;
}
return from_unixtime(timestamp, ctz);
from_unixtime(timestamp, ctz);
return true;
}

template <typename T>
bool DateV2Value<T>::from_unixtime(std::pair<int64_t, int64_t> timestamp,
void DateV2Value<T>::from_unixtime(std::pair<int64_t, int64_t> timestamp,
const cctz::time_zone& ctz) {
static const cctz::time_point<cctz::sys_seconds> epoch =
std::chrono::time_point_cast<cctz::sys_seconds>(
Expand All @@ -3240,7 +3243,6 @@ bool DateV2Value<T>::from_unixtime(std::pair<int64_t, int64_t> timestamp,

set_time(tp.year(), tp.month(), tp.day(), tp.hour(), tp.minute(), tp.second(),
timestamp.second);
return true;
}

template <typename T>
Expand All @@ -3250,11 +3252,12 @@ bool DateV2Value<T>::from_unixtime(int64_t timestamp, int32_t nano_seconds,
if (!TimezoneUtils::find_cctz_time_zone(timezone, ctz)) {
return false;
}
return from_unixtime(timestamp, nano_seconds, ctz, scale);
from_unixtime(timestamp, nano_seconds, ctz, scale);
return true;
}

template <typename T>
bool DateV2Value<T>::from_unixtime(int64_t timestamp, int32_t nano_seconds,
void DateV2Value<T>::from_unixtime(int64_t timestamp, int32_t nano_seconds,
const cctz::time_zone& ctz, int scale) {
static const cctz::time_point<cctz::sys_seconds> epoch =
std::chrono::time_point_cast<cctz::sys_seconds>(
Expand All @@ -3269,7 +3272,6 @@ bool DateV2Value<T>::from_unixtime(int64_t timestamp, int32_t nano_seconds,

set_time(tp.year(), tp.month(), tp.day(), tp.hour(), tp.minute(), tp.second(),
nano_seconds / int_exp10(9 - scale) * int_exp10(6 - scale));
return true;
}

template <typename T>
Expand Down
Loading