Skip to content

[fix](datetime) fix datetime rounding on BE #32075

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 5 commits into from
Mar 17, 2024
Merged

[fix](datetime) fix datetime rounding on BE #32075

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
588d2f5
FINISH
zhiqiang-hhhh Mar 12, 2024
7393369
MORE TEST
zhiqiang-hhhh Mar 12, 2024
efd7ab6
Merge branch 'master' into fix-datetime-round-be
zhiqiang-hhhh Mar 12, 2024
f1c933e
FORMAT
zhiqiang-hhhh Mar 12, 2024
f622759
Merge branch 'fix-datetime-round-be' of github.com:zhiqiang-hhhh/dori…
zhiqiang-hhhh Mar 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion be/src/exec/tablet_info.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,8 +465,10 @@ Status VOlapTablePartitionParam::_create_partition_key(const TExprNode& t_expr,
column->insert_data(reinterpret_cast<const char*>(&dt), 0);
} else if (TypeDescriptor::from_thrift(t_expr.type).is_datetime_v2_type()) {
DateV2Value<DateTimeV2ValueType> dt;
const int32_t scale =
t_expr.type.types.empty() ? -1 : t_expr.type.types.front().scalar_type.scale;
if (!dt.from_date_str(t_expr.date_literal.value.c_str(),
t_expr.date_literal.value.size())) {
t_expr.date_literal.value.size(), scale)) {
std::stringstream ss;
ss << "invalid date literal in partition column, date=" << t_expr.date_literal;
return Status::InternalError(ss.str());
Expand Down
4 changes: 2 additions & 2 deletions be/src/vec/data_types/data_type_time_v2.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,9 @@ void DataTypeDateTimeV2::to_string(const IColumn& column, size_t row_num,
}

Status DataTypeDateTimeV2::from_string(ReadBuffer& rb, IColumn* column) const {
auto* column_data = static_cast<ColumnUInt64*>(column);
auto* column_data = assert_cast<ColumnUInt64*>(column);
UInt64 val = 0;
if (!read_datetime_v2_text_impl<UInt64>(val, rb)) {
if (!read_datetime_v2_text_impl<UInt64>(val, rb, _scale)) {
return Status::InvalidArgument("parse date fail, string: '{}'",
std::string(rb.position(), rb.count()).c_str());
}
Expand Down
5 changes: 4 additions & 1 deletion be/src/vec/data_types/data_type_time_v2.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,10 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase<UInt64> {

Field get_field(const TExprNode& node) const override {
DateV2Value<DateTimeV2ValueType> value;
if (value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size())) {
const int32_t scale =
node.type.types.empty() ? -1 : node.type.types.front().scalar_type.scale;
if (value.from_date_str(node.date_literal.value.c_str(), node.date_literal.value.size(),
scale)) {
return value.to_date_int_val();
} else {
throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
Expand Down
84 changes: 59 additions & 25 deletions be/src/vec/runtime/vdatetime_value.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1994,7 +1994,6 @@ bool DateV2Value<T>::from_date_str_base(const char* date_str, int len, int scale
const static int allow_space_mask = 4 | 64;
uint32_t date_val[MAX_DATE_PARTS] = {0};
int32_t date_len[MAX_DATE_PARTS] = {0};
bool carry_bits[MAX_DATE_PARTS] = {false};

// Skip space character
while (ptr < end && isspace(*ptr)) {
Expand Down Expand Up @@ -2028,46 +2027,72 @@ bool DateV2Value<T>::from_date_str_base(const char* date_str, int len, int scale
int field_idx = 0;
int field_len = year_len;
long sec_offset = 0;

while (ptr < end && isdigit(*ptr) && field_idx < MAX_DATE_PARTS) {
const char* start = ptr;
int temp_val = 0;
bool scan_to_delim = (!is_interval_format) && (field_idx != 6);
while (ptr < end && isdigit(*ptr) && (scan_to_delim || field_len--)) { // field_len <= 6
temp_val = temp_val * 10 + (*ptr++ - '0');
while (ptr < end && isdigit(*ptr) && (scan_to_delim || field_len--)) { // field_len <= 7
temp_val = temp_val * 10 + (*ptr - '0');
ptr++;
}

if (field_idx == 6) {
// Microsecond
const auto ms_part = ptr - start;
temp_val *= int_exp10(std::max(0L, 6 - ms_part));
if constexpr (is_datetime) {
// round of microseconds
// 1. normalize to 7 digits for rounding
// 2. rounding
// 3. nomalize to 6 digits for storage
if (scale >= 0) {
if (scale == 6 && ms_part >= 6) {
if (ptr <= end && isdigit(*ptr) && *ptr >= '5') {
temp_val += 1;
}
// do normalization
const auto ms_digit_count = ptr - start;
const auto normalizer = int_exp10(std::abs(7 - ms_digit_count));
temp_val *= normalizer;

// check round
const auto rounder = int_exp10(std::abs(7 - scale));
const auto reminder = temp_val % rounder;
temp_val -= reminder;

if (reminder >= 5 * normalizer) {
temp_val += rounder;
}

// truncate to 6 digits
if (temp_val == int_exp10(7)) {
temp_val = 0;
sec_offset += 1;
} else {
const int divisor = int_exp10(6 - scale);
int remainder = temp_val % divisor;
temp_val /= divisor;
if (scale < 6 && std::abs(remainder) >= (divisor >> 1)) {
temp_val += 1;
}
temp_val *= divisor;
if (temp_val == 1000000L) {
temp_val = 0;
date_val[field_idx - 1] += 1;
carry_bits[field_idx] = true;
}
temp_val /= 10;
}
}

// move ptr to start of timezone or end
while (ptr < end && isdigit(*ptr)) {
ptr++;
}
} else {
// Microsecond
const auto ms_part = ptr - start;
temp_val *= int_exp10(std::max(0L, 6 - ms_part));
}
}

// Impossible
if (temp_val > 999999L) {
return false;
}

date_val[field_idx] = temp_val;
date_len[field_idx] = ptr - start;

if (field_idx == 6) {
// select cast("2020-01-01 12:00:00.12345" as Datetime(4))
// ptr - start will be 5, but scale is 4
date_len[field_idx] = std::min(static_cast<int>(ptr - start), scale);
} else {
date_len[field_idx] = ptr - start;
}

field_len = 2;

if (ptr == end) {
Expand Down Expand Up @@ -2114,7 +2139,14 @@ bool DateV2Value<T>::from_date_str_base(const char* date_str, int len, int scale
if (field_idx == 5) {
if (*ptr == '.') {
ptr++;
field_len = 6;
// for datetime, we need to discard the fraction part
// that beyond the scale + 1, and scale + 1 digit will
// be used to round the fraction part
if constexpr (is_datetime) {
field_len = std::min(7, scale + 1);
} else {
field_len = 6;
}
} else if (isdigit(*ptr)) {
field_idx++;
break;
Expand All @@ -2136,6 +2168,7 @@ bool DateV2Value<T>::from_date_str_base(const char* date_str, int len, int scale
}
field_idx++;
}

int num_field = field_idx;
if (!is_interval_format) {
year_len = date_len[0];
Expand All @@ -2158,11 +2191,12 @@ bool DateV2Value<T>::from_date_str_base(const char* date_str, int len, int scale
if (is_invalid(date_val[0], date_val[1], date_val[2], 0, 0, 0, 0)) {
return false;
}
format_datetime(date_val, carry_bits);

if (!check_range_and_set_time(date_val[0], date_val[1], date_val[2], date_val[3], date_val[4],
date_val[5], date_val[6])) {
return false;
}

return sec_offset ? date_add_interval<TimeUnit::SECOND>(
TimeInterval {TimeUnit::SECOND, sec_offset, false})
: true;
Expand Down
Loading