From 79401d5c8b951659cbacec1c7d8e7eb09052d51b Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Tue, 6 Aug 2019 14:44:04 +0800 Subject: [PATCH 1/6] Support time zone function in be --- be/src/common/daemon.cpp | 2 + be/src/exprs/timestamp_functions.cpp | 298 +++++------------- be/src/exprs/timestamp_functions.h | 69 ++-- be/src/exprs/timezone_db.cpp | 2 +- be/src/runtime/CMakeLists.txt | 1 + be/src/runtime/plan_fragment_executor.cpp | 2 +- be/src/runtime/runtime_state.cpp | 29 +- be/src/runtime/runtime_state.h | 22 +- be/src/runtime/test_env.cc | 2 +- be/src/runtime/timestamp_value.cpp | 86 +++++ be/src/runtime/timestamp_value.h | 178 +++++++++++ be/test/exec/broker_scan_node_test.cpp | 2 +- be/test/exec/broker_scanner_test.cpp | 2 +- be/test/exec/es_http_scan_node_test.cpp | 2 +- be/test/exec/es_predicate_test.cpp | 2 +- be/test/exec/es_scan_node_test.cpp | 2 +- be/test/exec/parquet_scanner_test.cpp | 2 +- be/test/exec/tablet_sink_test.cpp | 17 +- be/test/exprs/timestamp_functions_test.cpp | 33 ++ be/test/runtime/buffered_block_mgr2_test.cpp | 2 +- be/test/runtime/tablet_writer_mgr_test.cpp | 15 +- be/test/runtime/timestamp_value_test.cpp | 58 ++++ .../date-time-functions/convert_tz.md | 27 ++ .../java/org/apache/doris/qe/Coordinator.java | 14 +- gensrc/script/doris_builtins_functions.py | 5 +- gensrc/thrift/PaloInternalService.thrift | 1 + 26 files changed, 565 insertions(+), 310 deletions(-) create mode 100644 be/src/runtime/timestamp_value.cpp create mode 100644 be/src/runtime/timestamp_value.h create mode 100644 be/test/runtime/timestamp_value_test.cpp create mode 100644 docs/documentation/cn/sql-reference/sql-functions/date-time-functions/convert_tz.md diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 96a5a33f66afd7..f343277dd54560 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -33,6 +33,7 @@ #include "runtime/exec_env.h" #include "runtime/mem_tracker.h" #include "runtime/user_function_cache.h" +#include "runtime/timestamp_value.h" #include "exprs/operators.h" #include "exprs/is_null_predicate.h" #include "exprs/like_predicate.h" @@ -266,6 +267,7 @@ void init_daemon(int argc, char** argv, const std::vector& paths) { HllHashFunctions::init(); ESFunctions::init(); GeoFunctions::init(); + TimezoneDatabase::init(); pthread_t tc_malloc_pid; pthread_create(&tc_malloc_pid, NULL, tcmalloc_gc_thread, NULL); diff --git a/be/src/exprs/timestamp_functions.cpp b/be/src/exprs/timestamp_functions.cpp index 17174d631248d4..f781837cd23045 100644 --- a/be/src/exprs/timestamp_functions.cpp +++ b/be/src/exprs/timestamp_functions.cpp @@ -26,83 +26,17 @@ #include "exprs/anyval_util.h" #include "runtime/tuple_row.h" #include "runtime/datetime_value.h" +#include "runtime/timestamp_value.h" #include "runtime/runtime_state.h" #include "util/path_builder.h" #include "runtime/string_value.hpp" #include "util/debug_util.h" -#define TIMEZONE_DATABASE "be/files/date_time_zonespec.csv" - namespace doris { -boost::local_time::tz_database TimezoneDatabase::_s_tz_database; -std::vector TimezoneDatabase::_s_tz_region_list; - void TimestampFunctions::init() { } -StringVal TimestampFunctions::from_unix( - FunctionContext* context, const IntVal& unix_time) { - if (unix_time.is_null) { - return StringVal::null(); - } - DateTimeValue t; - if (!t.from_unixtime(unix_time.val)) { - return StringVal::null(); - } - // default fmt means return DATE_TIME - t.set_type(TIME_DATETIME); - char buf[64]; - t.to_string(buf); - return AnyValUtil::from_string_temp(context, buf); -} - -StringVal TimestampFunctions::from_unix( - FunctionContext* context, const IntVal& unix_time, const StringVal& fmt) { - if (unix_time.is_null || fmt.is_null) { - return StringVal::null(); - } - DateTimeValue t; - if (!t.from_unixtime(unix_time.val)) { - return StringVal::null(); - } - - if (!check_format(fmt, t)) { - return StringVal::null(); - } - - char buf[64]; - t.to_string(buf); - return AnyValUtil::from_string_temp(context, buf); -} - -IntVal TimestampFunctions::to_unix(FunctionContext* context) { - return IntVal(context->impl()->state()->now()->unix_timestamp()); -} - -IntVal TimestampFunctions::to_unix( - FunctionContext* context, const StringVal& string_val, const StringVal& fmt) { - if (string_val.is_null || fmt.is_null) { - return IntVal::null(); - } - // const DateTimeVal& tv_val = ToTimestamp(context, string_val, fmt); - DateTimeValue tv_val; - if (!tv_val.from_date_format_str( - (const char*)fmt.ptr, fmt.len, (const char*)string_val.ptr, string_val.len)) { - return IntVal::null(); - } - return IntVal(tv_val.unix_timestamp()); -} - -IntVal TimestampFunctions::to_unix( - FunctionContext* context, const DateTimeVal& ts_val) { - if (ts_val.is_null) { - return IntVal::null(); - } - const DateTimeValue& tv = DateTimeValue::from_datetime_val(ts_val); - return IntVal(tv.unix_timestamp()); -} - // TODO: accept Java data/time format strings: // http://docs.oracle.com/javase/1.4.2/docs/api/java/text/SimpleDateFormat.html // Convert them to boost format strings. @@ -228,33 +162,6 @@ IntVal TimestampFunctions::second( return IntVal(ts_value.second()); } -DateTimeVal TimestampFunctions::now(FunctionContext* context) { - const DateTimeValue* now = context->impl()->state()->now(); - DateTimeVal return_val; - now->to_datetime_val(&return_val); - return return_val; -} - -DateTimeVal TimestampFunctions::curtime(FunctionContext* context) { - DateTimeValue now = *context->impl()->state()->now(); - now.cast_to_time(); - DateTimeVal return_val; - now.to_datetime_val(&return_val); - return return_val; -} - -DateTimeVal TimestampFunctions::utc_timestamp(FunctionContext* context) { - TimeInterval interval; - // TODO(liuhy): we only support Beijing Timezone, so minus 28800 - interval.second = -28800; - DateTimeValue dtv = *(context->impl()->state()->now()); - dtv.date_add_interval(interval, SECOND); - - DateTimeVal return_val; - dtv.to_datetime_val(&return_val); - return return_val; -} - DateTimeVal TimestampFunctions::to_date( FunctionContext* ctx, const DateTimeVal& ts_val) { if (ts_val.is_null) { @@ -462,19 +369,16 @@ IntVal TimestampFunctions::to_days( return IntVal(ts_value.daynr()); } -// TODO(dhc): implement this funciton really -DateTimeVal TimestampFunctions::time_diff( +DoubleVal TimestampFunctions::time_diff( FunctionContext* ctx, const DateTimeVal& ts_val1, const DateTimeVal& ts_val2) { if (ts_val1.is_null || ts_val2.is_null) { - return DateTimeVal::null(); + return DoubleVal::null(); } const DateTimeValue& ts_value1 = DateTimeValue::from_datetime_val(ts_val1); const DateTimeValue& ts_value2 = DateTimeValue::from_datetime_val(ts_val2); - DateTimeValue ts = ts_value1 - ts_value2; - ts.cast_to_time(); - DateTimeVal result; - ts.to_datetime_val(&result); - return result; + int64_t timediff = ts_value1.unix_timestamp() - ts_value2.unix_timestamp(); + + return DoubleVal(timediff); } IntVal TimestampFunctions::date_diff( @@ -487,142 +391,104 @@ IntVal TimestampFunctions::date_diff( return IntVal(ts_value1.daynr() - ts_value2.daynr()); } +// TimeZone correlation functions. DateTimeVal TimestampFunctions::timestamp( FunctionContext* ctx, const DateTimeVal& val) { return val; } -void* TimestampFunctions::from_utc(Expr* e, TupleRow* row) { - return NULL; - // DCHECK_EQ(e->get_num_children(), 2); - // Expr* op1 = e->children()[0]; - // Expr* op2 = e->children()[1]; - // DateTimeValue* tv = reinterpret_cast(op1->get_value(row)); - // StringValue* tz = reinterpret_cast(op2->get_value(row)); - - // if (tv == NULL || tz == NULL) { - // return NULL; - // } - - // if (tv->not_a_date_time()) { - // return NULL; - // } - - // boost::local_time::time_zone_ptr timezone = TimezoneDatabase::find_timezone(tz->debug_string()); - - // This should raise some sort of error or at least null. Hive just ignores it. - // if (timezone == NULL) { - // LOG(ERROR) << "Unknown timezone '" << *tz << "'" << std::endl; - // e->_result.timestamp_val = *tv; - // return &e->_result.timestamp_val; - // } - - // boost::posix_time::ptime temp; - // tv->to_ptime(&temp); - // boost::local_time::local_date_time lt(temp, timezone); - // e->_result.timestamp_val = lt.local_time(); - // return &e->_result.timestamp_val; +StringVal TimestampFunctions::from_unix( + FunctionContext* context, const IntVal& unix_time) { + if (unix_time.is_null) { + return StringVal::null(); + } + TimestampValue timestamp(unix_time.val); + return AnyValUtil::from_string_temp(context, + timestamp.to_datetime_string(context->impl()->state()->timezone())); } -void* TimestampFunctions::to_utc(Expr* e, TupleRow* row) { - return NULL; - // DCHECK_EQ(e->get_num_children(), 2); - // Expr* op1 = e->children()[0]; - // Expr* op2 = e->children()[1]; - // DateTimeValue* tv = reinterpret_cast(op1->get_value(row)); - // StringValue* tz = reinterpret_cast(op2->get_value(row)); - - // if (tv == NULL || tz == NULL) { - // return NULL; - // } - - // if (tv->not_a_date_time()) { - // return NULL; - // } - - // boost::local_time::time_zone_ptr timezone = TimezoneDatabase::find_timezone(tz->debug_string()); - - // This should raise some sort of error or at least null. Hive just ignores it. - // if (timezone == NULL) { - // LOG(ERROR) << "Unknown timezone '" << *tz << "'" << std::endl; - // e->_result.timestamp_val = *tv; - // return &e->_result.timestamp_val; - // } +StringVal TimestampFunctions::from_unix( + FunctionContext* context, const IntVal& unix_time, const StringVal& fmt) { + if (unix_time.is_null || fmt.is_null) { + return StringVal::null(); + } - // boost::local_time::local_date_time lt(tv->date(), tv->time_of_day(), - // timezone, boost::local_time::local_date_time::NOT_DATE_TIME_ON_ERROR); - // e->_result.timestamp_val = DateTimeValue(lt.utc_time()); - // return &e->_result.timestamp_val; + TimestampValue timestamp(unix_time.val); + DateTimeValue tv; + timestamp.to_datetime_value(tv, context->impl()->state()->timezone()); + char buf[128]; + if (!tv.to_format_string((const char*)fmt.ptr, fmt.len, buf)){ + return StringVal::null(); + } + return AnyValUtil::from_string_temp(context, buf); } -TimezoneDatabase::TimezoneDatabase() { - // Create a temporary file and write the timezone information. The boost - // interface only loads this format from a file. We don't want to raise - // an error here since this is done when the backend is created and this - // information might not actually get used by any queries. - char filestr[] = "/tmp/doris.tzdb.XXXXXXX"; - FILE* file = NULL; - int fd = -1; - - if ((fd = mkstemp(filestr)) == -1) { - LOG(ERROR) << "Could not create temporary timezone file: " << filestr; - return; - } +IntVal TimestampFunctions::to_unix(FunctionContext* context) { + return IntVal(context->impl()->state()->timestamp() / 1000); +} - if ((file = fopen(filestr, "w")) == NULL) { - unlink(filestr); - close(fd); - LOG(ERROR) << "Could not open temporary timezone file: " << filestr; - return; +IntVal TimestampFunctions::to_unix( + FunctionContext* context, const StringVal& string_val, const StringVal& fmt) { + if (string_val.is_null || fmt.is_null) { + return IntVal::null(); } - - if (fputs(_s_timezone_database_str, file) == EOF) { - unlink(filestr); - close(fd); - fclose(file); - LOG(ERROR) << "Could not load temporary timezone file: " << filestr; - return; + DateTimeValue tv; + if (!tv.from_date_format_str( + (const char *)fmt.ptr, fmt.len, (const char *)string_val.ptr, string_val.len)) { + return IntVal::null(); } - - fclose(file); - _s_tz_database.load_from_file(std::string(filestr)); - _s_tz_region_list = _s_tz_database.region_list(); - unlink(filestr); - close(fd); + TimestampValue ts; + ts.from_date_time_value(tv, context->impl()->state()->timezone()); + return ts.val; } -TimezoneDatabase::~TimezoneDatabase() { } - -boost::local_time::time_zone_ptr TimezoneDatabase::find_timezone(const std::string& tz) { - // See if they specified a zone id - if (tz.find_first_of('/') != std::string::npos) { - return _s_tz_database.time_zone_from_region(tz); +IntVal TimestampFunctions::to_unix( + FunctionContext* context, const DateTimeVal& ts_val) { + if (ts_val.is_null) { + return IntVal::null(); } + const DateTimeValue &tv = DateTimeValue::from_datetime_val(ts_val); + TimestampValue ts; + ts.from_date_time_value(tv, context->impl()->state()->timezone()); + return ts.val; +} - for (std::vector::const_iterator iter = _s_tz_region_list.begin(); - iter != _s_tz_region_list.end(); ++iter) { - boost::local_time::time_zone_ptr tzp = _s_tz_database.time_zone_from_region(*iter); - DCHECK(tzp != NULL); - - if (tzp->dst_zone_abbrev() == tz) { - return tzp; - } +DateTimeVal TimestampFunctions::utc_timestamp(FunctionContext* context) { + TimestampValue timestamp(context->impl()->state()->timestamp()); + DateTimeVal return_val; + timestamp.to_datetime_val(&return_val); + return return_val; +} - if (tzp->std_zone_abbrev() == tz) { - return tzp; - } +DateTimeVal TimestampFunctions::now(FunctionContext* context) { + TimestampValue ts(context->impl()->state()->timestamp()); + DateTimeVal return_val; + ts.to_datetime_val(&return_val, context->impl()->state()->timezone()); + return return_val; +} - if (tzp->dst_zone_name() == tz) { - return tzp; - } +DoubleVal TimestampFunctions::curtime(FunctionContext* context) { + TimestampValue ts(context->impl()->state()->timestamp()); + DoubleVal return_val; + ts.to_time_val(&return_val, context->impl()->state()->timezone()); + return return_val; +} - if (tzp->std_zone_name() == tz) { - return tzp; - } +DateTimeVal TimestampFunctions::convert_tz(FunctionContext* ctx, const DateTimeVal& ts_val, + const StringVal& from_tz, const StringVal& to_tz) { + TimezoneDatabase::init(); + if (TimezoneDatabase::find_timezone(std::string((char *)from_tz.ptr, from_tz.len)) == nullptr || + TimezoneDatabase::find_timezone(std::string((char *)to_tz.ptr, to_tz.len)) == nullptr + ) { + return DateTimeVal::null(); } - return boost::local_time::time_zone_ptr(); - + const DateTimeValue &ts_value = DateTimeValue::from_datetime_val(ts_val); + TimestampValue ts; + ts.from_date_time_value(ts_value, std::string((char *)from_tz.ptr, from_tz.len)); + DateTimeVal return_val; + ts.to_datetime_val(&return_val, std::string((char *)to_tz.ptr, to_tz.len)); + return return_val; } } diff --git a/be/src/exprs/timestamp_functions.h b/be/src/exprs/timestamp_functions.h index c14edfe1e7355f..fbc62b385bc7cf 100644 --- a/be/src/exprs/timestamp_functions.h +++ b/be/src/exprs/timestamp_functions.h @@ -35,23 +35,6 @@ class TupleRow; class TimestampFunctions { public: static void init(); - /// Returns the current time. - static doris_udf::IntVal to_unix(doris_udf::FunctionContext* context); - /// Converts 'tv_val' to a unix time_t - static doris_udf::IntVal to_unix( - doris_udf::FunctionContext* context, const doris_udf::DateTimeVal& tv_val); - /// Parses 'string_val' based on the format 'fmt'. - static doris_udf::IntVal to_unix( - doris_udf::FunctionContext* context, const doris_udf::StringVal& string_val, - const doris_udf::StringVal& fmt); - /// Return a timestamp string from a unix time_t - /// Optional second argument is the format of the string. - /// TIME is the integer type of the unix time argument. - static doris_udf::StringVal from_unix( - doris_udf::FunctionContext* context, const doris_udf::IntVal& unix_time); - static doris_udf::StringVal from_unix( - doris_udf::FunctionContext* context, const doris_udf::IntVal& unix_time, - const doris_udf::StringVal& fmt); // Functions to extract parts of the timestamp, return integers. static doris_udf::IntVal year( @@ -76,18 +59,14 @@ class TimestampFunctions { doris_udf::FunctionContext* context, const doris_udf::DateTimeVal& ts_val); // Date/time functions. - static doris_udf::DateTimeVal now(doris_udf::FunctionContext* context); - static doris_udf::DateTimeVal curtime(doris_udf::FunctionContext* context); - static doris_udf::DateTimeVal utc_timestamp(doris_udf::FunctionContext* context); static doris_udf::DateTimeVal to_date( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val); static doris_udf::IntVal date_diff( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val1, const doris_udf::DateTimeVal& ts_val2); - static doris_udf::DateTimeVal time_diff( + static doris_udf::DoubleVal time_diff( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val1, const doris_udf::DateTimeVal& ts_val2); - static doris_udf::DateTimeVal years_add( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val, const doris_udf::IntVal& count); @@ -136,7 +115,6 @@ class TimestampFunctions { static doris_udf::DateTimeVal micros_sub( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val, const doris_udf::IntVal& count); - static doris_udf::StringVal date_format( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val, const doris_udf::StringVal& format); @@ -152,18 +130,37 @@ class TimestampFunctions { static doris_udf::StringVal day_name( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val); + // TimeZone correlation functions. static doris_udf::DateTimeVal timestamp( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& val); - // Helper for add/sub functions on the time portion. template static doris_udf::DateTimeVal timestamp_time_op( doris_udf::FunctionContext* ctx, const doris_udf::DateTimeVal& ts_val, const doris_udf::IntVal& count, bool is_add); - - // Convert a timestamp to or from a particular timezone based time. - static void* from_utc(Expr* e, TupleRow* row); - static void* to_utc(Expr* e, TupleRow* row); + static doris_udf::DateTimeVal now(doris_udf::FunctionContext* context); + static doris_udf::DoubleVal curtime(doris_udf::FunctionContext* context); + static doris_udf::DateTimeVal utc_timestamp(doris_udf::FunctionContext* context); + /// Returns the current time. + static doris_udf::IntVal to_unix(doris_udf::FunctionContext* context); + /// Converts 'tv_val' to a unix time_t + static doris_udf::IntVal to_unix( + doris_udf::FunctionContext* context, const doris_udf::DateTimeVal& tv_val); + /// Parses 'string_val' based on the format 'fmt'. + static doris_udf::IntVal to_unix( + doris_udf::FunctionContext* context, const doris_udf::StringVal& string_val, + const doris_udf::StringVal& fmt); + /// Return a timestamp string from a unix time_t + /// Optional second argument is the format of the string. + /// TIME is the integer type of the unix time argument. + static doris_udf::StringVal from_unix( + doris_udf::FunctionContext* context, const doris_udf::IntVal& unix_time); + static doris_udf::StringVal from_unix( + doris_udf::FunctionContext* context, const doris_udf::IntVal& unix_time, + const doris_udf::StringVal& fmt); + static doris_udf::DateTimeVal convert_tz(doris_udf::FunctionContext* ctx, + const doris_udf::DateTimeVal& ts_val, const doris_udf::StringVal& from_tz, + const doris_udf::StringVal& to_tz); // Helper function to check date/time format strings. // TODO: eventually return format converted from Java to Boost. @@ -171,23 +168,7 @@ class TimestampFunctions { // Issue a warning for a bad format string. static void report_bad_format(const StringVal* format); - }; - -// Functions to load and access the timestamp database. -class TimezoneDatabase { -public: - TimezoneDatabase(); - ~TimezoneDatabase(); - - static boost::local_time::time_zone_ptr find_timezone(const std::string& tz); - -private: - static const char* _s_timezone_database_str; - static boost::local_time::tz_database _s_tz_database; - static std::vector _s_tz_region_list; -}; - } #endif diff --git a/be/src/exprs/timezone_db.cpp b/be/src/exprs/timezone_db.cpp index 3ba36736b901c8..2ad08c106fbebc 100644 --- a/be/src/exprs/timezone_db.cpp +++ b/be/src/exprs/timezone_db.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "timestamp_functions.h" +#include "runtime/timestamp_value.h" namespace doris { const char* TimezoneDatabase::_s_timezone_database_str = diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt index 5ad5e3585b19b4..5c3225a0cb86af 100644 --- a/be/src/runtime/CMakeLists.txt +++ b/be/src/runtime/CMakeLists.txt @@ -100,6 +100,7 @@ set(RUNTIME_FILES routine_load/data_consumer_pool.cpp routine_load/routine_load_task_executor.cpp small_file_mgr.cpp + timestamp_value.cpp ) if (WITH_MYSQL) diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp index 51d4664336085d..79dd326aaabd03 100644 --- a/be/src/runtime/plan_fragment_executor.cpp +++ b/be/src/runtime/plan_fragment_executor.cpp @@ -76,7 +76,7 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request) { // VLOG(2) << "request:\n" << apache::thrift::ThriftDebugString(request); _runtime_state.reset(new RuntimeState( - request, request.query_options, request.query_globals.now_string, _exec_env)); + request, request.query_options, request.query_globals, _exec_env)); RETURN_IF_ERROR(_runtime_state->init_mem_trackers(_query_id)); _runtime_state->set_be_number(request.backend_num); diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 257c6f90364176..e155dd8bbdbdee 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -33,6 +33,7 @@ #include "runtime/exec_env.h" #include "runtime/initial_reservations.h" #include "runtime/runtime_state.h" +#include "runtime/timestamp_value.h" #include "runtime/load_path_mgr.h" #include "util/cpu_info.h" #include "util/mem_info.h" @@ -50,7 +51,7 @@ namespace doris { RuntimeState::RuntimeState( const TUniqueId& fragment_instance_id, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env) : + const TQueryGlobals& query_globals, ExecEnv* exec_env) : _obj_pool(new ObjectPool()), _data_stream_recvrs_pool(new ObjectPool()), _unreported_error_idx(0), @@ -68,14 +69,14 @@ RuntimeState::RuntimeState( _error_log_file_path(""), _error_log_file(nullptr), _instance_buffer_reservation(new ReservationTracker) { - Status status = init(fragment_instance_id, query_options, now, exec_env); + Status status = init(fragment_instance_id, query_options, query_globals, exec_env); DCHECK(status.ok()); } RuntimeState::RuntimeState( const TExecPlanFragmentParams& fragment_params, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env) : + const TQueryGlobals& query_globals, ExecEnv* exec_env) : _obj_pool(new ObjectPool()), _data_stream_recvrs_pool(new ObjectPool()), _unreported_error_idx(0), @@ -95,19 +96,23 @@ RuntimeState::RuntimeState( _error_log_file_path(""), _error_log_file(nullptr), _instance_buffer_reservation(new ReservationTracker) { - Status status = init(fragment_params.params.fragment_instance_id, query_options, now, exec_env); + Status status = init(fragment_params.params.fragment_instance_id, query_options, query_globals, exec_env); DCHECK(status.ok()); } -RuntimeState::RuntimeState(const std::string& now) +RuntimeState::RuntimeState(const TQueryGlobals& query_globals) : _obj_pool(new ObjectPool()), _data_stream_recvrs_pool(new ObjectPool()), _unreported_error_idx(0), _profile(_obj_pool.get(), ""), _per_fragment_instance_idx(0) { _query_options.batch_size = DEFAULT_BATCH_SIZE; - _now.reset(new DateTimeValue()); - _now->from_date_str(now.c_str(), now.size()); + _timestamp = atol(query_globals.now_string.c_str()); + if (query_globals.__isset.time_zone) { + _timezone = query_globals.time_zone; + } else { + _timezone = "Asia/Shanghai"; + } } RuntimeState::~RuntimeState() { @@ -161,11 +166,15 @@ RuntimeState::~RuntimeState() { Status RuntimeState::init( const TUniqueId& fragment_instance_id, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env) { + const TQueryGlobals& query_globals, ExecEnv* exec_env) { _fragment_instance_id = fragment_instance_id; _query_options = query_options; - _now.reset(new DateTimeValue()); - _now->from_date_str(now.c_str(), now.size()); + _timestamp = atol(query_globals.now_string.c_str()); + if (query_globals.__isset.time_zone) { + _timezone = query_globals.time_zone; + } else { + _timezone = "Asia/Shanghai"; + } _exec_env = exec_env; if (!query_options.disable_codegen) { diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 24aedd6a30f0de..f246b933cf55b8 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -68,15 +68,15 @@ class RuntimeState { // for ut only RuntimeState(const TUniqueId& fragment_instance_id, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env); + const TQueryGlobals& query_globals, ExecEnv* exec_env); RuntimeState( const TExecPlanFragmentParams& fragment_params, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env); + const TQueryGlobals& query_globals, ExecEnv* exec_env); // RuntimeState for executing expr in fe-support. - RuntimeState(const std::string& now); + RuntimeState(const TQueryGlobals& query_globals); // Empty d'tor to avoid issues with scoped_ptr. ~RuntimeState(); @@ -84,7 +84,7 @@ class RuntimeState { // Set per-query state. Status init(const TUniqueId& fragment_instance_id, const TQueryOptions& query_options, - const std::string& now, ExecEnv* exec_env); + const TQueryGlobals& query_globals, ExecEnv* exec_env); // Set up four-level hierarchy of mem trackers: process, query, fragment instance. // The instance tracker is tied to our profile. @@ -136,8 +136,11 @@ class RuntimeState { int num_scanner_threads() const { return _query_options.num_scanner_threads; } - const DateTimeValue* now() const { - return _now.get(); + int64_t timestamp() const { + return _timestamp; + } + const std::string timezone() const { + return _timezone; } const std::string& user() const { return _user; @@ -532,9 +535,10 @@ class RuntimeState { // Username of user that is executing the query to which this RuntimeState belongs. std::string _user; - // Query-global timestamp, e.g., for implementing now(). - // Use pointer to avoid inclusion of timestampvalue.h and avoid clang issues. - boost::scoped_ptr _now; + + //Query-global timestamp + int64_t _timestamp; + std::string _timezone; TUniqueId _query_id; TUniqueId _fragment_instance_id; diff --git a/be/src/runtime/test_env.cc b/be/src/runtime/test_env.cc index 691a08ee3d35ba..b0d5048d3b09b7 100644 --- a/be/src/runtime/test_env.cc +++ b/be/src/runtime/test_env.cc @@ -66,7 +66,7 @@ RuntimeState* TestEnv::create_runtime_state(int64_t query_id) { TExecPlanFragmentParams plan_params = TExecPlanFragmentParams(); plan_params.params.query_id.hi = 0; plan_params.params.query_id.lo = query_id; - return new RuntimeState(plan_params, TQueryOptions(), "", _exec_env.get()); + return new RuntimeState(plan_params, TQueryOptions(), TQueryGlobals(), _exec_env.get()); } Status TestEnv::create_query_state(int64_t query_id, int max_buffers, int block_size, diff --git a/be/src/runtime/timestamp_value.cpp b/be/src/runtime/timestamp_value.cpp new file mode 100644 index 00000000000000..6e84cdb3614cef --- /dev/null +++ b/be/src/runtime/timestamp_value.cpp @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#include "runtime/timestamp_value.h" + +#include +#include +#include +#include +#include +#include "runtime/string_value.h" + + + +namespace doris { + +boost::local_time::tz_database TimezoneDatabase::_s_tz_database; +std::vector TimezoneDatabase::_s_tz_region_list; + +TimezoneDatabase::TimezoneDatabase() { + // Create a temporary file and write the timezone information. The boost + // interface only loads this format from a file. We don't want to raise + // an error here since this is done when the backend is created and this + // information might not actually get used by any queries. + char filestr[] = "/tmp/doris.tzdb.XXXXXXX"; + FILE *file = NULL; + int fd = -1; + + if ((fd = mkstemp(filestr)) == -1) { + LOG(ERROR) << "Could not create temporary timezone file: " << filestr; + return; + } + + if ((file = fopen(filestr, "w")) == NULL) { + unlink(filestr); + close(fd); + LOG(ERROR) << "Could not open temporary timezone file: " << filestr; + return; + } + + if (fputs(_s_timezone_database_str, file) == EOF) { + unlink(filestr); + close(fd); + fclose(file); + LOG(ERROR) << "Could not load temporary timezone file: " << filestr; + return; + } + + fclose(file); + _s_tz_database.load_from_file(std::string(filestr)); + _s_tz_region_list = _s_tz_database.region_list(); + unlink(filestr); + close(fd); +} + +TimezoneDatabase::~TimezoneDatabase() {} + +boost::local_time::time_zone_ptr TimezoneDatabase::find_timezone(const std::string &tz) { + try { + // See if they specified a zone id + if (tz.find_first_of('/') != std::string::npos) { + return _s_tz_database.time_zone_from_region(tz); + } else { + //eg. +08:00 + boost::local_time::time_zone_ptr tzp(new boost::local_time::posix_time_zone(std::string("TMP") + tz)); + return tzp; + } + } catch (boost::exception& e) { + return nullptr; + } +} + +} diff --git a/be/src/runtime/timestamp_value.h b/be/src/runtime/timestamp_value.h new file mode 100644 index 00000000000000..3245f428a4b6eb --- /dev/null +++ b/be/src/runtime/timestamp_value.h @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_RUNTIME_TIMESTAMP_VALUE_H +#define DORIS_BE_RUNTIME_TIMESTAMP_VALUE_H + +#include + +#include +#include +#include +#include + +#include "udf/udf.h" +#include "util/hash_util.hpp" + +#include "timestamp_value.h" + +namespace doris { + +// Functions to load and access the timestamp database. +class TimezoneDatabase { +public: + TimezoneDatabase(); + + ~TimezoneDatabase(); + + static void init() { + TimezoneDatabase(); + } + + static boost::local_time::time_zone_ptr find_timezone(const std::string &tz); + +private: + static const char *_s_timezone_database_str; + static boost::local_time::tz_database _s_tz_database; + static std::vector _s_tz_region_list; +}; + +class TimestampValue { +public: + TimestampValue() { } + + TimestampValue(time_t timestamp) { + val = timestamp; + } + + bool from_date_time_value(DateTimeValue tv, std::string timezone) { + boost::local_time::time_zone_ptr local_time_zone = TimezoneDatabase::find_timezone(timezone); + if (local_time_zone == nullptr) { + return false; + } + + std::stringstream ss; + ss << tv; + boost::posix_time::ptime pt = boost::posix_time::time_from_string(ss.str()); + boost::local_time::local_date_time lt(pt.date(), pt.time_of_day(), local_time_zone, + boost::local_time::local_date_time::NOT_DATE_TIME_ON_ERROR); + + boost::posix_time::ptime utc_ptime = lt.utc_time(); + boost::posix_time::ptime utc_start(boost::gregorian::date(1970, 1, 1)); + boost::posix_time::time_duration dur = utc_ptime - utc_start; + val = dur.total_milliseconds(); + return true; + } + + bool to_datetime_value(DateTimeValue &dt_val, std::string timezone) { + boost::local_time::time_zone_ptr local_time_zone = TimezoneDatabase::find_timezone(timezone); + if (local_time_zone == nullptr) { + return false; + } + boost::local_time::local_date_time lt(boost::posix_time::from_time_t(val), local_time_zone); + boost::posix_time::ptime locat_ptime = lt.local_time(); + + dt_val.set_type(TIME_DATETIME); + dt_val.from_olap_datetime( + locat_ptime.date().year() * 10000000000 + + locat_ptime.date().month() * 100000000 + + locat_ptime.date().day() * 1000000 + + locat_ptime.time_of_day().hours() * 10000 + + locat_ptime.time_of_day().minutes() * 100 + + locat_ptime.time_of_day().seconds()); + return true; + } + + std::string to_datetime_string(std::string timezone) { + boost::local_time::time_zone_ptr local_time_zone = TimezoneDatabase::find_timezone(timezone); + if (local_time_zone == nullptr) { + return ""; + } + boost::local_time::local_date_time lt(boost::posix_time::from_time_t(val), local_time_zone); + boost::posix_time::ptime ret_ptime = lt.local_time(); + + std::stringstream ss; + ss << std::setw(4) << std::setfill('0') << ret_ptime.date().year() << "-" + << std::setw(2) << std::setfill('0') << ret_ptime.date().month().as_number() << "-" + << std::setw(2) << std::setfill('0') << ret_ptime.date().day() << " " + << std::setw(2) << std::setfill('0') << ret_ptime.time_of_day().hours() << ":" + << std::setw(2) << std::setfill('0') << ret_ptime.time_of_day().minutes() << ":" + << std::setw(2) << std::setfill('0') << ret_ptime.time_of_day().seconds(); + return ss.str(); + } + + void to_datetime_val(doris_udf::DateTimeVal *tv) const { + boost::posix_time::ptime p = boost::posix_time::from_time_t(val / 1000); + int _year = p.date().year(); + int _month = p.date().month(); + int _day = p.date().day(); + int64_t _hour = p.time_of_day().hours(); + int64_t _minute = p.time_of_day().minutes(); + int64_t _second = p.time_of_day().seconds(); + int _microsecond = 0; + + int64_t ymd = ((_year * 13 + _month) << 5) | _day; + int64_t hms = (_hour << 12) | (_minute << 6) | _second; + tv->packed_time = (((ymd << 17) | hms) << 24) + _microsecond; + tv->type = TIME_DATETIME; + } + + bool to_datetime_val(doris_udf::DateTimeVal *tv, std::string timezone) const { + boost::local_time::time_zone_ptr local_time_zone = TimezoneDatabase::find_timezone(timezone); + if (local_time_zone == nullptr) { + return false; + } + boost::local_time::local_date_time lt(boost::posix_time::from_time_t(val / 1000), local_time_zone); + boost::posix_time::ptime p = lt.local_time(); + + int _year = p.date().year(); + int _month = p.date().month(); + int _day = p.date().day(); + int64_t _hour = p.time_of_day().hours(); + int64_t _minute = p.time_of_day().minutes(); + int64_t _second = p.time_of_day().seconds(); + int _microsecond = 0; + + int64_t ymd = ((_year * 13 + _month) << 5) | _day; + int64_t hms = (_hour << 12) | (_minute << 6) | _second; + tv->packed_time = (((ymd << 17) | hms) << 24) + _microsecond; + tv->type = TIME_DATETIME; + return true; + } + + bool to_time_val(doris_udf::DoubleVal *tv, std::string timezone) const { + boost::local_time::time_zone_ptr local_time_zone = TimezoneDatabase::find_timezone(timezone); + if (local_time_zone == nullptr) { + return false; + } + boost::local_time::local_date_time lt(boost::posix_time::from_time_t(val / 1000), local_time_zone); + boost::posix_time::ptime p = lt.local_time(); + + int64_t _hour = p.time_of_day().hours(); + int64_t _minute = p.time_of_day().minutes(); + int64_t _second = p.time_of_day().seconds(); + + tv->val = _hour * 3600 + _minute * 60 + _second; + return true; + } + +public: + int64_t val; +}; + +} +#endif //DORIS_BE_RUNTIME_TIMESTAMP_VALUE_H diff --git a/be/test/exec/broker_scan_node_test.cpp b/be/test/exec/broker_scan_node_test.cpp index 066f468dfa99cb..5ccb5506e3c1d7 100644 --- a/be/test/exec/broker_scan_node_test.cpp +++ b/be/test/exec/broker_scan_node_test.cpp @@ -38,7 +38,7 @@ namespace doris { class BrokerScanNodeTest : public testing::Test { public: - BrokerScanNodeTest() : _runtime_state("BrokerScanNodeTest") { + BrokerScanNodeTest() : _runtime_state(TQueryGlobals()) { init(); _runtime_state._instance_mem_tracker.reset(new MemTracker()); } diff --git a/be/test/exec/broker_scanner_test.cpp b/be/test/exec/broker_scanner_test.cpp index 81c0cd1a8309b7..1c59c10d790a4f 100644 --- a/be/test/exec/broker_scanner_test.cpp +++ b/be/test/exec/broker_scanner_test.cpp @@ -38,7 +38,7 @@ namespace doris { class BrokerScannerTest : public testing::Test { public: - BrokerScannerTest() : _runtime_state("BrokerScannerTest") { + BrokerScannerTest() : _runtime_state(TQueryGlobals()) { init(); _profile = _runtime_state.runtime_profile(); _runtime_state._instance_mem_tracker.reset(new MemTracker()); diff --git a/be/test/exec/es_http_scan_node_test.cpp b/be/test/exec/es_http_scan_node_test.cpp index e3fd63a9d2e64f..0c8f73a15d52af 100644 --- a/be/test/exec/es_http_scan_node_test.cpp +++ b/be/test/exec/es_http_scan_node_test.cpp @@ -38,7 +38,7 @@ namespace doris { // mock class EsHttpScanNodeTest : public testing::Test { public: - EsHttpScanNodeTest() : _runtime_state("EsHttpScanNodeTest") { + EsHttpScanNodeTest() : _runtime_state(TQueryGlobals()) { _runtime_state._instance_mem_tracker.reset(new MemTracker()); TDescriptorTable t_desc_table; diff --git a/be/test/exec/es_predicate_test.cpp b/be/test/exec/es_predicate_test.cpp index 3c18bf1af4c454..0fd5abe1c30f88 100644 --- a/be/test/exec/es_predicate_test.cpp +++ b/be/test/exec/es_predicate_test.cpp @@ -40,7 +40,7 @@ class RuntimeState; class EsPredicateTest : public testing::Test { public: - EsPredicateTest() : _runtime_state("EsPredicateTest") { + EsPredicateTest() : _runtime_state(TQueryGlobals()) { _runtime_state._instance_mem_tracker.reset(new MemTracker()); TDescriptorTable t_desc_table; diff --git a/be/test/exec/es_scan_node_test.cpp b/be/test/exec/es_scan_node_test.cpp index adc2c0d8e3e9bd..77f2cb7cf69454 100644 --- a/be/test/exec/es_scan_node_test.cpp +++ b/be/test/exec/es_scan_node_test.cpp @@ -37,7 +37,7 @@ namespace doris { // mock class EsScanNodeTest : public testing::Test { public: - EsScanNodeTest() : _runtime_state("EsScanNodeTest") { + EsScanNodeTest() : _runtime_state(TQueryGlobals()) { _runtime_state._instance_mem_tracker.reset(new MemTracker()); TDescriptorTable t_desc_table; diff --git a/be/test/exec/parquet_scanner_test.cpp b/be/test/exec/parquet_scanner_test.cpp index 4bea0130c33672..2d96e44cdd5b95 100644 --- a/be/test/exec/parquet_scanner_test.cpp +++ b/be/test/exec/parquet_scanner_test.cpp @@ -38,7 +38,7 @@ namespace doris { class ParquetSannerTest : public testing::Test { public: - ParquetSannerTest() : _runtime_state("ParquetSannerTest") { + ParquetSannerTest() : _runtime_state(TQueryGlobals()) { init(); _runtime_state._instance_mem_tracker.reset(new MemTracker()); } diff --git a/be/test/exec/tablet_sink_test.cpp b/be/test/exec/tablet_sink_test.cpp index 0033a587db0c74..c1bf8c602b0bc5 100644 --- a/be/test/exec/tablet_sink_test.cpp +++ b/be/test/exec/tablet_sink_test.cpp @@ -21,6 +21,7 @@ #include "gen_cpp/HeartbeatService_types.h" #include "gen_cpp/internal_service.pb.h" +#include "common/config.h" #include "runtime/decimal_value.h" #include "runtime/exec_env.h" #include "runtime/row_batch.h" @@ -49,6 +50,8 @@ class OlapTableSinkTest : public testing::Test { _env._master_info = new TMasterInfo(); _env._load_stream_mgr = new LoadStreamMgr(); _env._brpc_stub_cache = new BrpcStubCache(); + + config::tablet_writer_rpc_timeout_sec = 600; } void TearDown() override { delete _env._brpc_stub_cache; @@ -332,7 +335,7 @@ TEST_F(OlapTableSinkTest, normal) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -438,7 +441,7 @@ TEST_F(OlapTableSinkTest, convert) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1024; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -565,7 +568,7 @@ TEST_F(OlapTableSinkTest, init_fail1) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -623,7 +626,7 @@ TEST_F(OlapTableSinkTest, init_fail3) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -682,7 +685,7 @@ TEST_F(OlapTableSinkTest, init_fail4) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -749,7 +752,7 @@ TEST_F(OlapTableSinkTest, add_batch_failed) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; @@ -844,7 +847,7 @@ TEST_F(OlapTableSinkTest, decimal) { TUniqueId fragment_id; TQueryOptions query_options; query_options.batch_size = 1; - RuntimeState state(fragment_id, query_options, "2018-05-25 12:14:15", &_env); + RuntimeState state(fragment_id, query_options, TQueryGlobals(), &_env); state._instance_mem_tracker.reset(new MemTracker()); ObjectPool obj_pool; diff --git a/be/test/exprs/timestamp_functions_test.cpp b/be/test/exprs/timestamp_functions_test.cpp index 2e521cda837013..8478347b6ca76d 100644 --- a/be/test/exprs/timestamp_functions_test.cpp +++ b/be/test/exprs/timestamp_functions_test.cpp @@ -37,6 +37,39 @@ TEST_F(TimestampFunctionsTest, day_of_week_test) { ASSERT_EQ(7, TimestampFunctions::day_of_week(context, tv).val); } +TEST_F(TimestampFunctionsTest, time_diff_test) { + doris_udf::FunctionContext *context = new doris_udf::FunctionContext(); + + DateTimeValue dt1(20190718120000); + dt1.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv1; + dt1.to_datetime_val(&tv1); + + DateTimeValue dt2(20190718130102); + dt2.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv2; + dt2.to_datetime_val(&tv2); + + ASSERT_EQ(-3662, TimestampFunctions::time_diff(context, tv1, tv2).val); +} + +TEST_F(TimestampFunctionsTest, convert_tz_test) { + doris_udf::FunctionContext *context = new doris_udf::FunctionContext(); + + DateTimeValue dt1(20190801175700); + dt1.set_type(TIME_DATETIME); + doris_udf::DateTimeVal tv1; + dt1.to_datetime_val(&tv1); + + std::cout << tv1.packed_time << std::endl; + + DateTimeVal t = TimestampFunctions::convert_tz(context, tv1, StringVal("Asia/Shanghai"), StringVal("America/Los_Angeles")); + std::cout << t.packed_time << std::endl; + + DateTimeValue dt2 = DateTimeValue::from_datetime_val(t); + ASSERT_EQ(20190801025700, dt2.to_int64()); +} + } int main(int argc, char** argv) { diff --git a/be/test/runtime/buffered_block_mgr2_test.cpp b/be/test/runtime/buffered_block_mgr2_test.cpp index eb4824840d361f..0b4cfa196d3320 100644 --- a/be/test/runtime/buffered_block_mgr2_test.cpp +++ b/be/test/runtime/buffered_block_mgr2_test.cpp @@ -557,7 +557,7 @@ class BufferedBlockMgrTest : public ::testing::Test { // RuntimeState* shared_state = new RuntimeState(TExecPlanFragmentParams(), "", // _test_env->exec_env()); RuntimeState* shared_state = new RuntimeState( - TUniqueId(), TQueryOptions(), "", _test_env->exec_env()); + TUniqueId(), TQueryOptions(), TQueryGlobals(), _test_env->exec_env()); for (int i = 0; i < num_threads; ++i) { thread* t = new boost::thread(boost::bind( &BufferedBlockMgrTest::CreateDestroyThread, this, i, shared_state)); diff --git a/be/test/runtime/tablet_writer_mgr_test.cpp b/be/test/runtime/tablet_writer_mgr_test.cpp index 65e02a83b0d6e3..1a1114e0416d62 100644 --- a/be/test/runtime/tablet_writer_mgr_test.cpp +++ b/be/test/runtime/tablet_writer_mgr_test.cpp @@ -39,6 +39,7 @@ std::unordered_map _k_tablet_recorder; OLAPStatus open_status; OLAPStatus add_status; OLAPStatus close_status; +int64_t wait_lock_time_ns; // mock DeltaWriter::DeltaWriter(WriteRequest* req) : _req(*req) { @@ -223,7 +224,7 @@ TEST_F(TabletWriterMgrTest, normal) { } row_batch.serialize(request.mutable_row_batch()); google::protobuf::RepeatedPtrField tablet_vec; - auto st = mgr.add_batch(request, &tablet_vec); + auto st = mgr.add_batch(request, &tablet_vec, &wait_lock_time_ns); request.release_id(); ASSERT_TRUE(st.ok()); } @@ -387,7 +388,7 @@ TEST_F(TabletWriterMgrTest, add_failed) { row_batch.serialize(request.mutable_row_batch()); add_status = OLAP_ERR_TABLE_NOT_FOUND; google::protobuf::RepeatedPtrField tablet_vec; - auto st = mgr.add_batch(request, &tablet_vec); + auto st = mgr.add_batch(request, &tablet_vec, &wait_lock_time_ns); request.release_id(); ASSERT_FALSE(st.ok()); } @@ -476,7 +477,7 @@ TEST_F(TabletWriterMgrTest, close_failed) { row_batch.serialize(request.mutable_row_batch()); close_status = OLAP_ERR_TABLE_NOT_FOUND; google::protobuf::RepeatedPtrField tablet_vec; - auto st = mgr.add_batch(request, &tablet_vec); + auto st = mgr.add_batch(request, &tablet_vec, &wait_lock_time_ns); request.release_id(); ASSERT_FALSE(st.ok()); } @@ -561,7 +562,7 @@ TEST_F(TabletWriterMgrTest, unknown_tablet) { } row_batch.serialize(request.mutable_row_batch()); google::protobuf::RepeatedPtrField tablet_vec; - auto st = mgr.add_batch(request, &tablet_vec); + auto st = mgr.add_batch(request, &tablet_vec, &wait_lock_time_ns); request.release_id(); ASSERT_FALSE(st.ok()); } @@ -646,10 +647,10 @@ TEST_F(TabletWriterMgrTest, duplicate_packet) { } row_batch.serialize(request.mutable_row_batch()); google::protobuf::RepeatedPtrField tablet_vec1; - auto st = mgr.add_batch(request, &tablet_vec1); + auto st = mgr.add_batch(request, &tablet_vec1, &wait_lock_time_ns); ASSERT_TRUE(st.ok()); google::protobuf::RepeatedPtrField tablet_vec2; - st = mgr.add_batch(request, &tablet_vec2); + st = mgr.add_batch(request, &tablet_vec2, &wait_lock_time_ns); request.release_id(); ASSERT_TRUE(st.ok()); } @@ -662,7 +663,7 @@ TEST_F(TabletWriterMgrTest, duplicate_packet) { request.set_eos(true); request.set_packet_seq(0); google::protobuf::RepeatedPtrField tablet_vec; - auto st = mgr.add_batch(request, &tablet_vec); + auto st = mgr.add_batch(request, &tablet_vec, &wait_lock_time_ns); request.release_id(); ASSERT_TRUE(st.ok()); } diff --git a/be/test/runtime/timestamp_value_test.cpp b/be/test/runtime/timestamp_value_test.cpp new file mode 100644 index 00000000000000..10f21dc4862fe9 --- /dev/null +++ b/be/test/runtime/timestamp_value_test.cpp @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "runtime/timestamp_value.h" + +#include + +#include + +#include "common/logging.h" +#include "util/logging.h" + +namespace doris { + +class TimestampValueTest : public testing::Test { +public: + TimestampValueTest() { + } + +protected: + virtual void SetUp() { + } + virtual void TearDown() { + } +}; + +// Assert size +TEST_F(TimestampValueTest, struct_size) { + ASSERT_EQ(8, sizeof(TimestampValue)); +} + +TEST_F(TimestampValueTest, construct) { + + DateTimeValue value1; + value1.from_date_int64(20190801114602); + TimestampValue t(value1, "Asia/Shanghai"); + ASSERT_EQ(1564631162,t.val); +} + + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/convert_tz.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/convert_tz.md new file mode 100644 index 00000000000000..ece25505aa3c09 --- /dev/null +++ b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/convert_tz.md @@ -0,0 +1,27 @@ +# convert_tz + +## Syntax + +`DATETIME CONVERT_TZ(DATETIME dt, VARCHAR from_tz, VARCHAR to_tz)` + +## Description + +转换datetime值dt,从 from_tz 由给定转到 to_tz 时区给出的时区,并返回的结果值。 如果参数无效该函数返回NULL。 + +## Examples + +``` +mysql> select convert_tz('2019-08-01 13:21:03', 'Asia/Shanghai', 'America/Los_Angeles'); ++---------------------------------------------------------------------------+ +| convert_tz('2019-08-01 13:21:03', 'Asia/Shanghai', 'America/Los_Angeles') | ++---------------------------------------------------------------------------+ +| 2019-07-31 22:21:03 | ++---------------------------------------------------------------------------+ + +mysql> select convert_tz('2019-08-01 13:21:03', '+08:00', 'America/Los_Angeles'); ++--------------------------------------------------------------------+ +| convert_tz('2019-08-01 13:21:03', '+08:00', 'America/Los_Angeles') | ++--------------------------------------------------------------------+ +| 2019-07-31 22:21:03 | ++--------------------------------------------------------------------+ +``` \ No newline at end of file diff --git a/fe/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/src/main/java/org/apache/doris/qe/Coordinator.java index 8d289b038d75ee..774ee255de6b40 100644 --- a/fe/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/src/main/java/org/apache/doris/qe/Coordinator.java @@ -83,8 +83,6 @@ import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -104,8 +102,6 @@ public class Coordinator { private static final Logger LOG = LogManager.getLogger(Coordinator.class); - private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - private static String localIP = FrontendOptions.getLocalHostAddress(); // Overall status of the entire query; set to the first reported fragment error @@ -191,7 +187,12 @@ public Coordinator(ConnectContext context, Analyzer analyzer, Planner planner) { this.descTable = analyzer.getDescTbl().toThrift(); this.returnedAllResults = false; this.queryOptions = context.getSessionVariable().toThrift(); - this.queryGlobals.setNow_string(DATE_FORMAT.format(new Date())); + this.queryGlobals.setNow_string(String.valueOf(new Date().getTime())); + if (context.getSessionVariable().getTimeZone().equals("CST")) { + this.queryGlobals.setTime_zone("Asia/Shanghai"); + } else { + this.queryGlobals.setTime_zone(context.getSessionVariable().getTimeZone()); + } this.tResourceInfo = new TResourceInfo(context.getQualifiedUser(), context.getSessionVariable().getResourceGroup()); this.needReport = context.getSessionVariable().isReportSucc(); @@ -211,7 +212,8 @@ public Coordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, this.fragments = fragments; this.scanNodes = scanNodes; this.queryOptions = new TQueryOptions(); - this.queryGlobals.setNow_string(DATE_FORMAT.format(new Date())); + this.queryGlobals.setNow_string(String.valueOf(new Date().getTime())); + this.queryGlobals.setTime_zone("Asia/Shanghai"); this.tResourceInfo = new TResourceInfo("", ""); this.needReport = true; this.clusterName = cluster; diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index df8b64176082fb..872ca1cb3968c0 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -108,7 +108,7 @@ [['from_unixtime'], 'VARCHAR', ['INT', 'VARCHAR'], '_ZN5doris18TimestampFunctions9from_unixEPN9doris_udf' '15FunctionContextERKNS1_6IntValERKNS1_9StringValE'], - [['now', 'current_timestamp'], 'DATETIME', [], + [['now', 'current_timestamp', 'localtime', 'localtimestamp'], 'DATETIME', [], '_ZN5doris18TimestampFunctions3nowEPN9doris_udf15FunctionContextE'], [['curtime', 'current_time'], 'DATETIME', [], '_ZN5doris18TimestampFunctions7curtimeEPN9doris_udf15FunctionContextE'], @@ -218,6 +218,9 @@ '_ZN5doris18TimestampFunctions10month_nameEPN9doris_udf' '15FunctionContextERKNS1_11DateTimeValE'], + [['convert_tz'], 'DATETIME', ['DATETIME', 'VARCHAR', 'VARCHAR'], + '_ZN5doris18TimestampFunctions10convert_tzEPN9doris_udf15FunctionContextERKNS1_11DateTimeValERKNS1_9StringValES9_'], + # Math builtin functions [['pi'], 'DOUBLE', [], '_ZN5doris13MathFunctions2piEPN9doris_udf15FunctionContextE'], diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 25428d880589a7..f8d373e5a3085d 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -182,6 +182,7 @@ struct TPlanFragmentExecParams { struct TQueryGlobals { // String containing a timestamp set as the current time. 1: required string now_string + 2: optional string time_zone } From 68357f99576e510b5fdebf5367b22d16c50a132c Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Fri, 9 Aug 2019 16:44:04 +0800 Subject: [PATCH 2/6] add default time zone --- be/src/runtime/datetime_value.cpp | 5 -- be/src/runtime/datetime_value.h | 5 -- be/src/runtime/runtime_state.cpp | 4 +- be/test/exprs/timestamp_functions_test.cpp | 6 +- be/test/runtime/timestamp_value_test.cpp | 60 ------------------- .../apache/doris/common/util/TimeUtils.java | 2 + .../java/org/apache/doris/qe/Coordinator.java | 5 +- 7 files changed, 9 insertions(+), 78 deletions(-) delete mode 100644 be/test/runtime/timestamp_value_test.cpp diff --git a/be/src/runtime/datetime_value.cpp b/be/src/runtime/datetime_value.cpp index ce4892bd81ce4e..24e1f0e4db6b0a 100644 --- a/be/src/runtime/datetime_value.cpp +++ b/be/src/runtime/datetime_value.cpp @@ -1529,12 +1529,8 @@ int64_t DateTimeValue::unix_timestamp(const std::string& timezone) const { if (local_time_zone == nullptr) { return false; } - //boost::posix_time::ptime pt( - // boost::date_time::date(_year, _month, _day), boost::date_time::time_duration(_hour, _minute, _second)); - //std::cout << "ptime : " << pt << std::endl; char buf[64]; char* to = to_datetime_string(buf); - std::cout <<"timezone : " << std::string(buf, to - buf - 1) << std::endl; boost::posix_time::ptime pt = boost::posix_time::time_from_string(std::string(buf, to - buf -1)); boost::local_time::local_date_time lt(pt.date(), pt.time_of_day(), local_time_zone, @@ -1553,7 +1549,6 @@ bool DateTimeValue::from_unixtime(int64_t timestamp, const std::string& timezone } boost::local_time::local_date_time lt(boost::posix_time::from_time_t(timestamp), local_time_zone); boost::posix_time::ptime local_ptime = lt.local_time(); - std::cout << local_ptime<< std::endl; _neg = 0; _type = TIME_DATETIME; diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h index 754755aa9973bc..46e3d46d333833 100644 --- a/be/src/runtime/datetime_value.h +++ b/be/src/runtime/datetime_value.h @@ -22,11 +22,6 @@ #include #include -#include -#include -#include -#include -#include #include "udf/udf.h" #include "util/hash_util.hpp" diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 80958cf0ea1ba9..172d80193a6857 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -111,7 +111,7 @@ RuntimeState::RuntimeState(const TQueryGlobals& query_globals) if (query_globals.__isset.time_zone) { _timezone = query_globals.time_zone; } else { - _timezone = "Asia/Shanghai"; + _timezone = TimezoneDatabase::default_time_zone; } } @@ -173,7 +173,7 @@ Status RuntimeState::init( if (query_globals.__isset.time_zone) { _timezone = query_globals.time_zone; } else { - _timezone = "Asia/Shanghai"; + _timezone = TimezoneDatabase::default_time_zone; } _exec_env = exec_env; diff --git a/be/test/exprs/timestamp_functions_test.cpp b/be/test/exprs/timestamp_functions_test.cpp index 9338d4082b9191..4fc33ccbe9e0f6 100644 --- a/be/test/exprs/timestamp_functions_test.cpp +++ b/be/test/exprs/timestamp_functions_test.cpp @@ -68,8 +68,6 @@ TEST_F(TimestampFunctionsTest, day_of_week_test) { } TEST_F(TimestampFunctionsTest, time_diff_test) { - doris_udf::FunctionContext *context = new doris_udf::FunctionContext(); - DateTimeValue dt1(20190718120000); dt1.set_type(TIME_DATETIME); doris_udf::DateTimeVal tv1; @@ -80,7 +78,7 @@ TEST_F(TimestampFunctionsTest, time_diff_test) { doris_udf::DateTimeVal tv2; dt2.to_datetime_val(&tv2); - ASSERT_EQ(-3662, TimestampFunctions::time_diff(context, tv1, tv2).val); + ASSERT_EQ(-3662, TimestampFunctions::time_diff(ctx, tv1, tv2).val); } TEST_F(TimestampFunctionsTest, now) { @@ -101,7 +99,7 @@ TEST_F(TimestampFunctionsTest, to_unix) { dt_val.type = TIME_DATETIME; ASSERT_EQ(1565080737, TimestampFunctions::to_unix(ctx).val); ASSERT_EQ(1565080737, TimestampFunctions::to_unix(ctx, dt_val).val); - //ASSERT_EQ(1565080737, TimestampFunctions::to_unix(ctx, StringVal("2019-08-06 01:38:57"), "%Y-%m-%d %H:%i:%S").val); + ASSERT_EQ(1565080737, TimestampFunctions::to_unix(ctx, StringVal("2019-08-06 01:38:57"), "%Y-%m-%d %H:%i:%S").val); } TEST_F(TimestampFunctionsTest, curtime) { diff --git a/be/test/runtime/timestamp_value_test.cpp b/be/test/runtime/timestamp_value_test.cpp deleted file mode 100644 index b9724d0bba1d7f..00000000000000 --- a/be/test/runtime/timestamp_value_test.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "runtime/timestamp_value.h" - -#include -#include -#include "common/logging.h" -#include "util/logging.h" - -namespace doris { - -class TimestampValueTest : public testing::Test { -public: - TimestampValueTest() { - TimezoneDatabase::init(); - } - -protected: - virtual void SetUp() { - } - virtual void TearDown() { - } -}; - -// Assert size -TEST_F(TimestampValueTest, struct_size) { - ASSERT_EQ(8, sizeof(TimestampValue)); -} - -TEST_F(TimestampValueTest, construct) { - DateTimeValue value1; - value1.from_date_int64(20190806163857); - TimestampValue ts; - ts.from_date_time_value(value1, std::string("Asia/Shanghai")); - ASSERT_EQ(1565080737, ts.val / 1000); -} - -TEST_F(TimestampValueTest, timezone) { - ASSERT_TRUE(TimezoneDatabase::find_timezone("Error timezone") == nullptr); -} -} -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java index 7adae22a674572..490cdd03d2c3d2 100644 --- a/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -66,6 +66,8 @@ public class TimeUtils { public static int MIN_TIME; public static int MAX_TIME; + public static String DEFAULT_TIME_ZONE = "Asia/Shanghai"; + static { TIME_ZONE = new SimpleTimeZone(8 * 3600 * 1000, ""); diff --git a/fe/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/src/main/java/org/apache/doris/qe/Coordinator.java index a05403f87ba0d4..8cded734b13dd8 100644 --- a/fe/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/src/main/java/org/apache/doris/qe/Coordinator.java @@ -29,6 +29,7 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.ListUtil; import org.apache.doris.common.util.RuntimeProfile; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.load.LoadErrorHub; import org.apache.doris.planner.DataPartition; import org.apache.doris.planner.DataSink; @@ -191,7 +192,7 @@ public Coordinator(ConnectContext context, Analyzer analyzer, Planner planner) { this.queryOptions = context.getSessionVariable().toThrift(); this.queryGlobals.setNow_string(String.valueOf(new Date().getTime())); if (context.getSessionVariable().getTimeZone().equals("CST")) { - this.queryGlobals.setTime_zone("Asia/Shanghai"); + this.queryGlobals.setTime_zone(TimeUtils.DEFAULT_TIME_ZONE); } else { this.queryGlobals.setTime_zone(context.getSessionVariable().getTimeZone()); } @@ -215,7 +216,7 @@ public Coordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, this.scanNodes = scanNodes; this.queryOptions = new TQueryOptions(); this.queryGlobals.setNow_string(String.valueOf(new Date().getTime())); - this.queryGlobals.setTime_zone("Asia/Shanghai"); + this.queryGlobals.setTime_zone(TimeUtils.DEFAULT_TIME_ZONE); this.tResourceInfo = new TResourceInfo("", ""); this.needReport = true; this.clusterName = cluster; From c6fcd5dab95e9afd0139dbb242e85d708a5c795e Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Fri, 9 Aug 2019 17:18:09 +0800 Subject: [PATCH 3/6] add time zone doc --- .../cn/administrator-guide/time-zone.md | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 docs/documentation/cn/administrator-guide/time-zone.md diff --git a/docs/documentation/cn/administrator-guide/time-zone.md b/docs/documentation/cn/administrator-guide/time-zone.md new file mode 100644 index 00000000000000..a4e42b0365bbea --- /dev/null +++ b/docs/documentation/cn/administrator-guide/time-zone.md @@ -0,0 +1,53 @@ +# 时区 + +Doris 支持多时区设置 + +## 名词解释 + +* FE:Frontend,Doris 的前端节点。负责元数据管理和请求接入。 +* BE:Backend,Doris 的后端节点。负责查询执行和数据存储。 + +## 基本概念 + +Doris 内部存在多个时区相关参数 + +* system_time_zone : + 当服务器启动时,会根据机器设置时区自动设置,设置后不可修改。 + +* time_zone : + 服务器当前时区,区分session级别和global级别 + +## 具体操作 + +1. show variables like '%time_zone%' + + 查看当前时区相关配置 + +2. SET time_zone = 'Asia/Shanghai' + + 该命令可以设置session级别的时区,连接断开后失效 + +3. SET global time_zone = 'Asia/Shanghai' + + 该命令可以设置global级别的时区参数,fe会将参数持久化,连接断开后不失效 + +### 时区的影响 + +时区设置会影响对时区敏感的时间值的显示和存储。 + +包括NOW()或CURTIME()等时间函数显示的值,也包括show load, show backends中的时间值。 + +但不会影响create table 中时间类型分区列的less than值,也不会影响存储为date/datetime类型的值的显示。 + +## 使用限制 + +时区值可以使用几种格式给出,不区分大小写: + +* 表示UTC偏移量的字符串,如'+10:00'或'-6:00' + +* 标准时区格式,如"Asia/Shanghai"、"America/Los_Angeles" + +* 不支持缩写时区格式,如"MET"、"CTT"。因为缩写时区在不同场景下存在歧义,不建议使用。 + +* 为了兼容Doris,支持CST缩写时区,内部会将CST转移为"Asia/Shanghai"的中国标准时区 + From fc0699331079bb80104849b2be3ae0af5658386d Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Mon, 12 Aug 2019 13:14:40 +0800 Subject: [PATCH 4/6] fix timediff --- be/src/exprs/timestamp_functions.cpp | 49 +++++++++++++++++-------- be/src/exprs/timezone_db.cpp | 4 +- be/src/exprs/timezone_db.h | 9 +---- be/src/runtime/datetime_value.cpp | 5 ++- be/src/runtime/datetime_value.h | 16 +++++--- be/src/runtime/runtime_state.h | 2 +- be/src/testutil/function_utils.cpp | 14 +++---- be/test/runtime/datetime_value_test.cpp | 20 ++++++---- 8 files changed, 71 insertions(+), 48 deletions(-) diff --git a/be/src/exprs/timestamp_functions.cpp b/be/src/exprs/timestamp_functions.cpp index a1bb5aa7f42143..9b90979dab7ed2 100644 --- a/be/src/exprs/timestamp_functions.cpp +++ b/be/src/exprs/timestamp_functions.cpp @@ -364,18 +364,15 @@ IntVal TimestampFunctions::to_days( return IntVal(ts_value.daynr()); } -// TODO(dhc): implement this funciton really DoubleVal TimestampFunctions::time_diff( FunctionContext* ctx, const DateTimeVal& ts_val1, const DateTimeVal& ts_val2) { if (ts_val1.is_null || ts_val2.is_null) { return DoubleVal::null(); } + const DateTimeValue& ts_value1 = DateTimeValue::from_datetime_val(ts_val1); const DateTimeValue& ts_value2 = DateTimeValue::from_datetime_val(ts_val2); - int64_t timediff = ts_value1.unix_timestamp(ctx->impl()->state()->timezone()) - - ts_value2.unix_timestamp(ctx->impl()->state()->timezone()); - - return DoubleVal(timediff); + return DoubleVal(ts_value1.second_diff(ts_value2)); } IntVal TimestampFunctions::date_diff( @@ -439,7 +436,13 @@ IntVal TimestampFunctions::to_unix( (const char *)fmt.ptr, fmt.len, (const char *)string_val.ptr, string_val.len)) { return IntVal::null(); } - return tv.unix_timestamp(context->impl()->state()->timezone()); + + int64_t timestamp; + if(!tv.unix_timestamp(×tamp, context->impl()->state()->timezone())) { + return IntVal::null(); + } else { + return IntVal(timestamp); + } } IntVal TimestampFunctions::to_unix( @@ -448,12 +451,20 @@ IntVal TimestampFunctions::to_unix( return IntVal::null(); } const DateTimeValue &tv = DateTimeValue::from_datetime_val(ts_val); - return tv.unix_timestamp(context->impl()->state()->timezone()); + + int64_t timestamp; + if(!tv.unix_timestamp(×tamp, context->impl()->state()->timezone())) { + return IntVal::null(); + } else { + return IntVal(timestamp); + } } DateTimeVal TimestampFunctions::utc_timestamp(FunctionContext* context) { DateTimeValue dtv; - dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, "+00:00"); + if (!dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, "+00:00")) { + return DateTimeVal::null(); + } DateTimeVal return_val; dtv.to_datetime_val(&return_val); @@ -462,8 +473,10 @@ DateTimeVal TimestampFunctions::utc_timestamp(FunctionContext* context) { DateTimeVal TimestampFunctions::now(FunctionContext* context) { DateTimeValue dtv; - dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, - context->impl()->state()->timezone()); + if (!dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, + context->impl()->state()->timezone())) { + return DateTimeVal::null(); + } DateTimeVal return_val; dtv.to_datetime_val(&return_val); @@ -472,8 +485,10 @@ DateTimeVal TimestampFunctions::now(FunctionContext* context) { DoubleVal TimestampFunctions::curtime(FunctionContext* context) { DateTimeValue dtv; - dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, - context->impl()->state()->timezone()); + if (!dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, + context->impl()->state()->timezone())) { + return DoubleVal::null(); + } return dtv.hour() * 3600 + dtv.minute() * 60 + dtv.second(); } @@ -486,10 +501,14 @@ DateTimeVal TimestampFunctions::convert_tz(FunctionContext* ctx, const DateTimeV return DateTimeVal::null(); } const DateTimeValue &ts_value = DateTimeValue::from_datetime_val(ts_val); - int timestamp = ts_value.unix_timestamp(std::string((char *)from_tz.ptr, from_tz.len)); - + int64_t timestamp; + if(!ts_value.unix_timestamp(×tamp, std::string((char *)from_tz.ptr, from_tz.len))) { + return DateTimeVal::null(); + } DateTimeValue ts_value2; - ts_value2.from_unixtime(timestamp, std::string((char *)to_tz.ptr, to_tz.len)); + if (!ts_value2.from_unixtime(timestamp, std::string((char *)to_tz.ptr, to_tz.len))) { + return DateTimeVal::null(); + } DateTimeVal return_val; ts_value2.to_datetime_val(&return_val); diff --git a/be/src/exprs/timezone_db.cpp b/be/src/exprs/timezone_db.cpp index 48e112c17fb1cb..448a0d18ad6a39 100644 --- a/be/src/exprs/timezone_db.cpp +++ b/be/src/exprs/timezone_db.cpp @@ -18,7 +18,7 @@ namespace doris { boost::local_time::tz_database TimezoneDatabase::_s_tz_database; -TimezoneDatabase::TimezoneDatabase() { +void TimezoneDatabase::init() { // Create a temporary file and write the timezone information. The boost // interface only loads this format from a file. We don't want to raise // an error here since this is done when the backend is created and this @@ -53,8 +53,6 @@ TimezoneDatabase::TimezoneDatabase() { close(fd); } -TimezoneDatabase::~TimezoneDatabase() {} - boost::local_time::time_zone_ptr TimezoneDatabase::find_timezone(const std::string &tz) { try { // See if they specified a zone id diff --git a/be/src/exprs/timezone_db.h b/be/src/exprs/timezone_db.h index e5319d684b3a82..d16d1d43b29430 100644 --- a/be/src/exprs/timezone_db.h +++ b/be/src/exprs/timezone_db.h @@ -34,14 +34,7 @@ namespace doris { class TimezoneDatabase { public: - TimezoneDatabase(); - - ~TimezoneDatabase(); - - static void init() { - TimezoneDatabase(); - } - + static void init(); static boost::local_time::time_zone_ptr find_timezone(const std::string &tz); static const std::string default_time_zone; private: diff --git a/be/src/runtime/datetime_value.cpp b/be/src/runtime/datetime_value.cpp index 24e1f0e4db6b0a..b11d7f8c0592af 100644 --- a/be/src/runtime/datetime_value.cpp +++ b/be/src/runtime/datetime_value.cpp @@ -1524,7 +1524,7 @@ bool DateTimeValue::date_add_interval(const TimeInterval& interval, TimeUnit uni return true; } -int64_t DateTimeValue::unix_timestamp(const std::string& timezone) const { +bool DateTimeValue::unix_timestamp(int64_t* timestamp, const std::string& timezone) const{ boost::local_time::time_zone_ptr local_time_zone = TimezoneDatabase::find_timezone(timezone); if (local_time_zone == nullptr) { return false; @@ -1539,7 +1539,8 @@ int64_t DateTimeValue::unix_timestamp(const std::string& timezone) const { boost::posix_time::ptime utc_ptime = lt.utc_time(); boost::posix_time::ptime utc_start(boost::gregorian::date(1970, 1, 1)); boost::posix_time::time_duration dur = utc_ptime - utc_start; - return dur.total_milliseconds() / 1000; + *timestamp = dur.total_milliseconds() / 1000; + return true; } bool DateTimeValue::from_unixtime(int64_t timestamp, const std::string& timezone) { diff --git a/be/src/runtime/datetime_value.h b/be/src/runtime/datetime_value.h index 46e3d46d333833..b6d04af4bd0e3d 100644 --- a/be/src/runtime/datetime_value.h +++ b/be/src/runtime/datetime_value.h @@ -332,9 +332,13 @@ class DateTimeValue { // Add interval bool date_add_interval(const TimeInterval& interval, TimeUnit unit); - - int64_t unix_timestamp(const std::string& timezone) const; - + + //unix_timestamp is called with a timezone argument, + //it returns seconds of the value of date literal since '1970-01-01 00:00:00' UTC + bool unix_timestamp(int64_t* timestamp, const std::string& timezone) const; + + //construct datetime_value from timestamp and timezone + //timestamp is an internal timestamp value representing seconds since '1970-01-01 00:00:00' UTC bool from_unixtime(int64_t, const std::string& timezone); bool operator==(const DateTimeValue& other) const { @@ -436,8 +440,10 @@ class DateTimeValue { } int64_t second_diff(const DateTimeValue& rhs) const { - return unix_timestamp(TimezoneDatabase::default_time_zone) - - rhs.unix_timestamp(TimezoneDatabase::default_time_zone); + int day_diff = daynr() - rhs.daynr(); + int time_diff = (hour() * 3600 + minute() * 60 + second()) + - (rhs.hour() * 3600 + rhs.minute() * 60 + rhs.second()); + return day_diff * 3600 * 24 + time_diff; } void set_type(int type); diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index f246b933cf55b8..64216e355602ee 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -139,7 +139,7 @@ class RuntimeState { int64_t timestamp() const { return _timestamp; } - const std::string timezone() const { + const std::string& timezone() const { return _timezone; } const std::string& user() const { diff --git a/be/src/testutil/function_utils.cpp b/be/src/testutil/function_utils.cpp index 94c1337b5131fd..28ac6c1d152f3f 100644 --- a/be/src/testutil/function_utils.cpp +++ b/be/src/testutil/function_utils.cpp @@ -35,13 +35,13 @@ FunctionUtils::FunctionUtils() { _state, _memory_pool, return_type, arg_types, 0, false); } FunctionUtils::FunctionUtils(RuntimeState* state) { - _state = state; - doris_udf::FunctionContext::TypeDesc return_type; - std::vector arg_types; - _mem_tracker = new MemTracker(); - _memory_pool = new MemPool(_mem_tracker); - _fn_ctx = FunctionContextImpl::create_context( - _state, _memory_pool, return_type, arg_types, 0, false); + _state = state; + doris_udf::FunctionContext::TypeDesc return_type; + std::vector arg_types; + _mem_tracker = new MemTracker(); + _memory_pool = new MemPool(_mem_tracker); + _fn_ctx = FunctionContextImpl::create_context( + _state, _memory_pool, return_type, arg_types, 0, false); } FunctionUtils::~FunctionUtils() { diff --git a/be/test/runtime/datetime_value_test.cpp b/be/test/runtime/datetime_value_test.cpp index 5155bb62d2abf4..83effa66dfdaef 100644 --- a/be/test/runtime/datetime_value_test.cpp +++ b/be/test/runtime/datetime_value_test.cpp @@ -303,19 +303,25 @@ TEST_F(DateTimeValueTest, from_unixtime) { // Calculate format TEST_F(DateTimeValueTest, unix_timestamp) { DateTimeValue value; - std::cout << TimezoneDatabase::default_time_zone << std::endl; + int64_t timestamp; value.from_date_int64(19691231); - ASSERT_EQ(-115200, value.unix_timestamp(TimezoneDatabase::default_time_zone)); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(-115200, timestamp); value.from_date_int64(19700101); - ASSERT_EQ(-28800, value.unix_timestamp(TimezoneDatabase::default_time_zone)); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(-28800, timestamp); value.from_date_int64(19700102); - ASSERT_EQ(86400 - 28800, value.unix_timestamp(TimezoneDatabase::default_time_zone)); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(86400 - 28800, timestamp); value.from_date_int64(19880201000000); - ASSERT_EQ(570672000 - 28800, value.unix_timestamp(TimezoneDatabase::default_time_zone)); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(570672000 - 28800, timestamp); value.from_date_int64(20380119); - ASSERT_EQ(2147472000 - 28800, value.unix_timestamp(TimezoneDatabase::default_time_zone)); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(2147472000 - 28800, timestamp); value.from_date_int64(20380120); - ASSERT_EQ(2147529600, value.unix_timestamp(TimezoneDatabase::default_time_zone)); + value.unix_timestamp(×tamp, TimezoneDatabase::default_time_zone); + ASSERT_EQ(2147529600, timestamp); } // Calculate format From 3fbd5bc27f002afdf762d845d204326012e17841 Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Tue, 13 Aug 2019 12:03:50 +0800 Subject: [PATCH 5/6] fix-time-zone-compatibility --- be/src/exprs/timestamp_functions.cpp | 8 +++---- be/src/runtime/runtime_state.cpp | 22 +++++++++++++++++-- be/src/runtime/runtime_state.h | 8 +++---- be/test/exprs/timestamp_functions_test.cpp | 3 ++- .../java/org/apache/doris/qe/Coordinator.java | 10 +++++++-- gensrc/thrift/PaloInternalService.thrift | 10 ++++++++- 6 files changed, 47 insertions(+), 14 deletions(-) diff --git a/be/src/exprs/timestamp_functions.cpp b/be/src/exprs/timestamp_functions.cpp index 9b90979dab7ed2..2b62c3258a8d67 100644 --- a/be/src/exprs/timestamp_functions.cpp +++ b/be/src/exprs/timestamp_functions.cpp @@ -423,7 +423,7 @@ StringVal TimestampFunctions::from_unix( } IntVal TimestampFunctions::to_unix(FunctionContext* context) { - return IntVal(context->impl()->state()->timestamp() / 1000); + return IntVal(context->impl()->state()->timestamp_ms() / 1000); } IntVal TimestampFunctions::to_unix( @@ -462,7 +462,7 @@ IntVal TimestampFunctions::to_unix( DateTimeVal TimestampFunctions::utc_timestamp(FunctionContext* context) { DateTimeValue dtv; - if (!dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, "+00:00")) { + if (!dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000, "+00:00")) { return DateTimeVal::null(); } @@ -473,7 +473,7 @@ DateTimeVal TimestampFunctions::utc_timestamp(FunctionContext* context) { DateTimeVal TimestampFunctions::now(FunctionContext* context) { DateTimeValue dtv; - if (!dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, + if (!dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000, context->impl()->state()->timezone())) { return DateTimeVal::null(); } @@ -485,7 +485,7 @@ DateTimeVal TimestampFunctions::now(FunctionContext* context) { DoubleVal TimestampFunctions::curtime(FunctionContext* context) { DateTimeValue dtv; - if (!dtv.from_unixtime(context->impl()->state()->timestamp() / 1000, + if (!dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000, context->impl()->state()->timezone())) { return DoubleVal::null(); } diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index 172d80193a6857..35b96dd5f40540 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -107,11 +107,20 @@ RuntimeState::RuntimeState(const TQueryGlobals& query_globals) _profile(_obj_pool.get(), ""), _per_fragment_instance_idx(0) { _query_options.batch_size = DEFAULT_BATCH_SIZE; - _timestamp = atol(query_globals.now_string.c_str()); if (query_globals.__isset.time_zone) { _timezone = query_globals.time_zone; + _timestamp_ms = query_globals.timestamp_ms; + } else if (!query_globals.now_string.empty()) { + _timezone = TimezoneDatabase::default_time_zone; + DateTimeValue dt; + dt.from_date_str(query_globals.now_string.c_str(), query_globals.now_string.size()); + int64_t timestamp; + dt.unix_timestamp(×tamp, _timezone); + _timestamp_ms = timestamp * 1000; } else { + //Unit test may set into here _timezone = TimezoneDatabase::default_time_zone; + _timestamp_ms = 0; } } @@ -169,11 +178,20 @@ Status RuntimeState::init( const TQueryGlobals& query_globals, ExecEnv* exec_env) { _fragment_instance_id = fragment_instance_id; _query_options = query_options; - _timestamp = atol(query_globals.now_string.c_str()); if (query_globals.__isset.time_zone) { _timezone = query_globals.time_zone; + _timestamp_ms = query_globals.timestamp_ms; + } else if (!query_globals.now_string.empty()) { + _timezone = TimezoneDatabase::default_time_zone; + DateTimeValue dt; + dt.from_date_str(query_globals.now_string.c_str(), query_globals.now_string.size()); + int64_t timestamp; + dt.unix_timestamp(×tamp, _timezone); + _timestamp_ms = timestamp * 1000; } else { + //Unit test may set into here _timezone = TimezoneDatabase::default_time_zone; + _timestamp_ms = 0; } _exec_env = exec_env; diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 64216e355602ee..ac5e2650d18136 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -136,8 +136,8 @@ class RuntimeState { int num_scanner_threads() const { return _query_options.num_scanner_threads; } - int64_t timestamp() const { - return _timestamp; + int64_t timestamp_ms() const { + return _timestamp_ms; } const std::string& timezone() const { return _timezone; @@ -536,8 +536,8 @@ class RuntimeState { // Username of user that is executing the query to which this RuntimeState belongs. std::string _user; - //Query-global timestamp - int64_t _timestamp; + //Query-global timestamp_ms + int64_t _timestamp_ms; std::string _timezone; TUniqueId _query_id; diff --git a/be/test/exprs/timestamp_functions_test.cpp b/be/test/exprs/timestamp_functions_test.cpp index 4fc33ccbe9e0f6..f1ae1ce2924e76 100644 --- a/be/test/exprs/timestamp_functions_test.cpp +++ b/be/test/exprs/timestamp_functions_test.cpp @@ -40,7 +40,8 @@ class TimestampFunctionsTest : public testing::Test { TimezoneDatabase::init(); TQueryGlobals globals; - globals.__set_now_string("1565080737805"); + globals.__set_now_string("2019-08-06 01:38:57"); + globals.__set_timestamp_ms(1565080737805); globals.__set_time_zone("America/Los_Angeles"); state = new RuntimeState(globals); utils = new FunctionUtils(state); diff --git a/fe/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/src/main/java/org/apache/doris/qe/Coordinator.java index 8cded734b13dd8..9d072e1180ee18 100644 --- a/fe/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/src/main/java/org/apache/doris/qe/Coordinator.java @@ -85,6 +85,8 @@ import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -105,6 +107,8 @@ public class Coordinator { private static final Logger LOG = LogManager.getLogger(Coordinator.class); + private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private static String localIP = FrontendOptions.getLocalHostAddress(); // Overall status of the entire query; set to the first reported fragment error @@ -190,7 +194,8 @@ public Coordinator(ConnectContext context, Analyzer analyzer, Planner planner) { this.descTable = analyzer.getDescTbl().toThrift(); this.returnedAllResults = false; this.queryOptions = context.getSessionVariable().toThrift(); - this.queryGlobals.setNow_string(String.valueOf(new Date().getTime())); + this.queryGlobals.setNow_string(DATE_FORMAT.format(new Date())); + this.queryGlobals.setTimestamp_ms(new Date().getTime()); if (context.getSessionVariable().getTimeZone().equals("CST")) { this.queryGlobals.setTime_zone(TimeUtils.DEFAULT_TIME_ZONE); } else { @@ -215,7 +220,8 @@ public Coordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, this.fragments = fragments; this.scanNodes = scanNodes; this.queryOptions = new TQueryOptions(); - this.queryGlobals.setNow_string(String.valueOf(new Date().getTime())); + this.queryGlobals.setNow_string(DATE_FORMAT.format(new Date())); + this.queryGlobals.setTimestamp_ms(new Date().getTime()); this.queryGlobals.setTime_zone(TimeUtils.DEFAULT_TIME_ZONE); this.tResourceInfo = new TResourceInfo("", ""); this.needReport = true; diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index f8d373e5a3085d..4aa70aae063401 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -181,8 +181,16 @@ struct TPlanFragmentExecParams { // Global query parameters assigned by the coordinator. struct TQueryGlobals { // String containing a timestamp set as the current time. + // Format is yyyy-MM-dd HH:mm:ss 1: required string now_string - 2: optional string time_zone + + // To support timezone in Doris. timestamp_ms is the millisecond uinix timestamp for + // this query to calculate time zone relative function + 2: optional i64 timestamp_ms + + // time_zone is the timezone this query used. + // If this value is set, BE will ignore now_string + 3: optional string time_zone } From 9aa37ca9c978325669526c61166e5b1cfb156d9e Mon Sep 17 00:00:00 2001 From: HangyuanLiu <460660596@qq.com> Date: Tue, 13 Aug 2019 16:24:05 +0800 Subject: [PATCH 6/6] fix doc to utf-8 --- .../cn/administrator-guide/load-data/broker-load-manual.md | 2 +- .../sql-reference/sql-functions/date-time-functions/curtime.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/documentation/cn/administrator-guide/load-data/broker-load-manual.md b/docs/documentation/cn/administrator-guide/load-data/broker-load-manual.md index bdcb4026f947f1..1a28ed0efb10cc 100644 --- a/docs/documentation/cn/administrator-guide/load-data/broker-load-manual.md +++ b/docs/documentation/cn/administrator-guide/load-data/broker-load-manual.md @@ -95,7 +95,7 @@ LOAD LABEL db1.label1 SET ( id=tmp_c2, - name=tmp_c1) + name=tmp_c1 ), DATA INFILE("hdfs://abc.com:8888/user/palo/test/ml/file2") INTO TABLE tbl2 diff --git a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curtime.md b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curtime.md index 83b9b05f3a6fc8..e3b8157617e884 100644 --- a/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curtime.md +++ b/docs/documentation/cn/sql-reference/sql-functions/date-time-functions/curtime.md @@ -6,7 +6,7 @@ ## Description -õǰʱ䣬TIMEͷ +获得当前的时间,以TIME类型返回 ## Examples