diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 745ceccfd3c34c..0c0ae3ae7eba54 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -58,6 +58,7 @@ endif() set(LLVM_BIN "${LLVM_HOME}/bin") option(MAKE_TEST "ON for make unit test or OFF for not" OFF) +option(WITH_MYSQL "Support access MySQL" ON) # Check gcc if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") @@ -154,8 +155,10 @@ set_target_properties(thriftnb PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/li add_library(lzo STATIC IMPORTED) set_target_properties(lzo PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/liblzo2.a) -add_library(mysql STATIC IMPORTED) -set_target_properties(mysql PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libmysqlclient.a) +if (WITH_MYSQL) + add_library(mysql STATIC IMPORTED) + set_target_properties(mysql PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libmysqlclient.a) +endif() add_library(libevent STATIC IMPORTED) set_target_properties(libevent PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libevent.a) @@ -274,6 +277,10 @@ set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -msse4.2") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DLLVM_ON_UNIX") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-attributes -DS2_USE_GFLAGS -DS2_USE_GLOG") +if (WITH_MYSQL) + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DDORIS_WITH_MYSQL") +endif() + if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -faligned-new") endif() @@ -443,9 +450,15 @@ set(DORIS_LINK_LIBS ${WL_END_GROUP} ) +if (WITH_MYSQL) + set(DORIS_DEPENDENCIES + mysql + ) +endif() + # Set thirdparty libraries set(DORIS_DEPENDENCIES - mysql + ${DORIS_DEPENDENCIES} ${WL_START_GROUP} rocksdb librdkafka_cpp diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index 554ecc13afdbac..387a61550ccdb6 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -58,8 +58,6 @@ set(EXEC_FILES olap_table_info.cpp olap_table_sink.cpp plain_text_line_reader.cpp - mysql_scan_node.cpp - mysql_scanner.cpp csv_scan_node.cpp csv_scanner.cpp es_scan_node.cpp @@ -95,6 +93,14 @@ set(EXEC_FILES broker_writer.cpp ) +if (WITH_MYSQL) + set(EXEC_FILES + ${EXEC_FILES} + mysql_scan_node.cpp + mysql_scanner.cpp + ) +endif() + if(EXISTS "${BASE_DIR}/src/exec/kudu_util.cpp") set(EXEC_FILES ${EXEC_FILES} #kudu_scan_node.cpp @@ -106,30 +112,3 @@ endif() add_library(Exec STATIC ${EXEC_FILES} ) - -# TODO: why is this test disabled? -#ADD_BE_TEST(es/es_query_builder_test) -#ADD_BE_TEST(es/es_scan_reader_test) -#ADD_BE_TEST(new_olap_scan_node_test) -#ADD_BE_TEST(pre_aggregation_node_test) -#ADD_BE_TEST(hash_table_test) -#ADD_BE_TEST(olap_scanner_test) -#ADD_BE_TEST(olap_meta_reader_test) -#ADD_BE_TEST(olap_common_test) -#ADD_BE_TEST(olap_scan_node_test) -#ADD_BE_TEST(mysql_scan_node_test) -#ADD_BE_TEST(mysql_scanner_test) -#ADD_BE_TEST(schema_scan_node_test) -#ADD_BE_TEST(schema_scanner_test) -##ADD_BE_TEST(set_executor_test) -#ADD_BE_TEST(schema_scanner/schema_authors_scanner_test) -#ADD_BE_TEST(schema_scanner/schema_columns_scanner_test) -#ADD_BE_TEST(schema_scanner/schema_create_table_scanner_test) -#ADD_BE_TEST(schema_scanner/schema_open_tables_scanner_test) -#ADD_BE_TEST(schema_scanner/schema_schemata_scanner_test) -#ADD_BE_TEST(schema_scanner/schema_table_names_scanner_test) -#ADD_BE_TEST(schema_scanner/schema_tables_scanner_test) -#ADD_BE_TEST(schema_scanner/schema_variables_scanner_test) -#ADD_BE_TEST(schema_scanner/schema_engines_scanner_test) -#ADD_BE_TEST(schema_scanner/schema_collations_scanner_test) -#ADD_BE_TEST(schema_scanner/schema_charsets_scanner_test) diff --git a/be/src/exec/data_sink.cpp b/be/src/exec/data_sink.cpp index a470702e397206..228f5afc4ad100 100644 --- a/be/src/exec/data_sink.cpp +++ b/be/src/exec/data_sink.cpp @@ -71,6 +71,7 @@ Status DataSink::create_data_sink( break; case TDataSinkType::MYSQL_TABLE_SINK: { +#ifdef DORIS_WITH_MYSQL if (!thrift_sink.__isset.mysql_table_sink) { return Status("Missing data buffer sink."); } @@ -80,6 +81,9 @@ Status DataSink::create_data_sink( pool, row_desc, output_exprs); sink->reset(mysql_tbl_sink); break; +#else + return Status("Don't support MySQL table, you should rebuild Doris with WITH_MYSQL option ON"); +#endif } case TDataSinkType::DATA_SPLIT_SINK: { diff --git a/be/src/exec/exec_node.cpp b/be/src/exec/exec_node.cpp index c934cf5fff20f7..ce7533aa49a083 100644 --- a/be/src/exec/exec_node.cpp +++ b/be/src/exec/exec_node.cpp @@ -360,8 +360,12 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN return Status::OK; case TPlanNodeType::MYSQL_SCAN_NODE: +#ifdef DORIS_WITH_MYSQL *node = pool->add(new MysqlScanNode(pool, tnode, descs)); return Status::OK; +#else + return Status("Don't support MySQL table, you should rebuild Doris with WITH_MYSQL option ON"); +#endif case TPlanNodeType::ES_SCAN_NODE: *node = pool->add(new EsScanNode(pool, tnode, descs)); diff --git a/be/src/exec/mysql_scanner.cpp b/be/src/exec/mysql_scanner.cpp index 8903b119ccdf7c..4aaf65af847253 100644 --- a/be/src/exec/mysql_scanner.cpp +++ b/be/src/exec/mysql_scanner.cpp @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +#include + +#define __DorisMysql MYSQL +#define __DorisMysqlRes MYSQL_RES #include "mysql_scanner.h" @@ -170,6 +174,13 @@ Status MysqlScanner::get_next_row(char** *buf, unsigned long** lengths, bool* eo return Status::OK; } +Status MysqlScanner::_error_status(const std::string& prefix) { + std::stringstream msg; + msg << prefix << " Err: " << mysql_error(_my_conn); + LOG(INFO) << msg.str(); + return Status(msg.str()); +} + } /* vim: set ts=4 sw=4 sts=4 tw=100 noet: */ diff --git a/be/src/exec/mysql_scanner.h b/be/src/exec/mysql_scanner.h index 662301638dae48..3bda992b0019f2 100644 --- a/be/src/exec/mysql_scanner.h +++ b/be/src/exec/mysql_scanner.h @@ -21,10 +21,18 @@ #include #include #include -#include #include "common/status.h" + +#ifndef __DorisMysql +#define __DorisMysql void +#endif + +#ifndef __DorisMysqlRes +#define __DorisMysqlRes void +#endif + namespace doris { struct MysqlScannerParam { @@ -55,16 +63,11 @@ class MysqlScanner { return _field_num; } private: - Status _error_status(const std::string& prefix) { - std::stringstream msg; - msg << prefix << " Err: " << mysql_error(_my_conn); - LOG(WARNING) << msg.str(); - return Status(msg.str()); - } + Status _error_status(const std::string& prefix); const MysqlScannerParam& _my_param; - MYSQL* _my_conn; - MYSQL_RES* _my_result; + __DorisMysql* _my_conn; + __DorisMysqlRes* _my_result; std::string _sql_str; bool _is_open; int _field_num; diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt index 7f19dcd0c7725b..22e8c5c0aad556 100644 --- a/be/src/runtime/CMakeLists.txt +++ b/be/src/runtime/CMakeLists.txt @@ -23,7 +23,7 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/runtime") # where to put generated binaries set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/runtime") -add_library(Runtime STATIC +set(RUNTIME_FILES broker_mgr.cpp buffered_block_mgr.cpp buffered_tuple_stream.cpp @@ -66,8 +66,6 @@ add_library(Runtime STATIC dpp_sink.cpp etl_job_mgr.cpp load_path_mgr.cpp - mysql_table_writer.cpp - mysql_table_sink.cpp types.cpp tmp_file_mgr.cc disk_io_mgr.cc @@ -103,26 +101,13 @@ add_library(Runtime STATIC routine_load/routine_load_task_executor.cpp ) -# This test runs forever so should not be part of 'make test' -#add_executable(disk_io_mgr_stress_test disk_io_mgr_stress_test.cpp) -#target_link_libraries(disk_io_mgr_stress_test ${IMPALA_TEST_LINK_LIBS}) +if (WITH_MYSQL) + set(RUNTIME_FILES ${RUNTIME_FILES} + mysql_table_writer.cpp + mysql_table_sink.cpp + ) +endif() -#ADD_BE_TEST(sorter_test) -#ADD_BE_TEST(buffered_tuple_stream_test) -#ADD_BE_TEST(result_writer_test) -#ADD_BE_TEST(buffer_control_block_test) -#ADD_BE_TEST(result_buffer_mgr_test) -#ADD_BE_TEST(result_receiver_test) -#ADD_BE_TEST(result_sink_test) -#ADD_BE_TEST(mem_pool_test) -#ADD_BE_TEST(free_list_test) -#ADD_BE_TEST(string_buffer_test) -#ADD_BE_TEST(data_stream_test) -#ADD_BE_TEST(timestamp_test) -#ADD_BE_TEST(disk_io_mgr_test) -#ADD_BE_TEST(parallel_executor_test) -#ADD_BE_TEST(datetime_value_test) -#ADD_BE_TEST(decimal_value_test) -#ADD_BE_TEST(decimalv2_value_test) -#ADD_BE_TEST(string_value_test) -#ADD_BE_TEST(thread_resource_mgr_test) +add_library(Runtime STATIC + ${RUNTIME_FILES} + ) diff --git a/be/src/runtime/decimal_value.cpp b/be/src/runtime/decimal_value.cpp index 8e6522d5bed3a9..0a7ba775007ffc 100755 --- a/be/src/runtime/decimal_value.cpp +++ b/be/src/runtime/decimal_value.cpp @@ -17,7 +17,6 @@ #include "runtime/decimal_value.h" -#include #include #include #include diff --git a/be/src/runtime/mysql_table_sink.cpp b/be/src/runtime/mysql_table_sink.cpp index b239941d895358..031b315b414375 100644 --- a/be/src/runtime/mysql_table_sink.cpp +++ b/be/src/runtime/mysql_table_sink.cpp @@ -39,7 +39,7 @@ MysqlTableSink::~MysqlTableSink() { } Status MysqlTableSink::init(const TDataSink& t_sink) { - RETURN_IF_ERROR(MysqlTableSink::init(t_sink)); + RETURN_IF_ERROR(DataSink::init(t_sink)); const TMysqlTableSink& t_mysql_sink = t_sink.mysql_table_sink; _conn_info.host = t_mysql_sink.host; diff --git a/be/src/runtime/mysql_table_writer.cpp b/be/src/runtime/mysql_table_writer.cpp index 467ca0006c4780..46c64aaf33c522 100644 --- a/be/src/runtime/mysql_table_writer.cpp +++ b/be/src/runtime/mysql_table_writer.cpp @@ -15,9 +15,13 @@ // specific language governing permissions and limitations // under the License. +#include + +#define __DorisMysql MYSQL #include "runtime/mysql_table_writer.h" #include + #include "runtime/row_batch.h" #include "runtime/tuple_row.h" #include "exprs/expr.h" diff --git a/be/src/runtime/mysql_table_writer.h b/be/src/runtime/mysql_table_writer.h index 0d8a4b18882225..ace1e87df4bdc9 100644 --- a/be/src/runtime/mysql_table_writer.h +++ b/be/src/runtime/mysql_table_writer.h @@ -20,10 +20,13 @@ #include #include -#include #include "common/status.h" +#ifndef __DorisMysql +#define __DorisMysql void +#endif + namespace doris { struct MysqlConnInfo { @@ -67,7 +70,7 @@ class MysqlTableWriter { const std::vector& _output_expr_ctxs; std::string _mysql_tbl; - MYSQL* _mysql_conn; + __DorisMysql* _mysql_conn; }; } diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index f28429f266d0cb..84acdb4481ab80 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -40,7 +40,7 @@ #include "util/disk_info.h" #include "util/file_utils.h" #include "util/pretty_printer.h" -#include "util/mysql_load_error_hub.h" +#include "util/load_error_hub.h" #include "runtime/mem_tracker.h" #include "runtime/bufferpool/reservation_tracker.h" diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt index 541a178c86d927..79dad23b50a692 100644 --- a/be/src/util/CMakeLists.txt +++ b/be/src/util/CMakeLists.txt @@ -21,7 +21,7 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/util") # where to put generated binaries set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/util") -add_library(Util STATIC +set(UTIL_FILES arena.cpp bfd_parser.cpp bitmap.cpp @@ -62,7 +62,6 @@ add_library(Util STATIC spinlock.cc filesystem_util.cc load_error_hub.cpp - mysql_load_error_hub.cpp broker_load_error_hub.cpp null_load_error_hub.cpp time.cpp @@ -77,15 +76,13 @@ add_library(Util STATIC frontend_helper.cpp ) -#ADD_BE_TEST(integer-array-test) -#ADD_BE_TEST(runtime-profile-test) -#ADD_BE_TEST(benchmark-test) -#ADD_BE_TEST(decompress-test) -#ADD_BE_TEST(metrics-test) -#ADD_BE_TEST(debug-util-test) -#ADD_BE_TEST(url-coding-test) -#ADD_BE_TEST(thrift-util-test) -#ADD_BE_TEST(bit-util-test) -#ADD_BE_TEST(rle-test) -##ADD_BE_TEST(perf-counters-test) -##ADD_BE_TEST(es-scan-reader-test) +if (WITH_MYSQL) + set(UTIL_FILES ${UTIL_FILES} + mysql_load_error_hub.cpp + ) +endif() + +add_library(Util STATIC + ${UTIL_FILES} +) + diff --git a/be/src/util/load_error_hub.cpp b/be/src/util/load_error_hub.cpp index 36e4a05cba2a09..671ab8cf6607d0 100644 --- a/be/src/util/load_error_hub.cpp +++ b/be/src/util/load_error_hub.cpp @@ -43,10 +43,14 @@ Status LoadErrorHub::create_hub( switch (t_hub_info->type) { case TErrorHubType::MYSQL: +#ifdef DORIS_WITH_MYSQL tmp_hub = new MysqlLoadErrorHub(t_hub_info->mysql_info); tmp_hub->prepare(); hub->reset(tmp_hub); break; +#else + return Status("Don't support MySQL table, you should rebuild Doris with WITH_MYSQL option ON"); +#endif case TErrorHubType::BROKER: { // the origin file name may contains __shard_0/xxx // replace the '/' with '_' diff --git a/be/src/util/mysql_load_error_hub.cpp b/be/src/util/mysql_load_error_hub.cpp index cc2228fcdcc16a..1d13bbeca15e79 100644 --- a/be/src/util/mysql_load_error_hub.cpp +++ b/be/src/util/mysql_load_error_hub.cpp @@ -15,6 +15,9 @@ // specific language governing permissions and limitations // under the License. +#include + +#define __DorisMysql MYSQL #include "mysql_load_error_hub.h" #include "util/defer_op.h" diff --git a/be/src/util/mysql_load_error_hub.h b/be/src/util/mysql_load_error_hub.h index b61611c4972a20..9ffbcf688b3f84 100644 --- a/be/src/util/mysql_load_error_hub.h +++ b/be/src/util/mysql_load_error_hub.h @@ -23,11 +23,13 @@ #include #include -#include - #include "util/load_error_hub.h" #include "gen_cpp/PaloInternalService_types.h" +#ifndef __DorisMysql +#define __DorisMysql void +#endif + namespace doris { // For now every load job has its own mysql connection, @@ -69,15 +71,15 @@ class MysqlLoadErrorHub : public LoadErrorHub { virtual std::string debug_string() const; private: - Status open_mysql_conn(MYSQL** my_conn); + Status open_mysql_conn(__DorisMysql** my_conn); Status write_mysql(); - Status gen_sql(MYSQL* my_conn, + Status gen_sql(__DorisMysql* my_conn, const LoadErrorHub::ErrorMsg& error_msg, std::stringstream* sql_stream); - Status error_status(const std::string& prefix, MYSQL* my_conn); + Status error_status(const std::string& prefix, __DorisMysql* my_conn); MysqlInfo _info; diff --git a/be/src/util/null_load_error_hub.h b/be/src/util/null_load_error_hub.h index 85e95fcff48be5..4cd4852203f913 100644 --- a/be/src/util/null_load_error_hub.h +++ b/be/src/util/null_load_error_hub.h @@ -23,8 +23,6 @@ #include #include -#include - #include "load_error_hub.h" namespace doris { diff --git a/build.sh b/build.sh index 743d3f99bc28d5..fc8575d208acb2 100755 --- a/build.sh +++ b/build.sh @@ -48,15 +48,18 @@ usage() { echo " Usage: $0 Optional options: - --be build Backend - --fe build Frontend - --clean clean and build target + --be build Backend + --fe build Frontend + --clean clean and build target + --with-mysql enable MySQL support + --without-mysql disable MySQL support Eg. - $0 build Backend and Frontend without clean - $0 --be build Backend without clean - $0 --fe --clean clean and build Frontend - $0 --fe --be --clean clean and build both Frontend and Backend + $0 build Backend and Frontend without clean + $0 --be build Backend without clean + $0 --be --without-mysql build Backend with MySQL disable + $0 --fe --clean clean and build Frontend + $0 --fe --be --clean clean and build both Frontend and Backend " exit 1 } @@ -64,9 +67,13 @@ Usage: $0 OPTS=$(getopt \ -n $0 \ -o '' \ + -o 'h' \ -l 'be' \ -l 'fe' \ -l 'clean' \ + -l 'with-mysql' \ + -l 'without-mysql' \ + -l 'help' \ -- "$@") if [ $? != 0 ] ; then @@ -79,6 +86,8 @@ BUILD_BE= BUILD_FE= CLEAN= RUN_UT= +WITH_MYSQL=ON +HELP=0 if [ $# == 1 ] ; then # defuat BUILD_BE=1 @@ -96,12 +105,21 @@ else --fe) BUILD_FE=1 ; shift ;; --clean) CLEAN=1 ; shift ;; --ut) RUN_UT=1 ; shift ;; + --with-mysql) WITH_MYSQL=ON; shift ;; + --without-mysql) WITH_MYSQL=OFF; shift ;; + -h) HELP=1; shift ;; + --help) HELP=1; shift ;; --) shift ; break ;; *) ehco "Internal error" ; exit 1 ;; esac done fi +if [[ ${HELP} -eq 1 ]]; then + usage + exit +fi + if [ ${CLEAN} -eq 1 -a ${BUILD_BE} -eq 0 -a ${BUILD_FE} -eq 0 ]; then echo "--clean can not be specified without --fe or --be" exit 1 @@ -132,7 +150,7 @@ if [ ${BUILD_BE} -eq 1 ] ; then fi mkdir -p ${DORIS_HOME}/be/build/ cd ${DORIS_HOME}/be/build/ - cmake ../ + cmake -DWITH_MYSQL=${WITH_MYSQL} ../ make -j${PARALLEL} make install cd ${DORIS_HOME} diff --git a/run-ut.sh b/run-ut.sh index 28c61aa48027f6..8f32a4a071aa9a 100755 --- a/run-ut.sh +++ b/run-ut.sh @@ -89,7 +89,7 @@ fi cd ${DORIS_HOME}/be/build/ -cmake ../ -DMAKE_TEST=ON +cmake ../ -DWITH_MYSQL=OFF -DMAKE_TEST=ON make -j${PARALLEL} if [ ${RUN} -ne 1 ]; then