diff --git a/.gitignore b/.gitignore index c6229863aaccce..ad81f3bec807b6 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,4 @@ tools/single-node-cluster/fe* data_test /conf/log4j2-spring.xml +/fe/fe-core/src/test/resources/real-help-resource.zip diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 733eb7a001f195..04c3ed9f8c2a83 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1647,7 +1647,7 @@ Status TaskWorkerPool::_move_dir(const TTabletId tablet_id, const std::string& s return loader.move(src, tablet, overwrite); } -void TaskWorkerPool::_handle_report(TReportRequest& request, ReportType type) { +void TaskWorkerPool::_handle_report(const TReportRequest& request, ReportType type) { TMasterResult result; Status status = MasterServerClient::instance()->report(request, &result); bool is_report_success = false; diff --git a/be/src/agent/task_worker_pool.h b/be/src/agent/task_worker_pool.h index d583a02495b071..3ea664014e24d9 100644 --- a/be/src/agent/task_worker_pool.h +++ b/be/src/agent/task_worker_pool.h @@ -203,7 +203,7 @@ class TaskWorkerPool { void _alter_tablet(const TAgentTaskRequest& alter_tablet_request, int64_t signature, const TTaskType::type task_type, TFinishTaskRequest* finish_task_request); - void _handle_report(TReportRequest& request, ReportType type); + void _handle_report(const TReportRequest& request, ReportType type); Status _get_tablet_info(const TTabletId tablet_id, const TSchemaHash schema_hash, int64_t signature, TTabletInfo* tablet_info); diff --git a/be/src/gen_cpp/CMakeLists.txt b/be/src/gen_cpp/CMakeLists.txt index e9bc3cd79343fb..0b4d42da5a756f 100644 --- a/be/src/gen_cpp/CMakeLists.txt +++ b/be/src/gen_cpp/CMakeLists.txt @@ -19,78 +19,13 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/gen_cpp") set (GEN_CPP_DIR ${GENSRC_DIR}/gen_cpp) -set(SRC_FILES - ${GEN_CPP_DIR}/AgentService_constants.cpp - ${GEN_CPP_DIR}/AgentService_types.cpp - ${GEN_CPP_DIR}/BackendService_constants.cpp - ${GEN_CPP_DIR}/BackendService.cpp - ${GEN_CPP_DIR}/BackendService_types.cpp - ${GEN_CPP_DIR}/PaloBrokerService_types.cpp - ${GEN_CPP_DIR}/TDorisExternalService.cpp - ${GEN_CPP_DIR}/DorisExternalService_types.cpp - ${GEN_CPP_DIR}/DorisExternalService_constants.cpp - ${GEN_CPP_DIR}/QueryPlanExtra_types.cpp - ${GEN_CPP_DIR}/QueryPlanExtra_constants.cpp - ${GEN_CPP_DIR}/TPaloBrokerService.cpp - ${GEN_CPP_DIR}/HeartbeatService_constants.cpp - ${GEN_CPP_DIR}/HeartbeatService.cpp - ${GEN_CPP_DIR}/HeartbeatService_types.cpp - ${GEN_CPP_DIR}/PaloInternalService_constants.cpp - ${GEN_CPP_DIR}/PaloInternalService_types.cpp - ${GEN_CPP_DIR}/FrontendService.cpp - ${GEN_CPP_DIR}/FrontendService_constants.cpp - ${GEN_CPP_DIR}/FrontendService_types.cpp - ${GEN_CPP_DIR}/PaloService_constants.cpp - ${GEN_CPP_DIR}/PaloService_types.cpp - ${GEN_CPP_DIR}/Data_constants.cpp - ${GEN_CPP_DIR}/Data_types.cpp - ${GEN_CPP_DIR}/DataSinks_constants.cpp - ${GEN_CPP_DIR}/DataSinks_types.cpp - ${GEN_CPP_DIR}/Ddl_constants.cpp - ${GEN_CPP_DIR}/Ddl_types.cpp - ${GEN_CPP_DIR}/Descriptors_constants.cpp - ${GEN_CPP_DIR}/Descriptors_types.cpp - ${GEN_CPP_DIR}/Exprs_constants.cpp - ${GEN_CPP_DIR}/Exprs_types.cpp - ${GEN_CPP_DIR}/MasterService_constants.cpp - ${GEN_CPP_DIR}/MasterService_types.cpp - ${GEN_CPP_DIR}/MetricDefs_constants.cpp - ${GEN_CPP_DIR}/MetricDefs_types.cpp - ${GEN_CPP_DIR}/Metrics_constants.cpp - ${GEN_CPP_DIR}/Metrics_types.cpp - ${GEN_CPP_DIR}/NetworkTest_constants.cpp - ${GEN_CPP_DIR}/NetworkTest_types.cpp - ${GEN_CPP_DIR}/NetworkTestService.cpp - ${GEN_CPP_DIR}/Opcodes_constants.cpp - ${GEN_CPP_DIR}/Opcodes_types.cpp - ${GEN_CPP_DIR}/PlanNodes_constants.cpp - ${GEN_CPP_DIR}/PlanNodes_types.cpp - ${GEN_CPP_DIR}/Partitions_constants.cpp - ${GEN_CPP_DIR}/Partitions_types.cpp - ${GEN_CPP_DIR}/Planner_constants.cpp - ${GEN_CPP_DIR}/Planner_types.cpp - ${GEN_CPP_DIR}/RuntimeProfile_constants.cpp - ${GEN_CPP_DIR}/RuntimeProfile_types.cpp - ${GEN_CPP_DIR}/Status_constants.cpp - ${GEN_CPP_DIR}/Status_types.cpp - ${GEN_CPP_DIR}/Types_constants.cpp - ${GEN_CPP_DIR}/Types_types.cpp - ${GEN_CPP_DIR}/olap_common.pb.cc - ${GEN_CPP_DIR}/olap_file.pb.cc - ${GEN_CPP_DIR}/column_data_file.pb.cc - ${GEN_CPP_DIR}/data.pb.cc - ${GEN_CPP_DIR}/descriptors.pb.cc - ${GEN_CPP_DIR}/internal_service.pb.cc - ${GEN_CPP_DIR}/function_service.pb.cc - ${GEN_CPP_DIR}/types.pb.cc - ${GEN_CPP_DIR}/segment_v2.pb.cc - ${GEN_CPP_DIR}/parquet_constants.cpp - ${GEN_CPP_DIR}/parquet_types.cpp - #$${GEN_CPP_DIR}/opcode/functions.cc - #$${GEN_CPP_DIR}/opcode/vector-functions.cc - #$${GEN_CPP_DIR}/opcode/opcode-registry-init.cc +file(GLOB SRC_FILES CONFIGURE_DEPENDS + ${GEN_CPP_DIR}/*.cpp + ${GEN_CPP_DIR}/*.cc ) +add_compile_options(-Wno-return-type) + # keep everything in one library, the object files reference # each other add_library(DorisGen STATIC ${SRC_FILES}) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/GenericPool.java b/fe/fe-core/src/main/java/org/apache/doris/common/GenericPool.java index 86d381f27a1d84..35a319861641a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/GenericPool.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/GenericPool.java @@ -28,10 +28,10 @@ import org.apache.logging.log4j.Logger; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.transport.TFramedTransport; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.transport.layered.TFramedTransport; import java.lang.reflect.Constructor; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServerContext.java b/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServerContext.java index 3237f72302c49e..f6d04332215ed3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServerContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServerContext.java @@ -34,4 +34,13 @@ public ThriftServerContext(TNetworkAddress clientAddress) { public TNetworkAddress getClient() { return client; } + + public boolean isWrapperFor(Class iface) { + return false; + } + + @Override + public T unwrap(Class iface) { + throw new UnsupportedOperationException("Unimplemented method 'unwrap'"); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServerEventProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServerEventProcessor.java index 253eaf5ee52708..33228cd1a14a23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServerEventProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ThriftServerEventProcessor.java @@ -25,9 +25,9 @@ import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.server.ServerContext; import org.apache.thrift.server.TServerEventHandler; -import org.apache.thrift.transport.TFramedTransport; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.layered.TFramedTransport; import java.net.InetSocketAddress; import java.net.SocketAddress; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java index fa051b983f5003..6d42f4a338b625 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java @@ -236,9 +236,10 @@ private void handleQuery(ConnectContext context, String requestDb, String reques tQueryPlanInfo.tablet_info = tabletInfo; // serialize TQueryPlanInfo and encode plan with Base64 to string in order to translate by json format - TSerializer serializer = new TSerializer(); + TSerializer serializer; String opaquedQueryPlan; try { + serializer = new TSerializer(); byte[] queryPlanStream = serializer.serialize(tQueryPlanInfo); opaquedQueryPlan = Base64.getEncoder().encodeToString(queryPlanStream); } catch (TException e) { diff --git a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 4a5acc4a2674eb..cafa639f22484d 100644 --- a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -222,10 +222,10 @@ import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TCompactProtocol; import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.transport.TFramedTransport; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.transport.layered.TFramedTransport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -653,7 +653,12 @@ private void open() throws MetaException { throw new MetaException(e.toString()); } } else { - transport = new TSocket(store.getHost(), store.getPort(), clientSocketTimeout); + try { + transport = new TSocket(store.getHost(), store.getPort(), clientSocketTimeout); + } catch (TTransportException e) { + tte = e; + throw new MetaException(e.toString()); + } } if (useSasl) { @@ -691,7 +696,12 @@ private void open() throws MetaException { } } else { if (useFramedTransport) { - transport = new TFramedTransport(transport); + try { + transport = new TFramedTransport(transport); + } catch (TTransportException e) { + tte = e; + throw new MetaException(e.toString()); + } } } diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JniUtil.java b/fe/java-udf/src/main/java/org/apache/doris/udf/JniUtil.java index d949b85586f712..af1cf6e3b0d779 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JniUtil.java +++ b/fe/java-udf/src/main/java/org/apache/doris/udf/JniUtil.java @@ -90,8 +90,8 @@ public static String throwableToStackTrace(Throwable t) { * Serializes input into a byte[] using the default protocol factory. */ public static > byte[] serializeToThrift(T input) throws InternalException { - TSerializer serializer = new TSerializer(protocolFactory_); try { + TSerializer serializer = new TSerializer(protocolFactory_); return serializer.serialize(input); } catch (TException e) { throw new InternalException(e.getMessage()); @@ -103,8 +103,8 @@ public static String throwableToStackTrace(Throwable t) { */ public static , F extends TProtocolFactory> byte[] serializeToThrift( T input, F protocolFactory) throws InternalException { - TSerializer serializer = new TSerializer(protocolFactory); try { + TSerializer serializer = new TSerializer(protocolFactory); return serializer.serialize(input); } catch (TException e) { throw new InternalException(e.getMessage()); @@ -122,8 +122,8 @@ public static String throwableToStackTrace(Throwable t) { public static , F extends TProtocolFactory> void deserializeThrift( F protocolFactory, T result, byte[] thriftData) throws InternalException { // TODO: avoid creating deserializer for each query? - TDeserializer deserializer = new TDeserializer(protocolFactory); try { + TDeserializer deserializer = new TDeserializer(protocolFactory); deserializer.deserialize(result, thriftData); } catch (TException e) { throw new InternalException(e.getMessage()); diff --git a/fe/pom.xml b/fe/pom.xml index 44cdd12e3777d2..e718a6d178cb01 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -189,7 +189,7 @@ under the License. 1.1.1 5.8.2 1.2.5 - 0.13.0 + 0.16.0 8.5.86 2.18.0 2.18.0 diff --git a/fs_brokers/apache_hdfs_broker/pom.xml b/fs_brokers/apache_hdfs_broker/pom.xml index bea0443d5645e5..b67c119fc94202 100644 --- a/fs_brokers/apache_hdfs_broker/pom.xml +++ b/fs_brokers/apache_hdfs_broker/pom.xml @@ -252,7 +252,7 @@ under the License. org.apache.thrift libthrift - 0.13.0 + 0.16.0 diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index c16c0d6a6f6b04..7f8f46432e15be 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,9 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## v20230228 +- Modified: thrift 0.13 -> 0.16 + ## v20230221 - Modified: clucene 2.4.4 -> 2.4.6 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index d50c07d1de3ed7..7a011374c8fe2c 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -367,11 +367,11 @@ build_thrift() { if [[ "${KERNEL}" != 'Darwin' ]]; then cflags="-I${TP_INCLUDE_DIR}" - cxxflags="-I${TP_INCLUDE_DIR} ${warning_unused_but_set_variable}" + cxxflags="-I${TP_INCLUDE_DIR} ${warning_unused_but_set_variable} -Wno-inconsistent-missing-override" ldflags="-L${TP_LIB_DIR} --static" else - cflags="-I${TP_INCLUDE_DIR} -Wno-implicit-function-declaration" - cxxflags="-I${TP_INCLUDE_DIR} ${warning_unused_but_set_variable}" + cflags="-I${TP_INCLUDE_DIR} -Wno-implicit-function-declaration -Wno-inconsistent-missing-override" + cxxflags="-I${TP_INCLUDE_DIR} ${warning_unused_but_set_variable} -Wno-inconsistent-missing-override" ldflags="-L${TP_LIB_DIR}" fi diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 002a0e513d7947..87c50edfc17494 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -73,10 +73,10 @@ OPENSSL_SOURCE=openssl-OpenSSL_1_1_1s OPENSSL_MD5SUM="7e79a7560dee77c0758baa33c61af4b4" # thrift -THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.13.0/thrift-0.13.0.tar.gz" -THRIFT_NAME=thrift-0.13.0.tar.gz -THRIFT_SOURCE=thrift-0.13.0 -THRIFT_MD5SUM="38a27d391a2b03214b444cb13d5664f1" +THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.16.0/thrift-0.16.0.tar.gz" +THRIFT_NAME=thrift-0.16.0.tar.gz +THRIFT_SOURCE=thrift-0.16.0 +THRIFT_MD5SUM="44cf1b54b4ec1890576c85804acfa637" # protobuf PROTOBUF_DOWNLOAD="https://github.com/google/protobuf/archive/v3.15.0.tar.gz"