diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 192ee3f406abf9..3bacf930a0e4b0 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -294,6 +294,18 @@ set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib add_library(minzip STATIC IMPORTED) set_target_properties(minzip PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libminizip.a) +add_library(hdfs3 STATIC IMPORTED) +set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libhdfs3.a) + +add_library(gsasl STATIC IMPORTED) +set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a) + +add_library(xml2 STATIC IMPORTED) +set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a) + +add_library(lzma STATIC IMPORTED) +set_target_properties(lzma PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/liblzma.a) + find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin) # Check if functions are supported in this platform. All flags will generated @@ -494,6 +506,10 @@ set(DORIS_DEPENDENCIES odbc cctz minzip + hdfs3 + gsasl + xml2 + lzma ${AWS_LIBS} ${WL_END_GROUP} ) diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index c6da8ede2e4eba..c958c67c9f5cc5 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -40,6 +40,7 @@ set(EXEC_FILES hash_join_node.cpp hash_join_node_ir.cpp hash_table.cpp + hdfs_file_reader.cpp local_file_reader.cpp merge_node.cpp merge_join_node.cpp diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp index c4254c0e6de230..c5087c147ce5be 100644 --- a/be/src/exec/broker_scanner.cpp +++ b/be/src/exec/broker_scanner.cpp @@ -36,6 +36,14 @@ #include "runtime/stream_load/load_stream_mgr.h" #include "runtime/stream_load/stream_load_pipe.h" #include "runtime/tuple.h" +#include "exprs/expr.h" +#include "exec/text_converter.h" +#include "exec/text_converter.hpp" +#include "exec/plain_text_line_reader.h" +#include "exec/hdfs_file_reader.h" +#include "exec/local_file_reader.h" +#include "exec/broker_reader.h" +#include "exec/decompressor.h" #include "util/utf8_check.h" namespace doris { @@ -159,6 +167,13 @@ Status BrokerScanner::open_file_reader() { _cur_file_reader = file_reader; break; } + case TFileType::FILE_HDFS: { + HdfsFileReader* file_reader = new HdfsFileReader( + range.hdfs_params, range.path, start_offset); + RETURN_IF_ERROR(file_reader->open()); + _cur_file_reader = file_reader; + break; + } case TFileType::FILE_BROKER: { BrokerReader* broker_reader = new BrokerReader(_state->exec_env(), _broker_addresses, _params.properties, diff --git a/be/src/exec/hdfs_file_reader.cpp b/be/src/exec/hdfs_file_reader.cpp new file mode 100644 index 00000000000000..2cecabe3032712 --- /dev/null +++ b/be/src/exec/hdfs_file_reader.cpp @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#include "exec/hdfs_file_reader.h" + +#include +#include + +#include "common/logging.h" + +namespace doris { +HdfsFileReader::HdfsFileReader(THdfsParams hdfs_params, + const std::string& path, int64_t start_offset) + : _hdfs_params(hdfs_params), _path(path), _current_offset(start_offset), + _file_size(-1), _hdfs_fs(nullptr), _hdfs_file(nullptr) { + std::stringstream namenode_ss; + namenode_ss << "hdfs://" << _hdfs_params.host<< ":" << _hdfs_params.port; + _namenode = namenode_ss.str(); +} + +HdfsFileReader::~HdfsFileReader() { + close(); +} + +Status HdfsFileReader::connect() { + hdfsBuilder* hdfs_builder = hdfsNewBuilder(); + hdfsBuilderSetNameNode(hdfs_builder, _namenode.c_str()); + // set hdfs user + if (_hdfs_params.__isset.user) { + hdfsBuilderSetUserName(hdfs_builder, _hdfs_params.user.c_str()); + } + // set kerberos conf + if (_hdfs_params.__isset.kerb_principal) { + hdfsBuilderSetPrincipal(hdfs_builder, _hdfs_params.kerb_principal.c_str()); + } + if (_hdfs_params.__isset.kerb_ticket_cache_path) { + hdfsBuilderSetKerbTicketCachePath(hdfs_builder, _hdfs_params.kerb_ticket_cache_path.c_str()); + } + // set token + if (_hdfs_params.__isset.token) { + hdfsBuilderSetToken(hdfs_builder, _hdfs_params.token.c_str()); + } + // set other conf + if (_hdfs_params.__isset.hdfs_conf) { + for (const THdfsConf& conf : _hdfs_params.hdfs_conf) { + hdfsBuilderConfSetStr(hdfs_builder, conf.key.c_str(), conf.value.c_str()); + } + } + _hdfs_fs = hdfsBuilderConnect(hdfs_builder); + if (_hdfs_fs == nullptr) { + std::stringstream ss; + ss << "connect failed. " << _namenode; + return Status::InternalError(ss.str()); + } + return Status::OK(); +} + +Status HdfsFileReader::open() { + if (!closed()) { + close(); + } + RETURN_IF_ERROR(connect()); + _hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_RDONLY, 0, 0, 0); + if (_hdfs_file == nullptr) { + std::stringstream ss; + ss << "open file failed. " << _namenode << _path; + return Status::InternalError(ss.str()); + } + LOG(INFO) << "open file. " << _namenode << _path; + return seek(_current_offset); +} + +void HdfsFileReader::close() { + if (!closed()) { + if (_hdfs_file != nullptr && _hdfs_fs != nullptr) { + std::stringstream ss; + ss << "close hdfs file: " << _namenode << _path; + LOG(INFO) << ss.str(); + //If the hdfs file was valid, the memory associated with it will + // be freed at the end of this call, even if there was an I/O error + hdfsCloseFile(_hdfs_fs, _hdfs_file); + } + if (_hdfs_fs != nullptr) { + // Even if there is an error, the resources associated with the hdfsFS will be freed. + hdfsDisconnect(_hdfs_fs); + } + } + _hdfs_file = nullptr; + _hdfs_fs = nullptr; +} + +bool HdfsFileReader::closed() { + return _hdfs_file == nullptr || _hdfs_fs == nullptr; +} + +// Read all bytes +Status HdfsFileReader::read_one_message(std::unique_ptr* buf, size_t* length) { + int64_t file_size = size() - _current_offset; + if (file_size <= 0) { + buf->reset(); + *length = 0; + return Status::OK(); + } + bool eof; + *length = file_size; + buf->reset(new uint8_t[file_size]); + read(buf->get(), length, &eof); + return Status::OK(); +} + +Status HdfsFileReader::read(uint8_t* buf, size_t* buf_len, bool* eof) { + readat(_current_offset, (int64_t)*buf_len, (int64_t*)buf_len, buf); + if (*buf_len == 0) { + *eof = true; + } else { + *eof = false; + } + return Status::OK(); +} + +Status HdfsFileReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_read, void* out) { + if (position != _current_offset) { + int ret = hdfsSeek(_hdfs_fs, _hdfs_file, position); + if (ret != 0) { // check fseek return value + std::stringstream ss; + ss << "hdfsSeek failed. " << _namenode << _path; + return Status::InternalError(ss.str()); + } + } + + *bytes_read = hdfsRead(_hdfs_fs, _hdfs_file, out, nbytes); + if (*bytes_read < 0) { + std::stringstream ss; + ss << "Read hdfs file failed. " << _namenode << _path; + return Status::InternalError(ss.str()); + } + _current_offset += *bytes_read; // save offset with file + return Status::OK(); +} + +int64_t HdfsFileReader::size() { + if (_file_size == -1) { + bool need_init_fs = false; + if (_hdfs_fs == nullptr) { + need_init_fs = true; + if (!connect().ok()) { + return -1; + } + } + hdfsFileInfo* file_info = hdfsGetPathInfo(_hdfs_fs, _path.c_str()); + if (file_info == nullptr) { + LOG(WARNING) << "get path info failed: " << _namenode << _path; + close(); + return -1; + } + _file_size = file_info->mSize; + hdfsFreeFileInfo(file_info, 1); + if (need_init_fs) { + close(); + } + } + return _file_size; +} + +Status HdfsFileReader::seek(int64_t position) { + int res = hdfsSeek(_hdfs_fs, _hdfs_file, position); + if (res != 0) { + char err_buf[64]; + std::stringstream ss; + ss << "Seek to offset failed. offset=" << position + << ", error=" << strerror_r(errno, err_buf, 64); + return Status::InternalError(ss.str()); + } + return Status::OK(); +} + +Status HdfsFileReader::tell(int64_t* position) { + *position = _current_offset; + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/hdfs_file_reader.h b/be/src/exec/hdfs_file_reader.h new file mode 100644 index 00000000000000..037e2c11d225af --- /dev/null +++ b/be/src/exec/hdfs_file_reader.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "exec/file_reader.h" + +#include "gen_cpp/PlanNodes_types.h" + +namespace doris { + +class HdfsFileReader : public FileReader { +public: + HdfsFileReader(THdfsParams hdfs_params, const std::string& path, int64_t start_offset); + virtual ~HdfsFileReader(); + + virtual Status open() override; + + // Read content to 'buf', 'buf_len' is the max size of this buffer. + // Return ok when read success, and 'buf_len' is set to size of read content + // If reach to end of file, the eof is set to true. meanwhile 'buf_len' + // is set to zero. + virtual Status read(uint8_t* buf, size_t* buf_len, bool* eof) override; + virtual Status readat(int64_t position, int64_t nbytes, int64_t* bytes_read, + void* out) override; + virtual Status read_one_message(std::unique_ptr* buf, size_t* length) override; + virtual int64_t size() override; + virtual Status seek(int64_t position) override; + virtual Status tell(int64_t* position) override; + virtual void close() override; + virtual bool closed() override; + +private: + Status connect(); +private: + THdfsParams _hdfs_params; + std::string _namenode; + std::string _path; + int64_t _current_offset; + int64_t _file_size; + hdfsFS _hdfs_fs; + hdfsFile _hdfs_file; +}; + +} // namespace doris diff --git a/be/src/exec/parquet_scanner.cpp b/be/src/exec/parquet_scanner.cpp index 50c255aa68c046..8d0bf57be7ca16 100644 --- a/be/src/exec/parquet_scanner.cpp +++ b/be/src/exec/parquet_scanner.cpp @@ -32,6 +32,16 @@ #include "runtime/stream_load/load_stream_mgr.h" #include "runtime/stream_load/stream_load_pipe.h" #include "runtime/tuple.h" +#include "exec/parquet_reader.h" +#include "exprs/expr.h" +#include "exec/text_converter.h" +#include "exec/text_converter.hpp" +#include "exec/hdfs_file_reader.h" +#include "exec/local_file_reader.h" +#include "exec/broker_reader.h" +#include "exec/buffered_reader.h" +#include "exec/decompressor.h" +#include "exec/parquet_reader.h" namespace doris { @@ -117,6 +127,11 @@ Status ParquetScanner::open_next_reader() { file_reader.reset(new LocalFileReader(range.path, range.start_offset)); break; } + case TFileType::FILE_HDFS: { + file_reader.reset(new HdfsFileReader( + range.hdfs_params, range.path, range.start_offset)); + break; + } case TFileType::FILE_BROKER: { int64_t file_size = 0; // for compatibility diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 6dbdad24aebbdf..239df52686750b 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -108,6 +108,21 @@ enum TFileFormatType { FORMAT_JSON, } +struct THdfsConf { + 1: required string key + 2: required string value +} + +struct THdfsParams { + 1: optional string host + 2: optional i32 port + 3: optional string user + 4: optional string kerb_principal + 5: optional string kerb_ticket_cache_path + 6: optional string token + 7: optional list hdfs_conf +} + // One broker range information. struct TBrokerRangeDesc { 1: required Types.TFileType file_type @@ -134,6 +149,7 @@ struct TBrokerRangeDesc { // it's usefull when format_type == FORMAT_JSON 14: optional bool num_as_string; 15: optional bool fuzzy_parse; + 16: optional THdfsParams hdfs_params } struct TBrokerScanRangeParams { diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index ca0ae841a6504d..bbe6e13c684e88 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -802,6 +802,50 @@ build_aws_sdk() { -DCMAKE_MODULE_PATH=$TP_INSTALL_DIR/lib64/cmake -DBUILD_ONLY="s3" ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } + +# lzma +build_lzma() { + check_if_source_exist $LZMA_SOURCE + cd $TP_SOURCE_DIR/$LZMA_SOURCE + export ACLOCAL_PATH=/usr/share/aclocal + sh autogen.sh + mkdir -p $BUILD_DIR && cd $BUILD_DIR + ../configure --prefix=$TP_INSTALL_DIR --enable-shared=no --with-pic + make -j $PARALLEL && make install +} + +# xml2 +build_xml2() { + check_if_source_exist $XML2_SOURCE + cd $TP_SOURCE_DIR/$XML2_SOURCE + export ACLOCAL_PATH=/usr/share/aclocal + sh autogen.sh + make distclean + mkdir -p $BUILD_DIR && cd $BUILD_DIR + ../configure --prefix=$TP_INSTALL_DIR --enable-shared=no --with-pic --with-python=no + make -j $PARALLEL && make install +} + +# gsasl +build_gsasl() { + check_if_source_exist $GSASL_SOURCE + cd $TP_SOURCE_DIR/$GSASL_SOURCE + mkdir -p $BUILD_DIR && cd $BUILD_DIR + ../configure --prefix=$TP_INSTALL_DIR --enable-shared=no --with-pic + make -j $PARALLEL && make install +} + +# hdfs3 +build_hdfs3() { + check_if_source_exist $HDFS3_SOURCE + cd $TP_SOURCE_DIR/$HDFS3_SOURCE + mkdir -p $BUILD_DIR && cd $BUILD_DIR + # export CC=/opt/compiler/gcc-10/bin/gcc + # export CXX=/opt/compiler/gcc-10/bin/g++ + ../bootstrap --dependency=$TP_INSTALL_DIR --prefix=$TP_INSTALL_DIR + make -j $PARALLEL && make install +} + # See https://github.com/apache/incubator-doris/issues/2910 # LLVM related codes have already be removed in master, so there is # no need to build llvm tool here. @@ -849,5 +893,9 @@ build_aws_checksums build_aws_c_event_stream build_aws_sdk build_js_and_css +build_lzma +build_xml2 +build_gsasl +build_hdfs3 echo "Finished to build all thirdparties" diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index d34bb4db1899f4..75a772834762cc 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -315,3 +315,11 @@ fi cd - echo "Finished patching $S2_SOURCE" +# hdfs3 patch to fix compile error +cd $TP_SOURCE_DIR/$HDFS3_SOURCE +if [ ! -f $PATCHED_MARK ]; then + patch -p1 < $TP_PATCH_DIR/libhdfs3-master.patch + touch $PATCHED_MARK +fi +cd - +echo "Finished patching $HDFS3_SOURCE" diff --git a/thirdparty/patches/libhdfs3-master.patch b/thirdparty/patches/libhdfs3-master.patch new file mode 100644 index 00000000000000..52cd1ef00443fa --- /dev/null +++ b/thirdparty/patches/libhdfs3-master.patch @@ -0,0 +1,104 @@ +diff --git a/src/client/FileSystem.cpp b/src/client/FileSystem.cpp +index 6c347c7..6aec1a3 100644 +--- a/src/client/FileSystem.cpp ++++ b/src/client/FileSystem.cpp +@@ -136,7 +136,7 @@ static std::string ExtractPrincipalFromTicketCache( + static std::string ExtractPrincipalFromToken(const Token & token) { + std::string realUser, owner; + std::string identifier = token.getIdentifier(); +- WritableUtils cin(identifier.data(), identifier.size()); ++ WritableUtils cin(&identifier[0], identifier.size()); + char version; + + try { +diff --git a/src/client/Token.cpp b/src/client/Token.cpp +index 1e23fed..8c88b2f 100644 +--- a/src/client/Token.cpp ++++ b/src/client/Token.cpp +@@ -156,10 +156,10 @@ Token & Token::fromString(const std::string & str) { + WritableUtils in(buffer.data(), buffer.size()); + len = in.ReadInt32(); + identifier.resize(len); +- in.ReadRaw(identifier.data(), len); ++ in.ReadRaw(&identifier[0], len); + len = in.ReadInt32(); + password.resize(len); +- in.ReadRaw(password.data(), len); ++ in.ReadRaw(&password[0], len); + kind = in.ReadText(); + service = in.ReadText(); + return *this; +diff --git a/src/common/ExceptionInternal.h b/src/common/ExceptionInternal.h +index 4ee661e..9d734af 100644 +--- a/src/common/ExceptionInternal.h ++++ b/src/common/ExceptionInternal.h +@@ -175,7 +175,7 @@ void ThrowException(bool nested, const char * f, int l, + int offset = buffer.size(); + buffer.resize(offset + size + 1); + va_start(ap, fmt); +- vsnprintf(buffer.data() + offset, size + 1, fmt, ap); ++ vsnprintf(&buffer[offset], size + 1, fmt, ap); + va_end(ap); + + if (!nested) { +@@ -218,7 +218,7 @@ void ThrowException(bool nested, const char * f, int l, + int offset = buffer.size(); + buffer.resize(offset + size + 1); + va_start(ap, fmt); +- vsnprintf(buffer.data() + offset, size + 1, fmt, ap); ++ vsnprintf(&buffer[offset], size + 1, fmt, ap); + va_end(ap); + + if (!nested) { +diff --git a/src/common/StringUtil.h b/src/common/StringUtil.h +index f9cba5d..3a8e76f 100644 +--- a/src/common/StringUtil.h ++++ b/src/common/StringUtil.h +@@ -41,7 +41,7 @@ static inline std::vector StringSplit(const std::string & str, + char * token, *lasts = NULL; + std::string s = str; + std::vector retval; +- token = strtok_r(s.data(), sep, &lasts); ++ token = strtok_r(&s[0], sep, &lasts); + + while (token) { + retval.push_back(token); +diff --git a/src/common/WritableUtils.cpp b/src/common/WritableUtils.cpp +index fdadd45..98bbc6a 100644 +--- a/src/common/WritableUtils.cpp ++++ b/src/common/WritableUtils.cpp +@@ -88,7 +88,7 @@ std::string WritableUtils::ReadText() { + std::string retval; + length = ReadInt32(); + retval.resize(length); +- ReadRaw(retval.data(), length); ++ ReadRaw(&retval[0], length); + return retval; + } + +diff --git a/src/rpc/RpcClient.cpp b/src/rpc/RpcClient.cpp +index bb19991..ebb6e28 100644 +--- a/src/rpc/RpcClient.cpp ++++ b/src/rpc/RpcClient.cpp +@@ -54,7 +54,7 @@ RpcClientImpl::RpcClientImpl() : + cleaning(false), running(true), count(0) { + auto id = boost::uuids::random_generator()(); + clientId.resize(boost::uuids::uuid::static_size()); +- memcpy(clientId.data(), id.begin(), boost::uuids::uuid::static_size()); ++ memcpy(&clientId[0], id.begin(), boost::uuids::uuid::static_size()); + #ifdef MOCK + stub = NULL; + #endif +diff --git a/src/rpc/SaslClient.cpp b/src/rpc/SaslClient.cpp +index 9aa9d7b..53893a1 100644 +--- a/src/rpc/SaslClient.cpp ++++ b/src/rpc/SaslClient.cpp +@@ -137,7 +137,7 @@ std::string SaslClient::evaluateChallenge(const std::string & challenge) { + + if (rc == GSASL_NEEDS_MORE || rc == GSASL_OK) { + retval.resize(outputSize); +- memcpy(retval.data(), output, outputSize); ++ memcpy(&retval[0], output, outputSize); + + if (output) { + free(output); diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 3a46bb2c4858f4..bb79834cd8d7f2 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -349,6 +349,30 @@ TSAN_HEADER_NAME="tsan_interface_atomic.h" TSAN_HEADER_FILE="tsan_interface_atomic.h" TSAN_HEADER_MD5SUM="d72679bea167d6a513d959f5abd149dc" +# lzma +LZMA_DOWNLOAD="https://github.com/kobolabs/liblzma/archive/refs/heads/master.zip" +LZMA_NAME="liblzma-master.zip" +LZMA_SOURCE="liblzma-master" +LZMA_MD5SUM="ef11f2fbbfa6893b629f207a32bf730e" + +# xml2 +XML2_DOWNLOAD="https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.9.10/libxml2-v2.9.10.tar.gz" +XML2_NAME="libxml2-v2.9.10.tar.gz" +XML2_SOURCE="libxml2-v2.9.10" +XML2_MD5SUM="b18faee9173c3378c910f6d7d1493115" + +# gsasl +GSASL_DOWNLOAD="https://ftp.gnu.org/gnu/gsasl/libgsasl-1.10.0.tar.gz" +GSASL_NAME="libgsasl-1.10.0.tar.gz" +GSASL_SOURCE="libgsasl-1.10.0" +GSASL_MD5SUM="9c8fc632da4ce108fb7581b33de2a5ce" + +# hdfs3 +HDFS3_DOWNLOAD="https://github.com/ClickHouse-Extras/libhdfs3/archive/refs/heads/master.zip" +HDFS3_NAME="libhdfs3-master.zip" +HDFS3_SOURCE="libhdfs3-master" +HDFS3_MD5SUM="9e46a16009cf86b5e187d302b3582628" + # all thirdparties which need to be downloaded is set in array TP_ARCHIVES export TP_ARCHIVES="LIBEVENT OPENSSL @@ -399,4 +423,8 @@ AWS_C_CAL AWS_C_IO AWS_CHECKSUMS AWS_S2N -AWS_SDK" +AWS_SDK +LZMA +XML2 +GSASL +HDFS3"