Skip to content

[Load]Support compressed csv file in stream load #5463

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yangzhg merged 1 commit into from
Mar 11, 2021
Merged

[Load]Support compressed csv file in stream load #5463

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions be/src/exec/broker_scanner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ Status BrokerScanner::open_file_reader() {
}

Status BrokerScanner::create_decompressor(TFileFormatType::type type) {
if (_cur_decompressor == nullptr) {
if (_cur_decompressor != nullptr) {
delete _cur_decompressor;
_cur_decompressor = nullptr;
}
Expand Down Expand Up @@ -208,7 +208,7 @@ Status BrokerScanner::create_decompressor(TFileFormatType::type type) {
break;
default: {
std::stringstream ss;
ss << "Unknown format type, type=" << type;
ss << "Unknown format type, cannot inference compress type, type=" << type;
return Status::InternalError(ss.str());
}
}
Expand Down Expand Up @@ -259,7 +259,7 @@ Status BrokerScanner::open_line_reader() {
break;
default: {
std::stringstream ss;
ss << "Unknown format type, type=" << range.format_type;
ss << "Unknown format type, cannot init line reader, type=" << range.format_type;
return Status::InternalError(ss.str());
}
}
Expand Down
54 changes: 41 additions & 13 deletions be/src/http/action/stream_load.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,18 +69,46 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(streaming_load_current_processing, MetricUnit
TStreamLoadPutResult k_stream_load_put_result;
#endif

static TFileFormatType::type parse_format(const std::string& format_str) {
static TFileFormatType::type parse_format(const std::string& format_str,
const std::string& compress_type) {
if (format_str.empty()) {
return parse_format("CSV", compress_type);
}
TFileFormatType::type format_type = TFileFormatType::FORMAT_UNKNOWN;
if (boost::iequals(format_str, "CSV")) {
return TFileFormatType::FORMAT_CSV_PLAIN;
if (compress_type.empty()) {
format_type = TFileFormatType::FORMAT_CSV_PLAIN;
}
if (boost::iequals(compress_type, "GZ")) {
format_type = TFileFormatType::FORMAT_CSV_GZ;
} else if (boost::iequals(compress_type, "LZO")) {
format_type = TFileFormatType::FORMAT_CSV_LZO;
} else if (boost::iequals(compress_type, "BZ2")) {
format_type = TFileFormatType::FORMAT_CSV_BZ2;
} else if (boost::iequals(compress_type, "LZ4FRAME")) {
format_type = TFileFormatType::FORMAT_CSV_LZ4FRAME;
} else if (boost::iequals(compress_type, "LZOP")) {
format_type = TFileFormatType::FORMAT_CSV_LZOP;
} else if (boost::iequals(compress_type, "DEFLATE")) {
format_type = TFileFormatType::FORMAT_CSV_DEFLATE;
}
} else if (boost::iequals(format_str, "JSON")) {
return TFileFormatType::FORMAT_JSON;
if (compress_type.empty()) {
format_type = TFileFormatType::FORMAT_JSON;
}
}
return TFileFormatType::FORMAT_UNKNOWN;
return format_type;
}

static bool is_format_support_streaming(TFileFormatType::type format) {
switch (format) {
case TFileFormatType::FORMAT_CSV_PLAIN:
case TFileFormatType::FORMAT_CSV_BZ2:
case TFileFormatType::FORMAT_CSV_DEFLATE:
case TFileFormatType::FORMAT_CSV_GZ:
case TFileFormatType::FORMAT_CSV_LZ4FRAME:
case TFileFormatType::FORMAT_CSV_LZO:
case TFileFormatType::FORMAT_CSV_LZOP:
case TFileFormatType::FORMAT_JSON:
return true;
default:
Expand Down Expand Up @@ -214,15 +242,15 @@ Status StreamLoadAction::_on_header(HttpRequest* http_req, StreamLoadContext* ct
}

// get format of this put
if (http_req->header(HTTP_FORMAT_KEY).empty()) {
ctx->format = TFileFormatType::FORMAT_CSV_PLAIN;
} else {
ctx->format = parse_format(http_req->header(HTTP_FORMAT_KEY));
if (ctx->format == TFileFormatType::FORMAT_UNKNOWN) {
std::stringstream ss;
ss << "unknown data format, format=" << http_req->header(HTTP_FORMAT_KEY);
return Status::InternalError(ss.str());
}
if (!http_req->header(HTTP_COMPRESS_TYPE).empty() && boost::iequals(http_req->header(HTTP_FORMAT_KEY), "JSON")) {
return Status::InternalError("compress data of JSON format is not supported.");
}
ctx->format =
parse_format(http_req->header(HTTP_FORMAT_KEY), http_req->header(HTTP_COMPRESS_TYPE));
if (ctx->format == TFileFormatType::FORMAT_UNKNOWN) {
std::stringstream ss;
ss << "unknown data format, format=" << http_req->header(HTTP_FORMAT_KEY);
return Status::InternalError(ss.str());
}

// check content length
Expand Down
1 change: 1 addition & 0 deletions be/src/http/http_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ static const std::string HTTP_MERGE_TYPE = "merge_type";
static const std::string HTTP_DELETE_CONDITION = "delete";
static const std::string HTTP_FUNCTION_COLUMN = "function_column";
static const std::string HTTP_SEQUENCE_COL = "sequence_col";
static const std::string HTTP_COMPRESS_TYPE = "compress_type";

static const std::string HTTP_100_CONTINUE = "100-continue";

Expand Down
2 changes: 1 addition & 1 deletion gensrc/thrift/PlanNodes.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ enum TFileFormatType {
FORMAT_PARQUET,
FORMAT_CSV_DEFLATE,
FORMAT_ORC,
FORMAT_JSON
FORMAT_JSON,
}

// One broker range information.
Expand Down