Skip to content

Pick "[enhance](S3) Print the oss request id for each error s3 request #32499" #32607

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 1 commit into from
Mar 21, 2024
Merged

Pick "[enhance](S3) Print the oss request id for each error s3 request #32499" #32607

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions be/src/io/fs/s3_file_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,11 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea
}
auto outcome = client->GetObject(request);
if (!outcome.IsSuccess()) {
return Status::IOError("failed to read from {}: {}, exception {}, error code {}",
_path.native(), outcome.GetError().GetMessage(),
outcome.GetError().GetExceptionName(),
outcome.GetError().GetResponseCode());
return Status::IOError(
"failed to read from {}: {}, exception {}, error code {}, request id {}",
_path.native(), outcome.GetError().GetMessage(),
outcome.GetError().GetExceptionName(), outcome.GetError().GetResponseCode(),
outcome.GetError().GetRequestId());
}
*bytes_read = outcome.GetResult().GetContentLength();
if (*bytes_read != bytes_req) {
Expand Down
4 changes: 2 additions & 2 deletions be/src/io/fs/s3_file_system.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,10 +526,10 @@ Status S3FileSystem::get_key(const Path& path, std::string* key) const {

template <typename AwsOutcome>
std::string S3FileSystem::error_msg(const std::string& key, const AwsOutcome& outcome) const {
return fmt::format("(endpoint: {}, bucket: {}, key:{}, {}), {}, error code {}",
return fmt::format("(endpoint: {}, bucket: {}, key:{}, {}), {}, error code {}, request id {}",
_s3_conf.endpoint, _s3_conf.bucket, key,
outcome.GetError().GetExceptionName(), outcome.GetError().GetMessage(),
outcome.GetError().GetResponseCode());
outcome.GetError().GetResponseCode(), outcome.GetError().GetRequestId());
}

std::string S3FileSystem::error_msg(const std::string& key, const std::string& err) const {
Expand Down
26 changes: 16 additions & 10 deletions be/src/io/fs/s3_file_writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,10 @@ Status S3FileWriter::_create_multi_upload_request() {
}
return Status::IOError(
"failed to create multipart upload(bucket={}, key={}, upload_id={}): {}, exception {}, "
"error code {}",
"error code {}, request id {}",
_bucket, _path.native(), _upload_id, outcome.GetError().GetMessage(),
outcome.GetError().GetExceptionName(), outcome.GetError().GetResponseCode());
outcome.GetError().GetExceptionName(), outcome.GetError().GetResponseCode(),
outcome.GetError().GetRequestId());
}

void S3FileWriter::_wait_until_finish(std::string_view task_name) {
Expand Down Expand Up @@ -174,9 +175,10 @@ Status S3FileWriter::abort() {
}
return Status::IOError(
"failed to abort multipart upload(bucket={}, key={}, upload_id={}): {}, exception {}, "
"error code {}",
"error code {}, request id {}",
_bucket, _path.native(), _upload_id, outcome.GetError().GetMessage(),
outcome.GetError().GetExceptionName(), outcome.GetError().GetResponseCode());
outcome.GetError().GetExceptionName(), outcome.GetError().GetResponseCode(),
outcome.GetError().GetRequestId());
}

Status S3FileWriter::close() {
Expand Down Expand Up @@ -306,11 +308,12 @@ void S3FileWriter::_upload_one_part(int64_t part_num, S3FileBuffer& buf) {
if (!upload_part_outcome.IsSuccess()) {
auto s = Status::IOError(
"failed to upload part (bucket={}, key={}, part_num={}, up_load_id={}): {}, "
"exception {}, error code {}",
"exception {}, error code {}, request id {}",
_bucket, _path.native(), part_num, _upload_id,
upload_part_outcome.GetError().GetMessage(),
upload_part_outcome.GetError().GetExceptionName(),
upload_part_outcome.GetError().GetResponseCode());
upload_part_outcome.GetError().GetResponseCode(),
upload_part_outcome.GetError().GetRequestId());
LOG_WARNING(s.to_string());
buf._on_failed(s);
return;
Expand Down Expand Up @@ -357,10 +360,11 @@ Status S3FileWriter::_complete() {
if (!compute_outcome.IsSuccess()) {
auto s = Status::IOError(
"failed to create complete multi part upload (bucket={}, key={}): {}, exception "
"{}, error code {}",
"{}, error code {}, request id {}",
_bucket, _path.native(), compute_outcome.GetError().GetMessage(),
compute_outcome.GetError().GetExceptionName(),
compute_outcome.GetError().GetResponseCode());
compute_outcome.GetError().GetResponseCode(),
compute_outcome.GetError().GetRequestId());
LOG_WARNING(s.to_string());
return s;
}
Expand Down Expand Up @@ -399,9 +403,11 @@ void S3FileWriter::_put_object(S3FileBuffer& buf) {
auto response = _client->PutObject(request);
if (!response.IsSuccess()) {
_st = Status::InternalError(
"failed to put object (bucket={}, key={}), Error: [{}:{}, responseCode:{}]",
"failed to put object (bucket={}, key={}), Error: [{}:{}, responseCode:{}, request "
"id:{}]",
_bucket, _path.native(), response.GetError().GetExceptionName(),
response.GetError().GetMessage(), response.GetError().GetResponseCode());
response.GetError().GetMessage(), response.GetError().GetResponseCode(),
response.GetError().GetRequestId());
LOG(WARNING) << _st;
buf._on_failed(_st);
return;
Expand Down