database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhance](S3) Print the oss request id for each error s3 request (#32499)

Browse Source
This commit is contained in:
AlexYue
2024-03-20 23:32:12 +08:00
committed by yiguolei
parent d2968dcf99
commit 02430e6e53
3 changed files with 31 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
be/src/io/fs/err_utils.cpp
View File

@ -115,16 +115,17 @@ Status s3fs_error(const Aws::S3::S3Error& err, std::string_view msg) {
using namespace Aws::Http;
switch (err.GetResponseCode()) {
case HttpResponseCode::NOT_FOUND:
return Status::Error<NOT_FOUND, false>("{}: {} {} type={}", msg, err.GetExceptionName(),
err.GetMessage(), err.GetErrorType());
return Status::Error<NOT_FOUND, false>("{}: {} {} type={}, request_id={}", msg,
err.GetExceptionName(), err.GetMessage(),
err.GetErrorType(), err.GetRequestId());
case HttpResponseCode::FORBIDDEN:
return Status::Error<PERMISSION_DENIED, false>("{}: {} {} type={}", msg,
return Status::Error<PERMISSION_DENIED, false>("{}: {} {} type={}, request_id={}", msg,
err.GetExceptionName(), err.GetMessage(),
err.GetErrorType());
err.GetErrorType(), err.GetRequestId());
default:
return Status::Error<ErrorCode::INTERNAL_ERROR, false>(
"{}: {} {} code={} type={}", msg, err.GetExceptionName(), err.GetMessage(),
err.GetResponseCode(), err.GetErrorType());
"{}: {} {} code={} type={}, request_id={}", msg, err.GetExceptionName(),
err.GetMessage(), err.GetResponseCode(), err.GetErrorType(), err.GetRequestId());
}
}

34
be/src/io/fs/s3_file_writer.cpp
View File

@ -179,13 +179,14 @@ Status S3FileWriter::_abort() {
outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) {
LOG(INFO) << "Abort multipart upload successfully"
<< "bucket=" << _bucket << ", key=" << _path.native()
<< ", upload_id=" << _upload_id;
<< ", upload_id=" << _upload_id << ", whole parts=" << _dump_completed_part();
_aborted = true;
return Status::OK();
}
return s3fs_error(outcome.GetError(),
fmt::format("failed to abort multipart upload {} upload_id={}",
_path.native(), _upload_id));
return s3fs_error(
outcome.GetError(),
fmt::format("failed to abort multipart upload {} upload_id={}, whole parts={}",
_path.native(), _upload_id, _dump_completed_part()));
}
Status S3FileWriter::close() {
@ -409,8 +410,9 @@ Status S3FileWriter::_complete() {
_wait_until_finish("Complete");
DBUG_EXECUTE_IF("s3_file_writer::_complete:1", { _cur_part_num++; });
if (_failed || _completed_parts.size() != _cur_part_num) {
_st = Status::InternalError("error status {}, complete parts {}, cur part num {}", _st,
_completed_parts.size(), _cur_part_num);
_st = Status::InternalError(
"error status {}, complete parts {}, cur part num {}, whole parts {}", _st,
_completed_parts.size(), _cur_part_num, _dump_completed_part());
LOG(WARNING) << _st;
return _st;
}
@ -423,8 +425,9 @@ Status S3FileWriter::_complete() {
for (size_t i = 0; i < _completed_parts.size(); i++) {
if (_completed_parts[i]->GetPartNumber() != i + 1) [[unlikely]] {
auto st = Status::InternalError(
"error status {}, part num not continous, expected num {}, actual num {}", _st,
i + 1, _completed_parts[i]->GetPartNumber());
"error status {}, part num not continous, expected num {}, actual num {}, "
"whole parts {}",
_st, i + 1, _completed_parts[i]->GetPartNumber(), _dump_completed_part());
LOG(WARNING) << st;
_st = st;
return st;
@ -445,9 +448,10 @@ Status S3FileWriter::_complete() {
auto complete_outcome = _client->CompleteMultipartUpload(complete_request);
if (!complete_outcome.IsSuccess()) {
_st = s3fs_error(complete_outcome.GetError(),
fmt::format("failed to complete multi part upload {}, upload_id={}",
_path.native(), _upload_id));
_st = s3fs_error(
complete_outcome.GetError(),
fmt::format("failed to complete multi part upload {}, upload_id={}, whole parts={}",
_path.native(), _upload_id, _dump_completed_part()));
LOG(WARNING) << _st;
return _st;
}
@ -504,4 +508,12 @@ void S3FileWriter::_put_object(UploadFileBuffer& buf) {
s3_file_created_total << 1;
}
std::string S3FileWriter::_dump_completed_part() const {
std::string view;
for (const auto& part : _completed_parts) {
view.append(fmt::format("part {}, ", view, part->GetPartNumber()));
}
return view;
}
} // namespace doris::io

1
be/src/io/fs/s3_file_writer.h
View File

@ -53,6 +53,7 @@ public:
private:
Status _abort();
[[nodiscard]] std::string _dump_completed_part() const;
void _wait_until_finish(std::string_view task_name);
Status _complete();
Status _create_multi_upload_request();