hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add Error Codes for S3 Region-Endpoint Mismatch and Invalid Object Storage Endpoint Validation

Browse Source
This commit is contained in:
obdev
2024-03-07 07:15:38 +00:00
committed by ob-robot
parent 82552b3f6b
commit fd55142354
16 changed files with 117 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
deps/oblib/src/lib/ob_errno.h vendored
View File

@ -410,6 +410,8 @@ constexpr int OB_TENANT_SNAPSHOT_LOCK_CONFLICT = -9112;
constexpr int OB_CHECKSUM_TYPE_NOT_SUPPORTED = -9113;
constexpr int OB_INVALID_STORAGE_DEST = -9114;
constexpr int OB_OBJECT_STORAGE_PERMISSION_DENIED = -9116;
constexpr int OB_S3_REGION_MISMATCH = -9117;
constexpr int OB_INVALID_OBJECT_STORAGE_ENDPOINT = -9118;
constexpr int OB_ERR_XML_PARSE = -9549;
constexpr int OB_ERR_XSLT_PARSE = -9574;
constexpr int OB_MAX_RAISE_APPLICATION_ERROR = -20000;

28
deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp vendored
View File

@ -45,6 +45,7 @@ constexpr int OB_COS_ERROR = -9060;
constexpr int OB_IO_LIMIT = -9061;
constexpr int OB_BACKUP_PERMISSION_DENIED = -9071;
constexpr int OB_BACKUP_PWRITE_OFFSET_NOT_MATCH = -9083;
constexpr int OB_INVALID_OBJECT_STORAGE_ENDPOINT = -9118;
const int COS_BAD_REQUEST = 400;
const int COS_OBJECT_NOT_EXIST = 404;
@ -94,8 +95,12 @@ static void convert_io_error(cos_status_t *cos_ret, int &ob_errcode)
break;
}
case COS_BAD_REQUEST: {
if (0 == strcmp("InvalidDigest", cos_ret->error_code)) {
if (nullptr == cos_ret->error_code) {
ob_errcode = OB_COS_ERROR;
} else if (0 == strcmp("InvalidDigest", cos_ret->error_code)) {
ob_errcode = OB_CHECKSUM_ERROR;
} else if (0 == strcmp("InvalidRegionName", cos_ret->error_code)) {
ob_errcode = OB_INVALID_OBJECT_STORAGE_ENDPOINT;
} else {
ob_errcode = OB_COS_ERROR;
}
@ -185,8 +190,6 @@ int ObCosAccount::parse_from(const char *storage_info, uint32_t size)
if (OB_SUCCESS != (ret = set_field(token + strlen(DELETE_MODE), delete_mode_, sizeof(delete_mode_)))) {
cos_warn_log("[COS]fail to set delete_mode=%s, ret=%d", token, ret);
}
} else {
cos_warn_log("[COS]unkown token:%s\n", token);
}
}
@ -245,26 +248,26 @@ static void log_status(cos_status_t *s, const int ob_errcode)
{
if (NULL != s) {
if (OB_CHECKSUM_ERROR == ob_errcode) {
cos_error_log("[COS]status->code: %d", s->code);
cos_error_log("[COS]status->code: %d, ret=%d", s->code, ob_errcode);
if (s->error_code) {
cos_error_log("[COS]status->error_code: %s", s->error_code);
cos_error_log("[COS]status->error_code: %s, ret=%d", s->error_code, ob_errcode);
}
if (s->error_msg) {
cos_error_log("[COS]status->error_msg: %s", s->error_msg);
cos_error_log("[COS]status->error_msg: %s, ret=%d", s->error_msg, ob_errcode);
}
if (s->req_id) {
cos_error_log("[COS]status->req_id: %s", s->req_id);
cos_error_log("[COS]status->req_id: %s, ret=%d", s->req_id, ob_errcode);
}
} else {
cos_warn_log("[COS]status->code: %d", s->code);
cos_warn_log("[COS]status->code: %d, ret=%d", s->code, ob_errcode);
if (s->error_code) {
cos_warn_log("[COS]status->error_code: %s", s->error_code);
cos_warn_log("[COS]status->error_code: %s, ret=%d", s->error_code, ob_errcode);
}
if (s->error_msg) {
cos_warn_log("[COS]status->error_msg: %s", s->error_msg);
cos_warn_log("[COS]status->error_msg: %s, ret=%d", s->error_msg, ob_errcode);
}
if (s->req_id) {
cos_warn_log("[COS]status->req_id: %s", s->req_id);
cos_warn_log("[COS]status->req_id: %s, ret=%d", s->req_id, ob_errcode);
}
}
}
@ -341,11 +344,10 @@ int ObCosWrapper::create_cos_handle(
if (check_md5) {
cos_set_content_md5_enable(ctx->options->ctl, COS_TRUE);
ctx->options->ctl->options->enable_crc = false;
} else {
cos_set_content_md5_enable(ctx->options->ctl, COS_FALSE);
ctx->options->ctl->options->enable_crc = true;
}
ctx->options->ctl->options->enable_crc = false;
}
}

19
deps/oblib/src/lib/restore/ob_storage.cpp vendored
View File

@ -343,7 +343,7 @@ int ObStorageUtil::open(common::ObObjectStorageInfo *storage_info)
STORAGE_LOG(WARN, "double init the storage util", K(ret));
} else if (OB_ISNULL(storage_info) || OB_UNLIKELY(!storage_info->is_valid())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), KP(storage_info));
STORAGE_LOG(WARN, "invalid arguments", K(ret), KPC(storage_info));
} else if (OB_FALSE_IT(device_type_ = storage_info->get_type())) {
} else if (OB_STORAGE_OSS == device_type_) {
util_ = &oss_util_;
@ -1370,6 +1370,9 @@ int ObStorageReader::open(const common::ObString &uri, common::ObObjectStorageIn
} else if (NULL != reader_) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "cannot open twice", K(ret), K(uri));
} else if (OB_ISNULL(storage_info) || OB_UNLIKELY(uri.empty() || !storage_info->is_valid())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid arguments", K(ret), K(uri), KPC(storage_info));
} else if (OB_FAIL(databuff_printf(uri_, sizeof(uri_), "%.*s", uri.length(), uri.ptr()))) {
STORAGE_LOG(WARN, "failed to fill uri", K(ret), K(uri));
} else if (OB_FAIL(get_storage_type_from_path(uri, type))) {
@ -1502,6 +1505,9 @@ int ObStorageAdaptiveReader::open(const common::ObString &uri,
} else if (NULL != reader_) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "cannot open twice", K(ret), K(uri));
} else if (OB_ISNULL(storage_info) || OB_UNLIKELY(uri.empty() || !storage_info->is_valid())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid arguments", K(ret), K(uri), KPC(storage_info));
} else if (OB_FAIL(databuff_printf(uri_, sizeof(uri_), "%.*s", uri.length(), uri.ptr()))) {
STORAGE_LOG(WARN, "failed to fill uri", K(ret), K(uri));
} else if (OB_FAIL(get_storage_type_from_path(uri, type))) {
@ -1701,6 +1707,9 @@ int ObStorageWriter::open(const common::ObString &uri, common::ObObjectStorageIn
} else if (NULL != writer_) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "cannot open twice", K(ret), K(uri));
} else if (OB_ISNULL(storage_info) || OB_UNLIKELY(uri.empty() || !storage_info->is_valid())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid arguments", K(ret), K(uri), KPC(storage_info));
} else if (OB_FAIL(databuff_printf(uri_, sizeof(uri_), "%.*s", uri.length(), uri.ptr()))) {
STORAGE_LOG(WARN, "failed to fill uri", K(ret), K(uri));
} else if (OB_FAIL(get_storage_type_from_path(uri, type))) {
@ -1842,9 +1851,9 @@ int ObStorageAppender::open(
} else if (NULL != appender_) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "cannot open twice", K(ret), K(uri));
} else if (OB_UNLIKELY(uri.empty()) || OB_ISNULL(storage_info)) {
} else if (OB_ISNULL(storage_info) || OB_UNLIKELY(uri.empty() || !storage_info->is_valid())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(uri), KP(storage_info));
STORAGE_LOG(WARN, "invalid arguments", K(ret), K(uri), KPC(storage_info));
} else if (OB_FAIL(databuff_printf(uri_, sizeof(uri_), "%.*s", uri.length(), uri.ptr()))) {
STORAGE_LOG(WARN, "failed to fill uri", K(ret), K(uri));
} else if (OB_FAIL(get_storage_type_from_path(uri, type_))) {
@ -2137,9 +2146,9 @@ int ObStorageMultiPartWriter::open(
} else if (NULL != multipart_writer_) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "multipart writer cannot open twice", K(ret), K(uri));
} else if (OB_UNLIKELY(uri.empty()) || OB_ISNULL(storage_info)) {
} else if (OB_ISNULL(storage_info) || OB_UNLIKELY(uri.empty() || !storage_info->is_valid())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(uri), KPC(storage_info));
STORAGE_LOG(WARN, "invalid arguments", K(ret), K(uri), KPC(storage_info));
} else if (OB_FAIL(databuff_printf(uri_, sizeof(uri_), "%.*s", uri.length(), uri.ptr()))) {
STORAGE_LOG(WARN, "failed to fill uri", K(ret), K(uri));
} else if (OB_FAIL(get_storage_type_from_path(uri, type))) {

7
deps/oblib/src/lib/restore/ob_storage_cos_base.cpp vendored
View File

@ -537,12 +537,12 @@ int ObStorageCosBase::open(
ObObjectStorageInfo *storage_info)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(uri.empty()) || OB_ISNULL(storage_info)) {
if (OB_ISNULL(storage_info) || OB_UNLIKELY(uri.empty() || !storage_info->is_valid())) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "uri is empty", K(ret), K(uri), KP(storage_info));
OB_LOG(WARN, "uri is empty", K(ret), K(uri), KPC(storage_info));
} else if (FALSE_IT(checksum_type_ = storage_info->get_checksum_type())) {
} else if (OB_UNLIKELY(!is_cos_supported_checksum(checksum_type_))) {
ret = OB_NOT_SUPPORTED;
ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
OB_LOG(WARN, "that checksum algorithm is not supported for cos", K(ret), K_(checksum_type));
} else if (OB_FAIL(init_handle(*storage_info))) {
OB_LOG(WARN, "failed to init cos wrapper handle", K(ret), K(uri));
@ -887,6 +887,7 @@ int ObStorageCosWriter::pwrite(const char *buf, const int64_t size, const int64_
int ObStorageCosWriter::write(const char *buf, const int64_t size)
{
int ret = OB_SUCCESS;
ObExternalIOCounterGuard io_guard;
if (!is_opened_) {
ret = OB_NOT_INIT;
OB_LOG(WARN, "cos writer not opened", K(ret));

10
deps/oblib/src/lib/restore/ob_storage_info.cpp vendored
View File

@ -140,9 +140,6 @@ int ObObjectStorageInfo::set(const common::ObStorageType device_type, const char
} else if (OB_STORAGE_COS == device_type && !has_needed_extension) {
ret = OB_INVALID_BACKUP_DEST;
LOG_WARN("invalid cos info, appid do not allow to be empty", K(ret), K_(extension));
} else if (OB_STORAGE_S3 == device_type && !has_needed_extension) {
ret = OB_INVALID_BACKUP_DEST;
LOG_WARN("invalid s3 info, region do not allow to be empty", K(ret), K_(extension));
} else if (OB_STORAGE_FILE == device_type
&& (0 != strlen(endpoint_) || 0 != strlen(access_id_) || 0 != strlen(access_key_))) {
ret = OB_INVALID_BACKUP_DEST;
@ -190,7 +187,6 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has
if (NULL == token) {
break;
} else if (0 == strncmp(REGION, token, strlen(REGION))) {
has_needed_extension = (OB_STORAGE_S3 == device_type_);
if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) {
LOG_WARN("failed to set region", K(ret), K(token));
}
@ -327,6 +323,7 @@ int ObObjectStorageInfo::assign(const ObObjectStorageInfo &storage_info)
{
int ret = OB_SUCCESS;
device_type_ = storage_info.device_type_;
checksum_type_ = storage_info.checksum_type_;
MEMCPY(endpoint_, storage_info.endpoint_, sizeof(endpoint_));
MEMCPY(access_id_, storage_info.access_id_, sizeof(access_id_));
MEMCPY(access_key_, storage_info.access_key_, sizeof(access_key_));
@ -349,8 +346,9 @@ int ObObjectStorageInfo::get_storage_info_str(char *storage_info, const int64_t
} else if (OB_STORAGE_FILE != device_type_) {
if (OB_FAIL(get_access_key_(key, sizeof(key)))) {
LOG_WARN("failed to get access key", K(ret));
} else if (OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s",
endpoint_, access_id_, key))) {
} else if (OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s&%s%s",
endpoint_, access_id_, key,
CHECKSUM_TYPE, get_checksum_type_str()))) {
LOG_WARN("failed to set storage info", K(ret), K(info_len));
}
}

7
deps/oblib/src/lib/restore/ob_storage_info.h vendored
View File

@ -79,7 +79,8 @@ public:
int64_t hash() const;
bool operator ==(const ObObjectStorageInfo &storage_info) const;
bool operator !=(const ObObjectStorageInfo &storage_info) const;
TO_STRING_KV(K_(endpoint), K_(access_id), K_(extension), "type", get_type_str());
TO_STRING_KV(K_(endpoint), K_(access_id), K_(extension),
"type", get_type_str(), K_(checksum_type));
protected:
virtual int get_access_key_(char *key_buf, const int64_t key_buf_len) const;
@ -89,6 +90,10 @@ protected:
int set_storage_info_field_(const char *info, char *field, const int64_t length);
public:
// TODO: Rename device_type_ to storage_protocol_type_ for better clarity
// Prefix in the storage_info string, such as 's3://', indicates the protocol used to access the target
// Currently, both OBS and GCS are accessed via the s3 protocol,
// hence s3_region is updated to be an optional parameter
common::ObStorageType device_type_;
// Optional parameter. If not provided, the default value OB_MD5_ALGO will be used.
// For OSS/COS, OB_NO_CHECKSUM_ALGO indicates that no checksum algorithm will be used.

21
deps/oblib/src/lib/restore/ob_storage_oss_base.cpp vendored
View File

@ -227,8 +227,12 @@ static void convert_io_error(aos_status_t *aos_ret, int &ob_errcode)
}
case OSS_BAD_REQUEST: {
if (0 == STRCMP("InvalidDigest", aos_ret->error_code)) {
if (OB_ISNULL(aos_ret->error_code)) {
ob_errcode = OB_OSS_ERROR;
} else if (0 == STRCMP("InvalidDigest", aos_ret->error_code)) {
ob_errcode = OB_CHECKSUM_ERROR;
} else if (0 == STRCMP("InvalidBucketName", aos_ret->error_code)) {
ob_errcode = OB_INVALID_OBJECT_STORAGE_ENDPOINT;
} else {
ob_errcode = OB_OSS_ERROR;
}
@ -375,9 +379,9 @@ int ObStorageOssBase::init_with_storage_info(common::ObObjectStorageInfo *storag
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
OB_LOG(WARN, "oss client init twice", K(ret));
} else if (OB_ISNULL(storage_info)) {
} else if (OB_ISNULL(storage_info) || OB_UNLIKELY(!storage_info->is_valid())) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "oss account is null, fail to init oss base!", K(ret));
OB_LOG(WARN, "oss account is invalid, fail to init oss base!", K(ret), KPC(storage_info));
} else if (OB_FAIL(storage_info->get_storage_info_str(info_str, sizeof(info_str)))) {
OB_LOG(WARN, "fail to get storage info str", K(ret), KPC(storage_info));
} else if (OB_FAIL(oss_account_.parse_oss_arg(info_str))) {
@ -390,15 +394,11 @@ int ObStorageOssBase::init_with_storage_info(common::ObObjectStorageInfo *storag
} else {
checksum_type_ = storage_info->get_checksum_type();
if (OB_UNLIKELY(!is_oss_supported_checksum(checksum_type_))) {
ret = OB_NOT_SUPPORTED;
ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
OB_LOG(WARN, "that checksum algorithm is not supported for oss", K(ret), K_(checksum_type));
} else {
is_inited_ = true;
if (checksum_type_ == ObStorageChecksumType::OB_MD5_ALGO) {
oss_option_->ctl->options->enable_crc = false;
} else {
oss_option_->ctl->options->enable_crc = true;
}
oss_option_->ctl->options->enable_crc = false;
}
}
return ret;
@ -722,6 +722,9 @@ int ObStorageOssMultiPartWriter::open(const ObString &uri, common::ObObjectStora
convert_io_error(aos_ret, ret);
OB_LOG(WARN, "oss init multipart upload error", K(uri), K(ret));
print_oss_info(resp_headers, aos_ret, ret);
} else if (OB_ISNULL(upload_id_.data) || OB_UNLIKELY(upload_id_.len <= 0)) {
ret = OB_OSS_ERROR;
OB_LOG(WARN, "upload id is invalid", K(ret), KP(upload_id_.data), K(upload_id_.len));
} else {
is_opened_ = true;
base_buf_pos_ = 0;

3
deps/oblib/src/lib/restore/ob_storage_oss_base.h vendored
View File

@ -156,7 +156,6 @@ private:
common::ObArenaAllocator allocator_;
common::ObString bucket_;
common::ObString object_;
ObStorageChecksumType checksum_type_;
DISALLOW_COPY_AND_ASSIGN(ObStorageOssWriter);
};
@ -190,7 +189,6 @@ private:
int partnum_;
bool is_opened_;
int64_t file_length_;
ObStorageChecksumType checksum_type_;
DISALLOW_COPY_AND_ASSIGN(ObStorageOssMultiPartWriter);
};
@ -285,7 +283,6 @@ private:
common::ObArenaAllocator allocator_;
common::ObString bucket_;
common::ObString object_;
ObStorageChecksumType checksum_type_;
DISALLOW_COPY_AND_ASSIGN(ObStorageOssAppendWriter);
};

18
deps/oblib/src/lib/restore/ob_storage_s3_base.cpp vendored
View File

@ -575,6 +575,9 @@ static void convert_http_error(const Aws::S3::S3Error &s3_err, int &ob_errcode)
case S3_BAD_REQUEST: {
if (exception == "InvalidRequest" && err_msg.find("x-amz-checksum") != std::string::npos) {
ob_errcode = OB_CHECKSUM_ERROR;
} else if (err_msg.find("region") != std::string::npos
&& err_msg.find("is wrong; expecting") != std::string::npos) {
ob_errcode = OB_S3_REGION_MISMATCH;
} else {
ob_errcode = OB_S3_ERROR;
}
@ -616,6 +619,10 @@ static void convert_io_error(const Aws::S3::S3Error &s3_err, int &ob_errcode)
ob_errcode = OB_BACKUP_PERMISSION_DENIED;
break;
}
case Aws::S3::S3Errors::NO_SUCH_BUCKET: {
ob_errcode = OB_INVALID_OBJECT_STORAGE_ENDPOINT;
break;
}
default: {
convert_http_error(s3_err, ob_errcode);
break;
@ -652,7 +659,7 @@ static int set_request_checkusum_algorithm(RequestType &request,
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_s3_supported_checksum(checksum_type))) {
ret = OB_NOT_SUPPORTED;
ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
OB_LOG(WARN, "that checksum algorithm is not supported for s3", K(ret), K(checksum_type));
} else {
if (checksum_type == ObStorageChecksumType::OB_CRC32_ALGO) {
@ -670,7 +677,7 @@ static int set_completed_part_checksum(Aws::S3::Model::CompletedPart &completed_
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_s3_supported_checksum(checksum_type))) {
ret = OB_NOT_SUPPORTED;
ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
OB_LOG(WARN, "that checksum algorithm is not supported for s3", K(ret), K(checksum_type));
} else {
if (checksum_type == ObStorageChecksumType::OB_CRC32_ALGO) {
@ -941,9 +948,9 @@ int ObStorageS3Base::open(const ObString &uri, ObObjectStorageInfo *storage_info
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
OB_LOG(WARN, "s3 base alreagy inited", K(ret));
} else if (OB_UNLIKELY(uri.empty()) || OB_ISNULL(storage_info)) {
} else if (OB_ISNULL(storage_info) || OB_UNLIKELY(uri.empty() || !storage_info->is_valid())) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "failed to init s3 base, invalid arguments", K(ret), K(uri), KP(storage_info));
OB_LOG(WARN, "failed to init s3 base, invalid arguments", K(ret), K(uri), KPC(storage_info));
} else if (OB_FAIL(build_bucket_and_object_name(allocator_, uri, bucket_, object_))) {
OB_LOG(WARN, "failed to parse uri", K(ret), K(uri));
} else if (OB_FAIL(storage_info->get_storage_info_str(info_str, sizeof(info_str)))) {
@ -955,7 +962,7 @@ int ObStorageS3Base::open(const ObString &uri, ObObjectStorageInfo *storage_info
} else {
checksum_type_ = storage_info->get_checksum_type();
if (OB_UNLIKELY(!is_s3_supported_checksum(checksum_type_))) {
ret = OB_NOT_SUPPORTED;
ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
OB_LOG(WARN, "that checksum algorithm is not supported for s3", K(ret), K_(checksum_type));
} else {
is_inited_ = true;
@ -1091,6 +1098,7 @@ int ObStorageS3Writer::write_(const char *buf, const int64_t size)
{
int ret = OB_SUCCESS;
Aws::S3::Model::PutObjectRequest request;
ObExternalIOCounterGuard io_guard;
if (OB_UNLIKELY(!is_opened_)) {
ret = OB_NOT_INIT;
OB_LOG(WARN, "s3 writer not opened", K(ret));