hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix object storage checksum

Browse Source
This commit is contained in:
LoLolobster
2024-08-12 14:59:02 +00:00
committed by ob-robot
parent d21904fa68
commit 11f38cff7c
7 changed files with 27 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
deps/oblib/src/lib/restore/ob_storage_cos_base.cpp vendored
View File

@ -551,6 +551,11 @@ int ObStorageCosBase::open(
} else if (OB_FAIL(handle_.build_bucket_and_object_name(uri))) {
OB_LOG(WARN, "failed to build bucket and object name", K(ret), K(uri));
}
#ifdef ERRSIM
if (OB_NOT_NULL(storage_info) && (OB_SUCCESS != EventTable::EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE)) {
OB_LOG(ERROR, "errsim backup io with checksum type", "checksum_type", storage_info->get_checksum_type_str());
}
#endif
return ret;
}

6
deps/oblib/src/lib/restore/ob_storage_info.cpp vendored
View File

@ -221,6 +221,8 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has
const char *checksum_type_str = token + strlen(CHECKSUM_TYPE);
if (OB_FAIL(set_checksum_type_(checksum_type_str))) {
OB_LOG(WARN, "fail to set checksum type", K(ret), K(checksum_type_str));
} else if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) {
LOG_WARN("fail to set checksum type into extension", K(ret), K(token));
}
} else {
}
@ -346,9 +348,7 @@ int ObObjectStorageInfo::get_storage_info_str(char *storage_info, const int64_t
} else if (OB_STORAGE_FILE != device_type_) {
if (OB_FAIL(get_access_key_(key, sizeof(key)))) {
LOG_WARN("failed to get access key", K(ret));
} else if (OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s&%s%s",
endpoint_, access_id_, key,
CHECKSUM_TYPE, get_checksum_type_str()))) {
} else if (OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s", endpoint_, access_id_, key))) {
LOG_WARN("failed to set storage info", K(ret), K(info_len));
}
}

5
deps/oblib/src/lib/restore/ob_storage_oss_base.cpp vendored
View File

@ -452,6 +452,11 @@ int ObStorageOssBase::init_with_storage_info(common::ObObjectStorageInfo *storag
OB_LOG(WARN, "aos pool or oss option is NULL", K(aos_pool_), K(oss_option_));
} else {
checksum_type_ = storage_info->get_checksum_type();
#ifdef ERRSIM
if (OB_NOT_NULL(storage_info) && (OB_SUCCESS != EventTable::EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE)) {
OB_LOG(ERROR, "errsim backup io with checksum type", "checksum_type", storage_info->get_checksum_type_str());
}
#endif
if (OB_UNLIKELY(!is_oss_supported_checksum(checksum_type_))) {
ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
OB_LOG(WARN, "that checksum algorithm is not supported for oss", K(ret), K_(checksum_type));

5
deps/oblib/src/lib/restore/ob_storage_s3_base.cpp vendored
View File

@ -992,6 +992,11 @@ int ObStorageS3Base::open(const ObString &uri, ObObjectStorageInfo *storage_info
OB_LOG(WARN, "faied to get s3 client", K(ret));
} else {
checksum_type_ = storage_info->get_checksum_type();
#ifdef ERRSIM
if (OB_NOT_NULL(storage_info) && (OB_SUCCESS != EventTable::EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE)) {
OB_LOG(ERROR, "errsim backup io with checksum type", "checksum_type", storage_info->get_checksum_type_str());
}
#endif
if (OB_UNLIKELY(!is_s3_supported_checksum(checksum_type_))) {
ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
OB_LOG(WARN, "that checksum algorithm is not supported for s3", K(ret), K_(checksum_type));

1
deps/oblib/src/lib/utility/ob_tracepoint_def.h vendored
View File

@ -495,6 +495,7 @@ GLOBAL_ERRSIM_POINT_DEF(1113, EN_RESTORE_TABLET_TASK_FAILED, "");
GLOBAL_ERRSIM_POINT_DEF(1114, EN_INSERT_USER_RECOVER_JOB_FAILED, "");
GLOBAL_ERRSIM_POINT_DEF(1115, EN_INSERT_AUX_TENANT_RESTORE_JOB_FAILED, "");
GLOBAL_ERRSIM_POINT_DEF(1116, EN_RESTORE_CREATE_LS_FAILED, "");
GLOBAL_ERRSIM_POINT_DEF(1117, EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE, "");
// END OF STORAGE HA - 1101 - 2000
// sql parameterization 1170-1180

8
deps/oblib/unittest/lib/restore/test_storage_info.cpp vendored
View File

@ -111,7 +111,7 @@ TEST(ObObjectStorageInfo, cos)
storage_info = "host=xxx.com&access_id=111&access_key=222";
ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
storage_info = "host=xxx.com&access_id=111&access_key=222&checksum_type=md5&appid=333";
storage_info = "host=xxx.com&access_id=111&access_key=222&appid=333&checksum_type=md5";
ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
@ -153,15 +153,15 @@ TEST(ObObjectStorageInfo, s3)
ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
info1.reset();
storage_info = "host=xxx.com&access_id=111&access_key=222&checksum_type=md5&s3_region=333";
storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&checksum_type=md5";
ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
ASSERT_EQ(0, ::strcmp("s3_region=333", info1.extension_));
ASSERT_EQ(0, ::strcmp("s3_region=333&checksum_type=md5", info1.extension_));
char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf)));
ASSERT_STREQ(storage_info, buf);
storage_info = "host=xxx.com&access_id=111&access_key=222&checksum_type=md5&s3_region=333&delete_mode=delete";
storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&delete_mode=delete&checksum_type=md5";
info1.reset();
ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf)));

8
unittest/share/backup/test_backup_struct.cpp
View File

@ -98,7 +98,7 @@ TEST(ObBackupDest, oss)
EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID,
obrpc::RootKeyType::DEFAULT, ObString()));
ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str)));
ASSERT_EQ(0, strcmp(backup_dest_str, "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&checksum_type=md5&delete_mode=tagging"));
ASSERT_EQ(0, strcmp(backup_dest_str, "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&delete_mode=tagging"));
ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str)));
ASSERT_EQ(0, strcmp(backup_path_str, "oss://backup_dir?host=xxx.com"));
ASSERT_TRUE(dest.is_root_path_equal(dest1));
@ -139,7 +139,7 @@ TEST(ObBackupDest, oss_encrypt)
char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 };
char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 };
ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str)));
ASSERT_EQ(0, strcmp(backup_dest_str, "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&checksum_type=md5"));
ASSERT_EQ(0, strcmp(backup_dest_str, "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F"));
ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str)));
ASSERT_EQ(0, strcmp(backup_path_str, "oss://backup_dir?host=xxx.com"));
@ -168,7 +168,7 @@ TEST(ObBackupDest, cos)
EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID,
obrpc::RootKeyType::DEFAULT, ObString()));
ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str)));
ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&checksum_type=md5&delete_mode=tagging&appid=333"));
ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&delete_mode=tagging&appid=333"));
ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str)));
ASSERT_EQ(0, strcmp(backup_path_str, "cos://backup_dir?host=xxx.com"));
ASSERT_TRUE(dest.is_root_path_equal(dest1));
@ -209,7 +209,7 @@ TEST(ObBackupDest, cos_encrypt)
char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 };
char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 };
ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str)));
ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&checksum_type=md5&appid=333"));
ASSERT_EQ(0, strcmp(backup_dest_str, "cos://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&appid=333"));
ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str)));
ASSERT_EQ(0, strcmp(backup_path_str, "cos://backup_dir?host=xxx.com"));