hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add tests for object storage to sensitive test

Browse Source
This commit is contained in:
obdev
2024-02-09 17:50:21 +00:00
committed by ob-robot
parent 1d31b939ec
commit 335cda36b4
9 changed files with 97 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp vendored
View File

@ -134,7 +134,7 @@ int ObCosAccount::parse_from(const char *storage_info, uint32_t size)
int ret = OB_SUCCESS;
if (NULL == storage_info || MAX_COS_DOMAIN_LENGTH <= size) {
ret = OB_INVALID_ARGUMENT;
cos_warn_log("[COS]cos parse account failed, storage_info=%s, size=%d, ret=%d\n", storage_info, size, ret);
cos_warn_log("[COS]cos parse account failed, storage_info=%p, size=%d, ret=%d\n", storage_info, size, ret);
} else {
// host=xxxx&access_id=xxx&access_key=xxx&appid=xxx
char tmp[MAX_COS_DOMAIN_LENGTH];
@ -169,7 +169,7 @@ int ObCosAccount::parse_from(const char *storage_info, uint32_t size)
}
} else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) {
if (OB_SUCCESS != (ret = set_field(token + strlen(ACCESS_KEY), access_key_, sizeof(access_key_)))) {
cos_warn_log("[COS]fail to set access_key=%s, ret=%d\n", token, ret);
cos_warn_log("[COS]fail to set access_key, ret=%d\n", ret);
} else {
bitmap |= (1 << 2);
}
@ -190,7 +190,7 @@ int ObCosAccount::parse_from(const char *storage_info, uint32_t size)
if (OB_SUCCESS == ret && bitmap != 0x0F) {
ret = OB_COS_ERROR;
cos_warn_log("[COS]fail to parse cos account storage_info=%s, bitmap=%x, ret=%d\n", storage_info, bitmap, ret);
cos_warn_log("[COS]fail to parse cos account storage_info=%p, bitmap=%x, ret=%d\n", storage_info, bitmap, ret);
}
}

4
deps/oblib/src/lib/restore/ob_object_device.cpp vendored
View File

@ -57,7 +57,7 @@ void ObObjectDevice::destroy()
ObObjectDevice::~ObObjectDevice()
{
destroy();
OB_LOG(INFO, "destory the device!", KCSTRING(storage_info_str_));
OB_LOG(INFO, "destory the device!", KP(storage_info_str_));
}
/*the app logical use call ObBackupIoAdapter::get_and_init_device*/
@ -83,7 +83,7 @@ int ObObjectDevice::start(const ObIODOpts &opts)
if (OB_SUCCESS != ret) {
//mem resource will be free with device destroy
} else if (OB_FAIL(util_.open(&storage_info_))) {
OB_LOG(WARN, "fail to open the util!", K(ret), KCSTRING(opts.opts_[0].value_.value_str));
OB_LOG(WARN, "fail to open the util!", K(ret), KP(opts.opts_[0].value_.value_str));
} else if (OB_FAIL(fd_mng_.init())) {
OB_LOG(WARN, "fail to init fd manager!", K(ret));
} else {

12
deps/oblib/src/lib/restore/ob_storage_info.cpp vendored
View File

@ -107,7 +107,7 @@ int ObObjectStorageInfo::set(const common::ObStorageType device_type, const char
LOG_WARN("storage info init twice", K(ret));
} else if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) {
ret = OB_INVALID_BACKUP_DEST;
LOG_WARN("storage info is invalid", K(ret), K(storage_info));
LOG_WARN("storage info is invalid", K(ret), KP(storage_info));
} else if (FALSE_IT(device_type_ = device_type)) {
} else if (0 == strlen(storage_info)) {
if (OB_STORAGE_FILE != device_type_) {
@ -163,7 +163,7 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has
has_needed_extension = false;
if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) {
ret = OB_INVALID_BACKUP_DEST;
LOG_WARN("storage info is invalid", K(ret), K(storage_info), K(strlen(storage_info)));
LOG_WARN("storage info is invalid", K(ret), KP(storage_info), K(strlen(storage_info)));
} else {
char tmp[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
char *token = NULL;
@ -192,7 +192,7 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has
}
} else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) {
if (OB_FAIL(set_storage_info_field_(token, access_key_, sizeof(access_key_)))) {
LOG_WARN("failed to set access key", K(ret), K(token));
LOG_WARN("failed to set access key", K(ret));
}
} else if (OB_STORAGE_FILE != device_type_ && 0 == strncmp(APPID, token, strlen(APPID))) {
has_needed_extension = (OB_STORAGE_COS == device_type_);
@ -250,14 +250,14 @@ int ObObjectStorageInfo::set_storage_info_field_(const char *info, char *field,
int64_t pos = strlen(field);
if (info_len >= length) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("info is too long ", K(ret), K(info), K(length));
LOG_WARN("info is too long ", K(ret), K(info_len), K(length));
} else if (pos > 0 && OB_FAIL(databuff_printf(field, length, pos, "&"))) {
// cos:host=xxxx&access_id=xxx&access_key=xxxappid=xxx&delete_mode=xxx
// extension_ may contain both appid and delete_mode
// so delimiter '&' should be included
LOG_WARN("failed to add delimiter to storage info field", K(ret), K(info), K(field), K(length));
LOG_WARN("failed to add delimiter to storage info field", K(ret), K(pos), KP(field), K(length));
} else if (OB_FAIL(databuff_printf(field, length, pos, "%s", info))) {
LOG_WARN("failed to set storage info field", K(ret), K(info), K(field), K(length));
LOG_WARN("failed to set storage info field", K(ret), K(pos), KP(field), K(length));
}
}
return ret;

12
deps/oblib/src/lib/restore/ob_storage_oss_base.cpp vendored
View File

@ -436,7 +436,7 @@ int ObOssAccount::parse_oss_arg(const common::ObString &storage_info)
OB_LOG(WARN, "oss client init twice", K(ret));
} else if (OB_ISNULL(storage_info.ptr()) || storage_info.length() >= OB_MAX_URI_LENGTH) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "uri is too long", K(ret), K(storage_info), K(storage_info.length()));
OB_LOG(WARN, "uri is too long", K(ret), K(storage_info.length()));
} else {
// host=xxxx&access_id=xxx&access_key=xxx
char tmp[OB_MAX_URI_LENGTH];
@ -464,24 +464,24 @@ int ObOssAccount::parse_oss_arg(const common::ObString &storage_info)
}
} else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) {
if (OB_FAIL(set_oss_field(token + strlen(ACCESS_KEY), oss_key_, sizeof(oss_key_)))) {
OB_LOG(WARN, "failed to set oss_key_", K(ret), KCSTRING(token));
OB_LOG(WARN, "failed to set oss_key_", K(ret));
}
} else if (0 == strncmp(DELETE_MODE, token, strlen(DELETE_MODE))) {
if (OB_FAIL(set_delete_mode(token + strlen(DELETE_MODE)))) {
OB_LOG(WARN, "failed to set delete mode", K(ret), K(token));
}
} else {
OB_LOG(DEBUG, "unkown oss info", K(*token), K(storage_info));
OB_LOG(DEBUG, "unkown oss info", K(*token), KP(storage_info.ptr()));
}
}
if (strlen(oss_domain_) == 0 || strlen(oss_id_) == 0 || strlen(oss_key_) == 0) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "failed to parse oss info",
K(ret), KCSTRING(oss_domain_), KCSTRING(oss_id_), KCSTRING(oss_key_), K(storage_info));
K(ret), KCSTRING(oss_domain_), KCSTRING(oss_id_));
} else {
STORAGE_LOG(DEBUG, "success to parse oss info",
K(ret), KCSTRING(oss_domain_), KCSTRING(oss_id_), KCSTRING(oss_key_), K(storage_info));
K(ret), KCSTRING(oss_domain_), KCSTRING(oss_id_));
}
}
return ret;
@ -498,7 +498,7 @@ int ObOssAccount::set_oss_field(const char *info, char *field, const int64_t len
const int64_t info_len = strlen(info);
if (info_len >= length) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "info is too long ", K(ret), KCSTRING(info), K(length));
OB_LOG(WARN, "info is too long ", K(ret), K(info_len), K(length));
} else {
MEMCPY(field, info, info_len);
field[info_len] = '\0';

4
deps/oblib/unittest/lib/CMakeLists.txt vendored
View File

@ -89,8 +89,8 @@ oblib_addtest(resource/test_resource_mgr.cpp)
#oblib_addtest(restore/test_storage_oss.cpp)
oblib_addtest(restore/test_storage_cos.cpp)
oblib_addtest(restore/test_storage_s3.cpp)
oblib_addtest(restore/test_object_storage.cpp)
oblib_addtest(restore/test_common_storage.cpp)
# oblib_addtest(restore/test_object_storage.cpp)
# oblib_addtest(restore/test_common_storage.cpp)
oblib_addtest(restore/test_storage_info.cpp)
#oblib_addtest(restore/test_storage.cpp)
oblib_addtest(stat/test_di_cache.cpp)

6
src/share/backup/ob_backup_struct.cpp
View File

@ -1176,9 +1176,9 @@ int ObBackupStorageInfo::parse_storage_info_(const char *storage_info, bool &has
int ret = OB_SUCCESS;
if (OB_ISNULL(storage_info) || strlen(storage_info) >= OB_MAX_BACKUP_STORAGE_INFO_LENGTH) {
ret = OB_INVALID_BACKUP_DEST;
LOG_WARN("storage info is invalid", K(ret), K(storage_info), K(strlen(storage_info)));
LOG_WARN("storage info is invalid", K(ret), KP(storage_info), K(strlen(storage_info)));
} else if (OB_FAIL(ObObjectStorageInfo::parse_storage_info_(storage_info, has_needed_extension))) {
LOG_WARN("failed to parse storage info", K(ret), K(storage_info));
LOG_WARN("failed to parse storage info", K(ret), KP(storage_info));
} else {
char tmp[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
char serialize_key[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 };
@ -1260,7 +1260,7 @@ int ObBackupStorageInfo::decrypt_access_key_(const char *buf)
} else if (OB_FAIL(ObEncryptionUtil::decrypt_sys_data(OB_SYS_TENANT_ID,
deserialize_buf, deserialize_size,
decrypt_key, sizeof(decrypt_key), key_len))) {
LOG_WARN("failed to decrypt authorization key", K(ret), K(deserialize_buf), K(deserialize_size));
LOG_WARN("failed to decrypt authorization key", K(ret), KP(deserialize_buf), K(deserialize_size));
} else if (key_len >= sizeof(decrypt_key) || (key_len + strlen(ACCESS_KEY)) >= sizeof(access_key_)) {
ret = OB_SIZE_OVERFLOW;
LOG_WARN("decrypt key size overflow", K(ret), K(key_len), K(sizeof(decrypt_key)));

31
src/share/ob_device_manager.cpp
View File

@ -167,7 +167,7 @@ int ObDeviceManager::alloc_device(ObDeviceInsInfo*& device_info,
}
if (-1 == avai_idx && -1 == last_no_ref_idx) {
OB_LOG(WARN, "devices too mang!", K(MAX_DEVICE_INSTANCE), K(storage_info), K(storage_type_prefix));
OB_LOG(WARN, "devices too mang!", K(MAX_DEVICE_INSTANCE), KP(storage_info.ptr()), K(storage_type_prefix));
//cannot insert into device manager
ret = OB_OUT_OF_ELEMENT;
} else {
@ -176,9 +176,11 @@ int ObDeviceManager::alloc_device(ObDeviceInsInfo*& device_info,
//erase from map
ObString old_key(device_ins_[last_no_ref_idx].storage_info_);
if (OB_FAIL(device_map_.erase_refactored(old_key))) {
OB_LOG(WARN, "fail to erase device from device map", K(old_key), K(ret), K(storage_info), K(storage_type_prefix));
OB_LOG(WARN, "fail to erase device from device map",
KP(old_key.ptr()), K(ret), KP(storage_info.ptr()), K(storage_type_prefix));
} else if (OB_FAIL(handle_map_.erase_refactored((int64_t)(device_ins_[last_no_ref_idx].device_)))) {
OB_LOG(WARN, "fail to erase device from handle map", K(device_ins_[last_no_ref_idx].device_), K(ret), K(storage_info), K(storage_type_prefix));
OB_LOG(WARN, "fail to erase device from handle map", K(ret),
K(device_ins_[last_no_ref_idx].device_), KP(storage_info.ptr()), K(storage_type_prefix));
} else {
/*free the resource*/
ObIODevice* del_device = device_ins_[last_no_ref_idx].device_;
@ -188,7 +190,8 @@ int ObDeviceManager::alloc_device(ObDeviceInsInfo*& device_info,
abort_unless(device_count_ == MAX_DEVICE_INSTANCE);
device_count_--;
avai_idx = last_no_ref_idx;
OB_LOG(INFO, "release one device for realloc another!", K(old_key), K(storage_info), K(storage_type_prefix));
OB_LOG(INFO, "release one device for realloc another!",
KP(old_key.ptr()), KP(storage_info.ptr()), K(storage_type_prefix));
}
}
@ -197,11 +200,14 @@ int ObDeviceManager::alloc_device(ObDeviceInsInfo*& device_info,
STRCPY(device_ins_[avai_idx].storage_info_, storage_info.ptr());
ObString cur_key(device_ins_[avai_idx].storage_info_);
if (OB_FAIL(device_map_.set_refactored(cur_key, &(device_ins_[avai_idx])))) {
OB_LOG(WARN, "fail to set device to device map!", K(ret), K(cur_key), K(storage_type_prefix));
OB_LOG(WARN, "fail to set device to device map!",
K(ret), KP(storage_info.ptr()), K(storage_type_prefix));
} else if (OB_FAIL(handle_map_.set_refactored((int64_t)(device_handle), &(device_ins_[avai_idx])))) {
OB_LOG(WARN, "fail to set device to handle map!", K(ret), K(storage_info), K(storage_type_prefix));
OB_LOG(WARN, "fail to set device to handle map!",
K(ret), KP(storage_info.ptr()), K(storage_type_prefix));
} else {
OB_LOG(INFO, "success insert into map!", K(storage_info), K(storage_type_prefix));
OB_LOG(INFO, "success insert into map!",
KP(storage_info.ptr()), K(storage_type_prefix));
}
}
}
@ -216,7 +222,8 @@ int ObDeviceManager::alloc_device(ObDeviceInsInfo*& device_info,
device_ins_[avai_idx].device_ = device_handle;
device_ins_[avai_idx].ref_cnt_ = 0;
device_count_++;
OB_LOG(INFO, "alloc a new device!", K(storage_info), K(storage_type_prefix), K(avai_idx), K(device_count_), K(device_handle));
OB_LOG(INFO, "alloc a new device!", KP(storage_info.ptr()),
K(storage_type_prefix), K(avai_idx), K(device_count_), K(device_handle));
device_info = &(device_ins_[avai_idx]);
}
@ -254,10 +261,12 @@ int ObDeviceManager::get_device(const common::ObString& storage_info,
if (OB_HASH_NOT_EXIST == ret) {
//alloc a device, and set into the map
if (OB_FAIL(alloc_device(dev_info, storage_info_tmp, storage_type_prefix))) {
OB_LOG(WARN, "fail to alloc device!", K(ret), K(storage_info_tmp), K(storage_type_prefix));
OB_LOG(WARN, "fail to alloc device!",
K(ret), KP(storage_info_tmp.ptr()), K(storage_type_prefix));
}
} else {
OB_LOG(WARN, "fail to get device from device manager ", K(ret), K(storage_info_tmp), K(storage_type_prefix));
OB_LOG(WARN, "fail to get device from device manager ",
K(ret), KP(storage_info_tmp.ptr()), K(storage_type_prefix));
}
}
@ -300,7 +309,7 @@ int ObDeviceManager::release_device(ObIODevice*& device_handle)
abort_unless(device_info->ref_cnt_ > 0);
device_info->ref_cnt_--;
if (0 == device_info->ref_cnt_) {
OB_LOG(DEBUG, "A Device has no others ref", K(device_info->device_), K(device_info->storage_info_));
OB_LOG(DEBUG, "A Device has no others ref", K(device_info->device_), KP(device_info->storage_info_));
} else {
OB_LOG(DEBUG, "released dev info", K(device_info->device_), K(device_info->ref_cnt_));
}

6
tools/ob_admin/io_bench/ob_admin_io_adapter_bench.cpp
View File

@ -42,7 +42,7 @@ int ObAdminIOAdapterBenchmarkExecutor::execute(int argc, char *argv[])
if (OB_FAIL(parse_cmd_(argc, argv))) {
OB_LOG(WARN, "failed to parse cmd", K(ret), K(argc), K(argv));
} else if (OB_FAIL(run_all_tests_())) {
OB_LOG(WARN, "failed to pass all tests", K(ret), K_(base_path), K_(storage_info_str));
OB_LOG(WARN, "failed to pass all tests", K(ret), K_(base_path));
}
return ret;
}
@ -85,7 +85,7 @@ int ObAdminIOAdapterBenchmarkExecutor::parse_cmd_(int argc, char *argv[])
}
case 's': {
if (OB_FAIL(databuff_printf(storage_info_str_, sizeof(storage_info_str_), "%s", optarg))) {
OB_LOG(WARN, "failed to copy storage info str", K(ret), K((char *)optarg));
OB_LOG(WARN, "failed to copy storage info str", K(ret));
}
break;
}
@ -240,7 +240,7 @@ int ObAdminIOAdapterBenchmarkExecutor::run_all_tests_()
if (FALSE_IT(init_random_content())) {
} else if (OB_FAIL(storage_info.set(base_path_, storage_info_str_))) {
OB_LOG(WARN, "failed to set storage info", K(ret), K_(base_path), K_(storage_info_str));
OB_LOG(WARN, "failed to set storage info", K(ret), K_(base_path));
} else if (OB_FAIL(runner.init(base_path_, &storage_info, config_))) {
OB_LOG(WARN, "fail to init ObBackupIoAdapterBenchmarkRunner",
K(ret), K_(base_path), K(storage_info), K_(config));

104
tools/ob_admin/io_device/ob_admin_test_io_device_executor.cpp
View File

@ -47,7 +47,7 @@ int ObAdminTestIODeviceExecutor::execute(int argc, char *argv[])
if(OB_FAIL(ret)) {
} else if (OB_FAIL(run_all_tests_())) {
STORAGE_LOG(WARN, "failed to pass all tests", K(ret), K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to pass all tests", K(ret), K_(backup_path));
}
return ret;
}
@ -102,15 +102,15 @@ int ObAdminTestIODeviceExecutor::run_all_tests_()
{
int ret = OB_SUCCESS;
if (OB_FAIL(test_backup_dest_connectivity_())) {
STORAGE_LOG(WARN, "test backup dest connectivity failed", K(ret), K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "test backup dest connectivity failed", K(ret), K_(backup_path));
} else if (OB_FAIL(test_archive_log_())) {
STORAGE_LOG(WARN, "test archive log failed", K(ret), K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "test archive log failed", K(ret), K_(backup_path));
} else if (OB_FAIL(test_backup_data_())) {
STORAGE_LOG(WARN, "test backup data failed", K(ret), K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "test backup data failed", K(ret), K_(backup_path));
} else if (OB_FAIL(test_consume_clog_file_())) {
STORAGE_LOG(WARN, "test consume clog file failed", K(ret));
} else if (OB_FAIL(test_clean_backup_file_())) {
STORAGE_LOG(WARN, "test clean backup file failed", K(ret), K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "test clean backup file failed", K(ret), K_(backup_path));
}
return ret;
}
@ -123,9 +123,9 @@ int ObAdminTestIODeviceExecutor::test_backup_dest_connectivity_()
bool is_empty_directory = false;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(util.is_empty_directory(backup_path_, &storage_info, is_empty_directory))) {
STORAGE_LOG(WARN, "failed check backup dest is empty dir", K(ret), K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed check backup dest is empty dir", K(ret), K_(backup_path));
} else if (!is_empty_directory) {
ret = OB_INVALID_BACKUP_DEST;
STORAGE_LOG(WARN, "back up dest is not empty!", K(ret), K_(backup_path));
@ -164,19 +164,19 @@ int ObAdminTestIODeviceExecutor::test_normal_check_file_(const char* check_file_
int64_t read_size = 0;
bool is_exist = false;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(check_file_path, OB_MAX_URI_LENGTH,
"%s%s", check_file_dir, check_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.write_single_file(check_file_path, &storage_info, check_file_content, real_len))) {
STORAGE_LOG(WARN, "failed to write check file", K(ret), K(check_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to write check file", K(ret), K(check_file_path));
} else if (OB_FAIL(util.is_exist(check_file_path, &storage_info, is_exist))) {
STORAGE_LOG(WARN, "failed to check if normal check file is exist", K(ret), K(check_file_path));
} else if (!is_exist) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "normal check file does not exist!", K(ret), K(check_file_path));
} else if (OB_FAIL(util.get_file_length(check_file_path, &storage_info, check_file_len))) {
STORAGE_LOG(WARN, "failed to get check file length", K(ret), K(check_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to get check file length", K(ret), K(check_file_path));
} else if (real_len != check_file_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get check file length does not match real length", K(ret), K(check_file_path), K(real_len), K(check_file_len));
@ -212,33 +212,33 @@ int ObAdminTestIODeviceExecutor::test_appendable_check_file_(const char* check_f
int64_t offset = 0;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(appendable_check_file_path,
OB_MAX_URI_LENGTH, "%s%s", check_file_dir, appendable_check_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.open_with_access_type(device_handle, fd, &storage_info,
appendable_check_file_path, access_type))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret),
K(appendable_check_file_path), K_(storage_info), K(access_type));
K(appendable_check_file_path), K(access_type));
} else if (OB_ISNULL(device_handle)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "device handle is NULL", K(ret), K(device_handle), K_(backup_path));
} else if (OB_FAIL(device_handle->pwrite(fd, offset, real_len,
appendable_check_file_content, write_size))) {
STORAGE_LOG(WARN, "failed to pwrite appendable check file", K(ret), K(appendable_check_file_path),
K_(storage_info), K(appendable_check_file_content), K(real_len));
K(appendable_check_file_content), K(real_len));
} else if (FALSE_IT(offset += real_len)){
} else if (OB_FAIL(device_handle->pwrite(fd, offset, real_len,
appendable_check_file_content, write_size))) {
STORAGE_LOG(WARN, "failed to repeat pwrite appendable check file", K(ret), K(appendable_check_file_path),
K_(storage_info), K(appendable_check_file_content), K(real_len));
K(appendable_check_file_content), K(real_len));
} else if (FALSE_IT(real_len += real_len)) {
} else if (OB_FAIL(device_handle->seal_file(fd))) {
STORAGE_LOG(WARN, "failed to seal appendable check file", K(ret), K(appendable_check_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to seal appendable check file", K(ret), K(appendable_check_file_path));
} else if (OB_FAIL(device_handle->seal_file(fd))) {
STORAGE_LOG(WARN, "failed to repeat seal appendable check file", K(ret), K(appendable_check_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to repeat seal appendable check file", K(ret), K(appendable_check_file_path));
} else if (OB_FAIL(util.close_device_and_fd(device_handle, fd))) {
STORAGE_LOG(WARN, "fail to close appendable file and release device!", K(ret), K(appendable_check_file_path), K_(storage_info));
STORAGE_LOG(WARN, "fail to close appendable file and release device!", K(ret), K(appendable_check_file_path));
} else if (OB_FAIL(util.adaptively_is_exist(appendable_check_file_path, &storage_info, is_exist))) {
STORAGE_LOG(WARN, "failed to check if normal check file is exist", K(ret), K(appendable_check_file_path));
} else if (!is_exist) {
@ -246,7 +246,7 @@ int ObAdminTestIODeviceExecutor::test_appendable_check_file_(const char* check_f
STORAGE_LOG(WARN, "appendable check file does not exist!", K(ret), K(appendable_check_file_path));
} else if (OB_FAIL(util.adaptively_get_file_length(appendable_check_file_path, &storage_info,
check_file_len))) {
STORAGE_LOG(WARN, "fail to get appendable check file length", K(ret), K(appendable_check_file_path), K_(storage_info));
STORAGE_LOG(WARN, "fail to get appendable check file length", K(ret), K(appendable_check_file_path));
} else if (check_file_len != real_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get appendable check file length does not match real length", K(ret), K(appendable_check_file_path),
@ -283,19 +283,19 @@ int ObAdminTestIODeviceExecutor::test_multipart_upload_check_file_(const char* c
int64_t read_size = 0;
bool is_exist = false;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(check_file_path, OB_MAX_URI_LENGTH,
"%s%s", check_file_dir, check_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.open_with_access_type(device_handle, fd, &storage_info,
check_file_path, access_type))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret),
K(check_file_path), K_(storage_info), K(access_type));
K(check_file_path), K(access_type));
} else if (OB_ISNULL(device_handle)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "device handle is NULL", K(ret), K(device_handle), K_(backup_path));
} else if (OB_FAIL(device_handle->write(fd, check_file_content, real_len, write_size))) {
STORAGE_LOG(WARN, "failed to write check file", K(ret), K_(storage_info), K(check_file_content), K(real_len));
STORAGE_LOG(WARN, "failed to write check file", K(ret), K(check_file_content), K(real_len));
} else if (OB_FAIL(util.close_device_and_fd(device_handle, fd))) {
STORAGE_LOG(WARN, "fail to close device and fd", K(ret), KP(device_handle), K(fd));
} else if (OB_FAIL(util.is_exist(check_file_path, &storage_info, is_exist))) {
@ -304,7 +304,7 @@ int ObAdminTestIODeviceExecutor::test_multipart_upload_check_file_(const char* c
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "normal check file does not exist!", K(ret), K(check_file_path));
} else if (OB_FAIL(util.get_file_length(check_file_path, &storage_info, check_file_len))) {
STORAGE_LOG(WARN, "failed to get check file length", K(ret), K(check_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to get check file length", K(ret), K(check_file_path));
} else if (real_len != check_file_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get check file length does not match real length", K(ret), K(check_file_path), K(real_len), K(check_file_len));
@ -365,7 +365,7 @@ int ObAdminTestIODeviceExecutor::test_clean_backup_file_()
char appendable_check_file_path[OB_MAX_URI_LENGTH] = { 0 };
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(check_file_dir_path, OB_MAX_URI_LENGTH, "%s%s%s",
backup_path_, "/", check_file_dir_name))) {
STORAGE_LOG(WARN, "fail to databuff printf", K(ret));
@ -373,14 +373,14 @@ int ObAdminTestIODeviceExecutor::test_clean_backup_file_()
check_file_dir_path, check_file_name))) {
STORAGE_LOG(WARN, "fail to databuff printf", K(ret));
} else if (OB_FAIL(util.del_file(check_file_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to del normal check file", K(ret), K(check_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to del normal check file", K(ret), K(check_file_path));
} else if (OB_FAIL(databuff_printf(appendable_check_file_path, OB_MAX_URI_LENGTH, "%s%s",
check_file_dir_path, appendable_check_file_name))) {
STORAGE_LOG(WARN, "fail to databuff printf", K(ret));
} else if (OB_FAIL(util.del_file(appendable_check_file_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to del appendable check file", K(ret), K(appendable_check_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to del appendable check file", K(ret), K(appendable_check_file_path));
} else if (OB_FAIL(util.del_dir(check_file_dir_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to del check file dir", K(ret), K(check_file_dir_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to del check file dir", K(ret), K(check_file_dir_path));
} else { // remove the whole test dir
if (ObStorageType::OB_STORAGE_FILE == storage_info.get_type()) {
char cmd[OB_MAX_URI_LENGTH] = { 0 };
@ -424,21 +424,21 @@ int ObAdminTestIODeviceExecutor::test_backup_data_()
int64_t total_size = 0;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(meta_file_path, OB_MAX_URI_LENGTH, "%s%s%s%s",
backup_path_, "/", meta_dir_name, meta_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.mk_parent_dir(meta_file_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to make parent dir for meta file", K(ret), K(meta_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to make parent dir for meta file", K(ret), K(meta_file_path));
} else if (OB_FAIL(util.write_single_file(meta_file_path, &storage_info, meta_file_content, real_len))) {
STORAGE_LOG(WARN, "failed to write single meta file", K(ret), K(meta_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to write single meta file", K(ret), K(meta_file_path));
} else if (OB_FAIL(databuff_printf(data_file_path, OB_MAX_URI_LENGTH, "%s%s%s",
backup_path_, "/", data_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.open_with_access_type(device_handle, fd, &storage_info,
data_file_path, access_type))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret),
K(data_file_path), K_(storage_info), K(access_type));
K(data_file_path), K(access_type));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < UPLOAD_TIMES; ++i) {
char* buf = nullptr;
@ -447,7 +447,7 @@ int ObAdminTestIODeviceExecutor::test_backup_data_()
STORAGE_LOG(WARN, "failed to generate random str", K(ret), K(buf_size));
} else if (OB_FAIL(device_handle->pwrite(fd, offset, buf_size,
buf, write_size))) {
STORAGE_LOG(WARN, "failed to upload part", K(ret), K(data_file_path), K_(storage_info), K(buf_size));
STORAGE_LOG(WARN, "failed to upload part", K(ret), K(data_file_path), K(buf_size));
} else if (write_size != buf_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "upload data size does not match buf size", K(ret), K(write_size), K(buf_size));
@ -457,9 +457,9 @@ int ObAdminTestIODeviceExecutor::test_backup_data_()
}
if(OB_SUCC(ret)) {
if (OB_FAIL(util.close_device_and_fd(device_handle, fd))) {
STORAGE_LOG(WARN, "fail to close multipart writer and release device!", K(ret), K(data_file_path), K_(storage_info));
STORAGE_LOG(WARN, "fail to close multipart writer and release device!", K(ret), K(data_file_path));
} else if (OB_FAIL(util.get_file_length(data_file_path, &storage_info, read_size))) {
STORAGE_LOG(WARN, "fail to get multipart upload file length", K(ret), K(data_file_path), K_(storage_info));
STORAGE_LOG(WARN, "fail to get multipart upload file length", K(ret), K(data_file_path));
} else if (read_size != total_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get multipart upload file length does not match real file length", K(ret), K(read_size), K(total_size));
@ -494,14 +494,14 @@ int ObAdminTestIODeviceExecutor::test_archive_log_() {
int64_t real_len = strlen(meta_file_content);
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(meta_file_path, OB_MAX_URI_LENGTH, "%s%s%s%s",
backup_path_, "/", meta_dir_name, meta_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.mk_parent_dir(meta_file_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to make parent dir for meta file", K(ret), K(meta_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to make parent dir for meta file", K(ret), K(meta_file_path));
} else if (OB_FAIL(util.write_single_file(meta_file_path, &storage_info, meta_file_content, real_len))) {
STORAGE_LOG(WARN, "failed to write single meta file", K(ret), K(meta_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to write single meta file", K(ret), K(meta_file_path));
} else if (OB_FAIL(test_full_clog_file_())) {
STORAGE_LOG(WARN, "test full clog file failed", K(ret));
} else if (OB_FAIL(test_partial_clog_file_())) {
@ -523,18 +523,18 @@ int ObAdminTestIODeviceExecutor::test_full_clog_file_()
int64_t file_size = 0;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(full_log_file_path, OB_MAX_URI_LENGTH, "%s%s%s",
backup_path_, "/", full_log_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
} else if (OB_FAIL(generate_random_str_(buf, buf_size))) {
STORAGE_LOG(WARN, "failed to generate random str", K(ret), K(buf_size));
} else if (OB_FAIL(util.mk_parent_dir(full_log_file_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to make parent dir for full clog file", K(ret), K(full_log_file_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to make parent dir for full clog file", K(ret), K(full_log_file_path));
} else if (OB_FAIL(util.write_single_file(full_log_file_path, &storage_info, buf, buf_size))) {
STORAGE_LOG(WARN, "failed to write single full clog file", K(full_log_file_path), K_(storage_info), K(buf_size));
STORAGE_LOG(WARN, "failed to write single full clog file", K(full_log_file_path), K(buf_size));
} else if (OB_FAIL(util.get_file_length(full_log_file_path, &storage_info,file_size))) {
STORAGE_LOG(WARN, "failed to get full clog file length", K(full_log_file_path), K_(storage_info), K(file_size));
STORAGE_LOG(WARN, "failed to get full clog file length", K(full_log_file_path), K(file_size));
} else if (CLOG_FILE_SIZE != file_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get full clog file length does not match real length", K(file_size), K(buf_size));
@ -558,14 +558,14 @@ int ObAdminTestIODeviceExecutor::test_partial_clog_file_()
int64_t total_size = 0;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(partial_log_file_path, OB_MAX_URI_LENGTH, "%s%s%s",
backup_path_, "/", partial_log_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.open_with_access_type(device_handle, fd, &storage_info,
partial_log_file_path, access_type))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret),
K(partial_log_file_path), K_(storage_info), K(access_type));
K(partial_log_file_path), K(access_type));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < APPEND_TIMES; ++i) {
char* buf = nullptr;
@ -574,7 +574,7 @@ int ObAdminTestIODeviceExecutor::test_partial_clog_file_()
STORAGE_LOG(WARN, "failed to generate random str", K(ret), K(buf_size));
} else if (OB_FAIL(device_handle->pwrite(fd, offset, buf_size,
buf, write_size))) {
STORAGE_LOG(WARN, "failed to append clog", K(ret), K(partial_log_file_path), K_(storage_info), K(offset), K(buf_size));
STORAGE_LOG(WARN, "failed to append clog", K(ret), K(partial_log_file_path), K(offset), K(buf_size));
} else if (write_size != buf_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "append data size does not match buf size", K(ret), K(write_size), K(buf_size));
@ -587,16 +587,16 @@ int ObAdminTestIODeviceExecutor::test_partial_clog_file_()
int64_t remain_size = 0;
char* remain_buf = nullptr;
if (OB_FAIL(util.close_device_and_fd(device_handle, fd))) {
STORAGE_LOG(WARN, "fail to close append writer and release device!", K(ret), K(partial_log_file_path), K_(storage_info));
STORAGE_LOG(WARN, "fail to close append writer and release device!", K(ret), K(partial_log_file_path));
} else if (OB_FAIL(util.adaptively_get_file_length(partial_log_file_path, &storage_info, read_size))) {
STORAGE_LOG(WARN, "fail to get appendable clog file length", K(ret), K(partial_log_file_path), K_(storage_info));
STORAGE_LOG(WARN, "fail to get appendable clog file length", K(ret), K(partial_log_file_path));
} else if (read_size != total_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get appendable clog file length does not match real file length", K(ret), K(read_size), K(total_size));
} else if (OB_FAIL(util.open_with_access_type(device_handle, fd, &storage_info,
partial_log_file_path, access_type))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret),
K(partial_log_file_path), K_(storage_info), K(access_type));
K(partial_log_file_path), K(access_type));
} else if (FALSE_IT(remain_size = CLOG_FILE_SIZE - total_size)) {
} else if (OB_FAIL(generate_random_str_(remain_buf, remain_size))) {
STORAGE_LOG(WARN, "failed to generate remaining random str for appendable clog file", K(ret), K(remain_size));
@ -606,9 +606,9 @@ int ObAdminTestIODeviceExecutor::test_partial_clog_file_()
} else if (OB_FAIL(device_handle->seal_file(fd))) {
STORAGE_LOG(WARN, "failed to seal for appendable clog file", K(ret), K(partial_log_file_path));
} else if (OB_FAIL(util.close_device_and_fd(device_handle, fd))) {
STORAGE_LOG(WARN, "fail to close append writer and release device!", K(ret), K(partial_log_file_path), K_(storage_info));
STORAGE_LOG(WARN, "fail to close append writer and release device!", K(ret), K(partial_log_file_path));
} else if (OB_FAIL(util.adaptively_get_file_length(partial_log_file_path, &storage_info, read_size))) {
STORAGE_LOG(WARN, "fail to get appendable clog file length", K(ret), K(partial_log_file_path), K_(storage_info));
STORAGE_LOG(WARN, "fail to get appendable clog file length", K(ret), K(partial_log_file_path));
} else if (read_size != total_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get appendable clog file length does not match real file length", K(ret), K(read_size), K(total_size));
@ -636,13 +636,13 @@ int ObAdminTestIODeviceExecutor::test_consume_clog_file_()
int64_t read_size = 0;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(ls_path, OB_MAX_URI_LENGTH, "%s%s%s",
backup_path_, "/", ls_dir))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
} else if (OB_FAIL(archive::ObArchiveFileUtils::get_file_range(ls_path,
&storage_info, min_file_id, max_file_id))) {
STORAGE_LOG(WARN, "failed to get clog file range", K(ret), K(ls_path), K_(storage_info));
STORAGE_LOG(WARN, "failed to get clog file range", K(ret), K(ls_path));
} else if (min_file_id != 1 || max_file_id != 2) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "got wrong clog file list", K(ret), K(min_file_id), K(max_file_id));
@ -654,7 +654,7 @@ int ObAdminTestIODeviceExecutor::test_consume_clog_file_()
STORAGE_LOG(WARN, "failed to allocate memory", K(ret), K(buf_len));
} else if (OB_FAIL(util.adaptively_read_part_file(full_clog_file_path, &storage_info, buf1, buf_len, offset, read_size))) {
STORAGE_LOG(WARN, "failed to read part from full clog file", K(ret), K(full_clog_file_path),
K_(storage_info), K(offset), K(buf_len));
K(offset), K(buf_len));
} else if (read_size != buf_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "read full clog file length does not match real buf length", K(ret), K(read_size), K(buf_len));
@ -668,7 +668,7 @@ int ObAdminTestIODeviceExecutor::test_consume_clog_file_()
STORAGE_LOG(WARN, "failed to allocate memory", K(ret), K(buf_len));
} else if (OB_FAIL(util.adaptively_read_part_file(partial_clog_file_path, &storage_info, buf2, buf_len, offset, read_size))) {
STORAGE_LOG(WARN, "failed to read part from full clog file", K(ret), K(full_clog_file_path),
K_(storage_info), K(offset), K(buf_len));
K(offset), K(buf_len));
} else if (read_size != buf_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "read partial clog file length does not match real buf length", K(ret), K(read_size), K(buf_len));