hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CP] improved readability of ob_admin test_io_device test results

Browse Source
This commit is contained in:
kpole 2024-11-07 07:13:57 +00:00 committed by ob-robot
parent 9c517debaf
commit 414c279c12
4 changed files with 319 additions and 266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

291
tools/ob_admin/io_device/ob_admin_test_io_device_executor.cpp
View File

@ -13,7 +13,6 @@
#include <time.h>
#include "ob_admin_test_io_device_executor.h"
#include "share/backup/ob_backup_io_adapter.h"
#include "../dumpsst/ob_admin_dumpsst_print_helper.h"
#include "src/logservice/archiveservice/ob_archive_file_utils.h"
#include "src/share/backup/ob_backup_path.h"
#include "src/share/backup/ob_backup_clean_util.h"
@ -25,7 +24,10 @@ using namespace oceanbase::common;
namespace oceanbase {
namespace tools {
ObAdminTestIODeviceExecutor::ObAdminTestIODeviceExecutor() : is_quiet_(false), allocator_()
ObAdminTestIODeviceExecutor::ObAdminTestIODeviceExecutor()
: is_quiet_(false),
test_id_(1),
allocator_()
{
MEMSET(backup_path_, 0, common::OB_MAX_URI_LENGTH);
MEMSET(storage_info_, 0, common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH);
@ -60,7 +62,7 @@ int ObAdminTestIODeviceExecutor::execute(int argc, char *argv[])
}
if (FAILEDx(parse_cmd_(argc, argv))) {
STORAGE_LOG(WARN, "failed to parse cmd", K(ret), K(argc), K(argv));
STORAGE_LOG_FILTER(ERROR, "failed to parse cmd", K(ret), K(argc), K(argv));
} else if (is_quiet_) {
OB_LOGGER.set_log_level("WARN");
} else {
@ -69,7 +71,7 @@ int ObAdminTestIODeviceExecutor::execute(int argc, char *argv[])
if(OB_FAIL(ret)) {
} else if (OB_FAIL(run_all_tests_())) {
STORAGE_LOG(WARN, "failed to pass all tests", K(ret), K_(backup_path));
STORAGE_LOG(ERROR, "failed to pass all tests", K(ret), K_(backup_path));
}
return ret;
}
@ -98,13 +100,13 @@ int ObAdminTestIODeviceExecutor::parse_cmd_(int argc, char *argv[])
char buffer[OB_MAX_TIME_STR_LENGTH];
strftime(buffer, sizeof(buffer), "%Y-%m-%d-%H:%M:%S", timeinfo);
if (OB_FAIL(databuff_printf(backup_path_, sizeof(backup_path_), "%s%s", optarg, buffer))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
}
break;
}
case 's': {
if (OB_FAIL(databuff_printf(storage_info_, sizeof(storage_info_), "%s", optarg))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
}
break;
}
@ -114,7 +116,7 @@ int ObAdminTestIODeviceExecutor::parse_cmd_(int argc, char *argv[])
}
case 'e': {
if (OB_FAIL(set_s3_url_encode_type(optarg))) {
STORAGE_LOG(WARN, "failed to set s3 url encode type", KR(ret));
STORAGE_LOG_FILTER(ERROR, "failed to set s3 url encode type", KR(ret));
}
break;
}
@ -131,19 +133,19 @@ int ObAdminTestIODeviceExecutor::run_all_tests_()
{
int ret = OB_SUCCESS;
if (OB_FAIL(test_backup_dest_connectivity_())) {
STORAGE_LOG(WARN, "test backup dest connectivity failed", K(ret), K_(backup_path));
STORAGE_LOG(ERROR, "test backup dest connectivity failed", K(ret), K_(backup_path));
} else if (OB_FAIL(test_archive_log_())) {
STORAGE_LOG(WARN, "test archive log failed", K(ret), K_(backup_path));
STORAGE_LOG(ERROR, "test archive log failed", K(ret), K_(backup_path));
} else if (OB_FAIL(test_backup_data_())) {
STORAGE_LOG(WARN, "test backup data failed", K(ret), K_(backup_path));
STORAGE_LOG(ERROR, "test backup data failed", K(ret), K_(backup_path));
} else if (OB_FAIL(test_consume_clog_file_())) {
STORAGE_LOG(WARN, "test consume clog file failed", K(ret));
STORAGE_LOG(ERROR, "test consume clog file failed", K(ret));
} else if (OB_FAIL(test_clean_backup_file_())) {
STORAGE_LOG(WARN, "test clean backup file failed", K(ret), K_(backup_path));
STORAGE_LOG(ERROR, "test clean backup file failed", K(ret), K_(backup_path));
} else if (OB_FAIL(test_object_storage_interface_())) {
STORAGE_LOG(WARN, "test object storage interface failed", K(ret), K_(backup_path));
STORAGE_LOG(ERROR, "test object storage interface failed", K(ret), K_(backup_path));
} else if (OB_FAIL(test_list_before_complete_multipart_write_())) {
STORAGE_LOG(WARN, "test list before complete multipart write failed", K(ret), K_(backup_path));
STORAGE_LOG(ERROR, "test list before complete multipart write failed", K(ret), K_(backup_path));
}
return ret;
}
@ -151,35 +153,35 @@ int ObAdminTestIODeviceExecutor::run_all_tests_()
int ObAdminTestIODeviceExecutor::test_backup_dest_connectivity_()
{
int ret = OB_SUCCESS;
TestExecGuard guard("Backup dest connectivity", test_id_, ret);
ObBackupIoAdapter util;
share::ObBackupStorageInfo storage_info;
bool is_empty_directory = false;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(util.is_empty_directory(backup_path_, &storage_info, is_empty_directory))) {
STORAGE_LOG(WARN, "failed check backup dest is empty dir", K(ret), K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "The path is inaccessible, please check the path and the storage info parameter", K(ret), K_(backup_path));
} else if (!is_empty_directory) {
ret = OB_INVALID_BACKUP_DEST;
STORAGE_LOG(WARN, "back up dest is not empty!", K(ret), K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "Path not empty directory", K(ret), K_(backup_path));
} else {
const char* check_file_dir_name = "check_file/";
char check_file_dir[OB_MAX_URI_LENGTH] = { 0 };
if (OB_FAIL(databuff_printf(check_file_dir, OB_MAX_URI_LENGTH,
"%s%s%s", backup_path_, "/", check_file_dir_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.mkdir(check_file_dir, &storage_info))) {
STORAGE_LOG(WARN, "failed to make check file dir", K(ret), K(check_file_dir));
STORAGE_LOG_FILTER(ERROR, "failed to make check file dir", K(ret), K(check_file_dir));
} else if (OB_FAIL(test_normal_check_file_(check_file_dir))) {
STORAGE_LOG(WARN, "test normal check file failed", K(ret));
STORAGE_LOG_FILTER(ERROR, "test normal check file failed", K(ret));
} else if (OB_FAIL(test_appendable_check_file_(check_file_dir))) {
STORAGE_LOG(WARN, "test appendable check file failed", K(ret));
STORAGE_LOG_FILTER(ERROR, "test appendable check file failed", K(ret));
} else if (OB_FAIL(test_multipart_upload_check_file_(check_file_dir))) {
STORAGE_LOG(WARN, "test multipart upload check file failed", K(ret));
STORAGE_LOG_FILTER(ERROR, "test multipart upload check file failed", K(ret));
}
}
PrintHelper::print_dump_line("test dest connnectivity result", OB_SUCC(ret) ? "SUCCESS" : "FAIL");
return ret;
}
@ -197,32 +199,32 @@ int ObAdminTestIODeviceExecutor::test_normal_check_file_(const char* check_file_
int64_t read_size = 0;
bool is_exist = false;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(check_file_path, OB_MAX_URI_LENGTH,
"%s%s", check_file_dir, check_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.write_single_file(check_file_path, &storage_info, check_file_content,
real_len, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to write check file", K(ret), K(check_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to write check file", K(ret), K(check_file_path));
} else if (OB_FAIL(util.is_exist(check_file_path, &storage_info, is_exist))) {
STORAGE_LOG(WARN, "failed to check if normal check file is exist", K(ret), K(check_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to check if normal check file is exist", K(ret), K(check_file_path));
} else if (!is_exist) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "normal check file does not exist!", K(ret), K(check_file_path));
STORAGE_LOG_FILTER(ERROR, "normal check file does not exist!", K(ret), K(check_file_path));
} else if (OB_FAIL(util.get_file_length(check_file_path, &storage_info, check_file_len))) {
STORAGE_LOG(WARN, "failed to get check file length", K(ret), K(check_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to get check file length", K(ret), K(check_file_path));
} else if (real_len != check_file_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get check file length does not match real length", K(ret), K(check_file_path), K(real_len), K(check_file_len));
STORAGE_LOG_FILTER(ERROR, "get check file length does not match real length", K(ret), K(check_file_path), K(real_len), K(check_file_len));
} else if (OB_ISNULL(read_file_buf = reinterpret_cast<char*>(allocator_.alloc(check_file_len)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to allocate buf", K(ret), K(check_file_path), K(check_file_len));
STORAGE_LOG_FILTER(ERROR, "failed to allocate buf", K(ret), K(check_file_path), K(check_file_len));
} else if (OB_FAIL(util.read_single_file(check_file_path, &storage_info, read_file_buf,
check_file_len, read_size, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to read check file", K(ret), K(check_file_path), K(check_file_len));
STORAGE_LOG_FILTER(ERROR, "failed to read check file", K(ret), K(check_file_path), K(check_file_len));
} else if (read_size != check_file_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "read check file length does not match real length", K(ret), K(check_file_path), K(read_size), K(check_file_len));
STORAGE_LOG_FILTER(ERROR, "read check file length does not match real length", K(ret), K(check_file_path), K(read_size), K(check_file_len));
}
return ret;
}
@ -247,55 +249,55 @@ int ObAdminTestIODeviceExecutor::test_appendable_check_file_(const char* check_f
int64_t offset = 0;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(appendable_check_file_path,
OB_MAX_URI_LENGTH, "%s%s", check_file_dir, appendable_check_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.open_with_access_type(device_handle, fd, &storage_info,
appendable_check_file_path, access_type,
ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret),
STORAGE_LOG_FILTER(ERROR, "failed to open device with access type", K(ret),
K(appendable_check_file_path), K(access_type));
} else if (OB_ISNULL(device_handle)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "device handle is NULL", K(ret), K(device_handle), K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "device handle is NULL", K(ret), K(device_handle), K_(backup_path));
} else if (OB_FAIL(device_handle->pwrite(fd, offset, real_len,
appendable_check_file_content, write_size))) {
STORAGE_LOG(WARN, "failed to pwrite appendable check file", K(ret), K(appendable_check_file_path),
STORAGE_LOG_FILTER(ERROR, "failed to pwrite appendable check file", K(ret), K(appendable_check_file_path),
K(appendable_check_file_content), K(real_len));
} else if (FALSE_IT(offset += real_len)){
} else if (OB_FAIL(device_handle->pwrite(fd, offset, real_len,
appendable_check_file_content, write_size))) {
STORAGE_LOG(WARN, "failed to repeat pwrite appendable check file", K(ret), K(appendable_check_file_path),
STORAGE_LOG_FILTER(ERROR, "failed to repeat pwrite appendable check file", K(ret), K(appendable_check_file_path),
K(appendable_check_file_content), K(real_len));
} else if (FALSE_IT(real_len += real_len)) {
} else if (OB_FAIL(device_handle->seal_file(fd))) {
STORAGE_LOG(WARN, "failed to seal appendable check file", K(ret), K(appendable_check_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to seal appendable check file", K(ret), K(appendable_check_file_path));
} else if (OB_FAIL(device_handle->seal_file(fd))) {
STORAGE_LOG(WARN, "failed to repeat seal appendable check file", K(ret), K(appendable_check_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to repeat seal appendable check file", K(ret), K(appendable_check_file_path));
} else if (OB_FAIL(util.close_device_and_fd(device_handle, fd))) {
STORAGE_LOG(WARN, "fail to close appendable file and release device!", K(ret), K(appendable_check_file_path));
STORAGE_LOG_FILTER(ERROR, "fail to close appendable file and release device!", K(ret), K(appendable_check_file_path));
} else if (OB_FAIL(util.adaptively_is_exist(appendable_check_file_path, &storage_info, is_exist))) {
STORAGE_LOG(WARN, "failed to check if normal check file is exist", K(ret), K(appendable_check_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to check if normal check file is exist", K(ret), K(appendable_check_file_path));
} else if (!is_exist) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "appendable check file does not exist!", K(ret), K(appendable_check_file_path));
STORAGE_LOG_FILTER(ERROR, "appendable check file does not exist!", K(ret), K(appendable_check_file_path));
} else if (OB_FAIL(util.adaptively_get_file_length(appendable_check_file_path, &storage_info,
check_file_len))) {
STORAGE_LOG(WARN, "fail to get appendable check file length", K(ret), K(appendable_check_file_path));
STORAGE_LOG_FILTER(ERROR, "fail to get appendable check file length", K(ret), K(appendable_check_file_path));
} else if (check_file_len != real_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get appendable check file length does not match real length", K(ret), K(appendable_check_file_path),
STORAGE_LOG_FILTER(ERROR, "get appendable check file length does not match real length", K(ret), K(appendable_check_file_path),
K(real_len), K(check_file_len));
} else if (OB_ISNULL(read_file_buf = reinterpret_cast<char*>(allocator_.alloc(check_file_len)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to allocate buf", K(ret), K(appendable_check_file_path), K(check_file_len));
STORAGE_LOG_FILTER(ERROR, "failed to allocate buf", K(ret), K(appendable_check_file_path), K(check_file_len));
} else if (OB_FAIL(util.adaptively_read_single_file(appendable_check_file_path, &storage_info, read_file_buf,
check_file_len, read_size, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to read appendable check file", K(ret), K(appendable_check_file_path), K(check_file_len));
STORAGE_LOG_FILTER(ERROR, "failed to read appendable check file", K(ret), K(appendable_check_file_path), K(check_file_len));
} else if (read_size != check_file_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "read appendable check file length does not match real length", K(ret), K(appendable_check_file_path),
STORAGE_LOG_FILTER(ERROR, "read appendable check file length does not match real length", K(ret), K(appendable_check_file_path),
K(read_size), K(check_file_len));
}
return ret;
@ -320,53 +322,53 @@ int ObAdminTestIODeviceExecutor::test_multipart_upload_check_file_(const char* c
int64_t read_size = 0;
bool is_exist = false;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(check_file_path, OB_MAX_URI_LENGTH,
"%s%s", check_file_dir, check_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.open_with_access_type(device_handle, fd, &storage_info,
check_file_path, access_type,
ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret),
STORAGE_LOG_FILTER(ERROR, "failed to open device with access type", K(ret),
K(check_file_path), K(access_type));
} else if (OB_ISNULL(device_handle)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "device handle is NULL", K(ret), K(device_handle), K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "device handle is NULL", K(ret), K(device_handle), K_(backup_path));
} else if (OB_FAIL(device_handle->write(fd, check_file_content, real_len, write_size))) {
STORAGE_LOG(WARN, "failed to write check file", K(ret), K(check_file_content), K(real_len));
STORAGE_LOG_FILTER(ERROR, "failed to write check file", K(ret), K(check_file_content), K(real_len));
} else if (OB_FAIL(device_handle->complete(fd))) {
STORAGE_LOG(WARN, "fail to complete multipart upload", K(ret), K(device_handle), K(fd));
STORAGE_LOG_FILTER(ERROR, "fail to complete multipart upload", K(ret), K(device_handle), K(fd));
}
if (OB_SUCCESS != ret) {
if (OB_TMP_FAIL(device_handle->abort(fd))) {
ret = COVER_SUCC(tmp_ret);
STORAGE_LOG(WARN, "fail to abort multipart upload", K(ret), K(tmp_ret), K(device_handle), K(fd));
STORAGE_LOG_FILTER(ERROR, "fail to abort multipart upload", K(ret), K(tmp_ret), K(device_handle), K(fd));
}
}
if (OB_TMP_FAIL(util.close_device_and_fd(device_handle, fd))) {
ret = COVER_SUCC(tmp_ret);
STORAGE_LOG(WARN, "fail to close device and fd", K(ret), K(tmp_ret), K(device_handle), K(fd));
STORAGE_LOG_FILTER(ERROR, "fail to close device and fd", K(ret), K(tmp_ret), K(device_handle), K(fd));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(util.is_exist(check_file_path, &storage_info, is_exist))) {
STORAGE_LOG(WARN, "failed to check if normal check file is exist", K(ret), K(check_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to check if normal check file is exist", K(ret), K(check_file_path));
} else if (!is_exist) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "normal check file does not exist!", K(ret), K(check_file_path));
STORAGE_LOG_FILTER(ERROR, "normal check file does not exist!", K(ret), K(check_file_path));
} else if (OB_FAIL(util.get_file_length(check_file_path, &storage_info, check_file_len))) {
STORAGE_LOG(WARN, "failed to get check file length", K(ret), K(check_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to get check file length", K(ret), K(check_file_path));
} else if (real_len != check_file_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get check file length does not match real length", K(ret), K(check_file_path), K(real_len), K(check_file_len));
STORAGE_LOG_FILTER(ERROR, "get check file length does not match real length", K(ret), K(check_file_path), K(real_len), K(check_file_len));
} else if (OB_ISNULL(read_file_buf = reinterpret_cast<char*>(allocator_.alloc(check_file_len)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to allocate buf", K(ret), K(check_file_path), K(check_file_len));
STORAGE_LOG_FILTER(ERROR, "failed to allocate buf", K(ret), K(check_file_path), K(check_file_len));
} else if (OB_FAIL(util.read_single_file(check_file_path, &storage_info, read_file_buf,
check_file_len, read_size, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to read check file", K(ret), K(check_file_path), K(check_file_len));
STORAGE_LOG_FILTER(ERROR, "failed to read check file", K(ret), K(check_file_path), K(check_file_len));
} else if (read_size != check_file_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "read check file length does not match real length", K(ret), K(check_file_path), K(read_size), K(check_file_len));
STORAGE_LOG_FILTER(ERROR, "read check file length does not match real length", K(ret), K(check_file_path), K(read_size), K(check_file_len));
}
}
return ret;
@ -407,6 +409,7 @@ int CleanBackupPathOp::func(const dirent *entry)
int ObAdminTestIODeviceExecutor::test_clean_backup_file_()
{
int ret = OB_SUCCESS;
TestExecGuard guard("clean backup file", test_id_, ret);
ObBackupIoAdapter util;
share::ObBackupStorageInfo storage_info;
const char* check_file_dir_name = "check_file/";
@ -417,47 +420,46 @@ int ObAdminTestIODeviceExecutor::test_clean_backup_file_()
char appendable_check_file_path[OB_MAX_URI_LENGTH] = { 0 };
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(check_file_dir_path, OB_MAX_URI_LENGTH, "%s%s%s",
backup_path_, "/", check_file_dir_name))) {
STORAGE_LOG(WARN, "fail to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "fail to databuff printf", K(ret));
} else if (OB_FAIL(databuff_printf(check_file_path, OB_MAX_URI_LENGTH, "%s%s",
check_file_dir_path, check_file_name))) {
STORAGE_LOG(WARN, "fail to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "fail to databuff printf", K(ret));
} else if (OB_FAIL(util.del_file(check_file_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to del normal check file", K(ret), K(check_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to del normal check file", K(ret), K(check_file_path));
} else if (OB_FAIL(databuff_printf(appendable_check_file_path, OB_MAX_URI_LENGTH, "%s%s",
check_file_dir_path, appendable_check_file_name))) {
STORAGE_LOG(WARN, "fail to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "fail to databuff printf", K(ret));
} else if (OB_FAIL(util.del_file(appendable_check_file_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to del appendable check file", K(ret), K(appendable_check_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to del appendable check file", K(ret), K(appendable_check_file_path));
} else if (OB_FAIL(util.del_dir(check_file_dir_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to del check file dir", K(ret), K(check_file_dir_path));
STORAGE_LOG_FILTER(ERROR, "failed to del check file dir", K(ret), K(check_file_dir_path));
} else { // remove the whole test dir
if (ObStorageType::OB_STORAGE_FILE == storage_info.get_type()) {
char cmd[OB_MAX_URI_LENGTH] = { 0 };
if (OB_FAIL(databuff_printf(cmd, OB_MAX_URI_LENGTH,
"rm -rf %s/*", backup_path_ + strlen(OB_FILE_PREFIX)))) {
OB_LOG(WARN, "fail to fill clean cmd", K(ret), K_(backup_path));
} else if (0 != std::system(cmd)) {
ret = OB_ERR_UNEXPECTED;
OB_LOG(WARN, "fail to delete dir", K(ret), K_(backup_path), K(cmd));
}
} else {
CleanBackupPathOp op(backup_path_, &storage_info);
if (OB_FAIL(util.list_files(backup_path_, &storage_info, op))) {
OB_LOG(WARN, "fail to clean", K(ret), K_(backup_path), K(storage_info));
char cmd[OB_MAX_URI_LENGTH] = { 0 };
if (OB_FAIL(databuff_printf(cmd, OB_MAX_URI_LENGTH,
"rm -rf %s/*", backup_path_ + strlen(OB_FILE_PREFIX)))) {
STORAGE_LOG_FILTER(ERROR, "fail to fill clean cmd", K(ret), K_(backup_path));
} else if (0 != std::system(cmd)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG_FILTER(ERROR, "fail to delete dir", K(ret), K_(backup_path), K(cmd));
}
} else {
CleanBackupPathOp op(backup_path_, &storage_info);
if (OB_FAIL(util.list_files(backup_path_, &storage_info, op))) {
STORAGE_LOG_FILTER(ERROR, "fail to clean", K(ret), K_(backup_path), K(storage_info));
}
}
}
}
PrintHelper::print_dump_line("test clean backup file result", OB_SUCC(ret) ? "SUCCESS" : "FAIL");
return ret;
}
int ObAdminTestIODeviceExecutor::test_backup_data_()
{
int ret = OB_SUCCESS;
TestExecGuard guard("backup data", test_id_, ret);
int tmp_ret = OB_SUCCESS;
ObBackupIoAdapter util;
share::ObBackupStorageInfo storage_info;
@ -477,34 +479,34 @@ int ObAdminTestIODeviceExecutor::test_backup_data_()
int64_t total_size = 0;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(meta_file_path, OB_MAX_URI_LENGTH, "%s%s%s%s",
backup_path_, "/", meta_dir_name, meta_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.mk_parent_dir(meta_file_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to make parent dir for meta file", K(ret), K(meta_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to make parent dir for meta file", K(ret), K(meta_file_path));
} else if (OB_FAIL(util.write_single_file(meta_file_path, &storage_info, meta_file_content,
real_len, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to write single meta file", K(ret), K(meta_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to write single meta file", K(ret), K(meta_file_path));
} else if (OB_FAIL(databuff_printf(data_file_path, OB_MAX_URI_LENGTH, "%s%s%s",
backup_path_, "/", data_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.open_with_access_type(device_handle, fd, &storage_info,
data_file_path, access_type, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret),
STORAGE_LOG_FILTER(ERROR, "failed to open device with access type", K(ret),
K(data_file_path), K(access_type));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < UPLOAD_TIMES; ++i) {
char* buf = nullptr;
const int64_t buf_size = static_cast<int64_t>(ObRandom::rand(MIN_PART_SIZE, MAX_PART_SIZE));
if(OB_FAIL(generate_random_str_(buf, buf_size))) {
STORAGE_LOG(WARN, "failed to generate random str", K(ret), K(buf_size));
STORAGE_LOG_FILTER(ERROR, "failed to generate random str", K(ret), K(buf_size));
} else if (OB_FAIL(device_handle->pwrite(fd, offset, buf_size,
buf, write_size))) {
STORAGE_LOG(WARN, "failed to upload part", K(ret), K(data_file_path), K(buf_size));
STORAGE_LOG_FILTER(ERROR, "failed to upload part", K(ret), K(data_file_path), K(buf_size));
} else if (write_size != buf_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "upload data size does not match buf size", K(ret), K(write_size), K(buf_size));
STORAGE_LOG_FILTER(ERROR, "upload data size does not match buf size", K(ret), K(write_size), K(buf_size));
} else {
total_size += buf_size;
offset = total_size;
@ -512,28 +514,27 @@ int ObAdminTestIODeviceExecutor::test_backup_data_()
}
if (OB_SUCC(ret)) {
if (OB_FAIL(device_handle->complete(fd))) {
STORAGE_LOG(WARN, "fail to complete multipart upload", K(ret), K(tmp_ret), K(device_handle), K(fd));
STORAGE_LOG_FILTER(ERROR, "fail to complete multipart upload", K(ret), K(tmp_ret), K(device_handle), K(fd));
}
} else {
if (OB_TMP_FAIL(device_handle->abort(fd))) {
ret = COVER_SUCC(tmp_ret);
STORAGE_LOG(WARN, "fail to abort multipart upload", K(ret), K(tmp_ret), K(device_handle), K(fd));
STORAGE_LOG_FILTER(ERROR, "fail to abort multipart upload", K(ret), K(tmp_ret), K(device_handle), K(fd));
}
}
if (OB_TMP_FAIL(util.close_device_and_fd(device_handle, fd))) {
ret = COVER_SUCC(tmp_ret);
STORAGE_LOG(WARN, "fail to close device and fd", K(ret), K(device_handle), K(fd));
STORAGE_LOG_FILTER(ERROR, "fail to close device and fd", K(ret), K(device_handle), K(fd));
}
if(OB_SUCC(ret)) {
if (OB_FAIL(util.get_file_length(data_file_path, &storage_info, read_size))) {
STORAGE_LOG(WARN, "fail to get multipart upload file length", K(ret), K(data_file_path));
STORAGE_LOG_FILTER(ERROR, "fail to get multipart upload file length", K(ret), K(data_file_path));
} else if (read_size != total_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get multipart upload file length does not match real file length", K(ret), K(read_size), K(total_size));
STORAGE_LOG_FILTER(ERROR, "get multipart upload file length does not match real file length", K(ret), K(read_size), K(total_size));
}
}
}
PrintHelper::print_dump_line("test backup data result", OB_SUCC(ret) ? "SUCCESS" : "FAIL");
return ret;
}
@ -542,7 +543,7 @@ int ObAdminTestIODeviceExecutor::generate_random_str_(char *&buf, const int64_t
int ret = OB_SUCCESS;
if (OB_ISNULL((buf = static_cast<char *>(allocator_.alloc(size))))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "fail to allocate memory", K(ret), K(size));
STORAGE_LOG_FILTER(ERROR, "fail to allocate memory", K(ret), K(size));
}
for (int64_t i = 0; OB_SUCC(ret) && i < size; ++i) {
buf[i] = static_cast<char> ('a' + ObRandom::rand(0, 25));
@ -552,6 +553,7 @@ int ObAdminTestIODeviceExecutor::generate_random_str_(char *&buf, const int64_t
int ObAdminTestIODeviceExecutor::test_archive_log_() {
int ret = OB_SUCCESS;
TestExecGuard guard("archive log", test_id_, ret);
ObBackupIoAdapter util;
share::ObBackupStorageInfo storage_info;
const char* meta_dir_name = "archive/rounds/";
@ -576,21 +578,20 @@ int ObAdminTestIODeviceExecutor::test_archive_log_() {
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(meta_file_path, OB_MAX_URI_LENGTH, "%s%s%s%s",
backup_path_, "/", meta_dir_name, meta_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.mk_parent_dir(meta_file_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to make parent dir for meta file", K(ret), K(meta_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to make parent dir for meta file", K(ret), K(meta_file_path));
} else if (OB_FAIL(util.write_single_file(meta_file_path, &storage_info, meta_file_content,
real_len, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to write single meta file", K(ret), K(meta_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to write single meta file", K(ret), K(meta_file_path));
} else if (OB_FAIL(test_full_clog_file_())) {
STORAGE_LOG(WARN, "test full clog file failed", K(ret));
STORAGE_LOG_FILTER(ERROR, "test full clog file failed", K(ret));
} else if (OB_FAIL(test_partial_clog_file_())) {
STORAGE_LOG(WARN, "test partial clog file failed", K(ret));
STORAGE_LOG_FILTER(ERROR, "test partial clog file failed", K(ret));
}
PrintHelper::print_dump_line("test archive log result", OB_SUCC(ret) ? "SUCCESS" : "FAIL");
return ret;
}
@ -606,22 +607,22 @@ int ObAdminTestIODeviceExecutor::test_full_clog_file_()
int64_t file_size = 0;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(full_log_file_path, OB_MAX_URI_LENGTH, "%s%s%s",
backup_path_, "/", full_log_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_FAIL(generate_random_str_(buf, buf_size))) {
STORAGE_LOG(WARN, "failed to generate random str", K(ret), K(buf_size));
STORAGE_LOG_FILTER(ERROR, "failed to generate random str", K(ret), K(buf_size));
} else if (OB_FAIL(util.mk_parent_dir(full_log_file_path, &storage_info))) {
STORAGE_LOG(WARN, "failed to make parent dir for full clog file", K(ret), K(full_log_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to make parent dir for full clog file", K(ret), K(full_log_file_path));
} else if (OB_FAIL(util.write_single_file(full_log_file_path, &storage_info, buf, buf_size,
ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to write single full clog file", K(full_log_file_path), K(buf_size));
STORAGE_LOG_FILTER(ERROR, "failed to write single full clog file", K(full_log_file_path), K(buf_size));
} else if (OB_FAIL(util.get_file_length(full_log_file_path, &storage_info,file_size))) {
STORAGE_LOG(WARN, "failed to get full clog file length", K(full_log_file_path), K(file_size));
STORAGE_LOG_FILTER(ERROR, "failed to get full clog file length", K(full_log_file_path), K(file_size));
} else if (CLOG_FILE_SIZE != file_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get full clog file length does not match real length", K(file_size), K(buf_size));
STORAGE_LOG_FILTER(ERROR, "get full clog file length does not match real length", K(file_size), K(buf_size));
}
return ret;
}
@ -642,26 +643,26 @@ int ObAdminTestIODeviceExecutor::test_partial_clog_file_()
int64_t total_size = 0;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(partial_log_file_path, OB_MAX_URI_LENGTH, "%s%s%s",
backup_path_, "/", partial_log_file_name))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_FAIL(util.open_with_access_type(device_handle, fd, &storage_info,
partial_log_file_path, access_type, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret),
STORAGE_LOG_FILTER(ERROR, "failed to open device with access type", K(ret),
K(partial_log_file_path), K(access_type));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < APPEND_TIMES; ++i) {
char* buf = nullptr;
const int64_t buf_size = static_cast<int64_t>(ObRandom::rand(MIN_APPEND_SIZE, MAX_APPEND_SIZE));
if(OB_FAIL(generate_random_str_(buf, buf_size))) {
STORAGE_LOG(WARN, "failed to generate random str", K(ret), K(buf_size));
STORAGE_LOG_FILTER(ERROR, "failed to generate random str", K(ret), K(buf_size));
} else if (OB_FAIL(device_handle->pwrite(fd, offset, buf_size,
buf, write_size))) {
STORAGE_LOG(WARN, "failed to append clog", K(ret), K(partial_log_file_path), K(offset), K(buf_size));
STORAGE_LOG_FILTER(ERROR, "failed to append clog", K(ret), K(partial_log_file_path), K(offset), K(buf_size));
} else if (write_size != buf_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "append data size does not match buf size", K(ret), K(write_size), K(buf_size));
STORAGE_LOG_FILTER(ERROR, "append data size does not match buf size", K(ret), K(write_size), K(buf_size));
} else {
offset += buf_size;
total_size += buf_size;
@ -671,31 +672,31 @@ int ObAdminTestIODeviceExecutor::test_partial_clog_file_()
int64_t remain_size = 0;
char* remain_buf = nullptr;
if (OB_FAIL(util.close_device_and_fd(device_handle, fd))) {
STORAGE_LOG(WARN, "fail to close append writer and release device!", K(ret), K(partial_log_file_path));
STORAGE_LOG_FILTER(ERROR, "fail to close append writer and release device!", K(ret), K(partial_log_file_path));
} else if (OB_FAIL(util.adaptively_get_file_length(partial_log_file_path, &storage_info, read_size))) {
STORAGE_LOG(WARN, "fail to get appendable clog file length", K(ret), K(partial_log_file_path));
STORAGE_LOG_FILTER(ERROR, "fail to get appendable clog file length", K(ret), K(partial_log_file_path));
} else if (read_size != total_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get appendable clog file length does not match real file length", K(ret), K(read_size), K(total_size));
STORAGE_LOG_FILTER(ERROR, "get appendable clog file length does not match real file length", K(ret), K(read_size), K(total_size));
} else if (OB_FAIL(util.open_with_access_type(device_handle, fd, &storage_info,
partial_log_file_path, access_type, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret),
STORAGE_LOG_FILTER(ERROR, "failed to open device with access type", K(ret),
K(partial_log_file_path), K(access_type));
} else if (FALSE_IT(remain_size = CLOG_FILE_SIZE - total_size)) {
} else if (OB_FAIL(generate_random_str_(remain_buf, remain_size))) {
STORAGE_LOG(WARN, "failed to generate remaining random str for appendable clog file", K(ret), K(remain_size));
STORAGE_LOG_FILTER(ERROR, "failed to generate remaining random str for appendable clog file", K(ret), K(remain_size));
} else if (OB_FAIL(device_handle->pwrite(fd, offset, remain_size, remain_buf, write_size))) {
STORAGE_LOG(WARN, "failed to append remaining str for appendable clog file", K(ret), K(remain_size));
STORAGE_LOG_FILTER(ERROR, "failed to append remaining str for appendable clog file", K(ret), K(remain_size));
} else if (FALSE_IT(read_size = 0) || FALSE_IT(total_size += remain_size)) {
} else if (OB_FAIL(device_handle->seal_file(fd))) {
STORAGE_LOG(WARN, "failed to seal for appendable clog file", K(ret), K(partial_log_file_path));
STORAGE_LOG_FILTER(ERROR, "failed to seal for appendable clog file", K(ret), K(partial_log_file_path));
} else if (OB_FAIL(util.close_device_and_fd(device_handle, fd))) {
STORAGE_LOG(WARN, "fail to close append writer and release device!", K(ret), K(partial_log_file_path));
STORAGE_LOG_FILTER(ERROR, "fail to close append writer and release device!", K(ret), K(partial_log_file_path));
} else if (OB_FAIL(util.adaptively_get_file_length(partial_log_file_path, &storage_info, read_size))) {
STORAGE_LOG(WARN, "fail to get appendable clog file length", K(ret), K(partial_log_file_path));
STORAGE_LOG_FILTER(ERROR, "fail to get appendable clog file length", K(ret), K(partial_log_file_path));
} else if (read_size != total_size) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get appendable clog file length does not match real file length", K(ret), K(read_size), K(total_size));
STORAGE_LOG_FILTER(ERROR, "get appendable clog file length does not match real file length", K(ret), K(read_size), K(total_size));
}
}
}
@ -705,6 +706,7 @@ int ObAdminTestIODeviceExecutor::test_partial_clog_file_()
int ObAdminTestIODeviceExecutor::test_consume_clog_file_()
{
int ret = OB_SUCCESS;
TestExecGuard guard("consume clog file", test_id_, ret);
ObBackupIoAdapter util;
share::ObBackupStorageInfo storage_info;
int64_t min_file_id = 0;
@ -720,47 +722,46 @@ int ObAdminTestIODeviceExecutor::test_consume_clog_file_()
int64_t read_size = 0;
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K_(backup_path));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K_(backup_path));
} else if (OB_FAIL(databuff_printf(ls_path, OB_MAX_URI_LENGTH, "%s%s%s",
backup_path_, "/", ls_dir))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_FAIL(archive::ObArchiveFileUtils::get_file_range(ls_path,
&storage_info, min_file_id, max_file_id))) {
STORAGE_LOG(WARN, "failed to get clog file range", K(ret), K(ls_path));
STORAGE_LOG_FILTER(ERROR, "failed to get clog file range", K(ret), K(ls_path));
} else if (min_file_id != 1 || max_file_id != 2) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "got wrong clog file list", K(ret), K(min_file_id), K(max_file_id));
STORAGE_LOG_FILTER(ERROR, "got wrong clog file list", K(ret), K(min_file_id), K(max_file_id));
} else if (OB_FAIL(databuff_printf(full_clog_file_path, OB_MAX_URI_LENGTH, "%s%s%s%s", backup_path_,
"/", ls_dir, "1.obarc"))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_ISNULL(buf1 = reinterpret_cast<char*>(allocator_.alloc(buf_len)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to allocate memory", K(ret), K(buf_len));
STORAGE_LOG_FILTER(ERROR, "failed to allocate memory", K(ret), K(buf_len));
} else if (OB_FAIL(util.adaptively_read_part_file(full_clog_file_path, &storage_info, buf1,
buf_len, offset, read_size, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to read part from full clog file", K(ret), K(full_clog_file_path),
STORAGE_LOG_FILTER(ERROR, "failed to read part from full clog file", K(ret), K(full_clog_file_path),
K(offset), K(buf_len));
} else if (read_size != buf_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "read full clog file length does not match real buf length", K(ret), K(read_size), K(buf_len));
STORAGE_LOG_FILTER(ERROR, "read full clog file length does not match real buf length", K(ret), K(read_size), K(buf_len));
} else {
read_size = 0;
if (OB_FAIL(databuff_printf(partial_clog_file_path, OB_MAX_URI_LENGTH, "%s%s%s%s", backup_path_,
"/", ls_dir, "2.obarc"))) {
STORAGE_LOG(WARN, "failed to databuff printf", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to databuff printf", K(ret));
} else if (OB_ISNULL(buf2 = reinterpret_cast<char*>(allocator_.alloc(buf_len)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to allocate memory", K(ret), K(buf_len));
STORAGE_LOG_FILTER(ERROR, "failed to allocate memory", K(ret), K(buf_len));
} else if (OB_FAIL(util.adaptively_read_part_file(partial_clog_file_path, &storage_info, buf2,
buf_len, offset, read_size, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to read part from full clog file", K(ret), K(full_clog_file_path),
STORAGE_LOG_FILTER(ERROR, "failed to read part from full clog file", K(ret), K(full_clog_file_path),
K(offset), K(buf_len));
} else if (read_size != buf_len) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "read partial clog file length does not match real buf length", K(ret), K(read_size), K(buf_len));
STORAGE_LOG_FILTER(ERROR, "read partial clog file length does not match real buf length", K(ret), K(read_size), K(buf_len));
}
}
PrintHelper::print_dump_line("test consume log result", OB_SUCC(ret) ? "SUCCESS" : "FAIL");
return ret;
}

21
tools/ob_admin/io_device/ob_admin_test_io_device_executor.h
View File

@ -17,10 +17,17 @@
#include "../ob_admin_executor.h"
#include "deps/oblib/src/lib/ob_define.h" // OB_MAX_URI_LENGTH
#include "share/backup/ob_backup_struct.h" // OB_MAX_BACKUP_STORAGE_INFO_LENGTH
#include "../dumpsst/ob_admin_dumpsst_print_helper.h"
namespace oceanbase {
namespace tools {
#define STORAGE_LOG_FILTER(level, info_string, args...) \
STORAGE_LOG(level, info_string, ##args); \
if (strcmp(#level, "ERROR") == 0) { \
std::cout << LIGHT_RED << "ERROR INFO: " << info_string << NONE_COLOR << std::endl; \
}
struct TestObjectStorageInterfaceContext
{
TestObjectStorageInterfaceContext();
@ -49,6 +56,19 @@ struct TestObjectStorageInterfaceContext
char upload_file_path_[OB_MAX_URI_LENGTH];
};
class TestExecGuard
{
public:
TestExecGuard(const char *title, int64_t &test_id, int &ret);
~TestExecGuard();
private:
const char *title_;
int64_t &test_id_;
int &ret_;
const int64_t start_time_us_;
};
class ObAdminTestIODeviceExecutor : public ObAdminExecutor
{
public:
@ -88,6 +108,7 @@ private:
char backup_path_[common::OB_MAX_URI_LENGTH];
char storage_info_[common::OB_MAX_BACKUP_STORAGE_INFO_LENGTH];
bool is_quiet_;
int64_t test_id_;
common::ObArenaAllocator allocator_;
private:

258
tools/ob_admin/io_device/ob_admin_test_object_storage_interface.cpp
View File

@ -13,7 +13,6 @@
#include <time.h>
#include "ob_admin_test_io_device_executor.h"
#include "share/backup/ob_backup_io_adapter.h"
#include "../dumpsst/ob_admin_dumpsst_print_helper.h"
using namespace oceanbase::share;
using namespace oceanbase::common;
@ -23,6 +22,30 @@ namespace oceanbase
namespace tools
{
TestExecGuard::TestExecGuard(const char *title, int64_t &test_id, int &ret)
: title_(title),
test_id_(test_id),
ret_(ret),
start_time_us_(ObTimeUtility::current_time())
{
if (OB_ISNULL(title_)) {
title_ = "";
}
std::cout << LIGHT_GREEN << "[RUN TEST" << test_id << "] " << NONE_COLOR << title_ << std::endl;
}
TestExecGuard::~TestExecGuard()
{
const int64_t cost_time_ms = (ObTimeUtility::current_time() - start_time_us_) / 1000;
if (ret_ == OB_SUCCESS) {
std::cout << LIGHT_GREEN << "[ OK] " << NONE_COLOR << title_ << " (" << cost_time_ms << " ms)" << std::endl;
} else {
std::cout << LIGHT_RED << "ERROR CODE: " << ret_ << ' ' << common::ob_error_name(ret_) << std::endl;
std::cout << LIGHT_RED << "[ FAILED! ] " << NONE_COLOR << title_ << " (" << cost_time_ms << " ms)" << std::endl;
}
test_id_++;
}
TestObjectStorageInterfaceContext::TestObjectStorageInterfaceContext()
: single_file_content_(nullptr),
appendable_file_content_(nullptr),
@ -37,24 +60,24 @@ TestObjectStorageInterfaceContext::TestObjectStorageInterfaceContext()
int ObAdminTestIODeviceExecutor::test_object_storage_interface_()
{
int ret = OB_SUCCESS;
TestExecGuard guard("test_object_storage_interface", test_id_, ret);
TestObjectStorageInterfaceContext ctx;
if (OB_FAIL(test_object_storage_interface_prepare_(ctx))) {
STORAGE_LOG(WARN, "failed to prepare envionment for test object storage interface", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to prepare envionment for test object storage interface", K(ret));
} else if (OB_FAIL(test_object_storage_interface_is_exist_(ctx))) {
STORAGE_LOG(WARN, "failed to test object storage interface is exist", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to test object storage interface is exist", K(ret));
} else if (OB_FAIL(test_object_storage_interface_list_file_(ctx))) {
STORAGE_LOG(WARN, "failed to test object storage interface list file", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to test object storage interface list file", K(ret));
} else if (OB_FAIL(test_object_storage_interface_list_directories_(ctx))) {
STORAGE_LOG(WARN, "failed to test object storage interface list directories", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to test object storage interface list directories", K(ret));
} else if (OB_FAIL(test_object_storage_interface_get_file_length_(ctx))) {
STORAGE_LOG(WARN, "failed to test object storage interface get file length", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to test object storage interface get file length", K(ret));
} else if (OB_FAIL(test_object_storage_interface_read_(ctx))) {
STORAGE_LOG(WARN, "failed to test object storage interface read", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to test object storage interface read", K(ret));
} else if (OB_FAIL(test_object_storage_interface_del_(ctx))) {
STORAGE_LOG(WARN, "failed to test object storage interface del", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to test object storage interface del", K(ret));
}
PrintHelper::print_dump_line("test object storage interface", OB_SUCC(ret) ? "SUCCESS" : "FAIL");
return ret;
}
@ -62,35 +85,35 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_prepare_(TestObje
{
int ret = OB_SUCCESS;
if (OB_FAIL(ctx.storage_info_.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K(ret), K_(backup_path), K_(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K(ret), K_(backup_path), K_(storage_info));
} else if (OB_FAIL(databuff_printf(ctx.test_dir_path_, OB_MAX_URI_LENGTH, "%s/%s",
backup_path_, ctx.test_dir_name_))) {
STORAGE_LOG(WARN, "failed to set test dir path", K(ret), K_(backup_path), K(ctx.test_dir_name_));
STORAGE_LOG_FILTER(ERROR, "failed to set test dir path", K(ret), K_(backup_path), K(ctx.test_dir_name_));
} else if (OB_FAIL(databuff_printf(ctx.single_file_path_, OB_MAX_URI_LENGTH, "%s/%s",
ctx.test_dir_path_, ctx.single_file_name_))) {
STORAGE_LOG(WARN, "failed to set single file path", K(ret), K(ctx.test_dir_path_), K(ctx.single_file_name_));
STORAGE_LOG_FILTER(ERROR, "failed to set single file path", K(ret), K(ctx.test_dir_path_), K(ctx.single_file_name_));
} else if (OB_FAIL(generate_random_str_(ctx.single_file_content_, ctx.SINGLE_FILE_LENGTH))) {
STORAGE_LOG(WARN, "failed to generate random str", K(ret), K(ctx.SINGLE_FILE_LENGTH));
STORAGE_LOG_FILTER(ERROR, "failed to generate random str", K(ret), K(ctx.SINGLE_FILE_LENGTH));
} else if (OB_FAIL(ctx.util_.mk_parent_dir(ctx.single_file_path_, &ctx.storage_info_))) {
STORAGE_LOG(WARN, "failed to make parent dir for full clog file", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to make parent dir for full clog file", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
} else if (OB_FAIL(databuff_printf(ctx.appendable_file_path_, OB_MAX_URI_LENGTH, "%s/%s",
ctx.test_dir_path_, ctx.appendable_file_name_))) {
STORAGE_LOG(WARN, "failed to set appendable file path", K(ret), K(ctx.test_dir_path_), K(ctx.appendable_file_name_));
STORAGE_LOG_FILTER(ERROR, "failed to set appendable file path", K(ret), K(ctx.test_dir_path_), K(ctx.appendable_file_name_));
} else if (OB_FAIL(generate_random_str_(ctx.appendable_file_content_, ctx.APPENDABLE_FILE_LENGTH))) {
STORAGE_LOG(WARN, "failed to generate random str", K(ret), K(ctx.APPENDABLE_FILE_LENGTH));
STORAGE_LOG_FILTER(ERROR, "failed to generate random str", K(ret), K(ctx.APPENDABLE_FILE_LENGTH));
} else if (OB_FAIL(ctx.util_.write_single_file(ctx.single_file_path_, &ctx.storage_info_,
ctx.single_file_content_, ctx.SINGLE_FILE_LENGTH, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to write single file", K(ret), K(ctx.SINGLE_FILE_LENGTH), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to write single file", K(ret), K(ctx.SINGLE_FILE_LENGTH), K(ctx.storage_info_));
} else if (OB_FAIL(test_multi_step_write_appendable_file_(ctx))) {
STORAGE_LOG(WARN, "failed to write appendable file",
STORAGE_LOG_FILTER(ERROR, "failed to write appendable file",
K(ret), K(ctx.appendable_file_path_), K(ctx.APPENDABLE_FILE_LENGTH), K(ctx.storage_info_));
} else if (OB_FAIL(databuff_printf(ctx.upload_file_path_, OB_MAX_URI_LENGTH, "%s/%s",
ctx.test_dir_path_, ctx.upload_file_name_))) {
STORAGE_LOG(WARN, "failed to set upload file path", K(ret), K(ctx.test_dir_path_), K(ctx.upload_file_name_));
STORAGE_LOG_FILTER(ERROR, "failed to set upload file path", K(ret), K(ctx.test_dir_path_), K(ctx.upload_file_name_));
} else if (OB_FAIL(generate_random_str_(ctx.upload_file_content_, ctx.UPLOAD_FILE_LENGTH))) {
STORAGE_LOG(WARN, "failed to generate random str", K(ret), K(ctx.UPLOAD_FILE_LENGTH));
STORAGE_LOG_FILTER(ERROR, "failed to generate random str", K(ret), K(ctx.UPLOAD_FILE_LENGTH));
} else if (OB_FAIL(test_object_storage_interface_async_upload_(ctx))) {
STORAGE_LOG(WARN, "failed to exec async upload", K(ret), K(ctx.upload_file_path_), K(ctx.UPLOAD_FILE_LENGTH));
STORAGE_LOG_FILTER(ERROR, "failed to exec async upload", K(ret), K(ctx.upload_file_path_), K(ctx.UPLOAD_FILE_LENGTH));
}
return ret;
}
@ -107,30 +130,30 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_async_upload_(Tes
if (OB_FAIL(ctx.util_.open_with_access_type(device_handle, fd, &ctx.storage_info_,
ctx.upload_file_path_, access_type,
ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret), K(ctx.upload_file_path_), K(access_type), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to open device with access type", K(ret), K(ctx.upload_file_path_), K(access_type), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.async_upload_data(*device_handle, fd, ctx.upload_file_content_,
0, ctx.UPLOAD_FILE_LENGTH, io_handle))) {
STORAGE_LOG(WARN, "failed to start async upload task!", K(ret), K(ctx.storage_info_),
STORAGE_LOG_FILTER(ERROR, "failed to start async upload task!", K(ret), K(ctx.storage_info_),
K(ctx.upload_file_path_), K(ctx.UPLOAD_FILE_LENGTH));
} else if (OB_FAIL(io_handle.wait())) {
STORAGE_LOG(WARN, "failed to wait async upload data finish", K(ret), K(ctx.storage_info_),
STORAGE_LOG_FILTER(ERROR, "failed to wait async upload data finish", K(ret), K(ctx.storage_info_),
K(ctx.upload_file_path_), K(ctx.UPLOAD_FILE_LENGTH));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ctx.util_.complete(*device_handle, fd))) {
STORAGE_LOG(WARN, "failed to complete", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to complete", K(ret));
}
}
if (OB_FAIL(ret)) {
if (OB_TMP_FAIL(ctx.util_.abort(*device_handle, fd))) {
ret = COVER_SUCC(tmp_ret);
STORAGE_LOG(WARN, "failed to abort", K(ret), K(tmp_ret));
STORAGE_LOG_FILTER(ERROR, "failed to abort", K(ret), K(tmp_ret));
}
}
if (OB_TMP_FAIL(ctx.util_.close_device_and_fd(device_handle, fd))) {
ret = COVER_SUCC(tmp_ret);
STORAGE_LOG(WARN, "failed to close file and release device!", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to close file and release device!", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
}
return ret;
@ -140,39 +163,39 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_is_exist_(TestObj
{
int ret = OB_SUCCESS;
if (OB_FAIL(ctx.util_.is_exist(ctx.single_file_path_, &ctx.storage_info_, ctx.single_file_is_exist_))) {
STORAGE_LOG(WARN, "failed to check if single file is exist", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to check if single file is exist", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(!ctx.single_file_is_exist_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "single file does not exist!", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "single file does not exist!", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.is_exist(ctx.appendable_file_path_, &ctx.storage_info_, ctx.appendable_file_is_exist_))) {
STORAGE_LOG(WARN, "failed to check if appendable file is exist",
STORAGE_LOG_FILTER(ERROR, "failed to check if appendable file is exist",
K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else if (ctx.storage_info_.get_type() == ObStorageType::OB_STORAGE_S3
&& OB_UNLIKELY(ctx.appendable_file_is_exist_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "appendable file should not be detected when storage type is S3",
STORAGE_LOG_FILTER(ERROR, "appendable file should not be detected when storage type is S3",
K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else if (ctx.storage_info_.get_type() != ObStorageType::OB_STORAGE_S3
&& OB_UNLIKELY(!ctx.appendable_file_is_exist_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "appendable file should be detected when storage type is not S3",
STORAGE_LOG_FILTER(ERROR, "appendable file should be detected when storage type is not S3",
K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else if (FALSE_IT(ctx.single_file_is_exist_ = false)) {
} else if (FALSE_IT(ctx.appendable_file_is_exist_ = false)) {
} else if (OB_FAIL(ctx.util_.adaptively_is_exist(ctx.single_file_path_, &ctx.storage_info_, ctx.single_file_is_exist_))) {
STORAGE_LOG(WARN, "failed to check if single file is exist",
STORAGE_LOG_FILTER(ERROR, "failed to check if single file is exist",
K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(!ctx.single_file_is_exist_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "single file does not exist!",
STORAGE_LOG_FILTER(ERROR, "single file does not exist!",
K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.adaptively_is_exist(ctx.appendable_file_path_, &ctx.storage_info_,
ctx.appendable_file_is_exist_))) {
STORAGE_LOG(WARN, "failed to check if appendable file is exist",
STORAGE_LOG_FILTER(ERROR, "failed to check if appendable file is exist",
K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(!ctx.appendable_file_is_exist_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "appendable file does not exist!", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "appendable file does not exist!", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
}
return ret;
@ -185,23 +208,23 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_list_file_(TestOb
ObFileListArrayOp file_list_array_op(file_names, allocator_);
if (OB_FAIL(ctx.util_.list_files(ctx.test_dir_path_, &ctx.storage_info_, file_list_array_op))) {
STORAGE_LOG(WARN, "failed to list test dir path", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to list test dir path", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
} else if (ctx.storage_info_.get_type() == ObStorageType::OB_STORAGE_S3
&& OB_UNLIKELY(file_names.size() != ctx.S3_FILE_COUNT)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "when the storage type is set to OB_STORAGE_S3, there should be 10 files present here",
STORAGE_LOG_FILTER(ERROR, "when the storage type is set to OB_STORAGE_S3, there should be 10 files present here",
K(ret), K(ctx.test_dir_path_), K(file_names.size()), K(ctx.storage_info_));
} else if (ctx.storage_info_.get_type() != ObStorageType::OB_STORAGE_S3
&& OB_UNLIKELY(file_names.size() != ctx.OTHER_FILE_COUNT)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "when the storage type is not set to OB_STORAGE_S3, there should be 3 files present here",
STORAGE_LOG_FILTER(ERROR, "when the storage type is not set to OB_STORAGE_S3, there should be 3 files present here",
K(ret), K(ctx.test_dir_path_), K(file_names.size()), K(ctx.storage_info_));
} else if (FALSE_IT(file_names.reuse())) {
} else if (OB_FAIL(ctx.util_.adaptively_list_files(ctx.test_dir_path_, &ctx.storage_info_, file_list_array_op))) {
STORAGE_LOG(WARN, "failed to list test dir path", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to list test dir path", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(file_names.count() != ctx.OTHER_FILE_COUNT)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "the length of file_names does not equal to 2", K(ret), K(file_names.count()), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "the length of file_names does not equal to 2", K(ret), K(file_names.count()), K(ctx.storage_info_));
}
return ret;
@ -214,16 +237,16 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_list_directories_
ObFileListArrayOp dir_list_array_op(dir_names, allocator_);
if (OB_FAIL(ctx.util_.list_directories(ctx.test_dir_path_, &ctx.storage_info_, dir_list_array_op))) {
STORAGE_LOG(WARN, "failed to list directories", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to list directories", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
} else if (ctx.storage_info_.get_type() == ObStorageType::OB_STORAGE_S3
&& (OB_UNLIKELY(dir_names.size() != 1) || OB_UNLIKELY(dir_names[0] != ObString(ctx.appendable_file_name_)))) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "appendable dir not find when storage type is s3",
STORAGE_LOG_FILTER(ERROR, "appendable dir not find when storage type is s3",
K(ret), K(ctx.test_dir_path_), K(ctx.appendable_file_name_), K(dir_names.size()), K(ctx.storage_info_));
} else if (ctx.storage_info_.get_type() != ObStorageType::OB_STORAGE_S3
&& OB_UNLIKELY(dir_names.size() != 0)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "length of the list directories result should be 0 when storage type is not s3",
STORAGE_LOG_FILTER(ERROR, "length of the list directories result should be 0 when storage type is not s3",
K(ret), K(ctx.test_dir_path_), K(ctx.appendable_file_name_), K(dir_names.size()), K(ctx.storage_info_));
}
@ -238,19 +261,20 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_get_file_length_(
// test get_file_length
if (OB_FAIL(ctx.util_.get_file_length(ctx.single_file_path_, &ctx.storage_info_, single_file_length_query))) {
STORAGE_LOG(WARN, "failed to get single file length", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to get single file length", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(single_file_length_query != ctx.SINGLE_FILE_LENGTH)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get single file length not equal to real length",
STORAGE_LOG_FILTER(ERROR, "get single file length not equal to real length",
K(ret), K(ctx.single_file_path_), K(ctx.SINGLE_FILE_LENGTH), K(single_file_length_query), K(ctx.storage_info_));
} else if (FALSE_IT(single_file_length_query = 0)) {
} else if (ctx.storage_info_.get_type() == ObStorageType::OB_STORAGE_S3) {
if (OB_SUCC(ctx.util_.get_file_length(ctx.appendable_file_path_, &ctx.storage_info_, appendable_file_length_query))) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "when the storage type is set to OB_STORAGE_S3, the get_file_length interface should fail to be invoked.",
STORAGE_LOG_FILTER(ERROR, "when the storage type is set to OB_STORAGE_S3, the get_file_length interface should fail to be invoked.",
K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(ret != OB_OBJECT_NOT_EXIST)) {
STORAGE_LOG(WARN, "the get_file_length interface should return OB_OBJECT_NOT_EXIST when storage type is set to OB_STORAGE_S3",
} else if (OB_UNLIKELY(ret != OB_BACKUP_FILE_NOT_EXIST)) {
STORAGE_LOG_FILTER(ERROR, "the get_file_length interface should return OB_BACKUP_FILE_NOT_EXIST when storage type is set to OB_STORAGE_S3",
K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else {
ret = OB_SUCCESS;
@ -258,11 +282,11 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_get_file_length_(
} else {
if (OB_FAIL(ctx.util_.get_file_length(ctx.appendable_file_path_, &ctx.storage_info_, appendable_file_length_query))) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "failed to get file length when the storage type is not S3",
STORAGE_LOG_FILTER(ERROR, "failed to get file length when the storage type is not S3",
K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(appendable_file_length_query != ctx.APPENDABLE_FILE_LENGTH)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get appendable file length not equal to real length",
STORAGE_LOG_FILTER(ERROR, "get appendable file length not equal to real length",
K(ret), K(ctx.APPENDABLE_FILE_LENGTH), K(appendable_file_length_query));
}
}
@ -270,17 +294,17 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_get_file_length_(
// test adaptively_get_file_length
if (FAILEDx(ctx.util_.adaptively_get_file_length(ctx.single_file_path_, &ctx.storage_info_,
single_file_length_query))) {
STORAGE_LOG(WARN, "failed to get single file length", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to get single file length", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(single_file_length_query != ctx.SINGLE_FILE_LENGTH)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get single file length not equal to real length",
STORAGE_LOG_FILTER(ERROR, "get single file length not equal to real length",
K(ret), K(ctx.single_file_path_), K(ctx.SINGLE_FILE_LENGTH), K(single_file_length_query), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.adaptively_get_file_length(ctx.appendable_file_path_, &ctx.storage_info_,
appendable_file_length_query))) {
STORAGE_LOG(WARN, "failed to get appendable file length", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to get appendable file length", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(appendable_file_length_query != ctx.APPENDABLE_FILE_LENGTH)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "get appendable file length not equal to real length",
STORAGE_LOG_FILTER(ERROR, "get appendable file length not equal to real length",
K(ret), K(ctx.appendable_file_path_), K(ctx.APPENDABLE_FILE_LENGTH), K(appendable_file_length_query), K(ctx.storage_info_));
}
return ret;
@ -316,13 +340,13 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_read_(TestObjectS
// test read_single_text_file
if (OB_ISNULL(single_file_buf = static_cast<char *>(allocator_.alloc(ctx.SINGLE_FILE_LENGTH + 1)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to allocate buf", K(ret), K(ctx.single_file_path_), K(ctx.SINGLE_FILE_LENGTH), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to allocate buf", K(ret), K(ctx.single_file_path_), K(ctx.SINGLE_FILE_LENGTH), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.read_single_text_file(ctx.single_file_path_, &ctx.storage_info_,
single_file_buf, ctx.SINGLE_FILE_LENGTH + 1, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to read single file text", K(ret), K(ctx.single_file_path_), K(ctx.SINGLE_FILE_LENGTH), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to read single file text", K(ret), K(ctx.single_file_path_), K(ctx.SINGLE_FILE_LENGTH), K(ctx.storage_info_));
} else if (OB_UNLIKELY(strlen(single_file_buf) != ctx.SINGLE_FILE_LENGTH)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "read single file length not equal to real length", K(ret), K(ctx.single_file_path_),
STORAGE_LOG_FILTER(ERROR, "read single file length not equal to real length", K(ret), K(ctx.single_file_path_),
K(ctx.SINGLE_FILE_LENGTH), K(strlen(appendable_file_buf)), K(ctx.storage_info_));
}
@ -330,14 +354,14 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_read_(TestObjectS
if (OB_FAIL(ret)) {
} else if (OB_ISNULL(appendable_file_buf = static_cast<char *>(allocator_.alloc(ctx.APPENDABLE_FILE_LENGTH + 1)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to allocate buf", K(ret), K(ctx.appendable_file_path_), K(ctx.APPENDABLE_FILE_LENGTH), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to allocate buf", K(ret), K(ctx.appendable_file_path_), K(ctx.APPENDABLE_FILE_LENGTH), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.adaptively_read_single_text_file(ctx.appendable_file_path_,
&ctx.storage_info_, appendable_file_buf, ctx.APPENDABLE_FILE_LENGTH + 1, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to read appendable text file",
STORAGE_LOG_FILTER(ERROR, "failed to read appendable text file",
K(ret), K(ctx.appendable_file_path_), K(ctx.APPENDABLE_FILE_LENGTH + 1), K(ctx.storage_info_));
} else if (OB_UNLIKELY(strlen(appendable_file_buf) != ctx.APPENDABLE_FILE_LENGTH)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "read appendable file length not equal to real length",
STORAGE_LOG_FILTER(ERROR, "read appendable file length not equal to real length",
K(ret), K(ctx.appendable_file_path_), K(ctx.APPENDABLE_FILE_LENGTH), K(strlen(appendable_file_buf)), K(ctx.storage_info_));
}
@ -345,14 +369,14 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_read_(TestObjectS
if (OB_FAIL(ret)) {
} else if (OB_ISNULL(single_file_read_part_buf = static_cast<char *>(allocator_.alloc(read_part_size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to allocate buf", K(ret), K(ctx.single_file_path_), K(ctx.SINGLE_FILE_LENGTH), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to allocate buf", K(ret), K(ctx.single_file_path_), K(ctx.SINGLE_FILE_LENGTH), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.read_part_file(ctx.single_file_path_, &ctx.storage_info_, single_file_read_part_buf,
read_part_size, single_file_read_part_offset, read_part_read_size, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to read part file",
STORAGE_LOG_FILTER(ERROR, "failed to read part file",
K(ret), K(ctx.single_file_path_), K(single_file_read_part_offset), K(read_part_size), K(ctx.storage_info_));
} else if (OB_UNLIKELY(read_part_size != read_part_read_size)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "read size is inaccurate",
STORAGE_LOG_FILTER(ERROR, "read size is inaccurate",
K(ret), K(ctx.single_file_path_), K(single_file_read_part_offset), K(read_part_size), K(read_part_read_size), K(ctx.storage_info_));
}
@ -361,14 +385,14 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_read_(TestObjectS
if (OB_FAIL(ret)) {
} else if (OB_ISNULL(appendable_file_read_part_buf = static_cast<char *>(allocator_.alloc(read_part_size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to allocate buf", K(ret), K(ctx.appendable_file_path_), K(read_part_size), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to allocate buf", K(ret), K(ctx.appendable_file_path_), K(read_part_size), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.adaptively_read_part_file(ctx.appendable_file_path_, &ctx.storage_info_,
appendable_file_read_part_buf, read_part_size, appendable_file_read_part_offset, read_part_read_size, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to adaptively read part file",
STORAGE_LOG_FILTER(ERROR, "failed to adaptively read part file",
K(ret), K(ctx.appendable_file_path_), K(appendable_file_read_part_offset), K(read_part_size), K(ctx.storage_info_));
} else if (OB_UNLIKELY(read_part_size != read_part_read_size)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "read size is inaccurate", K(ret), K(ctx.appendable_file_path_),
STORAGE_LOG_FILTER(ERROR, "read size is inaccurate", K(ret), K(ctx.appendable_file_path_),
K(appendable_file_read_part_offset), K(read_part_size), K(read_part_read_size), K(ctx.storage_info_));
}
@ -376,47 +400,47 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_read_(TestObjectS
if (OB_FAIL(ret)) {
} else if (OB_ISNULL(pread_buf = static_cast<char *>(allocator_.alloc(read_part_size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to allocate buf", K(ret), K(read_part_size), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to allocate buf", K(ret), K(read_part_size), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.pread(ctx.single_file_path_, &ctx.storage_info_,
pread_buf, read_part_size, pread_offset,
pread_read_size, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to pread single file", K(ret), K(ctx.storage_info_), K(ctx.single_file_path_), K(pread_offset), K(read_part_size));
STORAGE_LOG_FILTER(ERROR, "failed to pread single file", K(ret), K(ctx.storage_info_), K(ctx.single_file_path_), K(pread_offset), K(read_part_size));
} else if (OB_UNLIKELY(pread_read_size != read_part_size)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "pread size is inaccurate", K(ret), K(ctx.storage_info_), K(ctx.single_file_path_), K(pread_offset), K(read_part_size));
STORAGE_LOG_FILTER(ERROR, "pread size is inaccurate", K(ret), K(ctx.storage_info_), K(ctx.single_file_path_), K(pread_offset), K(read_part_size));
}
// test async_pread
if (OB_SUCC(ret)) {
if (OB_ISNULL(async_pread_buf = static_cast<char *>(allocator_.alloc(async_pread_read_size)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to allocate buf", K(ret), K(async_pread_read_size));
STORAGE_LOG_FILTER(ERROR, "failed to allocate buf", K(ret), K(async_pread_read_size));
} else if (OB_FAIL(ctx.util_.open_with_access_type(device_handle, fd, &ctx.storage_info_,
ctx.appendable_file_path_, access_type,
ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret), K(ctx.storage_info_), K(ctx.appendable_file_path_), K(access_type));
STORAGE_LOG_FILTER(ERROR, "failed to open device with access type", K(ret), K(ctx.storage_info_), K(ctx.appendable_file_path_), K(access_type));
} else if (OB_FAIL(ctx.util_.async_pread(*device_handle, fd, async_pread_buf,
async_pread_read_offset, async_pread_read_size,
io_handle))) {
STORAGE_LOG(WARN, "failed to start async pread task!", K(ret), K(ctx.storage_info_),
STORAGE_LOG_FILTER(ERROR, "failed to start async pread task!", K(ret), K(ctx.storage_info_),
K(async_pread_read_offset),
K(async_pread_read_size));
} else if (OB_FAIL(io_handle.wait())) {
STORAGE_LOG(WARN, "failed to wait async pread finish", K(ret), K(ctx.storage_info_),
STORAGE_LOG_FILTER(ERROR, "failed to wait async pread finish", K(ret), K(ctx.storage_info_),
K(async_pread_read_offset),
K(async_pread_read_size));
} else if (OB_UNLIKELY(memcmp(ctx.appendable_file_content_ + async_pread_read_offset,
async_pread_buf,
async_pread_read_size) != 0)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "incorrect read from async pread", K(ret), K(ctx.storage_info_),
STORAGE_LOG_FILTER(ERROR, "incorrect read from async pread", K(ret), K(ctx.storage_info_),
K(async_pread_read_offset), K(async_pread_read_size));
}
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(ctx.util_.close_device_and_fd(device_handle, fd))) {
ret = COVER_SUCC(tmp_ret);
STORAGE_LOG(WARN, "failed to close the file and release device!", K(ret));
STORAGE_LOG_FILTER(ERROR, "failed to close the file and release device!", K(ret));
}
}
return ret;
@ -426,35 +450,35 @@ int ObAdminTestIODeviceExecutor::test_object_storage_interface_del_(TestObjectSt
{
int ret = OB_SUCCESS;
if (OB_FAIL(ctx.util_.del_file(ctx.single_file_path_, &ctx.storage_info_))) {
STORAGE_LOG(WARN, "failed to del single file", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to del single file", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
} else if (FALSE_IT(ctx.single_file_is_exist_ = true)) {
} else if (OB_FAIL(ctx.util_.is_exist(ctx.single_file_path_, &ctx.storage_info_, ctx.single_file_is_exist_))) {
STORAGE_LOG(WARN, "failed to check if single file is exist", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to check if single file is exist", K(ret), K(ctx.single_file_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(ctx.single_file_is_exist_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "failed to del single file", K(ret), K(ctx.single_file_path_), K(ctx.single_file_is_exist_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to del single file", K(ret), K(ctx.single_file_path_), K(ctx.single_file_is_exist_), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.adaptively_del_file(ctx.appendable_file_path_, &ctx.storage_info_))) {
STORAGE_LOG(WARN, "failed to del appendable file path", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to del appendable file path", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else if (FALSE_IT(ctx.appendable_file_is_exist_ = true)) {
} else if (OB_FAIL(ctx.util_.adaptively_is_exist(ctx.appendable_file_path_, &ctx.storage_info_, ctx.appendable_file_is_exist_))) {
STORAGE_LOG(WARN, "failed to check if appendable file is exist", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to check if appendable file is exist", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(ctx.appendable_file_is_exist_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "failed to del appendable file", K(ret), K(ctx.appendable_file_path_), K(ctx.appendable_file_is_exist_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to del appendable file", K(ret), K(ctx.appendable_file_path_), K(ctx.appendable_file_is_exist_), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.del_dir(ctx.test_dir_path_, &ctx.storage_info_, true))) {
STORAGE_LOG(WARN, "failed to del dir", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to del dir", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
} else if (FALSE_IT(ctx.upload_file_is_exist_ = true)) {
} else if (OB_FAIL(ctx.util_.adaptively_is_exist(ctx.upload_file_path_,
&ctx.storage_info_,
ctx.upload_file_is_exist_))) {
STORAGE_LOG(WARN, "failed to check if upload file is exist", K(ret),
STORAGE_LOG_FILTER(ERROR, "failed to check if upload file is exist", K(ret),
K(ctx.upload_file_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(ctx.upload_file_is_exist_)) {
ret = OB_ERR_UNDEFINED;
STORAGE_LOG(WARN, "failed to del upload file", K(ret), K(ctx.upload_file_path_),
STORAGE_LOG_FILTER(ERROR, "failed to del upload file", K(ret), K(ctx.upload_file_path_),
K(ctx.storage_info_), K(ctx.upload_file_is_exist_));
} else if (OB_FAIL(ctx.util_.del_dir(ctx.test_dir_path_, &ctx.storage_info_))) {
STORAGE_LOG(WARN, "failed to del dir", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to del dir", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
}
return ret;
}
@ -483,22 +507,22 @@ int ObAdminTestIODeviceExecutor::test_multi_step_write_appendable_file_(TestObje
|| OB_ISNULL(ctx.appendable_file_content_)
|| OB_UNLIKELY(ctx.APPENDABLE_FILE_LENGTH <= STEP_COUNT * STEP_LEN)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "arguments is invalid",
STORAGE_LOG_FILTER(ERROR, "arguments is invalid",
K(ret), K(ctx.appendable_file_path_), K(ctx.appendable_file_content_), K(ctx.APPENDABLE_FILE_LENGTH), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.open_with_access_type(device_handle, fd, &ctx.storage_info_, ctx.appendable_file_path_, access_type, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to open device with access type",
STORAGE_LOG_FILTER(ERROR, "failed to open device with access type",
K(ret), K(ctx.appendable_file_path_), K(access_type), K(ctx.storage_info_));
} else if (OB_ISNULL(device_handle)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "device handle is null", K(ret), K(device_handle), K_(backup_path), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "device handle is null", K(ret), K(device_handle), K_(backup_path), K(ctx.storage_info_));
} else {
for (int i = 0; OB_SUCC(ret) && i < STEP_COUNT; i++) {
if (OB_FAIL(ctx.util_.pwrite(*device_handle, fd, ctx.appendable_file_content_ + offset, offset,
STEP_LEN, single_write_size, false/*is_can_seal*/))) {
STORAGE_LOG(WARN, "failed to pwrite", K(ret), K(offset), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to pwrite", K(ret), K(offset), K(ctx.storage_info_));
} else if (OB_UNLIKELY(single_write_size != STEP_LEN)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "write_size not equal to STEP_LEN", K(ret), K(single_write_size), K(STEP_LEN), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "write_size not equal to STEP_LEN", K(ret), K(single_write_size), K(STEP_LEN), K(ctx.storage_info_));
} else {
offset += single_write_size;
writed_len += single_write_size;
@ -508,52 +532,52 @@ int ObAdminTestIODeviceExecutor::test_multi_step_write_appendable_file_(TestObje
if (OB_SUCC(ret)) {
// test list_files, adaptively_list_files and list_directories before seal
if (OB_FAIL(ctx.util_.list_files(ctx.test_dir_path_, &ctx.storage_info_, file_list_array_op))) {
STORAGE_LOG(WARN, "failed to list test dir path", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to list test dir path", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
} else if (ctx.storage_info_.get_type() == ObStorageType::OB_STORAGE_S3
&& OB_UNLIKELY(file_names.size() != 7)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "when the storage type is set to OB_STORAGE_S3, there should be 7 files present here",
STORAGE_LOG_FILTER(ERROR, "when the storage type is set to OB_STORAGE_S3, there should be 7 files present here",
K(ret), K(ctx.test_dir_path_), K(file_names.size()), K(ctx.storage_info_));
} else if (ctx.storage_info_.get_type() != ObStorageType::OB_STORAGE_S3
&& OB_UNLIKELY(file_names.size() != 2)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "when the storage type is not set to OB_STORAGE_S3, there should be 2 files present here",
STORAGE_LOG_FILTER(ERROR, "when the storage type is not set to OB_STORAGE_S3, there should be 2 files present here",
K(ret), K(ctx.test_dir_path_), K(file_names.size()), K(ctx.storage_info_));
} else if (FALSE_IT(file_names.reuse())){
} else if (OB_FAIL(ctx.util_.adaptively_list_files(ctx.test_dir_path_, &ctx.storage_info_, file_list_array_op))) {
STORAGE_LOG(WARN, "failed to list test dir path", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to list test dir path", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
} else if (OB_UNLIKELY(file_names.size() != 2)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "the length of file_names does not equal to 2", K(ret), K(file_names.size()), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "the length of file_names does not equal to 2", K(ret), K(file_names.size()), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.list_directories(ctx.test_dir_path_, &ctx.storage_info_, dir_list_array_op))) {
STORAGE_LOG(WARN, "failed to list directories", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to list directories", K(ret), K(ctx.test_dir_path_), K(ctx.storage_info_));
} else if (ctx.storage_info_.get_type() == ObStorageType::OB_STORAGE_S3
&& OB_UNLIKELY(dir_names.size() != 1 || dir_names[0] != ObString(ctx.appendable_file_name_))) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "appendable dir not find when storage type is s3",
STORAGE_LOG_FILTER(ERROR, "appendable dir not find when storage type is s3",
K(ret), K(ctx.test_dir_path_), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else if (ctx.storage_info_.get_type() != ObStorageType::OB_STORAGE_S3
&& OB_UNLIKELY(dir_names.size() != 0)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "length of the list directories result should be 0 when storage type is not s3",
STORAGE_LOG_FILTER(ERROR, "length of the list directories result should be 0 when storage type is not s3",
K(ret), K(ctx.test_dir_path_), K(ctx.appendable_file_path_), K(ctx.storage_info_));
} else if (FALSE_IT(remain_len = ctx.APPENDABLE_FILE_LENGTH - writed_len)) {
} else if (OB_UNLIKELY(remain_len <= 0)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "remain_len must be greater than 0",
STORAGE_LOG_FILTER(ERROR, "remain_len must be greater than 0",
K(ret), K(remain_len), K(ctx.APPENDABLE_FILE_LENGTH), K(writed_len), K(ctx.storage_info_));
} else if (OB_FAIL(ctx.util_.pwrite(ctx.appendable_file_path_, &ctx.storage_info_,
ctx.appendable_file_content_ + offset, offset, remain_len,
access_type, write_size, true, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to pwrite", K(ret), K(offset), K(remain_len), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to pwrite", K(ret), K(offset), K(remain_len), K(ctx.storage_info_));
} else if (OB_FAIL(device_handle->seal_file(fd))) {
STORAGE_LOG(WARN, "failed to seal file", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to seal file", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
}
}
if (OB_TMP_FAIL(ctx.util_.close_device_and_fd(device_handle, fd))) {
ret = COVER_SUCC(tmp_ret);
STORAGE_LOG(WARN, "failed to close file and release device!", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
STORAGE_LOG_FILTER(ERROR, "failed to close file and release device!", K(ret), K(ctx.appendable_file_path_), K(ctx.storage_info_));
}
}
return ret;
@ -563,6 +587,7 @@ int ObAdminTestIODeviceExecutor::test_multi_step_write_appendable_file_(TestObje
int ObAdminTestIODeviceExecutor::test_list_before_complete_multipart_write_()
{
int ret = OB_SUCCESS;
TestExecGuard guard("test_list_before_complete_multipart_write", test_id_, ret);
int tmp_ret = OB_SUCCESS;
ObBackupIoAdapter util;
share::ObBackupStorageInfo storage_info;
@ -579,48 +604,47 @@ int ObAdminTestIODeviceExecutor::test_list_before_complete_multipart_write_()
ObFileListArrayOp file_list_array_op(file_names, allocator_);
if (OB_FAIL(storage_info.set(backup_path_, storage_info_))) {
STORAGE_LOG(WARN, "failed to set storage info", K(ret), K_(backup_path), K_(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to set storage info", K(ret), K_(backup_path), K_(storage_info));
} else if (OB_FAIL(databuff_printf(file_path, OB_MAX_URI_LENGTH, "%s/%s", backup_path_, file_name))) {
STORAGE_LOG(WARN, "failed to set file path", K(ret), K_(backup_path), K(file_name), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to set file path", K(ret), K_(backup_path), K(file_name), K(storage_info));
} else if (OB_FAIL(generate_random_str_(file_content, file_length))) {
STORAGE_LOG(WARN, "failed to generate content for file", K(ret), K(file_length), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to generate content for file", K(ret), K(file_length), K(storage_info));
} else if (OB_FAIL(util.open_with_access_type(device_handle, fd, &storage_info,
file_path, access_type, ObStorageIdMod::get_default_id_mod()))) {
STORAGE_LOG(WARN, "failed to open device with access type", K(ret), K(file_path), K(access_type), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to open device with access type", K(ret), K(file_path), K(access_type), K(storage_info));
} else if (OB_ISNULL(device_handle)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "device handle is null", K(ret), K(device_handle), K_(backup_path), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "device handle is null", K(ret), K(device_handle), K_(backup_path), K(storage_info));
} else if (OB_FAIL(device_handle->write(fd, file_content, file_length, write_size))) {
STORAGE_LOG(WARN, "failed to write file", K(ret), K(file_length), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to write file", K(ret), K(file_length), K(storage_info));
} else if (OB_FAIL(util.adaptively_list_files(backup_path_, &storage_info, file_list_array_op))) {
STORAGE_LOG(WARN, "failed to list files in dir", K(ret), K_(backup_path), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to list files in dir", K(ret), K_(backup_path), K(storage_info));
} else if (OB_UNLIKELY(std::find(file_names.begin(), file_names.end(), file_name) != file_names.end())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "uncompleted multipart file are invisible!", K(ret), K(file_path), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "uncompleted multipart file are invisible!", K(ret), K(file_path), K(storage_info));
} else if (OB_FAIL(device_handle->complete(fd))) {
STORAGE_LOG(WARN, "failed to complete multipart upload", K(ret), K(device_handle), K(fd), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to complete multipart upload", K(ret), K(device_handle), K(fd), K(storage_info));
} else if (OB_FAIL(util.adaptively_list_files(backup_path_, &storage_info, file_list_array_op))) {
STORAGE_LOG(WARN, "failed to list files in dir", K(ret), K_(backup_path), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to list files in dir", K(ret), K_(backup_path), K(storage_info));
} else if (OB_UNLIKELY(std::find(file_names.begin(), file_names.end(), file_name) == file_names.end())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "completed multipart file are visible!", K(ret), K(file_path), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "completed multipart file are visible!", K(ret), K(file_path), K(storage_info));
} else if (OB_FAIL(util.del_file(file_path, &storage_info))) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "failed to del file", K(ret), K(file_path), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to del file", K(ret), K(file_path), K(storage_info));
}
if (OB_FAIL(ret)) {
if (OB_TMP_FAIL(device_handle->abort(fd))) {
ret = COVER_SUCC(tmp_ret);
STORAGE_LOG(WARN, "failed to abort multipart upload", K(ret), K(tmp_ret), K(device_handle), K(fd), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to abort multipart upload", K(ret), K(tmp_ret), K(device_handle), K(fd), K(storage_info));
}
}
if (OB_TMP_FAIL(util.close_device_and_fd(device_handle, fd))) {
ret = COVER_SUCC(tmp_ret);
STORAGE_LOG(WARN, "failed to close device and fd", K(ret), K(tmp_ret), K(device_handle), K(fd), K(storage_info));
STORAGE_LOG_FILTER(ERROR, "failed to close device and fd", K(ret), K(tmp_ret), K(device_handle), K(fd), K(storage_info));
}
PrintHelper::print_dump_line("test list before complete", OB_SUCC(ret) ? "SUCCESS" : "FAIL");
return ret;
}

15
tools/ob_admin/main.cpp
View File

@ -76,10 +76,17 @@ int get_log_base_directory(char *log_file_name, const int64_t log_file_name_len,
int64_t ob_admin_log_dir_len = 0;
bool is_directory = false;
if (NULL == ob_admin_log_dir) {
fprintf(stderr, "\nThe OB_ADMIN_LOG_DIR environment variable not found, we will not generate ob_admin.log\n"
"If log files are required, please notice that log files should not be outputted to\n"
"OceanBase's clog directory.(for example, export OB_ADMIN_LOG_DIR=/tmp)\n");
ret = OB_ENTRY_NOT_EXIST;
// to improve the readability of the ob_admin results, the default path of log is set to /dev/null.
// and prompts the user to set the path manually when logging is required.
if (OB_FAIL(databuff_printf(log_file_name, log_file_name_len, "/dev/null"))) {
fprintf(stderr, "\nUnexpected error, databuff_printf failed\n");
} else if (OB_FAIL(databuff_printf(log_file_rs_name, log_file_rs_name_len, "/dev/null"))) {
fprintf(stderr, "\nUnexpected error, databuff_printf failed\n");
} else {
fprintf(stderr, "\033[1;33m[NOTICE]\033[m If specific log is required, you need to set the environment variable OB_ADMIN_LOG_DIR.\n"
" for example: export OB_ADMIN_LOG_DIR=~/.ob_admin_log\n"
" please notice that log files should not be outputted to OceanBase's clog directory.\n");
}
} else if (FALSE_IT(ob_admin_log_dir_len = strlen(ob_admin_log_dir))) {
} else if (OB_FAIL(FileDirectoryUtils::is_directory(ob_admin_log_dir, is_directory))) {
fprintf(stderr, "\nCheck is_directory failed, we will not generate ob_admin.log(errno:%d)\n", ret);