hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Test] Test the append file behavior after cos switching to s3 during the upgrade process

Browse Source
This commit is contained in:
obdev 2024-10-16 15:43:59 +00:00 committed by ob-robot
parent 8a5ba496e9
commit dac5e9c5a8
11 changed files with 21 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
deps/oblib/src/common/storage/ob_device_common.h vendored
View File

@ -105,29 +105,21 @@ enum ObStorageAccessType
OB_STORAGE_ACCESS_NOHEAD_READER = 1,
OB_STORAGE_ACCESS_ADAPTIVE_READER = 2,
OB_STORAGE_ACCESS_OVERWRITER = 3,
// OB_STORAGE_ACCESS_APPENDER and OB_STORAGE_ACCESS_RANDOMWRITER both correspond to ObStorageAppender.
// OB_STORAGE_ACCESS_APPENDER correspond to ObStorageAppender.
// In cases where the destination is S3, a special format is utilized to emulate the append interface.
// Upon completion of data writing,
// it is recommended to invoke the seal_file interface to write a seal meta file.
// This step is designed to enhance subsequent access performance to the object.
// Skipping this step does not compromise data integrity but may impact performance.
OB_STORAGE_ACCESS_APPENDER = 4,
// TODO @fangdan: remove this type
// OB_STORAGE_ACCESS_APPENDER and OB_STORAGE_ACCESS_RANDOMWRITER both correspond to ObStorageAppender.
// In cases where the destination is S3, a special format is utilized to emulate the append interface.
// Upon completion of data writing,
// it is recommended to invoke the seal_file interface to write a seal meta file.
// This step is designed to enhance subsequent access performance to the object.
// Skipping this step does not compromise data integrity but may impact performance.
OB_STORAGE_ACCESS_RANDOMWRITER = 5,
// When utilizing the multipart writer interface,
// it is crucial to invoke the complete interface upon successful data upload to ensure object visibility.
// The object remains invisible until the complete interface responds successfully.
// If an error occur before invoking complete or during its execution,
// it is imperative to call the abort interface to delete the already uploaded data.
OB_STORAGE_ACCESS_MULTIPART_WRITER = 6,
OB_STORAGE_ACCESS_DIRECT_MULTIPART_WRITER = 7,
OB_STORAGE_ACCESS_BUFFERED_MULTIPART_WRITER = 8,
OB_STORAGE_ACCESS_MULTIPART_WRITER = 5,
OB_STORAGE_ACCESS_DIRECT_MULTIPART_WRITER = 6,
OB_STORAGE_ACCESS_BUFFERED_MULTIPART_WRITER = 7,
OB_STORAGE_ACCESS_MAX_TYPE
};

16
deps/oblib/src/lib/restore/ob_object_device.cpp vendored
View File

@ -19,8 +19,8 @@ namespace common
const char *OB_STORAGE_ACCESS_TYPES_STR[] = {
"reader", "nohead_reader", "adaptive_reader",
"overwriter", "appender", "random_write",
"multipart_writer", "direct_multipart_writer", "buffered_multipart_writer"};
"overwriter", "appender", "multipart_writer",
"direct_multipart_writer", "buffered_multipart_writer"};
const char *get_storage_access_type_str(const ObStorageAccessType &type)
{
@ -162,8 +162,6 @@ int ObObjectDevice::get_access_type(ObIODOpts *opts, ObStorageAccessType& access
access_type_flag = OB_STORAGE_ACCESS_OVERWRITER;
} else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_APPENDER])) {
access_type_flag = OB_STORAGE_ACCESS_APPENDER;
} else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_RANDOMWRITER])) {
access_type_flag = OB_STORAGE_ACCESS_RANDOMWRITER;
} else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_MULTIPART_WRITER])) {
access_type_flag = OB_STORAGE_ACCESS_MULTIPART_WRITER;
} else if (0 == STRCMP(access_type , OB_STORAGE_ACCESS_TYPES_STR[OB_STORAGE_ACCESS_DIRECT_MULTIPART_WRITER])) {
@ -358,8 +356,7 @@ int ObObjectDevice::release_res(void *ctx, const ObIOFd &fd, ObStorageAccessType
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "ctx is null, invald para!");
} else {
if (OB_STORAGE_ACCESS_APPENDER == access_type ||
OB_STORAGE_ACCESS_RANDOMWRITER == access_type) {
if (OB_STORAGE_ACCESS_APPENDER == access_type) {
ObStorageAppender *appender = static_cast<ObStorageAppender*>(ctx);
if (OB_FAIL(appender->close())) {
OB_LOG(WARN, "fail to close the appender!", K(ret), K(access_type));
@ -449,8 +446,7 @@ int ObObjectDevice::open(const char *pathname, const int flags, const mode_t mod
ret = open_for_reader(pathname, ctx, false/*head_meta*/);
} else if (OB_STORAGE_ACCESS_ADAPTIVE_READER == access_type) {
ret = open_for_adaptive_reader_(pathname, ctx);
} else if (OB_STORAGE_ACCESS_APPENDER == access_type ||
OB_STORAGE_ACCESS_RANDOMWRITER == access_type) {
} else if (OB_STORAGE_ACCESS_APPENDER == access_type) {
ret = open_for_appender(pathname, opts, ctx);
} else if (OB_STORAGE_ACCESS_OVERWRITER == access_type) {
ret = open_for_overwriter(pathname, ctx);
@ -630,7 +626,7 @@ int ObObjectDevice::seal_for_adaptive(const ObIOFd &fd)
} else if (OB_ISNULL(ctx)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret));
} else if ((OB_STORAGE_ACCESS_RANDOMWRITER == flag) || (OB_STORAGE_ACCESS_APPENDER == flag)) {
} else if (OB_STORAGE_ACCESS_APPENDER == flag) {
ObStorageAppender *appender = static_cast<ObStorageAppender*>(ctx);
if (OB_FAIL(appender->seal_for_adaptive())) {
OB_LOG(WARN, "fail to seal!", K(ret), K(flag));
@ -883,7 +879,7 @@ int ObObjectDevice::pwrite(const ObIOFd &fd, const int64_t offset, const int64_t
} else if (OB_ISNULL(ctx)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "fd ctx is null!", K(flag), K(ret));
} else if ((OB_STORAGE_ACCESS_RANDOMWRITER == flag) || (OB_STORAGE_ACCESS_APPENDER == flag)) {
} else if (OB_STORAGE_ACCESS_APPENDER == flag) {
ObStorageAppender *appender = static_cast<ObStorageAppender*>(ctx);
if (OB_FAIL(appender->pwrite((char*)buf, size, offset))) {
OB_LOG(WARN, "fail to do appender pwrite!", K(ret));

2
src/logservice/archiveservice/ob_archive_io.cpp
View File

@ -59,7 +59,7 @@ int ObArchiveIO::push_log(const ObString &uri,
}
} else {
if (OB_FAIL(io_adapter.pwrite(uri, storage_info, data, offset, data_len,
common::ObStorageAccessType::OB_STORAGE_ACCESS_RANDOMWRITER, write_size, is_can_seal,
common::ObStorageAccessType::OB_STORAGE_ACCESS_APPENDER, write_size, is_can_seal,
common::ObStorageIdMod(backup_dest_id, common::ObStorageUsedMod::STORAGE_USED_ARCHIVE)))) {
ARCHIVE_LOG(WARN, "fail to write file", K(ret), K(uri), KP(storage_info), K(data), K(data_len));
}

2
src/logservice/logminer/ob_log_miner_file_manager.cpp
View File

@ -500,7 +500,7 @@ int ObLogMinerFileManager::append_file_(const ObString &uri,
if (nullptr == data || 0 == data_len) {
// do nothing
} else if (OB_FAIL(utils.open_with_access_type(device_handle, fd, output_dest_.get_storage_info(),
uri, common::OB_STORAGE_ACCESS_RANDOMWRITER, mod))) {
uri, common::OB_STORAGE_ACCESS_APPENDER, mod))) {
LOG_ERROR("failed to open device", K(uri), K(output_dest_), K(uri));
} else if (OB_FAIL(device_handle->pwrite(fd, offset, data_len, data, write_size))) {
LOG_ERROR("failed to write data into file", K(uri), K(output_dest_),

9
src/share/backup/ob_backup_io_adapter.cpp
View File

@ -99,7 +99,7 @@ int ObBackupIoAdapter::open_with_access_type(ObIODevice*& device_handle, ObIOFd
OB_LOG(WARN, "invalid access type!", KR(ret), K(access_type));
} else {
iod_opts.opts_[0].set("AccessType", OB_STORAGE_ACCESS_TYPES_STR[access_type]);
if (access_type == OB_STORAGE_ACCESS_RANDOMWRITER)
if (access_type == OB_STORAGE_ACCESS_APPENDER)
{
iod_opts.opts_[1].set("OpenMode", "CREATE_OPEN_NOLOCK");
iod_opts.opt_cnt_++;
@ -415,8 +415,7 @@ int ObBackupIoAdapter::pwrite(
ObIOFd fd;
ObIODevice *device_handle = NULL;
if (OB_UNLIKELY((ObStorageAccessType::OB_STORAGE_ACCESS_APPENDER != access_type)
&& (ObStorageAccessType::OB_STORAGE_ACCESS_RANDOMWRITER != access_type))) {
if (OB_UNLIKELY(ObStorageAccessType::OB_STORAGE_ACCESS_APPENDER != access_type)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "invalid access type", K(ret), K(access_type));
} else if (OB_FAIL(open_with_access_type(device_handle, fd, storage_info, uri, access_type, storage_id_mod))) {
@ -448,15 +447,13 @@ int ObBackupIoAdapter::pwrite(
int flag = -1;
ObFdSimulator::get_fd_flag(fd, flag);
if ((ObStorageAccessType::OB_STORAGE_ACCESS_APPENDER != flag)
&& (ObStorageAccessType::OB_STORAGE_ACCESS_RANDOMWRITER != flag)
&& (ObStorageAccessType::OB_STORAGE_ACCESS_MULTIPART_WRITER != flag)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid storage access type", K(ret), K(flag));
} else if (FALSE_IT(fd.device_handle_ = (&device_handle))) {
} else if (OB_FAIL(io_manager_write(buf, offset, size, fd, write_size))) {
STORAGE_LOG(WARN, "fail to io manager write", K(ret), K(offset), K(size), K(fd));
} else if (((ObStorageAccessType::OB_STORAGE_ACCESS_APPENDER == flag) ||
(ObStorageAccessType::OB_STORAGE_ACCESS_RANDOMWRITER == flag))
} else if (ObStorageAccessType::OB_STORAGE_ACCESS_APPENDER == flag
&& is_can_seal
&& OB_FAIL(device_handle.seal_file(fd))) {
STORAGE_LOG(WARN, "fail to seal file", K(ret), K(offset), K(size), K(fd));

1
src/share/io/ob_io_struct.cpp
View File

@ -2502,7 +2502,6 @@ int ObSyncIOChannel::do_sync_io(ObIORequest &req)
}
}
} else if ((ObStorageAccessType::OB_STORAGE_ACCESS_APPENDER == flag)
|| (ObStorageAccessType::OB_STORAGE_ACCESS_RANDOMWRITER == flag)
|| (ObStorageAccessType::OB_STORAGE_ACCESS_MULTIPART_WRITER == flag)) {
if (OB_FAIL(device_handle->pwrite(req.fd_, io_offset, req.io_result_->size_, req.calc_io_buf(), io_size))) {
LOG_WARN("pwrite failed", K(ret), K(req));

2
src/storage/backup/ob_backup_device_wrapper.cpp
View File

@ -113,7 +113,7 @@ int ObBackupWrapperIODevice::setup_io_opts_for_backup_device(
opts->opts_[2].set(SECOND_ID_STR, tmp_macro_id.second_id_);
opts->opts_[3].set(THIRD_ID_STR, tmp_macro_id.third_id_);
opts->opts_[4].set("AccessType", OB_STORAGE_ACCESS_TYPES_STR[access_type]);
if (access_type == OB_STORAGE_ACCESS_RANDOMWRITER)
if (access_type == OB_STORAGE_ACCESS_APPENDER)
{
opts->opts_[5].set("OpenMode", "CREATE_OPEN_NOLOCK");
}

2
tools/ob_admin/io_bench/task_executor.cpp
View File

@ -498,7 +498,7 @@ int AppendWriteTaskExecutor::execute()
ObBackupIoAdapter adapter;
ObIOFd fd;
ObIODevice *device_handle = nullptr;
ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER;
ObStorageAccessType access_type = OB_STORAGE_ACCESS_APPENDER;
const int64_t open_start_time_us = ObTimeUtility::current_time();
if (OB_FAIL(adapter.open_with_access_type(

4
tools/ob_admin/io_device/ob_admin_test_io_device_executor.cpp
View File

@ -230,7 +230,7 @@ int ObAdminTestIODeviceExecutor::test_appendable_check_file_(const char* check_f
share::ObBackupStorageInfo storage_info;
ObIOFd fd;
ObIODevice *device_handle = NULL;
ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER;
ObStorageAccessType access_type = OB_STORAGE_ACCESS_APPENDER;
bool is_exist = false;
const char* appendable_check_file_name = "appendable_check_file.obbak";
const char* appendable_check_file_content = "appendable check file";
@ -629,7 +629,7 @@ int ObAdminTestIODeviceExecutor::test_partial_clog_file_()
share::ObBackupStorageInfo storage_info;
ObIOFd fd;
ObIODevice *device_handle = NULL;
ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER;
ObStorageAccessType access_type = OB_STORAGE_ACCESS_APPENDER;
char partial_log_file_path[OB_MAX_URI_LENGTH] = { 0 };
const char* partial_log_file_name = "archive/logstream_1/log/2.obarc";
int64_t offset = 0;

2
unittest/storage/backup/test_backup_device_wrapper.cpp
View File

@ -68,7 +68,7 @@ TEST(TestBackupDeviceWrapper, test_alloc_block) {
OK(ObIOManager::get_instance().init(test_memory));
OK(ObBackupWrapperIODevice::setup_io_opts_for_backup_device(
backup_set_id, ls_id, backup_data_type, turn_id, retry_id, file_id, block_type, OB_STORAGE_ACCESS_RANDOMWRITER, &io_d_opts_write));
backup_set_id, ls_id, backup_data_type, turn_id, retry_id, file_id, block_type, OB_STORAGE_ACCESS_APPENDER, &io_d_opts_write));
ObBackupWrapperIODevice wrapper_io_write;

2
unittest/storage/backup/test_backup_sync_io_mock_async_io.cpp
View File

@ -285,7 +285,7 @@ TEST_F(TestBackupMockAsyncIO, test_sync_io_mock_async_io)
ObIOInfo write_io_info;
ObIOHandle write_io_handle;
ObIOFd write_io_fd;
const ObStorageAccessType writer_access_type = OB_STORAGE_ACCESS_RANDOMWRITER;
const ObStorageAccessType writer_access_type = OB_STORAGE_ACCESS_APPENDER;
EXPECT_TRUE(ObBackupDeviceMacroBlockId::is_backup_block_file(macro_id.first_id()));
// mock write