hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] Backup support AWS S3

Browse Source 
Co-authored-by: donglou-zhang <zhangleisoft2012@163.com>
Co-authored-by: xuhuleon <xuhuleon@qq.com>
This commit is contained in:
LoLolobster
2023-12-13 13:43:20 +00:00
committed by ob-robot
parent 051cb24288
commit dd84275032
68 changed files with 12175 additions and 1129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/storage/backup/ob_backup_ctx.cpp
View File

@ -400,7 +400,7 @@ int ObBackupDataCtx::open_file_writer_(const share::ObBackupPath &backup_path)
{
int ret = OB_SUCCESS;
common::ObBackupIoAdapter util;
const ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER;
const ObStorageAccessType access_type = OB_STORAGE_ACCESS_MULTIPART_WRITER;
if (OB_FAIL(util.mk_parent_dir(backup_path.get_obstr(), param_.backup_dest_.get_storage_info()))) {
LOG_WARN("failed to make parent dir", K(backup_path));
} else if (OB_FAIL(util.open_with_access_type(

2
src/storage/backup/ob_backup_extern_info_mgr.cpp
View File

@ -371,7 +371,7 @@ int ObExternTabletMetaWriter::prepare_backup_file_(const int64_t file_id)
int ret = OB_SUCCESS;
share::ObBackupPath backup_path;
common::ObBackupIoAdapter util;
const ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER;
const ObStorageAccessType access_type = OB_STORAGE_ACCESS_MULTIPART_WRITER;
const int64_t data_file_size = get_data_file_size();
if (OB_FAIL(ObBackupPathUtil::get_ls_data_tablet_info_path(
backup_set_dest_, ls_id_, turn_id_, retry_id_, file_id, backup_path))) {

2
src/storage/backup/ob_backup_index_merger.cpp
View File

@ -572,7 +572,7 @@ int ObIBackupIndexMerger::open_file_writer_(const share::ObBackupPath &path, con
{
int ret = OB_SUCCESS;
common::ObBackupIoAdapter util;
const ObStorageAccessType access_type = OB_STORAGE_ACCESS_RANDOMWRITER;
const ObStorageAccessType access_type = OB_STORAGE_ACCESS_MULTIPART_WRITER;
if (OB_FAIL(util.mk_parent_dir(path.get_obstr(), storage_info))) {
LOG_WARN("failed to make parent dir", K(path), K(path), KP(storage_info));
} else if (OB_FAIL(util.open_with_access_type(dev_handle_, io_fd_, storage_info, path.get_obstr(), access_type))) {

6
src/storage/backup/ob_backup_task.cpp
View File

@ -5280,7 +5280,7 @@ int ObLSBackupComplementLogTask::inner_get_piece_file_list_(const share::ObLSID
const share::SCN &start_scn = piece_attr.start_scn_;
if (OB_FAIL(get_src_backup_piece_dir_(ls_id, piece_attr, src_piece_dir_path))) {
LOG_WARN("failed to get src backup piece dir", K(ret), K(round_id), K(piece_id), K(ls_id), K(piece_attr));
} else if (OB_FAIL(util.list_files(src_piece_dir_path.get_obstr(), archive_dest_.get_storage_info(), op))) {
} else if (OB_FAIL(util.adaptively_list_files(src_piece_dir_path.get_obstr(), archive_dest_.get_storage_info(), op))) {
LOG_WARN("failed to list files", K(ret), K(src_piece_dir_path));
} else if (OB_FAIL(op.get_file_id_list(file_id_list))) {
LOG_WARN("failed to get files", K(ret));
@ -5551,7 +5551,7 @@ int ObLSBackupComplementLogTask::inner_transfer_clog_file_(
} else if (OB_ISNULL(buf = static_cast<char *>(allocator.alloc(transfer_len)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate memory", K(ret), K(transfer_len));
} else if (OB_FAIL(util.read_part_file(src_path.get_obstr(), archive_dest_.get_storage_info(), buf, transfer_len, dst_len, read_len))) {
} else if (OB_FAIL(util.adaptively_read_part_file(src_path.get_obstr(), archive_dest_.get_storage_info(), buf, transfer_len, dst_len, read_len))) {
LOG_WARN("failed to read part file", K(ret), K(src_path));
} else if (read_len != transfer_len) {
ret = OB_ERR_UNEXPECTED;
@ -5585,7 +5585,7 @@ int ObLSBackupComplementLogTask::get_file_length_(
{
int ret = OB_SUCCESS;
ObBackupIoAdapter util;
if (OB_FAIL(util.get_file_length(file_path, storage_info, length))) {
if (OB_FAIL(util.adaptively_get_file_length(file_path, storage_info, length))) {
if (OB_BACKUP_FILE_NOT_EXIST == ret) {
ret = OB_SUCCESS;
length = 0;