From f6417feb0b8be846d67156e86b595cf53b7e321e Mon Sep 17 00:00:00 2001 From: oceanoverflow Date: Wed, 25 Sep 2024 21:47:20 +0000 Subject: [PATCH] fix backup linked item in ss mode --- src/storage/backup/ob_backup_data_struct.cpp | 2 +- src/storage/backup/ob_backup_data_struct.h | 4 +-- .../backup/ob_backup_linked_block_reader.cpp | 4 +-- .../backup/ob_backup_linked_block_writer.cpp | 11 ++++--- src/storage/backup/ob_backup_linked_item.h | 2 +- src/storage/backup/ob_backup_task.cpp | 16 ++++++---- .../ob_storage_ha_reader.cpp | 11 ++++--- ...test_backup_linked_item_write_and_read.cpp | 29 +++++++++++-------- 8 files changed, 46 insertions(+), 33 deletions(-) diff --git a/src/storage/backup/ob_backup_data_struct.cpp b/src/storage/backup/ob_backup_data_struct.cpp index 3af5064e5..a926acc2c 100644 --- a/src/storage/backup/ob_backup_data_struct.cpp +++ b/src/storage/backup/ob_backup_data_struct.cpp @@ -1573,7 +1573,7 @@ bool ObBackupDeviceMacroBlockId::is_valid() const && static_cast(blocksstable::ObMacroBlockIdMode::ID_MODE_BACKUP) == id_mode_; } -int ObBackupDeviceMacroBlockId::get_backup_macro_block_index(const share::ObBackupDataType &backup_data_type, +int ObBackupDeviceMacroBlockId::get_backup_macro_block_index( const blocksstable::ObLogicMacroBlockId &logic_id, ObBackupMacroBlockIndex ¯o_index) const { int ret = OB_SUCCESS; diff --git a/src/storage/backup/ob_backup_data_struct.h b/src/storage/backup/ob_backup_data_struct.h index 6d9b97446..aee029a6d 100644 --- a/src/storage/backup/ob_backup_data_struct.h +++ b/src/storage/backup/ob_backup_data_struct.h @@ -275,7 +275,7 @@ public: }; struct ObBackupDeviceMacroBlockId; -typedef ObBackupPhysicalID ObBackupLinkedBlockAddr; +typedef ObBackupDeviceMacroBlockId ObBackupLinkedBlockAddr; struct ObBackupMacroBlockIndex { OB_UNIS_VERSION(1); @@ -475,7 +475,7 @@ public: public: ObBackupDeviceMacroBlockId(); ~ObBackupDeviceMacroBlockId(); - int get_backup_macro_block_index(const share::ObBackupDataType &backup_data_type, + int get_backup_macro_block_index( const blocksstable::ObLogicMacroBlockId &logic_id, ObBackupMacroBlockIndex ¯o_index) const; int set(const int64_t backup_set_id, const int64_t ls_id, const int64_t dir_id, const int64_t turn_id, const int64_t retry_id, const int64_t file_id, diff --git a/src/storage/backup/ob_backup_linked_block_reader.cpp b/src/storage/backup/ob_backup_linked_block_reader.cpp index 64e836242..da2da9e8d 100644 --- a/src/storage/backup/ob_backup_linked_block_reader.cpp +++ b/src/storage/backup/ob_backup_linked_block_reader.cpp @@ -112,8 +112,8 @@ int ObBackupLinkedBlockReader::pread_block_(const ObBackupLinkedBlockAddr &block { int ret = OB_SUCCESS; ObBackupPath backup_path; - const int64_t offset = static_cast(block_addr.aligned_offset_) * DIO_READ_ALIGN_SIZE; - const int64_t length = static_cast(block_addr.aligned_length_) * DIO_READ_ALIGN_SIZE; + const int64_t offset = static_cast(block_addr.offset_) * DIO_READ_ALIGN_SIZE; + const int64_t length = static_cast(block_addr.length_) * DIO_READ_ALIGN_SIZE; const ObBackupStorageInfo *storage_info = backup_set_dest_.get_storage_info(); if (!block_addr.is_valid()) { ret = OB_INVALID_ARGUMENT; diff --git a/src/storage/backup/ob_backup_linked_block_writer.cpp b/src/storage/backup/ob_backup_linked_block_writer.cpp index 9d2c34ca9..4014e605e 100644 --- a/src/storage/backup/ob_backup_linked_block_writer.cpp +++ b/src/storage/backup/ob_backup_linked_block_writer.cpp @@ -106,7 +106,7 @@ int ObBackupLinkedBlockWriter::calc_next_block_addr_() ret = OB_ERR_UNEXPECTED; LOG_WARN("file offset should not be null", K(ret), KP_(file_offset)); } else { - current_block_addr_.aligned_offset_ = *file_offset_ / DIO_READ_ALIGN_SIZE; + current_block_addr_.offset_ = *file_offset_ / DIO_READ_ALIGN_SIZE; } return ret; } @@ -241,13 +241,16 @@ int ObBackupLinkedBlockItemWriter::get_entry_block_addr_( ObBackupDataType backup_data_type; backup_data_type.set_user_data_backup(); block_addr.ls_id_ = param.ls_id_.id(); - block_addr.type_ = backup_data_type.type_; + block_addr.data_type_ = backup_data_type.type_; block_addr.turn_id_ = param.turn_id_; block_addr.retry_id_ = param.retry_id_; block_addr.file_id_ = file_id; block_addr.backup_set_id_ = param.backup_set_desc_.backup_set_id_; - block_addr.aligned_offset_ = file_offset / DIO_READ_ALIGN_SIZE; // aligned offset is real file offset divided by 4096 - block_addr.aligned_length_ = BLOCK_SIZE / DIO_READ_ALIGN_SIZE; + block_addr.offset_ = file_offset / DIO_READ_ALIGN_SIZE; // aligned offset is real file offset divided by 4096 + block_addr.length_ = BLOCK_SIZE / DIO_READ_ALIGN_SIZE; + block_addr.block_type_ = ObBackupDeviceMacroBlockId::DATA_BLOCK; + block_addr.id_mode_ = static_cast(blocksstable::ObMacroBlockIdMode::ID_MODE_BACKUP); + block_addr.version_ = ObBackupDeviceMacroBlockId::BACKUP_MACRO_BLOCK_ID_VERSION; } return ret; } diff --git a/src/storage/backup/ob_backup_linked_item.h b/src/storage/backup/ob_backup_linked_item.h index 7eb8b8426..6d3d36210 100644 --- a/src/storage/backup/ob_backup_linked_item.h +++ b/src/storage/backup/ob_backup_linked_item.h @@ -61,7 +61,7 @@ public: bool operator!=(const ObBackupLinkedItem &other) const; TO_STRING_KV(K_(macro_id), K_(backup_id)); blocksstable::MacroBlockId macro_id_; - ObBackupPhysicalID backup_id_; // TODO(yanfeng): change type when quick restore code merge + ObBackupDeviceMacroBlockId backup_id_; }; } diff --git a/src/storage/backup/ob_backup_task.cpp b/src/storage/backup/ob_backup_task.cpp index fb155c9e0..c1aab0014 100644 --- a/src/storage/backup/ob_backup_task.cpp +++ b/src/storage/backup/ob_backup_task.cpp @@ -2425,7 +2425,7 @@ int ObPrefetchBackupInfoTask::inner_check_backup_item_need_copy_when_change_retr } else { LOG_DEBUG("find, no need copy", K(logic_id)); need_copy = false; - if (OB_FAIL(iter->physical_id_.get_backup_macro_block_index(backup_data_type_, iter->logic_id_, macro_index))) { + if (OB_FAIL(iter->physical_id_.get_backup_macro_block_index(iter->logic_id_, macro_index))) { LOG_WARN("failed to get backup macro block index", K(ret), K_(backup_data_type)); } } @@ -3102,13 +3102,19 @@ int ObLSBackupDataTask::write_ddl_other_block_(const blocksstable::ObBufferReade if (OB_FAIL(backup_data_ctx_.write_other_block(buffer_reader, offset, length))) { LOG_WARN("failed to write ddl other block", K(ret), K(buffer_reader)); } else { + ObBackupDataType backup_data_type; + backup_data_type.set_user_data_backup(); physical_id.backup_set_id_ = param_.backup_set_desc_.backup_set_id_; physical_id.ls_id_ = param_.ls_id_.id(); physical_id.turn_id_ = param_.turn_id_; physical_id.retry_id_ = param_.retry_id_; physical_id.file_id_ = task_id_; - physical_id.aligned_offset_ = offset / DIO_READ_ALIGN_SIZE; - physical_id.aligned_length_ = length / DIO_READ_ALIGN_SIZE; + physical_id.data_type_ = backup_data_type.type_; + physical_id.offset_ = offset / DIO_READ_ALIGN_SIZE; + physical_id.length_ = length / DIO_READ_ALIGN_SIZE; + physical_id.block_type_ = ObBackupDeviceMacroBlockId::DATA_BLOCK; + physical_id.id_mode_ = static_cast(blocksstable::ObMacroBlockIdMode::ID_MODE_BACKUP); + physical_id.version_ = ObBackupDeviceMacroBlockId::BACKUP_MACRO_BLOCK_ID_VERSION; } return ret; } @@ -3165,14 +3171,14 @@ int ObLSBackupDataTask::deal_with_sstable_other_block_root_blocks_( } else if (OB_FAIL(linked_writer->write(link_item))) { LOG_WARN("failed to write link item", K(ret)); } else { - LOG_INFO("write link item", K(link_item)); + LOG_INFO("write link item", K(tablet_id), K(table_key), K(link_item)); } } if (FAILEDx(linked_writer->close())) { LOG_WARN("failed to close item writer", K(ret)); } + LOG_INFO("deal with ddl sstable root blocks", K(tablet_id), K(table_key)); } - LOG_INFO("deal with ddl sstable root blocks", K(tablet_id), K(table_key)); return ret; } diff --git a/src/storage/high_availability/ob_storage_ha_reader.cpp b/src/storage/high_availability/ob_storage_ha_reader.cpp index 27e93731a..17d701e44 100644 --- a/src/storage/high_availability/ob_storage_ha_reader.cpp +++ b/src/storage/high_availability/ob_storage_ha_reader.cpp @@ -618,7 +618,7 @@ int ObCopyMacroBlockRestoreReader::fetch_macro_block_index_( backup::ObBackupDeviceMacroBlockId macro_id; if (OB_FAIL(restore_macro_block_id_mgr_->get_macro_block_id(block_id_idx, logic_block_id, macro_id))) { LOG_WARN("failed to get macro block id", K(ret), K(block_id_idx)); - } else if (OB_FAIL(macro_id.get_backup_macro_block_index(backup_data_type, logic_block_id, macro_index))) { + } else if (OB_FAIL(macro_id.get_backup_macro_block_index(logic_block_id, macro_index))) { LOG_WARN("failed to get backup macro block index", K(ret)); } } else { @@ -811,8 +811,9 @@ int ObCopyDDLMacroBlockRestoreReader::get_next_macro_block( LOG_WARN("macro block count is bigger than link item count", K(ret), K(macro_block_count_), K(link_item_)); } else { const ObLogicMacroBlockId logic_block_id(1, 1, 1); //TODO(yanfeng.yyy)Fake logical block id - backup::ObBackupPhysicalID physic_block_id; + backup::ObBackupDeviceMacroBlockId physic_block_id; share::ObBackupDataType data_type; + data_type.set_user_data_backup(); share::ObBackupPath backup_path; const int64_t align_size = DIO_READ_ALIGN_SIZE; backup::ObBackupMacroBlockIndex macro_index; @@ -828,8 +829,6 @@ int ObCopyDDLMacroBlockRestoreReader::get_next_macro_block( physic_block_id = link_item_.at(macro_block_count_).backup_id_; if (OB_FAIL(physic_block_id.get_backup_macro_block_index(logic_block_id, macro_index))) { LOG_WARN("failed to get backup macro block index", K(ret), K(logic_block_id), K(physic_block_id)); - } else if (OB_FAIL(ObRestoreUtils::get_backup_data_type(table_key_, data_type))) { - LOG_WARN("fail to get backup data type", K(ret), K(table_key_)); } else if (OB_FAIL(restore_base_info_->get_restore_backup_set_dest(macro_index.backup_set_id_, backup_set_brief_info))) { LOG_WARN("fail to get backup set dest", K(ret), K(macro_index)); } else if (OB_FAIL(restore_base_info_->get_restore_data_dest_id(*GCTX.sql_proxy_, MTL_ID(), dest_id))) { @@ -837,7 +836,7 @@ int ObCopyDDLMacroBlockRestoreReader::get_next_macro_block( } else if (OB_FALSE_IT(mod.storage_id_ = static_cast(dest_id))) { } else if (OB_FAIL(backup_set_dest.set(backup_set_brief_info.backup_set_path_))) { LOG_WARN("fail to set backup set dest", K(ret)); - } else if (OB_FAIL(share::ObBackupPathUtil::get_macro_block_backup_path(backup_set_dest, macro_index.ls_id_, + } else if (OB_FAIL(share::ObBackupPathUtilV_4_3_2::get_macro_block_backup_path(backup_set_dest, macro_index.ls_id_, data_type, macro_index.turn_id_, macro_index.retry_id_, macro_index.file_id_, backup_path))) { LOG_WARN("failed to get macro block index", K(ret), K(restore_base_info_), K(macro_index), KPC(restore_base_info_)); } else if (OB_FAIL(backup::ObLSBackupRestoreUtil::read_macro_block_data(backup_path.get_obstr(), @@ -4243,7 +4242,7 @@ int ObCopyRemoteSSTableMacroBlockRestoreReader::get_backup_macro_block_index_( share::ObBackupDataType data_type; if (OB_FAIL(ObRestoreUtils::get_backup_data_type(table_key_, data_type))) { LOG_WARN("fail to get backup data type", K(ret), K(table_key_)); - } else if (OB_FAIL(macro_id.macro_id_.get_backup_macro_block_index(data_type, macro_id.logic_block_id_, macro_index))) { + } else if (OB_FAIL(macro_id.macro_id_.get_backup_macro_block_index(macro_id.logic_block_id_, macro_index))) { LOG_WARN("failed to get backup macro block index", K(ret), K(data_type), K(macro_id)); } return ret; diff --git a/unittest/storage/backup/test_backup_linked_item_write_and_read.cpp b/unittest/storage/backup/test_backup_linked_item_write_and_read.cpp index 3996b6c93..5f40c1942 100644 --- a/unittest/storage/backup/test_backup_linked_item_write_and_read.cpp +++ b/unittest/storage/backup/test_backup_linked_item_write_and_read.cpp @@ -154,14 +154,19 @@ int prepare_one_item(const int64_t idx, const int64_t file_id, const ObLSBackupD item.macro_id_.third_id_ = 0; item.macro_id_.fourth_id_ = 0; + ObBackupDataType backup_data_type; + backup_data_type.set_user_data_backup(); item.backup_id_.ls_id_ = param.ls_id_.id(); - item.backup_id_.type_ = 0; + item.backup_id_.data_type_ = backup_data_type.type_; item.backup_id_.turn_id_ = param.turn_id_; item.backup_id_.retry_id_ = param.retry_id_; item.backup_id_.file_id_ = file_id; item.backup_id_.backup_set_id_ = param.backup_set_desc_.backup_set_id_; - item.backup_id_.aligned_offset_ = idx * OB_DEFAULT_MACRO_BLOCK_SIZE / 4096; - item.backup_id_.aligned_length_ = OB_DEFAULT_MACRO_BLOCK_SIZE / 4096; + item.backup_id_.offset_ = idx * OB_DEFAULT_MACRO_BLOCK_SIZE / 4096; + item.backup_id_.length_ = OB_DEFAULT_MACRO_BLOCK_SIZE / 4096; + item.backup_id_.block_type_ = ObBackupDeviceMacroBlockId::DATA_BLOCK; + item.backup_id_.id_mode_ = static_cast(blocksstable::ObMacroBlockIdMode::ID_MODE_BACKUP); + item.backup_id_.version_ = ObBackupDeviceMacroBlockId::BACKUP_MACRO_BLOCK_ID_VERSION; return ret; } @@ -226,9 +231,9 @@ struct ObBackupLinkItemCompare struct ObBackupPhysicalIDCompare { - bool operator()(const ObBackupPhysicalID &lhs, const ObBackupPhysicalID &rhs) + bool operator()(const ObBackupDeviceMacroBlockId &lhs, const ObBackupDeviceMacroBlockId &rhs) { - return lhs.aligned_offset_ < rhs.aligned_offset_; + return lhs.offset_ < rhs.offset_; } }; @@ -257,8 +262,8 @@ int sort_and_compare_item_list( } int sort_and_compare_block_list( - common::ObArray &block_list1, - common::ObArray &block_list2) + common::ObArray &block_list1, + common::ObArray &block_list2) { int ret = OB_SUCCESS; ObBackupPhysicalIDCompare cmp; @@ -269,8 +274,8 @@ int sort_and_compare_block_list( std::sort(block_list1.begin(), block_list1.end(), cmp); std::sort(block_list2.begin(), block_list2.end(), cmp); ARRAY_FOREACH_X(block_list1, idx, cnt, OB_SUCC(ret)) { - const ObBackupPhysicalID &id1 = block_list1.at(idx); - const ObBackupPhysicalID &id2 = block_list2.at(idx); + const ObBackupDeviceMacroBlockId &id1 = block_list1.at(idx); + const ObBackupDeviceMacroBlockId &id2 = block_list2.at(idx); if (id1 != id2) { ret = OB_ERR_UNEXPECTED; LOG_WARN("item not same", K(ret), K(idx), K(cnt), K(id1), K(id2)); @@ -351,7 +356,7 @@ int write_and_read_items(const int64_t item_count, const int64_t expected_block_ ObArray write_item_list; ObBackupLinkedBlockItemWriter writer; int64_t file_offset = 0; - ObBackupPhysicalID entry_block_id; + ObBackupDeviceMacroBlockId entry_block_id; int64_t total_block_count = 0; ObArray read_item_list; ObBackupLinkedBlockItemReader reader; @@ -359,8 +364,8 @@ int write_and_read_items(const int64_t item_count, const int64_t expected_block_ ObStorageIdMod id_mod; id_mod.storage_id_ = 1; id_mod.storage_used_mod_ = ObStorageUsedMod::STORAGE_USED_BACKUP; - ObArray written_block_list; - ObArray read_block_list; + ObArray written_block_list; + ObArray read_block_list; ObInOutBandwidthThrottle bandwidth_throttle; if (OB_FAIL(bandwidth_throttle.init(1024 * 1024 * 60))) {