hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix backup linked item in ss mode

Browse Source
This commit is contained in:
oceanoverflow 2024-09-25 21:47:20 +00:00 committed by ob-robot
parent e7bd5eab11
commit f6417feb0b
8 changed files with 46 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/storage/backup/ob_backup_data_struct.cpp
View File

@ -1573,7 +1573,7 @@ bool ObBackupDeviceMacroBlockId::is_valid() const
&& static_cast<uint64_t>(blocksstable::ObMacroBlockIdMode::ID_MODE_BACKUP) == id_mode_;
}
int ObBackupDeviceMacroBlockId::get_backup_macro_block_index(const share::ObBackupDataType &backup_data_type,
int ObBackupDeviceMacroBlockId::get_backup_macro_block_index(
const blocksstable::ObLogicMacroBlockId &logic_id, ObBackupMacroBlockIndex &macro_index) const
{
int ret = OB_SUCCESS;

4
src/storage/backup/ob_backup_data_struct.h
View File

@ -275,7 +275,7 @@ public:
};
struct ObBackupDeviceMacroBlockId;
typedef ObBackupPhysicalID ObBackupLinkedBlockAddr;
typedef ObBackupDeviceMacroBlockId ObBackupLinkedBlockAddr;
struct ObBackupMacroBlockIndex {
OB_UNIS_VERSION(1);
@ -475,7 +475,7 @@ public:
public:
ObBackupDeviceMacroBlockId();
~ObBackupDeviceMacroBlockId();
int get_backup_macro_block_index(const share::ObBackupDataType &backup_data_type,
int get_backup_macro_block_index(
const blocksstable::ObLogicMacroBlockId &logic_id, ObBackupMacroBlockIndex &macro_index) const;
int set(const int64_t backup_set_id, const int64_t ls_id, const int64_t dir_id,
const int64_t turn_id, const int64_t retry_id, const int64_t file_id,

4
src/storage/backup/ob_backup_linked_block_reader.cpp
View File

@ -112,8 +112,8 @@ int ObBackupLinkedBlockReader::pread_block_(const ObBackupLinkedBlockAddr &block
{
int ret = OB_SUCCESS;
ObBackupPath backup_path;
const int64_t offset = static_cast<int64_t>(block_addr.aligned_offset_) * DIO_READ_ALIGN_SIZE;
const int64_t length = static_cast<int64_t>(block_addr.aligned_length_) * DIO_READ_ALIGN_SIZE;
const int64_t offset = static_cast<int64_t>(block_addr.offset_) * DIO_READ_ALIGN_SIZE;
const int64_t length = static_cast<int64_t>(block_addr.length_) * DIO_READ_ALIGN_SIZE;
const ObBackupStorageInfo *storage_info = backup_set_dest_.get_storage_info();
if (!block_addr.is_valid()) {
ret = OB_INVALID_ARGUMENT;

11
src/storage/backup/ob_backup_linked_block_writer.cpp
View File

@ -106,7 +106,7 @@ int ObBackupLinkedBlockWriter::calc_next_block_addr_()
ret = OB_ERR_UNEXPECTED;
LOG_WARN("file offset should not be null", K(ret), KP_(file_offset));
} else {
current_block_addr_.aligned_offset_ = *file_offset_ / DIO_READ_ALIGN_SIZE;
current_block_addr_.offset_ = *file_offset_ / DIO_READ_ALIGN_SIZE;
}
return ret;
}
@ -241,13 +241,16 @@ int ObBackupLinkedBlockItemWriter::get_entry_block_addr_(
ObBackupDataType backup_data_type;
backup_data_type.set_user_data_backup();
block_addr.ls_id_ = param.ls_id_.id();
block_addr.type_ = backup_data_type.type_;
block_addr.data_type_ = backup_data_type.type_;
block_addr.turn_id_ = param.turn_id_;
block_addr.retry_id_ = param.retry_id_;
block_addr.file_id_ = file_id;
block_addr.backup_set_id_ = param.backup_set_desc_.backup_set_id_;
block_addr.aligned_offset_ = file_offset / DIO_READ_ALIGN_SIZE; // aligned offset is real file offset divided by 4096
block_addr.aligned_length_ = BLOCK_SIZE / DIO_READ_ALIGN_SIZE;
block_addr.offset_ = file_offset / DIO_READ_ALIGN_SIZE; // aligned offset is real file offset divided by 4096
block_addr.length_ = BLOCK_SIZE / DIO_READ_ALIGN_SIZE;
block_addr.block_type_ = ObBackupDeviceMacroBlockId::DATA_BLOCK;
block_addr.id_mode_ = static_cast<uint64_t>(blocksstable::ObMacroBlockIdMode::ID_MODE_BACKUP);
block_addr.version_ = ObBackupDeviceMacroBlockId::BACKUP_MACRO_BLOCK_ID_VERSION;
}
return ret;
}

2
src/storage/backup/ob_backup_linked_item.h
View File

@ -61,7 +61,7 @@ public:
bool operator!=(const ObBackupLinkedItem &other) const;
TO_STRING_KV(K_(macro_id), K_(backup_id));
blocksstable::MacroBlockId macro_id_;
ObBackupPhysicalID backup_id_; // TODO(yanfeng): change type when quick restore code merge
ObBackupDeviceMacroBlockId backup_id_;
};
}

16
src/storage/backup/ob_backup_task.cpp
View File

@ -2425,7 +2425,7 @@ int ObPrefetchBackupInfoTask::inner_check_backup_item_need_copy_when_change_retr
} else {
LOG_DEBUG("find, no need copy", K(logic_id));
need_copy = false;
if (OB_FAIL(iter->physical_id_.get_backup_macro_block_index(backup_data_type_, iter->logic_id_, macro_index))) {
if (OB_FAIL(iter->physical_id_.get_backup_macro_block_index(iter->logic_id_, macro_index))) {
LOG_WARN("failed to get backup macro block index", K(ret), K_(backup_data_type));
}
}
@ -3102,13 +3102,19 @@ int ObLSBackupDataTask::write_ddl_other_block_(const blocksstable::ObBufferReade
if (OB_FAIL(backup_data_ctx_.write_other_block(buffer_reader, offset, length))) {
LOG_WARN("failed to write ddl other block", K(ret), K(buffer_reader));
} else {
ObBackupDataType backup_data_type;
backup_data_type.set_user_data_backup();
physical_id.backup_set_id_ = param_.backup_set_desc_.backup_set_id_;
physical_id.ls_id_ = param_.ls_id_.id();
physical_id.turn_id_ = param_.turn_id_;
physical_id.retry_id_ = param_.retry_id_;
physical_id.file_id_ = task_id_;
physical_id.aligned_offset_ = offset / DIO_READ_ALIGN_SIZE;
physical_id.aligned_length_ = length / DIO_READ_ALIGN_SIZE;
physical_id.data_type_ = backup_data_type.type_;
physical_id.offset_ = offset / DIO_READ_ALIGN_SIZE;
physical_id.length_ = length / DIO_READ_ALIGN_SIZE;
physical_id.block_type_ = ObBackupDeviceMacroBlockId::DATA_BLOCK;
physical_id.id_mode_ = static_cast<uint64_t>(blocksstable::ObMacroBlockIdMode::ID_MODE_BACKUP);
physical_id.version_ = ObBackupDeviceMacroBlockId::BACKUP_MACRO_BLOCK_ID_VERSION;
}
return ret;
}
@ -3165,14 +3171,14 @@ int ObLSBackupDataTask::deal_with_sstable_other_block_root_blocks_(
} else if (OB_FAIL(linked_writer->write(link_item))) {
LOG_WARN("failed to write link item", K(ret));
} else {
LOG_INFO("write link item", K(link_item));
LOG_INFO("write link item", K(tablet_id), K(table_key), K(link_item));
}
}
if (FAILEDx(linked_writer->close())) {
LOG_WARN("failed to close item writer", K(ret));
}
LOG_INFO("deal with ddl sstable root blocks", K(tablet_id), K(table_key));
}
LOG_INFO("deal with ddl sstable root blocks", K(tablet_id), K(table_key));
return ret;
}

11
src/storage/high_availability/ob_storage_ha_reader.cpp
View File

@ -618,7 +618,7 @@ int ObCopyMacroBlockRestoreReader::fetch_macro_block_index_(
backup::ObBackupDeviceMacroBlockId macro_id;
if (OB_FAIL(restore_macro_block_id_mgr_->get_macro_block_id(block_id_idx, logic_block_id, macro_id))) {
LOG_WARN("failed to get macro block id", K(ret), K(block_id_idx));
} else if (OB_FAIL(macro_id.get_backup_macro_block_index(backup_data_type, logic_block_id, macro_index))) {
} else if (OB_FAIL(macro_id.get_backup_macro_block_index(logic_block_id, macro_index))) {
LOG_WARN("failed to get backup macro block index", K(ret));
}
} else {
@ -811,8 +811,9 @@ int ObCopyDDLMacroBlockRestoreReader::get_next_macro_block(
LOG_WARN("macro block count is bigger than link item count", K(ret), K(macro_block_count_), K(link_item_));
} else {
const ObLogicMacroBlockId logic_block_id(1, 1, 1); //TODO(yanfeng.yyy)Fake logical block id
backup::ObBackupPhysicalID physic_block_id;
backup::ObBackupDeviceMacroBlockId physic_block_id;
share::ObBackupDataType data_type;
data_type.set_user_data_backup();
share::ObBackupPath backup_path;
const int64_t align_size = DIO_READ_ALIGN_SIZE;
backup::ObBackupMacroBlockIndex macro_index;
@ -828,8 +829,6 @@ int ObCopyDDLMacroBlockRestoreReader::get_next_macro_block(
physic_block_id = link_item_.at(macro_block_count_).backup_id_;
if (OB_FAIL(physic_block_id.get_backup_macro_block_index(logic_block_id, macro_index))) {
LOG_WARN("failed to get backup macro block index", K(ret), K(logic_block_id), K(physic_block_id));
} else if (OB_FAIL(ObRestoreUtils::get_backup_data_type(table_key_, data_type))) {
LOG_WARN("fail to get backup data type", K(ret), K(table_key_));
} else if (OB_FAIL(restore_base_info_->get_restore_backup_set_dest(macro_index.backup_set_id_, backup_set_brief_info))) {
LOG_WARN("fail to get backup set dest", K(ret), K(macro_index));
} else if (OB_FAIL(restore_base_info_->get_restore_data_dest_id(*GCTX.sql_proxy_, MTL_ID(), dest_id))) {
@ -837,7 +836,7 @@ int ObCopyDDLMacroBlockRestoreReader::get_next_macro_block(
} else if (OB_FALSE_IT(mod.storage_id_ = static_cast<uint64_t>(dest_id))) {
} else if (OB_FAIL(backup_set_dest.set(backup_set_brief_info.backup_set_path_))) {
LOG_WARN("fail to set backup set dest", K(ret));
} else if (OB_FAIL(share::ObBackupPathUtil::get_macro_block_backup_path(backup_set_dest, macro_index.ls_id_,
} else if (OB_FAIL(share::ObBackupPathUtilV_4_3_2::get_macro_block_backup_path(backup_set_dest, macro_index.ls_id_,
data_type, macro_index.turn_id_, macro_index.retry_id_, macro_index.file_id_, backup_path))) {
LOG_WARN("failed to get macro block index", K(ret), K(restore_base_info_), K(macro_index), KPC(restore_base_info_));
} else if (OB_FAIL(backup::ObLSBackupRestoreUtil::read_macro_block_data(backup_path.get_obstr(),
@ -4243,7 +4242,7 @@ int ObCopyRemoteSSTableMacroBlockRestoreReader::get_backup_macro_block_index_(
share::ObBackupDataType data_type;
if (OB_FAIL(ObRestoreUtils::get_backup_data_type(table_key_, data_type))) {
LOG_WARN("fail to get backup data type", K(ret), K(table_key_));
} else if (OB_FAIL(macro_id.macro_id_.get_backup_macro_block_index(data_type, macro_id.logic_block_id_, macro_index))) {
} else if (OB_FAIL(macro_id.macro_id_.get_backup_macro_block_index(macro_id.logic_block_id_, macro_index))) {
LOG_WARN("failed to get backup macro block index", K(ret), K(data_type), K(macro_id));
}
return ret;

29
unittest/storage/backup/test_backup_linked_item_write_and_read.cpp
View File

@ -154,14 +154,19 @@ int prepare_one_item(const int64_t idx, const int64_t file_id, const ObLSBackupD
item.macro_id_.third_id_ = 0;
item.macro_id_.fourth_id_ = 0;
ObBackupDataType backup_data_type;
backup_data_type.set_user_data_backup();
item.backup_id_.ls_id_ = param.ls_id_.id();
item.backup_id_.type_ = 0;
item.backup_id_.data_type_ = backup_data_type.type_;
item.backup_id_.turn_id_ = param.turn_id_;
item.backup_id_.retry_id_ = param.retry_id_;
item.backup_id_.file_id_ = file_id;
item.backup_id_.backup_set_id_ = param.backup_set_desc_.backup_set_id_;
item.backup_id_.aligned_offset_ = idx * OB_DEFAULT_MACRO_BLOCK_SIZE / 4096;
item.backup_id_.aligned_length_ = OB_DEFAULT_MACRO_BLOCK_SIZE / 4096;
item.backup_id_.offset_ = idx * OB_DEFAULT_MACRO_BLOCK_SIZE / 4096;
item.backup_id_.length_ = OB_DEFAULT_MACRO_BLOCK_SIZE / 4096;
item.backup_id_.block_type_ = ObBackupDeviceMacroBlockId::DATA_BLOCK;
item.backup_id_.id_mode_ = static_cast<uint64_t>(blocksstable::ObMacroBlockIdMode::ID_MODE_BACKUP);
item.backup_id_.version_ = ObBackupDeviceMacroBlockId::BACKUP_MACRO_BLOCK_ID_VERSION;
return ret;
}
@ -226,9 +231,9 @@ struct ObBackupLinkItemCompare
struct ObBackupPhysicalIDCompare
{
bool operator()(const ObBackupPhysicalID &lhs, const ObBackupPhysicalID &rhs)
bool operator()(const ObBackupDeviceMacroBlockId &lhs, const ObBackupDeviceMacroBlockId &rhs)
{
return lhs.aligned_offset_ < rhs.aligned_offset_;
return lhs.offset_ < rhs.offset_;
}
};
@ -257,8 +262,8 @@ int sort_and_compare_item_list(
}
int sort_and_compare_block_list(
common::ObArray<ObBackupPhysicalID> &block_list1,
common::ObArray<ObBackupPhysicalID> &block_list2)
common::ObArray<ObBackupDeviceMacroBlockId> &block_list1,
common::ObArray<ObBackupDeviceMacroBlockId> &block_list2)
{
int ret = OB_SUCCESS;
ObBackupPhysicalIDCompare cmp;
@ -269,8 +274,8 @@ int sort_and_compare_block_list(
std::sort(block_list1.begin(), block_list1.end(), cmp);
std::sort(block_list2.begin(), block_list2.end(), cmp);
ARRAY_FOREACH_X(block_list1, idx, cnt, OB_SUCC(ret)) {
const ObBackupPhysicalID &id1 = block_list1.at(idx);
const ObBackupPhysicalID &id2 = block_list2.at(idx);
const ObBackupDeviceMacroBlockId &id1 = block_list1.at(idx);
const ObBackupDeviceMacroBlockId &id2 = block_list2.at(idx);
if (id1 != id2) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("item not same", K(ret), K(idx), K(cnt), K(id1), K(id2));
@ -351,7 +356,7 @@ int write_and_read_items(const int64_t item_count, const int64_t expected_block_
ObArray<ObBackupLinkedItem> write_item_list;
ObBackupLinkedBlockItemWriter writer;
int64_t file_offset = 0;
ObBackupPhysicalID entry_block_id;
ObBackupDeviceMacroBlockId entry_block_id;
int64_t total_block_count = 0;
ObArray<ObBackupLinkedItem> read_item_list;
ObBackupLinkedBlockItemReader reader;
@ -359,8 +364,8 @@ int write_and_read_items(const int64_t item_count, const int64_t expected_block_
ObStorageIdMod id_mod;
id_mod.storage_id_ = 1;
id_mod.storage_used_mod_ = ObStorageUsedMod::STORAGE_USED_BACKUP;
ObArray<ObBackupPhysicalID> written_block_list;
ObArray<ObBackupPhysicalID> read_block_list;
ObArray<ObBackupDeviceMacroBlockId> written_block_list;
ObArray<ObBackupDeviceMacroBlockId> read_block_list;
ObInOutBandwidthThrottle bandwidth_throttle;
if (OB_FAIL(bandwidth_throttle.init(1024 * 1024 * 60))) {