hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix excessive usage of memory when backup

Browse Source
This commit is contained in:
oceanoverflow 2024-11-21 09:14:53 +00:00 committed by ob-robot
parent 8f3fafe9f2
commit b15bc996fc
8 changed files with 514 additions and 379 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/storage/backup/ob_backup_ctx.cpp
View File

@ -1236,7 +1236,8 @@ ObLSBackupCtx::ObLSBackupCtx()
index_builder_mgr_(),
bandwidth_throttle_(NULL),
mview_dep_tablet_set_(),
wait_reuse_across_sstable_time_(0)
wait_reuse_across_sstable_time_(0),
mv_mutex_()
{}
ObLSBackupCtx::~ObLSBackupCtx()

1
src/storage/backup/ob_backup_ctx.h
View File

@ -348,6 +348,7 @@ public:
common::ObInOutBandwidthThrottle *bandwidth_throttle_;
common::hash::ObHashSet<common::ObTabletID> mview_dep_tablet_set_;
int64_t wait_reuse_across_sstable_time_;
mutable lib::ObMutex mv_mutex_;
DISALLOW_COPY_AND_ASSIGN(ObLSBackupCtx);
};

70
src/storage/backup/ob_backup_index_block_builder_mgr.cpp
View File

@ -86,7 +86,7 @@ int ObBackupTaskIndexRebuilderMgr::prepare_index_block_rebuilder_if_need(
bool is_first = false;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("backup task index rebuilder mgr do not init", K(ret));
LOG_WARN("backup task index rebuilder mgr do not init", K(ret));
} else if (OB_FAIL(check_is_first_item_for_table_key_(item, is_first))) {
LOG_WARN("failed to check is first item for table key" ,K(ret), K(item));
} else if (!is_first) {
@ -369,8 +369,7 @@ int ObBackupTabletIndexBlockBuilderMgr::prepare_sstable_index_builders(
int ObBackupTabletIndexBlockBuilderMgr::open_sstable_index_builder(
const common::ObTabletID &tablet_id, const ObTabletHandle &tablet_handle,
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable,
const bool is_empty)
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable)
{
int ret = OB_SUCCESS;
ObBackupTabletSSTableIndexBuilderMgr *mgr = NULL;
@ -387,7 +386,7 @@ int ObBackupTabletIndexBlockBuilderMgr::open_sstable_index_builder(
} else if (OB_ISNULL(mgr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup tablet sstable index builder mgr should not be null", K(ret));
} else if (OB_FAIL(mgr->add_sstable_index_builder(ls_id_, tablet_handle, table_key, sstable, is_empty))) {
} else if (OB_FAIL(mgr->add_sstable_index_builder(ls_id_, tablet_handle, table_key, sstable))) {
LOG_WARN("failed to add sstable index builder", K(ret), K(tablet_id), K(table_key), KPC(sstable));
} else {
LOG_INFO("[INDEX_BUILDER_MGR] open sstable index builder", K(tablet_id), K(table_key));
@ -450,6 +449,39 @@ int ObBackupTabletIndexBlockBuilderMgr::get_sstable_index_builder(const common::
return ret;
}
int ObBackupTabletIndexBlockBuilderMgr::check_sstable_index_builder_mgr_exist(
const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key, bool &exist)
{
int ret = OB_SUCCESS;
exist = false;
ObBackupTabletSSTableIndexBuilderMgr *builder_mgr = NULL;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("storage ha table info mgr do not init", K(ret));
} else if (!tablet_id.is_valid() || !table_key.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid argument", K(ret), K(tablet_id), K(table_key));
} else {
ObMutexGuard guard(mutex_);
blocksstable::ObSSTableIndexBuilder *index_builder = NULL;
if (OB_FAIL(sstable_builder_map_.get_refactored(tablet_id, builder_mgr))) {
if (OB_HASH_NOT_EXIST == ret) {
ret = OB_SUCCESS;
exist = false;
} else {
LOG_WARN("failed to get tablet table key mgr", K(ret), K(tablet_id));
}
} else if (OB_FAIL(builder_mgr->get_sstable_index_builder(table_key, index_builder))) {
LOG_WARN("failed to get sstable index builder", K(ret), K(table_key));
} else if (OB_ISNULL(index_builder)) {
exist = false;
} else {
exist = true;
}
}
return ret;
}
int ObBackupTabletIndexBlockBuilderMgr::get_sstable_merge_result(const common::ObTabletID &tablet_id,
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTableMergeRes *&merge_res)
{
@ -570,7 +602,6 @@ ObBackupTabletSSTableIndexBuilderMgr::ObBackupTabletSSTableIndexBuilderMgr()
table_keys_(),
builders_(),
merge_results_(),
sstable_ready_list_(),
local_reuse_map_(),
is_major_compaction_mview_dep_tablet_(false)
{
@ -598,19 +629,13 @@ int ObBackupTabletSSTableIndexBuilderMgr::init(const uint64_t tenant_id, const c
} else {
builders_.set_attr(mem_attr);
merge_results_.set_attr(mem_attr);
sstable_ready_list_.set_attr(mem_attr);
if (OB_FAIL(builders_.prepare_allocate(table_key_array.count()))) {
LOG_WARN("failed to reserve table keys", K(ret));
} else if (OB_FAIL(merge_results_.prepare_allocate(table_key_array.count()))) {
LOG_WARN("failed to reserve merge res", K(ret));
} else if (OB_FAIL(sstable_ready_list_.prepare_allocate(table_key_array.count()))) {
LOG_WARN("failed to reserve merge res", K(ret));
} else if (is_major_compaction_mview_dep_tablet && OB_FAIL(local_reuse_map_.create(BUCKET_NUM, mem_attr))) {
LOG_WARN("failed to create local reuse map", K(ret));
} else {
ARRAY_FOREACH(sstable_ready_list_, idx) {
sstable_ready_list_.at(idx) = false;
}
tablet_id_ = tablet_id;
is_major_compaction_mview_dep_tablet_ = is_major_compaction_mview_dep_tablet;
LOG_INFO("init backup tablet sstable index builder mgr", K(tablet_id), K(is_major_compaction_mview_dep_tablet));
@ -637,8 +662,7 @@ void ObBackupTabletSSTableIndexBuilderMgr::reset()
int ObBackupTabletSSTableIndexBuilderMgr::add_sstable_index_builder(
const share::ObLSID &ls_id, const ObTabletHandle &tablet_handle,
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable,
const bool is_empty)
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable)
{
int ret = OB_SUCCESS;
ObMutexGuard guard(mutex_);
@ -659,10 +683,6 @@ int ObBackupTabletSSTableIndexBuilderMgr::add_sstable_index_builder(
LOG_WARN("failed to alloc sstable index builder", K(ret), K(table_key));
} else {
builders_.at(idx) = index_builder;
if (is_empty) {
sstable_ready_list_.at(idx) = true;
LOG_INFO("set sstable ready", K(ls_id), K(table_key));
}
}
return ret;
}
@ -741,8 +761,7 @@ int ObBackupTabletSSTableIndexBuilderMgr::close_sstable_index_builder(
} else if (OB_FAIL(merge_results_.at(idx).assign(sstable_merge_res))) {
LOG_WARN("failed to assign res", K(ret), K(sstable_merge_res));
} else {
LOG_INFO("close sstable index builders", K_(tablet_id), K_(table_keys), K(idx), K(sstable_merge_res));
sstable_ready_list_.at(idx) = true;
LOG_INFO("close sstable index builders", K_(tablet_id), K(table_key), K(idx), K(sstable_merge_res));
}
return ret;
}
@ -765,19 +784,6 @@ int ObBackupTabletSSTableIndexBuilderMgr::free_sstable_index_builder(const stora
return ret;
}
int ObBackupTabletSSTableIndexBuilderMgr::check_sstable_merge_ready_(bool &is_ready)
{
int ret = OB_SUCCESS;
is_ready = true;
ARRAY_FOREACH(merge_results_, idx) {
if (!sstable_ready_list_.at(idx)) {
is_ready = false;
break;
}
}
return ret;
}
int ObBackupTabletSSTableIndexBuilderMgr::get_merge_type_(
const storage::ObITable::TableKey &table_key, compaction::ObMergeType &merge_type)
{

9
src/storage/backup/ob_backup_index_block_builder_mgr.h
View File

@ -81,8 +81,9 @@ public:
const common::ObIArray<storage::ObITable::TableKey> &table_keys,
const bool is_major_compaction_mview_dep_tablet);
int open_sstable_index_builder(const common::ObTabletID &tablet_id, const ObTabletHandle &tablet_handle,
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable, const bool is_empty);
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable);
int get_sstable_index_builder_mgr(const common::ObTabletID &tablet_id, ObBackupTabletSSTableIndexBuilderMgr *&builder_mgr);
int check_sstable_index_builder_mgr_exist(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key, bool &exist);
int get_sstable_index_builder(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key,
blocksstable::ObSSTableIndexBuilder *&sstable_index_builder);
int get_sstable_merge_result(const common::ObTabletID &tablet_id,
@ -117,7 +118,7 @@ public:
const common::ObIArray<storage::ObITable::TableKey> &table_key_array,
const bool is_major_compaction_mview_dep_tablet);
int add_sstable_index_builder(const share::ObLSID &ls_id, const ObTabletHandle &tablet_handle,
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable, const bool is_empty);
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable);
int get_sstable_index_builder(const storage::ObITable::TableKey &table_key,
blocksstable::ObSSTableIndexBuilder *&index_builder);
int get_sstable_merge_result(const storage::ObITable::TableKey &table_key, blocksstable::ObSSTableMergeRes *&merge_res);
@ -125,10 +126,9 @@ public:
int free_sstable_index_builder(const storage::ObITable::TableKey &table_key);
int close_sstable_index_builder(const storage::ObITable::TableKey &table_key, ObIODevice *device_handle);
bool is_major_compaction_mview_dep_tablet() const { return is_major_compaction_mview_dep_tablet_; }
TO_STRING_KV(K_(tablet_id), K_(table_keys), K_(builders), K_(merge_results), K_(sstable_ready_list));
TO_STRING_KV(K_(tablet_id), K_(table_keys), K_(builders), K_(merge_results));
private:
int check_sstable_merge_ready_(bool &is_ready);
int get_merge_type_(const storage::ObITable::TableKey &table_key, compaction::ObMergeType &merge_type);
int prepare_data_store_desc_(const share::ObLSID &ls_id, const ObTabletHandle &tablet_handle,
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable,
@ -155,7 +155,6 @@ private:
common::ObArray<storage::ObITable::TableKey> table_keys_;
common::ObArray<blocksstable::ObSSTableIndexBuilder *> builders_;
common::ObArray<blocksstable::ObSSTableMergeRes> merge_results_;
common::ObArray<bool> sstable_ready_list_;
common::hash::ObHashMap<blocksstable::ObLogicMacroBlockId, ObBackupMacroBlockIndex> local_reuse_map_;
bool is_major_compaction_mview_dep_tablet_;
DISALLOW_COPY_AND_ASSIGN(ObBackupTabletSSTableIndexBuilderMgr);

662
src/storage/backup/ob_backup_task.cpp
View File

@ -2444,14 +2444,8 @@ int ObLSBackupDataTask::process()
LOG_WARN("failed to prepare companion index file handle for data file", K(ret), K_(task_id), K(meta_tree_type));
} else if (OB_FAIL(device_handle_array.push_back(meta_tree_device_handle))) {
LOG_WARN("failed to push back device handle", K(ret), KP(meta_tree_device_handle));
} else if (OB_FAIL(do_backup_macro_block_data_(device_handle_array))) {
LOG_WARN("failed to do backup macro block data", K(ret));
} else if (OB_FAIL(do_backup_ddl_other_blocks_())) {
LOG_WARN("failed to backup ddl other blocks", K(ret));
} else if (OB_FAIL(do_backup_other_blocks_addr_block_())) {
LOG_WARN("failed to do backup other blocks root block", K(ret));
} else if (OB_FAIL(do_backup_meta_data_(device_handle_array.at(0)))) {
LOG_WARN("failed to do backup meta data", K(ret));
} else if (OB_FAIL(do_iterate_backup_items_(device_handle_array))) {
LOG_WARN("failed to do iterate backup items", K(ret));
} else if (OB_FAIL(ObBackupUtils::check_ls_valid_for_backup(
param_.tenant_id_, param_.ls_id_, ls_backup_ctx_->rebuild_seq_))) {
LOG_WARN("failed to check ls valid for backup", K(ret), K_(param));
@ -2565,127 +2559,311 @@ int ObLSBackupDataTask::do_write_file_header_()
return ret;
}
int ObLSBackupDataTask::do_backup_macro_block_data_(common::ObIArray<ObIODevice *> &device_handle_array)
// TODO(yanfeng): consider wrapping device_handle_array into a structure
int ObLSBackupDataTask::prepare_macro_block_readers_(ObMultiMacroBlockBackupReader *&macro_reader,
ObMultiMacroBlockBackupReader *&ddl_macro_reader,
common::ObIArray<ObIODevice *> &device_handle_array)
{
int ret = OB_SUCCESS;
macro_reader = NULL;
ddl_macro_reader = NULL;
ObArray<ObBackupMacroBlockId> ddl_macro_list;
ObArray<ObBackupMacroBlockId> macro_list;
ObArray<ObBackupProviderItem> item_list;
ObArray<ObBackupMacroBlockId> need_copy_macro_list;
ObMultiMacroBlockBackupReader *reader = NULL;
ObArenaAllocator io_allocator("BUR_IOUB", OB_MALLOC_NORMAL_BLOCK_SIZE, param_.tenant_id_);
if (OB_ISNULL(ls_backup_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls backup ctx should not be null", K(ret));
} else if (OB_FAIL(get_macro_block_id_list_(macro_list, item_list))) {
if (OB_FAIL(get_ddl_block_id_list_(ddl_macro_list))) {
LOG_WARN("failed to get macro block id list", K(ret));
} else if (OB_FAIL(get_need_copy_macro_block_id_list_(need_copy_macro_list))) {
} else if (OB_UNLIKELY(ddl_macro_list.empty())) {
LOG_INFO("no macro list need to backup");
} else if (OB_FAIL(prepare_macro_block_reader_(param_.tenant_id_, ddl_macro_list, ddl_macro_reader))) {
LOG_WARN("failed to prepare macro block reader", K(ret), K_(param), K(ddl_macro_list));
}
if (FAILEDx(get_macro_block_id_list_(macro_list, item_list))) {
LOG_WARN("failed to get macro block id list", K(ret));
} else if (OB_UNLIKELY(macro_list.empty())) {
LOG_INFO("no macro list need to backup");
} else if (OB_FAIL(prepare_macro_block_reader_(param_.tenant_id_, macro_list, reader))) {
LOG_WARN("failed to prepare macro block reader", K(ret), K(macro_list));
} else if (OB_FAIL(prepare_macro_block_reader_(param_.tenant_id_, macro_list, macro_reader))) {
LOG_WARN("failed to prepare macro block reader", K(ret), K_(param), K(macro_list));
} else if (OB_FAIL(rebuilder_mgr_.init(item_list, index_builder_mgr_, device_handle_array))) {
LOG_WARN("failed to init rebuilder mg", K(ret), K(item_list));
} else {
const share::ObLSID &ls_id = param_.ls_id_;
const int64_t turn_id = param_.turn_id_;
const int64_t retry_id = param_.retry_id_;
const int64_t file_id = task_id_;
LOG_INFO("start backup macro block data", K_(backup_data_type), K(ls_id), K(turn_id), K(retry_id), K(file_id), K(need_copy_macro_list));
#ifdef ERRSIM
bool has_need_copy = false;
ObLogicMacroBlockId first_logic_id;
#endif
while (OB_SUCC(ret)) {
io_allocator.reuse();
ObBufferReader buffer_reader;
common::ObTabletID tablet_id;
ObITable::TableKey table_key;
ObLogicMacroBlockId logic_id;
MacroBlockId macro_id;
ObBackupMacroBlockIndex macro_index;
ObBackupProviderItem backup_item;
ObBackupDeviceMacroBlockId physical_id;
bool need_copy = true; // when the macro block is neither in inc backup, or backed up in previous turn or retry, we backup it.
bool need_reuse_across_sstable = false; // when two major sstable share one macro block, we reuse it
if (REACH_TIME_INTERVAL(CHECK_DISK_SPACE_INTERVAL)) {
if (OB_FAIL(check_disk_space_())) {
LOG_WARN("failed to check disk space", K(ret));
}
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(macro_reader)) {
ObLSBackupFactory::free(macro_reader);
}
if (OB_NOT_NULL(ddl_macro_reader)) {
ObLSBackupFactory::free(ddl_macro_reader);
}
}
return ret;
}
int ObLSBackupDataTask::do_iterate_backup_items_(common::ObIArray<ObIODevice *> &device_handle_array)
{
int ret = OB_SUCCESS;
ObMultiMacroBlockBackupReader *macro_reader = NULL;
ObMultiMacroBlockBackupReader *ddl_macro_reader = NULL;
if (OB_FAIL(prepare_macro_block_readers_(macro_reader, ddl_macro_reader, device_handle_array))) {
LOG_WARN("failed to prepare readers", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < backup_items_.count(); ++i) {
const ObBackupProviderItem &item = backup_items_.at(i);
if (REACH_TIME_INTERVAL(CHECK_DISK_SPACE_INTERVAL)) {
if (OB_FAIL(check_disk_space_())) {
LOG_WARN("failed to check disk space", K(ret));
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(get_next_macro_block_data_(reader, buffer_reader, table_key, logic_id, macro_id, &io_allocator))) {
if (OB_ITER_END == ret) {
LOG_INFO("iterator meet end", K(logic_id));
ret = OB_SUCCESS;
}
if (OB_SUCC(ret)) {
switch (item.get_item_type()) {
case PROVIDER_ITEM_TABLET_SSTABLE_INDEX_BUILDER_PREPARE: {
if (OB_FAIL(do_prepare_sstable_builders_(item))) {
LOG_WARN("failed to do prepare sstable builders", K(ret), K(item));
}
break;
} else {
LOG_WARN("failed to get next macro block data", K(ret));
}
} else if (OB_FAIL(get_backup_item_(table_key, logic_id, backup_item))) {
LOG_WARN("failed to get backup item", K(ret), K(table_key), K(logic_id));
} else if (FALSE_IT(tablet_id = table_key.tablet_id_)) {
} else if (FALSE_IT(need_copy = backup_item.get_need_copy())) {
} else if (FALSE_IT(need_reuse_across_sstable = backup_item.get_need_reuse_across_sstable())) {
} else if (OB_FAIL(check_macro_block_data_(buffer_reader))) {
LOG_WARN("failed to check macro block data", K(ret), K(buffer_reader));
case PROVIDER_ITEM_MACRO_ID: {
if (OB_FAIL(do_backup_single_macro_block_data_(macro_reader, item, device_handle_array))) {
LOG_WARN("failed to do backup single macro block data", K(ret), K(item));
}
break;
}
case PROVIDER_ITEM_DDL_OTHER_BLOCK_ID: {
if (OB_FAIL(do_backup_single_ddl_other_block_(ddl_macro_reader, item))) {
LOG_WARN("failed to do backup single ddl other block", K(ret), K(item));
}
break;
}
case PROVIDER_ITEM_TABLET_AND_SSTABLE_META: {
const ObTabletID &tablet_id = item.get_tablet_id();
const storage::ObITable::TableKey &table_key = item.get_table_key();
if (OB_FAIL(deal_with_sstable_other_block_root_blocks_(tablet_id, table_key))) {
LOG_WARN("failed to deal with ddl sstable root blocks", K(ret), K(tablet_id), K(table_key));
} else if (OB_FAIL(do_backup_single_meta_data_(item, device_handle_array.at(0)))) {
LOG_WARN("failed to do backup single meta data", K(ret), K(item));
}
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("item type is not valid", K(ret), K(item));
break;
}
}
}
}
if (OB_NOT_NULL(macro_reader)) {
ObLSBackupFactory::free(macro_reader);
}
if (OB_NOT_NULL(ddl_macro_reader)) {
ObLSBackupFactory::free(ddl_macro_reader);
}
return ret;
}
int ObLSBackupDataTask::do_prepare_sstable_builders_(const ObBackupProviderItem &item)
{
int ret = OB_SUCCESS;
const ObTabletID &tablet_id = item.get_tablet_id();
ObBackupTabletHandleRef *tablet_ref = NULL;
ObArray<storage::ObSSTableWrapper> sstable_array;
bool is_major_compaction_mview_dep_tablet = false;
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
if (!item.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid arg", K(ret), K(item));
} else if (OB_ISNULL(ls_backup_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls backup ctx should not be null", K(ret));
} else if (OB_FAIL(get_tablet_handle_(tablet_id, tablet_ref))) {
LOG_WARN("failed to get tablet handle", K(ret), K(tablet_id));
} else if (OB_FAIL(tablet_ref->tablet_handle_.get_obj()->fetch_table_store(table_store_wrapper))) {
LOG_WARN("fail to fetch table store", K(ret), K(tablet_id));
} else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet_id, is_major_compaction_mview_dep_tablet))) {
LOG_WARN("failed to check is mview dep tablet", K(ret), K(tablet_id));
} else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(tablet_ref->tablet_handle_, backup_data_type_, *table_store_wrapper.get_member(), is_major_compaction_mview_dep_tablet, sstable_array))) {
LOG_WARN("failed to get sstables by data type", K(ret), KPC(tablet_ref), K_(backup_data_type));
} else if (OB_FAIL(prepare_tablet_sstable_index_builders_(tablet_id, is_major_compaction_mview_dep_tablet, sstable_array))) {
LOG_WARN("failed to prepare tablet sstable index builders", K(ret), K_(param), K(tablet_id), K(sstable_array));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < sstable_array.count(); ++i) {
storage::ObSSTableWrapper &sstable_wrapper = sstable_array.at(i);
ObSSTable *sstable_ptr = NULL;
if (OB_ISNULL(sstable_ptr = sstable_wrapper.get_sstable())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table should not be null", K(ret));
} else {
if (need_reuse_across_sstable) {
if (OB_FAIL(wait_reuse_other_block_ready_(tablet_id, logic_id, macro_index))) {
LOG_WARN("failed to wait reuse other block ready", K(ret), K(tablet_id), K(logic_id));
} else {
LOG_INFO("wait reuse other block ready", K(tablet_id), K(table_key), K(logic_id), K(macro_index));
}
} else {
if (need_copy) {
if (OB_FAIL(write_macro_block_data_(buffer_reader, table_key, logic_id, macro_index))) {
LOG_WARN("failed to write macro block data", K(ret), K(buffer_reader), K(table_key), K(logic_id));
} else {
LOG_INFO("write macro block", K(tablet_id), K(table_key), K(logic_id), K(macro_index));
}
} else {
macro_index = backup_item.get_macro_index();
}
}
}
if (FAILEDx(macro_index.get_backup_physical_id(backup_data_type_, physical_id))) {
LOG_WARN("failed to get backup physical id", K(ret), K_(backup_data_type), K(macro_index));
} else if (OB_FAIL(prepare_index_block_rebuilder_if_need_(backup_item, &task_id_))) {
LOG_WARN("failed to prepare index block rebuilder if need", K(ret));
} else if (OB_FAIL(append_macro_row_to_rebuilder_(backup_item, buffer_reader, physical_id))) {
LOG_WARN("failed to append macro row to rebuilder", K(ret), K(backup_item));
} else if (OB_FAIL(update_logic_id_to_macro_index_(tablet_id, table_key, logic_id, macro_index))) {
LOG_WARN("failed to update logic id to macro index", K(ret), K(logic_id), K(table_key), K(macro_index));
} else if (OB_FAIL(close_index_block_rebuilder_if_need_(backup_item))) {
LOG_WARN("failed to close index block rebuilder if need", K(ret));
} else if (OB_FAIL(mark_backup_item_finished_(backup_item, physical_id))) {
LOG_WARN("failed to mark backup item finished", K(ret), K(backup_item), K(macro_index), K(physical_id));
}
#ifdef ERRSIM
if (OB_SUCC(ret)) {
bool is_last = false;
const int64_t ERRSIM_TASK_ID = GCONF.errsim_backup_task_id;
const int64_t ERRSIM_TABLET_ID = GCONF.errsim_backup_tablet_id;
if (task_id_ != ERRSIM_TASK_ID && backup_item.get_tablet_id().id() != ERRSIM_TABLET_ID && !backup_item.get_table_key().is_major_sstable()) {
const ObITable::TableKey &table_key = sstable_ptr->get_key();
if (GCTX.is_shared_storage_mode() && table_key.is_ddl_dump_sstable()) {
// do nothing
} else if (OB_FAIL(rebuilder_mgr_.check_is_last_item_for_table_key(backup_item, is_last))) {
LOG_WARN("failed to check is last item for table key", K(ret));
} else if (is_last) {
SERVER_EVENT_SYNC_ADD("backup", "BEFORE_CLOSE_BACKUP_INDEX_BUILDER",
"turn_id", param_.turn_id_,
"retry_id", param_.retry_id_,
"tablet_id", backup_item.get_tablet_id(),
"table_key", backup_item.get_table_key(),
"task_id", task_id_);
DEBUG_SYNC(BEFORE_CLOSE_BACKUP_INDEX_BUILDER);
} else if (OB_FAIL(open_tablet_sstable_index_builder_(tablet_id, tablet_ref->tablet_handle_, table_key, sstable_ptr))) {
LOG_WARN("failed to open tablet sstable index builder", K(ret), K(tablet_id), KPC(tablet_ref), K(table_key));
}
}
}
}
return ret;
}
int ObLSBackupDataTask::prepare_tablet_sstable_index_builders_(const common::ObTabletID &tablet_id,
const bool is_major_compaction_mview_dep_tablet, common::ObIArray<storage::ObSSTableWrapper> &sstable_array)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(index_builder_mgr_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("index builder mgr should not be null", K(ret));
} else {
ObArray<storage::ObITable::TableKey> table_key_array;
ARRAY_FOREACH_X(sstable_array, idx, cnt, OB_SUCC(ret)) {
storage::ObSSTableWrapper &sstable_wrapper = sstable_array.at(idx);
ObSSTable *sstable_ptr = NULL;
if (OB_ISNULL(sstable_ptr = sstable_wrapper.get_sstable())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table should not be null", K(ret));
} else if (OB_FAIL(table_key_array.push_back(sstable_ptr->get_key()))) {
LOG_WARN("failed to push back", K(ret), KPC(sstable_ptr));
}
}
if (FAILEDx(index_builder_mgr_->prepare_sstable_index_builders(tablet_id, table_key_array, is_major_compaction_mview_dep_tablet))) {
LOG_WARN("failed to open sstable index builder", K(ret), K(tablet_id), K(table_key_array), K(is_major_compaction_mview_dep_tablet));
}
}
return ret;
}
int ObLSBackupDataTask::open_tablet_sstable_index_builder_(
const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle,
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(index_builder_mgr_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("index builder mgr should not be null", K(ret));
} else if (OB_FAIL(index_builder_mgr_->open_sstable_index_builder(tablet_id, tablet_handle, table_key, sstable))) {
LOG_WARN("failed to open sstable index builder", K(ret), K(tablet_id), K(tablet_handle), K(table_key), KPC(sstable));
}
return ret;
}
int ObLSBackupDataTask::do_wait_index_builder_ready_(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key)
{
int ret = OB_SUCCESS;
ObBackupTabletSSTableIndexBuilderMgr *mgr = NULL;
bool exist = false;
static const int64_t DEFAULT_SLEEP_US = 10_ms;
while (OB_SUCC(ret)) {
if (OB_ISNULL(ls_backup_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls back ctx should not be null", K(ret));
} else if (OB_SUCCESS != ls_backup_ctx_->get_result_code()) {
ret = OB_EAGAIN;
LOG_WARN("ctx already failed", K(ret));
} else if (OB_ISNULL(index_builder_mgr_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("mgr should not be null", K(ret));
} else if (OB_FAIL(index_builder_mgr_->check_sstable_index_builder_mgr_exist(tablet_id, table_key, exist))) {
LOG_WARN("failed to check sstable index builder mgr exist", K(ret), K(tablet_id), K(table_key));
} else if (!exist) {
LOG_INFO("index builder mgr still not exist", K(ret), K(tablet_id), K(table_key));
usleep(DEFAULT_SLEEP_US);
} else {
break;
}
}
return ret;
}
int ObLSBackupDataTask::do_backup_single_macro_block_data_(ObMultiMacroBlockBackupReader *macro_reader,
const ObBackupProviderItem &backup_item, common::ObIArray<ObIODevice *> &device_handle)
{
int ret = OB_SUCCESS;
ObArenaAllocator io_allocator("BackupMacroBlk", OB_MALLOC_NORMAL_BLOCK_SIZE, param_.tenant_id_);
ObBufferReader buffer_reader;
common::ObTabletID tablet_id;
ObITable::TableKey table_key;
ObLogicMacroBlockId logic_id;
MacroBlockId macro_id;
ObBackupMacroBlockIndex macro_index;
ObBackupDeviceMacroBlockId physical_id;
bool need_copy = true; // when the macro block is neither in inc backup, or backed up in previous turn or retry, we backup it.
bool need_reuse_for_mv = false; // when two major sstable share one macro block, we reuse it
#ifdef ERRSIM
bool has_need_copy = false;
ObLogicMacroBlockId first_logic_id;
#endif
if (FAILEDx(ls_backup_ctx_->stat_mgr_.add_macro_block(backup_item.get_backup_data_type(), logic_id, backup_item.get_need_copy()))) {
LOG_WARN("failed to add macro block", K(ret), K(backup_item));
} else if (OB_FAIL(ls_backup_ctx_->stat_mgr_.add_bytes(backup_data_type_, macro_index.length_))) {
LOG_WARN("failed to add bytes", K(ret));
if (OB_ISNULL(macro_reader)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("macro reader should not be null", K(ret));
} else if (OB_FAIL(do_wait_index_builder_ready_(backup_item.get_tablet_id(), backup_item.get_table_key()))) {
LOG_WARN("failed to do wait index builder ready", K(ret), K(backup_item));
} else if (OB_FAIL(get_next_macro_block_data_(macro_reader, buffer_reader, table_key, logic_id, macro_id, &io_allocator))) {
if (OB_ITER_END == ret) {
LOG_INFO("iterator meet end", K(logic_id));
ret = OB_SUCCESS;
} else {
LOG_WARN("failed to get next macro block data", K(ret));
}
} else if (FALSE_IT(tablet_id = table_key.tablet_id_)) {
} else if (FALSE_IT(need_copy = backup_item.get_need_copy())) {
} else if (OB_FAIL(check_need_reuse_sstable_macro_block_for_mv_(tablet_id, table_key, logic_id, need_reuse_for_mv))) {
LOG_WARN("failed to check need reuse across sstable", K(ret));
} else if (OB_FAIL(check_macro_block_data_(buffer_reader))) {
LOG_WARN("failed to check macro block data", K(ret), K(buffer_reader));
} else {
if (need_reuse_for_mv) {
if (OB_FAIL(wait_reuse_other_block_ready_(tablet_id, logic_id, macro_index))) {
LOG_WARN("failed to wait reuse other block ready", K(ret), K(tablet_id), K(logic_id));
} else {
LOG_INFO("wait reuse other block ready", K(tablet_id), K(table_key), K(logic_id), K(macro_index));
}
} else {
if (need_copy) {
if (OB_FAIL(write_macro_block_data_(buffer_reader, table_key, logic_id, macro_index))) {
LOG_WARN("failed to write macro block data", K(ret), K(buffer_reader), K(table_key), K(logic_id));
} else {
LOG_INFO("write macro block", K(tablet_id), K(table_key), K(logic_id), K(macro_index));
}
} else {
macro_index = backup_item.get_macro_index();
}
}
if (FAILEDx(macro_index.get_backup_physical_id(backup_data_type_, physical_id))) {
LOG_WARN("failed to get backup physical id", K(ret), K_(backup_data_type), K(macro_index));
} else if (OB_FAIL(prepare_index_block_rebuilder_if_need_(backup_item, &task_id_))) {
LOG_WARN("failed to prepare index block rebuilder if need", K(ret));
} else if (OB_FAIL(append_macro_row_to_rebuilder_(backup_item, buffer_reader, physical_id))) {
LOG_WARN("failed to append macro row to rebuilder", K(ret), K(backup_item));
} else if (OB_FAIL(update_logic_id_to_macro_index_(tablet_id, table_key, logic_id, macro_index))) {
LOG_WARN("failed to update logic id to macro index", K(ret), K(logic_id), K(table_key), K(macro_index));
} else if (OB_FAIL(close_index_block_rebuilder_if_need_(backup_item))) {
LOG_WARN("failed to close index block rebuilder if need", K(ret));
} else if (OB_FAIL(mark_backup_item_finished_(backup_item, physical_id))) {
LOG_WARN("failed to mark backup item finished", K(ret), K(backup_item), K(macro_index), K(physical_id));
}
#ifdef ERRSIM
if (OB_SUCC(ret)) {
bool is_last = false;
const int64_t ERRSIM_TASK_ID = GCONF.errsim_backup_task_id;
const int64_t ERRSIM_TABLET_ID = GCONF.errsim_backup_tablet_id;
if (task_id_ != ERRSIM_TASK_ID && backup_item.get_tablet_id().id() != ERRSIM_TABLET_ID && !backup_item.get_table_key().is_major_sstable()) {
// do nothing
} else if (OB_FAIL(rebuilder_mgr_.check_is_last_item_for_table_key(backup_item, is_last))) {
LOG_WARN("failed to check is last item for table key", K(ret));
} else if (is_last) {
SERVER_EVENT_SYNC_ADD("backup", "BEFORE_CLOSE_BACKUP_INDEX_BUILDER",
"turn_id", param_.turn_id_,
"retry_id", param_.retry_id_,
"tablet_id", backup_item.get_tablet_id(),
"table_key", backup_item.get_table_key(),
"task_id", task_id_);
DEBUG_SYNC(BEFORE_CLOSE_BACKUP_INDEX_BUILDER);
}
}
#endif
if (FAILEDx(ls_backup_ctx_->stat_mgr_.add_macro_block(backup_item.get_backup_data_type(), logic_id, backup_item.get_need_copy()))) {
LOG_WARN("failed to add macro block", K(ret), K(backup_item));
} else if (OB_FAIL(ls_backup_ctx_->stat_mgr_.add_bytes(backup_data_type_, macro_index.length_))) {
LOG_WARN("failed to add bytes", K(ret));
} else {
#ifdef ERRSIM
if (need_copy && table_key.is_major_sstable()) {
if (!has_need_copy) {
@ -2694,10 +2872,10 @@ int ObLSBackupDataTask::do_backup_macro_block_data_(common::ObIArray<ObIODevice
}
}
#endif
backup_stat_.input_bytes_ += OB_DEFAULT_MACRO_BLOCK_SIZE;
backup_stat_.finish_macro_block_count_ += 1;
}
backup_stat_.input_bytes_ += OB_DEFAULT_MACRO_BLOCK_SIZE;
backup_stat_.finish_macro_block_count_ += 1;
}
}
#ifdef ERRSIM
if (has_need_copy) {
SERVER_EVENT_SYNC_ADD("backup_data", "first_need_copy_logic_id",
@ -2707,83 +2885,51 @@ int ObLSBackupDataTask::do_backup_macro_block_data_(common::ObIArray<ObIODevice
"turn_id", param_.turn_id_,
"retry_id", param_.retry_id_,
"first_logic_id", first_logic_id,
to_cstring(file_id));
to_cstring(task_id_));
}
#endif
}
if (OB_NOT_NULL(reader)) {
ObLSBackupFactory::free(reader);
}
return ret;
}
int ObLSBackupDataTask::do_backup_ddl_other_blocks_()
int ObLSBackupDataTask::do_backup_single_ddl_other_block_(
ObMultiMacroBlockBackupReader *reader, const ObBackupProviderItem &item)
{
int ret = OB_SUCCESS;
ObArray<ObBackupMacroBlockId> macro_list;
ObMultiMacroBlockBackupReader *reader = NULL;
ObArenaAllocator io_allocator("BUR_IOUB", OB_MALLOC_NORMAL_BLOCK_SIZE, param_.tenant_id_);
if (OB_ISNULL(ls_backup_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls backup ctx should not be null", K(ret));
} else if (OB_FAIL(get_ddl_block_id_list_(macro_list))) {
LOG_WARN("failed to get macro block id list", K(ret));
} else if (OB_UNLIKELY(macro_list.empty())) {
LOG_INFO("no macro list need to backup");
} else if (OB_FAIL(prepare_macro_block_reader_(param_.tenant_id_, macro_list, reader))) {
LOG_WARN("failed to prepare macro block reader", K(ret), K(macro_list));
} else {
LOG_INFO("start backup ddl other blocks", K(macro_list));
ObBufferReader buffer_reader;
ObITable::TableKey table_key;
ObLogicMacroBlockId unused_logic_id;
MacroBlockId macro_id;
ObBackupLinkedBlockAddr physical_id;
ObTabletID tablet_id;
ObBackupOtherBlocksMgr *other_block_mgr = NULL;
ObBackupLinkedBlockItemWriter *linked_writer = NULL;
while (OB_SUCC(ret)) {
io_allocator.reuse();
macro_id.reset();
physical_id.reset();
tablet_id.reset();
if (REACH_TIME_INTERVAL(CHECK_DISK_SPACE_INTERVAL)) {
if (OB_FAIL(check_disk_space_())) {
LOG_WARN("failed to check disk space", K(ret));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(get_next_macro_block_data_(reader,
buffer_reader,
table_key,
unused_logic_id,
macro_id,
&io_allocator))) {
if (OB_ITER_END == ret) {
LOG_INFO("iterator meet end");
ret = OB_SUCCESS;
break;
} else {
LOG_WARN("failed to get next macro block data", K(ret));
}
} else if (OB_FAIL(check_macro_block_data_(buffer_reader))) {
LOG_WARN("failed to check macro block data", K(ret), K(buffer_reader));
} else if (OB_FAIL(get_tablet_id_for_macro_id_(macro_id, tablet_id))) {
LOG_WARN("failed to get tablet id for macro id", K(ret), K(macro_id));
} else if (OB_FAIL(get_other_block_mgr_for_tablet_(tablet_id, other_block_mgr, linked_writer))) {
LOG_WARN("failed to get other block mgr for tablet", K(ret), K(tablet_id));
} else if (OB_FAIL(write_ddl_other_block_(buffer_reader, physical_id))) {
LOG_WARN("failed to write ddl other blocks", K(ret));
} else if (OB_FAIL(add_item_to_other_block_mgr_(macro_id, physical_id, other_block_mgr))) {
LOG_WARN("failed to add item to other block mgr", K(ret), K(macro_id), K(physical_id));
} else {
backup_stat_.input_bytes_ += OB_DEFAULT_MACRO_BLOCK_SIZE;
backup_stat_.finish_macro_block_count_ += 1;
}
ObArenaAllocator io_allocator("BackupDDLBlock", OB_MALLOC_NORMAL_BLOCK_SIZE, param_.tenant_id_);
ObBufferReader buffer_reader;
ObITable::TableKey table_key;
ObLogicMacroBlockId unused_logic_id;
MacroBlockId macro_id;
ObBackupLinkedBlockAddr physical_id;
ObTabletID tablet_id;
ObBackupOtherBlocksMgr *other_block_mgr = NULL;
ObBackupLinkedBlockItemWriter *linked_writer = NULL;
if (OB_FAIL(get_next_macro_block_data_(reader,
buffer_reader,
table_key,
unused_logic_id,
macro_id,
&io_allocator))) {
if (OB_ITER_END == ret) {
LOG_INFO("iterator meet end");
ret = OB_SUCCESS;
} else {
LOG_WARN("failed to get next macro block data", K(ret));
}
}
if (OB_NOT_NULL(reader)) {
ObLSBackupFactory::free(reader);
} else if (OB_FAIL(check_macro_block_data_(buffer_reader))) {
LOG_WARN("failed to check macro block data", K(ret), K(buffer_reader));
} else if (OB_FAIL(get_tablet_id_for_macro_id_(macro_id, tablet_id))) {
LOG_WARN("failed to get tablet id for macro id", K(ret), K(macro_id));
} else if (OB_FAIL(get_other_block_mgr_for_tablet_(tablet_id, other_block_mgr, linked_writer))) {
LOG_WARN("failed to get other block mgr for tablet", K(ret), K(tablet_id));
} else if (OB_FAIL(write_ddl_other_block_(buffer_reader, physical_id))) {
LOG_WARN("failed to write ddl other blocks", K(ret));
} else if (OB_FAIL(add_item_to_other_block_mgr_(macro_id, physical_id, other_block_mgr))) {
LOG_WARN("failed to add item to other block mgr", K(ret), K(macro_id), K(physical_id));
} else {
backup_stat_.input_bytes_ += OB_DEFAULT_MACRO_BLOCK_SIZE;
backup_stat_.finish_macro_block_count_ += 1;
}
return ret;
}
@ -2916,30 +3062,6 @@ int ObLSBackupDataTask::deal_with_sstable_other_block_root_blocks_(
return ret;
}
int ObLSBackupDataTask::do_backup_other_blocks_addr_block_()
{
int ret = OB_SUCCESS;
ObArray<ObBackupProviderItem> item_list;
if (OB_ISNULL(ls_backup_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls backup ctx should not be null", K(ret));
} else if (OB_FAIL(get_sstable_meta_item_list_(item_list))) {
LOG_WARN("failed to get sstable meta item list", K(ret));
} else if (item_list.empty()) {
LOG_INFO("no meta need to back up");
} else {
ARRAY_FOREACH_X(item_list, idx, cnt, OB_SUCC(ret)) {
const ObBackupProviderItem &item = item_list.at(idx);
const ObTabletID &tablet_id = item.get_tablet_id();
const storage::ObITable::TableKey &table_key = item.get_table_key();
if (OB_FAIL(deal_with_sstable_other_block_root_blocks_(tablet_id, table_key))) {
LOG_WARN("failed to deal with ddl sstable root blocks", K(ret), K(tablet_id), K(table_key));
}
}
}
return ret;
}
int ObLSBackupDataTask::get_sstable_meta_item_list_(common::ObIArray<ObBackupProviderItem> &list)
{
int ret = OB_SUCCESS;
@ -2973,48 +3095,75 @@ int ObLSBackupDataTask::get_tablet_id_for_macro_id_(const blocksstable::MacroBlo
return ret;
}
int ObLSBackupDataTask::do_backup_meta_data_(ObIODevice *device_handle)
int ObLSBackupDataTask::do_backup_single_meta_data_(const ObBackupProviderItem &item, ObIODevice *device_handle)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObArray<ObBackupProviderItem> item_list;
ObBackupTabletStat *tablet_stat = NULL;
const common::ObTabletID &tablet_id = item.get_tablet_id();
const share::ObBackupDataType &backup_data_type = item.get_backup_data_type();
ObBackupTabletHandleRef *tablet_ref = NULL;
bool can_release = false;
if (OB_ISNULL(ls_backup_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls backup ctx should not be null", K(ret));
} else if (OB_FAIL(get_meta_item_list_(item_list))) {
LOG_WARN("failed to get meta item list", K(ret));
} else if (item_list.empty()) {
LOG_INFO("no meta need to back up");
} else if (FALSE_IT(tablet_stat = &ls_backup_ctx_->tablet_stat_)) {
} else if (OB_FAIL(get_tablet_handle_(tablet_id, tablet_ref))) {
LOG_WARN("failed to get tablet handle", K(ret), K_(task_id), K(tablet_id), K(item));
} else if (OB_FAIL(do_wait_sstable_index_builder_ready_(tablet_ref->tablet_handle_))) {
LOG_WARN("failed to wait sstable index builder ready", K(ret), K(item));
} else if (OB_FAIL(do_backup_tablet_meta_(SSTABLE_META_READER, BACKUP_SSTABLE_META,
backup_data_type, tablet_id, tablet_ref->tablet_handle_, device_handle))) {
LOG_WARN("failed to backup sstable meta", K(ret), K(backup_data_type), K(tablet_id), KPC(tablet_ref));
} else if (OB_FAIL(do_backup_tablet_meta_(TABLET_META_READER, BACKUP_TABLET_META,
backup_data_type, tablet_id, tablet_ref->tablet_handle_, device_handle))) {
LOG_WARN("failed to backup tablet meta", K(ret), K(backup_data_type), K(tablet_id), KPC(tablet_ref));
} else if (OB_FAIL(tablet_stat->add_finished_tablet_meta_count(tablet_id))) {
LOG_WARN("failed to add finished tablt meta count", K(ret), K(tablet_id));
} else if (OB_FAIL(tablet_stat->check_can_release_tablet(tablet_id, can_release))) {
LOG_WARN("failed to check can release tablet", K(ret), K(tablet_id));
} else if (!can_release) {
// do nothing
} else if (OB_FAIL(tablet_stat->free_tablet_stat(tablet_id))) {
LOG_WARN("failed to free tablet stat", K(ret), K_(finished_tablet_list), K(tablet_id));
} else if (OB_FAIL(release_tablet_handle_(tablet_id))) {
LOG_WARN("failed to release tablet handle", K(ret), K(tablet_id));
} else if (OB_FAIL(remove_sstable_index_builder_(tablet_id))) {
LOG_WARN("failed to remove sstable index builder", K(ret), K(tablet_id));
}
return ret;
}
int ObLSBackupDataTask::do_wait_sstable_index_builder_ready_(ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
ObTablet *tablet = nullptr;
bool is_major_compaction_mview_dep = false;
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
common::ObArray<storage::ObSSTableWrapper> sstable_array;
const ObTabletTableStore *table_store = nullptr;
if (OB_ISNULL(tablet = tablet_handle.get_obj())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet is nullptr.", K(ret), K(tablet_handle));
} else if (OB_FAIL(tablet->fetch_table_store(table_store_wrapper))) {
LOG_WARN("fail to fetch table store", K(ret));
} else if (OB_FAIL(table_store_wrapper.get_member(table_store))) {
LOG_WARN("failed to get table store", K(ret), KPC(tablet));
} else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet->get_tablet_id(), is_major_compaction_mview_dep))) {
LOG_WARN("failed to check is mview dep tablet", K(ret), KPC(tablet));
} else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(
tablet_handle, backup_data_type_, *table_store_wrapper.get_member(), is_major_compaction_mview_dep, sstable_array))) {
LOG_WARN("failed to get sstables by data type", K(ret), K(tablet_handle));
} else {
LOG_INFO("start backup meta data", K(item_list));
for (int64_t i = 0; OB_SUCC(ret) && i < item_list.count(); ++i) {
const ObBackupProviderItem &item = item_list.at(i);
const common::ObTabletID &tablet_id = item.get_tablet_id();
const share::ObBackupDataType &backup_data_type = item.get_backup_data_type();
ObBackupTabletHandleRef *tablet_ref = NULL;
bool can_release = false;
if (OB_FAIL(get_tablet_handle_(tablet_id, tablet_ref))) {
LOG_WARN("failed to get tablet handle", K(ret), K_(task_id), K(tablet_id), K(item), K(i), K(item_list));
} else if (OB_FAIL(do_backup_tablet_meta_(SSTABLE_META_READER, BACKUP_SSTABLE_META,
backup_data_type, tablet_id, tablet_ref->tablet_handle_, device_handle))) {
LOG_WARN("failed to backup sstable meta", K(ret), K(backup_data_type), K(tablet_id), KPC(tablet_ref));
} else if (OB_FAIL(do_backup_tablet_meta_(TABLET_META_READER, BACKUP_TABLET_META,
backup_data_type, tablet_id, tablet_ref->tablet_handle_, device_handle))) {
LOG_WARN("failed to backup tablet meta", K(ret), K(backup_data_type), K(tablet_id), KPC(tablet_ref));
} else if (OB_FAIL(tablet_stat->add_finished_tablet_meta_count(tablet_id))) {
LOG_WARN("failed to add finished tablt meta count", K(ret), K(tablet_id));
} else if (OB_FAIL(tablet_stat->check_can_release_tablet(tablet_id, can_release))) {
LOG_WARN("failed to check can release tablet", K(ret), K(tablet_id));
} else if (!can_release) {
// do nothing
} else if (OB_FAIL(tablet_stat->free_tablet_stat(tablet_id))) {
LOG_WARN("failed to free tablet stat", K(ret), K_(finished_tablet_list), K(tablet_id));
} else if (OB_FAIL(release_tablet_handle_(tablet_id))) {
LOG_WARN("failed to release tablet handle", K(ret), K(tablet_id));
} else if (OB_FAIL(remove_sstable_index_builder_(tablet_id))) {
LOG_WARN("failed to remove sstable index builder", K(ret), K(tablet_id));
for (int64_t i = 0; OB_SUCC(ret) && i < sstable_array.count(); ++i) {
storage::ObSSTableWrapper &sstable_wrapper = sstable_array.at(i);
ObSSTable *sstable_ptr = NULL;
if (OB_ISNULL(sstable_ptr = sstable_wrapper.get_sstable())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table should not be null", K(ret));
} else {
if (OB_FAIL(do_wait_index_builder_ready_(tablet->get_tablet_id(), sstable_ptr->get_key()))) {
LOG_WARN("failed to do wait index builder ready", K(ret));
}
}
}
}
@ -3953,6 +4102,39 @@ int ObLSBackupDataTask::inner_check_reuse_block_ready_(
return ret;
}
int ObLSBackupDataTask::check_need_reuse_sstable_macro_block_for_mv_(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key,
const blocksstable::ObLogicMacroBlockId &logic_id, bool &need_reuse_for_mv)
{
int ret = OB_SUCCESS;
need_reuse_for_mv = false;
ObBackupTabletIndexBlockBuilderMgr *mgr = index_builder_mgr_;
ObBackupTabletSSTableIndexBuilderMgr *sstable_mgr = NULL;
bool macro_index_exist = false;
if (!table_key.is_major_sstable()) {
need_reuse_for_mv = false;
} else if (OB_ISNULL(ls_backup_ctx_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls backup ctx should not be null", K(ret));
} else {
ObMutexGuard guard(ls_backup_ctx_->mv_mutex_);
if (OB_FAIL(mgr->get_sstable_index_builder_mgr(tablet_id, sstable_mgr))) {
LOG_WARN("failed to get sstable index builder mgr", K(ret), K(tablet_id));
} else if (!sstable_mgr->is_major_compaction_mview_dep_tablet()) {
need_reuse_for_mv = false;
} else if (OB_FAIL(sstable_mgr->check_place_holder_macro_index_exist(logic_id, macro_index_exist))) {
LOG_WARN("failed to check place holder macro index exist", K(ret), K(logic_id));
} else if (macro_index_exist) {
need_reuse_for_mv = true;
LOG_INFO("macro index exist, reuse local", K(tablet_id), K(table_key), K(logic_id));
} else if (OB_FAIL(sstable_mgr->insert_place_holder_macro_index(logic_id))) {
LOG_WARN("failed to insert empty macro index", K(ret), K(logic_id));
} else {
need_reuse_for_mv = false;
}
}
return ret;
}
/* ObLSBackupMetaTask */
ObLSBackupMetaTask::ObLSBackupMetaTask()

21
src/storage/backup/ob_backup_task.h
View File

@ -487,10 +487,21 @@ private:
private:
int build_backup_file_header_(ObBackupFileHeader &file_header);
int do_write_file_header_();
int do_backup_ddl_other_blocks_();
int do_backup_other_blocks_addr_block_();
int do_backup_macro_block_data_(common::ObIArray<ObIODevice *> &device_handle);
int do_backup_meta_data_(ObIODevice *device_handle);
int prepare_macro_block_readers_(ObMultiMacroBlockBackupReader *&macro_reader,
ObMultiMacroBlockBackupReader *&ddl_macro_reader,
common::ObIArray<ObIODevice *> &device_handle_array);
int do_iterate_backup_items_(common::ObIArray<ObIODevice *> &device_handle);
int do_prepare_sstable_builders_(const ObBackupProviderItem &item);
int do_backup_single_ddl_other_block_(ObMultiMacroBlockBackupReader *reader, const ObBackupProviderItem &item);
int do_wait_index_builder_ready_(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key);
int do_backup_single_macro_block_data_(ObMultiMacroBlockBackupReader *macro_reader,
const ObBackupProviderItem &item, common::ObIArray<ObIODevice *> &device_handle);
int do_backup_single_meta_data_(const ObBackupProviderItem &item, ObIODevice *device_handle);
int do_wait_sstable_index_builder_ready_(ObTabletHandle &tablet_handle);
int prepare_tablet_sstable_index_builders_(const common::ObTabletID &tablet_id,
const bool is_major_compaction_mview_dep_tablet, common::ObIArray<storage::ObSSTableWrapper> &sstable_array);
int open_tablet_sstable_index_builder_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle,
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable);
int do_backup_tablet_meta_(const ObTabletMetaReaderType reader_type, const ObBackupMetaType meta_type,
const share::ObBackupDataType &backup_data_type, const common::ObTabletID &tablet_id,
ObTabletHandle &tablet_handle, ObIODevice *device_handle);
@ -567,6 +578,8 @@ private:
const blocksstable::ObLogicMacroBlockId &logic_id, ObBackupMacroBlockIndex &macro_index);
int inner_check_reuse_block_ready_(const common::ObTabletID &tablet_id,
const blocksstable::ObLogicMacroBlockId &logic_id, ObBackupMacroBlockIndex &macro_index, bool &is_ready);
int check_need_reuse_sstable_macro_block_for_mv_(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key,
const blocksstable::ObLogicMacroBlockId &logic_id, bool &need_reuse_for_mv);
private:
static const int64_t CHECK_DISK_SPACE_INTERVAL = 5 * 1000 * 1000; // 5s;

114
src/storage/backup/ob_backup_utils.cpp
View File

@ -1299,7 +1299,7 @@ ObBackupProviderItem::ObBackupProviderItem()
table_key_(), tablet_id_(),
nested_offset_(0), nested_size_(0),
timestamp_(0), need_copy_(true), macro_index_(),
absolute_row_offset_(0), need_reuse_across_sstable_(false)
absolute_row_offset_(0)
{}
ObBackupProviderItem::~ObBackupProviderItem()
@ -1309,9 +1309,10 @@ int ObBackupProviderItem::set_with_fake(const ObBackupProviderItemType &item_typ
const common::ObTabletID &tablet_id, const share::ObBackupDataType &backup_data_type)
{
int ret = OB_SUCCESS;
if (PROVIDER_ITEM_TABLET_AND_SSTABLE_META != item_type) {
if (PROVIDER_ITEM_TABLET_AND_SSTABLE_META != item_type
&& PROVIDER_ITEM_TABLET_SSTABLE_INDEX_BUILDER_PREPARE != item_type) {
ret = OB_ERR_SYS;
LOG_WARN("get invalid args", K(ret));
LOG_WARN("get invalid args", K(ret), K(item_type));
} else if (!tablet_id.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(tablet_id));
@ -1382,8 +1383,7 @@ bool ObBackupProviderItem::operator==(const ObBackupProviderItem &other) const
return item_type_ == other.item_type_ && backup_data_type_ == other.backup_data_type_ && logic_id_ == other.logic_id_
&& macro_block_id_ == other.macro_block_id_ && table_key_ == other.table_key_
&& tablet_id_ == other.tablet_id_ && nested_size_ == other.nested_size_ && nested_offset_ == other.nested_offset_
&& need_copy_ == other.need_copy_ && macro_index_ == other.macro_index_ && absolute_row_offset_ == other.absolute_row_offset_
&& need_reuse_across_sstable_ == other.need_reuse_across_sstable_;
&& need_copy_ == other.need_copy_ && macro_index_ == other.macro_index_ && absolute_row_offset_ == other.absolute_row_offset_;
}
bool ObBackupProviderItem::operator!=(const ObBackupProviderItem &other) const
@ -1450,7 +1450,6 @@ int ObBackupProviderItem::deep_copy(const ObBackupProviderItem &src)
need_copy_ = src.need_copy_;
macro_index_ = src.macro_index_;
absolute_row_offset_ = src.absolute_row_offset_;
need_reuse_across_sstable_ = src.need_reuse_across_sstable_;
return ret;
}
@ -1469,7 +1468,6 @@ int ObBackupProviderItem::deep_copy(const ObBackupProviderItem &src, char *buf,
need_copy_ = src.need_copy_;
macro_index_ = src.macro_index_;
absolute_row_offset_ = src.absolute_row_offset_;
need_reuse_across_sstable_ = src.need_reuse_across_sstable_;
return ret;
}
@ -1478,7 +1476,8 @@ bool ObBackupProviderItem::is_valid() const
bool bret = false;
if (PROVIDER_ITEM_MACRO_ID != item_type_
&& PROVIDER_ITEM_DDL_OTHER_BLOCK_ID != item_type_
&& PROVIDER_ITEM_TABLET_AND_SSTABLE_META != item_type_) {
&& PROVIDER_ITEM_TABLET_AND_SSTABLE_META != item_type_
&& PROVIDER_ITEM_TABLET_SSTABLE_INDEX_BUILDER_PREPARE != item_type_) {
bret = false;
} else {
bret = logic_id_.is_valid() && macro_block_id_.is_valid()
@ -1500,7 +1499,6 @@ void ObBackupProviderItem::reset()
need_copy_ = true;
macro_index_.reset();
absolute_row_offset_ = 0;
need_reuse_across_sstable_ = false;
}
ObITable::TableKey ObBackupProviderItem::get_fake_table_key_()
@ -1992,8 +1990,8 @@ int ObBackupTabletProvider::prepare_tablet_(const uint64_t tenant_id, const shar
} else if (OB_FAIL(fetch_tablet_sstable_array_(
tablet_id, tablet_ref->tablet_handle_, *table_store_wrapper.get_member(), backup_data_type, is_major_compaction_mview_dep_tablet, sstable_array))) {
LOG_WARN("failed to fetch tablet sstable array", K(ret), K(tablet_id), KPC(tablet_ref), K(backup_data_type));
} else if (OB_FAIL(prepare_tablet_sstable_index_builders_(tablet_id, is_major_compaction_mview_dep_tablet, sstable_array))) {
LOG_WARN("failed to prepare tablet sstable index builders", K(ret), K_(param), K(tablet_id), K(sstable_array));
} else if (OB_FAIL(add_prepare_tablet_item_(tablet_id))) {
LOG_WARN("failed to prepare tablet item", K(ret), K(tablet_id));
} else {
ObITable::TableKey ss_ddl_table_key;
bool has_ss_ddl = false;
@ -2017,8 +2015,6 @@ int ObBackupTabletProvider::prepare_tablet_(const uint64_t tenant_id, const shar
} else {
if (OB_FAIL(fetch_all_logic_macro_block_id_(tablet_id, tablet_ref->tablet_handle_, table_key, *sstable_ptr, count))) {
LOG_WARN("failed to fetch all logic macro block id", K(ret), K(tablet_id), KPC(tablet_ref), K(table_key));
} else if (OB_FAIL(open_tablet_sstable_index_builder_(tablet_id, tablet_ref->tablet_handle_, table_key, sstable_ptr, 0 == count/*is_empty*/))) {
LOG_WARN("failed to open tablet sstable index builder", K(ret), K(tablet_id), KPC(tablet_ref), K(table_key));
} else {
total_count += count;
}
@ -2407,47 +2403,6 @@ int ObBackupTabletProvider::fetch_tablet_sstable_array_(const common::ObTabletID
return ret;
}
int ObBackupTabletProvider::prepare_tablet_sstable_index_builders_(const common::ObTabletID &tablet_id,
const bool is_major_compaction_mview_dep_tablet, common::ObIArray<storage::ObSSTableWrapper> &sstable_array)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(index_builder_mgr_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("index builder mgr should not be null", K(ret));
} else {
ObArray<storage::ObITable::TableKey> table_key_array;
ARRAY_FOREACH_X(sstable_array, idx, cnt, OB_SUCC(ret)) {
storage::ObSSTableWrapper &sstable_wrapper = sstable_array.at(idx);
ObSSTable *sstable_ptr = NULL;
if (OB_ISNULL(sstable_ptr = sstable_wrapper.get_sstable())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table should not be null", K(ret));
} else if (OB_FAIL(table_key_array.push_back(sstable_ptr->get_key()))) {
LOG_WARN("failed to push back", K(ret), KPC(sstable_ptr));
}
}
if (FAILEDx(index_builder_mgr_->prepare_sstable_index_builders(tablet_id, table_key_array, is_major_compaction_mview_dep_tablet))) {
LOG_WARN("failed to open sstable index builder", K(ret), K(tablet_id), K(table_key_array), K(is_major_compaction_mview_dep_tablet));
}
}
return ret;
}
int ObBackupTabletProvider::open_tablet_sstable_index_builder_(
const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle,
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable, const bool is_empty_sstable)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(index_builder_mgr_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("index builder mgr should not be null", K(ret));
} else if (OB_FAIL(index_builder_mgr_->open_sstable_index_builder(
tablet_id, tablet_handle, table_key, sstable, is_empty_sstable))) {
LOG_WARN("failed to open sstable index builder", K(ret), K(tablet_id), K(tablet_handle), K(table_key), KPC(sstable));
}
return ret;
}
int ObBackupTabletProvider::prepare_tablet_logic_id_reader_(const common::ObTabletID &tablet_id,
const storage::ObTabletHandle &tablet_handle, const ObITable::TableKey &table_key,
const blocksstable::ObSSTable &sstable, ObITabletLogicMacroIdReader *&reader)
@ -2574,14 +2529,10 @@ int ObBackupTabletProvider::add_macro_block_id_item_list_(const common::ObTablet
ObBackupProviderItem item;
bool need_skip = false;
share::ObBackupDataType backup_data_type;
bool need_reuse_across_sstable = false;
if (OB_FAIL(get_backup_data_type_(table_key, backup_data_type))) {
LOG_WARN("failed to get backup data type", K(ret), K(table_key));
} else if (OB_FAIL(check_need_reuse_across_sstable_(tablet_id, table_key, logic_id, need_reuse_across_sstable))) {
LOG_WARN("failed to check need reuse local", K(ret), K(tablet_id), K(table_key), K(macro_id));
} else if (OB_FAIL(item.set(PROVIDER_ITEM_MACRO_ID, backup_data_type, macro_id, table_key, tablet_id))) {
} else if (FAILEDx(item.set(PROVIDER_ITEM_MACRO_ID, backup_data_type, macro_id, table_key, tablet_id))) {
LOG_WARN("failed to set item", K(ret), K(macro_id), K(table_key), K(tablet_id));
} else if (need_reuse_across_sstable && OB_FALSE_IT(item.set_need_reuse_across_sstable())) {
} else if (!item.is_valid()) {
ret = OB_INVALID_DATA;
LOG_WARN("backup item is not valid", K(ret), K(item));
@ -2626,6 +2577,22 @@ bool ObBackupTabletProvider::is_same_type_(const storage::ObITable::TableKey &lh
return bret;
}
int ObBackupTabletProvider::add_prepare_tablet_item_(const common::ObTabletID &tablet_id)
{
int ret = OB_SUCCESS;
ObBackupProviderItem item;
ObBackupDataType backup_data_type;
backup_data_type.set_user_data_backup();
if (OB_FAIL(item.set_with_fake(PROVIDER_ITEM_TABLET_SSTABLE_INDEX_BUILDER_PREPARE, tablet_id, backup_data_type))) {
LOG_WARN("failed to set item", K(ret), K(tablet_id), K(backup_data_type));
} else if (OB_FAIL(push_item_to_queue_(item))) {
LOG_WARN("failed to push item to queue", K(ret), K(item));
} else {
LOG_INFO("add tablet item", K(tablet_id), K(backup_data_type));
}
return ret;
}
int ObBackupTabletProvider::add_tablet_item_(const common::ObTabletID &tablet_id,
const bool has_ss_ddl, const storage::ObITable::TableKey &table_key)
{
@ -2834,33 +2801,6 @@ void ObBackupTabletProvider::free_queue_item_()
item_queue_.reset();
}
int ObBackupTabletProvider::check_need_reuse_across_sstable_(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key,
const blocksstable::ObLogicMacroBlockId &logic_id, bool &need_reuse_across_sstable)
{
int ret = OB_SUCCESS;
need_reuse_across_sstable = false;
ObBackupTabletIndexBlockBuilderMgr *mgr = index_builder_mgr_;
ObBackupTabletSSTableIndexBuilderMgr *sstable_mgr = NULL;
bool macro_index_exist = false;
if (!table_key.is_major_sstable()) {
need_reuse_across_sstable = false;
} else if (OB_FAIL(mgr->get_sstable_index_builder_mgr(tablet_id, sstable_mgr))) {
LOG_WARN("failed to get sstable index builder mgr", K(ret), K(tablet_id));
} else if (!sstable_mgr->is_major_compaction_mview_dep_tablet()) {
need_reuse_across_sstable = false;
} else if (OB_FAIL(sstable_mgr->check_place_holder_macro_index_exist(logic_id, macro_index_exist))) {
LOG_WARN("failed to check place holder macro index exist", K(ret), K(logic_id));
} else if (macro_index_exist) {
need_reuse_across_sstable = true;
LOG_INFO("macro index exist, reuse local", K(tablet_id), K(table_key), K(logic_id));
} else if (OB_FAIL(sstable_mgr->insert_place_holder_macro_index(logic_id))) {
LOG_WARN("failed to insert empty macro index", K(ret), K(logic_id));
} else {
need_reuse_across_sstable = false;
}
return ret;
}
/* ObBackupMacroBlockTaskMgr */
ObBackupMacroBlockTaskMgr::ObBackupMacroBlockTaskMgr()
@ -3172,7 +3112,7 @@ bool ObBackupMacroBlockTaskMgr::all_item_is_reused(
}
OB_SERIALIZE_MEMBER(ObBackupProviderItem, item_type_, backup_data_type_, logic_id_, macro_block_id_, table_key_, tablet_id_,
nested_offset_, nested_size_, timestamp_, need_copy_, macro_index_, absolute_row_offset_, need_reuse_across_sstable_);
nested_offset_, nested_size_, timestamp_, need_copy_, macro_index_, absolute_row_offset_);
} // namespace backup
} // namespace oceanbase

13
src/storage/backup/ob_backup_utils.h
View File

@ -224,6 +224,7 @@ enum ObBackupProviderItemType {
PROVIDER_ITEM_MACRO_ID = 0,
PROVIDER_ITEM_DDL_OTHER_BLOCK_ID = 1,
PROVIDER_ITEM_TABLET_AND_SSTABLE_META = 2,
PROVIDER_ITEM_TABLET_SSTABLE_INDEX_BUILDER_PREPARE = 3,
PROVIDER_ITEM_MAX,
};
@ -260,9 +261,7 @@ public:
void set_no_need_copy() { need_copy_ = false; }
bool get_need_copy() const { return need_copy_; }
void set_macro_index(const ObBackupMacroBlockIndex &macro_index) { macro_index_ = macro_index; }
void set_need_reuse_across_sstable() { need_reuse_across_sstable_ = true; }
bool get_need_reuse_across_sstable() const { return need_reuse_across_sstable_; }
const ObBackupMacroBlockIndex &get_macro_index() { return macro_index_; }
const ObBackupMacroBlockIndex &get_macro_index() const { return macro_index_; }
int64_t get_absolute_row_offset() const { return absolute_row_offset_; }
TO_STRING_KV(K_(item_type), K_(backup_data_type), K_(logic_id), K_(macro_block_id), K_(table_key), K_(tablet_id), K_(nested_offset), K_(nested_size), K_(timestamp));
private:
@ -284,7 +283,6 @@ private:
bool need_copy_;
ObBackupMacroBlockIndex macro_index_;
int64_t absolute_row_offset_;
bool need_reuse_across_sstable_;
};
class ObBackupProviderItemCompare {
@ -384,10 +382,6 @@ private:
int fetch_tablet_sstable_array_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle,
const ObTabletTableStore &table_store, const share::ObBackupDataType &backup_data_type,
const bool is_major_compaction_mview_dep_tablet, common::ObIArray<storage::ObSSTableWrapper> &sstable_array);
int prepare_tablet_sstable_index_builders_(const common::ObTabletID &tablet_id,
const bool is_major_compaction_mview_dep_tablet, common::ObIArray<storage::ObSSTableWrapper> &sstable_array);
int open_tablet_sstable_index_builder_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle,
const storage::ObITable::TableKey &table_key, blocksstable::ObSSTable *sstable, const bool is_empty_sstable);
int prepare_tablet_logic_id_reader_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle,
const storage::ObITable::TableKey &table_key, const blocksstable::ObSSTable &sstable,
ObITabletLogicMacroIdReader *&reader);
@ -400,6 +394,7 @@ private:
const common::ObIArray<ObBackupMacroBlockId> &list, int64_t &added_count);
int get_backup_data_type_(const storage::ObITable::TableKey &table_key, share::ObBackupDataType &backup_data_type);
bool is_same_type_(const storage::ObITable::TableKey &lhs, const storage::ObITable::TableKey &rhs);
int add_prepare_tablet_item_(const common::ObTabletID &tablet_id);
int add_tablet_item_(const common::ObTabletID &tablet_id, const bool has_ss_ddl, const storage::ObITable::TableKey &table_key);
int remove_duplicates_(common::ObIArray<ObBackupProviderItem> &array);
int check_tablet_status_(const storage::ObTabletHandle &tablet_handle, bool &is_normal, bool &is_split_dst);
@ -413,8 +408,6 @@ private:
int push_item_to_queue_(const ObBackupProviderItem &item);
int pop_item_from_queue_(ObBackupProviderItem &item);
void free_queue_item_();
int check_need_reuse_across_sstable_(const common::ObTabletID &tablet_id, const storage::ObITable::TableKey &table_key,
const blocksstable::ObLogicMacroBlockId &logic_id, bool &need_reuse_across_sstable);
private:
static const int64_t BATCH_SIZE = 2000;