From ad4cf1e9c113025c601a637458af4faafda72007 Mon Sep 17 00:00:00 2001 From: oceanoverflow Date: Sat, 28 Jan 2023 16:16:06 +0800 Subject: [PATCH] fix backup and restore do not support -4725 issue --- .../backup/ob_backup_data_set_task_mgr.cpp | 103 +++++++++++++++++- .../backup/ob_backup_data_set_task_mgr.h | 2 + src/share/backup/ob_backup_data_store.cpp | 63 +++++++++++ src/share/backup/ob_backup_data_store.h | 22 ++++ .../backup/ob_backup_data_table_operator.cpp | 38 +++++-- .../backup/ob_backup_data_table_operator.h | 5 +- src/share/backup/ob_backup_path.cpp | 13 +++ src/share/backup/ob_backup_path.h | 3 + src/share/backup/ob_backup_struct.cpp | 43 +++++++- src/share/backup/ob_backup_struct.h | 32 +++++- src/share/ob_debug_sync_point.h | 1 + src/storage/backup/ob_backup_data_struct.cpp | 5 +- src/storage/backup/ob_backup_data_struct.h | 4 +- src/storage/backup/ob_backup_operator.cpp | 6 +- src/storage/backup/ob_backup_task.cpp | 18 ++- src/storage/backup/ob_backup_task.h | 3 +- src/storage/backup/ob_backup_utils.cpp | 22 +++- src/storage/backup/ob_backup_utils.h | 3 +- .../high_availability/ob_ls_restore.cpp | 14 ++- .../ob_admin_dump_backup_data_executor.cpp | 26 +++++ .../ob_admin_dump_backup_data_executor.h | 1 + 21 files changed, 389 insertions(+), 38 deletions(-) diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp index c629075c80..f4b9c3363f 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp @@ -1122,13 +1122,14 @@ int ObBackupSetTaskMgr::get_change_turn_tablets_( { int ret = OB_SUCCESS; ObArray tablet_attrs; - if (OB_FAIL(ObBackupSkippedTabletOperator::get_skip_tablet(trans_, true/*lock*/, set_task_attr_.tenant_id_, set_task_attr_.task_id_, tablet_attrs))) { + ObBackupSkippedType skipped_type(ObBackupSkippedType::TRANSFER); + if (OB_FAIL(ObBackupSkippedTabletOperator::get_skip_tablet(trans_, true/*lock*/, set_task_attr_.tenant_id_, set_task_attr_.task_id_, skipped_type, tablet_attrs))) { LOG_WARN("[DATA_BACKUP]failed to get skip tablet", K(ret), "teannt_id", set_task_attr_.tenant_id_, "task_id", set_task_attr_.task_id_); } else if (tablet_attrs.empty()) { LOG_INFO("[DATA_BACKUP]no change turn tablets found", K(ret)); } else if (OB_FAIL(do_get_change_turn_tablets_(ls_tasks, tablet_attrs, tablet_to_ls, new_ls_ids))) { LOG_WARN("[DATA_BACKUP]failed to do get change turn tables", K(ret), K(tablet_attrs), K(set_task_attr_)); - } else if (OB_FAIL(ObBackupSkippedTabletOperator::move_skip_tablet_to_his(trans_, set_task_attr_.tenant_id_, set_task_attr_.task_id_))) { + } else if (OB_FAIL(ObBackupSkippedTabletOperator::move_skip_tablet_to_his(trans_, set_task_attr_.tenant_id_, set_task_attr_.task_id_, skipped_type))) { LOG_WARN("[DATA_BACKUP]failed to move skip tablet to history", K(ret), K(tablet_attrs)); } return ret; @@ -1224,6 +1225,78 @@ int ObBackupSetTaskMgr::do_get_change_turn_tablets_( return ret; } +int ObBackupSetTaskMgr::persist_deleted_tablets_info_(const common::ObIArray &skipped_tablets) +{ + int ret = OB_SUCCESS; + const int64_t MAX_BUCKET = 1024; + ObHashMap *> deleted_tablet_to_ls_map; + if (OB_FAIL(deleted_tablet_to_ls_map.create(MAX_BUCKET, ObModIds::BACKUP))) { + LOG_WARN("[DATA_BACKUP]failed to create map", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < skipped_tablets.count(); ++i) { + const ObBackupSkipTabletAttr &skip_tablet = skipped_tablets.at(i); + const share::ObLSID &ls_id = skip_tablet.ls_id_; + ObArray *tablet_ids_ptr = NULL; + if (OB_FAIL(deleted_tablet_to_ls_map.get_refactored(ls_id, tablet_ids_ptr))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + if (OB_ISNULL(tablet_ids_ptr = OB_NEW(ObArray, ObModIds::BACKUP))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("[DATA_BACKUP]failed to construct tablet id array", KR(ret)); + } else if (OB_FAIL(deleted_tablet_to_ls_map.set_refactored(ls_id, tablet_ids_ptr, 1/*overwrite*/))) { + LOG_WARN("[DATA_BACKUP]failed to set refactored", K(ret), K(ls_id), KP(tablet_ids_ptr)); + } + if (OB_FAIL(ret)) { + OB_DELETE(ObArray, ObModIds::BACKUP, tablet_ids_ptr); + } + } else { + LOG_WARN("[DATA_BACKUP]failed to get refactored", K(ret), K(skip_tablet)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(tablet_ids_ptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[DATA_BACKUP]tmp tablet id array is null", K(ret)); + } else if (OB_FAIL(tablet_ids_ptr->push_back(skip_tablet.tablet_id_))) { + LOG_WARN("[DATA_BACKUP]failed to push back", K(ret), K(skip_tablet)); + } + } + } + if (OB_SUCC(ret)) { + ObBackupDeletedTabletToLSDesc deleted_tablet_to_ls; + ObHashMap *>::hashtable::const_iterator map_iter = deleted_tablet_to_ls_map.begin(); + for (; OB_SUCC(ret) && map_iter != deleted_tablet_to_ls_map.end(); ++map_iter) { + share::ObBackupDataTabletToLSInfo ls_info; + ls_info.ls_id_ = map_iter->first; + if (OB_ISNULL(map_iter->second)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[DATA_BACKUP]map iter should not be null", K(ret), K(ls_info)); + } else if (map_iter->second->empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[DATA_BACKUP]failed to get tablet to ls", K(ret), K(ls_info)); + } else if (OB_FAIL(append(ls_info.tablet_id_list_, *map_iter->second))) { + LOG_WARN("[DATA_BACKUP]failed to append tablet to ls array", K(ret)); + } else if (OB_FAIL(deleted_tablet_to_ls.deleted_tablet_to_ls_.push_back(ls_info))) { + LOG_WARN("[DATA_BACKUP]failed to push backup ls info", K(ret)); + } + } + if (FAILEDx(store_.write_deleted_tablet_info(deleted_tablet_to_ls))) { + LOG_WARN("[DATA_BACKUP]failed to write deleted tablet info", K(ret)); + } else { + LOG_INFO("write deleted tablet info", K(deleted_tablet_to_ls)); + } + } + if (deleted_tablet_to_ls_map.created()) { + ObHashMap *>::hashtable::iterator del_map_iter = deleted_tablet_to_ls_map.begin(); + for (; del_map_iter != deleted_tablet_to_ls_map.end(); ++del_map_iter) { + if (OB_NOT_NULL(del_map_iter->second)) { + OB_DELETE(ObArray, ObModIds::BACKUP, del_map_iter->second); + } + } + } + return ret; +} + int ObBackupSetTaskMgr::construct_cur_ls_set_( const ObIArray &ls_tasks, ObHashSet &ls_id_set) @@ -1436,6 +1509,7 @@ int ObBackupSetTaskMgr::do_backup_completing_log_(ObArray &l int ObBackupSetTaskMgr::do_clean_up(ObMySQLTransaction &trans) { int ret = OB_SUCCESS; + ObBackupSkippedType skipped_type(ObBackupSkippedType::DELETED); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("[DATA_BACKUP]not init", K(ret)); @@ -1448,7 +1522,10 @@ int ObBackupSetTaskMgr::do_clean_up(ObMySQLTransaction &trans) LOG_WARN("[DATA_BACKUP]failed to move ls to history", K(ret), K(set_task_attr_)); } else if (OB_FAIL(ObBackupTaskOperator::move_task_to_his(trans, set_task_attr_.tenant_id_, set_task_attr_.job_id_))) { LOG_WARN("[DATA_BACKUP]failed to move task to history", K(ret), K(set_task_attr_)); - } + } else if (OB_FAIL(ObBackupSkippedTabletOperator::move_skip_tablet_to_his( + trans, set_task_attr_.tenant_id_, set_task_attr_.task_id_, skipped_type))) { + LOG_WARN("[DATA_BACKUP]failed to move skip tablet to history", K(ret), K(set_task_attr_)); + } return ret; } @@ -1526,7 +1603,9 @@ int ObBackupSetTaskMgr::write_extern_infos() LOG_WARN("[DATA_BACKUP]failed to write extern tenant diagnose info", K(ret), KPC(job_attr_)); } else if (OB_FAIL(write_backup_set_placeholder(false/*finish job*/))) { LOG_WARN("[DATA_BACKUP]failed to write backup set finish placeholder", K(ret), KPC(job_attr_)); - } + } else if (OB_FAIL(write_deleted_tablet_infos_())) { + LOG_WARN("[DATA_BACKUP]failed to write deleted tablet infos", K(ret), KPC(job_attr_)); + } } return ret; } @@ -1581,6 +1660,22 @@ int ObBackupSetTaskMgr::write_backup_set_info_( return ret; } +int ObBackupSetTaskMgr::write_deleted_tablet_infos_() +{ + int ret = OB_SUCCESS; + ObArray tablet_attrs; + ObBackupSkippedType skipped_type(ObBackupSkippedType::DELETED); + if (OB_FAIL(ObBackupSkippedTabletOperator::get_skip_tablet(*sql_proxy_, false/*lock*/, + set_task_attr_.tenant_id_, set_task_attr_.task_id_, skipped_type, tablet_attrs))) { + LOG_WARN("[DATA_BACKUP]failed to get skip tablet", K(ret), "teannt_id", set_task_attr_.tenant_id_, "task_id", set_task_attr_.task_id_); + } else if (tablet_attrs.empty()) { + LOG_INFO("[DATA_BACKUP]no change turn tablets found", K(ret)); + } else if (OB_FAIL(persist_deleted_tablets_info_(tablet_attrs))) { + LOG_WARN("[DATA_BACKUP]failed to do persist deleted tablets info", K(ret), K(tablet_attrs), K(set_task_attr_)); + } + return ret; +} + int ObBackupSetTaskMgr::calculate_start_replay_scn_(SCN &start_replay_scn) { int ret = OB_SUCCESS; diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.h b/src/rootserver/backup/ob_backup_data_set_task_mgr.h index af2916ba57..33f1b62708 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.h +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.h @@ -87,6 +87,7 @@ private: int get_change_turn_ls_(ObIArray &ls_task, const ObIArray &tablets_to_ls, ObIArray &need_change_turn_ls_tasks); + int persist_deleted_tablets_info_(const common::ObIArray &skip_tablets); int update_inner_task_(const ObIArray &new_ls_ids, const ObIArray &need_change_turn_ls_tasks); int update_task_type_(const ObIArray &ls_task); @@ -107,6 +108,7 @@ private: int write_extern_ls_info_(const ObArray &ls_tasks); int write_tablet_to_ls_infos_(const ObIArray &tablets_to_ls, const int64_t turn_id); + int write_deleted_tablet_infos_(); int set_backup_set_files_failed_(ObMySQLTransaction &trans); int advance_status_(ObMySQLTransaction &trans, const share::ObBackupStatus &next_status, const int result = OB_SUCCESS, diff --git a/src/share/backup/ob_backup_data_store.cpp b/src/share/backup/ob_backup_data_store.cpp index aeb95f1dd3..17c2e84892 100644 --- a/src/share/backup/ob_backup_data_store.cpp +++ b/src/share/backup/ob_backup_data_store.cpp @@ -64,6 +64,16 @@ bool ObBackupDataTabletToLSDesc::is_valid() const return backup_scn_.is_valid() && !tablet_to_ls_.empty(); } +/* + *------------------------------ObBackupDeletedTabletToLSDesc---------------------------- + */ +OB_SERIALIZE_MEMBER(ObBackupDeletedTabletToLSDesc, deleted_tablet_to_ls_); + +bool ObBackupDeletedTabletToLSDesc::is_valid() const +{ + return true; +} + /* *------------------------------ObExternTenantLocalityInfo---------------------------- */ @@ -501,6 +511,59 @@ int ObBackupDataStore::read_tablet_to_ls_info(const ObLSID &ls_id, const int64_t return ret; } +int ObBackupDataStore::write_deleted_tablet_info(const ObBackupDeletedTabletToLSDesc &deleted_tablet_info) +{ + int ret = OB_SUCCESS; + ObBackupPathString full_path; + share::ObBackupPath path; + if (!is_init()) { + ret = OB_NOT_INIT; + LOG_WARN("backup data extern mgr not init", K(ret)); + } else if (OB_FAIL(ObBackupPathUtil::get_deleted_tablet_info_path(backup_set_dest_, path))) { + LOG_WARN("fail to get path", K(ret)); + } else if (OB_FAIL(full_path.assign(path.get_obstr()))) { + LOG_WARN("fail to assign full path", K(ret)); + } else if (OB_FAIL(write_single_file(full_path, deleted_tablet_info))) { + LOG_WARN("fail to write single file", K(ret)); + } + return ret; +} + +int ObBackupDataStore::read_deleted_tablet_info(const ObLSID &ls_id, ObIArray &deleted_tablet_ids) +{ + int ret = OB_SUCCESS; + deleted_tablet_ids.reset(); + share::ObBackupPath path; + ObBackupPathString full_path; + ObBackupDeletedTabletToLSDesc deleted_tablet_info; + if (!is_init()) { + ret = OB_NOT_INIT; + LOG_WARN("ObBackupDataStore not init", K(ret)); + } else if (OB_FAIL(ObBackupPathUtil::get_deleted_tablet_info_path(backup_set_dest_, path))) { + LOG_WARN("fail to get tenant ls attr info path", K(ret)); + } else if (OB_FAIL(full_path.assign(path.get_obstr()))) { + LOG_WARN("fail to assign full path", K(ret)); + } else if (OB_FAIL(read_single_file(full_path, deleted_tablet_info))) { + if (OB_BACKUP_FILE_NOT_EXIST == ret) { + ret = OB_SUCCESS; + LOG_INFO("backup deleted file not exist", K(ret)); + } else { + LOG_WARN("failed to read single file", K(ret), K(full_path)); + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < deleted_tablet_info.deleted_tablet_to_ls_.count(); ++i) { + const ObBackupDataTabletToLSInfo &info = deleted_tablet_info.deleted_tablet_to_ls_.at(i); + if (info.ls_id_ == ls_id) { + if (OB_FAIL(deleted_tablet_ids.assign(info.tablet_id_list_))) { + LOG_WARN("failed to assign", K(ret), K(info)); + } + break; + } + } + } + return ret; +} + int ObBackupDataStore::write_tenant_backup_set_infos(const ObTenantBackupSetInfosDesc &tenant_backup_set_infos) { int ret = OB_SUCCESS; diff --git a/src/share/backup/ob_backup_data_store.h b/src/share/backup/ob_backup_data_store.h index 8c88774763..bbc42adc6d 100644 --- a/src/share/backup/ob_backup_data_store.h +++ b/src/share/backup/ob_backup_data_store.h @@ -103,6 +103,25 @@ private: DISALLOW_COPY_AND_ASSIGN(ObBackupDataTabletToLSDesc); }; +struct ObBackupDeletedTabletToLSDesc final : public ObExternBackupDataDesc +{ +public: + static const uint8_t FILE_VERSION = 1; + OB_UNIS_VERSION(FILE_VERSION); +public: + ObBackupDeletedTabletToLSDesc() + : ObExternBackupDataDesc(ObBackupFileType::BACKUP_DELETED_TABLET_INFO, FILE_VERSION), + deleted_tablet_to_ls_() {} + virtual ~ObBackupDeletedTabletToLSDesc() {} + + bool is_valid() const override; + INHERIT_TO_STRING_KV("ObExternBackupDataDesc", ObExternBackupDataDesc, K_(deleted_tablet_to_ls)); +public: + ObSArray deleted_tablet_to_ls_; +private: + DISALLOW_COPY_AND_ASSIGN(ObBackupDeletedTabletToLSDesc); +}; + struct ObExternBackupSetPlaceholderDesc final : public ObExternBackupDataDesc { public: @@ -273,6 +292,9 @@ public: int read_tablet_to_ls_info(const int64_t turn_id, ObBackupDataTabletToLSDesc &tablet_to_ls_info); int read_tablet_to_ls_info(const ObLSID &ls_id, const int64_t turn_id, const int64_t retry_id, ObBackupDataTabletToLSDesc &tablet_to_ls_info); + // write and read deleted_tablet_info + int write_deleted_tablet_info(const ObBackupDeletedTabletToLSDesc &deleted_tablet_info); + int read_deleted_tablet_info(const ObLSID &ls_id, ObIArray &deleted_tablet_ids); // write tenant backup set infos int write_tenant_backup_set_infos(const ObTenantBackupSetInfosDesc &tenant_backup_set_infos); diff --git a/src/share/backup/ob_backup_data_table_operator.cpp b/src/share/backup/ob_backup_data_table_operator.cpp index 28cf61714a..a1a790a295 100644 --- a/src/share/backup/ob_backup_data_table_operator.cpp +++ b/src/share/backup/ob_backup_data_table_operator.cpp @@ -2195,6 +2195,7 @@ int ObBackupSkippedTabletOperator::get_skip_tablet( const bool need_lock, const uint64_t tenant_id, const int64_t task_id, + const share::ObBackupSkippedType &skipped_type, ObIArray &tablet_attrs) { int ret = OB_SUCCESS; @@ -2207,7 +2208,10 @@ int ObBackupSkippedTabletOperator::get_skip_tablet( ObMySQLResult *result = NULL; if (OB_FAIL(fill_select_skip_tablet_sql_(sql))) { LOG_WARN("[DATA_BACKUP]failed to fill select ls task sql", K(ret)); - } else if (OB_FAIL(sql.append_fmt(" where %s=%lu and %s=%ld", OB_STR_TENANT_ID, tenant_id, OB_STR_TASK_ID, task_id))) { + } else if (OB_FAIL(sql.append_fmt(" where %s=%lu and %s=%ld and %s='%s'", + OB_STR_TENANT_ID, tenant_id, OB_STR_TASK_ID, task_id, + OB_STR_BACKUP_SKIPPED_TYPE, skipped_type.str()))) { + LOG_WARN("[DATA_BACKUP]failed to append sql", K(ret), K(tenant_id), K(task_id), K(skipped_type)); } else if (need_lock && OB_FAIL(sql.append_fmt(" for update"))) { LOG_WARN("[DATA_BACKUP]failed to append sql", K(ret)); } else if (OB_FAIL(proxy.read(res, get_exec_tenant_id(tenant_id), sql.ptr()))) { @@ -2226,9 +2230,9 @@ int ObBackupSkippedTabletOperator::get_skip_tablet( int ObBackupSkippedTabletOperator::fill_select_skip_tablet_sql_(ObSqlString &sql) { int ret = OB_SUCCESS; - if (OB_FAIL(sql.assign_fmt("select %s", OB_STR_TASK_ID))) { + if (OB_FAIL(sql.assign_fmt("select %s", OB_STR_TENANT_ID))) { LOG_WARN("[DATA_BACKUP]failed to assign fmt", K(ret)); - } else if (OB_FAIL(sql.append_fmt(", %s", OB_STR_TENANT_ID))) { + } else if (OB_FAIL(sql.append_fmt(", %s", OB_STR_TASK_ID))) { LOG_WARN("[DATA_BACKUP]failed to append fmt", K(ret)); } else if (OB_FAIL(sql.append_fmt(", %s", OB_STR_TURN_ID))) { LOG_WARN("[DATA_BACKUP]failed to append fmt", K(ret)); @@ -2240,6 +2244,8 @@ int ObBackupSkippedTabletOperator::fill_select_skip_tablet_sql_(ObSqlString &sql LOG_WARN("[DATA_BACKUP]failed to append fmt", K(ret)); } else if (OB_FAIL(sql.append_fmt(", %s", OB_STR_BACKUP_SET_ID))) { LOG_WARN("[DATA_BACKUP]failed to append fmt", K(ret)); + } else if (OB_FAIL(sql.append_fmt(", %s", OB_STR_BACKUP_SKIPPED_TYPE))) { + LOG_WARN("[DATA_BACKUP]failed to append fmt", K(ret)); } else if (OB_FAIL(sql.append_fmt(" from %s", OB_ALL_BACKUP_SKIPPED_TABLET_TNAME))) { LOG_WARN("[DATA_BACKUP]failed to append fmt", K(ret)); } @@ -2275,24 +2281,32 @@ int ObBackupSkippedTabletOperator::do_parse_skip_tablet_result_( ObBackupSkipTabletAttr &tablet_attr) { int ret = OB_SUCCESS; + int64_t real_length = 0; int64_t tablet_id = -1; int64_t ls_id = -1; - EXTRACT_INT_FIELD_MYSQL(result, OB_STR_TASK_ID, tablet_attr.task_id_, int64_t); + char skipped_type_str[OB_INNER_TABLE_DEFAULT_VALUE_LENTH] = ""; EXTRACT_INT_FIELD_MYSQL(result, OB_STR_TENANT_ID, tablet_attr.tenant_id_, uint64_t); + EXTRACT_INT_FIELD_MYSQL(result, OB_STR_TASK_ID, tablet_attr.task_id_, int64_t); EXTRACT_INT_FIELD_MYSQL(result, OB_STR_TURN_ID, tablet_attr.turn_id_, int64_t); EXTRACT_INT_FIELD_MYSQL(result, OB_STR_RETRY_ID, tablet_attr.retry_id_, int64_t); - EXTRACT_INT_FIELD_MYSQL(result, OB_STR_BACKUP_SET_ID, tablet_attr.backup_set_id_, int64_t); EXTRACT_INT_FIELD_MYSQL(result, OB_STR_TABLET_ID, tablet_id, int64_t); EXTRACT_INT_FIELD_MYSQL(result, OB_STR_LS_ID, ls_id, int64_t); + EXTRACT_INT_FIELD_MYSQL(result, OB_STR_BACKUP_SET_ID, tablet_attr.backup_set_id_, int64_t); + EXTRACT_STRBUF_FIELD_MYSQL(result, OB_STR_BACKUP_SKIPPED_TYPE, skipped_type_str, OB_INNER_TABLE_DEFAULT_VALUE_LENTH, real_length); tablet_attr.tablet_id_ = tablet_id; tablet_attr.ls_id_ = ls_id; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(tablet_attr.skipped_type_.parse_from_str(skipped_type_str))) { + LOG_WARN("failed to parse from str", K(ret), K(skipped_type_str)); + } return ret; } int ObBackupSkippedTabletOperator::move_skip_tablet_to_his( common::ObISQLClient &proxy, const uint64_t tenant_id, - const int64_t task_id) + const int64_t task_id, + const share::ObBackupSkippedType &skipped_type) { int ret = OB_SUCCESS; ObSqlString sql; @@ -2301,18 +2315,20 @@ int ObBackupSkippedTabletOperator::move_skip_tablet_to_his( ret = OB_INVALID_ARGUMENT; LOG_WARN("[DATA_BACKUP]invalid argument", K(ret), K(tenant_id), K(task_id)); } else if (OB_FAIL(sql.assign_fmt( - "insert into %s select * from %s where %s=%lu and %s=%lu", + "insert into %s select * from %s where %s=%lu and %s=%lu and %s='%s'", OB_ALL_BACKUP_SKIPPED_TABLET_HISTORY_TNAME, OB_ALL_BACKUP_SKIPPED_TABLET_TNAME, - OB_STR_TENANT_ID, tenant_id, OB_STR_TASK_ID, task_id))) { + OB_STR_TENANT_ID, tenant_id, OB_STR_TASK_ID, task_id, + OB_STR_BACKUP_SKIPPED_TYPE, skipped_type.str()))) { LOG_WARN("[DATA_BACKUP]failed to init sql", K(ret)); } else if (OB_FAIL(proxy.write(get_exec_tenant_id(tenant_id), sql.ptr(), affected_rows))) { LOG_WARN("[DATA_BACKUP]failed to exec sql", K(ret), K(sql)); } else if (OB_FALSE_IT(sql.reset())) { } else if (OB_FAIL(sql.assign_fmt( - "delete from %s where %s=%lu and %s=%lu", - OB_ALL_BACKUP_SKIPPED_TABLET_TNAME, OB_STR_TENANT_ID, tenant_id, OB_STR_TASK_ID, task_id))) { + "delete from %s where %s=%lu and %s=%lu and %s='%s'", + OB_ALL_BACKUP_SKIPPED_TABLET_TNAME, OB_STR_TENANT_ID, tenant_id, OB_STR_TASK_ID, task_id, + OB_STR_BACKUP_SKIPPED_TYPE, skipped_type.str()))) { LOG_WARN("[DATA_BACKUP]failed to init sql", K(ret)); - } else if (OB_FAIL(proxy.write(sql.ptr(), affected_rows))) { + } else if (OB_FAIL(proxy.write(get_exec_tenant_id(tenant_id), sql.ptr(), affected_rows))) { LOG_WARN("[DATA_BACKUP]failed to exec sql", K(ret), K(sql)); } return ret; diff --git a/src/share/backup/ob_backup_data_table_operator.h b/src/share/backup/ob_backup_data_table_operator.h index cfadc2e082..0bd852bbe6 100644 --- a/src/share/backup/ob_backup_data_table_operator.h +++ b/src/share/backup/ob_backup_data_table_operator.h @@ -47,8 +47,9 @@ class ObBackupSkippedTabletOperator : public ObBackupBaseTableOperator { public: static int get_skip_tablet(common::ObISQLClient &proxy, const bool need_lock, const uint64_t tenant_id, - const int64_t task_id, ObIArray &tablet_attrs); - static int move_skip_tablet_to_his(common::ObISQLClient &proxy, const uint64_t tenant_id, const int64_t task_id); + const int64_t task_id, const share::ObBackupSkippedType &skipped_type, ObIArray &tablet_attrs); + static int move_skip_tablet_to_his(common::ObISQLClient &proxy, const uint64_t tenant_id, const int64_t task_id, + const share::ObBackupSkippedType &skipped_type); private: static int fill_select_skip_tablet_sql_(ObSqlString &sql); static int parse_skip_tablet_result_(sqlclient::ObMySQLResult &result, ObIArray &tablet_attrs); diff --git a/src/share/backup/ob_backup_path.cpp b/src/share/backup/ob_backup_path.cpp index 58a7d1cb40..614eeeea0f 100644 --- a/src/share/backup/ob_backup_path.cpp +++ b/src/share/backup/ob_backup_path.cpp @@ -1047,6 +1047,19 @@ int ObBackupPathUtil::get_backup_data_tablet_ls_info_path(const share::ObBackupD return ret; } +// file:///obbackup/backup_set_1_full/infos/deleted_tablet_info + +int ObBackupPathUtil::get_deleted_tablet_info_path(const share::ObBackupDest &backup_set_dest, share::ObBackupPath &path) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(get_ls_info_dir_path(backup_set_dest, path))) { + LOG_WARN("failed to get ls info dir path", K(ret), K(backup_set_dest)); + } else if (OB_FAIL(path.join(OB_STR_DELETED_TABLET_INFO))) { + LOG_WARN("failed to join tablet_log_stream_info", K(ret)); + } + return ret; +} + // file:///obbackup/backup_set_1_full_20211231/log_stream_1/complement_log/ int ObBackupPathUtil::get_ls_complement_log_dir_path(const share::ObBackupDest &backup_set_dest, const share::ObLSID &ls_id, ObBackupPath &backup_path) diff --git a/src/share/backup/ob_backup_path.h b/src/share/backup/ob_backup_path.h index c02283d522..3045f54e0c 100644 --- a/src/share/backup/ob_backup_path.h +++ b/src/share/backup/ob_backup_path.h @@ -231,6 +231,9 @@ struct ObBackupPathUtil static int get_backup_data_tablet_ls_info_path(const share::ObBackupDest &backup_set_dest, const uint64_t turn_id, share::ObBackupPath &path); + // file:///obbackup/backup_set_1_full/infos/deleted_tablet_info + static int get_deleted_tablet_info_path(const share::ObBackupDest &backup_set_dest, share::ObBackupPath &path); + // file:///obbackup/backup_set_1_full/log_stream_1/complement_log/ static int get_ls_complement_log_dir_path(const share::ObBackupDest &backup_set_dest, const share::ObLSID &ls_id, share::ObBackupPath &backup_path); diff --git a/src/share/backup/ob_backup_struct.cpp b/src/share/backup/ob_backup_struct.cpp index f1059c018b..a6ed9911b0 100644 --- a/src/share/backup/ob_backup_struct.cpp +++ b/src/share/backup/ob_backup_struct.cpp @@ -2087,7 +2087,6 @@ bool ObBackupUtils::is_need_retry_error(const int err) case OB_BACKUP_FILE_NOT_EXIST : case OB_LOG_ARCHIVE_INTERRUPTED : case OB_LOG_ARCHIVE_NOT_RUNNING : - case OB_TABLET_NOT_EXIST : case OB_CHECKSUM_ERROR : bret = false; break; @@ -3669,7 +3668,8 @@ ObBackupSkipTabletAttr::ObBackupSkipTabletAttr() retry_id_(0), backup_set_id_(0), tablet_id_(0), - ls_id_() + ls_id_(), + skipped_type_() { } @@ -3681,7 +3681,8 @@ bool ObBackupSkipTabletAttr::is_valid() const && retry_id_ > 0 && tablet_id_.is_valid() && backup_set_id_ > 0 - && ls_id_.is_valid(); + && ls_id_.is_valid() + && skipped_type_.is_valid(); } ObBackupLSTaskInfoAttr::ObBackupLSTaskInfoAttr() @@ -4107,3 +4108,39 @@ int ObRestoreBackupSetBriefInfo::assign(const ObRestoreBackupSetBriefInfo &that) } return ret; } + +/* ObBackupSkippedType */ + +const char *ObBackupSkippedType::str() const +{ + const char *str = "INVALID_TYPE"; + switch (type_) { + case DELETED: { + str = "DELETED"; + break; + } + case TRANSFER: { + str = "TRANSFER"; + break; + } + default: { + str = "INVALID_TYPE"; + } + } + return str; +} + +int ObBackupSkippedType::parse_from_str(const ObString &str) +{ + int ret = OB_SUCCESS; + if (0 == str.case_compare("DELETED")) { + type_ = DELETED; + } else if (0 == str.case_compare("TRANSFER")) { + type_ = TRANSFER; + } else { + type_ = MAX_TYPE; + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid backup skipped type str", KR(ret), K(str)); + } + return ret; +} \ No newline at end of file diff --git a/src/share/backup/ob_backup_struct.h b/src/share/backup/ob_backup_struct.h index b35ba8236a..7ffec2fba2 100644 --- a/src/share/backup/ob_backup_struct.h +++ b/src/share/backup/ob_backup_struct.h @@ -258,6 +258,7 @@ const char *const OB_STR_LOG_ARCHIVE_SOURCE_ID = "id"; const char *const OB_STR_LOG_ARCHIVE_SOURCE_TYPE = "type"; const char *const OB_STR_LOG_ARCHIVE_SOURCE_VALUE = "value"; const char *const OB_STR_LOG_ARCHIVE_SOURCE_UNTIL_SCN = "recovery_until_scn"; +const char *const OB_STR_BACKUP_SKIPPED_TYPE = "skipped_type"; const char *const OB_STR_TENANT = "tenant"; const char *const OB_STR_DATA = "data"; @@ -280,6 +281,7 @@ const char *const OB_STR_DATA_INTO_TURN = "data_info_turn"; const char *const OB_STR_META_INFO_TURN = "meta_info_turn"; const char *const OB_STR_LS_META_INFO = "ls_meta_info"; const char *const OB_STR_TABLET_LOG_STREAM_INFO = "tablet_log_stream_info"; +const char *const OB_STR_DELETED_TABLET_INFO = "deleted_tablet_info"; const char *const OB_STR_TENANT_MINOR_MACRO_INDEX = "tenant_minor_data_macro_range_index"; const char *const OB_STR_TENANT_MINOR_META_INDEX = "tenant_minor_data_meta_index"; const char *const OB_STR_TENANT_MINOR_SEC_META_INDEX = "tenant_minor_data_sec_meta_index"; @@ -420,6 +422,7 @@ enum ObBackupFileType BACKUP_CHECK_FILE = 33, BACKUP_LS_META_INFOS_FILE = 34, BACKUP_TENANT_ARCHIVE_PIECE_INFOS = 35, + BACKUP_DELETED_TABLET_INFO = 36, // type <=255 is write header struct to disk directly // type > 255 is use serialization to disk BACKUP_MAX_DIRECT_WRITE_TYPE = 255, @@ -429,6 +432,30 @@ enum ObBackupFileType BACKUP_TYPE_MAX }; +struct ObBackupSkippedType final +{ +public: + enum TYPE : uint8_t + { + DELETED = 0, + TRANSFER = 1, + MAX_TYPE + }; +public: + ObBackupSkippedType() : type_(MAX_TYPE) {} + ~ObBackupSkippedType() = default; + explicit ObBackupSkippedType(const TYPE &type) : type_(type) {} + + bool is_valid() const { return DELETED <= type_ && type_ < MAX_TYPE; } + void reset() { type_ = MAX_TYPE; } + const char *str() const; + int parse_from_str(const ObString &str); + + TO_STRING_KV(K_(type), "type", str()); +private: + TYPE type_; +}; + enum ObBackupMetaType { PARTITION_GROUP_META = 0, @@ -1441,6 +1468,8 @@ public: int64_t meta_turn_id_; }; +struct ObBackupSkippedType; + struct ObBackupSkipTabletAttr final { public: @@ -1448,7 +1477,7 @@ public: ~ObBackupSkipTabletAttr() = default; bool is_valid() const; TO_STRING_KV(K_(task_id), K_(tenant_id), K_(turn_id), K_(retry_id), K_(backup_set_id), K_(ls_id), - K_(tablet_id)); + K_(tablet_id), K_(skipped_type)); public: int64_t task_id_; uint64_t tenant_id_; @@ -1457,6 +1486,7 @@ public: int64_t backup_set_id_; ObTabletID tablet_id_; ObLSID ls_id_; + share::ObBackupSkippedType skipped_type_; }; struct ObBackupLSTaskInfoAttr final diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index 82f3d279ad..fb1019e80e 100644 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -380,6 +380,7 @@ class ObString; ACT(BEFORE_REPLAY_DDL_PREPRARE,)\ ACT(BEFORE_REPLAY_DDL_COMMIT,)\ ACT(BEFORE_BACKUP_UESR_META,)\ + ACT(BEFORE_BACKUP_SYS_TABLETS,)\ ACT(BEFORE_WRITE_DDL_PREPARE_LOG,)\ ACT(AFTER_REMOTE_WRITE_DDL_PREPARE_LOG,)\ ACT(BEFORE_CHECK_ALL_LS_HAS_LEADER,)\ diff --git a/src/storage/backup/ob_backup_data_struct.cpp b/src/storage/backup/ob_backup_data_struct.cpp index f46a81599a..5712c6b8c1 100644 --- a/src/storage/backup/ob_backup_data_struct.cpp +++ b/src/storage/backup/ob_backup_data_struct.cpp @@ -18,6 +18,7 @@ #include "lib/oblog/ob_log_module.h" #include "storage/backup/ob_backup_task.h" #include "storage/blocksstable/ob_logic_macro_id.h" +#include "share/backup/ob_backup_struct.h" using namespace oceanbase::common; using namespace oceanbase::share; @@ -1204,7 +1205,7 @@ void ObBackupLSTaskInfo::reset() /* ObBackupSkippedTablet */ ObBackupSkippedTablet::ObBackupSkippedTablet() - : task_id_(), tenant_id_(), turn_id_(), retry_id_(), tablet_id_(), ls_id_(), backup_set_id_() + : task_id_(), tenant_id_(), turn_id_(), retry_id_(), tablet_id_(), ls_id_(), backup_set_id_(), skipped_type_() {} ObBackupSkippedTablet::~ObBackupSkippedTablet() @@ -1213,7 +1214,7 @@ ObBackupSkippedTablet::~ObBackupSkippedTablet() bool ObBackupSkippedTablet::is_valid() const { return task_id_ > 0 && OB_INVALID_ID != tenant_id_ && turn_id_ > 0 && retry_id_ >= 0 && tablet_id_.is_valid() && - ls_id_.is_valid() && backup_set_id_ > 0; + ls_id_.is_valid() && backup_set_id_ > 0 && skipped_type_.is_valid(); } /* ObBackupReportCtx */ diff --git a/src/storage/backup/ob_backup_data_struct.h b/src/storage/backup/ob_backup_data_struct.h index 38d9f8776d..e63d7dc110 100644 --- a/src/storage/backup/ob_backup_data_struct.h +++ b/src/storage/backup/ob_backup_data_struct.h @@ -665,7 +665,8 @@ struct ObBackupSkippedTablet { ObBackupSkippedTablet(); ~ObBackupSkippedTablet(); bool is_valid() const; - TO_STRING_KV(K_(task_id), K_(tenant_id), K_(turn_id), K_(retry_id), K_(tablet_id), K_(ls_id), K_(backup_set_id)); + TO_STRING_KV(K_(task_id), K_(tenant_id), K_(turn_id), K_(retry_id), K_(tablet_id), + K_(ls_id), K_(backup_set_id), K_(skipped_type)); int64_t task_id_; uint64_t tenant_id_; int64_t turn_id_; @@ -673,6 +674,7 @@ struct ObBackupSkippedTablet { common::ObTabletID tablet_id_; share::ObLSID ls_id_; int64_t backup_set_id_; + share::ObBackupSkippedType skipped_type_; }; struct ObBackupReportCtx final { diff --git a/src/storage/backup/ob_backup_operator.cpp b/src/storage/backup/ob_backup_operator.cpp index dde8f73a97..d7b2a5356d 100644 --- a/src/storage/backup/ob_backup_operator.cpp +++ b/src/storage/backup/ob_backup_operator.cpp @@ -291,8 +291,8 @@ int ObLSBackupOperator::report_tablet_skipped( LOG_WARN("get invalid args", K(ret), K(tenant_id), K(skipped_tablet)); } else if (OB_FAIL(fill_backup_skipped_tablet_(skipped_tablet, dml_splicer))) { LOG_WARN("failed to fill backup skipped tablet", K(ret), K(skipped_tablet)); - } else if (OB_FAIL(dml_splicer.splice_update_sql(OB_ALL_BACKUP_SKIPPED_TABLET_TNAME, sql))) { - LOG_WARN("failed to splice update sql", K(ret), K(sql)); + } else if (OB_FAIL(dml_splicer.splice_insert_update_sql(OB_ALL_BACKUP_SKIPPED_TABLET_TNAME, sql))) { + LOG_WARN("failed to splice insert update sql", K(ret), K(tenant_id), K(skipped_tablet), K(sql)); } else if (OB_FAIL(sql_client.write(gen_meta_tenant_id(tenant_id), sql.ptr(), affected_rows))) { LOG_WARN("failed to execute sql", K(ret), K(sql)); } else { @@ -405,6 +405,8 @@ int ObLSBackupOperator::fill_backup_skipped_tablet_(const ObBackupSkippedTablet LOG_WARN("failed to add column", K(task_info)); } else if (OB_FAIL(dml.add_column("backup_set_id", task_info.backup_set_id_))) { LOG_WARN("failed to add column", K(task_info)); + } else if (OB_FAIL(dml.add_column("skipped_type", task_info.skipped_type_.str()))) { + LOG_WARN("failed to add column", K(task_info)); } return ret; } diff --git a/src/storage/backup/ob_backup_task.cpp b/src/storage/backup/ob_backup_task.cpp index a65e9a4082..5dffd36201 100644 --- a/src/storage/backup/ob_backup_task.cpp +++ b/src/storage/backup/ob_backup_task.cpp @@ -39,6 +39,7 @@ #include "share/ls/ob_ls_table_operator.h" #include "storage/ob_storage_rpc.h" #include "storage/blocksstable/ob_logic_macro_id.h" +#include "share/backup/ob_backup_struct.h" #include using namespace oceanbase::blocksstable; @@ -2478,6 +2479,13 @@ int ObLSBackupDataTask::process() } } } + if (OB_SUCC(ret)) { + if (param_.ls_id_.is_sys_ls()) { + SERVER_EVENT_SYNC_ADD("backup_errsim", "before_backup_sys_tablets", + "ls_id", param_.ls_id_.id()); + DEBUG_SYNC(BEFORE_BACKUP_SYS_TABLETS); + } + } #endif if (OB_FAIL(ret)) { } else if (IS_NOT_INIT) { @@ -2580,7 +2588,8 @@ int ObLSBackupDataTask::may_inject_simulated_error_() return ret; } -int ObLSBackupDataTask::report_tablet_skipped_(const common::ObTabletID &tablet_id) +int ObLSBackupDataTask::report_tablet_skipped_(const common::ObTabletID &tablet_id, + const share::ObBackupSkippedType &skipped_type) { int ret = OB_SUCCESS; ObBackupSkippedTablet skipped_tablet; @@ -2591,14 +2600,15 @@ int ObLSBackupDataTask::report_tablet_skipped_(const common::ObTabletID &tablet_ skipped_tablet.tablet_id_ = tablet_id; skipped_tablet.backup_set_id_ = param_.backup_set_desc_.backup_set_id_; skipped_tablet.ls_id_ = param_.ls_id_; - if (!tablet_id.is_valid()) { + skipped_tablet.skipped_type_ = skipped_type; + if (!skipped_tablet.is_valid()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid args", K(ret), K(tablet_id)); + LOG_WARN("get invalid args", K(ret), K(skipped_tablet)); } else if (OB_FAIL(ObLSBackupOperator::report_tablet_skipped( param_.tenant_id_, skipped_tablet, *report_ctx_.sql_proxy_))) { LOG_WARN("failed to report tablet skipped", K(ret), K_(param), K(tablet_id)); } else { - FLOG_INFO("report tablet skipping", K(tablet_id)); + FLOG_INFO("report tablet skipping", K(tablet_id), K(skipped_tablet)); } return ret; } diff --git a/src/storage/backup/ob_backup_task.h b/src/storage/backup/ob_backup_task.h index 8376fa895c..607df16b12 100644 --- a/src/storage/backup/ob_backup_task.h +++ b/src/storage/backup/ob_backup_task.h @@ -526,7 +526,8 @@ public: private: int may_inject_simulated_error_(); - int report_tablet_skipped_(const common::ObTabletID &tablet_id); + int report_tablet_skipped_(const common::ObTabletID &tablet_id, + const share::ObBackupSkippedType &skipped_type); private: int build_backup_file_header_(ObBackupFileHeader &file_header); diff --git a/src/storage/backup/ob_backup_utils.cpp b/src/storage/backup/ob_backup_utils.cpp index af7da631f8..2feb43e789 100644 --- a/src/storage/backup/ob_backup_utils.cpp +++ b/src/storage/backup/ob_backup_utils.cpp @@ -1580,9 +1580,14 @@ int ObBackupTabletProvider::prepare_tablet_(const uint64_t tenant_id, const shar bool is_deleted = false; if (OB_FAIL(check_tablet_deleted_(param_.tenant_id_, tablet_id, is_deleted))) { LOG_WARN("failed to check tablet deleted", K(ret), K(tablet_id)); - } else if (is_deleted) { - ret = OB_TABLET_NOT_EXIST; - LOG_ERROR("do not support backup deleted tablet for now", K(ret), K(tablet_id)); + } else { + ObBackupSkippedType skipped_type = is_deleted ? ObBackupSkippedType(ObBackupSkippedType::DELETED) + : ObBackupSkippedType(ObBackupSkippedType::TRANSFER); + if (OB_FAIL(report_tablet_skipped_(tablet_id, skipped_type))) { + LOG_WARN("failed to report tablet skipped", K(ret), K(tablet_id), K_(param), K(skipped_type)); + } else { + LOG_INFO("report tablet skipped", K(ret), K(tablet_id), K_(param), K(skipped_type)); + } } } else { LOG_WARN("failed to get tablet handle", K(ret), K(tenant_id), K(ls_id), K(tablet_id)); @@ -1651,6 +1656,7 @@ int ObBackupTabletProvider::get_tablet_handle_(const uint64_t tenant_id, const s ObLS *ls = NULL; ObLSHandle ls_handle; ObLSService *ls_svr = NULL; + ObTabletTxMultiSourceDataUnit tx_data; const int64_t timeout_us = ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US; if (OB_ISNULL(ls_svr = MTL(ObLSService *))) { ret = OB_ERR_UNEXPECTED; @@ -1664,6 +1670,12 @@ int ObBackupTabletProvider::get_tablet_handle_(const uint64_t tenant_id, const s LOG_WARN("failed to check ls valid for backup", K(ret), K(tenant_id), K(ls_id), K(rebuild_seq)); } else if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle, timeout_us))) { LOG_WARN("failed to get tablet handle", K(ret), K(tenant_id), K(ls_id), K(tablet_id)); + } else if (OB_FAIL(tablet_handle.get_obj()->get_tx_data(tx_data))) { + LOG_WARN("failed to get_tx_data", K(ret), K(ls_id), K(tablet_id)); + } else if (ObTabletStatus::DELETED == tx_data.tablet_status_.get_status() + || ObTabletStatus::DELETING == tx_data.tablet_status_.get_status()) { + ret = OB_TABLET_NOT_EXIST; + LOG_INFO("tablet has been deleted", K(ret), K(ls_id), K(tablet_id), K(tx_data)); } else if (OB_FAIL(ObBackupUtils::check_ls_valid_for_backup(tenant_id, ls_id, rebuild_seq))) { LOG_WARN("failed to check ls valid for backup", K(ret), K(tenant_id), K(ls_id), K(rebuild_seq)); } @@ -1704,7 +1716,8 @@ int ObBackupTabletProvider::check_tablet_deleted_( return ret; } -int ObBackupTabletProvider::report_tablet_skipped_(const common::ObTabletID &tablet_id) +int ObBackupTabletProvider::report_tablet_skipped_(const common::ObTabletID &tablet_id, + const share::ObBackupSkippedType &skipped_type) { int ret = OB_SUCCESS; ObBackupSkippedTablet skipped_tablet; @@ -1715,6 +1728,7 @@ int ObBackupTabletProvider::report_tablet_skipped_(const common::ObTabletID &tab skipped_tablet.tablet_id_ = tablet_id; skipped_tablet.backup_set_id_ = param_.backup_set_desc_.backup_set_id_; skipped_tablet.ls_id_ = param_.ls_id_; + skipped_tablet.skipped_type_ = skipped_type; if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid args", K(ret), K(tablet_id)); diff --git a/src/storage/backup/ob_backup_utils.h b/src/storage/backup/ob_backup_utils.h index 23160b2459..a30c64a6b4 100644 --- a/src/storage/backup/ob_backup_utils.h +++ b/src/storage/backup/ob_backup_utils.h @@ -21,6 +21,7 @@ #include "lib/allocator/page_arena.h" #include "lib/mysqlclient/ob_mysql_proxy.h" #include "share/ob_ls_id.h" +#include "share/backup/ob_backup_struct.h" #include "storage/backup/ob_backup_data_struct.h" #include "storage/blocksstable/ob_macro_block_id.h" #include "storage/ob_i_table.h" @@ -261,7 +262,7 @@ private: int get_tablet_handle_(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, storage::ObTabletHandle &tablet_handle); int check_tablet_deleted_(const uint64_t tenant_id, const common::ObTabletID &tablet_id, bool &is_deleted); - int report_tablet_skipped_(const common::ObTabletID &tablet_id); + int report_tablet_skipped_(const common::ObTabletID &tablet_id, const share::ObBackupSkippedType &skipped_type); int hold_tablet_handle_(const common::ObTabletID &tablet_id, storage::ObTabletHandle &tablet_handle); int fetch_tablet_sstable_array_(const common::ObTabletID &tablet_id, storage::ObTabletHandle &tablet_handle, const share::ObBackupDataType &backup_data_type, common::ObIArray &sstable_array); diff --git a/src/storage/high_availability/ob_ls_restore.cpp b/src/storage/high_availability/ob_ls_restore.cpp index 120cedcf8b..ab639f53d6 100644 --- a/src/storage/high_availability/ob_ls_restore.cpp +++ b/src/storage/high_availability/ob_ls_restore.cpp @@ -986,6 +986,8 @@ int ObStartLSRestoreTask::choose_leader_src_() { int ret = OB_SUCCESS; ObArray tablet_id_array; + ObArray deleted_tablet_id_array; + ObArray need_schedule_tablet_id_array; ObMigrationStatus migration_status; ObLSRestoreStatus restore_status; share::ObBackupDataStore store; @@ -1016,13 +1018,21 @@ int ObStartLSRestoreTask::choose_leader_src_() ctx_->arg_.ls_id_, tablet_id_array))) { LOG_WARN("failed to read tablet to ls info", K(ret), KPC(ctx_)); + } else if (OB_FAIL(store.read_deleted_tablet_info( + ctx_->arg_.ls_id_, + deleted_tablet_id_array))) { + LOG_WARN("failed to read deleted tablet info", K(ret), KPC(ctx_)); + } else if (OB_FAIL(get_difference(tablet_id_array, deleted_tablet_id_array, need_schedule_tablet_id_array))) { + LOG_WARN("failed to get difference", K(ret), K(tablet_id_array), K(deleted_tablet_id_array)); } else { FLOG_INFO("succeed get backup ls meta info and tablet id array", K(ls_meta_package), K(tablet_id_array)); ctx_->src_ls_meta_package_ = ls_meta_package; ctx_->need_check_seq_ = false; ctx_->ls_rebuild_seq_ = -1; - if (OB_FAIL(generate_tablet_id_array_(tablet_id_array))) { - LOG_WARN("failed to generate tablet id array", K(ret), K(ls_meta_package), K(tablet_id_array)); + if (OB_FAIL(generate_tablet_id_array_(need_schedule_tablet_id_array))) { + LOG_WARN("failed to generate tablet id array", K(ret), K(ls_meta_package), K(need_schedule_tablet_id_array)); + } else { + LOG_INFO("get deleted tablet ids", KPC(ctx_), K(deleted_tablet_id_array)); } } } diff --git a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp index 00cf8c0597..22d9896e13 100644 --- a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp +++ b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp @@ -690,6 +690,12 @@ int ObAdminDumpBackupDataExecutor::do_execute_() } break; } + case share::ObBackupFileType::BACKUP_DELETED_TABLET_INFO: { + if (OB_FAIL(print_deleted_tablet_info_())) { + STORAGE_LOG(WARN, "failed to print deleted tablet info", K(ret)); + } + break; + } case share::ObBackupFileType::BACKUP_TENANT_LOCALITY_INFO: { if (OB_FAIL(print_tenant_locality_info_())) { STORAGE_LOG(WARN, "failed to print meta index file", K(ret)); @@ -1307,6 +1313,26 @@ int ObAdminDumpBackupDataExecutor::print_tablet_to_ls_info_() return ret; } +int ObAdminDumpBackupDataExecutor::print_deleted_tablet_info_() +{ + int ret = OB_SUCCESS; + share::ObBackupDeletedTabletToLSDesc deleted_tablet_to_ls; + if (OB_FAIL(inner_print_common_header_(backup_path_, storage_info_))) { + STORAGE_LOG(WARN, "fail to inner print common header", K(ret)); + } else if (OB_FAIL(ObAdminDumpBackupDataUtil::read_backup_info_file(ObString(backup_path_), ObString(storage_info_), deleted_tablet_to_ls))) { + STORAGE_LOG(WARN, "fail to read tablet to ls info", K(ret), K(backup_path_), K(storage_info_)); + } else { + ARRAY_FOREACH_X(deleted_tablet_to_ls.deleted_tablet_to_ls_, i , cnt, OB_SUCC(ret)) { + const share::ObBackupDataTabletToLSInfo tablet_to_ls = deleted_tablet_to_ls.deleted_tablet_to_ls_.at(i); + if (OB_FAIL(dump_tablet_to_ls_info_(tablet_to_ls))) { + STORAGE_LOG(WARN, "fail to dump ls attr info", K(ret), K(tablet_to_ls)); + } + } + } + PrintHelper::print_end_line(); + return ret; +} + int ObAdminDumpBackupDataExecutor::print_tenant_locality_info_() { int ret = OB_SUCCESS; diff --git a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.h b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.h index 5da87779c0..78c9c317c5 100644 --- a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.h +++ b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.h @@ -105,6 +105,7 @@ private: int print_meta_index_index_list_(); int print_ls_attr_info_(); int print_tablet_to_ls_info_(); + int print_deleted_tablet_info_(); int print_tenant_locality_info_(); int print_tenant_diagnose_info_(); int print_backup_set_info_();