From 95d786e47cd64a902f4f8e5526e8ff3fb85961e8 Mon Sep 17 00:00:00 2001 From: oceanoverflow Date: Tue, 17 Dec 2024 09:44:41 +0000 Subject: [PATCH] optimize backup mview tablet process --- .../backup/ob_backup_data_set_task_mgr.cpp | 6 +- .../mview/ob_mview_push_refresh_scn_task.cpp | 7 -- .../backup/ob_backup_data_table_operator.cpp | 29 +++-- .../backup/ob_backup_data_table_operator.h | 3 +- src/share/backup/ob_backup_struct.cpp | 1 + src/share/ob_debug_sync_point.h | 1 + src/share/ob_errno.cpp | 15 +++ src/share/ob_errno.def | 1 + src/share/ob_errno.h | 4 + src/storage/backup/ob_backup_ctx.cpp | 31 +++-- src/storage/backup/ob_backup_ctx.h | 4 +- src/storage/backup/ob_backup_data_store.cpp | 19 ++-- src/storage/backup/ob_backup_data_store.h | 3 +- src/storage/backup/ob_backup_reader.cpp | 5 +- src/storage/backup/ob_backup_task.cpp | 11 +- src/storage/backup/ob_backup_utils.cpp | 107 ++++++++++++++++-- src/storage/backup/ob_backup_utils.h | 14 ++- 17 files changed, 203 insertions(+), 58 deletions(-) diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp index 080785487..52e513668 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp @@ -633,6 +633,7 @@ int ObBackupSetTaskMgr::backup_meta_finish_() } } } + DEBUG_SYNC(AFTER_BACKUP_META_FINISH); return ret; } @@ -841,6 +842,7 @@ int ObBackupSetTaskMgr::backup_major_compaction_mview_dep_tablet_list_() int ret = OB_SUCCESS; ObBackupMajorCompactionMViewDepTabletListDesc desc; common::ObArray mview_tablet_list; + common::ObArray mview_dep_scn_list; share::SCN backup_scn; if (OB_ISNULL(sql_proxy_)) { ret = OB_ERR_UNEXPECTED; @@ -848,10 +850,12 @@ int ObBackupSetTaskMgr::backup_major_compaction_mview_dep_tablet_list_() } else if (OB_FAIL(ObBackupDataScheduler::get_backup_scn(*sql_proxy_, set_task_attr_.tenant_id_, true/*is_backup_start*/, backup_scn))) { LOG_WARN("failed to get backup scn", K(ret), K(set_task_attr_)); } else if (OB_FAIL(ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list( - *sql_proxy_, set_task_attr_.tenant_id_, backup_scn, mview_tablet_list))) { + *sql_proxy_, set_task_attr_.tenant_id_, backup_scn, mview_tablet_list, mview_dep_scn_list))) { LOG_WARN("failed to get all major compaction mveiw dep tablet list", K(ret), K(backup_scn)); } else if (OB_FAIL(desc.tablet_id_list_.assign(mview_tablet_list))) { LOG_WARN("failed to assign tablet list", K(ret)); + } else if (OB_FAIL(desc.mview_dep_scn_list_.assign(mview_dep_scn_list))) { + LOG_WARN("failed to assign major scn list", K(ret)); } else if (OB_FAIL(store_.write_major_compaction_mview_dep_tablet_list(desc))) { LOG_WARN("failed to write mview dep tablet list", K(ret)); } diff --git a/src/rootserver/mview/ob_mview_push_refresh_scn_task.cpp b/src/rootserver/mview/ob_mview_push_refresh_scn_task.cpp index 288ffcc1f..25c5c32e7 100644 --- a/src/rootserver/mview/ob_mview_push_refresh_scn_task.cpp +++ b/src/rootserver/mview/ob_mview_push_refresh_scn_task.cpp @@ -123,7 +123,6 @@ void ObMViewPushRefreshScnTask::runTimerTask() share::ObGlobalStatProxy stat_proxy(trans, tenant_id_); share::SCN major_refresh_mv_merge_scn; ObArray backup_jobs; - uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); if (OB_FAIL(stat_proxy.get_major_refresh_mv_merge_scn(true /*select for update*/, major_refresh_mv_merge_scn))) { LOG_WARN("fail to get major_refresh_mv_merge_scn", KR(ret), K(tenant_id_)); @@ -131,12 +130,6 @@ void ObMViewPushRefreshScnTask::runTimerTask() ret = OB_ERR_UNEXPECTED; LOG_WARN("major_refresh_mv_merge_scn is invalid", KR(ret), K(tenant_id_), K(major_refresh_mv_merge_scn)); - } else if (OB_FAIL(share::ObBackupJobOperator::get_jobs( - *sql_proxy, meta_tenant_id, false /*select for update*/, backup_jobs))) { - LOG_WARN("failed to get backup jobs", K(ret), K(tenant_id_)); - } else if (!backup_jobs.empty()) { - LOG_INFO("[MAJ_REF_MV] backup jobs exist, skip push major refresh mv scn", KR(ret), - K(tenant_id_)); } else if (OB_FAIL(update_major_refresh_mview_scn_(tenant_id_, major_refresh_mv_merge_scn, trans))) { LOG_WARN("fail to update major_refresh_mview_scn", KR(ret), K(tenant_id_), K(major_refresh_mv_merge_scn)); } else { diff --git a/src/share/backup/ob_backup_data_table_operator.cpp b/src/share/backup/ob_backup_data_table_operator.cpp index 9c39f852e..9bdf3700d 100644 --- a/src/share/backup/ob_backup_data_table_operator.cpp +++ b/src/share/backup/ob_backup_data_table_operator.cpp @@ -2437,13 +2437,16 @@ int ObBackupSkippedTabletOperator::move_skip_tablet_to_his( return ret; } -int ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list(common::ObMySQLProxy &proxy, - const uint64_t tenant_id, - const share::SCN &snapshot, - common::ObIArray &tablet_list) +int ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list( + common::ObMySQLProxy &proxy, + const uint64_t tenant_id, + const share::SCN &snapshot, + common::ObIArray &tablet_list, + common::ObIArray &tablet_mview_dep_scn_list) { int ret = OB_SUCCESS; tablet_list.reset(); + tablet_mview_dep_scn_list.reset(); ObSqlString sql; int64_t affected_rows = -1; if (OB_INVALID_TENANT_ID == tenant_id || !snapshot.is_valid()) { @@ -2452,12 +2455,11 @@ int ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list(common } else { HEAP_VAR(ObMySQLProxy::ReadResult, res) { ObMySQLResult *result = NULL; - if (OB_FAIL(sql.assign_fmt("select tablet_id from %s as of snapshot %ld where table_id in (" - " select p_obj from %s as of snapshot %ld" - " where mview_id in (select mview_id from %s as of snapshot %ld where refresh_mode=%ld) " - " union all " - " select data_table_id from %s as of snapshot %ld" - " where table_id in (select mview_id from %s as of snapshot %ld where refresh_mode=%ld))", + if (OB_FAIL(sql.assign_fmt("select t1.tablet_id tablet_id,t2.major_version major_version from %s as of snapshot %ld t1" + " join (select p_obj table_id,min(b.last_refresh_scn) major_version from %s as of snapshot %ld a" + " join %s as of snapshot %ld b on a.mview_id=b.mview_id where b.refresh_mode=%ld group by p_obj" + " union all (select a.data_table_id table_id,b.last_refresh_scn major_version from %s as of snapshot %ld a" + " join %s as of snapshot %ld b on a.table_id=b.mview_id and b.refresh_mode=%ld)) t2 on t1.table_id=t2.table_id", OB_ALL_TABLET_TO_LS_TNAME, snapshot.get_val_for_inner_table_field(), OB_ALL_MVIEW_DEP_TNAME, snapshot.get_val_for_inner_table_field(), OB_ALL_MVIEW_TNAME, snapshot.get_val_for_inner_table_field(), schema::ObMVRefreshMode::MAJOR_COMPACTION, @@ -2480,9 +2482,16 @@ int ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list(common } } else { int64_t tablet_id = -1; + uint64_t scn_val = -1; + share::SCN major_scn; EXTRACT_INT_FIELD_MYSQL(*result, OB_STR_TABLET_ID, tablet_id, int64_t); + EXTRACT_UINT_FIELD_MYSQL(*result, "major_version", scn_val, uint64_t); if (FAILEDx(tablet_list.push_back(ObTabletID(tablet_id)))) { LOG_WARN("failed to push back", K(ret), K(tablet_id)); + } else if (OB_FAIL(major_scn.convert_for_inner_table_field(scn_val))) { + LOG_WARN("failed to convert for inner table field", K(ret)); + } else if (OB_FAIL(tablet_mview_dep_scn_list.push_back(major_scn))) { + LOG_WARN("failed to push back", K(ret), K(major_scn)); } } } diff --git a/src/share/backup/ob_backup_data_table_operator.h b/src/share/backup/ob_backup_data_table_operator.h index a85e14586..d62e384fb 100644 --- a/src/share/backup/ob_backup_data_table_operator.h +++ b/src/share/backup/ob_backup_data_table_operator.h @@ -63,7 +63,8 @@ public: static int get_all_major_compaction_mview_dep_tablet_list(common::ObMySQLProxy &proxy, const uint64_t tenant_id, const share::SCN &snapshot, - common::ObIArray &tablet_list); + common::ObIArray &tablet_list, + common::ObIArray &tablet_mview_dep_scn_list); }; class ObBackupSetFileOperator : public ObBackupBaseTableOperator diff --git a/src/share/backup/ob_backup_struct.cpp b/src/share/backup/ob_backup_struct.cpp index 6838f4cf4..d5e452b42 100644 --- a/src/share/backup/ob_backup_struct.cpp +++ b/src/share/backup/ob_backup_struct.cpp @@ -2094,6 +2094,7 @@ bool ObBackupUtils::is_need_retry_error(const int err) case OB_CHECKSUM_ERROR : case OB_VERSION_NOT_MATCH: case OB_INVALID_DATA: + case OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE: bret = false; break; default: diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index 53b9c2264..d3279c3dc 100755 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -396,6 +396,7 @@ class ObString; ACT(BEFORE_REPLAY_DDL_COMMIT,)\ ACT(BEFORE_BACKUP_UESR_META,)\ ACT(BEFORE_BACKUP_META_FINISH,)\ + ACT(AFTER_BACKUP_META_FINISH,)\ ACT(BEFORE_INSERT_UERR_RECOVER_TABLE_JOB,)\ ACT(BEFORE_GENERATE_IMPORT_TABLE_TASK,)\ ACT(BEFORE_RECOVER_UESR_RECOVER_TABLE_JOB,)\ diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index e3ae84fb7..659879da1 100644 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -27733,6 +27733,20 @@ static const _error _error_OB_TMP_FILE_EXCEED_DISK_QUOTA = { .ob_str_error = "OBE-00600: internal error code, arguments: -9135, tmp file exceeds disk quota", .ob_str_user_error = "OBE-00600: internal error code, arguments: -9135, tmp file exceeds disk quota" }; +static const _error _error_OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE = { + .error_name = "OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE", + .error_cause = "Internal Error", + .error_solution = "Contact OceanBase Support", + .mysql_errno = -1, + .sqlstate = "HY000", + .str_error = "Backup missing mview dependent tablet sstable, sstable maybe recycled", + .str_user_error = "Backup missing mview dependent tablet sstable, sstable maybe recycled", + .oracle_errno = 600, + .oracle_str_error = "ORA-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled", + .oracle_str_user_error = "ORA-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled", + .ob_str_error = "OBE-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled", + .ob_str_user_error = "OBE-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled" +}; static const _error _error_OB_ERR_RESIZE_FILE_TO_SMALLER = { .error_name = "OB_ERR_RESIZE_FILE_TO_SMALLER", .error_cause = "Internal Error", @@ -35515,6 +35529,7 @@ struct ObStrErrorInit _errors[-OB_BACKUP_ZONE_IDC_REGION_INVALID] = &_error_OB_BACKUP_ZONE_IDC_REGION_INVALID; _errors[-OB_ERR_TMP_FILE_ALREADY_SEALED] = &_error_OB_ERR_TMP_FILE_ALREADY_SEALED; _errors[-OB_TMP_FILE_EXCEED_DISK_QUOTA] = &_error_OB_TMP_FILE_EXCEED_DISK_QUOTA; + _errors[-OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE] = &_error_OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE; _errors[-OB_ERR_RESIZE_FILE_TO_SMALLER] = &_error_OB_ERR_RESIZE_FILE_TO_SMALLER; _errors[-OB_MARK_BLOCK_INFO_TIMEOUT] = &_error_OB_MARK_BLOCK_INFO_TIMEOUT; _errors[-OB_NOT_READY_TO_EXTEND_FILE] = &_error_OB_NOT_READY_TO_EXTEND_FILE; diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index 69452b2cc..d513bdb0e 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -2351,6 +2351,7 @@ DEFINE_ERROR_EXT_DEP(OB_BACKUP_ZONE_IDC_REGION_INVALID, -9133, -1, "HY000", "bac DEFINE_ERROR_DEP(OB_ERR_TMP_FILE_ALREADY_SEALED, -9134, -1, "HY000", "tmp file has already sealed"); DEFINE_ERROR_DEP(OB_TMP_FILE_EXCEED_DISK_QUOTA, -9135, -1, "HY000", "tmp file exceeds disk quota"); +DEFINE_ERROR(OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE, -9136, -1, "HY000", "Backup missing mview dependent tablet sstable, sstable maybe recycled"); // 余留位置，在该行之前继续添加storage错误码 //////////////////////////////////////////////////////////////// diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index 2d74987a0..4bcb5a8ec 100644 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -1592,6 +1592,7 @@ constexpr int OB_STORAGE_DEST_NOT_CONNECT = -9115; constexpr int OB_TABLET_IS_SPLIT_SRC = -9123; constexpr int OB_TABLET_STATUS_NO_NEED_TO_SPLIT = -9127; constexpr int OB_FILE_DELETE_FAILED = -9128; +constexpr int OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE = -9136; constexpr int OB_ERR_RESIZE_FILE_TO_SMALLER = -9200; constexpr int OB_MARK_BLOCK_INFO_TIMEOUT = -9201; constexpr int OB_NOT_READY_TO_EXTEND_FILE = -9202; @@ -3947,6 +3948,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_BACKUP_ZONE_IDC_REGION_INVALID__USER_ERROR_MSG "%s" #define OB_ERR_TMP_FILE_ALREADY_SEALED__USER_ERROR_MSG "tmp file has already sealed" #define OB_TMP_FILE_EXCEED_DISK_QUOTA__USER_ERROR_MSG "tmp file exceeds disk quota" +#define OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE__USER_ERROR_MSG "Backup missing mview dependent tablet sstable, sstable maybe recycled" #define OB_ERR_RESIZE_FILE_TO_SMALLER__USER_ERROR_MSG "Extend ssblock file to smaller is not allowed" #define OB_MARK_BLOCK_INFO_TIMEOUT__USER_ERROR_MSG "Mark blocks timeout(5s) in auto extend process when alloc block fail" #define OB_NOT_READY_TO_EXTEND_FILE__USER_ERROR_MSG "Auto extend param is not ready to start extending file" @@ -8321,6 +8323,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_TMP_FILE_ALREADY_SEALED__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9134, tmp file has already sealed" #define OB_TMP_FILE_EXCEED_DISK_QUOTA__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9135, tmp file exceeds disk quota" #define OB_TMP_FILE_EXCEED_DISK_QUOTA__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9135, tmp file exceeds disk quota" +#define OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled" +#define OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled" #define OB_ERR_RESIZE_FILE_TO_SMALLER__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9200, Extend ssblock file to smaller is not allowed" #define OB_ERR_RESIZE_FILE_TO_SMALLER__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9200, Extend ssblock file to smaller is not allowed" #define OB_MARK_BLOCK_INFO_TIMEOUT__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9201, Mark blocks timeout(5s) in auto extend process when alloc block fail" diff --git a/src/storage/backup/ob_backup_ctx.cpp b/src/storage/backup/ob_backup_ctx.cpp index 051d1cecb..1c2d5c2fb 100644 --- a/src/storage/backup/ob_backup_ctx.cpp +++ b/src/storage/backup/ob_backup_ctx.cpp @@ -1235,7 +1235,7 @@ ObLSBackupCtx::ObLSBackupCtx() backup_tx_table_filled_tx_scn_(share::SCN::min_scn()), index_builder_mgr_(), bandwidth_throttle_(NULL), - mview_dep_tablet_set_(), + mview_dep_tablet_map_(), wait_reuse_across_sstable_time_(0), mv_mutex_() {} @@ -1260,6 +1260,7 @@ int ObLSBackupCtx::open( int ret = OB_SUCCESS; ObArray tablet_list; ObILSTabletIdReader *reader = NULL; + ObMemAttr attr(MTL_ID(), ObModIds::BACKUP); if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); @@ -1279,8 +1280,8 @@ int ObLSBackupCtx::open( LOG_WARN("failed to init index builder mg", K(ret), K(param)); } else if (OB_FAIL(param_.assign(param))) { LOG_WARN("failed to assign param", K(ret), K(param)); - } else if (OB_FAIL(mview_dep_tablet_set_.create(DEFAULT_MVIEW_DEP_TABLET_SET_SIZE))) { - LOG_WARN("failed to create mveiw dep tablet set", K(ret)); + } else if (OB_FAIL(mview_dep_tablet_map_.create(DEFAULT_MVIEW_DEP_TABLET_SET_SIZE, attr))) { + LOG_WARN("failed to create mview dep tablet map", K(ret)); } else { max_file_id_ = 0; prefetch_task_id_ = 0; @@ -1519,7 +1520,8 @@ int ObLSBackupCtx::finish_task(const int64_t file_id) return ret; } -int ObLSBackupCtx::check_is_major_compaction_mview_dep_tablet(const common::ObTabletID &tablet_id, bool &is_dep) const +int ObLSBackupCtx::check_is_major_compaction_mview_dep_tablet( + const common::ObTabletID &tablet_id, share::SCN &mview_dep_scn, bool &is_dep) const { int ret = OB_SUCCESS; is_dep = false; @@ -1527,12 +1529,11 @@ int ObLSBackupCtx::check_is_major_compaction_mview_dep_tablet(const common::ObTa ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid args", K(ret), K(tablet_id)); } else { - int hash_ret = mview_dep_tablet_set_.exist_refactored(tablet_id); - if (OB_HASH_EXIST == hash_ret) { - is_dep = true; - LOG_INFO("is mview dependent tablet", K(tablet_id)); - } else if (OB_HASH_NOT_EXIST == hash_ret) { + int hash_ret = mview_dep_tablet_map_.get_refactored(tablet_id, mview_dep_scn); + if (OB_HASH_NOT_EXIST == hash_ret) { is_dep = false; + } else if (OB_SUCCESS == hash_ret) { + is_dep = true; } else { ret = hash_ret; LOG_WARN("failed to check is mview dep tablet", K(ret), K(tablet_id)); @@ -1642,11 +1643,17 @@ int ObLSBackupCtx::prepare_mview_dep_tablet_set_(const ObLSBackupParam ¶m, LOG_WARN("failed to read mview dep tablet list", K(ret)); } else { const common::ObSArray &mview_tablet_list = desc.tablet_id_list_;; + const ObSArray &tablet_mview_dep_scn_list = desc.mview_dep_scn_list_; + if (mview_tablet_list.count() != tablet_mview_dep_scn_list.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mview tablet list not equal to tablet version list", K(ret)); + } ARRAY_FOREACH_X(mview_tablet_list, idx, cnt, OB_SUCC(ret)) { const ObTabletID &tablet_id = mview_tablet_list.at(idx); + const SCN &tablet_version = tablet_mview_dep_scn_list.at(idx); const int flag = 1; - if (OB_FAIL(mview_dep_tablet_set_.set_refactored(tablet_id, flag))) { - LOG_WARN("failed to set refactored", K(ret), K(tablet_id)); + if (OB_FAIL(mview_dep_tablet_map_.set_refactored(tablet_id, tablet_version, flag))) { + LOG_WARN("failed to set refactored", K(ret), K(tablet_id), K(tablet_version)); } } } @@ -1662,7 +1669,7 @@ int ObLSBackupCtx::check_mview_tablet_set_(const uint64_t tenant_id, common::ObM LOG_WARN("failed to get tenant info", K(ret), K(tenant_id)); } else if (tenant_info.is_primary()) { LOG_INFO("tenant is primary", K(tenant_id), K(tenant_info)); - } else if (!mview_dep_tablet_set_.empty()) { + } else if (!mview_dep_tablet_map_.empty()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tenant not primary but mview dep tablet is not empty", K(tenant_id), K(tenant_info)); } diff --git a/src/storage/backup/ob_backup_ctx.h b/src/storage/backup/ob_backup_ctx.h index 06aa5780e..b317e9497 100644 --- a/src/storage/backup/ob_backup_ctx.h +++ b/src/storage/backup/ob_backup_ctx.h @@ -300,7 +300,7 @@ public: return tablet_holder_; } - int check_is_major_compaction_mview_dep_tablet(const common::ObTabletID &tablet_id, bool &is_major_compaction_mview_dep_tablet) const; + int check_is_major_compaction_mview_dep_tablet(const common::ObTabletID &tablet_id, share::SCN &mview_dep_scn, bool &is_major_compaction_mview_dep_tablet) const; void add_wait_reuse_across_sstable_time(int64_t cost_time); private: @@ -346,7 +346,7 @@ public: share::SCN backup_tx_table_filled_tx_scn_; ObBackupTabletIndexBlockBuilderMgr index_builder_mgr_; common::ObInOutBandwidthThrottle *bandwidth_throttle_; - common::hash::ObHashSet mview_dep_tablet_set_; + common::hash::ObHashMap mview_dep_tablet_map_; int64_t wait_reuse_across_sstable_time_; mutable lib::ObMutex mv_mutex_; DISALLOW_COPY_AND_ASSIGN(ObLSBackupCtx); diff --git a/src/storage/backup/ob_backup_data_store.cpp b/src/storage/backup/ob_backup_data_store.cpp index 8c2581a19..4b5e3ce9a 100644 --- a/src/storage/backup/ob_backup_data_store.cpp +++ b/src/storage/backup/ob_backup_data_store.cpp @@ -339,21 +339,26 @@ bool ObBackupTableListMetaInfoDesc::is_valid() const *-----------------------------ObBackupMajorCompactionMViewDepTabletListDesc----------------------- */ -OB_SERIALIZE_MEMBER(ObBackupMajorCompactionMViewDepTabletListDesc, tablet_id_list_); +OB_SERIALIZE_MEMBER(ObBackupMajorCompactionMViewDepTabletListDesc, tablet_id_list_, mview_dep_scn_list_); ObBackupMajorCompactionMViewDepTabletListDesc::ObBackupMajorCompactionMViewDepTabletListDesc() : ObExternBackupDataDesc(ObBackupFileType::BACKUP_MVIEW_DEP_TABLET_LIST_FILE, FILE_VERSION), - tablet_id_list_() {} + tablet_id_list_(), + mview_dep_scn_list_() {} bool ObBackupMajorCompactionMViewDepTabletListDesc::is_valid() const { int ret = OB_SUCCESS; bool bret = true; - ARRAY_FOREACH(tablet_id_list_, i) { - const ObTabletID &tablet_id = tablet_id_list_.at(i); - if (!tablet_id.is_valid()) { - bret = false; - break; + if (tablet_id_list_.count() != mview_dep_scn_list_.count()) { + bret = false; + } else { + ARRAY_FOREACH(tablet_id_list_, i) { + const ObTabletID &tablet_id = tablet_id_list_.at(i); + if (!tablet_id.is_valid()) { + bret = false; + break; + } } } return bret; diff --git a/src/storage/backup/ob_backup_data_store.h b/src/storage/backup/ob_backup_data_store.h index 5183ca330..6383ae50a 100644 --- a/src/storage/backup/ob_backup_data_store.h +++ b/src/storage/backup/ob_backup_data_store.h @@ -346,9 +346,10 @@ public: ObBackupMajorCompactionMViewDepTabletListDesc(); ~ObBackupMajorCompactionMViewDepTabletListDesc() {} bool is_valid() const override; - TO_STRING_KV(K_(tablet_id_list)); + TO_STRING_KV(K_(tablet_id_list), K_(mview_dep_scn_list)); public: ObSArray tablet_id_list_; + ObSArray mview_dep_scn_list_; private: DISALLOW_COPY_AND_ASSIGN(ObBackupMajorCompactionMViewDepTabletListDesc); }; diff --git a/src/storage/backup/ob_backup_reader.cpp b/src/storage/backup/ob_backup_reader.cpp index 8274868b2..64b75a8a5 100644 --- a/src/storage/backup/ob_backup_reader.cpp +++ b/src/storage/backup/ob_backup_reader.cpp @@ -670,12 +670,13 @@ int ObSSTableMetaBackupReader::init(const common::ObTabletID &tablet_id, linked_writer_ = linked_writer; ObTablet &tablet = *tablet_handle_->get_obj(); bool is_major_compaction_mview_dep = false; + share::SCN mview_dep_scn; if (OB_FAIL(tablet.fetch_table_store(table_store_wrapper_))) { LOG_WARN("failed to fetch table store from tablet", K(ret)); - } else if (OB_FAIL(ls_backup_ctx.check_is_major_compaction_mview_dep_tablet(tablet_id, is_major_compaction_mview_dep))) { + } else if (OB_FAIL(ls_backup_ctx.check_is_major_compaction_mview_dep_tablet(tablet_id, mview_dep_scn, is_major_compaction_mview_dep))) { LOG_WARN("failed to check is mview dep tablet", K(ret), K(tablet_id)); } else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type( - tablet_handle, backup_data_type, *table_store_wrapper_.get_member(), is_major_compaction_mview_dep, sstable_array_))) { + tablet_handle, backup_data_type, *table_store_wrapper_.get_member(), is_major_compaction_mview_dep, mview_dep_scn, sstable_array_))) { LOG_WARN("failed to get sstables by data type", K(ret), K(tablet_handle)); } else { builder_mgr_ = &index_block_builder_mgr; diff --git a/src/storage/backup/ob_backup_task.cpp b/src/storage/backup/ob_backup_task.cpp index 432f5ffa0..ffa5bd0f1 100644 --- a/src/storage/backup/ob_backup_task.cpp +++ b/src/storage/backup/ob_backup_task.cpp @@ -2684,6 +2684,7 @@ int ObLSBackupDataTask::do_prepare_sstable_builders_(const ObBackupProviderItem ObBackupTabletHandleRef *tablet_ref = NULL; ObArray sstable_array; bool is_major_compaction_mview_dep_tablet = false; + share::SCN mview_dep_scn; ObTabletMemberWrapper table_store_wrapper; if (!item.is_valid()) { ret = OB_INVALID_ARGUMENT; @@ -2695,9 +2696,10 @@ int ObLSBackupDataTask::do_prepare_sstable_builders_(const ObBackupProviderItem LOG_WARN("failed to get tablet handle", K(ret), K(tablet_id)); } else if (OB_FAIL(tablet_ref->tablet_handle_.get_obj()->fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret), K(tablet_id)); - } else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet_id, is_major_compaction_mview_dep_tablet))) { + } else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet_id, mview_dep_scn, is_major_compaction_mview_dep_tablet))) { LOG_WARN("failed to check is mview dep tablet", K(ret), K(tablet_id)); - } else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(tablet_ref->tablet_handle_, backup_data_type_, *table_store_wrapper.get_member(), is_major_compaction_mview_dep_tablet, sstable_array))) { + } else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(tablet_ref->tablet_handle_, backup_data_type_, + *table_store_wrapper.get_member(), is_major_compaction_mview_dep_tablet, mview_dep_scn, sstable_array))) { LOG_WARN("failed to get sstables by data type", K(ret), KPC(tablet_ref), K_(backup_data_type)); } else if (OB_FAIL(prepare_tablet_sstable_index_builders_(tablet_id, is_major_compaction_mview_dep_tablet, sstable_array))) { LOG_WARN("failed to prepare tablet sstable index builders", K(ret), K_(param), K(tablet_id), K(sstable_array)); @@ -3156,6 +3158,7 @@ int ObLSBackupDataTask::do_wait_sstable_index_builder_ready_(ObTabletHandle &tab int ret = OB_SUCCESS; ObTablet *tablet = nullptr; bool is_major_compaction_mview_dep = false; + share::SCN mview_dep_scn; ObTabletMemberWrapper table_store_wrapper; common::ObArray sstable_array; const ObTabletTableStore *table_store = nullptr; @@ -3166,10 +3169,10 @@ int ObLSBackupDataTask::do_wait_sstable_index_builder_ready_(ObTabletHandle &tab LOG_WARN("fail to fetch table store", K(ret)); } else if (OB_FAIL(table_store_wrapper.get_member(table_store))) { LOG_WARN("failed to get table store", K(ret), KPC(tablet)); - } else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet->get_tablet_id(), is_major_compaction_mview_dep))) { + } else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet->get_tablet_id(), mview_dep_scn, is_major_compaction_mview_dep))) { LOG_WARN("failed to check is mview dep tablet", K(ret), KPC(tablet)); } else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type( - tablet_handle, backup_data_type_, *table_store_wrapper.get_member(), is_major_compaction_mview_dep, sstable_array))) { + tablet_handle, backup_data_type_, *table_store_wrapper.get_member(), is_major_compaction_mview_dep, mview_dep_scn, sstable_array))) { LOG_WARN("failed to get sstables by data type", K(ret), K(tablet_handle)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < sstable_array.count(); ++i) { diff --git a/src/storage/backup/ob_backup_utils.cpp b/src/storage/backup/ob_backup_utils.cpp index 307c9230a..2634fcea6 100644 --- a/src/storage/backup/ob_backup_utils.cpp +++ b/src/storage/backup/ob_backup_utils.cpp @@ -84,7 +84,8 @@ int ObBackupUtils::calc_start_replay_scn( } int ObBackupUtils::get_sstables_by_data_type(const storage::ObTabletHandle &tablet_handle, const share::ObBackupDataType &backup_data_type, - const storage::ObTabletTableStore &tablet_table_store, const bool is_major_compaction_mview_dep_tablet, common::ObIArray &sstable_array) + const storage::ObTabletTableStore &tablet_table_store, const bool is_major_compaction_mview_dep_tablet, const share::SCN &mview_dep_scn, + common::ObIArray &sstable_array) { int ret = OB_SUCCESS; sstable_array.reset(); @@ -98,7 +99,7 @@ int ObBackupUtils::get_sstables_by_data_type(const storage::ObTabletHandle &tabl } else { if (OB_FAIL(fetch_minor_and_ddl_sstables_(tablet_handle, tablet_table_store, sstable_array))) { LOG_WARN("failed to fetch minor and ddl sstables", K(ret), K(tablet_handle), K(tablet_table_store)); - } else if (OB_FAIL(fetch_major_sstables_(tablet_handle, tablet_table_store, is_major_compaction_mview_dep_tablet, sstable_array))) { + } else if (OB_FAIL(fetch_major_sstables_(tablet_handle, tablet_table_store, is_major_compaction_mview_dep_tablet, mview_dep_scn, sstable_array))) { LOG_WARN("failed to fetch major sstables", K(ret), K(tablet_handle), K(tablet_table_store)); } } @@ -164,7 +165,7 @@ int ObBackupUtils::fetch_minor_and_ddl_sstables_(const storage::ObTabletHandle & int ObBackupUtils::fetch_major_sstables_(const storage::ObTabletHandle &tablet_handle, const storage::ObTabletTableStore &tablet_table_store, const bool is_major_compaction_mview_dep_tablet, - common::ObIArray &sstable_array) + const share::SCN &mview_dep_scn, common::ObIArray &sstable_array) { int ret = OB_SUCCESS; const storage::ObSSTableArray *major_sstable_array_ptr = NULL; @@ -190,8 +191,8 @@ int ObBackupUtils::fetch_major_sstables_(const storage::ObTabletHandle &tablet_h LOG_WARN("medium list is invalid for last major sstable", K(ret), KPC(medium_info_list), KPC(last_major_sstable_ptr), K(tablet_handle)); } else if (is_major_compaction_mview_dep_tablet) { - if (OB_FAIL(major_sstable_array_ptr->get_all_table_wrappers(sstable_array))) { - LOG_WARN("failed to get all table wrappers", K(ret)); + if (OB_FAIL(check_and_filter_major_sstables_for_mview_(mview_dep_scn, major_sstable_array_ptr, sstable_array))) { + LOG_WARN("failed to check and filter major sstable for mview", K(ret)); } } else { // not mview dep tablet if (last_major_sstable_ptr->is_co_sstable()) { @@ -209,6 +210,92 @@ int ObBackupUtils::fetch_major_sstables_(const storage::ObTabletHandle &tablet_h return ret; } +int ObBackupUtils::check_and_filter_major_sstables_for_mview_( + const share::SCN &mview_dep_scn, + const storage::ObSSTableArray *major_sstable_array_ptr, + common::ObIArray &sstable_array) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_major_sstables_for_mview_(mview_dep_scn, major_sstable_array_ptr))) { + LOG_WARN("failed to check major sstable for mview", K(ret)); + } else if (OB_FAIL(filter_major_sstables_for_mview_(mview_dep_scn, major_sstable_array_ptr, sstable_array))) { + LOG_WARN("failed to filter major sstables for mview", K(ret)); + } else { + LOG_INFO("check and filter major sstable for mview", K(sstable_array)); + } + return ret; +} + +int ObBackupUtils::check_major_sstables_for_mview_(const share::SCN &mview_dep_scn, + const storage::ObSSTableArray *major_sstable_array_ptr) +{ + int ret = OB_SUCCESS; + common::ObArray tmp_sstable_array; + if (OB_ISNULL(major_sstable_array_ptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("major sstable array ptr should not be null", K(ret)); + } else if (OB_FAIL(major_sstable_array_ptr->get_all_table_wrappers(tmp_sstable_array))) { + LOG_WARN("failed to get all table wrappers", K(ret), KPC(major_sstable_array_ptr)); + } else { + bool exist = false; + ARRAY_FOREACH_X(tmp_sstable_array, idx, cnt, OB_SUCC(ret)) { + storage::ObSSTableWrapper &sstable_wrapper = tmp_sstable_array.at(idx); + ObSSTable *sstable_ptr = NULL; + if (OB_ISNULL(sstable_ptr = sstable_wrapper.get_sstable())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table should not be null", K(ret)); + } else { + const ObITable::TableKey &table_key = sstable_ptr->get_key(); + if (table_key.get_end_scn() == mview_dep_scn) { + exist = true; + break; + } + } + } + if (OB_SUCC(ret) && !exist) { + ret = OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE; + LOG_WARN("missing sstable for mview tablet", K(ret), K(mview_dep_scn), K(tmp_sstable_array)); + } + } + return ret; +} + +int ObBackupUtils::filter_major_sstables_for_mview_( + const share::SCN &mview_dep_scn, + const storage::ObSSTableArray *major_sstable_array_ptr, + common::ObIArray &sstable_array) +{ + int ret = OB_SUCCESS; + common::ObArray tmp_sstable_array; + if (OB_ISNULL(major_sstable_array_ptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("major sstable array ptr should not be null", K(ret)); + } else if (OB_FAIL(major_sstable_array_ptr->get_all_table_wrappers(tmp_sstable_array))) { + LOG_WARN("failed to get all table wrappers", K(ret)); + } else { + ARRAY_FOREACH_X(tmp_sstable_array, idx, cnt, OB_SUCC(ret)) { + storage::ObSSTableWrapper &sstable_wrapper = tmp_sstable_array.at(idx); + ObSSTable *sstable_ptr = NULL; + if (OB_ISNULL(sstable_ptr = sstable_wrapper.get_sstable())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table should not be null", K(ret)); + } else { + const ObITable::TableKey &table_key = sstable_ptr->get_key(); + if (table_key.get_end_scn() < mview_dep_scn) { + // skip + } else if (OB_FAIL(sstable_array.push_back(sstable_wrapper))) { + LOG_WARN("failed to push back", K(ret), K(sstable_wrapper)); + } + } + } + if (OB_SUCC(ret) && sstable_array.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sstable array should not be empty", K(ret), K(mview_dep_scn), KPC(major_sstable_array_ptr)); + } + } + return ret; +} + int ObBackupUtils::check_tablet_with_major_sstable(const storage::ObTabletHandle &tablet_handle, bool &with_major) { int ret = OB_SUCCESS; @@ -1941,6 +2028,7 @@ int ObBackupTabletProvider::prepare_tablet_(const uint64_t tenant_id, const shar ObTabletMemberWrapper table_store_wrapper; bool need_skip_tablet = false; bool is_major_compaction_mview_dep_tablet = false; + share::SCN mview_dep_scn; bool is_split_dst = false; if (OB_ISNULL(ls_backup_ctx_)) { ret = OB_ERR_UNEXPECTED; @@ -1985,10 +2073,10 @@ int ObBackupTabletProvider::prepare_tablet_(const uint64_t tenant_id, const shar LOG_WARN("failed to check tablet replica validity", K(ret), K(tenant_id), K(ls_id), K(tablet_id), K(backup_data_type)); } else if (OB_FAIL(tablet_ref->tablet_handle_.get_obj()->fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); - } else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet_id, is_major_compaction_mview_dep_tablet))) { + } else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet_id, mview_dep_scn, is_major_compaction_mview_dep_tablet))) { LOG_WARN("failed to check is mview dep tablet", K(ret)); } else if (OB_FAIL(fetch_tablet_sstable_array_( - tablet_id, tablet_ref->tablet_handle_, *table_store_wrapper.get_member(), backup_data_type, is_major_compaction_mview_dep_tablet, sstable_array))) { + tablet_id, tablet_ref->tablet_handle_, *table_store_wrapper.get_member(), backup_data_type, is_major_compaction_mview_dep_tablet, mview_dep_scn, sstable_array))) { LOG_WARN("failed to fetch tablet sstable array", K(ret), K(tablet_id), KPC(tablet_ref), K(backup_data_type)); } else if (OB_FAIL(add_prepare_tablet_item_(tablet_id))) { LOG_WARN("failed to prepare tablet item", K(ret), K(tablet_id)); @@ -2388,14 +2476,15 @@ int ObBackupTabletProvider::hold_tablet_handle_( int ObBackupTabletProvider::fetch_tablet_sstable_array_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle, const ObTabletTableStore &table_store, const share::ObBackupDataType &backup_data_type, const bool is_major_compaction_mview_dep_tablet, - common::ObIArray &sstable_array) + const share::SCN &mview_dep_scn, common::ObIArray &sstable_array) { int ret = OB_SUCCESS; sstable_array.reset(); if (!tablet_id.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid args", K(ret), K(tablet_id)); - } else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(tablet_handle, backup_data_type, table_store, is_major_compaction_mview_dep_tablet, sstable_array))) { + } else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(tablet_handle, backup_data_type, table_store, + is_major_compaction_mview_dep_tablet, mview_dep_scn, sstable_array))) { LOG_WARN("failed to get sstables by data type", K(ret), K(tablet_handle), K(backup_data_type), K(table_store)); } else { LOG_INFO("fetch tablet sstable array", K(ret), K(tablet_id), K(backup_data_type), K(sstable_array)); diff --git a/src/storage/backup/ob_backup_utils.h b/src/storage/backup/ob_backup_utils.h index 72ea540be..15f262572 100644 --- a/src/storage/backup/ob_backup_utils.h +++ b/src/storage/backup/ob_backup_utils.h @@ -55,7 +55,7 @@ class ObBackupMacroBlockIndexStore; class ObBackupUtils { public: static int get_sstables_by_data_type(const storage::ObTabletHandle &tablet_handle, const share::ObBackupDataType &backup_data_type, - const storage::ObTabletTableStore &table_store, const bool is_major_compaction_mview_dep_tablet, + const storage::ObTabletTableStore &table_store, const bool is_major_compaction_mview_dep_tablet, const share::SCN &mview_dep_scn, common::ObIArray &sstable_array); static int check_tablet_with_major_sstable(const storage::ObTabletHandle &tablet_handle, bool &with_major); static int fetch_macro_block_logic_id_list(const storage::ObTabletHandle &tablet_handle, @@ -85,6 +85,16 @@ private: const storage::ObTabletTableStore &tablet_table_store, common::ObIArray &sstable_array); static int fetch_major_sstables_(const storage::ObTabletHandle &tablet_handle, const storage::ObTabletTableStore &tablet_table_store, const bool is_major_compaction_mview_dep_tablet, + const share::SCN &mview_dep_scn, common::ObIArray &sstable_array); + static int check_and_filter_major_sstables_for_mview_( + const share::SCN &mview_dep_scn, + const storage::ObSSTableArray *major_sstable_array_ptr, + common::ObIArray &sstable_array); + static int check_major_sstables_for_mview_(const share::SCN &mview_dep_scn, + const storage::ObSSTableArray *major_sstable_array_ptr); + static int filter_major_sstables_for_mview_( + const share::SCN &mview_dep_scn, + const storage::ObSSTableArray *major_sstable_array_ptr, common::ObIArray &sstable_array); }; @@ -381,7 +391,7 @@ private: int hold_tablet_handle_(const common::ObTabletID &tablet_id, ObBackupTabletHandleRef *tablet_handle); int fetch_tablet_sstable_array_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle, const ObTabletTableStore &table_store, const share::ObBackupDataType &backup_data_type, - const bool is_major_compaction_mview_dep_tablet, common::ObIArray &sstable_array); + const bool is_major_compaction_mview_dep_tablet, const share::SCN &mview_dep_scn, common::ObIArray &sstable_array); int prepare_tablet_logic_id_reader_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle, const storage::ObITable::TableKey &table_key, const blocksstable::ObSSTable &sstable, ObITabletLogicMacroIdReader *&reader);