hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

optimize backup mview tablet process

Browse Source
This commit is contained in:
oceanoverflow 2024-12-17 09:44:41 +00:00 committed by ob-robot
parent e352718516
commit 95d786e47c
17 changed files with 203 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/rootserver/backup/ob_backup_data_set_task_mgr.cpp
View File

@ -633,6 +633,7 @@ int ObBackupSetTaskMgr::backup_meta_finish_()
}
}
}
DEBUG_SYNC(AFTER_BACKUP_META_FINISH);
return ret;
}
@ -841,6 +842,7 @@ int ObBackupSetTaskMgr::backup_major_compaction_mview_dep_tablet_list_()
int ret = OB_SUCCESS;
ObBackupMajorCompactionMViewDepTabletListDesc desc;
common::ObArray<common::ObTabletID> mview_tablet_list;
common::ObArray<share::SCN> mview_dep_scn_list;
share::SCN backup_scn;
if (OB_ISNULL(sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
@ -848,10 +850,12 @@ int ObBackupSetTaskMgr::backup_major_compaction_mview_dep_tablet_list_()
} else if (OB_FAIL(ObBackupDataScheduler::get_backup_scn(*sql_proxy_, set_task_attr_.tenant_id_, true/*is_backup_start*/, backup_scn))) {
LOG_WARN("failed to get backup scn", K(ret), K(set_task_attr_));
} else if (OB_FAIL(ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list(
*sql_proxy_, set_task_attr_.tenant_id_, backup_scn, mview_tablet_list))) {
*sql_proxy_, set_task_attr_.tenant_id_, backup_scn, mview_tablet_list, mview_dep_scn_list))) {
LOG_WARN("failed to get all major compaction mveiw dep tablet list", K(ret), K(backup_scn));
} else if (OB_FAIL(desc.tablet_id_list_.assign(mview_tablet_list))) {
LOG_WARN("failed to assign tablet list", K(ret));
} else if (OB_FAIL(desc.mview_dep_scn_list_.assign(mview_dep_scn_list))) {
LOG_WARN("failed to assign major scn list", K(ret));
} else if (OB_FAIL(store_.write_major_compaction_mview_dep_tablet_list(desc))) {
LOG_WARN("failed to write mview dep tablet list", K(ret));
}

7
src/rootserver/mview/ob_mview_push_refresh_scn_task.cpp
View File

@ -123,7 +123,6 @@ void ObMViewPushRefreshScnTask::runTimerTask()
share::ObGlobalStatProxy stat_proxy(trans, tenant_id_);
share::SCN major_refresh_mv_merge_scn;
ObArray<share::ObBackupJobAttr> backup_jobs;
uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_);
if (OB_FAIL(stat_proxy.get_major_refresh_mv_merge_scn(true /*select for update*/,
major_refresh_mv_merge_scn))) {
LOG_WARN("fail to get major_refresh_mv_merge_scn", KR(ret), K(tenant_id_));
@ -131,12 +130,6 @@ void ObMViewPushRefreshScnTask::runTimerTask()
ret = OB_ERR_UNEXPECTED;
LOG_WARN("major_refresh_mv_merge_scn is invalid", KR(ret), K(tenant_id_),
K(major_refresh_mv_merge_scn));
} else if (OB_FAIL(share::ObBackupJobOperator::get_jobs(
*sql_proxy, meta_tenant_id, false /*select for update*/, backup_jobs))) {
LOG_WARN("failed to get backup jobs", K(ret), K(tenant_id_));
} else if (!backup_jobs.empty()) {
LOG_INFO("[MAJ_REF_MV] backup jobs exist, skip push major refresh mv scn", KR(ret),
K(tenant_id_));
} else if (OB_FAIL(update_major_refresh_mview_scn_(tenant_id_, major_refresh_mv_merge_scn, trans))) {
LOG_WARN("fail to update major_refresh_mview_scn", KR(ret), K(tenant_id_), K(major_refresh_mv_merge_scn));
} else {

29
src/share/backup/ob_backup_data_table_operator.cpp
View File

@ -2437,13 +2437,16 @@ int ObBackupSkippedTabletOperator::move_skip_tablet_to_his(
return ret;
}
int ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list(common::ObMySQLProxy &proxy,
const uint64_t tenant_id,
const share::SCN &snapshot,
common::ObIArray<common::ObTabletID> &tablet_list)
int ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list(
common::ObMySQLProxy &proxy,
const uint64_t tenant_id,
const share::SCN &snapshot,
common::ObIArray<common::ObTabletID> &tablet_list,
common::ObIArray<share::SCN> &tablet_mview_dep_scn_list)
{
int ret = OB_SUCCESS;
tablet_list.reset();
tablet_mview_dep_scn_list.reset();
ObSqlString sql;
int64_t affected_rows = -1;
if (OB_INVALID_TENANT_ID == tenant_id || !snapshot.is_valid()) {
@ -2452,12 +2455,11 @@ int ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list(common
} else {
HEAP_VAR(ObMySQLProxy::ReadResult, res) {
ObMySQLResult *result = NULL;
if (OB_FAIL(sql.assign_fmt("select tablet_id from %s as of snapshot %ld where table_id in ("
" select p_obj from %s as of snapshot %ld"
" where mview_id in (select mview_id from %s as of snapshot %ld where refresh_mode=%ld) "
" union all "
" select data_table_id from %s as of snapshot %ld"
" where table_id in (select mview_id from %s as of snapshot %ld where refresh_mode=%ld))",
if (OB_FAIL(sql.assign_fmt("select t1.tablet_id tablet_id,t2.major_version major_version from %s as of snapshot %ld t1"
" join (select p_obj table_id,min(b.last_refresh_scn) major_version from %s as of snapshot %ld a"
" join %s as of snapshot %ld b on a.mview_id=b.mview_id where b.refresh_mode=%ld group by p_obj"
" union all (select a.data_table_id table_id,b.last_refresh_scn major_version from %s as of snapshot %ld a"
" join %s as of snapshot %ld b on a.table_id=b.mview_id and b.refresh_mode=%ld)) t2 on t1.table_id=t2.table_id",
OB_ALL_TABLET_TO_LS_TNAME, snapshot.get_val_for_inner_table_field(),
OB_ALL_MVIEW_DEP_TNAME, snapshot.get_val_for_inner_table_field(),
OB_ALL_MVIEW_TNAME, snapshot.get_val_for_inner_table_field(), schema::ObMVRefreshMode::MAJOR_COMPACTION,
@ -2480,9 +2482,16 @@ int ObBackupMViewOperator::get_all_major_compaction_mview_dep_tablet_list(common
}
} else {
int64_t tablet_id = -1;
uint64_t scn_val = -1;
share::SCN major_scn;
EXTRACT_INT_FIELD_MYSQL(*result, OB_STR_TABLET_ID, tablet_id, int64_t);
EXTRACT_UINT_FIELD_MYSQL(*result, "major_version", scn_val, uint64_t);
if (FAILEDx(tablet_list.push_back(ObTabletID(tablet_id)))) {
LOG_WARN("failed to push back", K(ret), K(tablet_id));
} else if (OB_FAIL(major_scn.convert_for_inner_table_field(scn_val))) {
LOG_WARN("failed to convert for inner table field", K(ret));
} else if (OB_FAIL(tablet_mview_dep_scn_list.push_back(major_scn))) {
LOG_WARN("failed to push back", K(ret), K(major_scn));
}
}
}

3
src/share/backup/ob_backup_data_table_operator.h
View File

@ -63,7 +63,8 @@ public:
static int get_all_major_compaction_mview_dep_tablet_list(common::ObMySQLProxy &proxy,
const uint64_t tenant_id,
const share::SCN &snapshot,
common::ObIArray<common::ObTabletID> &tablet_list);
common::ObIArray<common::ObTabletID> &tablet_list,
common::ObIArray<share::SCN> &tablet_mview_dep_scn_list);
};
class ObBackupSetFileOperator : public ObBackupBaseTableOperator

1
src/share/backup/ob_backup_struct.cpp
View File

@ -2094,6 +2094,7 @@ bool ObBackupUtils::is_need_retry_error(const int err)
case OB_CHECKSUM_ERROR :
case OB_VERSION_NOT_MATCH:
case OB_INVALID_DATA:
case OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE:
bret = false;
break;
default:

1
src/share/ob_debug_sync_point.h
View File

@ -396,6 +396,7 @@ class ObString;
ACT(BEFORE_REPLAY_DDL_COMMIT,)\
ACT(BEFORE_BACKUP_UESR_META,)\
ACT(BEFORE_BACKUP_META_FINISH,)\
ACT(AFTER_BACKUP_META_FINISH,)\
ACT(BEFORE_INSERT_UERR_RECOVER_TABLE_JOB,)\
ACT(BEFORE_GENERATE_IMPORT_TABLE_TASK,)\
ACT(BEFORE_RECOVER_UESR_RECOVER_TABLE_JOB,)\

15
src/share/ob_errno.cpp
View File

@ -27733,6 +27733,20 @@ static const _error _error_OB_TMP_FILE_EXCEED_DISK_QUOTA = {
.ob_str_error = "OBE-00600: internal error code, arguments: -9135, tmp file exceeds disk quota",
.ob_str_user_error = "OBE-00600: internal error code, arguments: -9135, tmp file exceeds disk quota"
};
static const _error _error_OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE = {
.error_name = "OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE",
.error_cause = "Internal Error",
.error_solution = "Contact OceanBase Support",
.mysql_errno = -1,
.sqlstate = "HY000",
.str_error = "Backup missing mview dependent tablet sstable, sstable maybe recycled",
.str_user_error = "Backup missing mview dependent tablet sstable, sstable maybe recycled",
.oracle_errno = 600,
.oracle_str_error = "ORA-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled",
.oracle_str_user_error = "ORA-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled",
.ob_str_error = "OBE-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled",
.ob_str_user_error = "OBE-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled"
};
static const _error _error_OB_ERR_RESIZE_FILE_TO_SMALLER = {
.error_name = "OB_ERR_RESIZE_FILE_TO_SMALLER",
.error_cause = "Internal Error",
@ -35515,6 +35529,7 @@ struct ObStrErrorInit
_errors[-OB_BACKUP_ZONE_IDC_REGION_INVALID] = &_error_OB_BACKUP_ZONE_IDC_REGION_INVALID;
_errors[-OB_ERR_TMP_FILE_ALREADY_SEALED] = &_error_OB_ERR_TMP_FILE_ALREADY_SEALED;
_errors[-OB_TMP_FILE_EXCEED_DISK_QUOTA] = &_error_OB_TMP_FILE_EXCEED_DISK_QUOTA;
_errors[-OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE] = &_error_OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE;
_errors[-OB_ERR_RESIZE_FILE_TO_SMALLER] = &_error_OB_ERR_RESIZE_FILE_TO_SMALLER;
_errors[-OB_MARK_BLOCK_INFO_TIMEOUT] = &_error_OB_MARK_BLOCK_INFO_TIMEOUT;
_errors[-OB_NOT_READY_TO_EXTEND_FILE] = &_error_OB_NOT_READY_TO_EXTEND_FILE;

1
src/share/ob_errno.def
View File

@ -2351,6 +2351,7 @@ DEFINE_ERROR_EXT_DEP(OB_BACKUP_ZONE_IDC_REGION_INVALID, -9133, -1, "HY000", "bac
DEFINE_ERROR_DEP(OB_ERR_TMP_FILE_ALREADY_SEALED, -9134, -1, "HY000", "tmp file has already sealed");
DEFINE_ERROR_DEP(OB_TMP_FILE_EXCEED_DISK_QUOTA, -9135, -1, "HY000", "tmp file exceeds disk quota");
DEFINE_ERROR(OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE, -9136, -1, "HY000", "Backup missing mview dependent tablet sstable, sstable maybe recycled");
// 余留位置，在该行之前继续添加storage错误码
////////////////////////////////////////////////////////////////

4
src/share/ob_errno.h
View File

@ -1592,6 +1592,7 @@ constexpr int OB_STORAGE_DEST_NOT_CONNECT = -9115;
constexpr int OB_TABLET_IS_SPLIT_SRC = -9123;
constexpr int OB_TABLET_STATUS_NO_NEED_TO_SPLIT = -9127;
constexpr int OB_FILE_DELETE_FAILED = -9128;
constexpr int OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE = -9136;
constexpr int OB_ERR_RESIZE_FILE_TO_SMALLER = -9200;
constexpr int OB_MARK_BLOCK_INFO_TIMEOUT = -9201;
constexpr int OB_NOT_READY_TO_EXTEND_FILE = -9202;
@ -3947,6 +3948,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219;
#define OB_BACKUP_ZONE_IDC_REGION_INVALID__USER_ERROR_MSG "%s"
#define OB_ERR_TMP_FILE_ALREADY_SEALED__USER_ERROR_MSG "tmp file has already sealed"
#define OB_TMP_FILE_EXCEED_DISK_QUOTA__USER_ERROR_MSG "tmp file exceeds disk quota"
#define OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE__USER_ERROR_MSG "Backup missing mview dependent tablet sstable, sstable maybe recycled"
#define OB_ERR_RESIZE_FILE_TO_SMALLER__USER_ERROR_MSG "Extend ssblock file to smaller is not allowed"
#define OB_MARK_BLOCK_INFO_TIMEOUT__USER_ERROR_MSG "Mark blocks timeout(5s) in auto extend process when alloc block fail"
#define OB_NOT_READY_TO_EXTEND_FILE__USER_ERROR_MSG "Auto extend param is not ready to start extending file"
@ -8321,6 +8323,8 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219;
#define OB_ERR_TMP_FILE_ALREADY_SEALED__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9134, tmp file has already sealed"
#define OB_TMP_FILE_EXCEED_DISK_QUOTA__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9135, tmp file exceeds disk quota"
#define OB_TMP_FILE_EXCEED_DISK_QUOTA__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9135, tmp file exceeds disk quota"
#define OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled"
#define OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9136, Backup missing mview dependent tablet sstable, sstable maybe recycled"
#define OB_ERR_RESIZE_FILE_TO_SMALLER__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9200, Extend ssblock file to smaller is not allowed"
#define OB_ERR_RESIZE_FILE_TO_SMALLER__OBE_USER_ERROR_MSG "OBE-00600: internal error code, arguments: -9200, Extend ssblock file to smaller is not allowed"
#define OB_MARK_BLOCK_INFO_TIMEOUT__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9201, Mark blocks timeout(5s) in auto extend process when alloc block fail"

31
src/storage/backup/ob_backup_ctx.cpp
View File

@ -1235,7 +1235,7 @@ ObLSBackupCtx::ObLSBackupCtx()
backup_tx_table_filled_tx_scn_(share::SCN::min_scn()),
index_builder_mgr_(),
bandwidth_throttle_(NULL),
mview_dep_tablet_set_(),
mview_dep_tablet_map_(),
wait_reuse_across_sstable_time_(0),
mv_mutex_()
{}
@ -1260,6 +1260,7 @@ int ObLSBackupCtx::open(
int ret = OB_SUCCESS;
ObArray<common::ObTabletID> tablet_list;
ObILSTabletIdReader *reader = NULL;
ObMemAttr attr(MTL_ID(), ObModIds::BACKUP);
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("init twice", K(ret));
@ -1279,8 +1280,8 @@ int ObLSBackupCtx::open(
LOG_WARN("failed to init index builder mg", K(ret), K(param));
} else if (OB_FAIL(param_.assign(param))) {
LOG_WARN("failed to assign param", K(ret), K(param));
} else if (OB_FAIL(mview_dep_tablet_set_.create(DEFAULT_MVIEW_DEP_TABLET_SET_SIZE))) {
LOG_WARN("failed to create mveiw dep tablet set", K(ret));
} else if (OB_FAIL(mview_dep_tablet_map_.create(DEFAULT_MVIEW_DEP_TABLET_SET_SIZE, attr))) {
LOG_WARN("failed to create mview dep tablet map", K(ret));
} else {
max_file_id_ = 0;
prefetch_task_id_ = 0;
@ -1519,7 +1520,8 @@ int ObLSBackupCtx::finish_task(const int64_t file_id)
return ret;
}
int ObLSBackupCtx::check_is_major_compaction_mview_dep_tablet(const common::ObTabletID &tablet_id, bool &is_dep) const
int ObLSBackupCtx::check_is_major_compaction_mview_dep_tablet(
const common::ObTabletID &tablet_id, share::SCN &mview_dep_scn, bool &is_dep) const
{
int ret = OB_SUCCESS;
is_dep = false;
@ -1527,12 +1529,11 @@ int ObLSBackupCtx::check_is_major_compaction_mview_dep_tablet(const common::ObTa
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(tablet_id));
} else {
int hash_ret = mview_dep_tablet_set_.exist_refactored(tablet_id);
if (OB_HASH_EXIST == hash_ret) {
is_dep = true;
LOG_INFO("is mview dependent tablet", K(tablet_id));
} else if (OB_HASH_NOT_EXIST == hash_ret) {
int hash_ret = mview_dep_tablet_map_.get_refactored(tablet_id, mview_dep_scn);
if (OB_HASH_NOT_EXIST == hash_ret) {
is_dep = false;
} else if (OB_SUCCESS == hash_ret) {
is_dep = true;
} else {
ret = hash_ret;
LOG_WARN("failed to check is mview dep tablet", K(ret), K(tablet_id));
@ -1642,11 +1643,17 @@ int ObLSBackupCtx::prepare_mview_dep_tablet_set_(const ObLSBackupParam &param,
LOG_WARN("failed to read mview dep tablet list", K(ret));
} else {
const common::ObSArray<common::ObTabletID> &mview_tablet_list = desc.tablet_id_list_;;
const ObSArray<share::SCN> &tablet_mview_dep_scn_list = desc.mview_dep_scn_list_;
if (mview_tablet_list.count() != tablet_mview_dep_scn_list.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("mview tablet list not equal to tablet version list", K(ret));
}
ARRAY_FOREACH_X(mview_tablet_list, idx, cnt, OB_SUCC(ret)) {
const ObTabletID &tablet_id = mview_tablet_list.at(idx);
const SCN &tablet_version = tablet_mview_dep_scn_list.at(idx);
const int flag = 1;
if (OB_FAIL(mview_dep_tablet_set_.set_refactored(tablet_id, flag))) {
LOG_WARN("failed to set refactored", K(ret), K(tablet_id));
if (OB_FAIL(mview_dep_tablet_map_.set_refactored(tablet_id, tablet_version, flag))) {
LOG_WARN("failed to set refactored", K(ret), K(tablet_id), K(tablet_version));
}
}
}
@ -1662,7 +1669,7 @@ int ObLSBackupCtx::check_mview_tablet_set_(const uint64_t tenant_id, common::ObM
LOG_WARN("failed to get tenant info", K(ret), K(tenant_id));
} else if (tenant_info.is_primary()) {
LOG_INFO("tenant is primary", K(tenant_id), K(tenant_info));
} else if (!mview_dep_tablet_set_.empty()) {
} else if (!mview_dep_tablet_map_.empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tenant not primary but mview dep tablet is not empty", K(tenant_id), K(tenant_info));
}

4
src/storage/backup/ob_backup_ctx.h
View File

@ -300,7 +300,7 @@ public:
return tablet_holder_;
}
int check_is_major_compaction_mview_dep_tablet(const common::ObTabletID &tablet_id, bool &is_major_compaction_mview_dep_tablet) const;
int check_is_major_compaction_mview_dep_tablet(const common::ObTabletID &tablet_id, share::SCN &mview_dep_scn, bool &is_major_compaction_mview_dep_tablet) const;
void add_wait_reuse_across_sstable_time(int64_t cost_time);
private:
@ -346,7 +346,7 @@ public:
share::SCN backup_tx_table_filled_tx_scn_;
ObBackupTabletIndexBlockBuilderMgr index_builder_mgr_;
common::ObInOutBandwidthThrottle *bandwidth_throttle_;
common::hash::ObHashSet<common::ObTabletID> mview_dep_tablet_set_;
common::hash::ObHashMap<common::ObTabletID, share::SCN> mview_dep_tablet_map_;
int64_t wait_reuse_across_sstable_time_;
mutable lib::ObMutex mv_mutex_;
DISALLOW_COPY_AND_ASSIGN(ObLSBackupCtx);

19
src/storage/backup/ob_backup_data_store.cpp
View File

@ -339,21 +339,26 @@ bool ObBackupTableListMetaInfoDesc::is_valid() const
*-----------------------------ObBackupMajorCompactionMViewDepTabletListDesc-----------------------
*/
OB_SERIALIZE_MEMBER(ObBackupMajorCompactionMViewDepTabletListDesc, tablet_id_list_);
OB_SERIALIZE_MEMBER(ObBackupMajorCompactionMViewDepTabletListDesc, tablet_id_list_, mview_dep_scn_list_);
ObBackupMajorCompactionMViewDepTabletListDesc::ObBackupMajorCompactionMViewDepTabletListDesc()
: ObExternBackupDataDesc(ObBackupFileType::BACKUP_MVIEW_DEP_TABLET_LIST_FILE, FILE_VERSION),
tablet_id_list_() {}
tablet_id_list_(),
mview_dep_scn_list_() {}
bool ObBackupMajorCompactionMViewDepTabletListDesc::is_valid() const
{
int ret = OB_SUCCESS;
bool bret = true;
ARRAY_FOREACH(tablet_id_list_, i) {
const ObTabletID &tablet_id = tablet_id_list_.at(i);
if (!tablet_id.is_valid()) {
bret = false;
break;
if (tablet_id_list_.count() != mview_dep_scn_list_.count()) {
bret = false;
} else {
ARRAY_FOREACH(tablet_id_list_, i) {
const ObTabletID &tablet_id = tablet_id_list_.at(i);
if (!tablet_id.is_valid()) {
bret = false;
break;
}
}
}
return bret;

3
src/storage/backup/ob_backup_data_store.h
View File

@ -346,9 +346,10 @@ public:
ObBackupMajorCompactionMViewDepTabletListDesc();
~ObBackupMajorCompactionMViewDepTabletListDesc() {}
bool is_valid() const override;
TO_STRING_KV(K_(tablet_id_list));
TO_STRING_KV(K_(tablet_id_list), K_(mview_dep_scn_list));
public:
ObSArray<common::ObTabletID> tablet_id_list_;
ObSArray<share::SCN> mview_dep_scn_list_;
private:
DISALLOW_COPY_AND_ASSIGN(ObBackupMajorCompactionMViewDepTabletListDesc);
};

5
src/storage/backup/ob_backup_reader.cpp
View File

@ -670,12 +670,13 @@ int ObSSTableMetaBackupReader::init(const common::ObTabletID &tablet_id,
linked_writer_ = linked_writer;
ObTablet &tablet = *tablet_handle_->get_obj();
bool is_major_compaction_mview_dep = false;
share::SCN mview_dep_scn;
if (OB_FAIL(tablet.fetch_table_store(table_store_wrapper_))) {
LOG_WARN("failed to fetch table store from tablet", K(ret));
} else if (OB_FAIL(ls_backup_ctx.check_is_major_compaction_mview_dep_tablet(tablet_id, is_major_compaction_mview_dep))) {
} else if (OB_FAIL(ls_backup_ctx.check_is_major_compaction_mview_dep_tablet(tablet_id, mview_dep_scn, is_major_compaction_mview_dep))) {
LOG_WARN("failed to check is mview dep tablet", K(ret), K(tablet_id));
} else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(
tablet_handle, backup_data_type, *table_store_wrapper_.get_member(), is_major_compaction_mview_dep, sstable_array_))) {
tablet_handle, backup_data_type, *table_store_wrapper_.get_member(), is_major_compaction_mview_dep, mview_dep_scn, sstable_array_))) {
LOG_WARN("failed to get sstables by data type", K(ret), K(tablet_handle));
} else {
builder_mgr_ = &index_block_builder_mgr;

11
src/storage/backup/ob_backup_task.cpp
View File

@ -2684,6 +2684,7 @@ int ObLSBackupDataTask::do_prepare_sstable_builders_(const ObBackupProviderItem
ObBackupTabletHandleRef *tablet_ref = NULL;
ObArray<storage::ObSSTableWrapper> sstable_array;
bool is_major_compaction_mview_dep_tablet = false;
share::SCN mview_dep_scn;
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
if (!item.is_valid()) {
ret = OB_INVALID_ARGUMENT;
@ -2695,9 +2696,10 @@ int ObLSBackupDataTask::do_prepare_sstable_builders_(const ObBackupProviderItem
LOG_WARN("failed to get tablet handle", K(ret), K(tablet_id));
} else if (OB_FAIL(tablet_ref->tablet_handle_.get_obj()->fetch_table_store(table_store_wrapper))) {
LOG_WARN("fail to fetch table store", K(ret), K(tablet_id));
} else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet_id, is_major_compaction_mview_dep_tablet))) {
} else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet_id, mview_dep_scn, is_major_compaction_mview_dep_tablet))) {
LOG_WARN("failed to check is mview dep tablet", K(ret), K(tablet_id));
} else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(tablet_ref->tablet_handle_, backup_data_type_, *table_store_wrapper.get_member(), is_major_compaction_mview_dep_tablet, sstable_array))) {
} else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(tablet_ref->tablet_handle_, backup_data_type_,
*table_store_wrapper.get_member(), is_major_compaction_mview_dep_tablet, mview_dep_scn, sstable_array))) {
LOG_WARN("failed to get sstables by data type", K(ret), KPC(tablet_ref), K_(backup_data_type));
} else if (OB_FAIL(prepare_tablet_sstable_index_builders_(tablet_id, is_major_compaction_mview_dep_tablet, sstable_array))) {
LOG_WARN("failed to prepare tablet sstable index builders", K(ret), K_(param), K(tablet_id), K(sstable_array));
@ -3156,6 +3158,7 @@ int ObLSBackupDataTask::do_wait_sstable_index_builder_ready_(ObTabletHandle &tab
int ret = OB_SUCCESS;
ObTablet *tablet = nullptr;
bool is_major_compaction_mview_dep = false;
share::SCN mview_dep_scn;
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
common::ObArray<storage::ObSSTableWrapper> sstable_array;
const ObTabletTableStore *table_store = nullptr;
@ -3166,10 +3169,10 @@ int ObLSBackupDataTask::do_wait_sstable_index_builder_ready_(ObTabletHandle &tab
LOG_WARN("fail to fetch table store", K(ret));
} else if (OB_FAIL(table_store_wrapper.get_member(table_store))) {
LOG_WARN("failed to get table store", K(ret), KPC(tablet));
} else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet->get_tablet_id(), is_major_compaction_mview_dep))) {
} else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet->get_tablet_id(), mview_dep_scn, is_major_compaction_mview_dep))) {
LOG_WARN("failed to check is mview dep tablet", K(ret), KPC(tablet));
} else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(
tablet_handle, backup_data_type_, *table_store_wrapper.get_member(), is_major_compaction_mview_dep, sstable_array))) {
tablet_handle, backup_data_type_, *table_store_wrapper.get_member(), is_major_compaction_mview_dep, mview_dep_scn, sstable_array))) {
LOG_WARN("failed to get sstables by data type", K(ret), K(tablet_handle));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < sstable_array.count(); ++i) {

107
src/storage/backup/ob_backup_utils.cpp
View File

@ -84,7 +84,8 @@ int ObBackupUtils::calc_start_replay_scn(
}
int ObBackupUtils::get_sstables_by_data_type(const storage::ObTabletHandle &tablet_handle, const share::ObBackupDataType &backup_data_type,
const storage::ObTabletTableStore &tablet_table_store, const bool is_major_compaction_mview_dep_tablet, common::ObIArray<storage::ObSSTableWrapper> &sstable_array)
const storage::ObTabletTableStore &tablet_table_store, const bool is_major_compaction_mview_dep_tablet, const share::SCN &mview_dep_scn,
common::ObIArray<storage::ObSSTableWrapper> &sstable_array)
{
int ret = OB_SUCCESS;
sstable_array.reset();
@ -98,7 +99,7 @@ int ObBackupUtils::get_sstables_by_data_type(const storage::ObTabletHandle &tabl
} else {
if (OB_FAIL(fetch_minor_and_ddl_sstables_(tablet_handle, tablet_table_store, sstable_array))) {
LOG_WARN("failed to fetch minor and ddl sstables", K(ret), K(tablet_handle), K(tablet_table_store));
} else if (OB_FAIL(fetch_major_sstables_(tablet_handle, tablet_table_store, is_major_compaction_mview_dep_tablet, sstable_array))) {
} else if (OB_FAIL(fetch_major_sstables_(tablet_handle, tablet_table_store, is_major_compaction_mview_dep_tablet, mview_dep_scn, sstable_array))) {
LOG_WARN("failed to fetch major sstables", K(ret), K(tablet_handle), K(tablet_table_store));
}
}
@ -164,7 +165,7 @@ int ObBackupUtils::fetch_minor_and_ddl_sstables_(const storage::ObTabletHandle &
int ObBackupUtils::fetch_major_sstables_(const storage::ObTabletHandle &tablet_handle,
const storage::ObTabletTableStore &tablet_table_store, const bool is_major_compaction_mview_dep_tablet,
common::ObIArray<ObSSTableWrapper> &sstable_array)
const share::SCN &mview_dep_scn, common::ObIArray<ObSSTableWrapper> &sstable_array)
{
int ret = OB_SUCCESS;
const storage::ObSSTableArray *major_sstable_array_ptr = NULL;
@ -190,8 +191,8 @@ int ObBackupUtils::fetch_major_sstables_(const storage::ObTabletHandle &tablet_h
LOG_WARN("medium list is invalid for last major sstable", K(ret), KPC(medium_info_list),
KPC(last_major_sstable_ptr), K(tablet_handle));
} else if (is_major_compaction_mview_dep_tablet) {
if (OB_FAIL(major_sstable_array_ptr->get_all_table_wrappers(sstable_array))) {
LOG_WARN("failed to get all table wrappers", K(ret));
if (OB_FAIL(check_and_filter_major_sstables_for_mview_(mview_dep_scn, major_sstable_array_ptr, sstable_array))) {
LOG_WARN("failed to check and filter major sstable for mview", K(ret));
}
} else { // not mview dep tablet
if (last_major_sstable_ptr->is_co_sstable()) {
@ -209,6 +210,92 @@ int ObBackupUtils::fetch_major_sstables_(const storage::ObTabletHandle &tablet_h
return ret;
}
int ObBackupUtils::check_and_filter_major_sstables_for_mview_(
const share::SCN &mview_dep_scn,
const storage::ObSSTableArray *major_sstable_array_ptr,
common::ObIArray<storage::ObSSTableWrapper> &sstable_array)
{
int ret = OB_SUCCESS;
if (OB_FAIL(check_major_sstables_for_mview_(mview_dep_scn, major_sstable_array_ptr))) {
LOG_WARN("failed to check major sstable for mview", K(ret));
} else if (OB_FAIL(filter_major_sstables_for_mview_(mview_dep_scn, major_sstable_array_ptr, sstable_array))) {
LOG_WARN("failed to filter major sstables for mview", K(ret));
} else {
LOG_INFO("check and filter major sstable for mview", K(sstable_array));
}
return ret;
}
int ObBackupUtils::check_major_sstables_for_mview_(const share::SCN &mview_dep_scn,
const storage::ObSSTableArray *major_sstable_array_ptr)
{
int ret = OB_SUCCESS;
common::ObArray<ObSSTableWrapper> tmp_sstable_array;
if (OB_ISNULL(major_sstable_array_ptr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("major sstable array ptr should not be null", K(ret));
} else if (OB_FAIL(major_sstable_array_ptr->get_all_table_wrappers(tmp_sstable_array))) {
LOG_WARN("failed to get all table wrappers", K(ret), KPC(major_sstable_array_ptr));
} else {
bool exist = false;
ARRAY_FOREACH_X(tmp_sstable_array, idx, cnt, OB_SUCC(ret)) {
storage::ObSSTableWrapper &sstable_wrapper = tmp_sstable_array.at(idx);
ObSSTable *sstable_ptr = NULL;
if (OB_ISNULL(sstable_ptr = sstable_wrapper.get_sstable())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table should not be null", K(ret));
} else {
const ObITable::TableKey &table_key = sstable_ptr->get_key();
if (table_key.get_end_scn() == mview_dep_scn) {
exist = true;
break;
}
}
}
if (OB_SUCC(ret) && !exist) {
ret = OB_BACKUP_MISSING_MVIEW_DEP_TABLET_SSTABLE;
LOG_WARN("missing sstable for mview tablet", K(ret), K(mview_dep_scn), K(tmp_sstable_array));
}
}
return ret;
}
int ObBackupUtils::filter_major_sstables_for_mview_(
const share::SCN &mview_dep_scn,
const storage::ObSSTableArray *major_sstable_array_ptr,
common::ObIArray<storage::ObSSTableWrapper> &sstable_array)
{
int ret = OB_SUCCESS;
common::ObArray<ObSSTableWrapper> tmp_sstable_array;
if (OB_ISNULL(major_sstable_array_ptr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("major sstable array ptr should not be null", K(ret));
} else if (OB_FAIL(major_sstable_array_ptr->get_all_table_wrappers(tmp_sstable_array))) {
LOG_WARN("failed to get all table wrappers", K(ret));
} else {
ARRAY_FOREACH_X(tmp_sstable_array, idx, cnt, OB_SUCC(ret)) {
storage::ObSSTableWrapper &sstable_wrapper = tmp_sstable_array.at(idx);
ObSSTable *sstable_ptr = NULL;
if (OB_ISNULL(sstable_ptr = sstable_wrapper.get_sstable())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table should not be null", K(ret));
} else {
const ObITable::TableKey &table_key = sstable_ptr->get_key();
if (table_key.get_end_scn() < mview_dep_scn) {
// skip
} else if (OB_FAIL(sstable_array.push_back(sstable_wrapper))) {
LOG_WARN("failed to push back", K(ret), K(sstable_wrapper));
}
}
}
if (OB_SUCC(ret) && sstable_array.empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sstable array should not be empty", K(ret), K(mview_dep_scn), KPC(major_sstable_array_ptr));
}
}
return ret;
}
int ObBackupUtils::check_tablet_with_major_sstable(const storage::ObTabletHandle &tablet_handle, bool &with_major)
{
int ret = OB_SUCCESS;
@ -1941,6 +2028,7 @@ int ObBackupTabletProvider::prepare_tablet_(const uint64_t tenant_id, const shar
ObTabletMemberWrapper<ObTabletTableStore> table_store_wrapper;
bool need_skip_tablet = false;
bool is_major_compaction_mview_dep_tablet = false;
share::SCN mview_dep_scn;
bool is_split_dst = false;
if (OB_ISNULL(ls_backup_ctx_)) {
ret = OB_ERR_UNEXPECTED;
@ -1985,10 +2073,10 @@ int ObBackupTabletProvider::prepare_tablet_(const uint64_t tenant_id, const shar
LOG_WARN("failed to check tablet replica validity", K(ret), K(tenant_id), K(ls_id), K(tablet_id), K(backup_data_type));
} else if (OB_FAIL(tablet_ref->tablet_handle_.get_obj()->fetch_table_store(table_store_wrapper))) {
LOG_WARN("fail to fetch table store", K(ret));
} else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet_id, is_major_compaction_mview_dep_tablet))) {
} else if (OB_FAIL(ls_backup_ctx_->check_is_major_compaction_mview_dep_tablet(tablet_id, mview_dep_scn, is_major_compaction_mview_dep_tablet))) {
LOG_WARN("failed to check is mview dep tablet", K(ret));
} else if (OB_FAIL(fetch_tablet_sstable_array_(
tablet_id, tablet_ref->tablet_handle_, *table_store_wrapper.get_member(), backup_data_type, is_major_compaction_mview_dep_tablet, sstable_array))) {
tablet_id, tablet_ref->tablet_handle_, *table_store_wrapper.get_member(), backup_data_type, is_major_compaction_mview_dep_tablet, mview_dep_scn, sstable_array))) {
LOG_WARN("failed to fetch tablet sstable array", K(ret), K(tablet_id), KPC(tablet_ref), K(backup_data_type));
} else if (OB_FAIL(add_prepare_tablet_item_(tablet_id))) {
LOG_WARN("failed to prepare tablet item", K(ret), K(tablet_id));
@ -2388,14 +2476,15 @@ int ObBackupTabletProvider::hold_tablet_handle_(
int ObBackupTabletProvider::fetch_tablet_sstable_array_(const common::ObTabletID &tablet_id,
const storage::ObTabletHandle &tablet_handle, const ObTabletTableStore &table_store,
const share::ObBackupDataType &backup_data_type, const bool is_major_compaction_mview_dep_tablet,
common::ObIArray<storage::ObSSTableWrapper> &sstable_array)
const share::SCN &mview_dep_scn, common::ObIArray<storage::ObSSTableWrapper> &sstable_array)
{
int ret = OB_SUCCESS;
sstable_array.reset();
if (!tablet_id.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(tablet_id));
} else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(tablet_handle, backup_data_type, table_store, is_major_compaction_mview_dep_tablet, sstable_array))) {
} else if (OB_FAIL(ObBackupUtils::get_sstables_by_data_type(tablet_handle, backup_data_type, table_store,
is_major_compaction_mview_dep_tablet, mview_dep_scn, sstable_array))) {
LOG_WARN("failed to get sstables by data type", K(ret), K(tablet_handle), K(backup_data_type), K(table_store));
} else {
LOG_INFO("fetch tablet sstable array", K(ret), K(tablet_id), K(backup_data_type), K(sstable_array));

14
src/storage/backup/ob_backup_utils.h
View File

@ -55,7 +55,7 @@ class ObBackupMacroBlockIndexStore;
class ObBackupUtils {
public:
static int get_sstables_by_data_type(const storage::ObTabletHandle &tablet_handle, const share::ObBackupDataType &backup_data_type,
const storage::ObTabletTableStore &table_store, const bool is_major_compaction_mview_dep_tablet,
const storage::ObTabletTableStore &table_store, const bool is_major_compaction_mview_dep_tablet, const share::SCN &mview_dep_scn,
common::ObIArray<storage::ObSSTableWrapper> &sstable_array);
static int check_tablet_with_major_sstable(const storage::ObTabletHandle &tablet_handle, bool &with_major);
static int fetch_macro_block_logic_id_list(const storage::ObTabletHandle &tablet_handle,
@ -85,6 +85,16 @@ private:
const storage::ObTabletTableStore &tablet_table_store, common::ObIArray<storage::ObSSTableWrapper> &sstable_array);
static int fetch_major_sstables_(const storage::ObTabletHandle &tablet_handle,
const storage::ObTabletTableStore &tablet_table_store, const bool is_major_compaction_mview_dep_tablet,
const share::SCN &mview_dep_scn, common::ObIArray<storage::ObSSTableWrapper> &sstable_array);
static int check_and_filter_major_sstables_for_mview_(
const share::SCN &mview_dep_scn,
const storage::ObSSTableArray *major_sstable_array_ptr,
common::ObIArray<storage::ObSSTableWrapper> &sstable_array);
static int check_major_sstables_for_mview_(const share::SCN &mview_dep_scn,
const storage::ObSSTableArray *major_sstable_array_ptr);
static int filter_major_sstables_for_mview_(
const share::SCN &mview_dep_scn,
const storage::ObSSTableArray *major_sstable_array_ptr,
common::ObIArray<storage::ObSSTableWrapper> &sstable_array);
};
@ -381,7 +391,7 @@ private:
int hold_tablet_handle_(const common::ObTabletID &tablet_id, ObBackupTabletHandleRef *tablet_handle);
int fetch_tablet_sstable_array_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle,
const ObTabletTableStore &table_store, const share::ObBackupDataType &backup_data_type,
const bool is_major_compaction_mview_dep_tablet, common::ObIArray<storage::ObSSTableWrapper> &sstable_array);
const bool is_major_compaction_mview_dep_tablet, const share::SCN &mview_dep_scn, common::ObIArray<storage::ObSSTableWrapper> &sstable_array);
int prepare_tablet_logic_id_reader_(const common::ObTabletID &tablet_id, const storage::ObTabletHandle &tablet_handle,
const storage::ObITable::TableKey &table_key, const blocksstable::ObSSTable &sstable,
ObITabletLogicMacroIdReader *&reader);