diff --git a/src/share/transfer/ob_transfer_info.cpp b/src/share/transfer/ob_transfer_info.cpp index fa87dc2ef4..44c4959387 100644 --- a/src/share/transfer/ob_transfer_info.cpp +++ b/src/share/transfer/ob_transfer_info.cpp @@ -752,6 +752,28 @@ int ObTransferTaskInfo::assign(const ObTransferTaskInfo &task_info) return ret; } +int ObTransferTaskInfo::fill_tablet_ids(ObIArray &tablet_ids) const +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!tablet_ids.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument, tablet_ids should be empty", K(ret)); + } else if (OB_FAIL(tablet_ids.reserve(tablet_list_.count()))) { + LOG_WARN("failed to reserve tablet_ids", K(ret)); + } else { + FOREACH_X(it, tablet_list_, OB_SUCC(ret)) { + const ObTabletID &tablet_id = it->tablet_id(); + if (OB_UNLIKELY(!tablet_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tablet id", K(ret), K(tablet_id)); + } else if (OB_FAIL(tablet_ids.push_back(tablet_id))) { + LOG_WARN("failed to push back tablet_id", K(ret), K(tablet_id)); + } + } + } + return ret; +} + int ObTransferLockUtil::lock_tablet_on_dest_ls_for_table_lock( ObMySQLTransaction &trans, const uint64_t tenant_id, diff --git a/src/share/transfer/ob_transfer_info.h b/src/share/transfer/ob_transfer_info.h index 5210d7344d..a9bab01aa7 100644 --- a/src/share/transfer/ob_transfer_info.h +++ b/src/share/transfer/ob_transfer_info.h @@ -397,7 +397,7 @@ struct ObTransferTaskInfo final bool is_valid() const; int convert_from(const uint64_t tenant_id, const ObTransferTask &task); int assign(const ObTransferTaskInfo &task_info); - + int fill_tablet_ids(ObIArray &tablet_ids) const; TO_STRING_KV(K_(tenant_id), K_(src_ls_id), K_(dest_ls_id), K_(task_id), K_(trace_id), K_(status), K_(table_lock_owner_id), K_(table_lock_tablet_list), K_(tablet_list), K_(start_scn), K_(finish_scn), K_(result)); diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp index 3a625cbd08..b239b0ea5e 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp @@ -615,7 +615,7 @@ const char *ObProhibitScheduleMediumMap::ProhibitFlagStr[] = { ObProhibitScheduleMediumMap::ObProhibitScheduleMediumMap() : transfer_flag_cnt_(0), lock_(), - ls_id_map_() + tablet_id_map_() { STATIC_ASSERT(static_cast(ProhibitFlag::FLAG_MAX) == ARRAYSIZEOF(ProhibitFlagStr), "flag str len is mismatch"); } @@ -623,73 +623,87 @@ ObProhibitScheduleMediumMap::ObProhibitScheduleMediumMap() int ObProhibitScheduleMediumMap::init() { int ret = OB_SUCCESS; - if (OB_FAIL(ls_id_map_.create(OB_MAX_LS_NUM_PER_TENANT_PER_SERVER, "MediumMap", "MediumMap", MTL_ID()))) { - LOG_WARN("Fail to create prohibit medium ls id map", K(ret)); + obsys::ObWLockGuard lock_guard(lock_); + if (OB_FAIL(tablet_id_map_.create(TABLET_ID_MAP_BUCKET_NUM, "MediumTabletMap", "MediumTabletMap", MTL_ID()))) { + LOG_WARN("Fail to create prohibit medium tablet id map", K(ret)); } return ret; } -int ObProhibitScheduleMediumMap::add_flag(const ObLSID &ls_id, const ProhibitFlag &input_flag) +int ObProhibitScheduleMediumMap::add_flag(const ObTabletID &tablet_id, const ProhibitFlag &input_flag) { int ret = OB_SUCCESS; - ProhibitMediumStatus tmp_status(ProhibitFlag::FLAG_MAX); - ProhibitMediumStatus input_status(input_flag); - if (OB_UNLIKELY(!ls_id.is_valid() || !input_status.is_valid())) { + ProhibitFlag tmp_flag = ProhibitFlag::FLAG_MAX; + if (OB_UNLIKELY(!tablet_id.is_valid() || !is_valid_flag(input_flag))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(input_flag)); + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(input_flag)); } else { obsys::ObWLockGuard lock_guard(lock_); - if (OB_FAIL(ls_id_map_.get_refactored(ls_id, tmp_status))) { - if (OB_HASH_NOT_EXIST == ret) { - if (OB_FAIL(ls_id_map_.set_refactored(ls_id, input_status))) { - LOG_WARN("failed to stop ls schedule medium", K(ret), K(ls_id), K(input_flag)); - } else if (ProhibitFlag::TRANSFER == input_flag) { - ++transfer_flag_cnt_; - } + if (OB_FAIL(tablet_id_map_.get_refactored(tablet_id, tmp_flag))) { + if (OB_HASH_NOT_EXIST == ret && OB_FAIL(tablet_id_map_.set_refactored(tablet_id, input_flag))) { + LOG_WARN("failed to stop tablet schedule medium", K(ret), K(tablet_id), K(input_flag)); } else { - LOG_WARN("failed to get map", K(ret), K(ls_id), K(tmp_status)); + LOG_WARN("failed to get map", K(ret), K(tablet_id), K(tmp_flag)); } - } else if (tmp_status.flag_ != input_flag) { + } else if (tmp_flag != input_flag) { ret = OB_EAGAIN; if (REACH_TENANT_TIME_INTERVAL(PRINT_LOG_INVERVAL)) { - LOG_INFO("flag in conflict", K(ret), K(ls_id), K(tmp_status), K(input_flag)); + LOG_INFO("flag in conflict", K(ret), K(tablet_id), K(tmp_flag), K(input_flag)); } - } else if (tmp_status.is_medium()) { - tmp_status.inc_ref(); - if (OB_FAIL(ls_id_map_.set_refactored(ls_id, tmp_status, 1/*overwrite*/))) { - LOG_WARN("failed to set map", K(ret), K(ls_id), K(tmp_status)); - } - } else { + } else { // tmp_flag == input_flag ret = OB_ERR_UNEXPECTED; - LOG_TRACE("flag in already exist", K(ret), K(ls_id), K(tmp_status), K(input_flag)); + LOG_TRACE("flag in already exist", K(ret), K(tablet_id), K(tmp_flag), K(input_flag)); } } return ret; } -int ObProhibitScheduleMediumMap::clear_flag(const ObLSID &ls_id, const ProhibitFlag &input_flag) +int ObProhibitScheduleMediumMap::clear_flag(const ObTabletID &tablet_id, const ProhibitFlag &input_flag) { int ret = OB_SUCCESS; - ProhibitMediumStatus tmp_status(ProhibitFlag::FLAG_MAX); - ProhibitMediumStatus input_status(input_flag); - if (OB_UNLIKELY(!ls_id.is_valid() || !input_status.is_valid())) { + if (OB_UNLIKELY(!tablet_id.is_valid() || !is_valid_flag(input_flag))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(input_flag)); + LOG_WARN("invalid argument", K(ret), K(input_flag)); } else { obsys::ObWLockGuard lock_guard(lock_); - if (OB_FAIL(ls_id_map_.get_refactored(ls_id, tmp_status))) { - LOG_WARN("failed to get from map", K(ret), K(ls_id), K(tmp_status)); - } else if (!tmp_status.is_equal(input_flag)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("task do not match", K(ret), K(ls_id), K(tmp_status), K(input_flag)); - } else if (FALSE_IT(tmp_status.dec_ref())) { - } else if (!tmp_status.can_erase()) { - // need overwrite old status in map - if (OB_FAIL(ls_id_map_.set_refactored(ls_id, tmp_status, 1/*overwrite*/))) { - LOG_WARN("failed to set map", K(ret), K(ls_id), K(tmp_status)); + if (OB_FAIL(inner_clear_flag_(tablet_id, input_flag))) { + LOG_WARN("failed to inner clear flag", K(ret), K(tablet_id), K(input_flag)); + } + } + return ret; +} + +int ObProhibitScheduleMediumMap::batch_add_flags(const ObIArray &tablet_ids, const ProhibitFlag &input_flag) +{ + int ret = OB_SUCCESS; + obsys::ObWLockGuard lock_guard(lock_); + if (OB_UNLIKELY(ProhibitFlag::TRANSFER != input_flag)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument, batch_add_flags only support TRANSFER now", K(ret), K(input_flag)); + } else if (OB_FAIL(inner_batch_check_tablets_not_prohibited_(tablet_ids))) { + LOG_WARN("failed to check all tablets not prohibited", K(ret), K(tablet_ids)); + } else if (OB_FAIL(inner_batch_add_tablets_prohibit_flags_(tablet_ids, input_flag))){ + LOG_WARN("failed to add tablets prohibit_flags", K(ret), K(tablet_ids), K(input_flag)); + } else if (ProhibitFlag::TRANSFER == input_flag){ + ++transfer_flag_cnt_; + } + return ret; +} + +int ObProhibitScheduleMediumMap::batch_clear_flags(const ObIArray &tablet_ids, const ProhibitFlag &input_flag) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(ProhibitFlag::TRANSFER != input_flag)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument, batch_clear_flags only support TRANSFER now", K(ret), K(input_flag)); + } else { + const int64_t tablets_cnt = tablet_ids.count(); + obsys::ObWLockGuard lock_guard(lock_); + for (int64_t idx = 0; OB_SUCC(ret) && idx < tablets_cnt; idx++) { + const ObTabletID &tablet_id = tablet_ids.at(idx); + if (OB_FAIL(inner_clear_flag_(tablet_id, input_flag))) { + LOG_WARN("failed to clear transfer flag", K(ret), K(tablet_id), K(input_flag)); } - } else if (OB_FAIL(ls_id_map_.erase_refactored(ls_id))) { - LOG_WARN("failed to resume ls schedule medium", K(ret), K(ls_id), K(tmp_status)); } if (OB_SUCC(ret) && ProhibitFlag::TRANSFER == input_flag) { --transfer_flag_cnt_; @@ -700,9 +714,10 @@ int ObProhibitScheduleMediumMap::clear_flag(const ObLSID &ls_id, const ProhibitF void ObProhibitScheduleMediumMap::destroy() { + obsys::ObWLockGuard lock_guard(lock_); transfer_flag_cnt_ = 0; - if (ls_id_map_.created()) { - ls_id_map_.destroy(); + if (tablet_id_map_.created()) { + tablet_id_map_.destroy(); } } @@ -714,19 +729,19 @@ int64_t ObProhibitScheduleMediumMap::to_string(char *buf, const int64_t buf_len) if (OB_ISNULL(buf) || buf_len <= 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KP(buf), K(buf_len)); - } else if (0 == ls_id_map_.size()) { + } else if (0 == tablet_id_map_.size()) { // do nothing } else { J_ARRAY_START(); int64_t idx = 0; - FOREACH_X(it, ls_id_map_, OB_SUCC(ret)) { - const ObLSID &ls_id = it->first; - if (OB_UNLIKELY(!it->second.is_valid())) { + FOREACH_X(it, tablet_id_map_, OB_SUCC(ret)) { + const ObTabletID &tablet_id = it->first; + if (OB_UNLIKELY(!is_valid_flag(it->second))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("prihibit medium flag is not expected", K(ret), "flag", it->second); } else { J_OBJ_START(); - J_KV(K(idx), "ls_id", ls_id.id(), "flag", ProhibitFlagStr[static_cast(it->second.flag_)], "ref", it->second.ref_); + J_KV(K(idx), "tablet_id", tablet_id.id(), "flag", ProhibitFlagStr[static_cast(it->second)]); J_OBJ_END(); ++idx; } @@ -736,41 +751,118 @@ int64_t ObProhibitScheduleMediumMap::to_string(char *buf, const int64_t buf_len) return pos; } +// ATTENTION: hold lock outside !! +int ObProhibitScheduleMediumMap::inner_batch_check_tablets_not_prohibited_(const ObIArray &tablet_ids) +{ + int ret = OB_SUCCESS; + const int64_t tablets_cnt = tablet_ids.count(); + ProhibitFlag tmp_flag = ProhibitFlag::FLAG_MAX; + for (int64_t idx = 0; OB_SUCC(ret) && idx < tablets_cnt; idx++) { + const ObTabletID &tablet_id = tablet_ids.at(idx); + if (OB_UNLIKELY(!tablet_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tablet id", K(ret), K(tablet_id)); + } else if (OB_FAIL(tablet_id_map_.get_refactored(tablet_id, tmp_flag))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get flag from tablet_id_map", K(ret), K(tablet_id), K(tmp_flag)); + } + } else { + ret = OB_EAGAIN; + LOG_WARN("tablet is already set flag", K(ret), K(tablet_id), K(tmp_flag)); + } + } + return ret; +} + +// ATTENTION: hold lock outside !! +int ObProhibitScheduleMediumMap::inner_batch_add_tablets_prohibit_flags_(const ObIArray &tablet_ids, const ProhibitFlag &input_flag) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + const int64_t tablets_cnt = tablet_ids.count(); + int64_t idx = 0; + for (idx = 0; OB_SUCC(ret) && idx < tablets_cnt; idx++) { + const ObTabletID &tablet_id = tablet_ids.at(idx); + if (OB_UNLIKELY(!tablet_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tablet id", K(ret), K(tablet_id)); + } else if (OB_FAIL(tablet_id_map_.set_refactored(tablet_id, input_flag))) { + LOG_WARN("failed to add flag for tablet", K(ret), K(tablet_id), K(input_flag)); + // set partially, rollback. suppose tablet_ids.at(idx) not set input_flag + for (idx--; idx >= 0; idx--) { + const ObTabletID &tmp_tablet_id = tablet_ids.at(idx); // checked valid before + if (OB_TMP_FAIL(inner_clear_flag_(tmp_tablet_id, input_flag))) { + LOG_ERROR("failed clear transfer flag", K(tmp_ret), K(tmp_tablet_id), K(input_flag)); + } + } + } + } + return ret; +} + +// ATTENTION: hold lock outside !! +int ObProhibitScheduleMediumMap::inner_clear_flag_(const ObTabletID &tablet_id, const ProhibitFlag &input_flag) +{ + int ret = OB_SUCCESS; + ProhibitFlag tmp_flag = ProhibitFlag::FLAG_MAX; + if (OB_FAIL(tablet_id_map_.get_refactored(tablet_id, tmp_flag))) { + LOG_ERROR("failed to get from map", K(ret), K(tablet_id), K(tmp_flag)); + } else if (OB_UNLIKELY(tmp_flag != input_flag)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("task do not match", K(ret), K(tablet_id), K(tmp_flag), K(input_flag)); + } else if (OB_FAIL(tablet_id_map_.erase_refactored(tablet_id))) { + LOG_ERROR("failed to resume tablet schedule medium", K(ret), K(tablet_id), K(input_flag)); + } + return ret; +} + int64_t ObProhibitScheduleMediumMap::get_transfer_flag_cnt() const { obsys::ObRLockGuard lock_guard(lock_); return transfer_flag_cnt_; } -int ObTenantTabletScheduler::stop_ls_schedule_medium(const ObLSID &ls_id) +int ObTenantTabletScheduler::stop_tablets_schedule_medium(const ObIArray &tablet_ids, const ObProhibitScheduleMediumMap::ProhibitFlag &input_flag) { int ret = OB_SUCCESS; - if (OB_FAIL(prohibit_medium_map_.add_flag(ls_id, ObProhibitScheduleMediumMap::ProhibitFlag::TRANSFER))) { - if (OB_EAGAIN == ret) { - LOG_WARN("need wait ls already schedule medium end", K(ret), K(ls_id)); - } else { - LOG_WARN("failed to add flag for stopping", K(ret), K(ls_id)); - } + if (OB_UNLIKELY(ObProhibitScheduleMediumMap::ProhibitFlag::MEDIUM == input_flag)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument, input flag could not be MEDIUM", K(ret), K(input_flag)); + } else if (OB_FAIL(prohibit_medium_map_.batch_add_flags(tablet_ids, input_flag))) { + LOG_WARN("failed to add flag for stopping medium", K(ret), K(tablet_ids), K(input_flag)); } else { - FLOG_INFO("stopped ls schedule medium for transfer", K(ret), K(ls_id)); + LOG_INFO("stopped tablets schedule medium", K(ret), K(tablet_ids), K(input_flag)); } return ret; } // When executing the medium task, set the flag in the normal task process of the log stream -int ObTenantTabletScheduler::ls_start_schedule_medium(const ObLSID &ls_id, bool &ls_could_schedule_medium) +int ObTenantTabletScheduler::tablet_start_schedule_medium(const ObTabletID &tablet_id, bool &tablet_could_schedule_medium) { int ret = OB_SUCCESS; - ls_could_schedule_medium = false; - if (OB_FAIL(prohibit_medium_map_.add_flag(ls_id, ObProhibitScheduleMediumMap::ProhibitFlag::MEDIUM))) { + tablet_could_schedule_medium = false; + if (OB_FAIL(prohibit_medium_map_.add_flag(tablet_id, ObProhibitScheduleMediumMap::ProhibitFlag::MEDIUM))) { if (OB_EAGAIN == ret) { - ls_could_schedule_medium = false; + tablet_could_schedule_medium = false; ret = OB_SUCCESS; } else { - LOG_WARN("failed to add flag for ls schedule medium", K(ret), K(ls_id)); + LOG_WARN("failed to add flag for tablet schedule medium", K(ret)); } } else { - ls_could_schedule_medium = true; + tablet_could_schedule_medium = true; + } + return ret; +} + +int ObTenantTabletScheduler::clear_tablets_prohibit_medium_flag(const ObIArray &tablet_ids, const ObProhibitScheduleMediumMap::ProhibitFlag &input_flag) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(prohibit_medium_map_.batch_clear_flags(tablet_ids, input_flag))) { + LOG_WARN("failed to clear tablets prohibit medium flag", K(ret), K(tablet_ids), K(input_flag)); + } else { + LOG_INFO("allow tablets schedule medium", K(ret), K(input_flag)); } return ret; } @@ -1293,11 +1385,11 @@ int ObTenantTabletScheduler::schedule_next_round_for_leader( ObTabletHandle tablet_handle; ObArenaAllocator tmp_allocator("MediumChecker", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); const compaction::ObMediumCompactionInfoList *medium_list = nullptr; - bool ls_could_schedule_medium = false; + bool tablet_could_schedule_medium = false; //#TODO @jingshui sort tablet_ls_info with ls id - if (OB_FAIL(ls_start_schedule_medium(ls_id, ls_could_schedule_medium))) { + if (OB_FAIL(tablet_start_schedule_medium(tablet_id, tablet_could_schedule_medium))) { LOG_WARN("failed to set start schedule medium", K(ret), K(tmp_ret), K(ls_id)); - } else if (!ls_could_schedule_medium) { + } else if (!tablet_could_schedule_medium) { if (REACH_TENANT_TIME_INTERVAL(PRINT_LOG_INVERVAL)) { LOG_INFO("tenant is blocking schedule medium", KR(ret), K(MTL_ID()), K(ls_id)); } @@ -1313,8 +1405,8 @@ int ObTenantTabletScheduler::schedule_next_round_for_leader( } // clear flags set by ls_start_schedule_medium //#TODO @jingshui sort tablet_ls_info with ls id - if (ls_could_schedule_medium - && OB_TMP_FAIL(clear_prohibit_medium_flag(ls_id, ObProhibitScheduleMediumMap::ProhibitFlag::MEDIUM))) { + if (tablet_could_schedule_medium + && OB_TMP_FAIL(clear_prohibit_medium_flag(tablet_id, ObProhibitScheduleMediumMap::ProhibitFlag::MEDIUM))) { LOG_WARN("failed to clear prohibit schedule medium flag", K(tmp_ret), K(ret), K(ls_id)); } } // end of for @@ -1346,7 +1438,7 @@ int ObTenantTabletScheduler::schedule_ls_medium_merge( bool need_merge = false; ObLS &ls = *ls_handle.get_ls(); const ObLSID &ls_id = ls.get_ls_id(); - bool ls_could_schedule_medium = false; + bool tablet_could_schedule_medium = false; if (OB_FAIL(ObTabletMergeChecker::check_ls_state_in_major(ls, need_merge))) { LOG_WARN("failed to check ls can schedule medium", K(ret), K(ls)); } else if (!need_merge) { @@ -1384,14 +1476,6 @@ int ObTenantTabletScheduler::schedule_ls_medium_merge( } } - if (OB_FAIL(ret) || !is_leader) { - } else if (could_major_merge && OB_TMP_FAIL(ls_start_schedule_medium(ls_id, ls_could_schedule_medium))) { - LOG_WARN("failed to set start schedule medium", K(ret), K(tmp_ret), K(ls_id)); - } else if (!ls_could_schedule_medium) { // not allow schedule medium - if (REACH_TENANT_TIME_INTERVAL(PRINT_LOG_INVERVAL)) { - LOG_INFO("tenant is blocking schedule medium", KR(ret), K(MTL_ID()), K(ls_id), K(is_leader), K(could_major_merge)); - } - } bool enable_adaptive_compaction = get_enable_adaptive_compaction(); bool tablet_need_freeze_flag = false; @@ -1422,11 +1506,20 @@ int ObTenantTabletScheduler::schedule_ls_medium_merge( } else if (FALSE_IT(tablet_id = tablet->get_tablet_meta().tablet_id_)) { } else if (tablet_id.is_ls_inner_tablet()) { // do nothing + } else if (FALSE_IT(tablet_could_schedule_medium = false)) { + } else if (is_leader && could_major_merge + && OB_TMP_FAIL(tablet_start_schedule_medium(tablet_id, tablet_could_schedule_medium))) { + LOG_WARN("failed to set start schedule medium", K(ret), K(tmp_ret), K(ls_id)); + } else if (FALSE_IT(report_blocking_medium(is_leader, tablet_could_schedule_medium, could_major_merge, ls_id))) { } else if (OB_TMP_FAIL(schedule_tablet_medium( ls, tablet_handle, major_frozen_scn, weak_read_ts, - could_major_merge, ls_could_schedule_medium, merge_version, enable_adaptive_compaction, + could_major_merge, tablet_could_schedule_medium, merge_version, enable_adaptive_compaction, is_leader, tablet_merge_finish, tablet_need_freeze_flag, tablet_time_guard))) { LOG_WARN("failed to schedule tablet medium", KR(tmp_ret), K(ls_id), K(tablet_id)); + } else if (tablet_could_schedule_medium + && OB_TMP_FAIL(clear_prohibit_medium_flag(tablet_id, ObProhibitScheduleMediumMap::ProhibitFlag::MEDIUM))) { + // clear flags set by tablet_start_schedule_medium + LOG_WARN("failed to clear prohibit schedule medium flag", K(tmp_ret), K(ret), K(ls_id)); } medium_ls_tablet_iter_.update_merge_finish(tablet_merge_finish); if (tablet_need_freeze_flag) { @@ -1444,12 +1537,7 @@ int ObTenantTabletScheduler::schedule_ls_medium_merge( } } - // clear flags set by ls_start_schedule_medium - if (FALSE_IT(ls_time_guard.click(ObCompactionScheduleTimeGuard::FAST_FREEZE))) { - } else if (ls_could_schedule_medium - && OB_TMP_FAIL(clear_prohibit_medium_flag(ls_id, ObProhibitScheduleMediumMap::ProhibitFlag::MEDIUM))) { - LOG_WARN("failed to clear prohibit schedule medium flag", K(tmp_ret), K(ret), K(ls_id)); - } + ls_time_guard.click(ObCompactionScheduleTimeGuard::FAST_FREEZE); time_guard_.add_time_guard(ls_time_guard); } // else return ret; @@ -1488,7 +1576,7 @@ int ObTenantTabletScheduler::schedule_tablet_medium( const int64_t major_frozen_scn, const share::SCN &weak_read_ts, const bool could_major_merge, - const bool ls_could_schedule_medium, + const bool tablet_could_schedule_medium, const int64_t merge_version, const bool enable_adaptive_compaction, bool &is_leader, @@ -1505,7 +1593,7 @@ int ObTenantTabletScheduler::schedule_tablet_medium( bool tablet_could_schedule_merge = false; bool create_dag_flag = false; - if (ls_could_schedule_medium + if (tablet_could_schedule_medium && OB_TMP_FAIL(ObTabletMergeChecker::check_could_merge_for_medium(tablet, tablet_could_schedule_merge))) { LOG_WARN("failed to check tablet counld schedule merge", K(tmp_ret), K(tablet_id)); } @@ -1867,5 +1955,18 @@ int ObTenantTabletScheduler::update_report_scn_as_ls_leader(ObLS &ls) return ret; } +void ObTenantTabletScheduler::report_blocking_medium( + const bool &is_leader, + const bool &tablet_could_schedule_medium, + const bool &could_major_merge, + const share::ObLSID &ls_id) +{ + if (is_leader && !tablet_could_schedule_medium) { // not allow schedule medium + if (REACH_TENANT_TIME_INTERVAL(PRINT_LOG_INVERVAL)) { + LOG_INFO("tenant is blocking schedule medium", K(MTL_ID()), K(ls_id), K(is_leader), K(could_major_merge)); + } + } +} + } // namespace storage } // namespace oceanbase diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.h b/src/storage/compaction/ob_tenant_tablet_scheduler.h index 75b549e16b..cb72d13e8b 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.h +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.h @@ -90,45 +90,32 @@ public: MEDIUM = 1, FLAG_MAX }; - struct ProhibitMediumStatus - { - ProhibitMediumStatus() - : flag_(ProhibitFlag::MEDIUM), - ref_(0) - {} - explicit ProhibitMediumStatus(ProhibitFlag flag) - : flag_(flag), - ref_(1) - {} - ~ProhibitMediumStatus() { reset(); } - OB_INLINE void reset() { ref_ = 0; } - OB_INLINE bool is_valid() const { return flag_ >= ProhibitFlag::TRANSFER && flag_ < ProhibitFlag::FLAG_MAX; } - OB_INLINE bool is_transfer() const { return ProhibitFlag::TRANSFER == flag_; } - OB_INLINE bool is_medium() const { return ProhibitFlag::MEDIUM == flag_; } - OB_INLINE bool is_equal(const ProhibitFlag &type) { return type == flag_; } - OB_INLINE bool can_erase() { return 0 == ref_; } - OB_INLINE void dec_ref() { --ref_; } - OB_INLINE void inc_ref() { ++ref_; } - TO_STRING_KV(K_(flag), K_(ref)); - ProhibitFlag flag_; - int32_t ref_; - }; static const char *ProhibitFlagStr[]; + static bool is_valid_flag(const ProhibitFlag &flag) + { + return flag >= ProhibitFlag::TRANSFER && flag < ProhibitFlag::FLAG_MAX; + } ObProhibitScheduleMediumMap(); ~ObProhibitScheduleMediumMap() { destroy(); } int init(); void destroy(); - int clear_flag(const share::ObLSID &ls_id, const ProhibitFlag &input_flag); - int add_flag(const share::ObLSID &ls_id, const ProhibitFlag &input_flag); + int clear_flag(const ObTabletID &tablet_id, const ProhibitFlag &input_flag); + int add_flag(const ObTabletID &tablet_id, const ProhibitFlag &input_flag); + int batch_clear_flags(const ObIArray &tablet_ids, const ProhibitFlag &input_flag); + int batch_add_flags(const ObIArray &tablet_ids, const ProhibitFlag &input_flag); int64_t to_string(char *buf, const int64_t buf_len) const; int64_t get_transfer_flag_cnt() const; private: static const int64_t PRINT_LOG_INVERVAL = 2 * 60 * 1000 * 1000L; // 2m + static const int64_t TABLET_ID_MAP_BUCKET_NUM = OB_MAX_LS_NUM_PER_TENANT_PER_SERVER * 1024; + int inner_batch_check_tablets_not_prohibited_(const ObIArray &tablet_ids); // hold lock outside !! + int inner_batch_add_tablets_prohibit_flags_(const ObIArray &tablet_ids, const ProhibitFlag &input_flag); // hold lock outside !! + int inner_clear_flag_(const ObTabletID &tablet_id, const ProhibitFlag &input_flag); // hold lock outside !! int64_t transfer_flag_cnt_; mutable obsys::ObRWLock lock_; - common::hash::ObHashMap ls_id_map_; + common::hash::ObHashMap tablet_id_map_; // tablet is used for transfer of medium compaction }; class ObTenantTabletScheduler @@ -166,12 +153,13 @@ public: void resume_major_merge(); OB_INLINE bool could_major_merge_start() const { return ATOMIC_LOAD(&major_merge_status_); } // The transfer task sets the flag that prohibits the scheduling of medium when the log stream is src_ls of transfer - int stop_ls_schedule_medium(const share::ObLSID &ls_id); - int clear_prohibit_medium_flag(const share::ObLSID &ls_id, const ObProhibitScheduleMediumMap::ProhibitFlag &input_flag) + int stop_tablets_schedule_medium(const ObIArray &tablet_ids, const ObProhibitScheduleMediumMap::ProhibitFlag &input_flag); + int clear_tablets_prohibit_medium_flag(const ObIArray &tablet_ids, const ObProhibitScheduleMediumMap::ProhibitFlag &input_flag); + int clear_prohibit_medium_flag(const ObTabletID &tablet_id, const ObProhibitScheduleMediumMap::ProhibitFlag &input_flag) { - return prohibit_medium_map_.clear_flag(ls_id, input_flag); + return prohibit_medium_map_.clear_flag(tablet_id, input_flag); } - int ls_start_schedule_medium(const share::ObLSID &ls_id, bool &ls_could_schedule_medium); + int tablet_start_schedule_medium(const ObTabletID &tablet_id, bool &tablet_could_schedule_medium); const ObProhibitScheduleMediumMap& get_prohibit_medium_ls_map() const { return prohibit_medium_map_; } @@ -262,7 +250,7 @@ private: const int64_t major_frozen_scn, const share::SCN &weak_read_ts, const bool could_major_merge, - const bool ls_could_schedule_medium, + const bool tablet_could_schedule_medium, const int64_t merge_version, const bool enable_adaptive_compaction, bool &is_leader, @@ -295,6 +283,11 @@ private: ObTabletHandle &tablet_handle, const compaction::ObMediumCompactionInfoList *&medium_list, share::SCN &weak_read_ts); + void report_blocking_medium( + const bool &is_leader, + const bool &tablet_could_schedule_medium, + const bool &could_major_merge, + const share::ObLSID &ls_id); public: static const int64_t INIT_COMPACTION_SCN = 1; typedef common::ObSEArray MinorParallelResultArray; diff --git a/src/storage/high_availability/ob_transfer_handler.cpp b/src/storage/high_availability/ob_transfer_handler.cpp index 1ee7fd85fc..8b1b5695c4 100644 --- a/src/storage/high_availability/ob_transfer_handler.cpp +++ b/src/storage/high_availability/ob_transfer_handler.cpp @@ -451,6 +451,8 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta DEBUG_SYNC(START_TRANSFER_TRANS); } omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID())); + ObSEArray tablet_ids; // (0, 100], default 32, see ObTenantTransferService::get_tablet_count_threshold_(), + tablet_ids.set_attr(ObMemAttr(MTL_ID(), "TransferTblts")); if (tenant_config.is_valid()) { enable_kill_trx = tenant_config->_enable_balance_kill_transaction; } @@ -467,8 +469,10 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta LOG_WARN("transfer src is not leader", K(ret), K(task_info)); } else if (OB_FAIL(precheck_ls_replay_scn_(task_info))) { LOG_WARN("failed to precheck ls replay scn", K(ret), K(task_info)); - } else if (OB_FAIL(stop_ls_schedule_medium_(task_info.src_ls_id_, succ_stop_medium))) { - LOG_WARN("failed to stop ls schedule medium", K(ret), K(task_info)); + } else if (OB_FAIL(task_info.fill_tablet_ids(tablet_ids))) { + LOG_WARN("failed to init tablet_ids", K(ret), K(task_info)); + } else if (OB_FAIL(stop_tablets_schedule_medium_(tablet_ids, succ_stop_medium))) { + LOG_WARN("failed to stop tablets schedule medium", K(ret), K(task_info)); } else if (OB_FAIL(check_start_status_transfer_tablets_(task_info))) { LOG_WARN("failed to check start status transfer tablets", K(ret), K(task_info)); } else if (!enable_kill_trx && OB_FAIL(check_src_ls_has_active_trans_(task_info.src_ls_id_))) { @@ -504,7 +508,7 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta commit_succ = false; } - clear_prohibit_(task_info, succ_block_tx, succ_stop_medium); + clear_prohibit_(task_info, tablet_ids, succ_block_tx, succ_stop_medium); } @@ -2188,29 +2192,30 @@ void ObTransferHandler::online() transfer_worker_mgr_.reset_task_id(); } -int ObTransferHandler::stop_ls_schedule_medium_(const share::ObLSID &ls_id, bool &succ_stop) +int ObTransferHandler::stop_tablets_schedule_medium_(const ObIArray &tablet_ids, bool &succ_stop) { int ret = OB_SUCCESS; succ_stop = false; - if (OB_FAIL(MTL(ObTenantTabletScheduler*)->stop_ls_schedule_medium(ls_id))) { - LOG_WARN("failed to stop ls schedule medium", K(ret), K(ls_id)); + if (OB_FAIL(MTL(ObTenantTabletScheduler*)->stop_tablets_schedule_medium(tablet_ids, compaction::ObProhibitScheduleMediumMap::ProhibitFlag::TRANSFER))) { + LOG_WARN("failed to stop tablets schedule medium", K(ret)); } else { succ_stop = true; } return ret; } -int ObTransferHandler::clear_prohibit_medium_flag_(const share::ObLSID &ls_id) +int ObTransferHandler::clear_prohibit_medium_flag_(const ObIArray &tablet_ids) { int ret = OB_SUCCESS; - if (OB_FAIL(MTL(ObTenantTabletScheduler*)->clear_prohibit_medium_flag(ls_id, compaction::ObProhibitScheduleMediumMap::ProhibitFlag::TRANSFER))) { - LOG_WARN("failed to clear prohibit schedule medium flag", K(ret), K(ls_id)); + if (OB_FAIL(MTL(ObTenantTabletScheduler*)->clear_tablets_prohibit_medium_flag(tablet_ids, compaction::ObProhibitScheduleMediumMap::ProhibitFlag::TRANSFER))) { + LOG_WARN("failed to clear prohibit schedule medium flag", K(ret)); } return ret; } int ObTransferHandler::clear_prohibit_( const share::ObTransferTaskInfo &task_info, + const ObIArray &tablet_ids, const bool is_block_tx, const bool is_medium_stop) { @@ -2228,7 +2233,7 @@ int ObTransferHandler::clear_prohibit_( } if (OB_FAIL(ret)) { - } else if (is_medium_stop && OB_FAIL(clear_prohibit_medium_flag_(task_info.src_ls_id_))) { + } else if (is_medium_stop && OB_FAIL(clear_prohibit_medium_flag_(tablet_ids))) { LOG_WARN("failed to clear prohibit medium flag", K(ret), K(task_info)); ob_abort(); } diff --git a/src/storage/high_availability/ob_transfer_handler.h b/src/storage/high_availability/ob_transfer_handler.h index b58e6bfb05..e75e9c497c 100644 --- a/src/storage/high_availability/ob_transfer_handler.h +++ b/src/storage/high_availability/ob_transfer_handler.h @@ -236,8 +236,8 @@ private: const uint64_t tenant_id, const share::ObLSID &ls_id); int record_server_event_(const int32_t ret, const int64_t round, const share::ObTransferTaskInfo &task_info) const; - int clear_prohibit_medium_flag_(const share::ObLSID &ls_id); - int stop_ls_schedule_medium_(const share::ObLSID &ls_id, bool &succ_stop); + int clear_prohibit_medium_flag_(const ObIArray &tablet_ids); + int stop_tablets_schedule_medium_(const ObIArray &tablet_ids, bool &succ_stop); int get_next_tablet_info_( const share::ObLSID &dest_ls_id, const ObTransferTabletInfo &transfer_tablet_info, @@ -245,6 +245,7 @@ private: obrpc::ObCopyTabletInfo &tablet_info); int clear_prohibit_( const share::ObTransferTaskInfo &task_info, + const ObIArray &tablet_ids, const bool is_block_tx, const bool is_medium_stop); int get_config_version_(