Files
oceanbase/src/rootserver/ob_ls_recovery_stat_handler.cpp
xuhuleon 9dae112952 [FEAT MERGE] merge transfer
Co-authored-by: wxhwang <wxhwang@126.com>
Co-authored-by: godyangfight <godyangfight@gmail.com>
Co-authored-by: Tyshawn <tuyunshan@gmail.com>
2023-06-21 11:42:28 +00:00

511 lines
21 KiB
C++
Executable File

/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX RS
#include "ob_ls_recovery_stat_handler.h" // ObLSRecoveryStatHandler
#include "lib/utility/ob_macro_utils.h" // OB_FAIL
#include "lib/oblog/ob_log_module.h" // LOG_*
#include "lib/utility/ob_print_utils.h" // TO_STRING_KV
#include "logservice/ob_log_service.h" // ObLogService
#include "rootserver/ob_tenant_info_loader.h" // ObTenantInfoLoader
#include "rootserver/ob_ls_recovery_reportor.h" // ObLSRecoveryReportor
#include "rootserver/ob_ls_service_helper.h"//ObLSServiceHelper
namespace oceanbase
{
namespace rootserver
{
int ObLSRecoveryStatHandler::init(const uint64_t tenant_id, ObLS *ls)
{
int ret = OB_SUCCESS;
if (is_inited_) {
ret = OB_INIT_TWICE;
LOG_WARN("ObLSRecoveryStatHandler init twice", KR(ret), K_(is_inited));
} else if (OB_ISNULL(ls) || !is_valid_tenant_id(tenant_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), KP(ls), K(tenant_id));
} else {
ls_ = ls;
tenant_id_ = tenant_id;
is_inited_ = true;
LOG_INFO("ObLSRecoveryStatHandler init success", K(this));
}
return ret;
}
void ObLSRecoveryStatHandler::reset()
{
is_inited_ = false;
ls_ = NULL;
tenant_id_ = OB_INVALID_TENANT_ID;
}
int ObLSRecoveryStatHandler::check_inner_stat_()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret));
} else if (OB_ISNULL(ls_) || !is_valid_tenant_id(tenant_id_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Member variables is not init", KR(ret), KP(ls_), K_(tenant_id));
}
return ret;
}
int ObLSRecoveryStatHandler::get_ls_replica_readable_scn(share::SCN &readable_scn)
{
int ret = OB_SUCCESS;
readable_scn = SCN::min_scn();
share::SCN readable_scn_to_increase = SCN::min_scn();
if (OB_FAIL(check_inner_stat_())) {
LOG_WARN("inner stat error", KR(ret), K(is_inited_));
} else if (OB_FAIL(ls_->get_max_decided_scn(readable_scn))) {
LOG_WARN("failed to get_max_decided_scn", KR(ret), KPC_(ls));
} else if (FALSE_IT(readable_scn_to_increase = readable_scn)) {
} else if (OB_FAIL(increase_ls_replica_readable_scn_(readable_scn_to_increase))) {
if (OB_NOT_MASTER == ret) {
// if not master, do not increase_ls_replica_readable_scn
ret = OB_SUCCESS;
} else {
LOG_WARN("failed to increase_ls_replica_readable_scn_", KR(ret), K(readable_scn_to_increase), KPC_(ls));
}
} else {
readable_scn = readable_scn_to_increase;
}
return ret;
}
int ObLSRecoveryStatHandler::increase_ls_replica_readable_scn_(SCN &readable_scn)
{
int ret = OB_SUCCESS;
SCN sync_scn = SCN::min_scn();
int64_t first_proposal_id = palf::INVALID_PROPOSAL_ID;
int64_t second_proposal_id = palf::INVALID_PROPOSAL_ID;
common::ObRole first_role;
common::ObRole second_role;
logservice::ObLogService *ls_svr = MTL(logservice::ObLogService*);
rootserver::ObTenantInfoLoader *tenant_info_loader = MTL(rootserver::ObTenantInfoLoader*);
SCN replayable_scn = SCN::base_scn();
if (OB_FAIL(check_inner_stat_())) {
LOG_WARN("inner stat error", KR(ret), K_(is_inited));
} else if (OB_ISNULL(ls_svr) || OB_ISNULL(tenant_info_loader)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("pointer is null", KR(ret), KP(ls_svr), KP(tenant_info_loader));
} else if (OB_FAIL(ls_svr->get_palf_role(ls_->get_ls_id(), first_role, first_proposal_id))) {
LOG_WARN("failed to get first role", KR(ret), K(ls_->get_ls_id()), KP(ls_svr), KPC_(ls));
} else if (!is_strong_leader(first_role)) {
ret = OB_NOT_MASTER;
// Since the follower replica also call this function, return OB_NOT_MASTER does not LOG_WARN
} else if (!readable_scn.is_valid_and_not_min()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", KR(ret), K(readable_scn));
// scn get order: read_scn before replayable_scn before sync_scn
} else if (OB_FAIL(tenant_info_loader->get_replayable_scn(replayable_scn))) {
LOG_WARN("failed to get replayable_scn", KR(ret));
} else if (OB_FAIL(ObLSServiceHelper::get_ls_replica_sync_scn(MTL_ID(), ls_->get_ls_id(), sync_scn))) {
LOG_WARN("failed to get ls sync scn", KR(ret), "tenant_id", MTL_ID());
} else if (OB_FAIL(ls_svr->get_palf_role(ls_->get_ls_id(), second_role, second_proposal_id))) {
LOG_WARN("failed to get second role", KR(ret), K(ls_->get_ls_id()), KP(ls_svr), KPC_(ls));
} else if (!(first_proposal_id == second_proposal_id
&& first_role == second_role)) {
ret = OB_NOT_MASTER;
LOG_WARN("not leader", KR(ret), K(first_proposal_id), K(second_proposal_id), K(first_role),
K(second_role), KPC_(ls));
} else {
if (sync_scn < replayable_scn && readable_scn == sync_scn
&& sync_scn.is_valid_and_not_min() && replayable_scn.is_valid_and_not_min()
&& readable_scn.is_valid_and_not_min()) {
// two scenarios
// 1. when sync scn is pushed forward in switchover
// 2. wait offline LS
readable_scn = replayable_scn;
}
}
return ret;
}
int ObLSRecoveryStatHandler::get_ls_level_recovery_stat(ObLSRecoveryStat &ls_recovery_stat)
{
int ret = OB_SUCCESS;
share::SCN sync_scn = SCN::min_scn();
share::SCN readable_scn = SCN::min_scn();
logservice::ObLogService *ls_svr = MTL(logservice::ObLogService*);
common::ObRole role;
int64_t first_proposal_id = palf::INVALID_PROPOSAL_ID;
int64_t second_proposal_id = palf::INVALID_PROPOSAL_ID;
ls_recovery_stat.reset();
if (OB_FAIL(check_inner_stat_())) {
LOG_WARN("inner stat error", KR(ret), K_(is_inited));
} else if (OB_ISNULL(ls_svr)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("pointer is null", KR(ret), KP(ls_svr));
} else if (OB_FAIL(ls_svr->get_palf_role(ls_->get_ls_id(), role, first_proposal_id))) {
LOG_WARN("failed to get first role", KR(ret), K(ls_id), KPC_(ls));
} else if (!is_strong_leader(role)) {
ret = OB_NOT_MASTER;
LOG_TRACE("not leader", KR(ret), K(role), KPC_(ls));
} else if (OB_FAIL(do_get_ls_level_readable_scn_(readable_scn))) {
LOG_WARN("failed to do_get_ls_level_readable_scn_", KR(ret), KPC_(ls));
// scn get order: read_scn before replayable_scn before sync_scn
} else if (OB_FAIL(ObLSServiceHelper::get_ls_replica_sync_scn(MTL_ID(), ls_->get_ls_id(), sync_scn))) {
LOG_WARN("failed to get ls sync scn", KR(ret), "tenant_id", MTL_ID());
} else if (OB_FAIL(ls_recovery_stat.init_only_recovery_stat(tenant_id_, ls_->get_ls_id(),
sync_scn, readable_scn))) {
LOG_WARN("failed to init ls recovery stat", KR(ret), KPC_(ls), K_(tenant_id), K(sync_scn), K(readable_scn));
} else if (OB_FAIL(ls_svr->get_palf_role(ls_->get_ls_id(), role, second_proposal_id))) {
LOG_WARN("failed to get palf role again", KR(ret), K(role), KPC_(ls));
} else if (first_proposal_id != second_proposal_id || !is_strong_leader(role)) {
ret = OB_EAGAIN;
LOG_INFO("role changed, try again", KR(ret), K(role),
K(first_proposal_id), K(second_proposal_id), KPC_(ls));
}
return ret;
}
int ObLSRecoveryStatHandler::do_get_ls_level_readable_scn_(SCN &read_scn)
{
int ret = OB_SUCCESS;
palf::AccessMode access_mode;
int64_t unused_mode_version;
share::SCN majority_min_readable_scn = SCN::min_scn();
read_scn = SCN::min_scn();
if (OB_FAIL(check_inner_stat_())) {
LOG_WARN("inner stat error", KR(ret), K_(is_inited));
// scn get order: read_scn before replayable_scn before sync_scn
} else if (OB_FAIL(ls_->get_max_decided_scn(read_scn))) {
LOG_WARN("failed to get_max_decided_scn", KR(ret), KPC_(ls));
} else if (OB_FAIL(get_majority_readable_scn_(read_scn /* leader_readable_scn */, majority_min_readable_scn))) {
LOG_WARN("failed to get_majority_readable_scn_", KR(ret), K(read_scn), KPC_(ls));
} else {
read_scn = share::SCN::min(majority_min_readable_scn, read_scn /* leader_readable_scn */);
}
LOG_TRACE("do_get_ls_level_readable_scn_ finished", KR(ret), KPC_(ls), K(read_scn),
K(majority_min_readable_scn));
return ret;
}
int ObLSRecoveryStatHandler::construct_new_member_list_(
const common::ObMemberList &member_list_ori,
const common::GlobalLearnerList &degraded_list,
const int64_t paxos_replica_number_ori,
ObIArray<common::ObAddr> &member_list_new,
int64_t &paxos_replica_number_new)
{
int ret = OB_SUCCESS;
bool found_me = false;
member_list_new.reset();
paxos_replica_number_new = paxos_replica_number_ori;
if (OB_UNLIKELY(0 >= member_list_ori.get_member_number()
|| 0 > degraded_list.get_member_number()
|| 0 >= paxos_replica_number_ori)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(member_list_ori), K(degraded_list), K(paxos_replica_number_ori));
} else {
common::ObMember member;
for (int64_t i = 0; OB_SUCC(ret) && i < member_list_ori.get_member_number(); ++i) {
member.reset();
if (OB_FAIL(member_list_ori.get_member_by_index(i, member))) {
LOG_WARN("get_member_by_index failed", KR(ret), K(i));
} else if (OB_UNLIKELY(!member.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("member is invalid", KR(ret), K(member));
} else if (degraded_list.contains(member.get_server())) {
paxos_replica_number_new--;
// do not count degraded member
} else if (OB_FAIL(member_list_new.push_back(member.get_server()))) {
LOG_WARN("fail to push back member_list_new", KR(ret), K(member), K(member_list_new));
} else if (member.get_server() == GCTX.self_addr()) {
found_me = true;
}
}
if (OB_FAIL(ret)) {
} else if (!found_me) {
ret = OB_EAGAIN;
LOG_WARN("current leader degraded, try again", KR(ret), K(member_list_ori), K(degraded_list),
K(paxos_replica_number_ori), K(member_list_new), K(paxos_replica_number_new));
}
}
return ret;
}
int ObLSRecoveryStatHandler::get_palf_stat_(
palf::PalfStat &palf_stat)
{
int ret = OB_SUCCESS;
palf_stat.reset();
logservice::ObLogService *log_service = NULL;
palf::PalfHandleGuard palf_handle_guard;
if (OB_FAIL(check_inner_stat_())) {
LOG_WARN("inner stat error", KR(ret), K_(is_inited));
} else if (OB_ISNULL(log_service = MTL(logservice::ObLogService*))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get MTL log_service", KR(ret), K_(tenant_id), KPC_(ls));
} else if (OB_FAIL(log_service->open_palf(ls_->get_ls_id(), palf_handle_guard))) {
LOG_WARN("failed to open palf", KR(ret), K_(tenant_id), KPC_(ls));
} else if (OB_FAIL(palf_handle_guard.stat(palf_stat))) {
LOG_WARN("get palf_stat failed", KR(ret), KPC_(ls));
}
return ret;
}
int ObLSRecoveryStatHandler::get_latest_palf_stat_(
palf::PalfStat &palf_stat)
{
int ret = OB_SUCCESS;
palf_stat.reset();
common::ObMemberList ob_member_list_latest;
int64_t paxos_replica_number_latest = 0;
if (OB_FAIL(check_inner_stat_())) {
LOG_WARN("inner stat error", KR(ret), K_(is_inited));
} else if (OB_FAIL(get_palf_stat_(palf_stat))) {
LOG_WARN("get palf_stat failed", KR(ret), KPC_(ls));
} else if (OB_FAIL(ls_->get_paxos_member_list(ob_member_list_latest, paxos_replica_number_latest))) {
LOG_WARN("get latest paxos member_list failed", KR(ret), KPC_(ls));
} else if (!ob_member_list_latest.member_addr_equal(palf_stat.paxos_member_list_)
|| paxos_replica_number_latest != palf_stat.paxos_replica_num_) {
ret = OB_EAGAIN;
LOG_WARN("palf_stat is not latest, try again", KR(ret), KPC_(ls), K(ob_member_list_latest),
K(paxos_replica_number_latest), K(palf_stat));
}
return ret;
}
int ObLSRecoveryStatHandler::get_majority_readable_scn_(
const share::SCN &leader_readable_scn,
share::SCN &majority_min_readable_scn)
{
int ret = OB_SUCCESS;
majority_min_readable_scn = leader_readable_scn;
palf::PalfStat palf_stat_first;
palf::PalfStat palf_stat_second;
ObArray<common::ObAddr> member_list_new;
int64_t paxos_replica_number_new = 0;
if (OB_FAIL(check_inner_stat_())) {
LOG_WARN("inner stat error", KR(ret), K_(is_inited));
} else if (!leader_readable_scn.is_valid_and_not_min()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(leader_readable_scn));
} else if (OB_FAIL(get_latest_palf_stat_(palf_stat_first))) {
LOG_WARN("get latest palf_stat failed", KR(ret), KPC_(ls));
} else if (OB_FAIL(construct_new_member_list_(palf_stat_first.paxos_member_list_,
palf_stat_first.degraded_list_,
palf_stat_first.paxos_replica_num_,
member_list_new,
paxos_replica_number_new))) {
LOG_WARN("construct_new_member_list failed", KR(ret), KPC_(ls), K(palf_stat_first));
} else if (OB_FAIL(do_get_majority_readable_scn_(member_list_new,
leader_readable_scn, rootserver::majority(paxos_replica_number_new), majority_min_readable_scn))) {
LOG_WARN("do_get_majority_readable_scn_ failed", KR(ret), K(member_list_new), K(leader_readable_scn),
K(paxos_replica_number_new), K(palf_stat_first), K(majority_min_readable_scn));
} else if (OB_FAIL(get_latest_palf_stat_(palf_stat_second))) {
LOG_WARN("get latest palf_stat failed", KR(ret), KPC_(ls));
} else if (palf_stat_first.config_version_ != palf_stat_second.config_version_) {
ret = OB_EAGAIN;
LOG_WARN("config_version changed, try again", KR(ret), K(palf_stat_first), K(palf_stat_second));
}
return ret;
}
int ObLSRecoveryStatHandler::do_get_majority_readable_scn_(
const ObIArray<common::ObAddr> &ob_member_list,
const share::SCN &leader_readable_scn,
const int64_t majority_cnt,
share::SCN &majority_min_readable_scn)
{
int ret = OB_SUCCESS;
majority_min_readable_scn = SCN::min_scn();
const common::ObAddr self_addr = GCTX.self_addr();
ObTimeoutCtx ctx;
ObSEArray<ObAddr, 3> inactive_members;
obrpc::ObGetLSReplayedScnArg arg;
const int64_t need_query_member_cnt = majority_cnt - 1;
if (OB_FAIL(check_inner_stat_())) {
LOG_WARN("inner stat error", KR(ret), K_(is_inited));
} else if (!is_user_tenant(tenant_id_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument. only support for user tenant", KR(ret), K_(tenant_id));
} else if (OB_ISNULL(GCTX.server_tracer_) || OB_ISNULL(GCTX.srv_rpc_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("pointer is null", KR(ret), KP(GCTX.server_tracer_), KP(GCTX.srv_rpc_proxy_));
} else if (!leader_readable_scn.is_valid_and_not_min()
|| ob_member_list.count() <= 0
|| !self_addr.is_valid()
|| 0 >= majority_cnt
|| 0 > need_query_member_cnt) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(leader_readable_scn), K(self_addr),
K(majority_cnt), K(need_query_member_cnt), K(ob_member_list));
} else if (0 == need_query_member_cnt) {
ret = OB_SUCCESS;
majority_min_readable_scn = leader_readable_scn;
LOG_INFO("single replica, majority_min_readable_scn = leader_readable_scn", KR(ret),
K(ob_member_list), K(leader_readable_scn));
} else if (OB_FAIL(arg.init(tenant_id_, ls_->get_ls_id()))) {
LOG_WARN("failed to init arg", KR(ret), K_(tenant_id), KPC_(ls));
} else {
int tmp_ret = OB_SUCCESS;
ObGetLSReplayedScnProxy proxy(
*GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::get_ls_replayed_scn);
int64_t rpc_count = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < ob_member_list.count(); ++i) {
const auto member = ob_member_list.at(i);
bool alive = true;
int64_t trace_time;
if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
} else if (OB_UNLIKELY(!member.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("member is invalid", KR(ret), K(member));
} else if (self_addr == member) {
//skip myself
} else if (OB_FAIL(GCTX.server_tracer_->is_alive(member, alive, trace_time))) {
LOG_WARN("check server alive failed", KR(ret), K(member));
} else if (!alive) {
//not send to alive
if (OB_FAIL(inactive_members.push_back(member))) {
LOG_WARN("fail to push back inactive_members", KR(ret), K(member), K(inactive_members));
}
// use meta rpc process thread
} else if (OB_TMP_FAIL(proxy.call(member, ctx.get_timeout(), gen_meta_tenant_id(tenant_id_), arg))) {
LOG_WARN("failed to send rpc", KR(tmp_ret), K(member), K(i), K(ctx), K_(tenant_id), K(arg), K(ob_member_list));
} else {
rpc_count++;
}
}
if (OB_FAIL(ret)) {
} else if (need_query_member_cnt > rpc_count) {
// If the number of alive servers is not enough for a majority, send to majority servers
for (int64_t i = 0; OB_SUCC(ret) && i < inactive_members.count(); ++i) {
if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
} else if (OB_TMP_FAIL(proxy.call(inactive_members.at(i), ctx.get_timeout(), tenant_id_, arg))) {
LOG_WARN("failed to send rpc", KR(tmp_ret), K(i), K(ctx), K_(tenant_id), K(arg), K(inactive_members));
} else {
rpc_count++;
}
}
}
//get result
ObArray<int> return_code_array;
if (OB_SUCCESS != (tmp_ret = proxy.wait_all(return_code_array))) {
LOG_WARN("wait all batch result failed", KR(ret), KR(tmp_ret));
ret = OB_SUCCESS == ret ? tmp_ret : ret;
} else if (OB_FAIL(ret)) {
} else if (OB_FAIL(calc_majority_min_readable_scn_(
leader_readable_scn,
majority_cnt,
return_code_array,
proxy,
rpc_count,
majority_min_readable_scn))) {
LOG_WARN("failed to calc_majority_min_readable_scn", KR(ret), K(leader_readable_scn),
K(ob_member_list), K(return_code_array), K(rpc_count));
}
}
return ret;
}
int ObLSRecoveryStatHandler::calc_majority_min_readable_scn_(
const SCN &leader_readable_scn,
const int64_t majority_cnt,
const ObIArray<int> &return_code_array,
const ObGetLSReplayedScnProxy &proxy,
const int64_t rpc_count,
SCN &majority_min_readable_scn)
{
int ret = OB_SUCCESS;
ObArray<SCN> readable_scn_list;
majority_min_readable_scn = SCN::max_scn();
if (OB_FAIL(check_inner_stat_())) {
LOG_WARN("inner stat error", KR(ret), K_(is_inited));
} else if (!leader_readable_scn.is_valid_and_not_min() || 0 >= majority_cnt || 0 >= rpc_count) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(leader_readable_scn), K(majority_cnt), K(rpc_count));
} else if (rpc_count != return_code_array.count() ||
rpc_count != proxy.get_results().count()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("rpc count not equal to result count", KR(ret),
K(rpc_count), K(return_code_array), "arg count",
proxy.get_args().count(), K(proxy.get_results().count()));
} else if (OB_FAIL(readable_scn_list.push_back(leader_readable_scn))) {
LOG_WARN("failed to push back", KR(ret), K(leader_readable_scn), K(readable_scn_list));
} else if (OB_FAIL(ret)) {
} else {
ObGetLSReplayedScnRes res;
int tmp_ret = OB_SUCCESS;
for (int64_t i = 0; OB_SUCC(ret) && i < return_code_array.count(); ++i) {
tmp_ret = return_code_array.at(i);
// skip error server
if (OB_SUCCESS != tmp_ret) {
LOG_WARN("send rpc is failed", KR(tmp_ret), K(i), K(return_code_array));
} else {
const auto *result = proxy.get_results().at(i);
if (OB_ISNULL(result)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("result is null", KR(ret), K(i), K(return_code_array));
} else if (!result->get_cur_readable_scn().is_valid_and_not_min()) {
LOG_WARN("not valid scn", KR(ret), K(i), KPC(result));
// skip this server
} else if (OB_FAIL(readable_scn_list.push_back(result->get_cur_readable_scn()))) {
LOG_WARN("failed to push back", KR(ret), K(i), KPC(result), K(readable_scn_list));
}
}
}
if (OB_FAIL(ret)) {
} else if (readable_scn_list.count() < majority_cnt) {
ret = OB_EAGAIN;
LOG_WARN("can not get majority readable_scn count", KR(ret), K(majority_cnt), K(readable_scn_list), K(return_code_array));
} else {
(void)std::sort(readable_scn_list.begin(), readable_scn_list.end(), std::greater<share::SCN>());
for (int64_t i = 0; OB_SUCC(ret) && i < readable_scn_list.count() && i < majority_cnt; ++i) {
if (majority_min_readable_scn > readable_scn_list.at(i)) {
majority_min_readable_scn = readable_scn_list.at(i);
}
}
LOG_TRACE("calculate majority min readable_scn finished", KR(ret), K(leader_readable_scn), K(ls_id),
K(majority_min_readable_scn), K(readable_scn_list), K(majority_cnt), K(return_code_array));
}
}
return ret;
}
}
}