hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Election] fix arb server compare priority bug

Browse Source
This commit is contained in:
fengdeyiji
2023-06-05 08:50:59 +00:00
committed by ob-robot
parent 61b129e849
commit a795853f8e
13 changed files with 334 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/logservice/leader_coordinator/election_priority_impl/election_priority_impl.cpp
View File

@ -190,13 +190,15 @@ private:
const AbstractPriority *closest_priority_;
};
int ElectionPriorityImpl::compare_with(const ElectionPriority &rhs, int &result, ObStringHolder &reason) const
int ElectionPriorityImpl::compare_with(const ElectionPriority &rhs,
const uint64_t compare_version,
int &result,
ObStringHolder &reason) const
{
LC_TIME_GUARD(1_s);
int ret = OB_SUCCESS;
// 这里如果转型失败直接抛异常，但设计上转型不会失败
const ElectionPriorityImpl &rhs_impl = dynamic_cast<const ElectionPriorityImpl &>(rhs);
uint64_t compare_version = GET_MIN_CLUSTER_VERSION();
GetClosestVersionPriority functor1(compare_version);
GetClosestVersionPriority functor2(compare_version);
(void) priority_tuple_.for_each(functor1);
@ -242,8 +244,12 @@ struct RefeshPriority
LC_TIME_GUARD(1_s);
int ret = OB_SUCCESS;
if (CLICK_FAIL(element.refresh(ls_id_))) {
ret_ = ret;
COORDINATOR_LOG(WARN, "refresh priority failed", KR(ret), K(MTL_ID()), K(ls_id_), K(element));
if (OB_NO_NEED_UPDATE == ret) {
ret = OB_SUCCESS;
} else {
ret_ = ret;
COORDINATOR_LOG(WARN, "refresh priority failed", KR(ret), K(MTL_ID()), K(ls_id_), K(element));
}
}
return ret;
}

5
src/logservice/leader_coordinator/election_priority_impl/election_priority_impl.h
View File

@ -161,7 +161,10 @@ public:
// 主动刷新选举优先级的方法
virtual int refresh();
// 在priority间进行比较的方法
virtual int compare_with(const palf::election::ElectionPriority &rhs, int &result, ObStringHolder &reason) const;
virtual int compare_with(const palf::election::ElectionPriority &rhs,
const uint64_t compare_version,
int &result,
ObStringHolder &reason) const;
virtual int get_size_of_impl_type() const;
virtual void placement_new_impl(void *ptr) const;
// fatal failure跳过RCS直接切主

4
src/logservice/leader_coordinator/election_priority_impl/election_priority_v1.cpp
View File

@ -12,6 +12,7 @@
#include "lib/utility/ob_macro_utils.h"
#include "logservice/ob_log_service.h"
#include "share/ob_errno.h"
#include "share/ob_occam_time_guard.h"
#include "election_priority_impl.h"
#include "lib/list/ob_dlist.h"
@ -151,7 +152,8 @@ int PriorityV1::refresh_(const share::ObLSID &ls_id)
ObFailureDetector* detector = MTL(ObFailureDetector*);
LsElectionReferenceInfo election_reference_info;
SCN scn = SCN::min_scn();
if (OB_ISNULL(coordinator) || OB_ISNULL(detector)) {
if (observer::ObServer::get_instance().is_arbitration_mode()) {
} else if (OB_ISNULL(coordinator) || OB_ISNULL(detector)) {
ret = OB_ERR_UNEXPECTED;
COORDINATOR_LOG_(ERROR, "unexpected nullptr");
} else if (CLICK_FAIL(detector->get_specified_level_event(FailureLevel::FATAL, fatal_failures_))) {

15
src/logservice/palf/election/algorithm/election_acceptor.cpp
View File

@ -61,6 +61,7 @@ public:
T reject_msg = create_reject_message_(p_acceptor->p_election_->get_self_addr(),
p_acceptor->p_election_->inner_priority_seed_,
p_acceptor->p_election_->get_membership_version_(),
p_acceptor->p_election_->get_ls_biggest_min_cluster_version_ever_seen_(),
msg);
reject_msg.set_rejected(p_acceptor->ballot_number_);
p_acceptor->p_election_->send_(reject_msg);
@ -75,18 +76,20 @@ private:
static ElectionPrepareResponseMsg create_reject_message_(const common::ObAddr &addr,
const uint64_t inner_priority_seed,
const LogConfigVersion &membership_version,
const LsBiggestMinClusterVersionEverSeen &version,
const ElectionPrepareRequestMsg &msg)
{
UNUSED(inner_priority_seed),
UNUSED(membership_version);
return ElectionPrepareResponseMsg(addr, msg);
return ElectionPrepareResponseMsg(addr, version, msg);
}
static ElectionAcceptResponseMsg create_reject_message_(const common::ObAddr &addr,
const uint64_t inner_priority_seed,
const LogConfigVersion &membership_version,
const LsBiggestMinClusterVersionEverSeen &version,
const ElectionAcceptRequestMsg &msg)
{
return ElectionAcceptResponseMsg(addr, inner_priority_seed, membership_version, msg);
return ElectionAcceptResponseMsg(addr, inner_priority_seed, membership_version, version, msg);
}
};
@ -153,7 +156,9 @@ int ElectionAcceptor::start()
last_record_lease_owner = lease_.get_owner();
}
if (is_time_window_opened_) {
ElectionPrepareResponseMsg prepare_res_accept(p_election_->get_self_addr(), highest_priority_prepare_req_);
ElectionPrepareResponseMsg prepare_res_accept(p_election_->get_self_addr(),
p_election_->get_ls_biggest_min_cluster_version_ever_seen_(),
highest_priority_prepare_req_);
bool can_vote = false;
if (last_record_lease_valid_state && !lease_valid_state) {// 这个定时任务可能是被延迟致lease到期时触发的，为了在lease到期的第一时间投票
can_vote = true;
@ -234,7 +239,8 @@ void ElectionAcceptor::on_prepare_request(const ElectionPrepareRequestMsg &prepa
} else {
advance_ballot_number_and_reset_related_states_(prepare_req.get_ballot_number(), phase);
ElectionPrepareResponseMsg prepare_res_accept(p_election_->get_self_addr(),
prepare_req);
p_election_->get_ls_biggest_min_cluster_version_ever_seen_(),
prepare_req);
prepare_res_accept.set_accepted(ballot_number_, lease_);
if (CLICK_FAIL(p_election_->msg_handler_->send(prepare_res_accept))) {
LOG_PHASE(WARN, phase, "send prepare response to leader prepare failed");
@ -317,6 +323,7 @@ void ElectionAcceptor::on_accept_request(const ElectionAcceptRequestMsg &accept_
ElectionAcceptResponseMsg accept_res_accept(p_election_->get_self_addr(),
p_election_->inner_priority_seed_,
p_election_->get_membership_version_(),
p_election_->get_ls_biggest_min_cluster_version_ever_seen_(),
accept_req);
(void) p_election_->refresh_priority_();
if (CLICK_FAIL(accept_res_accept.set_accepted(ballot_number_,

32
src/logservice/palf/election/algorithm/election_impl.cpp
View File

@ -10,6 +10,8 @@
* See the Mulan PubL v2 for more details.
*/
#include "lib/list/ob_dlist.h"
#include "observer/ob_server.h"
#include "share/ob_errno.h"
#include "share/ob_occam_time_guard.h"
#include "election_impl.h"
@ -57,13 +59,14 @@ void DefaultRoleChangeCallBack::operator()(ElectionImpl *election,
}
ElectionImpl::ElectionImpl()
: lock_(common::ObLatchIds::ELECTION_LOCK),
: is_inited_(false),
is_running_(false),
lock_(common::ObLatchIds::ELECTION_LOCK),
proposer_(this),
acceptor_(this),
priority_(nullptr),
msg_handler_(nullptr),
is_inited_(false),
is_running_(false),
ls_biggest_min_cluster_version_ever_seen_(0),
event_recorder_(id_, self_addr_, timer_)
{}
@ -209,6 +212,10 @@ int ElectionImpl::reset_priority()
return OB_SUCCESS;
}
void ElectionImpl::handle_message_base_(const ElectionMsgBase &message_base)
{
ls_biggest_min_cluster_version_ever_seen_.try_advance(message_base.get_ls_biggest_min_cluster_version_ever_seen().version_);
}
int ElectionImpl::handle_message(const ElectionPrepareRequestMsg &msg)
{
@ -219,6 +226,7 @@ int ElectionImpl::handle_message(const ElectionPrepareRequestMsg &msg)
bool need_register_devote_task = false;
{
LockGuard lock_guard(lock_);
handle_message_base_(msg);
msg_counter_.add_received_count(msg);
CHECK_ELECTION_INIT_AND_START();
if (msg.get_sender() != self_addr_) {
@ -244,6 +252,7 @@ int ElectionImpl::handle_message(const ElectionAcceptRequestMsg &msg)
int64_t us_to_expired = 0;
{
LockGuard lock_guard(lock_);
handle_message_base_(msg);
msg_counter_.add_received_count(msg);
CHECK_ELECTION_INIT_AND_START();
if (msg.get_ballot_number() > proposer_.ballot_number_) {
@ -271,6 +280,7 @@ int ElectionImpl::handle_message(const ElectionPrepareResponseMsg &msg)
ELECT_TIME_GUARD(500_ms);
int ret = OB_SUCCESS;
LockGuard lock_guard(lock_);
handle_message_base_(msg);
msg_counter_.add_received_count(msg);
CHECK_ELECTION_INIT_AND_START();
proposer_.on_prepare_response(msg);
@ -283,6 +293,7 @@ int ElectionImpl::handle_message(const ElectionAcceptResponseMsg &msg)
ELECT_TIME_GUARD(500_ms);
int ret = OB_SUCCESS;
LockGuard lock_guard(lock_);
handle_message_base_(msg);
msg_counter_.add_received_count(msg);
CHECK_ELECTION_INIT_AND_START();
proposer_.on_accept_response(msg);
@ -295,6 +306,7 @@ int ElectionImpl::handle_message(const ElectionChangeLeaderMsg &msg)
ELECT_TIME_GUARD(500_ms);
int ret = OB_SUCCESS;
LockGuard lock_guard(lock_);
handle_message_base_(msg);
msg_counter_.add_received_count(msg);
CHECK_ELECTION_INIT_AND_START();
proposer_.on_change_leader(msg);
@ -467,6 +479,20 @@ int ElectionImpl::set_inner_priority_seed(const uint64_t seed)
return ret;
}
uint64_t ElectionImpl::get_ls_biggest_min_cluster_version_ever_seen_() const
{
#define PRINT_WRAPPER K(*this)
int ret = OB_SUCCESS;
uint64_t ls_biggest_min_cluster_version_ever_seen = 0;
if (observer::ObServer::get_instance().is_arbitration_mode()) {
} else {
ls_biggest_min_cluster_version_ever_seen = std::max(GET_MIN_CLUSTER_VERSION(),
ls_biggest_min_cluster_version_ever_seen_.version_);
}
return ls_biggest_min_cluster_version_ever_seen;
#undef PRINT_WRAPPER
}
}
}
}

44
src/logservice/palf/election/algorithm/election_impl.h
View File

@ -129,8 +129,10 @@ public:
int add_inner_priority_seed_bit(const PRIORITY_SEED_BIT new_bit);
int clear_inner_priority_seed_bit(const PRIORITY_SEED_BIT old_bit);
int set_inner_priority_seed(const uint64_t seed);
TO_STRING_KV(K_(is_inited), K_(is_running), K_(proposer), K_(acceptor), KPC_(priority));
TO_STRING_KV(K_(is_inited), K_(is_running), K_(proposer), K_(acceptor),
K_(ls_biggest_min_cluster_version_ever_seen), KPC_(priority));
private:// 定向暴露给友元类
void handle_message_base_(const ElectionMsgBase &message_base);
void refresh_priority_();
/**
* @description: 比较两个消息的优先级哪个更高
@ -200,7 +202,16 @@ private:// 定向暴露给友元类
if (!rhs.is_buffer_valid()) {// rhs优先级是空的，判定rhs优先级更低
(void) reason.assign("old message priority is valid and new message priority is invalid");
} else {// rhs优先级非空，具备可比较的基础
bool can_only_compare_ip_port = false;
uint64_t compare_version = get_ls_biggest_min_cluster_version_ever_seen_();
if (OB_ISNULL(self_priority)) {// 本机的优先级还没有设置，无法感知子类类型，此时没办法进行比较，只能比较IP大小
can_only_compare_ip_port = true;
LOG_PHASE(WARN, phase, "self priority not setted, can only compare IP-PORT");
} else if (compare_version == 0) {
can_only_compare_ip_port = true;
LOG_PHASE(WARN, phase, "self ever seen min_cluster_version is 0, can only compare IP-PORT");
}
if (can_only_compare_ip_port) {
if (compare_with_ip_port && rhs.get_sender() < lhs.get_sender()) {
rhs_is_higher = true;
(void) reason.assign("IP-PORT(priority invalid)");
@ -226,7 +237,7 @@ private:// 定向暴露给友元类
pos2))) {
LOG_PHASE(WARN, phase, "deserialize new message priority failed");
(void) reason.assign("DESERIALIZE FAIL");
} else if (CLICK_FAIL(lhs_priority->compare_with(*rhs_priority, compare_result, reason))) {
} else if (CLICK_FAIL(lhs_priority->compare_with(*rhs_priority, compare_version, compare_result, reason))) {
LOG_PHASE(WARN, phase, "compare priority failed");
(void) reason.assign("COMPARE FAIL");
} else {
@ -261,22 +272,25 @@ private:// 定向暴露给友元类
int send_(const ElectionPrepareResponseMsg &msg) const;
int send_(const ElectionAcceptResponseMsg &msg) const;
int send_(const ElectionChangeLeaderMsg &msg) const;
uint64_t get_ls_biggest_min_cluster_version_ever_seen_() const;
const char *print_version_pretty_(const uint64_t version) const;
private:
mutable common::ObSpinLock lock_;
int64_t id_;
ElectionProposer proposer_;
ElectionAcceptor acceptor_;
ElectionPriority *priority_;
ElectionMsgSender *msg_handler_;
common::ObAddr self_addr_;
ObFunction<int(const int64_t, const ObAddr &)> prepare_change_leader_cb_;
ObFunction<void(ElectionImpl *,common::ObRole,common::ObRole,RoleChangeReason)> role_change_cb_;
bool is_inited_;
bool is_running_;
uint64_t inner_priority_seed_;
common::ObOccamTimer *timer_;
EventRecorder event_recorder_;
mutable ElectionMsgCounter msg_counter_;
mutable common::ObSpinLock lock_;
int64_t id_;
ElectionProposer proposer_;// 对应PAXOS算法中的proposer
ElectionAcceptor acceptor_;// 对应PAXOS算法中的acceptor
ElectionPriority *priority_;// 由外部实现的选举优先级模块
ElectionMsgSender *msg_handler_;// 由外部实现的消息收发模块
common::ObAddr self_addr_;
ObFunction<int(const int64_t, const ObAddr &)> prepare_change_leader_cb_;// 切主回调
ObFunction<void(ElectionImpl *,common::ObRole,common::ObRole,RoleChangeReason)> role_change_cb_;// 角色状态变更回调
uint64_t inner_priority_seed_;// 协议内选举优先级
common::ObOccamTimer *timer_;// 选举定时任务的定时器
LsBiggestMinClusterVersionEverSeen ls_biggest_min_cluster_version_ever_seen_;// 为仲裁副本维护的日志流级别的min_cluster_version值，用于处理选举兼容性升级相关问题
EventRecorder event_recorder_;// 事件汇报模块
mutable ElectionMsgCounter msg_counter_;// 监控模块
};
}// namespace election

7
src/logservice/palf/election/algorithm/election_proposer.cpp
View File

@ -354,6 +354,7 @@ void ElectionProposer::prepare(const ObRole role)
p_election_->get_self_addr(),
restart_counter_,
ballot_number_,
p_election_->get_ls_biggest_min_cluster_version_ever_seen_(),
p_election_->inner_priority_seed_,
p_election_->get_membership_version_());
(void) p_election_->refresh_priority_();
@ -383,6 +384,7 @@ void ElectionProposer::on_prepare_request(const ElectionPrepareRequestMsg &prepa
if (prepare_req.get_ballot_number() <= ballot_number_) {
if (prepare_req.get_ballot_number() < ballot_number_) {// 对于旧消息发送拒绝响应
ElectionPrepareResponseMsg prepare_res_reject(p_election_->get_self_addr(),
p_election_->get_ls_biggest_min_cluster_version_ever_seen_(),
prepare_req);
prepare_res_reject.set_rejected(ballot_number_);
if (CLICK_FAIL(p_election_->send_(prepare_res_reject))) {
@ -414,6 +416,7 @@ void ElectionProposer::on_prepare_request(const ElectionPrepareRequestMsg &prepa
p_election_->get_self_addr(),
restart_counter_,
ballot_number_,
p_election_->get_ls_biggest_min_cluster_version_ever_seen_(),
p_election_->inner_priority_seed_,
p_election_->get_membership_version_());
if (CLICK_FAIL(prepare_followed_req.set(p_election_->get_priority_(),
@ -490,6 +493,7 @@ void ElectionProposer::propose()
p_election_->get_self_addr(),
restart_counter_,
prepare_success_ballot_,
p_election_->get_ls_biggest_min_cluster_version_ever_seen_(),
current_ts,
new_lease_interval,
memberlist_with_states_.get_member_list()
@ -561,10 +565,12 @@ void ElectionProposer::on_accept_response(const ElectionAcceptResponseMsg &accep
ElectionAcceptResponseMsg mock_self_accept_response_msg(p_election_->self_addr_,
p_election_->inner_priority_seed_,
p_election_->get_membership_version_(),
p_election_->get_ls_biggest_min_cluster_version_ever_seen_(),
ElectionAcceptRequestMsg(p_election_->id_,
p_election_->self_addr_,
restart_counter_,
ballot_number_,
p_election_->get_ls_biggest_min_cluster_version_ever_seen_(),
0,
record_lease_interval_,
p_election_->get_membership_version_()));
@ -635,6 +641,7 @@ void ElectionProposer::inner_change_leader_to(const ObAddr &dst)
p_election_->get_self_addr(),
restart_counter_,
ballot_number_,
p_election_->get_ls_biggest_min_cluster_version_ever_seen_(),
switch_source_leader_ballot,
p_election_->get_membership_version_());
if (OB_LIKELY(leader_revoke_if_lease_expired_(RoleChangeReason::ChangeLeaderToRevoke))) {

1
src/logservice/palf/election/interface/election_priority.h
View File

@ -38,6 +38,7 @@ public:
virtual int refresh() = 0;
// 在priority间进行比较的方法
virtual int compare_with(const ElectionPriority &rhs,
const uint64_t compare_version,
int &result,
common::ObStringHolder &reason) const = 0;
virtual int get_size_of_impl_type() const = 0;

16
src/logservice/palf/election/message/election_message.cpp
View File

@ -101,11 +101,13 @@ ElectionMsgBase::ElectionMsgBase(const int64_t id,
const common::ObAddr &self_addr,
const int64_t restart_counter,
const int64_t ballot_number,
const LsBiggestMinClusterVersionEverSeen &version,
const ElectionMsgType msg_type) :
id_(id),
sender_(self_addr),
restart_counter_(restart_counter),
ballot_number_(ballot_number),
biggest_min_cluster_version_ever_seen_(version),
msg_type_(static_cast<int64_t>(msg_type)) {
debug_ts_.src_construct_ts_ = ObClockGenerator::getRealClock();
}
@ -116,6 +118,7 @@ void ElectionMsgBase::reset()
receiver_.reset();
restart_counter_ = INVALID_VALUE;
ballot_number_ = INVALID_VALUE;
biggest_min_cluster_version_ever_seen_.version_ = 0;
msg_type_ = static_cast<int64_t>(ElectionMsgType::INVALID_TYPE);
}
@ -125,6 +128,9 @@ int64_t ElectionMsgBase::get_restart_counter() const { return restart_counter_;
int64_t ElectionMsgBase::get_ballot_number() const { return ballot_number_; }
const LsBiggestMinClusterVersionEverSeen &ElectionMsgBase::get_ls_biggest_min_cluster_version_ever_seen() const
{ return biggest_min_cluster_version_ever_seen_; }
bool ElectionMsgBase::is_valid() const
{
return sender_.is_valid() && receiver_.is_valid() && restart_counter_ != INVALID_VALUE && ballot_number_ != INVALID_VALUE &&
@ -155,12 +161,14 @@ ElectionPrepareRequestMsgMiddle::ElectionPrepareRequestMsgMiddle(const int64_t i
const common::ObAddr &self_addr,
const int64_t restart_counter,
const int64_t ballot_number,
const LsBiggestMinClusterVersionEverSeen &version,
const uint64_t inner_priority_seed,
const LogConfigVersion membership_version) :
ElectionMsgBase(id,
self_addr,
restart_counter,
ballot_number,
version,
ElectionMsgType::PREPARE_REQUEST),
role_(ObRole::INVALID_ROLE),
is_buffer_valid_(false),
@ -210,11 +218,13 @@ accepted_(false) {}
ElectionPrepareResponseMsgMiddle::
ElectionPrepareResponseMsgMiddle(const ObAddr &self_addr,
const LsBiggestMinClusterVersionEverSeen &version,
const ElectionPrepareRequestMsgMiddle &request) :
ElectionMsgBase(request.get_id(),
self_addr,
request.get_restart_counter(),
INVALID_VALUE,
version,
ElectionMsgType::PREPARE_RESPONSE),
accepted_(false) {
set_receiver(request.get_sender());
@ -250,6 +260,7 @@ ElectionAcceptRequestMsgMiddle::ElectionAcceptRequestMsgMiddle(const int64_t id,
const ObAddr &self_addr,
const int64_t restart_counter,
const int64_t ballot_number,
const LsBiggestMinClusterVersionEverSeen &version,
const int64_t lease_start_ts_on_proposer,
const int64_t lease_interval,
const LogConfigVersion membership_version) :
@ -257,6 +268,7 @@ ElectionMsgBase(id,
self_addr,
restart_counter,
ballot_number,
version,
ElectionMsgType::ACCEPT_REQUEST),
lease_start_ts_on_proposer_(lease_start_ts_on_proposer),
lease_interval_(lease_interval),
@ -289,11 +301,13 @@ ElectionAcceptResponseMsgMiddle::
ElectionAcceptResponseMsgMiddle(const ObAddr &self_addr,
const uint64_t inner_priority_seed,
const LogConfigVersion &membership_version,
const LsBiggestMinClusterVersionEverSeen &version,
const ElectionAcceptRequestMsgMiddle &request) :
ElectionMsgBase(request.get_id(),
self_addr,
request.get_restart_counter(),
INVALID_VALUE,
version,
ElectionMsgType::ACCEPT_RESPONSE),
lease_started_ts_on_proposer_(request.get_lease_start_ts_on_proposer()),
lease_interval_(request.get_lease_interval()),
@ -361,12 +375,14 @@ ElectionChangeLeaderMsgMiddle::ElectionChangeLeaderMsgMiddle(const int64_t id,
const ObAddr &self_addr,
const int64_t restart_counter,
const int64_t ballot_number,
const LsBiggestMinClusterVersionEverSeen &version,
int64_t switch_source_leader_ballot,
const LogConfigVersion membership_version) :
ElectionMsgBase(id,
self_addr,
restart_counter,
ballot_number,
version,
ElectionMsgType::CHANGE_LEADER),
switch_source_leader_ballot_(switch_source_leader_ballot),
membership_version_(membership_version) {}

49
src/logservice/palf/election/message/election_message.h
View File

@ -34,6 +34,27 @@ namespace palf
namespace election
{
struct LsBiggestMinClusterVersionEverSeen {// this is for maintain min_cluster_version on arb server
LsBiggestMinClusterVersionEverSeen() : version_(0) {}
LsBiggestMinClusterVersionEverSeen(const uint64_t version)
: version_(version) {};
void try_advance(uint64_t new_version) {
if (version_ < new_version) {
version_ = new_version;
ELECT_LOG(INFO, "advance ls ever seen biggest min_cluster_verson");
}
}
int64_t to_string(char *buf, const int64_t len) const {
int64_t pos = 0;
databuff_printf(buf, len, pos, "%u.", OB_VSN_MAJOR(version_));
databuff_printf(buf, len, pos, "%u.", OB_VSN_MINOR(version_));
databuff_printf(buf, len, pos, "%u.", OB_VSN_MAJOR_PATCH(version_));
databuff_printf(buf, len, pos, "%u", OB_VSN_MINOR_PATCH(version_));
return pos;
}
uint64_t version_;
};
struct CompatHelper {
static constexpr int64_t BIT_MASK_NOT_LESS_THAN_4_2 = (1LL << 63);// this bit is used for mark message needed seralized in lazy-mode, and for compat reason
static void set_msg_flag_not_less_than_4_2(const ElectionMsgBase &msg);
@ -76,6 +97,7 @@ public:
const common::ObAddr &self_addr,
const int64_t restart_counter,
const int64_t ballot_number,
const LsBiggestMinClusterVersionEverSeen &version,
const ElectionMsgType msg_type);
void reset();
void set_receiver(const common::ObAddr &addr);
@ -83,6 +105,7 @@ public:
int64_t get_ballot_number() const;
const common::ObAddr &get_sender() const;
const common::ObAddr &get_receiver() const;
const LsBiggestMinClusterVersionEverSeen &get_ls_biggest_min_cluster_version_ever_seen() const;
ElectionMsgType get_msg_type() const;
bool is_valid() const;
ElectionMsgDebugTs get_debug_ts() const;
@ -90,7 +113,7 @@ public:
int64_t get_id() const;
#define MSG_TYPE "msg_type", msg_type_to_string(static_cast<ElectionMsgType>(msg_type_))
TO_STRING_KV(MSG_TYPE, K_(id), K_(sender), K_(receiver), K_(restart_counter),
K_(ballot_number), K_(debug_ts));
K_(ballot_number), K_(debug_ts), K_(biggest_min_cluster_version_ever_seen));
#undef MSG_TYPE
protected:
int64_t id_;
@ -98,11 +121,13 @@ protected:
common::ObAddr receiver_;
int64_t restart_counter_;
int64_t ballot_number_;
LsBiggestMinClusterVersionEverSeen biggest_min_cluster_version_ever_seen_;
int64_t msg_type_;
ElectionMsgDebugTs debug_ts_;
};
OB_SERIALIZE_MEMBER_TEMP(inline, ElectionMsgBase, id_, sender_, receiver_,
restart_counter_, ballot_number_, msg_type_, debug_ts_);
restart_counter_, ballot_number_, msg_type_, debug_ts_,
biggest_min_cluster_version_ever_seen_.version_);
class ElectionPrepareRequestMsgMiddle : public ElectionMsgBase
{
@ -113,6 +138,7 @@ public:
const common::ObAddr &self_addr,
const int64_t restart_counter,
const int64_t ballot_number,
const LsBiggestMinClusterVersionEverSeen &version,
const uint64_t inner_priority_seed,
const LogConfigVersion membership_version);
int set(const ElectionPriority *priority, const common::ObRole role);
@ -146,9 +172,10 @@ public:
const common::ObAddr &self_addr,
const int64_t restart_counter,
const int64_t ballot_number,
const LsBiggestMinClusterVersionEverSeen &version,
const uint64_t inner_priority_seed,
const LogConfigVersion membership_version) :
ElectionPrepareRequestMsgMiddle(id, self_addr, restart_counter, ballot_number, inner_priority_seed, membership_version) {}
ElectionPrepareRequestMsgMiddle(id, self_addr, restart_counter, ballot_number, version, inner_priority_seed, membership_version) {}
int deserialize(const char* buf, const int64_t data_len, int64_t& pos) {
int ret = OB_SUCCESS;
if (OB_FAIL(ElectionPrepareRequestMsgMiddle::deserialize(buf, data_len, pos))) {
@ -173,6 +200,7 @@ class ElectionPrepareResponseMsgMiddle : public ElectionMsgBase
public:
ElectionPrepareResponseMsgMiddle();// default constructor is required by deserialization, but not actually worked
ElectionPrepareResponseMsgMiddle(const common::ObAddr &self_addr,
const LsBiggestMinClusterVersionEverSeen &version,
const ElectionPrepareRequestMsgMiddle &request);
void set_accepted(const int64_t ballot_number, const Lease lease);
void set_rejected(const int64_t ballot_number);
@ -196,8 +224,9 @@ class ElectionPrepareResponseMsg : public ElectionPrepareResponseMsgMiddle
public:
ElectionPrepareResponseMsg() : ElectionPrepareResponseMsgMiddle() {}// default constructor is required by deserialization, but not actually worked
ElectionPrepareResponseMsg(const common::ObAddr &self_addr,
const LsBiggestMinClusterVersionEverSeen &version,
const ElectionPrepareRequestMsgMiddle &request) :
ElectionPrepareResponseMsgMiddle(self_addr, request) {}
ElectionPrepareResponseMsgMiddle(self_addr, version, request) {}
int deserialize(const char* buf, const int64_t data_len, int64_t& pos) {
int ret = OB_SUCCESS;
if (OB_FAIL(ElectionPrepareResponseMsgMiddle::deserialize(buf, data_len, pos))) {
@ -225,6 +254,7 @@ public:
const common::ObAddr &self_addr,
const int64_t restart_counter,
const int64_t ballot_number,
const LsBiggestMinClusterVersionEverSeen &version,
const int64_t lease_start_ts_on_proposer,
const int64_t lease_interval,
const LogConfigVersion membership_version);
@ -253,10 +283,11 @@ public:
const common::ObAddr &self_addr,
const int64_t restart_counter,
const int64_t ballot_number,
const LsBiggestMinClusterVersionEverSeen &version,
const int64_t lease_start_ts_on_proposer,
const int64_t lease_interval,
const LogConfigVersion membership_version) :
ElectionAcceptRequestMsgMiddle(id, self_addr, restart_counter, ballot_number,
ElectionAcceptRequestMsgMiddle(id, self_addr, restart_counter, ballot_number, version,
lease_start_ts_on_proposer, lease_interval, membership_version) {}
int deserialize(const char* buf, const int64_t data_len, int64_t& pos) {
int ret = OB_SUCCESS;
@ -332,6 +363,7 @@ public:
ElectionAcceptResponseMsgMiddle(const common::ObAddr &self_addr,
const uint64_t inner_priority_seed,
const LogConfigVersion &membership_version,
const LsBiggestMinClusterVersionEverSeen &version,
const ElectionAcceptRequestMsgMiddle &request);
int set_accepted(const int64_t ballot_number, const ElectionPriority *priority);
void set_rejected(const int64_t ballot_number);
@ -373,8 +405,9 @@ public:
ElectionAcceptResponseMsg(const common::ObAddr &self_addr,
const uint64_t inner_priority_seed,
const LogConfigVersion &membership_version,
const LsBiggestMinClusterVersionEverSeen &version,
const ElectionAcceptRequestMsgMiddle &request) :
ElectionAcceptResponseMsgMiddle(self_addr, inner_priority_seed, membership_version, request) {}
ElectionAcceptResponseMsgMiddle(self_addr, inner_priority_seed, membership_version, version, request) {}
int deserialize(const char* buf, const int64_t data_len, int64_t& pos) {
int ret = OB_SUCCESS;
if (OB_FAIL(ElectionAcceptResponseMsgMiddle::deserialize(buf, data_len, pos))) {
@ -402,6 +435,7 @@ public:
const common::ObAddr &self_addr,
const int64_t restart_counter,
const int64_t get_ballot_number,
const LsBiggestMinClusterVersionEverSeen &version,
int64_t switch_source_leader_ballot,
const LogConfigVersion membership_version);
int64_t get_old_ballot_number() const;
@ -425,9 +459,10 @@ public:
const common::ObAddr &self_addr,
const int64_t restart_counter,
const int64_t get_ballot_number,
const LsBiggestMinClusterVersionEverSeen &version,
int64_t switch_source_leader_ballot,
const LogConfigVersion membership_version) :
ElectionChangeLeaderMsgMiddle(id, self_addr, restart_counter, get_ballot_number,
ElectionChangeLeaderMsgMiddle(id, self_addr, restart_counter, get_ballot_number, version,
switch_source_leader_ballot, membership_version) {}
int deserialize(const char* buf, const int64_t data_len, int64_t& pos) {
int ret = OB_SUCCESS;

27
unittest/logservice/test_ob_election_message_compat.cpp
View File

@ -101,7 +101,26 @@ protected:
OB_SERIALIZE_MEMBER_TEMP(inline, ElectionMsgBase, id_, sender_, receiver_,
restart_counter_, ballot_number_, msg_type_, debug_ts_);
class ElectionPrepareRequestMsgMiddleOld : public oceanbase::palf::election::ElectionMsgBase
ElectionMsgBase::ElectionMsgBase() :
id_(INVALID_VALUE),
restart_counter_(INVALID_VALUE),
ballot_number_(INVALID_VALUE),
msg_type_(static_cast<int64_t>(ElectionMsgType::INVALID_TYPE)) {}
ElectionMsgBase::ElectionMsgBase(const int64_t id,
const common::ObAddr &self_addr,
const int64_t restart_counter,
const int64_t ballot_number,
const ElectionMsgType msg_type) :
id_(id),
sender_(self_addr),
restart_counter_(restart_counter),
ballot_number_(ballot_number),
msg_type_(static_cast<int64_t>(msg_type)) {
debug_ts_.src_construct_ts_ = ObClockGenerator::getRealClock();
}
class ElectionPrepareRequestMsgMiddleOld : public oceanbase::unittest::ElectionMsgBase
{
OB_UNIS_VERSION(1);
public:
@ -189,13 +208,13 @@ public:
int deserialize(const char* buf, const int64_t data_len, int64_t& pos) {
int ret = ElectionPrepareRequestMsgMiddleOld::deserialize(buf, data_len, pos);
debug_ts_.dest_deserialize_ts_ = ObClockGenerator::getRealClock();
print_debug_ts_if_reach_warn_threshold(*this, MSG_DELAY_WARN_THRESHOLD);
// print_debug_ts_if_reach_warn_threshold(*this, MSG_DELAY_WARN_THRESHOLD);
return ret;
}
int64_t get_serialize_size() const {
if (debug_ts_.src_serialize_ts_ == 0) {// cause get_serialize_size maybe call more than once
const_cast<int64_t&>(debug_ts_.src_serialize_ts_) = ObClockGenerator::getRealClock();
print_debug_ts_if_reach_warn_threshold(*this, MSG_DELAY_WARN_THRESHOLD);
// print_debug_ts_if_reach_warn_threshold(*this, MSG_DELAY_WARN_THRESHOLD);
}
return ElectionPrepareRequestMsgMiddleOld::get_serialize_size();
}
@ -206,7 +225,7 @@ TEST_F(TestElectionMsgCompat, old_new_msg_serialize) {
config_version.generate(1, 1);
ElectionPrepareRequestMsgOld prepare_msg_old1(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, 1, config_version);
ElectionPrepareRequestMsg prepare_msg_new1;
ElectionPrepareRequestMsg prepare_msg_new2(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, 1, (1 << 10), config_version);
ElectionPrepareRequestMsg prepare_msg_new2(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, 1, LsBiggestMinClusterVersionEverSeen(CLUSTER_VERSION_4_1_0_0), (1 << 10), config_version);
ElectionPrepareRequestMsgOld prepare_msg_old2;
constexpr int64_t buffer_size = 2_KB;
char buffer[buffer_size];

12
unittest/logservice/test_ob_election_message_compat2.cpp
View File

@ -764,7 +764,7 @@ TEST_F(TestElectionMsgCompat2, old_to_new) {
LogConfigVersion config_version;
config_version.generate(1, 1);
int64_t pos = 0;
unittest::ElectionAcceptRequestMsg msg_request_old(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, 1, 123, 234, config_version);
unittest::ElectionAcceptRequestMsg msg_request_old(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, 1, LsBiggestMinClusterVersionEverSeen(CLUSTER_VERSION_4_1_0_0), 123, 234, config_version);
palf::election::ElectionAcceptRequestMsg msg_request_new;
ASSERT_EQ(msg_request_old.serialize(buffer, BUFFER_SIZE, pos), OB_SUCCESS);
int64_t pos2 = 0;
@ -773,7 +773,7 @@ TEST_F(TestElectionMsgCompat2, old_to_new) {
ASSERT_EQ(msg_request_new.flag_not_less_than_4_2_, false);
ElectionPriorityImpl priority;
unittest::ElectionAcceptResponseMsg msg_response_old(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, config_version, msg_request_old);
unittest::ElectionAcceptResponseMsg msg_response_old(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, config_version, LsBiggestMinClusterVersionEverSeen(CLUSTER_VERSION_4_1_0_0), msg_request_old);
palf::election::ElectionAcceptResponseMsg msg_response_new;
ASSERT_EQ(msg_response_old.set_accepted(1, &priority), OB_SUCCESS);
pos = 0;
@ -794,7 +794,7 @@ TEST_F(TestElectionMsgCompat2, new_to_old_fake_new) {
int64_t pos = 0;
ASSERT_EQ(observer::ObServer::get_instance().is_arbitration_mode(), false);
ASSERT_EQ(GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_2_0_0, true);
palf::election::ElectionAcceptRequestMsg msg_request_new(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, 1, 123, 234, config_version);
palf::election::ElectionAcceptRequestMsg msg_request_new(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, 1, LsBiggestMinClusterVersionEverSeen(CLUSTER_VERSION_4_1_0_0), 123, 234, config_version);
ASSERT_EQ(msg_request_new.flag_not_less_than_4_2_, false);
unittest::ElectionAcceptRequestMsg msg_request_old;
int64_t serialize_size = msg_request_new.get_serialize_size();
@ -805,7 +805,7 @@ TEST_F(TestElectionMsgCompat2, new_to_old_fake_new) {
ASSERT_EQ(pos, pos2);
ElectionPriorityImpl priority;
palf::election::ElectionAcceptResponseMsg msg_response_new(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, config_version, msg_request_old);
palf::election::ElectionAcceptResponseMsg msg_response_new(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, config_version, LsBiggestMinClusterVersionEverSeen(CLUSTER_VERSION_4_1_0_0), msg_request_old);
ASSERT_EQ(msg_response_new.flag_not_less_than_4_2_, false);
unittest::ElectionAcceptResponseMsg msg_response_old;
ASSERT_EQ(msg_response_new.set_accepted(1, &priority), OB_SUCCESS);
@ -828,7 +828,7 @@ TEST_F(TestElectionMsgCompat2, new_to_new_real_new) {
int64_t pos = 0;
ASSERT_EQ(observer::ObServer::get_instance().is_arbitration_mode(), false);
ASSERT_EQ(GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_2_0_0, false);
palf::election::ElectionAcceptRequestMsg msg_request_src(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, 1, 123, 234, config_version);
palf::election::ElectionAcceptRequestMsg msg_request_src(1, ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, 1, LsBiggestMinClusterVersionEverSeen(CLUSTER_VERSION_4_1_0_0), 123, 234, config_version);
ASSERT_EQ(msg_request_src.flag_not_less_than_4_2_, true);
palf::election::ElectionAcceptRequestMsg msg_request_dst;
int64_t serialize_size = msg_request_src.get_serialize_size();
@ -840,7 +840,7 @@ TEST_F(TestElectionMsgCompat2, new_to_new_real_new) {
ASSERT_EQ(msg_request_dst.flag_not_less_than_4_2_, true);
ElectionPriorityImpl priority;
palf::election::ElectionAcceptResponseMsg msg_response_src(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, config_version, msg_request_dst);
palf::election::ElectionAcceptResponseMsg msg_response_src(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", 1), 1, config_version, LsBiggestMinClusterVersionEverSeen(CLUSTER_VERSION_4_1_0_0), msg_request_dst);
ASSERT_EQ(msg_response_src.flag_not_less_than_4_2_, true);
palf::election::ElectionAcceptResponseMsg msg_response_dst;
ASSERT_EQ(msg_response_src.set_accepted(1, &priority), OB_SUCCESS);

158
unittest/logservice/test_ob_election_with_priority.cpp
View File

@ -28,6 +28,7 @@
#include "share/ob_occam_timer.h"
#include "share/rc/ob_tenant_base.h"
#include "mock_logservice_container/mock_election_user.h"
#include "observer/ob_server.h"
#include <iostream>
#include <vector>
@ -55,6 +56,10 @@ namespace logservice
{
namespace coordinator
{
int PriorityV0::refresh_(const share::ObLSID &)
{
return OB_SUCCESS;
}
int PriorityV1::refresh_(const share::ObLSID &)
{
return OB_SUCCESS;
@ -113,7 +118,7 @@ public:
};
template <typename TAKEOVER_OP>
vector<ElectionImpl *> create_election_group(vector<ElectionPriorityImpl> &v_pri, TAKEOVER_OP &&op)
vector<ElectionImpl *> create_election_group(vector<ElectionPriorityImpl> &v_pri, TAKEOVER_OP &&op, const vector<int> &v_port = {})
{
vector<ElectionImpl *> v;
int election_num = v_pri.size();
@ -121,8 +126,12 @@ vector<ElectionImpl *> create_election_group(vector<ElectionPriorityImpl> &v_pri
MemberList member_list;
ObArray<ObAddr> addr_list;
static int port = 1;
for (int i = 0; i < election_num; ++i)
addr_list.push_back(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", port + i));
if (v_port.empty())
for (int i = 0; i < election_num; ++i)
addr_list.push_back(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", port + i));
else
for (int port : v_port)
addr_list.push_back(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", port));
palf::LogConfigVersion version;
version.proposal_id_ = 1;
version.config_seq_ = 1;
@ -131,7 +140,7 @@ vector<ElectionImpl *> create_election_group(vector<ElectionPriorityImpl> &v_pri
int ret = OB_SUCCESS;
for (int i = 0; i < election_num; ++i) {
ElectionImpl *election = new ElectionImpl();
election->self_addr_ = ObAddr(ObAddr::VER::IPV4, "127.0.0.1", port + i);
election->self_addr_ = ObAddr(ObAddr::VER::IPV4, "127.0.0.1", addr_list[i].port_);
v.push_back(election);
}
for (auto &election_1 : v) {
@ -145,7 +154,7 @@ vector<ElectionImpl *> create_election_group(vector<ElectionPriorityImpl> &v_pri
1,
&timer,
&GlobalNetService,
ObAddr(ObAddr::VER::IPV4, "127.0.0.1", port + index),
election->self_addr_,
true,
1,
[election](int64_t, const ObAddr &dest_addr) {
@ -357,9 +366,148 @@ TEST_F(TestElectionWithPriority, meet_fatal_failure)
delete election;
ASSERT_EQ(change_leader_from_prepare_change_leader_cb, false);
}
}
}
namespace oceanbase
{
namespace palf
{
namespace election
{
uint64_t ElectionImpl::get_ls_biggest_min_cluster_version_ever_seen_() const// 让port=5555的副本认为自己是A副本
{
#define PRINT_WRAPPER K(*this)
int ret = OB_SUCCESS;
uint64_t ls_biggest_min_cluster_version_ever_seen = 0;
//if (observer::ObServer::get_instance().is_arbitration_mode()) {
if (observer::ObServer::get_instance().is_arbitration_mode() || self_addr_.port_ == 5555) {
if (CLUSTER_CURRENT_VERSION < ls_biggest_min_cluster_version_ever_seen_.version_) {
ret = OB_ERR_UNEXPECTED;
LOG_NONE(ERROR, "ls_biggest_min_cluster_version_ever_seen_ less than arb binary version");
} else if (ls_biggest_min_cluster_version_ever_seen_.version_ == 0) {
LOG_NONE(WARN, "ls_biggest_min_cluster_version_ever_seen_ not setted yet");
}
ls_biggest_min_cluster_version_ever_seen = ls_biggest_min_cluster_version_ever_seen_.version_;
} else {
ls_biggest_min_cluster_version_ever_seen = std::max(GET_MIN_CLUSTER_VERSION(),
ls_biggest_min_cluster_version_ever_seen_.version_);
}
return ls_biggest_min_cluster_version_ever_seen;
#undef PRINT_WRAPPER
}
}
}
namespace unittest
{
TEST_F(TestElectionWithPriority, arb_server_split_vote_cause_not_set_priority)// 复现仲裁bug场景
{
// oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_3_2_3_0;// 此时采用V0版本的优先级逻辑比较，投给IP
vector<ElectionPriorityImpl> v_pri(4);
for (auto &pri : v_pri)
init_pri(pri);
v_pri[0].priority_tuple_.element<1>().zone_priority_ = 3;//F
v_pri[1].priority_tuple_.element<1>().zone_priority_ = 2;//F
v_pri[2].priority_tuple_.element<1>().zone_priority_ = 1;//F
v_pri[3].priority_tuple_.element<1>().zone_priority_ = 0;//A
auto election_group = create_election_group(v_pri, [](){}, {1,2,3,5555/*仲裁*/});
election_group[0]->stop();// kill掉一个，还有2F1A
election_group[3]->set_inner_priority_seed(0ULL | static_cast<uint64_t>(palf::election::PRIORITY_SEED_BIT::SEED_NOT_NORMOL_REPLICA_BIT));
election_group[3]->reset_priority();// 移除优先级，模拟A副本，会投票给IP最小的副本
this_thread::sleep_for(chrono::seconds(5));// 等待选出第一任Leader
ASSERT_EQ(election_group[1]->proposer_.role_, ObRole::FOLLOWER);
ASSERT_EQ(election_group[2]->proposer_.role_, ObRole::FOLLOWER);
ASSERT_EQ(election_group[3]->proposer_.role_, ObRole::FOLLOWER);
ASSERT_EQ(leader_takeover_times, 0);
ASSERT_EQ(leader_revoke_times, 0);
ASSERT_EQ(devote_to_be_leader_count, 0);
ASSERT_EQ(lease_expired_to_be_follower_count, 0);
ASSERT_EQ(change_leader_to_be_leader_count, 0);
ASSERT_EQ(change_leader_to_be_follower_count, 0);
ASSERT_EQ(stop_to_be_follower_count, 0);
for (auto &election : election_group)
election->stop();
this_thread::sleep_for(chrono::seconds(1));
for (auto &election : election_group)
delete election;
}
TEST_F(TestElectionWithPriority, arb_server_won_t_split_vote_cause_set_priority)// 测试修复后的行为
{
oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_3_2_3_0;// 此时采用V0版本的优先级逻辑比较，投给port_number_较大的副本
vector<ElectionPriorityImpl> v_pri(4);
for (auto &pri : v_pri)
init_pri(pri);
// 优先级V0
v_pri[0].priority_tuple_.element<0>().port_number_ = 3;//F
v_pri[1].priority_tuple_.element<0>().port_number_ = 2;//F
v_pri[2].priority_tuple_.element<0>().port_number_ = 1;//F
v_pri[3].priority_tuple_.element<0>().port_number_ = 0;//A
// 优先级V1
v_pri[0].priority_tuple_.element<1>().zone_priority_ = 3;//F
v_pri[1].priority_tuple_.element<1>().zone_priority_ = 2;//F
v_pri[2].priority_tuple_.element<1>().zone_priority_ = 1;//F
v_pri[3].priority_tuple_.element<1>().zone_priority_ = 0;//A
auto election_group = create_election_group(v_pri, [](){}, {1,2,3,5555/*仲裁*/});
election_group[0]->stop();// kill掉一个，还有2F1A
election_group[3]->set_inner_priority_seed(0ULL | static_cast<uint64_t>(palf::election::PRIORITY_SEED_BIT::SEED_NOT_NORMOL_REPLICA_BIT));
this_thread::sleep_for(chrono::seconds(5));// 等待选出第一任Leader
ASSERT_EQ(election_group[1]->proposer_.role_, ObRole::LEADER);
ASSERT_EQ(election_group[2]->proposer_.role_, ObRole::FOLLOWER);
ASSERT_EQ(election_group[3]->proposer_.role_, ObRole::FOLLOWER);
ASSERT_EQ(leader_takeover_times, 1);
ASSERT_EQ(leader_revoke_times, 0);
ASSERT_EQ(devote_to_be_leader_count, 1);
ASSERT_EQ(lease_expired_to_be_follower_count, 0);
ASSERT_EQ(change_leader_to_be_leader_count, 0);
ASSERT_EQ(change_leader_to_be_follower_count, 0);
ASSERT_EQ(stop_to_be_follower_count, 0);
// 升级版本号
oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_4_2_0_0;// 此时将根据V1版本的优先级，将leader切换至election 2
this_thread::sleep_for(chrono::seconds(2));// 等待执行切主
ASSERT_EQ(election_group[1]->proposer_.role_, ObRole::FOLLOWER);
ASSERT_EQ(election_group[2]->proposer_.role_, ObRole::LEADER);
ASSERT_EQ(election_group[3]->proposer_.role_, ObRole::FOLLOWER);
ASSERT_EQ(leader_takeover_times, 2);
ASSERT_EQ(leader_revoke_times, 1);
ASSERT_EQ(devote_to_be_leader_count, 1);
ASSERT_EQ(lease_expired_to_be_follower_count, 0);
ASSERT_EQ(change_leader_to_be_leader_count, 1);
ASSERT_EQ(change_leader_to_be_follower_count, 1);
ASSERT_EQ(stop_to_be_follower_count, 0);
// 给leader断网，触发无主选举
GlobalNetService.disconnect_two_side(election_group[2], election_group[1]);
GlobalNetService.disconnect_two_side(election_group[2], election_group[3]);
this_thread::sleep_for(chrono::seconds(5));// 等待leader卸任
// 恢复leader的网络，预期无主时，就算是仲裁副本也正确采用了V1版本的优先级
GlobalNetService.connect_two_side(election_group[2], election_group[1]);
GlobalNetService.connect_two_side(election_group[2], election_group[3]);
this_thread::sleep_for(chrono::seconds(5));// 等待无主选举
ASSERT_EQ(election_group[1]->proposer_.role_, ObRole::FOLLOWER);
ASSERT_EQ(election_group[2]->proposer_.role_, ObRole::LEADER);
ASSERT_EQ(election_group[3]->proposer_.role_, ObRole::FOLLOWER);
ASSERT_EQ(leader_takeover_times, 3);
ASSERT_EQ(leader_revoke_times, 2);
ASSERT_EQ(devote_to_be_leader_count, 2);
ASSERT_EQ(lease_expired_to_be_follower_count, 1);
ASSERT_EQ(change_leader_to_be_leader_count, 1);
ASSERT_EQ(change_leader_to_be_follower_count, 1);
ASSERT_EQ(stop_to_be_follower_count, 0);
for (auto &election : election_group)
election->stop();
this_thread::sleep_for(chrono::seconds(1));
for (auto &election : election_group)
delete election;
}
}
}
int main(int argc, char **argv)
{