hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Election] optimize election accept response message size

Browse Source
This commit is contained in:
fengdeyiji
2023-05-29 06:41:43 +00:00
committed by ob-robot
parent 95f9b3a05e
commit 774a89db04
5 changed files with 1016 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/logservice/palf/election/algorithm/election_proposer.cpp
View File

@ -246,7 +246,7 @@ int ElectionProposer::register_renew_lease_task_()
} else if (role_ != ObRole::LEADER) {
LOG_RENEW_LEASE(ERROR, "unexpected role status");
} else if (OB_UNLIKELY(leader_revoke_if_lease_expired_(RoleChangeReason::LeaseExpiredToRevoke))) {
LOG_RENEW_LEASE(ERROR, "leader lease expired, leader revoked");
LOG_RENEW_LEASE(WARN, "leader lease expired, leader revoked");
} else if (prepare_success_ballot_ != ballot_number_) {// 需要进行leader prepare推大用于续约的ballot number
LOG_RENEW_LEASE(INFO, "prepare_success_ballot_ not same as ballot_number_, maybe in Leader Prepare phase, gie up renew lease this time");
} else {

92
src/logservice/palf/election/message/election_message.cpp
View File

@ -22,6 +22,7 @@
#include "logservice/palf/election/algorithm/election_proposer.h"
#include "logservice/palf/election/utils/election_args_checker.h"
#include "logservice/palf/election/utils/election_common_define.h"
#include "observer/ob_server.h"
namespace oceanbase
{
@ -33,6 +34,18 @@ namespace election
using namespace common;
using namespace share;
void CompatHelper::set_msg_flag_not_less_than_4_2(const ElectionMsgBase &msg) {
ElectionMsgBase &cast_msg = const_cast<ElectionMsgBase &>(msg);
cast_msg.msg_type_ |= BIT_MASK_NOT_LESS_THAN_4_2;
}
bool CompatHelper::fetch_msg_flag_not_less_than_4_2(const ElectionMsgBase &msg) {
ElectionMsgBase &cast_msg = const_cast<ElectionMsgBase &>(msg);
bool ret = (cast_msg.msg_type_ & BIT_MASK_NOT_LESS_THAN_4_2);
cast_msg.msg_type_ &= ~BIT_MASK_NOT_LESS_THAN_4_2;
return ret;
}
// this is important debug info when meet Lease Expired ERROR! which is a high frequecy error in election
void print_debug_ts_if_reach_warn_threshold(const ElectionMsgBase &msg, const int64_t warn_threshold)
{
@ -227,7 +240,9 @@ const Lease &ElectionPrepareResponseMsgMiddle::get_lease() const { return lease_
ElectionAcceptRequestMsgMiddle::ElectionAcceptRequestMsgMiddle() :
ElectionMsgBase(),
lease_start_ts_on_proposer_(0),
lease_interval_(0) {}
lease_interval_(0),
membership_version_(),
flag_not_less_than_4_2_(false) {}
ElectionMsgDebugTs ElectionPrepareResponseMsgMiddle::get_request_debug_ts() const { return request_debug_ts_; }
@ -245,13 +260,16 @@ ElectionMsgBase(id,
ElectionMsgType::ACCEPT_REQUEST),
lease_start_ts_on_proposer_(lease_start_ts_on_proposer),
lease_interval_(lease_interval),
membership_version_(membership_version) {}
membership_version_(membership_version),
flag_not_less_than_4_2_(!observer::ObServer::get_instance().is_arbitration_mode() && GET_MIN_CLUSTER_VERSION() >= CLUSTER_VERSION_4_2_0_0) {}
int64_t ElectionAcceptRequestMsgMiddle::get_lease_start_ts_on_proposer() const
{
return lease_start_ts_on_proposer_;
}
LogConfigVersion ElectionAcceptRequestMsgMiddle::get_membership_version() const { return membership_version_; }
int64_t ElectionAcceptRequestMsgMiddle::get_lease_interval() const { return lease_interval_; }
ElectionAcceptResponseMsgMiddle::ElectionAcceptResponseMsgMiddle() :
@ -260,12 +278,12 @@ lease_started_ts_on_proposer_(0),
lease_interval_(0),
accepted_(false),
is_buffer_valid_(false),
inner_priority_seed_(static_cast<uint64_t>(PRIORITY_SEED_BIT::DEFAULT_SEED))
{
memset(priority_buffer_, 0, PRIORITY_BUFFER_SIZE);
}
LogConfigVersion ElectionAcceptRequestMsgMiddle::get_membership_version() const { return membership_version_; }
inner_priority_seed_(static_cast<uint64_t>(PRIORITY_SEED_BIT::DEFAULT_SEED)),
responsed_membership_version_(),
membership_version_(),
request_debug_ts_(),
fixed_buffer_(priority_buffer_),
flag_not_less_than_4_2_(false) {}
ElectionAcceptResponseMsgMiddle::
ElectionAcceptResponseMsgMiddle(const ObAddr &self_addr,
@ -283,10 +301,11 @@ accepted_(false),
is_buffer_valid_(false),
inner_priority_seed_(inner_priority_seed),
responsed_membership_version_(request.get_membership_version()),
membership_version_(membership_version)
{
membership_version_(membership_version),
request_debug_ts_(),
fixed_buffer_(priority_buffer_),
flag_not_less_than_4_2_(request.not_less_than_4_2()) {
set_receiver(request.get_sender());
memset(priority_buffer_, 0, PRIORITY_BUFFER_SIZE);
request_debug_ts_ = request.get_debug_ts();
}
@ -298,11 +317,12 @@ int ElectionAcceptResponseMsgMiddle::set_accepted(const int64_t ballot_number,
accepted_ = true;
int64_t pos = 0;
if (OB_NOT_NULL(priority)) {
if (CLICK_FAIL(priority->serialize((char*)priority_buffer_, PRIORITY_BUFFER_SIZE, pos))) {
if (CLICK_FAIL(priority->serialize((char*)fixed_buffer_.priority_buffer_, PRIORITY_BUFFER_SIZE, pos))) {
ELECT_LOG(ERROR, "fail to serialize priority");
} else {
is_buffer_valid_ = true;
}
fixed_buffer_.buffer_used_size_ = pos;
}
return ret;
}
@ -355,6 +375,54 @@ LogConfigVersion ElectionChangeLeaderMsgMiddle::get_membership_version() const {
int64_t ElectionChangeLeaderMsgMiddle::get_old_ballot_number() const { return switch_source_leader_ballot_; }
#define OLD_SERIALIZE_MEMBER_LIST lease_started_ts_on_proposer_, lease_interval_, reserved_flag_,\
accepted_, is_buffer_valid_, responsed_membership_version_, membership_version_,\
request_debug_ts_, priority_buffer_, inner_priority_seed_
#define NEW_SERIALIZE_MEMBER_LIST lease_started_ts_on_proposer_, lease_interval_,\
accepted_, is_buffer_valid_, responsed_membership_version_, membership_version_,\
request_debug_ts_, fixed_buffer_, inner_priority_seed_
OB_UNIS_SERIALIZE(ElectionAcceptResponseMsgMiddle);
OB_UNIS_DESERIALIZE(ElectionAcceptResponseMsgMiddle);
OB_UNIS_SERIALIZE_SIZE(ElectionAcceptResponseMsgMiddle);
int ElectionAcceptResponseMsgMiddle::serialize_(char* buf, const int64_t buf_len, int64_t& pos) const {
int ret = ElectionMsgBase::serialize(buf, buf_len, pos);
if (OB_SUCC(ret)) {
if (flag_not_less_than_4_2_) {
LST_DO_CODE(OB_UNIS_ENCODE, NEW_SERIALIZE_MEMBER_LIST);
} else {
LST_DO_CODE(OB_UNIS_ENCODE, OLD_SERIALIZE_MEMBER_LIST);
}
}
return ret;
}
int ElectionAcceptResponseMsgMiddle::deserialize_(const char* buf, const int64_t data_len, int64_t& pos) {
int ret = ElectionMsgBase::deserialize(buf, data_len, pos);
flag_not_less_than_4_2_ = CompatHelper::fetch_msg_flag_not_less_than_4_2(*this);
if (OB_SUCC(ret)) {
if (flag_not_less_than_4_2_) {
LST_DO_CODE(OB_UNIS_DECODE, NEW_SERIALIZE_MEMBER_LIST);
} else {
LST_DO_CODE(OB_UNIS_DECODE, OLD_SERIALIZE_MEMBER_LIST);
}
}
return ret;
}
int64_t ElectionAcceptResponseMsgMiddle::get_serialize_size_(void) const {
if (flag_not_less_than_4_2_) {
CompatHelper::set_msg_flag_not_less_than_4_2(*this);
}
int64_t len = ElectionMsgBase::get_serialize_size();
if (flag_not_less_than_4_2_) {
LST_DO_CODE(OB_UNIS_ADD_LEN, NEW_SERIALIZE_MEMBER_LIST);
} else {
LST_DO_CODE(OB_UNIS_ADD_LEN, OLD_SERIALIZE_MEMBER_LIST);
}
return len;
}
#undef NEW_SERIALIZE_MEMBER_LIST
#undef OLD_SERIALIZE_MEMBER_LIST
}// namespace election
}// namespace palf
}// namesapce oceanbase

72
src/logservice/palf/election/message/election_message.h
View File

@ -16,10 +16,13 @@
#include "common/ob_clock_generator.h"
#include "lib/net/ob_addr.h"
#include "lib/ob_define.h"
#include "lib/ob_errno.h"
#include "lib/oblog/ob_log_module.h"
#include "lib/oblog/ob_log_time_fmt.h"
#include "lib/utility/ob_macro_utils.h"
#include "lib/utility/ob_unify_serialize.h"
#include "lib/utility/serialization.h"
#include "logservice/palf/election/utils/election_common_define.h"
#include "logservice/palf/election/utils/election_utils.h"
#include "logservice/palf/election/interface/election_priority.h"
#include "logservice/palf/election/interface/election.h"
@ -31,6 +34,12 @@ namespace palf
namespace election
{
struct CompatHelper {
static constexpr int64_t BIT_MASK_NOT_LESS_THAN_4_2 = (1LL << 63);// this bit is used for mark message needed seralized in lazy-mode, and for compat reason
static void set_msg_flag_not_less_than_4_2(const ElectionMsgBase &msg);
static bool fetch_msg_flag_not_less_than_4_2(const ElectionMsgBase &msg);
};
struct ElectionMsgDebugTs
{
OB_UNIS_VERSION(1);
@ -59,6 +68,7 @@ void print_debug_ts_if_reach_warn_threshold(const ElectionMsgBase &msg, const in
class ElectionMsgBase
{
friend class CompatHelper;
OB_UNIS_VERSION(1);
public:
ElectionMsgBase();// default constructor is required by deserialization, but not actually worked
@ -221,13 +231,15 @@ public:
int64_t get_lease_start_ts_on_proposer() const;
int64_t get_lease_interval() const;
LogConfigVersion get_membership_version() const;
bool not_less_than_4_2() const { return flag_not_less_than_4_2_; }
#define BASE "BASE", *(static_cast<const ElectionMsgBase*>(this))
TO_STRING_KV(BASE, K_(lease_start_ts_on_proposer), K_(lease_interval), K_(membership_version));
#undef BASE
private:
protected:
int64_t lease_start_ts_on_proposer_;
int64_t lease_interval_;
LogConfigVersion membership_version_;
bool flag_not_less_than_4_2_;
};
OB_SERIALIZE_MEMBER_TEMP(inline, (ElectionAcceptRequestMsgMiddle, ElectionMsgBase),
lease_start_ts_on_proposer_, lease_interval_, membership_version_);
@ -251,6 +263,7 @@ public:
if (OB_FAIL(ElectionAcceptRequestMsgMiddle::deserialize(buf, data_len, pos))) {
ELECT_LOG(WARN, "deserialize failed", KR(ret));
}
flag_not_less_than_4_2_ = CompatHelper::fetch_msg_flag_not_less_than_4_2(*this);
debug_ts_.dest_deserialize_ts_ = ObClockGenerator::getRealClock();
print_debug_ts_if_reach_warn_threshold(*this, MSG_DELAY_WARN_THRESHOLD);
return ret;
@ -260,10 +273,57 @@ public:
const_cast<int64_t&>(debug_ts_.src_serialize_ts_) = ObClockGenerator::getRealClock();
print_debug_ts_if_reach_warn_threshold(*this, MSG_DELAY_WARN_THRESHOLD);
}
if (flag_not_less_than_4_2_) {
CompatHelper::set_msg_flag_not_less_than_4_2(*this);
}
return ElectionAcceptRequestMsgMiddle::get_serialize_size();
}
};
struct ElectionPriorityAdaptivedSerializationBuffer
{
ElectionPriorityAdaptivedSerializationBuffer(unsigned char *buf)
: priority_buffer_(buf), buffer_used_size_(0) { memset(priority_buffer_, 0, PRIORITY_BUFFER_SIZE); }
int serialize(char *buf, const int64_t buf_len, int64_t &pos) const {// serialize all buffer
int ret = OB_SUCCESS;
if (OB_FAIL(serialization::encode(buf, buf_len, pos, buffer_used_size_))) {
ELECT_LOG(ERROR, "fail to serialize buffer size", K(pos), KR(ret), K_(buffer_used_size));
} else {
for (int64_t idx = 0; idx < buffer_used_size_ && OB_SUCC(ret); ++idx) {
if (OB_FAIL(serialization::encode(buf, buf_len, pos, priority_buffer_[idx]))) {
ELECT_LOG(ERROR, "fail to serialize buffer", K(idx), K(pos), KR(ret), K_(buffer_used_size));
}
}
}
return ret;
}
int deserialize(const char *buf, const int64_t buf_len, int64_t &pos) {
int ret = OB_SUCCESS;
if (OB_FAIL(serialization::decode(buf, buf_len, pos, buffer_used_size_))) {
ELECT_LOG(ERROR, "fail to serialize buffer size", K(pos), KR(ret), K_(buffer_used_size));
} else if (buffer_used_size_ > PRIORITY_BUFFER_SIZE) {
ret = OB_SIZE_OVERFLOW;
ELECT_LOG(ERROR, "too big deserialized buffer size", K(pos), KR(ret), K_(buffer_used_size));
} else {
for (int64_t idx = 0; idx < buffer_used_size_ && OB_SUCC(ret); ++idx) {
if (OB_FAIL(serialization::decode(buf, buf_len, pos, priority_buffer_[idx]))) {
ELECT_LOG(ERROR, "fail to serialize buffer", K(idx), K(pos), KR(ret), K_(buffer_used_size));
}
}
}
return ret;
}
int64_t get_serialize_size() const {
int64_t size = 0;
size += serialization::encoded_length(buffer_used_size_);
for (int64_t idx =0; idx < buffer_used_size_; ++idx) {
size += serialization::encoded_length(priority_buffer_[idx]);
}
return size;
}
unsigned char *priority_buffer_;
int64_t buffer_used_size_;
};
class ElectionAcceptResponseMsgMiddle : public ElectionMsgBase
{
OB_UNIS_VERSION(1);
@ -286,14 +346,14 @@ public:
ElectionMsgDebugTs get_request_debug_ts() const;
#define BASE "BASE", *(static_cast<const ElectionMsgBase*>(this))
TO_STRING_KV(BASE, K_(lease_started_ts_on_proposer), K_(lease_interval),
KTIMERANGE_(process_request_ts, MINUTE, USECOND), K_(accepted),
K_(reserved_flag), K_(accepted), K_(flag_not_less_than_4_2),
K_(is_buffer_valid), K_(responsed_membership_version), K_(inner_priority_seed),
K_(membership_version), K_(request_debug_ts));
#undef BASE
protected:
int64_t lease_started_ts_on_proposer_;
int64_t lease_interval_;
int64_t process_request_ts_;
int64_t reserved_flag_;
bool accepted_;
bool is_buffer_valid_;
uint64_t inner_priority_seed_;
@ -301,11 +361,9 @@ protected:
LogConfigVersion membership_version_;
ElectionMsgDebugTs request_debug_ts_;
unsigned char priority_buffer_[PRIORITY_BUFFER_SIZE];
ElectionPriorityAdaptivedSerializationBuffer fixed_buffer_;
bool flag_not_less_than_4_2_;
};
OB_SERIALIZE_MEMBER_TEMP(inline, (ElectionAcceptResponseMsgMiddle, ElectionMsgBase),
lease_started_ts_on_proposer_, lease_interval_, process_request_ts_,
accepted_, is_buffer_valid_, responsed_membership_version_, membership_version_,
request_debug_ts_, priority_buffer_, inner_priority_seed_);
// design wrapper class to record serialize/deserialize time, for debugging
class ElectionAcceptResponseMsg : public ElectionAcceptResponseMsgMiddle