hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[fix] set server to election blacklist if removing leader failed

Browse Source
This commit is contained in:
BinChenn
2023-08-23 10:14:09 +00:00
committed by ob-robot
parent 6f363580e2
commit 61145110f7
7 changed files with 150 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
deps/oblib/src/lib/string/ob_string_holder.h vendored
View File

@ -107,6 +107,12 @@ public:
}
return ret;
}
bool operator==(const ObString &str) const {
return str == ObString(len_, buffer_);
}
bool operator!=(const ObString &str) const {
return !(*this == str);
}
private:
void copy_from_tiny_ob_str_(const ObString &tiny_str) {
reset();

77
src/logservice/leader_coordinator/table_accessor.cpp
View File

@ -234,6 +234,83 @@ int LsElectionReferenceInfoRow::delete_server_from_blacklist(const common::ObAdd
#undef PRINT_WRAPPER
}
int LsElectionReferenceInfoRow::set_or_replace_server_in_blacklist(
const common::ObAddr &server,
InsertElectionBlacklistReason reason)
{
LC_TIME_GUARD(1_s);
#define PRINT_WRAPPER K(*this), KR(ret), K(server), K(reason)
int ret = OB_SUCCESS;
if (!server.is_valid()) {
ret = OB_INVALID_ARGUMENT;
COORDINATOR_LOG_(WARN, "server is invalid or reason is empty");
} else if (CLICK_FAIL(start_and_read_())) {
COORDINATOR_LOG_(WARN, "failed when start trans, read row, convert info");
} else if (CLICK_FAIL(set_user_row_for_specific_reason_(server, reason))) {
COORDINATOR_LOG_(WARN, "set_user_row_for_specific_reason_ failed");
} else if (CLICK_FAIL(write_and_commit_())) {
COORDINATOR_LOG_(WARN, "failed when convert info, write row, end trans");
} else {
COORDINATOR_LOG_(INFO, "set_or_replace_server_in_blacklist", K(server), "reason", to_cstring(reason));
}
if (trans_.is_started()) {
COORDINATOR_LOG_(WARN, "transaction execute failed");
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = end_(false))) {
COORDINATOR_LOG_(WARN, "fail to roll back transaction");
}
}
if (OB_SUCC(ret) && OB_FAIL(schedule_refresh_priority_task_())) {
COORDINATOR_LOG_(WARN, "failed to schedule refresh priority task", KR(ret));
}
return ret;
#undef PRINT_WRAPPER
}
int LsElectionReferenceInfoRow::set_user_row_for_specific_reason_(
const common::ObAddr &server,
InsertElectionBlacklistReason reason)
{
LC_TIME_GUARD(1_s);
#define PRINT_WRAPPER K(*this), KR(ret), K(server), K(reason)
int ret = OB_SUCCESS;
const char *reason_str = to_cstring(reason);
int64_t server_idx = -1;
int64_t number_of_the_reason = 0;
for (int64_t idx = 0; idx < row_for_user_.element<4>().count(); ++idx) {
if (row_for_user_.element<4>().at(idx).element<0>() == server) {
server_idx = idx;
}
if (row_for_user_.element<4>().at(idx).element<1>() == reason_str) {
number_of_the_reason += 1;
}
}
if (-1 != server_idx && 1 == number_of_the_reason) {
ret = OB_ENTRY_EXIST;
} else {
ObArray<ObTuple<ObAddr, ObStringHolder>> &old_array = row_for_user_.element<4>();
ObArray<ObTuple<ObAddr, ObStringHolder>> new_array;
for (int64_t idx = 0; idx < old_array.count() && OB_SUCC(ret); ++idx) {
if (old_array.at(idx).element<1>() != reason_str) {
if (CLICK_FAIL(new_array.push_back(old_array.at(idx)))) {
COORDINATOR_LOG_(WARN, "push tuple to new array failed");
}
}
}
if (OB_FAIL(ret)) {
} else if (CLICK_FAIL(old_array.assign(new_array))) {
COORDINATOR_LOG_(WARN, "replace old array with new array failed");
} else if (CLICK_FAIL(row_for_user_.element<4>().push_back(ObTuple<ObAddr, ObStringHolder>()))) {
COORDINATOR_LOG_(WARN, "failed to create new tuple for reason");
} else if (FALSE_IT(row_for_user_.element<4>().at(row_for_user_.element<4>().count() - 1).element<0>() = server)) {
} else if (CLICK_FAIL(row_for_user_.element<4>().at(row_for_user_.element<4>().count() - 1).element<1>().assign(to_cstring(reason)))) {
COORDINATOR_LOG_(WARN, "copy reason failed");
}
}
return ret;
#undef PRINT_WRAPPER
}
int LsElectionReferenceInfoRow::start_and_read_()
{
LC_TIME_GUARD(1_s);

9
src/logservice/leader_coordinator/table_accessor.h
View File

@ -105,6 +105,14 @@ public:
* @Date: 2022-01-29 16:44:51
*/
int delete_server_from_blacklist(const common::ObAddr &server);
/**
* @description: 将该原因的选举黑名单设置为对应 server,会清除掉选举黑名单中已有的该原因的 server
* @param {ObAddr} &server 不允许当leader的server
* @param {ObString} &reason 不允许当leader的原因
* @return {*}
* @Date: 2022-01-29 16:43:19
*/
int set_or_replace_server_in_blacklist(const common::ObAddr &server, InsertElectionBlacklistReason reason);
TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(exec_tenant_id), K_(row_for_table), K_(row_for_user));
private:
int begin_();
@ -116,6 +124,7 @@ private:
int start_and_read_();
int write_and_commit_();
int schedule_refresh_priority_task_();
int set_user_row_for_specific_reason_(const common::ObAddr &server, InsertElectionBlacklistReason reason);
private:
const uint64_t tenant_id_;
const share::ObLSID ls_id_;

9
src/logservice/ob_log_handler.cpp
View File

@ -1313,12 +1313,13 @@ int ObLogHandler::submit_config_change_cmd_(const LogConfigChangeCmd &req,
ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
// if removed_member is leader, switch leadership to another node and try again
// if meta tenant's leader is down, add_to_election_blacklist may return fail,
// need to retry add_to_election_blacklist until timeout/success
// if meta tenant's leader is down, set_election_blacklist may return fail,
// need to retry set_election_blacklist until timeout/success
// Note: set_election_blacklist will clear other MIGRATE servers in blacklist
if (true == has_added_to_blacklist ||
OB_SUCCESS != (tmp_ret = switch_leader_adapter.add_to_election_blacklist(id_, leader))) {
OB_SUCCESS != (tmp_ret = switch_leader_adapter.set_election_blacklist(id_, leader))) {
if (tmp_ret != OB_SUCCESS && REACH_TIME_INTERVAL(1 * 1000 * 1000)) {
CLOG_LOG(WARN, "add_to_election_blacklist failed", KR(tmp_ret), K_(id), K_(self));
CLOG_LOG(WARN, "set_election_blacklist failed", KR(tmp_ret), K_(id), K_(self));
}
} else {
has_added_to_blacklist = true;

18
src/logservice/ob_switch_leader_adapter.cpp
View File

@ -68,6 +68,24 @@ int ObSwitchLeaderAdapter::remove_from_election_blacklist(const int64_t palf_id,
return ret;
}
int ObSwitchLeaderAdapter::set_election_blacklist(const int64_t palf_id, const common::ObAddr &server)
{
int ret = OB_SUCCESS;
if (palf_id < 0 || !server.is_valid()) {
ret = OB_INVALID_ARGUMENT;
} else {
logservice::coordinator::LsElectionReferenceInfoRow row(MTL_ID(), share::ObLSID(palf_id));
if (OB_FAIL(row.set_or_replace_server_in_blacklist(server, logservice::coordinator::InsertElectionBlacklistReason::MIGRATE)) &&
OB_ENTRY_EXIST != ret) {
CLOG_LOG(WARN, "set_election_blacklist failed", K(ret), K(palf_id), K(server));
} else {
ret = OB_SUCCESS;
CLOG_LOG(INFO, "set_election_blacklist success", K(ret), K(palf_id), K(server));
}
}
return ret;
}
int ObSwitchLeaderAdapter::is_meta_tenant_dropped_(const uint64_t tenant_id, bool &is_dropped)
{
int ret = OB_SUCCESS;

1
src/logservice/ob_switch_leader_adapter.h
View File

@ -28,6 +28,7 @@ public:
static int add_to_election_blacklist(const int64_t palf_id, const common::ObAddr &server);
static int remove_from_election_blacklist(const int64_t palf_id, const common::ObAddr &server);
static int set_election_blacklist(const int64_t palf_id, const common::ObAddr &server);
private:
static int is_meta_tenant_dropped_(const uint64_t tenant_id, bool &is_dropped);

34
unittest/logservice/test_ob_election_priority_seperate_blacklist.cpp
View File

@ -64,6 +64,40 @@ TEST_F(TestPrioritySeperateBlackList, main)
ELECT_LOG(INFO, "print", K(row));
}
TEST_F(TestPrioritySeperateBlackList, test_set_user_row_for_specific_reason)
{
// OB_ENTRY_EXIST
{
logservice::coordinator::LsElectionReferenceInfoRow row(1, share::ObLSID(1));
row.row_for_table_.element<0>() = 1;
row.row_for_table_.element<1>() = 1;
row.row_for_table_.element<2>().assign("z4,z5;z3;z2,z1");
row.row_for_table_.element<3>().assign("127.0.0.1:1080");
row.row_for_table_.element<4>().assign("127.0.0.1:1080(MIGRATE)");
row.convert_table_info_to_user_info_();
ASSERT_EQ(OB_ENTRY_EXIST, row.set_user_row_for_specific_reason_(ObAddr(ObAddr::IPV4, "127.0.0.1", 1080),
logservice::coordinator::InsertElectionBlacklistReason::MIGRATE));
ASSERT_EQ(row.row_for_user_.element<4>().count(), 1);
ELECT_LOG(INFO, "print", K(row));
}
// OB_SUCCESS
{
logservice::coordinator::LsElectionReferenceInfoRow row(1, share::ObLSID(1));
row.row_for_table_.element<0>() = 1;
row.row_for_table_.element<1>() = 1;
row.row_for_table_.element<2>().assign("z4,z5;z3;z2,z1");
row.row_for_table_.element<3>().assign("127.0.0.1:1080");
row.row_for_table_.element<4>().assign("127.0.0.1:1080(MIGRATE);127.0.0.1:1081(MIGRATE);127.0.0.1:1082(SWITCH REPLICA)");
row.convert_table_info_to_user_info_();
const ObAddr new_addr = ObAddr(ObAddr::IPV4, "127.0.0.1", 1083);
ASSERT_EQ(OB_SUCCESS, row.set_user_row_for_specific_reason_(new_addr,
logservice::coordinator::InsertElectionBlacklistReason::MIGRATE));
ASSERT_EQ(row.row_for_user_.element<4>().count(), 2);
ASSERT_EQ(new_addr, row.row_for_user_.element<4>().at(1).element<0>());
ELECT_LOG(INFO, "print", K(row));
}
}
}
}