oceanbase/unittest/logservice/test_ob_election_with_priority.cpp

// Copyright (c) 2021 OceanBase
// OceanBase is licensed under Mulan PubL v2.
// You can use this software according to the terms and conditions of the Mulan PubL v2.
// You may obtain a copy of Mulan PubL v2 at:
//          http://license.coscl.org.cn/MulanPubL-2.0
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PubL v2 for more details.
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#define private public
#define protected public
#include "logservice/leader_coordinator/ob_failure_detector.h"
#include "common/ob_role.h"
#include "lib/list/ob_dlist.h"
#include "logservice/leader_coordinator/failure_event.h"
#include "logservice/palf/election/interface/election_msg_handler.h"
#include <algorithm>
#include <atomic>
#include <chrono>
#include "logservice/palf/election/interface/election.h"
#include "logservice/palf/log_meta_info.h"
#define UNITTEST
#include "logservice/palf/election/utils/election_common_define.h"
#include "logservice/palf/election/algorithm/election_impl.h"
#include "logservice/leader_coordinator/election_priority_impl/election_priority_impl.h"
#include "share/ob_occam_timer.h"
#include "share/rc/ob_tenant_base.h"
#include "mock_logservice_container/mock_election_user.h"
#include <iostream>
#include <vector>

using namespace oceanbase::obrpc;
using namespace std;

#define SUCC_(stmt) ASSERT_EQ((stmt), OB_SUCCESS)
#define FAIL_(stmt) ASSERT_EQ((stmt), OB_FAIL)
#define TRUE_(stmt) ASSERT_EQ((stmt), true)
#define FALSE_(stmt) ASSERT_EQ((stmt), false)

namespace oceanbase
{
namespace palf
{
namespace election
{
int EventRecorder::report_event_(ElectionEventType, const common::ObString &)
{
  return OB_SUCCESS;
}
}
}
namespace logservice
{
namespace coordinator
{
int PriorityV1::refresh_(const share::ObLSID &)
{
  return OB_SUCCESS;
}
}
}
namespace unittest {

using namespace common;
using namespace palf::election;
using namespace std;
using namespace logservice::coordinator;

int64_t MSG_DELAY = 1_ms;
std::atomic_int leader_takeover_times(0);
std::atomic_int leader_revoke_times(0);
std::atomic_int devote_to_be_leader_count(0);
std::atomic_int lease_expired_to_be_follower_count(0);
std::atomic_int change_leader_to_be_leader_count(0);
std::atomic_int change_leader_to_be_follower_count(0);
std::atomic_int stop_to_be_follower_count(0);
std::atomic_bool change_leader_from_prepare_change_leader_cb(false);
MockNetService GlobalNetService;
ObOccamThreadPool thread_pool;
ObOccamTimer timer;

void reset_global_status()
{
  leader_takeover_times = 0;
  leader_revoke_times = 0;
  devote_to_be_leader_count = 0;
  lease_expired_to_be_follower_count = 0;
  change_leader_to_be_leader_count = 0;
  change_leader_to_be_follower_count = 0;
  stop_to_be_follower_count = 0;
}

class TestElectionWithPriority : public ::testing::Test {
public:
  TestElectionWithPriority() {}
  ~TestElectionWithPriority() {}
  virtual void SetUp() {
    MAX_TST = 100_ms;
    change_leader_from_prepare_change_leader_cb = false;
    oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_4_0_0_0;
    share::ObTenantSwitchGuard guard;
    guard.switch_to(OB_SYS_TENANT_ID);
    GlobalNetService.clear();
    reset_global_status();
    new(&thread_pool) ObOccamThreadPool();
    new(&timer) ObOccamTimer();
    thread_pool.init_and_start(5);
    timer.init_and_start(thread_pool, 1_ms, "timer");
  }
  virtual void TearDown() { timer.~ObOccamTimer(); thread_pool.~ObOccamThreadPool(); }
};

template <typename TAKEOVER_OP>
vector<ElectionImpl *> create_election_group(vector<ElectionPriorityImpl> &v_pri, TAKEOVER_OP &&op)
{
  vector<ElectionImpl *> v;
  int election_num = v_pri.size();

  MemberList member_list;
  ObArray<ObAddr> addr_list;
  static int port = 1;
  for (int i = 0; i < election_num; ++i)
    addr_list.push_back(ObAddr(ObAddr::VER::IPV4, "127.0.0.1", port + i));
  palf::LogConfigVersion version;
  version.proposal_id_ = 1;
  version.config_seq_ = 1;
  member_list.set_new_member_list(addr_list, version, election_num);

  int ret = OB_SUCCESS;
  for (int i = 0; i < election_num; ++i) {
    ElectionImpl *election = new ElectionImpl();
    election->self_addr_ = ObAddr(ObAddr::VER::IPV4, "127.0.0.1", port + i);
    v.push_back(election);
  }
  for (auto &election_1 : v) {
    for (auto &election_2 : v) {
      GlobalNetService.connect(election_1, election_2);
    }
  }
  int index = 0;
  for (auto &election : v) {
    ret = election->init_and_start(
      1,
      &timer,
      &GlobalNetService,
      ObAddr(ObAddr::VER::IPV4, "127.0.0.1", port + index),
      true,
      1,
      [election](int64_t, const ObAddr &dest_addr) {
        change_leader_from_prepare_change_leader_cb = true;
        THREAD_POOL.commit_task_ignore_ret([election, dest_addr]() { election->change_leader_to(dest_addr); });
        return OB_SUCCESS;
      },
      [op, ret](Election *election, ObRole before, ObRole after, RoleChangeReason reason) {
        if (before == ObRole::FOLLOWER && after == ObRole::LEADER) {
          ELECT_LOG(INFO, "i become LEADER", K(obj_to_string(reason)), KPC(election));
          op();
        } else if (before == ObRole::LEADER && after == ObRole::FOLLOWER) {
          ELECT_LOG(INFO, "i become FOLLOWER", K(obj_to_string(reason)), KPC(election));
        } else {
          ELECT_LOG(ERROR, "i don't know why call me");
        }
        switch (reason) {
          case RoleChangeReason::DevoteToBeLeader:
            devote_to_be_leader_count++;
            leader_takeover_times++;
            break;
          case RoleChangeReason::LeaseExpiredToRevoke:
            lease_expired_to_be_follower_count++;
            leader_revoke_times++;
            break;
          case RoleChangeReason::ChangeLeaderToBeLeader:
            change_leader_to_be_leader_count++;
            leader_takeover_times++;
            break;
          case RoleChangeReason::ChangeLeaderToRevoke:
            change_leader_to_be_follower_count++;
            leader_revoke_times++;
            break;
          case RoleChangeReason::StopToRevoke:
            stop_to_be_follower_count++;
            leader_revoke_times++;
            break;
          default:
            ELECT_LOG(ERROR, "should not go here");
        }
      }
    );
    assert(ret == OB_SUCCESS);
    election->set_priority(&v_pri[index]);
    ret = election->set_memberlist(member_list);
    assert(ret == OB_SUCCESS);
    index++;
  }
  port += election_num;
  return v;
}

void init_pri(ElectionPriorityImpl &pri)
{
  pri.priority_tuple_.element<1>().is_valid_ = true;
  pri.priority_tuple_.element<1>().is_primary_region_ = true;
  pri.priority_tuple_.element<1>().scn_.set_base();
  pri.priority_tuple_.element<1>().zone_priority_ = 0;
}

TEST_F(TestElectionWithPriority, elect_leader_because_primary_region_and_change_leader_bacause_zone_priority) {
  vector<ElectionPriorityImpl> v_pri(3);
  for (auto &pri : v_pri)
    init_pri(pri);
  v_pri[0].priority_tuple_.element<1>().is_primary_region_ = false;
  auto election_group = create_election_group(v_pri, [](){});
  this_thread::sleep_for(chrono::seconds(5));// 等待选出第一任Leader
  ASSERT_EQ(election_group[1]->proposer_.role_, ObRole::LEADER);
  ASSERT_EQ(leader_takeover_times, 1);
  ASSERT_EQ(leader_revoke_times, 0);
  ASSERT_EQ(devote_to_be_leader_count, 1);
  ASSERT_EQ(lease_expired_to_be_follower_count, 0);
  ASSERT_EQ(change_leader_to_be_leader_count, 0);
  ASSERT_EQ(change_leader_to_be_follower_count, 0);
  ASSERT_EQ(stop_to_be_follower_count, 0);

  v_pri[1].priority_tuple_.element<1>().zone_priority_ = 1;// 1的zone priority会降低，Leader自动切换至2
  this_thread::sleep_for(chrono::seconds(1));// 等待选出第二任Leader

  ASSERT_EQ(election_group[2]->proposer_.role_, ObRole::LEADER);
  ASSERT_EQ(leader_takeover_times, 2);
  ASSERT_EQ(leader_revoke_times, 1);
  ASSERT_EQ(devote_to_be_leader_count, 1);
  ASSERT_EQ(lease_expired_to_be_follower_count, 0);
  ASSERT_EQ(change_leader_to_be_leader_count, 1);
  ASSERT_EQ(change_leader_to_be_follower_count, 1);
  ASSERT_EQ(stop_to_be_follower_count, 0);

  for (auto &election : election_group)
    election->stop();
  this_thread::sleep_for(chrono::seconds(1));
  for (auto &election : election_group)
    delete election;
  
  ASSERT_EQ(leader_takeover_times, 2);
  ASSERT_EQ(leader_revoke_times, 2);
  ASSERT_EQ(devote_to_be_leader_count, 1);
  ASSERT_EQ(lease_expired_to_be_follower_count, 0);
  ASSERT_EQ(change_leader_to_be_leader_count, 1);
  ASSERT_EQ(change_leader_to_be_follower_count, 1);
  ASSERT_EQ(stop_to_be_follower_count, 1);
  ASSERT_EQ(change_leader_from_prepare_change_leader_cb, true);
}

TEST_F(TestElectionWithPriority, not_change_leader_because_follower_memership_version_not_update_enough_and_change_leader_later_when_follwer_membership_version_update_endough) {
  vector<ElectionPriorityImpl> v_pri(3);
  for (auto &pri : v_pri)
    init_pri(pri);
  auto election_group = create_election_group(v_pri, [](){});
  // 0的IP-PORT最小，但因为membership version的原因，1会当选Leader
  election_group[1]->proposer_.memberlist_with_states_.p_impl_->member_list_.membership_version_.config_seq_ += 1;
  this_thread::sleep_for(chrono::seconds(5));// 等待选出第一任Leader
  ASSERT_EQ(election_group[1]->proposer_.role_, ObRole::LEADER);
  ASSERT_EQ(leader_takeover_times, 1);
  ASSERT_EQ(leader_revoke_times, 0);
  ASSERT_EQ(devote_to_be_leader_count, 1);
  ASSERT_EQ(lease_expired_to_be_follower_count, 0);
  ASSERT_EQ(change_leader_to_be_leader_count, 0);
  ASSERT_EQ(change_leader_to_be_follower_count, 0);
  ASSERT_EQ(stop_to_be_follower_count, 0);
  // 0的membership version追上来了，但是并不会触发切主，尽管它的IP-PORT更小，但是优先级字段同Leader相同
  election_group[0]->proposer_.memberlist_with_states_.p_impl_->member_list_.membership_version_.config_seq_ += 1;
  this_thread::sleep_for(chrono::seconds(1));// 等待切主
  ASSERT_EQ(election_group[1]->proposer_.role_, ObRole::LEADER);
  ASSERT_EQ(leader_takeover_times, 1);
  ASSERT_EQ(leader_revoke_times, 0);
  ASSERT_EQ(devote_to_be_leader_count, 1);
  ASSERT_EQ(lease_expired_to_be_follower_count, 0);
  ASSERT_EQ(change_leader_to_be_leader_count, 0);
  ASSERT_EQ(change_leader_to_be_follower_count, 0);
  ASSERT_EQ(stop_to_be_follower_count, 0);
  // 1的membership version再加1，同时让2的log_ts超过比较阈值
  election_group[1]->proposer_.memberlist_with_states_.p_impl_->member_list_.membership_version_.config_seq_ += 1;
  v_pri[2].priority_tuple_.element<1>().scn_.convert_for_logservice(100 * 1000 * 1000 * 1000L);// 此时并不会触发切主
  this_thread::sleep_for(chrono::seconds(1));// 等待切主
  ASSERT_EQ(election_group[1]->proposer_.role_, ObRole::LEADER);
  ASSERT_EQ(leader_takeover_times, 1);
  ASSERT_EQ(leader_revoke_times, 0);
  ASSERT_EQ(devote_to_be_leader_count, 1);
  ASSERT_EQ(lease_expired_to_be_follower_count, 0);
  ASSERT_EQ(change_leader_to_be_leader_count, 0);
  ASSERT_EQ(change_leader_to_be_follower_count, 0);
  ASSERT_EQ(stop_to_be_follower_count, 0);
  // 2的membership version追上Leader后触发切主
  election_group[2]->proposer_.memberlist_with_states_.p_impl_->member_list_.membership_version_.config_seq_ += 2;
  this_thread::sleep_for(chrono::seconds(1));// 等待切主
  ASSERT_EQ(election_group[2]->proposer_.role_, ObRole::LEADER);
  ASSERT_EQ(leader_takeover_times, 2);
  ASSERT_EQ(leader_revoke_times, 1);
  ASSERT_EQ(devote_to_be_leader_count, 1);
  ASSERT_EQ(lease_expired_to_be_follower_count, 0);
  ASSERT_EQ(change_leader_to_be_leader_count, 1);
  ASSERT_EQ(change_leader_to_be_follower_count, 1);
  ASSERT_EQ(stop_to_be_follower_count, 0);

  for (auto &election : election_group)
    election->stop();
  this_thread::sleep_for(chrono::seconds(1));
  for (auto &election : election_group)
    delete election;
  
  ASSERT_EQ(leader_takeover_times, 2);
  ASSERT_EQ(leader_revoke_times, 2);
  ASSERT_EQ(devote_to_be_leader_count, 1);
  ASSERT_EQ(lease_expired_to_be_follower_count, 0);
  ASSERT_EQ(change_leader_to_be_leader_count, 1);
  ASSERT_EQ(change_leader_to_be_follower_count, 1);
  ASSERT_EQ(stop_to_be_follower_count, 1);
  ASSERT_EQ(change_leader_from_prepare_change_leader_cb, true);
}

TEST_F(TestElectionWithPriority, meet_fatal_failure)
{
  vector<ElectionPriorityImpl> v_pri(3);
  for (auto &pri : v_pri)
    init_pri(pri);
  v_pri[0].priority_tuple_.element<1>().zone_priority_ = 0;
  v_pri[1].priority_tuple_.element<1>().zone_priority_ = 1;
  v_pri[2].priority_tuple_.element<1>().zone_priority_ = 2;
  auto election_group = create_election_group(v_pri, [](){});
  this_thread::sleep_for(chrono::seconds(5));// 等待选出第一任Leader
  ASSERT_EQ(election_group[0]->proposer_.role_, ObRole::LEADER);
  ASSERT_EQ(leader_takeover_times, 1);
  ASSERT_EQ(leader_revoke_times, 0);
  ASSERT_EQ(devote_to_be_leader_count, 1);
  ASSERT_EQ(lease_expired_to_be_follower_count, 0);
  ASSERT_EQ(change_leader_to_be_leader_count, 0);
  ASSERT_EQ(change_leader_to_be_follower_count, 0);
  ASSERT_EQ(stop_to_be_follower_count, 0);

  FailureEvent event(FailureType::PROCESS_HANG, FailureModule::LOG, FailureLevel::FATAL);
  event.set_info("test");
  ASSERT_EQ(OB_SUCCESS, v_pri[0].priority_tuple_.element<1>().fatal_failures_.push_back(event));
  ELECT_LOG(INFO, "add fatal failure");
  this_thread::sleep_for(chrono::seconds(5));// 等待切主
  ASSERT_EQ(election_group[1]->proposer_.role_, ObRole::LEADER);
  ASSERT_EQ(leader_takeover_times, 2);
  ASSERT_EQ(leader_revoke_times, 1);
  ASSERT_EQ(devote_to_be_leader_count, 1);
  ASSERT_EQ(lease_expired_to_be_follower_count, 0);
  ASSERT_EQ(change_leader_to_be_leader_count, 1);
  ASSERT_EQ(change_leader_to_be_follower_count, 1);
  ASSERT_EQ(stop_to_be_follower_count, 0);

  for (auto &election : election_group)
    election->stop();
  this_thread::sleep_for(chrono::seconds(1));
  for (auto &election : election_group)
    delete election;
  ASSERT_EQ(change_leader_from_prepare_change_leader_cb, false);
}

}
}

int main(int argc, char **argv)
{
  system("rm -rf test_ob_election_with_priority.log");
  oceanbase::palf::election::GLOBAL_INIT_ELECTION_MODULE();
  oceanbase::unittest::MockNetService::init();
  oceanbase::palf::election::INIT_TS = 0;
  oceanbase::common::ObClockGenerator::init();
  oceanbase::common::ObLogger &logger = oceanbase::common::ObLogger::get_logger();
  oceanbase::share::ObTenantBase tenant_base(OB_SYS_TENANT_ID);
  tenant_base.init();
  oceanbase::share::ObTenantEnv::set_tenant(&tenant_base);
  logger.set_file_name("test_ob_election_with_priority.log", false);
  logger.set_log_level(OB_LOG_LEVEL_INFO);
  testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}