oceanbase/mittest/logservice/test_ob_simple_log_rebuild.cpp

/**
 * Copyright (c) 2021 OceanBase
 * OceanBase CE is licensed under Mulan PubL v2.
 * You can use this software according to the terms and conditions of the Mulan PubL v2.
 * You may obtain a copy of Mulan PubL v2 at:
 *          http://license.coscl.org.cn/MulanPubL-2.0
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 * See the Mulan PubL v2 for more details.
 */

#include <cstdio>
#include <gtest/gtest.h>
#include <signal.h>
#include "share/scn.h"
#define private public
#include "env/ob_simple_log_cluster_env.h"
#undef private

const std::string TEST_NAME = "test_rebuild";

using namespace oceanbase::common;
using namespace oceanbase;
namespace oceanbase
{
using namespace logservice;
using namespace palf;
namespace unittest
{
class TestObSimpleLogClusterRebuild : public ObSimpleLogClusterTestEnv
{
public:
  TestObSimpleLogClusterRebuild() : ObSimpleLogClusterTestEnv()
  {}
public:
  class TestRebuildCbImpl : public PalfRebuildCb, public share::ObThreadPool
  {
  public:
    TestRebuildCbImpl():
        test_base_(NULL),
        server_idx_(-1),
        rebuild_palf_id_(-1),
        rebuild_lsn_(),
        is_inited_(false) {}
    virtual ~TestRebuildCbImpl() { destroy(); }
  public:
    int init(TestObSimpleLogClusterRebuild *ptr, const int64_t server_idx)
    {
      int ret = OB_SUCCESS;
      if (OB_ISNULL(ptr) || server_idx < 0 || server_idx >= ptr->get_member_cnt()) {
        ret = OB_INVALID_ARGUMENT;
        PALF_LOG(ERROR, "invalid_argument", KP(ptr), K(server_idx));
      } else {
        test_base_ = ptr;
        server_idx_ = server_idx;
        is_inited_ = true;
      }
      return ret;
    }

    int start()
    {
      int ret = OB_SUCCESS;
      if (IS_NOT_INIT) {
        ret = OB_NOT_INIT;
      } else if (OB_FAIL(share::ObThreadPool::start())) {
        PALF_LOG(ERROR, "RebuildCB thread failed to start");
      } else {
        PALF_LOG(INFO, "RebuildCB start success", K(ret));
      }
      return ret;
    }

    void destroy()
    {
      is_inited_ = false;
      stop();
      wait();
      server_idx_ = -1;
      rebuild_palf_id_ = -1;
      rebuild_lsn_.reset();
      test_base_ = NULL;
    }

    int on_rebuild(const int64_t id, const LSN &lsn)
    {
      int ret = OB_SUCCESS;
      if (!is_inited_) {
        ret = OB_NOT_INIT;
      } else if (id < 0 || !lsn.is_valid()) {
        ret = OB_INVALID_ARGUMENT;
        SERVER_LOG(ERROR, "invalid rebuild argument", K(server_idx_), K(id), K(lsn));
      } else {
        rebuild_palf_id_ = id;
        rebuild_lsn_ = lsn;
        SERVER_LOG(INFO, "on_rebuild success", K(server_idx_), K(id), K(lsn));
      }
      return ret;
    }

    void run1()
    {
      share::ObTenantBase tenant_base(OB_SYS_TENANT_ID);
      tenant_base.init();
      ObTenantEnv::set_tenant(&tenant_base);
      lib::set_thread_name("RebuildCB");
      while (!has_set_stop()) {
        if (rebuild_palf_id_ != -1 && rebuild_lsn_.is_valid()) {
          PalfHandleGuard leader;
          PalfHandleGuard *rebuild_palf;
          int64_t leader_idx;
          PalfBaseInfo rebuild_base_info;
          std::vector<PalfHandleGuard*> palf_list;
          EXPECT_EQ(OB_SUCCESS, test_base_->get_cluster_palf_handle_guard(rebuild_palf_id_, palf_list));
          rebuild_palf = palf_list[server_idx_];
          EXPECT_EQ(OB_SUCCESS, test_base_->get_leader(test_base_->palf_id_, leader, leader_idx));
          EXPECT_EQ(OB_SUCCESS, rebuild_palf->palf_handle_.disable_sync());
          EXPECT_EQ(OB_SUCCESS, leader.palf_handle_.get_base_info(rebuild_lsn_, rebuild_base_info));
          EXPECT_EQ(OB_SUCCESS, rebuild_palf->palf_handle_.advance_base_info(rebuild_base_info, true));
          EXPECT_EQ(OB_SUCCESS, rebuild_palf->palf_handle_.enable_sync());
          rebuild_palf = NULL;
          test_base_->revert_cluster_palf_handle_guard(palf_list);
          rebuild_palf_id_ = -1;
          rebuild_lsn_.reset();
        }
        SERVER_LOG(INFO, "RebuildCB", K(server_idx_));
        usleep(500 * 1000);
      }
    }
  public:
    TestObSimpleLogClusterRebuild *test_base_;
    int64_t server_idx_;
    int64_t rebuild_palf_id_;
    LSN rebuild_lsn_;
    bool is_inited_;
  };
};

int64_t ObSimpleLogClusterTestBase::member_cnt_ = 3;
int64_t ObSimpleLogClusterTestBase::node_cnt_ = 3;
std::string ObSimpleLogClusterTestBase::test_name_ = TEST_NAME;

//TEST_F(TestObSimpleLogClusterRebuild, test_old_leader_rebuild)
//{
//  SET_CASE_LOG_FILE(TEST_NAME, "old_leader_rebuild");
//  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
//  constexpr int64_t KB = 1024;
//  constexpr int64_t MB = 1024 * 1024;
//  PALF_LOG(INFO, "start test old_leader_rebuild", K(id));
//  int64_t leader_idx = 0;
//  palf::PalfHandleGuard leader;
//  palf::PalfHandleGuard *rebuild_server = NULL;
//  std::vector<PalfHandleGuard*> palf_list;
//  int64_t follower_idx1, follower_idx2;
//  EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
//  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
//  follower_idx1 = (leader_idx + 1) % 3;
//  follower_idx2 = (leader_idx + 2) % 3;
//  TestRebuildCbImpl rebuild_cb;
//  EXPECT_EQ(OB_SUCCESS, rebuild_cb.init(this, leader_idx));
//  EXPECT_EQ(OB_SUCCESS, rebuild_cb.start());
//  // reigster rebuild cb
//  rebuild_server = &leader;
//  EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.register_rebuild_cb(&rebuild_cb));
//  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1000, id, 6 * KB));
//  sleep(1);
//  PALF_LOG(INFO, "begin block net", K(id), K(leader_idx), K(follower_idx1), K(follower_idx2));
//  block_net(leader_idx, follower_idx1);
//  block_net(leader_idx, follower_idx2);
//  PALF_LOG(INFO, "begin submit logs", K(id), K(leader_idx), K(follower_idx1), K(follower_idx2));
//  (void) submit_log(leader, 1000, leader_idx, MB);
//  // sleep to wait leader switching
//  sleep(16);
//  PALF_LOG(INFO, "after sleep 16s, begin get_leader", K(id), K(leader_idx), K(follower_idx1), K(follower_idx2));
//  int64_t new_leader_idx = 0;
//  palf::PalfHandleGuard new_leader;
//  EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx));
//  PALF_LOG(INFO, "after get_leader", K(id), K(leader_idx), K(new_leader_idx));
//  // submit logs
//  EXPECT_EQ(OB_SUCCESS, submit_log(new_leader, 64 * 6, id, MB));
//
//  // update new_leader's disk option, only reserves 4 * 80% log blocks,
//  // that means 2 blocks will be recycled
//  PALF_LOG(INFO, "begin advance_base_lsn", K(id), K(leader_idx), K(new_leader_idx));
//  LSN recycle_lsn(2 * PALF_BLOCK_SIZE);
//  EXPECT_EQ(OB_SUCCESS, new_leader.advance_base_lsn(recycle_lsn));
//  update_disk_options(new_leader_idx, 4);
//  // recycle 2 block
//  sleep(5);
//  block_id_t leader_min_block_id;
//  int64_t min_ts_ns;
//  block_id_t min_block_id, max_block_id;
//  while (true) {
//    EXPECT_EQ(OB_SUCCESS, new_leader.palf_handle_.palf_handle_impl_->get_min_block_info_for_gc(leader_min_block_id, min_ts_ns));
//    // palf will reserve one more block
//    if (1 ==leader_min_block_id) {
//      break;
//    } else {
//      sleep(2);
//    }
//  }
//
//  EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.palf_handle_impl_->log_engine_.get_block_id_range(min_block_id, max_block_id));
//  PALF_LOG(ERROR, "runlin trace get_block_id_range", K(min_block_id), K(max_block_id));
//
//  // after unblocking net, old leader will do rebuild
//  unblock_net(leader_idx, follower_idx1);
//  unblock_net(leader_idx, follower_idx2);
//  sleep(10);
//
//  PalfBaseInfo base_info_in_leader;
//  PalfBaseInfo base_info_after_rebuild;
//  // check block_id
//  block_id_t rebuild_server_min_block_id;
//  EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.palf_handle_impl_->get_min_block_info_for_gc(rebuild_server_min_block_id, min_ts_ns));
//  EXPECT_EQ(2, rebuild_server_min_block_id);
//  // check base_lsn
//  EXPECT_EQ(recycle_lsn, rebuild_server->palf_handle_.palf_handle_impl_->log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_);
//  EXPECT_EQ(OB_SUCCESS, new_leader.palf_handle_.get_base_info(recycle_lsn, base_info_in_leader));
//  EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.get_base_info(recycle_lsn, base_info_after_rebuild));
//  // check prev_log_info
//  EXPECT_EQ(base_info_in_leader.prev_log_info_, base_info_after_rebuild.prev_log_info_);
//
//  LSN leader_end_lsn, rebuild_server_end_lsn;
//  new_leader.get_end_lsn(leader_end_lsn);
//  rebuild_server->get_end_lsn(rebuild_server_end_lsn);
//  PALF_LOG(INFO, "rebuild result", K(leader_end_lsn), K(rebuild_server_end_lsn));
//  // EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id));
//  revert_cluster_palf_handle_guard(palf_list);
//  EXPECT_EQ(OB_SUCCESS, new_leader.advance_base_lsn(64*6*MB));
//  sleep(1);
//  EXPECT_EQ(OB_SUCCESS, update_disk_options(new_leader_idx, 40));
//  PALF_LOG(INFO, "end test old_leader_rebuild", K(id));
//}

TEST_F(TestObSimpleLogClusterRebuild, test_follower_rebuild)
{
  SET_CASE_LOG_FILE(TEST_NAME, "follower_rebuild");
  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
  constexpr int64_t MB = 1024 * 1024;
  PALF_LOG(INFO, "start test follower_rebuild", K(id));
  int64_t leader_idx = 0;
  palf::PalfHandleGuard leader;
  palf::PalfHandleGuard *rebuild_server = NULL;
  std::vector<PalfHandleGuard*> palf_list;
  int64_t follower_idx;
  EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
  follower_idx = (leader_idx + 1) % 3;
  TestRebuildCbImpl rebuild_cb;
  EXPECT_EQ(OB_SUCCESS, rebuild_cb.init(this, follower_idx));
  EXPECT_EQ(OB_SUCCESS, rebuild_cb.start());
  // reigster rebuild cb
  rebuild_server = palf_list[follower_idx];
  EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.register_rebuild_cb(&rebuild_cb));
  // the follower is empty
  block_net(leader_idx, follower_idx);
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 64, leader_idx, MB));
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 64 * 5, leader_idx, MB));
  // recycle one block
  LSN recycle_lsn(2 * PALF_BLOCK_SIZE);
  EXPECT_EQ(OB_SUCCESS, leader.advance_base_lsn(recycle_lsn));
  update_disk_options(leader_idx, 4);
  sleep(1);
  block_id_t leader_min_block_id, follower_min_block_id;
  share::SCN min_scn;
  while (true) {
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_.palf_handle_impl_->get_min_block_info_for_gc(leader_min_block_id, min_scn));
    if (1 == leader_min_block_id) {
      break;
    } else {
      sleep(2);
    }
  }
  EXPECT_EQ(recycle_lsn, leader.palf_handle_.palf_handle_impl_->log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_);

  // follower_idx will trigger rebuild when fetching log
  unblock_net(leader_idx, follower_idx);
  sleep(10);
  PalfBaseInfo base_info_in_leader;
  PalfBaseInfo base_info_after_rebuild;
  // check block_id
  EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.palf_handle_impl_->get_min_block_info_for_gc(follower_min_block_id, min_scn));
  EXPECT_EQ(2, follower_min_block_id);
  // check base_lsn
  EXPECT_EQ(recycle_lsn, rebuild_server->palf_handle_.palf_handle_impl_->log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_);
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_.get_base_info(recycle_lsn, base_info_in_leader));
  EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.get_base_info(recycle_lsn, base_info_after_rebuild));
  // check prev_log_info
  EXPECT_EQ(base_info_in_leader.prev_log_info_, base_info_after_rebuild.prev_log_info_);

  LSN leader_end_lsn, rebuild_server_end_lsn;
  leader.get_end_lsn(leader_end_lsn);
  rebuild_server->get_end_lsn(rebuild_server_end_lsn);
  PALF_LOG(INFO, "rebuild result", K(leader_end_lsn), K(rebuild_server_end_lsn));
  // EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id));
  revert_cluster_palf_handle_guard(palf_list);
  PALF_LOG(INFO, "end test follower_rebuild", K(id));
}

//TEST_F(TestObSimpleLogClusterRebuild, block_gc)
//{
//  SET_CASE_LOG_FILE(TEST_NAME, "block_gc");
//  update_disk_options(0);
//  update_disk_options(1);
//  update_disk_options(2);
//
//  int64_t log_ts = 1;
//  PalfHandleGuard leader1;
//  const int64_t id1 = ATOMIC_AAF(&palf_id_, 1);
//  PALF_LOG(INFO, "start block gc", K(id1));
//  int64_t leader_idx1 = 0;
//  EXPECT_EQ(OB_SUCCESS, create_paxos_group(id1, leader_idx1, leader1));
//  EXPECT_EQ(OB_SUCCESS, submit_log(leader1, 1024, leader_idx1));
//  EXPECT_EQ(OB_ITER_END, read_log(leader1));
//  PALF_LOG(INFO, "create_paxos_group1");
//
//  PalfHandleGuard leader2;
//  const int64_t id2 = ATOMIC_AAF(&palf_id_, 1);
//  int64_t leader_idx2 = 0;
//  EXPECT_EQ(OB_SUCCESS, create_paxos_group(id2, leader_idx2, leader2));
//  EXPECT_EQ(OB_SUCCESS, submit_log(leader2, 4096, leader_idx2));
//  EXPECT_EQ(OB_ITER_END, read_log(leader2));
//
//  PALF_LOG(INFO, "create_paxos_group2");
//
//  PalfHandleGuard leader3;
//  const int64_t id3 = ATOMIC_AAF(&palf_id_, 1);
//  int64_t leader_idx3 = 0;
//  EXPECT_EQ(OB_SUCCESS, create_paxos_group(id3, leader_idx3, leader3));
//  EXPECT_EQ(OB_SUCCESS, submit_log(leader3, 2048, leader_idx3));
//  EXPECT_EQ(OB_ITER_END, read_log(leader3));
//
//  PALF_LOG(INFO, "create_paxos_group3");
//
//  LSN lsn1(0), lsn2(0), lsn3(0);
//  EXPECT_EQ(OB_SUCCESS, leader1.advance_base_lsn(lsn1));
//  EXPECT_EQ(OB_SUCCESS, leader2.advance_base_lsn(lsn2));
//  EXPECT_EQ(OB_SUCCESS, leader3.advance_base_lsn(lsn3));
//
//  sleep(1);
//
//  EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id1));
//  EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id2));
//  EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id3));
//}

} // end unittest
} // end oceanbase

int main(int argc, char **argv)
{
  RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME);
}