/** * Copyright (c) 2021 OceanBase * OceanBase CE is licensed under Mulan PubL v2. * You can use this software according to the terms and conditions of the Mulan PubL v2. * You may obtain a copy of Mulan PubL v2 at: * http://license.coscl.org.cn/MulanPubL-2.0 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. * See the Mulan PubL v2 for more details. */ #include #include #include #include "share/scn.h" #define private public #include "env/ob_simple_log_cluster_env.h" #undef private const std::string TEST_NAME = "test_rebuild"; using namespace oceanbase::common; using namespace oceanbase; namespace oceanbase { using namespace logservice; using namespace palf; namespace unittest { class TestObSimpleLogClusterRebuild : public ObSimpleLogClusterTestEnv { public: TestObSimpleLogClusterRebuild() : ObSimpleLogClusterTestEnv() {} public: class TestRebuildCbImpl : public PalfRebuildCb, public share::ObThreadPool { public: TestRebuildCbImpl(): test_base_(NULL), server_idx_(-1), rebuild_palf_id_(-1), rebuild_lsn_(), is_inited_(false) {} virtual ~TestRebuildCbImpl() { destroy(); } public: int init(TestObSimpleLogClusterRebuild *ptr, const int64_t server_idx) { int ret = OB_SUCCESS; if (OB_ISNULL(ptr) || server_idx < 0 || server_idx >= ptr->get_member_cnt()) { ret = OB_INVALID_ARGUMENT; PALF_LOG(ERROR, "invalid_argument", KP(ptr), K(server_idx)); } else { test_base_ = ptr; server_idx_ = server_idx; is_inited_ = true; } return ret; } int start() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; } else if (OB_FAIL(share::ObThreadPool::start())) { PALF_LOG(ERROR, "RebuildCB thread failed to start"); } else { PALF_LOG(INFO, "RebuildCB start success", K(ret)); } return ret; } void destroy() { is_inited_ = false; stop(); wait(); server_idx_ = -1; rebuild_palf_id_ = -1; rebuild_lsn_.reset(); test_base_ = NULL; } int on_rebuild(const int64_t id, const LSN &lsn) { int ret = OB_SUCCESS; if (!is_inited_) { ret = OB_NOT_INIT; } else if (id < 0 || !lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; SERVER_LOG(ERROR, "invalid rebuild argument", K(server_idx_), K(id), K(lsn)); } else { rebuild_palf_id_ = id; rebuild_lsn_ = lsn; SERVER_LOG(INFO, "on_rebuild success", K(server_idx_), K(id), K(lsn)); } return ret; } void run1() { share::ObTenantBase tenant_base(OB_SYS_TENANT_ID); tenant_base.init(); ObTenantEnv::set_tenant(&tenant_base); lib::set_thread_name("RebuildCB"); while (!has_set_stop()) { if (rebuild_palf_id_ != -1 && rebuild_lsn_.is_valid()) { PalfHandleGuard leader; PalfHandleGuard *rebuild_palf; int64_t leader_idx; PalfBaseInfo rebuild_base_info; std::vector palf_list; EXPECT_EQ(OB_SUCCESS, test_base_->get_cluster_palf_handle_guard(rebuild_palf_id_, palf_list)); rebuild_palf = palf_list[server_idx_]; EXPECT_EQ(OB_SUCCESS, test_base_->get_leader(test_base_->palf_id_, leader, leader_idx)); EXPECT_EQ(OB_SUCCESS, rebuild_palf->palf_handle_.disable_sync()); EXPECT_EQ(OB_SUCCESS, leader.palf_handle_.get_base_info(rebuild_lsn_, rebuild_base_info)); EXPECT_EQ(OB_SUCCESS, rebuild_palf->palf_handle_.advance_base_info(rebuild_base_info, true)); EXPECT_EQ(OB_SUCCESS, rebuild_palf->palf_handle_.enable_sync()); rebuild_palf = NULL; test_base_->revert_cluster_palf_handle_guard(palf_list); rebuild_palf_id_ = -1; rebuild_lsn_.reset(); } SERVER_LOG(INFO, "RebuildCB", K(server_idx_)); usleep(500 * 1000); } } public: TestObSimpleLogClusterRebuild *test_base_; int64_t server_idx_; int64_t rebuild_palf_id_; LSN rebuild_lsn_; bool is_inited_; }; }; int64_t ObSimpleLogClusterTestBase::member_cnt_ = 3; int64_t ObSimpleLogClusterTestBase::node_cnt_ = 3; std::string ObSimpleLogClusterTestBase::test_name_ = TEST_NAME; //TEST_F(TestObSimpleLogClusterRebuild, test_old_leader_rebuild) //{ // SET_CASE_LOG_FILE(TEST_NAME, "old_leader_rebuild"); // const int64_t id = ATOMIC_AAF(&palf_id_, 1); // constexpr int64_t KB = 1024; // constexpr int64_t MB = 1024 * 1024; // PALF_LOG(INFO, "start test old_leader_rebuild", K(id)); // int64_t leader_idx = 0; // palf::PalfHandleGuard leader; // palf::PalfHandleGuard *rebuild_server = NULL; // std::vector palf_list; // int64_t follower_idx1, follower_idx2; // EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader)); // EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list)); // follower_idx1 = (leader_idx + 1) % 3; // follower_idx2 = (leader_idx + 2) % 3; // TestRebuildCbImpl rebuild_cb; // EXPECT_EQ(OB_SUCCESS, rebuild_cb.init(this, leader_idx)); // EXPECT_EQ(OB_SUCCESS, rebuild_cb.start()); // // reigster rebuild cb // rebuild_server = &leader; // EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.register_rebuild_cb(&rebuild_cb)); // EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1000, id, 6 * KB)); // sleep(1); // PALF_LOG(INFO, "begin block net", K(id), K(leader_idx), K(follower_idx1), K(follower_idx2)); // block_net(leader_idx, follower_idx1); // block_net(leader_idx, follower_idx2); // PALF_LOG(INFO, "begin submit logs", K(id), K(leader_idx), K(follower_idx1), K(follower_idx2)); // (void) submit_log(leader, 1000, leader_idx, MB); // // sleep to wait leader switching // sleep(16); // PALF_LOG(INFO, "after sleep 16s, begin get_leader", K(id), K(leader_idx), K(follower_idx1), K(follower_idx2)); // int64_t new_leader_idx = 0; // palf::PalfHandleGuard new_leader; // EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx)); // PALF_LOG(INFO, "after get_leader", K(id), K(leader_idx), K(new_leader_idx)); // // submit logs // EXPECT_EQ(OB_SUCCESS, submit_log(new_leader, 64 * 6, id, MB)); // // // update new_leader's disk option, only reserves 4 * 80% log blocks, // // that means 2 blocks will be recycled // PALF_LOG(INFO, "begin advance_base_lsn", K(id), K(leader_idx), K(new_leader_idx)); // LSN recycle_lsn(2 * PALF_BLOCK_SIZE); // EXPECT_EQ(OB_SUCCESS, new_leader.advance_base_lsn(recycle_lsn)); // update_disk_options(new_leader_idx, 4); // // recycle 2 block // sleep(5); // block_id_t leader_min_block_id; // int64_t min_ts_ns; // block_id_t min_block_id, max_block_id; // while (true) { // EXPECT_EQ(OB_SUCCESS, new_leader.palf_handle_.palf_handle_impl_->get_min_block_info_for_gc(leader_min_block_id, min_ts_ns)); // // palf will reserve one more block // if (1 ==leader_min_block_id) { // break; // } else { // sleep(2); // } // } // // EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.palf_handle_impl_->log_engine_.get_block_id_range(min_block_id, max_block_id)); // PALF_LOG(ERROR, "runlin trace get_block_id_range", K(min_block_id), K(max_block_id)); // // // after unblocking net, old leader will do rebuild // unblock_net(leader_idx, follower_idx1); // unblock_net(leader_idx, follower_idx2); // sleep(10); // // PalfBaseInfo base_info_in_leader; // PalfBaseInfo base_info_after_rebuild; // // check block_id // block_id_t rebuild_server_min_block_id; // EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.palf_handle_impl_->get_min_block_info_for_gc(rebuild_server_min_block_id, min_ts_ns)); // EXPECT_EQ(2, rebuild_server_min_block_id); // // check base_lsn // EXPECT_EQ(recycle_lsn, rebuild_server->palf_handle_.palf_handle_impl_->log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_); // EXPECT_EQ(OB_SUCCESS, new_leader.palf_handle_.get_base_info(recycle_lsn, base_info_in_leader)); // EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.get_base_info(recycle_lsn, base_info_after_rebuild)); // // check prev_log_info // EXPECT_EQ(base_info_in_leader.prev_log_info_, base_info_after_rebuild.prev_log_info_); // // LSN leader_end_lsn, rebuild_server_end_lsn; // new_leader.get_end_lsn(leader_end_lsn); // rebuild_server->get_end_lsn(rebuild_server_end_lsn); // PALF_LOG(INFO, "rebuild result", K(leader_end_lsn), K(rebuild_server_end_lsn)); // // EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id)); // revert_cluster_palf_handle_guard(palf_list); // EXPECT_EQ(OB_SUCCESS, new_leader.advance_base_lsn(64*6*MB)); // sleep(1); // EXPECT_EQ(OB_SUCCESS, update_disk_options(new_leader_idx, 40)); // PALF_LOG(INFO, "end test old_leader_rebuild", K(id)); //} TEST_F(TestObSimpleLogClusterRebuild, test_follower_rebuild) { SET_CASE_LOG_FILE(TEST_NAME, "follower_rebuild"); const int64_t id = ATOMIC_AAF(&palf_id_, 1); constexpr int64_t MB = 1024 * 1024; PALF_LOG(INFO, "start test follower_rebuild", K(id)); int64_t leader_idx = 0; palf::PalfHandleGuard leader; palf::PalfHandleGuard *rebuild_server = NULL; std::vector palf_list; int64_t follower_idx; EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader)); EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list)); follower_idx = (leader_idx + 1) % 3; TestRebuildCbImpl rebuild_cb; EXPECT_EQ(OB_SUCCESS, rebuild_cb.init(this, follower_idx)); EXPECT_EQ(OB_SUCCESS, rebuild_cb.start()); // reigster rebuild cb rebuild_server = palf_list[follower_idx]; EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.register_rebuild_cb(&rebuild_cb)); // the follower is empty block_net(leader_idx, follower_idx); EXPECT_EQ(OB_SUCCESS, submit_log(leader, 64, leader_idx, MB)); EXPECT_EQ(OB_SUCCESS, submit_log(leader, 64 * 5, leader_idx, MB)); // recycle one block LSN recycle_lsn(2 * PALF_BLOCK_SIZE); EXPECT_EQ(OB_SUCCESS, leader.advance_base_lsn(recycle_lsn)); update_disk_options(leader_idx, 4); sleep(1); block_id_t leader_min_block_id, follower_min_block_id; share::SCN min_scn; while (true) { EXPECT_EQ(OB_SUCCESS, leader.palf_handle_.palf_handle_impl_->get_min_block_info_for_gc(leader_min_block_id, min_scn)); if (1 == leader_min_block_id) { break; } else { sleep(2); } } EXPECT_EQ(recycle_lsn, leader.palf_handle_.palf_handle_impl_->log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_); // follower_idx will trigger rebuild when fetching log unblock_net(leader_idx, follower_idx); sleep(10); PalfBaseInfo base_info_in_leader; PalfBaseInfo base_info_after_rebuild; // check block_id EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.palf_handle_impl_->get_min_block_info_for_gc(follower_min_block_id, min_scn)); EXPECT_EQ(2, follower_min_block_id); // check base_lsn EXPECT_EQ(recycle_lsn, rebuild_server->palf_handle_.palf_handle_impl_->log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_); EXPECT_EQ(OB_SUCCESS, leader.palf_handle_.get_base_info(recycle_lsn, base_info_in_leader)); EXPECT_EQ(OB_SUCCESS, rebuild_server->palf_handle_.get_base_info(recycle_lsn, base_info_after_rebuild)); // check prev_log_info EXPECT_EQ(base_info_in_leader.prev_log_info_, base_info_after_rebuild.prev_log_info_); LSN leader_end_lsn, rebuild_server_end_lsn; leader.get_end_lsn(leader_end_lsn); rebuild_server->get_end_lsn(rebuild_server_end_lsn); PALF_LOG(INFO, "rebuild result", K(leader_end_lsn), K(rebuild_server_end_lsn)); // EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id)); revert_cluster_palf_handle_guard(palf_list); PALF_LOG(INFO, "end test follower_rebuild", K(id)); } //TEST_F(TestObSimpleLogClusterRebuild, block_gc) //{ // SET_CASE_LOG_FILE(TEST_NAME, "block_gc"); // update_disk_options(0); // update_disk_options(1); // update_disk_options(2); // // int64_t log_ts = 1; // PalfHandleGuard leader1; // const int64_t id1 = ATOMIC_AAF(&palf_id_, 1); // PALF_LOG(INFO, "start block gc", K(id1)); // int64_t leader_idx1 = 0; // EXPECT_EQ(OB_SUCCESS, create_paxos_group(id1, leader_idx1, leader1)); // EXPECT_EQ(OB_SUCCESS, submit_log(leader1, 1024, leader_idx1)); // EXPECT_EQ(OB_ITER_END, read_log(leader1)); // PALF_LOG(INFO, "create_paxos_group1"); // // PalfHandleGuard leader2; // const int64_t id2 = ATOMIC_AAF(&palf_id_, 1); // int64_t leader_idx2 = 0; // EXPECT_EQ(OB_SUCCESS, create_paxos_group(id2, leader_idx2, leader2)); // EXPECT_EQ(OB_SUCCESS, submit_log(leader2, 4096, leader_idx2)); // EXPECT_EQ(OB_ITER_END, read_log(leader2)); // // PALF_LOG(INFO, "create_paxos_group2"); // // PalfHandleGuard leader3; // const int64_t id3 = ATOMIC_AAF(&palf_id_, 1); // int64_t leader_idx3 = 0; // EXPECT_EQ(OB_SUCCESS, create_paxos_group(id3, leader_idx3, leader3)); // EXPECT_EQ(OB_SUCCESS, submit_log(leader3, 2048, leader_idx3)); // EXPECT_EQ(OB_ITER_END, read_log(leader3)); // // PALF_LOG(INFO, "create_paxos_group3"); // // LSN lsn1(0), lsn2(0), lsn3(0); // EXPECT_EQ(OB_SUCCESS, leader1.advance_base_lsn(lsn1)); // EXPECT_EQ(OB_SUCCESS, leader2.advance_base_lsn(lsn2)); // EXPECT_EQ(OB_SUCCESS, leader3.advance_base_lsn(lsn3)); // // sleep(1); // // EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id1)); // EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id2)); // EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id3)); //} } // end unittest } // end oceanbase int main(int argc, char **argv) { RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME); }