/** * Copyright (c) 2021 OceanBase * OceanBase CE is licensed under Mulan PubL v2. * You can use this software according to the terms and conditions of the Mulan PubL v2. * You may obtain a copy of Mulan PubL v2 at: * http://license.coscl.org.cn/MulanPubL-2.0 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. * See the Mulan PubL v2 for more details. */ #include #include #include #include "lib/ob_define.h" #include "logservice/palf/i_log_location_cache.h" #include "logservice/palf/i_log_role_change_cb.h" #include "logservice/palf/i_ls_state.h" #include "logservice/palf/i_log_replay_engine.h" #include "logservice/palf/i_log_apply_service.h" #include "logservice/palf/log_meta_info.h" #include "logservice/palf/palf_options.h" #include "share/ls_id.h" #include "logservice/palf/log_writer_utils.h" #include "share/allocator/ob_tenant_mutil_allocator.h" #include "storage/blocksstable/ob_data_file_prepare.h" #include "lib/oblog/log_print_kv.h" #include "lib/oblog/log_module.h" #include "logservice/palf/log_engine.h" #include "logservice/palf/palf_handle_impl.h" #include "logservice/palf/lsn.h" #include "logservice/palf/log_io_task_cb_thread_pool.h" #include "logservice/palf/log_reader_utils.h" #include "logservice/palf/log_rpc.h" #include "logservice/palf/palf_env_impl.h" #include "logservice/palf/palf_handle_impl_guard.h" #include "logservice/palf/log_group_entry_header.h" #include "logservice/palf/log_group_entry_header.h" #include "logservice/palf/log_entry_header.h" #include "storage/ob_file_system_router.h" namespace oceanbase { using namespace common; using namespace palf; namespace unittest { class MockLogCtx : public logservice::AppendCb { public: explicit MockLogCtx(int j, int i) : j_(j), i_(i) {} ~MockLogCtx() {} int on_success() override { PALF_LOG(INFO, "on_success", K(j_), K(i_)); return OB_SUCCESS; } // 日志未形成多数派时会调用此函数,调用此函数后对象不再使用 int on_failure() override { PALF_LOG(INFO, "on_failure", K(j_), K(i_)); return OB_SUCCESS; } int j_; int i_; }; class MockLogRoleChangeCB : public ILogRoleChangeCB { virtual int on_leader_revoke(const int64_t palf_id) override { PALF_LOG(INFO, "on_leader_revoke", K(palf_id)); UNUSED(palf_id); return OB_SUCCESS; } virtual int is_leader_revoke_done(const int64_t palf_id, bool &is_done) const override { PALF_LOG(INFO, "is_leader_revoke_done", K(palf_id)); UNUSED(palf_id); is_done = true; return OB_SUCCESS; } virtual int on_leader_takeover(const int64_t palf_id) override { PALF_LOG(INFO, "on_leader_takeover", K(palf_id)); UNUSED(palf_id); return OB_SUCCESS; } virtual int is_leader_takeover_done(const int64_t palf_id, bool &is_done) const override { PALF_LOG(INFO, "is_leader_takeover_done", K(palf_id)); UNUSED(palf_id); is_done = true; return OB_SUCCESS; } virtual int on_leader_active(const int64_t palf_id) override { PALF_LOG(INFO, "on_leader_active", K(palf_id)); UNUSED(palf_id); return OB_SUCCESS; } }; class TestLogSubmitLog : public TestDataFilePrepare { public: TestLogSubmitLog(); virtual ~TestLogSubmitLog(); virtual void SetUp(); virtual void TearDown(); int generate_data(char *&buf, int buf_len, int wanted_size); int generate_data(LogWriteBuf &write_buf, char *&buf, int buf_len, int &wanted_size); protected: char log_dir_[1024]; int64_t palf_id_; PalfEnvImpl palf_env_impl_; MockLogRoleChangeCB role_change_cb_; PalfHandleImplGuard palf_handle_impl_guard_; }; TestLogSubmitLog::TestLogSubmitLog() : TestDataFilePrepare("TestLogSubmitLog"), log_dir_("unit_01"), palf_id_(1), role_change_cb_(), palf_handle_impl_guard_() { } TestLogSubmitLog::~TestLogSubmitLog() { } void TestLogSubmitLog::SetUp() { int ret = OB_SUCCESS; TestDataFilePrepare::SetUp(); #ifdef ERRSIM TP_SET_EVENT(EventTable::EN_LOGSERVICE_IO_TIMEOUT, OB_TIMEOUT, 0, 0); #endif const ObReplicaType replica_type = common::REPLICA_TYPE_FULL; ILogReplayExecutor *executor= reinterpret_cast(0x123); rpc::frame::ObReqTransport *transport = reinterpret_cast(0x123); const char *fake_ip = "127.0.0.1"; int32_t fake_port = 2882; ObAddr self(ObAddr::IPV4, fake_ip, fake_port); ObMemberList member_list; (void) member_list.add_server(self); const int64_t tenant_id = 1; ASSERT_EQ(OB_SUCCESS, TMA_MGR_INSTANCE.init()); ObILogAllocator *tenant_allocator = NULL; EXPECT_EQ(OB_SUCCESS, OB_FILE_SYSTEM_ROUTER.get_instance().init("dummy", "dummy", 1, "dummy", self)); EXPECT_EQ(OB_SUCCESS, ObTenantMutilAllocatorMgr::get_instance().get_tenant_log_allocator(tenant_id, tenant_allocator)); ASSERT_EQ(OB_SUCCESS, palf_env_impl_.init(&role_change_cb_, executor, tenant_allocator, transport, log_dir_, self)); palf_env_impl_.set_member_list(member_list); ASSERT_EQ(OB_SUCCESS, palf_env_impl_.start()); ASSERT_EQ(OB_SUCCESS, palf_env_impl_.create_palf_handle_impl(palf_id_, palf_handle_impl_guard_)); int64_t leader_epoch = 9; // set leader to self palf_handle_impl_guard_.get_palf_handle_impl()->set_leader(self, leader_epoch); // sleep 1s to wait leader takeover usleep(1 * 1000 * 1000); } void TestLogSubmitLog::TearDown() { #ifdef ERRSIM TP_SET_EVENT(EventTable::EN_LOGSERVICE_IO_TIMEOUT, OB_TIMEOUT, 0, 0); #endif PALF_LOG(INFO, "TestLogSubmitLog has TearDown"); PALF_LOG(INFO, "TearDown success"); TestDataFilePrepare::TearDown(); } constexpr int MAX_BUF_SIZE = 2 * 1024 * 1024; int TestLogSubmitLog::generate_data(char *&buf, int buf_len, int wanted_data_size) { int ret = OB_SUCCESS; if (buf_len < wanted_data_size) { ret = OB_BUF_NOT_ENOUGH; } else { std::mt19937 generator{std::random_device{}()}; std::uniform_int_distribution distribution{'a', 'z'}; int generate_len = wanted_data_size; std::string rand_str(generate_len, '\0'); for(auto& dis: rand_str) { dis = distribution(generator); } memcpy(buf, rand_str.c_str(), wanted_data_size); } return ret; } TEST_F(TestLogSubmitLog, test_submit_group_log) { ReadBufGuard read_guard("TestLogSubmitLog", MAX_LOG_BUFFER_SIZE); ReadBuf &read_buf = read_guard.read_buf_; char *buf = static_cast(ob_malloc(MAX_BUF_SIZE)); int real_log_data_size = 0; LogGroupEntryHeader entry_header; const int64_t LOG_HEADER_SER_SIZE = entry_header.get_serialize_size(); LogEntryHeader group_header; const int64_t LOG_HEADER_SIZE = group_header.get_serialize_size(); const int64_t LOG_LOG_CNT = 20; int64_t last_submit_log_ts = -1; PalfAppendOptions opts; opts.need_check_proposal_id = false; opts.need_nonblock = false; for (int64_t j = 0; j < 10; j++) { // 调大循环次数可以测试写多个clog文件场景 LSN offset_array[LOG_LOG_CNT]; int64_t log_size_array[LOG_LOG_CNT]; int64_t data_checksum_array[LOG_LOG_CNT]; for (int i = 0; i < LOG_LOG_CNT; i++) { int64_t log_ts; int64_t base_ts = i + j*LOG_LOG_CNT; log_size_array[i] = rand() % MAX_BUF_SIZE + 1; generate_data(buf, MAX_BUF_SIZE, log_size_array[i]); data_checksum_array[i] = static_cast(ob_crc64(buf, log_size_array[i])); MockLogCtx *log_ctx = new MockLogCtx(j, i); log_ctx->__palf_set_id(palf_id); EXPECT_EQ(OB_SUCCESS, palf_handle_impl_guard_.get_palf_handle_impl()->submit_log(opts, buf, log_size_array[i], base_ts, log_ctx, offset_array[i], log_ts)); // check if log_ts backwards if (-1 == last_submit_log_ts) { // skip } else if (log_ts <= last_submit_log_ts) { PALF_LOG(ERROR, "log_ts is less than last_submit_log_ts", K(j), K(i), "lsn", offset_array[i], K(log_ts), K(last_submit_log_ts)); abort(); } else {} // update last_submit_log_ts last_submit_log_ts = log_ts; } usleep(1 * 1000); for (int i = 0; i < LOG_LOG_CNT; i++) { int64_t read_total_len = log_size_array[i] + LOG_HEADER_SIZE; int ret = OB_SUCCESS; int64_t read_size = 0; while (OB_FAIL(palf_handle_impl_guard_.get_palf_handle_impl()->read_log(offset_array[i], read_total_len, read_buf, read_size))) { // sleep a while before retry when read log failed PALF_LOG(WARN, "read log failed, sleep 5ms", K(ret), K(j), "lsn", offset_array[i]); usleep(1 * 1000); } char *tmp_buf = read_buf.buf_ + LOG_HEADER_SIZE; PALF_LOG(INFO, "read log", K(j), K(i), "lsn", offset_array[i], K(read_total_len), K(read_size), K(LOG_HEADER_SER_SIZE), K(LOG_HEADER_SIZE)); const int64_t read_data_checksum = static_cast(ob_crc64(tmp_buf, log_size_array[i])); EXPECT_EQ(data_checksum_array[i], read_data_checksum); bool is_equal = (data_checksum_array[i] == read_data_checksum); PALF_LOG(INFO, "read log, compare finished", K(j), K(i), K(is_equal), "lsn", offset_array[i], K(read_data_checksum)); } } } } // END of unittest } // end of oceanbase int main(int argc, char **argv) { system("rm -fr ./unit_01"); system("mkdir ./unit_01"); system("rm -f ./test_log_submit_log.log"); OB_LOGGER.set_file_name("test_log_submit_log.log", true); OB_LOGGER.set_log_level("TRACE"); PALF_LOG(INFO, "begin unittest::test_log_submit_log"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); }