hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Patch some commits to open source branch

Browse Source
This commit is contained in:
rl0
2021-07-22 00:31:09 +08:00
committed by wangzelin.wzl
parent 69fa727d91
commit 5839aec035
30 changed files with 2827 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

407
unittest/clog/clog_mock_utils.h Normal file
View File

@ -0,0 +1,407 @@
// Copyright 2014 Alibaba Inc. All Rights Reserved.
// Author:
// zhangshuai.zs@alibaba-inc.com
//
// This file is for...
//
#ifndef OCEANBASE_UNITTEST_MOCK_LOG_UTILS_H_
#define OCEANBASE_UNITTEST_MOCK_LOG_UTILS_H_
#include "gtest/gtest.h"
#include "lib/allocator/ob_malloc.h"
#include <locale.h>
#include "clog/ob_log_state_mgr.h"
#include "clog/ob_i_net_log_buffer.h"
#include "clog/ob_i_disk_log_buffer.h"
#include "clog/ob_i_net_log_buffer_mgr.h"
#include "clog/ob_log_engine.h"
#include "storage/ob_i_partition_mgr.h"
#include "../storage/mockcontainer/mock_ob_partition.h"
#include "../storage/mockcontainer/mock_ob_election_mgr.h"
#include "clog/ob_log_callback_engine.h"
#include "../storage/mockcontainer/mock_ob_partition_mgr.h"
#include "election/ob_election_rpc.h"
namespace oceanbase
{
namespace clog
{
using namespace oceanbase::election;
class MockSubmitLogCb : public ObISubmitLogCb
{
public:
MockSubmitLogCb() {}
~MockSubmitLogCb() {}
//virtual int on_success(const common::ObPartitionKey &parition_key, const uint64_t log_id, const int64_t version) = 0;
virtual int on_success(const common::ObPartitionKey &partition_key, const uint64_t log_id,
const int64_t trans_version)
{
UNUSED(partition_key);
UNUSED(log_id);
UNUSED(trans_version);
return OB_SUCCESS;
}
};
/*
class MockLogEngine: public ObILogEngine
{
public:
MockLogEngine() {}
virtual ~MockLogEngine() {}
virtual ObIRawLogIterator *alloc_raw_log_iterator(const uint64_t file_id, const int64_t timeout)
{
UNUSED(file_id);
UNUSED(timeout);
MockRawLogIterator *ptr = static_cast<MockRawLogIterator *>(common::ob_malloc(sizeof(
MockRawLogIterator), common::ObModIds::OB_UPS_LOG));
ptr->init();
return ptr;
}
virtual void revert_raw_log_iterator(ObIRawLogIterator *iter)
{
common::ob_free(iter);
}
virtual int read_log_by_location(const clog::ObReadParam &param, ObLogEntry &entry)
{
UNUSED(param);
UNUSED(entry);
return common::OB_SUCCESS;
}
virtual int read_log_by_id(const clog::ObReadParam &param, ObLogEntry &entry)
{
UNUSED(param);
UNUSED(entry);
return common::OB_SUCCESS;
}
virtual int submit_flush_task(FlushTask *task)
{
ObLogCursor log_cursor;
task->after_consume(common::OB_SUCCESS, &log_cursor);
return common::OB_SUCCESS;
};
virtual int post_log(const common::ObAddr &server,
const common::ObPartitionKey &key,
const char *buf,
int64_t len,
common::ObProposalID propose_id)
{
UNUSED(server);
UNUSED(key);
UNUSED(buf);
UNUSED(len);
UNUSED(propose_id);
return common::OB_SUCCESS;
}
virtual int submit_net_task(const common::ObMemberList &mem_list,
const common::ObPartitionKey &key,
ObILogNetTask *task)
{
UNUSED(mem_list);
UNUSED(key);
UNUSED(task);
return common::OB_SUCCESS;
}
virtual int submit_log_ack(const common::ObAddr &server,
const common::ObPartitionKey &key,
const uint64_t log_id,
const common::ObProposalID propose_id,
ObLogType type)
{
UNUSED(server);
UNUSED(key);
UNUSED(log_id);
UNUSED(propose_id);
UNUSED(type);
return common::OB_SUCCESS;
}
virtual int fetch_log_from_all_follower(const common::ObMemberList &mem_list,
const common::ObPartitionKey &key,
const uint64_t start_id,
const common::ObProposalID propose_id)
{
UNUSED(mem_list);
UNUSED(key);
UNUSED(start_id);
UNUSED(propose_id);
return common::OB_SUCCESS;
}
virtual int fetch_log_from_leader(const common::ObAddr &server,
const common::ObPartitionKey &key,
const uint64_t start_id,
const uint64_t end_id,
const common::ObProposalID propose_id)
{
UNUSED(server);
UNUSED(key);
UNUSED(start_id);
UNUSED(end_id);
UNUSED(propose_id);
return common::OB_SUCCESS;
}
virtual int submit_prepare_rqst(const common::ObMemberList &mem_list,
const common::ObPartitionKey &key,
const common::ObProposalID propose_id)
{
UNUSED(mem_list);
UNUSED(key);
UNUSED(propose_id);
return common::OB_SUCCESS;
}
virtual int broadcast_mc_log_to_members(const common::ObPartitionKey &partition_key,
const common::ObMemberList &sendout_member_list,
const char *buff,
const int64_t buff_len)
{
UNUSED(partition_key);
UNUSED(sendout_member_list);
UNUSED(buff);
UNUSED(buff_len);
int ret = common::OB_SUCCESS;
return ret;
}
virtual int fetch_latest_mc_log(const common::ObPartitionKey &partition_key,
const common::ObAddr &leader)
{
UNUSED(partition_key);
UNUSED(leader);
int ret = common::OB_SUCCESS;
return ret;
}
virtual int ack_mc_log_to_leader(const common::ObPartitionKey &partition_key,
const common::ObAddr &leader,
const common::ObAddr &server,
const int64_t mc_timestamp,
const common::ObProposalID proposal_id)
{
UNUSED(partition_key);
UNUSED(leader);
UNUSED(server);
UNUSED(mc_timestamp);
UNUSED(proposal_id);
int ret = common::OB_SUCCESS;
return ret;
}
virtual int prepare_response(const common::ObAddr &server,
const common::ObPartitionKey &partition_key,
const uint64_t max_log_id,
const common::ObProposalID propose_id)
{
UNUSED(server);
UNUSED(partition_key);
UNUSED(max_log_id);
UNUSED(propose_id);
return common::OB_SUCCESS;
}
virtual int get_current_cursor(uint64_t &file_id, int64_t &offset)
{
UNUSED(file_id);
UNUSED(offset);
return common::OB_SUCCESS;
}
virtual int get_search_min_fid(const oceanbase::clog::ObReadParam &a, uint64_t &b)
{
UNUSED(a);
UNUSED(b);
return common::OB_SUCCESS;
}
virtual int submit_fetch_log_resp(const oceanbase::common::ObMemberList &a,
const oceanbase::common::ObPartitionKey &b, oceanbase::clog::ObILogNetTask *c)
{
UNUSED(a);
UNUSED(b);
UNUSED(c);
return common::OB_SUCCESS;
}
virtual int get_cur_min_fid(uint64_t &file_id)
{
UNUSED(file_id);
return common::OB_SUCCESS;
}
virtual ObILogPartitionMetaReader *alloc_partition_meta_reader()
{
return NULL;
}
virtual void revert_partition_meta_reader(ObILogPartitionMetaReader *reader)
{
UNUSED(reader);
return;
}
};
*/
class MockElectionMgr : public election::MockObIElectionMgr
{
public:
MockElectionMgr() : leader_() {}
virtual int start_all()
{
return 0;
}
int stop_all()
{
return 0;
}
virtual int handle_election_msg(const ObElectionMsgBuffer &msgbuf,
obrpc::ObElectionRpcResult &result)
{
UNUSED(msgbuf);
UNUSED(result);
return 0;
}
int init(const common::ObAddr &self, obrpc::ObElectionRpcProxy *client_manager)
{
UNUSED(self);
UNUSED(client_manager);
return 0;
}
int add_partition(const common::ObPartitionKey &partition, int64_t replica_num,
election::ObIElectionCallback *election_cb)
{
UNUSED(partition);
UNUSED(replica_num);
UNUSED(election_cb);
return 0;
}
int remove_partition(const common::ObPartitionKey &partition)
{
UNUSED(partition);
return 0;
}
int change_leader_async(const common::ObPartitionKey &partition, const common::ObAddr &leader)
{
UNUSED(partition);
UNUSED(leader);
change_leader_ = true;
return 0;
}
int start(const common::ObPartitionKey &partition)
{
UNUSED(partition);
return 0;
}
int stop(const common::ObPartitionKey &partition)
{
UNUSED(partition);
return 0;
}
virtual int set_candidate(const common::ObPartitionKey &partition,
const common::ObMemberList &prev_mlist,
const common::ObMemberList &curr_mlist)
{
UNUSED(partition);
UNUSED(prev_mlist);
UNUSED(curr_mlist);
return 0;
}
int get_prev_candidate(const common::ObPartitionKey &partition, common::ObMemberList &mlist) const
{
UNUSED(partition);
UNUSED(mlist);
return 0;
}
int get_curr_candidate(const common::ObPartitionKey &partition, common::ObMemberList &mlist) const
{
UNUSED(partition);
UNUSED(mlist);
return 0;
}
int get_leader(const common::ObPartitionKey &partition, common::ObAddr &leader) const
{
UNUSED(partition);
leader = leader_;
return 0;
}
public:
bool change_leader_;
ObAddr leader_;
};
class MockLogCallbackEngine: public ObILogCallbackEngine
{
public:
virtual int init(common::S2MQueueThread *worker_thread_pool,
common::S2MQueueThread *sp_thread_pool)
{
UNUSED(worker_thread_pool);
UNUSED(sp_thread_pool);
return common::OB_SUCCESS;
}
virtual void destroy()
{
return;
}
virtual int submit_flush_cb_task(const common::ObPartitionKey &partition_key,
const ObLogFlushCbArg &flush_cb_arg)
{
UNUSED(partition_key);
UNUSED(flush_cb_arg);
return common::OB_SUCCESS;
}
virtual int submit_member_change_success_cb_task(const common::ObPartitionKey &partition_key,
const int64_t mc_timestamp,
const common::ObMemberList &prev_member_list,
const common::ObMemberList &curr_member_list)
{
UNUSED(partition_key);
UNUSED(mc_timestamp);
UNUSED(prev_member_list);
UNUSED(curr_member_list);
return common::OB_SUCCESS;
}
virtual int submit_leader_takeover_cb_task(const common::ObPartitionKey &partition_key)
{
UNUSED(partition_key);
return common::OB_SUCCESS;
}
virtual int submit_leader_revoke_cb_task(const common::ObPartitionKey &partition_key)
{
UNUSED(partition_key);
return common::OB_SUCCESS;
}
};
class MockObPSCb : public ObIPSCb
{
public:
virtual int64_t get_min_using_file_id() const {return 0;}
virtual int on_leader_revoke(const common::ObPartitionKey &partition_key)
{UNUSED(partition_key); return 0;}
virtual int on_leader_takeover(const common::ObPartitionKey &partition_key)
{UNUSED(partition_key); return 0;}
virtual int on_leader_active(const common::ObPartitionKey &partition_key)
{UNUSED(partition_key); return 0;}
virtual int on_member_change_success(
const common::ObPartitionKey &partition_key,
const int64_t mc_timestamp,
const common::ObMemberList &prev_member_list,
const common::ObMemberList &curr_member_list)
{
UNUSED(partition_key);
UNUSED(mc_timestamp);
UNUSED(prev_member_list);
UNUSED(curr_member_list);
return 0;
}
virtual int handle_log_missing(const common::ObPartitionKey &pkey,
const common::ObAddr &server)
{
UNUSED(pkey);
UNUSED(server);
return common::OB_SUCCESS;
}
virtual int get_server_locality_array(
common::ObIArray<share::ObServerLocality> &server_locality_array,
bool &has_readonly_zone) const
{
UNUSED(server_locality_array);
UNUSED(has_readonly_zone);
return common::OB_SUCCESS;
}
};
} // namespace clog
} // namespace oceanbase
#endif // OCEANBASE_UNITTEST_MOCK_LOG_UTILS_H_

326
unittest/clog/test_log_callback_engine.cpp Normal file
View File

@ -0,0 +1,326 @@
// Copyright 2014 Alibaba Inc. All Rights Reserved.
// Author:
// zhenzhong.jzz@alibaba-inc.com
// Owner:
// zhenzhong.jzz@alibaba-inc.com
//
//This file is for the unit test of log_callback_engine and the related
//threadpool and handler.
#include <gtest/gtest.h>
#include "clog/ob_log_callback_engine.h"
#include "clog/ob_log_callback_task.h"
#include "clog/ob_log_callback_thread_pool.h"
#include "clog/ob_log_callback_handler.h"
#include "clog/ob_log_define.h"
//#include "storage/ob_replay_status.h"
#include "storage/ob_partition_component_factory.h"
#include "storage/ob_i_partition_group.h"
#include "common/storage/ob_freeze_define.h"
#include "../storage/mockcontainer/mock_ob_partition.h"
#include "../storage/mockcontainer/mock_ob_partition_service.h"
#include "common/ob_queue_thread.h"
#include "common/ob_partition_key.h"
#include "share/ob_errno.h"
#include "lib/net/ob_addr.h"
#include "lib/allocator/ob_concurrent_fifo_allocator.h"
#include "lib/utility/ob_tracepoint.h"
#include "gtest/gtest.h"
#include "share/ob_i_ps_cb.h"
#include "clog_mock_container/mock_ps_cb.h"
namespace oceanbase
{
using namespace common;
using namespace clog;
using namespace storage;
namespace storage
{
class ObPartitionComponentFactory;
}
namespace unittest
{
class MockPartition: public MockObIPartition
{
public:
MockPartition() {partition_key_.init(1, 1, 1);}
~MockPartition() {}
const common::ObPartitionKey &get_partition_key() const
{
return partition_key_;
}
virtual int get_safe_publish_version(int64_t& publish_version)
{
UNUSED(publish_version);
return 0;
}
private:
ObPartitionKey partition_key_;
ObReplayStatus relay_ststus_;
//Uncommenting this line will cause coverity to report an error, if necessary,
//please perform related initialization operations.
// ObPartitionMCState partition_mc_state_;
};
class MockPartitionMgr : public MockObIPartitionService
{
public:
MockPartitionMgr(): is_inited_(true) {}
int get_partition(const ObPartitionKey &partition_key,
ObIPartitionGroup *&partition)
{
int ret = OB_SUCCESS;
if (partition_key == mock_partition_.get_partition_key()) {
partition = &mock_partition_;
} else {
ret = OB_ENTRY_NOT_EXIST;
}
return ret;
}
private:
bool is_inited_;
MockPartition mock_partition_;
};
//class MockObPSCb : public share::ObIPSCb
//{
//public:
// int on_leader_revoke(const ObPartitionKey &partition_key)
// {
// UNUSED(partition_key);
// return OB_SUCCESS;
// }
// int on_leader_takeover(const ObPartitionKey &partition_key)
// {
// UNUSED(partition_key);
// return OB_SUCCESS;
//
// }
// int on_leader_active(const ObPartitionKey &partition_key)
// {
// UNUSED(partition_key);
// return OB_SUCCESS;
//
// }
// int on_member_change_success(
// const ObPartitionKey &partition_key,
// const int64_t mc_timestamp,
// const ObMemberList &prev_member_list,
// const ObMemberList &curr_member_list)
// {
// UNUSED(partition_key);
// UNUSED(mc_timestamp);
// UNUSED(prev_member_list);
// UNUSED(curr_member_list);
// return OB_SUCCESS;
// }
// int64_t get_min_using_file_id() const { return 0; }
//};
class ObTestLogCallback : public ::testing::Test
{
public :
void SetUp()
{
int ret = OB_SUCCESS;
const uint64_t TABLE_ID = 198;
const int32_t PARTITION_IDX = 125;
const int32_t PARTITION_CNT = 168;
const int64_t TOTAL_LIMIT = 1 << 30;
const int64_t HOLD_LIMIT = 1 << 29;
const int64_t PAGE_SIZE = 1 << 16;
const char *ip = "127.0.0.1";
const int32_t PORT = 80;
if (OB_SUCCESS != (ret = partition_key_.init(TABLE_ID, PARTITION_IDX, PARTITION_CNT))) {
CLOG_LOG(ERROR, "partition_key_.init failed");
}
if (OB_SUCCESS != (ret = allocator_.init(TOTAL_LIMIT, HOLD_LIMIT, PAGE_SIZE))) {
CLOG_LOG(ERROR, "allocator_.init failed");
}
self_addr_.set_ip_addr(ip, PORT);
}
void TearDown()
{
allocator_.destroy();
}
MockPartitionMgr partition_service_;
common::ObAddr self_addr_ ;
common::ObConcurrentFIFOAllocator allocator_;
common::ObPartitionKey partition_key_;
MockObPSCb partition_service_cb_;
clog::ObLogCallbackThreadPool log_callback_thread_pool_;
clog::ObLogCallbackThreadPool worker_thread_pool_;
clog::ObLogCallbackThreadPool sp_thread_pool_;
};// class ObTestLogCallback
TEST_F(ObTestLogCallback, init_test)
{
const int64_t THREAD_NUM = 15;
const int64_t TASK_LIMIT_NUM = 12;
clog::ObLogCallbackEngine log_callback_engine;
clog::ObLogCallbackHandler log_callback_handler;
//test the init method of ObLogCallbackHandler
EXPECT_EQ(OB_INVALID_ARGUMENT, log_callback_handler.init(NULL, NULL));
EXPECT_EQ(OB_SUCCESS, log_callback_handler.init(&partition_service_, &partition_service_cb_));
EXPECT_EQ(OB_INIT_TWICE, log_callback_handler.init(&partition_service_, &partition_service_cb_));
//test the init method of ObLogCallbackThreadPool
EXPECT_EQ(OB_INVALID_ARGUMENT, log_callback_thread_pool_.init(NULL, THREAD_NUM, TASK_LIMIT_NUM,
self_addr_));
TP_SET_ERROR("ob_log_callback_thread_pool_.cpp", "init", "test_a", OB_ERROR);
// EXPECT_EQ(OB_ERROR, log_callback_thread_pool_.init(&log_callback_handler, THREAD_NUM,
// TASK_LIMIT_NUM,
// self_addr_));
TP_SET("ob_log_callback_thread_pool_.cpp", "init", "test_a", NULL);
EXPECT_EQ(OB_SUCCESS, log_callback_thread_pool_.init(&log_callback_handler, THREAD_NUM,
TASK_LIMIT_NUM, self_addr_));
EXPECT_EQ(OB_INIT_TWICE, log_callback_thread_pool_.init(&log_callback_handler, THREAD_NUM,
TASK_LIMIT_NUM, self_addr_));
//test the init method of ObLogCallbackEngine
worker_thread_pool_.init(&log_callback_handler, THREAD_NUM, TASK_LIMIT_NUM, self_addr_);
sp_thread_pool_.init(&log_callback_handler, THREAD_NUM, TASK_LIMIT_NUM, self_addr_);
EXPECT_EQ(OB_INVALID_ARGUMENT, log_callback_engine.init(NULL, NULL));
EXPECT_EQ(OB_SUCCESS, log_callback_engine.init(&worker_thread_pool_, &sp_thread_pool_));
EXPECT_EQ(OB_INIT_TWICE, log_callback_engine.init(&worker_thread_pool_, &sp_thread_pool_));
sleep(10);
log_callback_engine.destroy();
log_callback_thread_pool_.destroy();
worker_thread_pool_.destroy();
sp_thread_pool_.destroy();
}
//TEST_F(ObTestLogCallback, submit_flush_cb_task_test)
//{
// const uint64_t LOG_ID = 190;
// const int64_t MC_TIMESTAMP = 865;
// const common::ObProposalID PROPOSAL_ID;
// const int64_t THREAD_NUM = 15;
// const int64_t TASK_LIMIT_NUM = 12;
// bool NEED_ACK = false;
//
// clog::ObLogCallbackEngine log_callback_engine;
// clog::ObLogCallbackThreadPool log_callback_thread_pool_;
// clog::ObLogCallbackHandler log_callback_handler;
// clog::ObLogCallbackThreadPool worker_thread_pool_;
// clog::ObLogCallbackThreadPool sp_thread_pool_;
//
// ObLogType type = OB_LOG_MEMBER_CHANGE;
// clog::ObLogCursor log_cursor;
// log_cursor.file_id_ = 0;
// log_cursor.offset_ = 0;
// log_cursor.size_ = 0;
// clog::ObLogFlushCbArg flush_cb_arg(type, LOG_ID, MC_TIMESTAMP, PROPOSAL_ID, NEED_ACK, self_addr_,
// log_cursor, 0);
//
// EXPECT_EQ(OB_NOT_INIT, log_callback_engine.submit_flush_cb_task(partition_key_, flush_cb_arg));
// ASSERT_EQ(OB_SUCCESS, log_callback_handler.init(&partition_service_, &partition_service_cb_));
// ASSERT_EQ(OB_SUCCESS, log_callback_thread_pool_.init(&log_callback_handler, THREAD_NUM,
// TASK_LIMIT_NUM, self_addr_));
// worker_thread_pool_.init(&log_callback_handler, THREAD_NUM, TASK_LIMIT_NUM, self_addr_);
// sp_thread_pool_.init(&log_callback_handler, THREAD_NUM, TASK_LIMIT_NUM, self_addr_);
// ASSERT_EQ(OB_SUCCESS, log_callback_engine.init(&worker_thread_pool_, &sp_thread_pool_));
// EXPECT_EQ(OB_SUCCESS, log_callback_engine.submit_flush_cb_task(partition_key_, flush_cb_arg));
//
// sleep(10);
// log_callback_engine.destroy();
// log_callback_thread_pool_.destroy();
// worker_thread_pool_.destroy();
// sp_thread_pool_.destroy();
//}
TEST_F(ObTestLogCallback, submit_member_change_success_cb_task_test)
{
const int64_t THREAD_NUM = 15;
const int64_t TASK_LIMIT_NUM = 12;
clog::ObLogCallbackEngine log_callback_engine;
clog::ObLogCallbackHandler log_callback_handler;
const int64_t MC_TIMESTAMP = 64;
const common::ObMemberList PREV_MEMBER_LIST;
const common::ObMemberList CURR_MEMBER_LIST;
EXPECT_EQ(OB_NOT_INIT, log_callback_engine.submit_member_change_success_cb_task(partition_key_,
MC_TIMESTAMP, PREV_MEMBER_LIST, CURR_MEMBER_LIST));
ASSERT_EQ(OB_SUCCESS, log_callback_handler.init(&partition_service_, &partition_service_cb_));
ASSERT_EQ(OB_SUCCESS, log_callback_thread_pool_.init(&log_callback_handler, THREAD_NUM,
TASK_LIMIT_NUM, self_addr_));
worker_thread_pool_.init(&log_callback_handler, THREAD_NUM, TASK_LIMIT_NUM, self_addr_);
sp_thread_pool_.init(&log_callback_handler, THREAD_NUM, TASK_LIMIT_NUM, self_addr_);
ASSERT_EQ(OB_SUCCESS, log_callback_engine.init(&worker_thread_pool_, &sp_thread_pool_));
EXPECT_EQ(OB_SUCCESS, log_callback_engine.submit_member_change_success_cb_task(partition_key_,
MC_TIMESTAMP, PREV_MEMBER_LIST, CURR_MEMBER_LIST));
sleep(10);
log_callback_engine.destroy();
log_callback_thread_pool_.destroy();
worker_thread_pool_.destroy();
sp_thread_pool_.destroy();
}
//TEST_F(ObTestLogCallback, submit_leader_takeover_cb_task_test)
//{
// const int64_t THREAD_NUM = 15;
// const int64_t TASK_LIMIT_NUM = 12;
// clog::ObLogCallbackEngine log_callback_engine;
// clog::ObLogCallbackThreadPool log_callback_thread_pool_;
// clog::ObLogCallbackHandler log_callback_handler;
// clog::ObLogCallbackThreadPool worker_thread_pool_;
// clog::ObLogCallbackThreadPool sp_thread_pool_;
//
// EXPECT_EQ(OB_NOT_INIT, log_callback_engine.submit_leader_takeover_cb_task(partition_key_));
// ASSERT_EQ(OB_SUCCESS, log_callback_handler.init(&partition_service_, &partition_service_cb_));
// ASSERT_EQ(OB_SUCCESS, log_callback_thread_pool_.init(&log_callback_handler, THREAD_NUM,
// TASK_LIMIT_NUM, self_addr_));
// worker_thread_pool_.init(&log_callback_handler, THREAD_NUM, TASK_LIMIT_NUM, self_addr_);
// sp_thread_pool_.init(&log_callback_handler, THREAD_NUM, TASK_LIMIT_NUM, self_addr_);
// ASSERT_EQ(OB_SUCCESS, log_callback_engine.init(&worker_thread_pool_, &sp_thread_pool_));
// EXPECT_EQ(OB_SUCCESS, log_callback_engine.submit_leader_takeover_cb_task(partition_key_));
//
// sleep(10);
// log_callback_engine.destroy();
// log_callback_thread_pool_.destroy();
// worker_thread_pool_.destroy();
// sp_thread_pool_.destroy();
//}
//TEST_F(ObTestLogCallback, submit_leader_revoke_cb_task_test)
//{
// const int64_t THREAD_NUM = 15;
// const int64_t TASK_LIMIT_NUM = 12;
// clog::ObLogCallbackEngine log_callback_engine;
// clog::ObLogCallbackThreadPool log_callback_thread_pool_;
// clog::ObLogCallbackHandler log_callback_handler;
// clog::ObLogCallbackThreadPool worker_thread_pool_;
// clog::ObLogCallbackThreadPool sp_thread_pool_;
//
// EXPECT_EQ(OB_NOT_INIT, log_callback_engine.submit_leader_revoke_cb_task(partition_key_));
// ASSERT_EQ(OB_SUCCESS, log_callback_handler.init(&partition_service_, &partition_service_cb_));
// ASSERT_EQ(OB_SUCCESS, log_callback_thread_pool_.init(&log_callback_handler, THREAD_NUM,
// TASK_LIMIT_NUM, self_addr_));
// worker_thread_pool_.init(&log_callback_handler, THREAD_NUM, TASK_LIMIT_NUM, self_addr_);
// sp_thread_pool_.init(&log_callback_handler, THREAD_NUM, TASK_LIMIT_NUM, self_addr_);
// ASSERT_EQ(OB_SUCCESS, log_callback_engine.init(&worker_thread_pool_, &sp_thread_pool_));
// EXPECT_EQ(OB_SUCCESS, log_callback_engine.submit_leader_revoke_cb_task(partition_key_));
//
// sleep(10);
// log_callback_engine.destroy();
// log_callback_thread_pool_.destroy();
// worker_thread_pool_.destroy();
// sp_thread_pool_.destroy();
//}
}//namespace unittest
}//namespace oceanbase
int main(int argc, char **argv)
{
OB_LOGGER.set_file_name("test_log_callback_engine.log", true);
OB_LOGGER.set_log_level("INFO");
CLOG_LOG(INFO, "begin unittest: test_ob_log_callback_engine");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

530
unittest/clog/test_log_reconfirm.cpp Normal file
View File

@ -0,0 +1,530 @@
// Copyright 2014 Alibaba Inc. All Rights Reserved.
// Author:
// zhenzhong.jzz@alibaba-inc.com
// Owner:
// zhenzhong.jzz@alibaba-inc.com
//
// This file is for unit test of ObLogReconfirm
#include "clog/ob_log_reconfirm.h"
#include "gtest/gtest.h"
#include "lib/allocator/ob_malloc.h"
#include "clog/ob_log_state_mgr.h"
#include "clog/ob_i_net_log_buffer.h"
#include "clog/ob_i_disk_log_buffer.h"
#include "clog/ob_i_net_log_buffer_mgr.h"
#include "clog/ob_i_log_engine.h"
#include "clog/ob_log_define.h"
#include "test_accessor.h"
#include "lib/utility/ob_tracepoint.h"
#include "clog_mock_container/mock_log_membership_mgr.h"
#include "clog_mock_container/mock_log_engine.h"
#include "clog_mock_container/mock_log_replay_engine.h"
#include "clog_mock_container/mock_log_allocator.h"
#include "clog_mock_container/mock_log_state_mgr.h"
#include "clog_mock_container/mock_submit_log_cb.h"
#include "clog_mock_container/mock_log_sliding_window.h"
using namespace oceanbase::common;
namespace oceanbase
{
using namespace common;
namespace clog
{
class MockLogSWForReconfirm: public ObILogSWForReconfirm
{
public:
MockLogSWForReconfirm() : start_id_(0), epoch_id_(0), max_id_(0), next_id_(0),
submit_ret_(OB_SUCCESS),
sw_ret_(OB_SUCCESS), array_(NULL), call_start_working_cnt_(0), call_submit_cnt_(0),
get_log_ok_(true), get_log_task_ret_(OB_SUCCESS), use_cnt_(0) {}
uint64_t get_start_id() const
{
return start_id_;
}
int64_t get_epoch_id() const
{
return epoch_id_;
}
int try_update_max_log_id(const uint64_t log_id)
{
max_id_ = log_id;
return OB_SUCCESS;
}
int submit_log(const ObLogEntryHeader &header, const char *buff,
ObISubmitLogCb *cb)
{
UNUSED(header);
UNUSED(buff);
UNUSED(cb);
ATOMIC_INC(&call_submit_cnt_);
return submit_ret_;
}
void revert_log_task(const int64_t *ref)
{
UNUSED(ref);
}
//The even number returns the log_task that exists,
//and the odd number returns the local non-existent
int get_log_task(const uint64_t log_id, ObLogTask *&log_task, const int64_t *&ref)
{
int ret = OB_SUCCESS;
UNUSED(ref);
if (get_log_ok_) {
if (array_ != NULL && log_id % 2 == 0) {
log_task = array_ + log_id;
} else {
ret = OB_ERR_NULL_VALUE;
}
} else {
ret = OB_ERR_UNEXPECTED;
}
if (get_log_task_ret_ != OB_SUCCESS && use_cnt_ < 3) {
use_cnt_++;
return get_log_task_ret_;
} else {
return ret;
}
}
uint64_t get_max_log_id() const
{
return max_id_;
}
void submit_replay_task(const bool is_appending) { UNUSED(is_appending); }
public:
uint64_t start_id_;
int64_t epoch_id_;
uint64_t max_id_;
uint64_t next_id_;
int submit_ret_;
int sw_ret_;
ObLogTask *array_;
int64_t call_start_working_cnt_;
int64_t call_submit_cnt_;
bool get_log_ok_;
int get_log_task_ret_;
uint32_t use_cnt_;
};
class MockLogStateMgr : public MockObLogStateMgr
{
public:
MockLogStateMgr() {}
~MockLogStateMgr() {}
public:
void set_proposal_id(common::ObProposalID proposal_id)
{
proposal_id_ = proposal_id;
}
common::ObProposalID get_proposal_id() const
{
return proposal_id_;
}
public:
common::ObProposalID proposal_id_;
};
class ReLogAllocator : public MockObLogAllocator
{
public:
ReLogAllocator()
{
alloc_.init(512 * 1024 *1024, 512 * 1024 * 1024, 64 * 1024);
}
void *re_alloc(const int64_t sz)
{
return malloc(sz);
}
void re_free(void *ptr)
{
free(ptr);
}
common::ObIAllocator *get_re_allocator()
{
return &alloc_;
}
private:
common::ObConcurrentFIFOAllocator alloc_;
};
class MockLogMembershipMgr : public MockObLogMembershipMgr
{
public:
MockLogMembershipMgr() {}
virtual ~MockLogMembershipMgr() {}
public:
int add_member(const common::ObMember &member)
{
return curr_member_list_.add_member(member);
}
const common::ObMemberList &get_curr_member_list() const
{
return curr_member_list_;
}
private:
common::ObMemberList curr_member_list_;
};
}//namespace clog
namespace unittest
{
class ReconfirmStateAccessorForTest
{
public:
ObLogReconfirm::State get_fetch_max_lsn()
{
return ObLogReconfirm::FETCH_MAX_LSN;
}
ObLogReconfirm::State get_reconfirming()
{
return ObLogReconfirm::RECONFIRMING;
}
ObLogReconfirm::State get_start_working()
{
return ObLogReconfirm::START_WORKING;
}
int64_t get_majority_tag_bit()
{
return ObLogReconfirm::MAJORITY_TAG_BIT;
}
};
class TestReconfirm : public ::testing::Test
{
public:
virtual void SetUp()
{
const uint64_t TABLE_ID = 120;
const int32_t PARTITION_IDX = 1400;
const int32_t PARTITION_CNT = 3;
partition_key_.init(TABLE_ID, PARTITION_IDX, PARTITION_CNT);
self_.parse_from_cstring("127.0.0.1:8111");
helper_.set_assert_on(reconfirm_, false);
common::ObMember self_member(self_, ObTimeUtility::current_time());
mm_.add_member(self_member);
}
virtual void TearDown()
{
helper_.set_assert_on(reconfirm_, true);
}
protected:
ReconfirmStateAccessorForTest state_accessor_;
ObLogReconfirmAccessor helper_;
clog::MockLogSWForReconfirm sw_;
clog::MockLogStateMgr state_mgr_;
clog::MockObLogEngine log_engine_;
ReLogAllocator alloc_;
MockObSubmitLogCb submit_cb_;
ObPartitionKey partition_key_;
ObLogReconfirm reconfirm_;
MockLogMembershipMgr mm_;
ObAddr self_;
};
TEST_F(TestReconfirm, reconfirm_test)
{
ASSERT_EQ(OB_SUCCESS, reconfirm_.init(&sw_, &state_mgr_, &mm_, &log_engine_, &alloc_,
partition_key_, self_));
//The state is INITED, and the maximum log_id stored by oob
//is greater than the sliding window
ASSERT_TRUE(mm_.get_curr_member_list().contains(self_));
sw_.max_id_ = 15;
EXPECT_EQ(OB_EAGAIN, reconfirm_.reconfirm());
//Handle FLUSHING_PREPARE_LOG
ObProposalID proposal_id_tmp;
proposal_id_tmp.addr_ = self_;
proposal_id_tmp.ts_ = 1500;
state_mgr_.set_proposal_id(proposal_id_tmp);
proposal_id_tmp.ts_ -= 100;
helper_.get_new_proposal_id(reconfirm_) = proposal_id_tmp;
EXPECT_EQ(OB_EAGAIN, reconfirm_.reconfirm());
//New proposal_id has flushed
state_mgr_.set_proposal_id(proposal_id_tmp);
EXPECT_EQ(OB_EAGAIN, reconfirm_.reconfirm());
helper_.set_state(reconfirm_, state_accessor_.get_reconfirming());
}
TEST_F(TestReconfirm, receive_max_log_id_test)
{
const uint64_t LOG_ID = 1400;
ASSERT_EQ(OB_SUCCESS, reconfirm_.init(&sw_, &state_mgr_, &mm_, &log_engine_, &alloc_,
partition_key_, self_));
int64_t max_log_ts = 0;
//Incorrect state
EXPECT_EQ(OB_STATE_NOT_MATCH, reconfirm_.receive_max_log_id(self_, LOG_ID, max_log_ts));
helper_.set_state(reconfirm_, state_accessor_.get_fetch_max_lsn());
EXPECT_EQ(OB_SUCCESS, reconfirm_.receive_max_log_id(self_, LOG_ID, max_log_ts));
helper_.get_max_log_ack_map(reconfirm_).test_and_set(state_accessor_.get_majority_tag_bit());
EXPECT_EQ(OB_SUCCESS, reconfirm_.receive_max_log_id(self_, LOG_ID, max_log_ts));
helper_.get_max_log_ack_map(reconfirm_).reset_map(state_accessor_.get_majority_tag_bit());
EXPECT_EQ(OB_SUCCESS, reconfirm_.receive_max_log_id(self_, LOG_ID, max_log_ts));
}
TEST_F(TestReconfirm, receive_log_test)
{
ObProposalID proposal_id_tmp;
proposal_id_tmp.addr_ = self_;
proposal_id_tmp.ts_ = 1500;
uint64_t log_id = 2000;
int64_t data_len = 0;
int64_t generate_timestamp = 1990;
int64_t epoch_id = 1800;
int64_t submit_timestamp = 1990;
ObLogEntry log_entry;
ObLogEntryHeader header;
common::ObVersion freeze_version;
ASSERT_EQ(OB_SUCCESS, reconfirm_.init(&sw_, &state_mgr_, &mm_, &log_engine_, &alloc_,
partition_key_, self_));
EXPECT_TRUE(reconfirm_.need_start_up());
EXPECT_EQ(OB_STATE_NOT_MATCH, reconfirm_.receive_log(log_entry, self_));
helper_.set_state(reconfirm_, state_accessor_.get_reconfirming());
helper_.get_max_flushed_id(reconfirm_) = 1800;
helper_.prepare_map(reconfirm_);
sw_.start_id_ = 1000;
header.generate_header(OB_LOG_SUBMIT, partition_key_, log_id, NULL, data_len, generate_timestamp,
epoch_id,
proposal_id_tmp, submit_timestamp,
freeze_version);
log_entry.generate_entry(header, NULL);
EXPECT_EQ(OB_INVALID_ARGUMENT, reconfirm_.receive_log(log_entry, ObAddr()));
log_id = 100;
header.generate_header(OB_LOG_SUBMIT, partition_key_, log_id, NULL, data_len, generate_timestamp,
epoch_id,
proposal_id_tmp, submit_timestamp,
freeze_version);
log_entry.generate_entry(header, NULL);
EXPECT_EQ(OB_SUCCESS, reconfirm_.receive_log(log_entry, self_));
log_id = 1500;
header.generate_header(OB_LOG_NOT_EXIST, partition_key_, log_id, NULL, data_len, generate_timestamp,
epoch_id,
proposal_id_tmp, submit_timestamp,
freeze_version);
log_entry.generate_entry(header, NULL);
EXPECT_EQ(OB_SUCCESS, reconfirm_.receive_log(log_entry, self_));
header.generate_header(OB_LOG_SUBMIT, partition_key_, log_id, NULL, data_len, generate_timestamp,
epoch_id,
proposal_id_tmp, submit_timestamp,
freeze_version);
log_entry.generate_entry(header, NULL);
EXPECT_EQ(OB_SUCCESS, reconfirm_.receive_log(log_entry, self_));
EXPECT_EQ(OB_SUCCESS, reconfirm_.receive_log(log_entry, self_));
}
TEST_F(TestReconfirm, private_test)
{
ObProposalID proposal_id_tmp;
proposal_id_tmp.addr_ = self_;
proposal_id_tmp.ts_ = 1500;
ObLogEntry log_entry;
ObLogEntryHeader header;
common::ObVersion freeze_version;
ASSERT_EQ(OB_SUCCESS, reconfirm_.init(&sw_, &state_mgr_, &mm_, &log_engine_, &alloc_,
partition_key_, self_));
helper_.set_state(reconfirm_, state_accessor_.get_reconfirming());
helper_.get_max_flushed_id(reconfirm_) = 1800;
helper_.prepare_map(reconfirm_);
//Test the processing of try_fetch_log_ under normal circumstances when
//the starting ID of the sliding window is less than the maximum fetching ID
sw_.start_id_ = 120;
helper_.get_start_id(reconfirm_) = 120;
helper_.get_max_flushed_id(reconfirm_) = 150;
EXPECT_EQ(OB_SUCCESS, helper_.execute_try_fetch_log(reconfirm_));
//Test the processing of try_filter_invalid_log_ function when the timestamp
//and leader_ts disagree
uint64_t log_id = 1500;
int64_t data_len = 0;
int64_t generate_timestamp = 1990;
int64_t epoch_id = 1800;
int64_t submit_timestamp = 1990;
header.generate_header(OB_LOG_SUBMIT, partition_key_, log_id, NULL, data_len, generate_timestamp,
epoch_id,
proposal_id_tmp,
submit_timestamp,
freeze_version);
helper_.get_leader_ts(reconfirm_) = 1600;
helper_.get_start_id(reconfirm_) = 140;
helper_.get_next_id(reconfirm_) = 150;
int idx = 10;
ObLogEntry *log_array = helper_.get_log_array(reconfirm_);
(log_array + idx) -> generate_entry(header, NULL);
EXPECT_EQ(OB_ERR_UNEXPECTED, helper_.execute_try_filter_invalid_log(reconfirm_));
//Handle start-working log
log_id = 1500;
header.generate_header(OB_LOG_START_MEMBERSHIP, partition_key_, log_id, NULL, data_len,
generate_timestamp,
epoch_id, proposal_id_tmp,
submit_timestamp, freeze_version);
(log_array + idx) -> generate_entry(header, NULL);
EXPECT_EQ(OB_SUCCESS, helper_.execute_try_filter_invalid_log(reconfirm_));
//Replace ghost log, may be non-empty log
epoch_id = 1000;
helper_.get_start_id(reconfirm_) = 140;
header.generate_header(OB_LOG_SUBMIT, partition_key_, log_id, NULL, data_len, generate_timestamp,
epoch_id,
proposal_id_tmp,
submit_timestamp, freeze_version);
(log_array + idx) -> generate_entry(header, NULL);
EXPECT_EQ(OB_SUCCESS, helper_.execute_try_filter_invalid_log(reconfirm_));
// sw_.start_id_ = 100;
// helper_.get_start_id(reconfirm_) = 100;
// helper_.get_next_id(reconfirm_) = 150;
// TP_SET_ERROR("ob_log_reconfirm.cpp", "retry_confirm_log_", "test_c", OB_ERROR);
// EXPECT_EQ(OB_ERROR, helper_.execute_retry_confirm_log(reconfirm_));
// TP_SET("ob_log_reconfirm.cpp", "retry_confirm_log_", "test_c", NULL);
//
//
// helper_.get_last_retry_reconfirm_ts(reconfirm_) = 0;
// sw_.start_id_ = 100;
// helper_.get_last_check_start_id(reconfirm_) = 120;
// EXPECT_TRUE(helper_.execute_need_retry_reconfirm(reconfirm_));
// sw_.start_id_ = 120;
// EXPECT_TRUE(helper_.execute_need_retry_reconfirm(reconfirm_));
//init_reconfirm, when the maximum logid stored in oob_log_handler_ is greater
//than in the sliding window, that is, when there is a log that cannot enter the sliding window
sw_.max_id_ = 1300;
EXPECT_EQ(OB_SUCCESS, helper_.execute_init_reconfirm(reconfirm_));
//The operation of the processing code for several cases of failure to obtain
//the log in get_start_id_and_leader_ts_
sw_.get_log_task_ret_ = OB_ERROR_OUT_OF_RANGE;
EXPECT_EQ(OB_SUCCESS, helper_.execute_get_start_id_and_leader_ts(reconfirm_));
}
//majority is 1
TEST(SampleRecnofirmTest, single_member)
{
ObLogReconfirmAccessor helper;
MockLogSWForReconfirm sw;
MockLogStateMgr state_mgr;
MockObLogEngine log_engine;
clog::MockObSlidingCallBack sliding_cb;
ReLogAllocator alloc;
MockObSubmitLogCb submit_cb;
ObPartitionKey partition_key;
ObLogReconfirm reconfirm;
MockLogMembershipMgr mm;
ObAddr self;
self.parse_from_cstring("127.0.0.1:8111");
common::ObMemberList curr_member_list;
common::ObMember self_member(self, ObTimeUtility::current_time());
mm.add_member(self_member);
ASSERT_EQ(OB_INVALID_ARGUMENT, reconfirm.init(NULL, &state_mgr, &mm, &log_engine, &alloc,
partition_key, self));
const uint64_t TABLE_ID = 120;
const int32_t PARTITION_IDX = 1400;
const int32_t PARTITION_CNT = 3;
partition_key.init(TABLE_ID, PARTITION_IDX, PARTITION_CNT);
ASSERT_EQ(OB_SUCCESS, reconfirm.init(&sw, &state_mgr, &mm, &log_engine, &alloc,
partition_key, self
));
reconfirm.clear();
////////////////////////////////////////////////////////////////////////////////////////////////////////
//Verify that there are no pending logs
sw.start_id_ = 1;
sw.max_id_ = 0;
ASSERT_EQ(OB_EAGAIN, reconfirm.reconfirm());
//Wait for the new proposal_id to refresh successfully
// Modify the proposal_id of StateMgr to indicate that the flash disk and
// the callback are executed successfully
state_mgr.set_proposal_id(helper.get_new_proposal_id(reconfirm));
//Verification enters the START_WORKING stage
ASSERT_EQ(OB_EAGAIN, reconfirm.reconfirm());
// Retransmit after timeout
usleep(CLOG_LEADER_RECONFIRM_SYNC_TIMEOUT + 100);
ASSERT_EQ(OB_EAGAIN, reconfirm.reconfirm());
sw.start_id_ = 2;
ASSERT_EQ(OB_EAGAIN, reconfirm.reconfirm());
reconfirm.clear();
////////////////////////////////////////////////////////////////////////////////////////////////////////
// Verify that there are three logs that require Reconfirm, and their log_ids are 0, 1, and 2,
// where log 2 is already in the local confirmation state, log 1 does not exist locally, and
// log 0 has a generate_header() parameter comparison table locally.
ObProposalID pid;
pid.ts_ = 123456;
pid.addr_ = self;
sw.start_id_ = 0;
sw.max_id_ = 2;
ObLogTask task_array[3];
ObLogEntryHeader header;
ObLogSimpleBitMap bit_map;
common::ObVersion freeze_version;
ObConfirmedInfo confirmed_info;
header.generate_header(OB_LOG_SUBMIT, partition_key, 0, NULL, 0, ::oceanbase::common::ObTimeUtility::current_time(), 0,
pid, 0, freeze_version);
task_array[0].init(&alloc, &sliding_cb, &submit_cb, 1);
task_array[0].set_log(header, NULL, true);
header.generate_header(OB_LOG_SUBMIT, partition_key, 1, NULL, 0, ::oceanbase::common::ObTimeUtility::current_time(), 0,
pid, 0, freeze_version);
task_array[1].init(&alloc, &sliding_cb, &submit_cb, 1);
task_array[1].set_log(header, NULL, true);
header.generate_header(OB_LOG_SUBMIT, partition_key, 2, NULL, 0, ::oceanbase::common::ObTimeUtility::current_time(), 0,
pid, 0, freeze_version);
task_array[2].init(&alloc, &sliding_cb, &submit_cb, 1);
task_array[2].set_log(header, NULL, true);
// Set log 2 to confirm the status
task_array[2].set_confirmed_info(confirmed_info);
sw.array_ = task_array;
state_mgr.proposal_id_ = pid;
EXPECT_EQ(OB_EAGAIN, reconfirm.reconfirm());
// Set proposal_id to brush disk successfully and call back
state_mgr.proposal_id_ = helper.get_new_proposal_id(reconfirm);
//Verification timeout does not form a majority, resend
usleep(CLOG_LEADER_RECONFIRM_SYNC_TIMEOUT + 1);
ASSERT_EQ(OB_EAGAIN, reconfirm.reconfirm());
// Move the left edge of the sliding window, the synchronization of pending
// logs is completed, and enter the writing START_WORKING phase
sw.start_id_ = sw.max_id_ + 1;
ASSERT_EQ(OB_EAGAIN, reconfirm.reconfirm());
//Move the left border of the sliding window, START_WORKING is complete,
//and the RECONFIRM process is complete
sw.start_id_ = sw.max_id_ + 2;
ASSERT_EQ(OB_EAGAIN, reconfirm.reconfirm());
reconfirm.clear();
}
}
}
int main(int argc, char **argv)
{
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
return 0;
}

301
unittest/clog/test_ob_log_partition_meta_reader.cpp Normal file
View File

@ -0,0 +1,301 @@
// Copyright 2014 Alibaba Inc. All Rights Reserved.
// Author:
// qiaoli.xql@alibaba-inc.com
// Owner:
// lujun.wlj@alibaba-inc.com
//
// This file tests ObLogPartitionMetaReader.
//
//l1: Write the correct trailer information and read the correct PartitionMeta information
//l2: Write the correct trailer information and read the incorrect PartitionMeta information
//l3: Write the correct trailer information and can not read PartitionMeta information
//14: not write trailer
//
#include "clog/ob_log_partition_meta_reader.h"
#include "clog/ob_log_file_trailer.h"
#include "clog/ob_log_direct_reader.h"
#include "clog/ob_log_file_pool.h"
#include "clog/ob_log_common.h"
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "lib/tbsys.h"
#include <libaio.h>
using namespace oceanbase::common;
namespace oceanbase
{
using namespace clog;
namespace unittest
{
class MyMinUsingFileIDGetter: public ObIGetMinUsingFID
{
public:
MyMinUsingFileIDGetter() {}
virtual ~MyMinUsingFileIDGetter() {}
int on_leader_revoke(const common::ObPartitionKey &partition_key)
{ UNUSED(partition_key); return OB_SUCCESS; }
int on_leader_takeover(const common::ObPartitionKey &partition_key)
{ UNUSED(partition_key); return OB_SUCCESS; }
int on_member_change_success(
const common::ObPartitionKey &partition_key,
const int64_t mc_timestamp,
const common::ObMemberList &prev_member_list,
const common::ObMemberList &curr_member_list)
{ UNUSED(partition_key); UNUSED(mc_timestamp); UNUSED(prev_member_list); UNUSED(curr_member_list); return OB_SUCCESS; }
int64_t get_min_using_file_id() const
{ return 1; }
};
class TestObLogPartitionMetaReader: public ::testing::Test
{
public:
virtual void SetUp();
virtual void TearDown();
int write_data(const int fd, const char *buf, const int64_t buf_len, const int64_t offset);
int submit_aio(struct iocb *p);
int wait_event(struct iocb *p);
int write_file();
static const int64_t data_size = 1 << 9; //512
static const int64_t align_size = 1 << 10; //1K
static const char *label = "2";
static const char *label2 = "3";
io_context_t ctx_;
ObAlignedBuffer buffer;
ObAlignedBuffer buffer2;
ObLogFileTrailer trailer;
ObLogCache cache;
const char *path;
int64_t blk_len;
char buf[align_size];
};
void TestObLogPartitionMetaReader::SetUp()
{
path = "readers";
EXPECT_LE(0, system("mkdir readers"));
EXPECT_LE(0, system("rm readers/*"));
memset(&ctx_, 0, sizeof(ctx_));
memset(buf, 0, sizeof(buf));
EXPECT_EQ(0, io_setup(1, &ctx_));
EXPECT_EQ(OB_SUCCESS, buffer.init(align_size, align_size, label));
EXPECT_EQ(OB_SUCCESS, buffer2.init(data_size, data_size, label2));
const int64_t bucket_num = 1024;
const int64_t max_cache_size = 1024 * 1024 * 512;
const int64_t block_size = common::OB_MALLOC_BIG_BLOCK_SIZE;
ObKVGlobalCache::get_instance().init(bucket_num, max_cache_size, block_size);
EXPECT_EQ(OB_SUCCESS, cache.init("TestObLogCache", 5));
}
void TestObLogPartitionMetaReader::TearDown()
{
if (NULL != ctx_) {
io_destroy(ctx_);
}
buffer.destroy();
cache.destroy();
ObKVGlobalCache::get_instance().destroy();
}
int TestObLogPartitionMetaReader::write_data(const int fd, const char *buf, const int64_t buf_len,
const int64_t offset)
{
int ret = OB_SUCCESS;
struct iocb io;
struct iocb *p = &io;
memset(&io, 0x0, sizeof(io));
io_prep_pwrite(&io, fd, (void *)buf, buf_len, offset);
EXPECT_EQ(1, io_submit(ctx_, 1, &p));
EXPECT_EQ(OB_SUCCESS, wait_event(p));
return ret;
}
int TestObLogPartitionMetaReader::wait_event(struct iocb *p)
{
int ret = OB_SUCCESS;
struct io_event e;
struct timespec timeout;
OB_ASSERT(p->u.c.offset != -1);
OB_ASSERT(static_cast<int64_t>(p->u.c.nbytes) >= 0);
timeout.tv_sec = 100;
timeout.tv_nsec = 0; //100s
EXPECT_EQ(1, io_getevents(ctx_, 1, 1, &e, &timeout));
EXPECT_EQ(0U, e.res2);
EXPECT_EQ(p->u.c.nbytes, e.res);
EXPECT_EQ(p->data, e.data);
return ret;
}
int TestObLogPartitionMetaReader::write_file()
{
int ret = OB_SUCCESS;
int fd = -1;
int64_t pos = 0;
int64_t num = 2;
int64_t file_size = 1 << 22; //4K
int64_t percent = 100;
file_id_t start_file_id = 11; //11-14 are test file
ObLogBlockMetaV2::MetaContent block;
ObLogWriteFilePool write_pool;
MyMinUsingFileIDGetter min_getter;
EXPECT_EQ(OB_SUCCESS, write_pool.init(path, file_size, percent, &min_getter));
EXPECT_EQ(OB_SUCCESS, block.generate_block(buf, data_size, OB_DATA_BLOCK));
CLOG_LOG(INFO, "block", "meta", to_cstring(block), "buf", buf, "data_size", (int64_t)data_size);
EXPECT_EQ(OB_SUCCESS, block.serialize(buffer.get_align_buf(), align_size, pos));
memcpy((buffer.get_align_buf() + pos), buf, data_size);
offset_t start_pos = -1;
file_id_t next_file_id = 0;
blk_len = block.get_total_len();
//--------------test case 1-----------------------
//Write file which file id is 11, content is correct, write tow block
EXPECT_EQ(OB_SUCCESS, write_pool.get_fd(start_file_id, fd));
for (int64_t i = 0; i < num; i++) {
if (OB_SUCCESS != (ret = write_data(fd, buffer.get_align_buf(), blk_len, i * blk_len))) {
CLOG_LOG(ERROR, "write_data fail", KERRMSG);
}
}
pos = 0;
start_pos = 1024;
next_file_id = start_file_id + 1;
//Write the trailer of file which file id is 11, include start address and next file id
EXPECT_EQ(OB_SUCCESS, trailer.build_serialized_trailer(buffer2.get_align_buf(), data_size, start_pos, next_file_id, pos));
EXPECT_EQ(OB_SUCCESS, write_data(fd, buffer2.get_align_buf(), data_size, CLOG_TRAILER_OFFSET));
if (fd >= 0) {
close(fd);
}
//--------------test case 2-----------------------
// Write file whild file id is 13, write data failed, check block integrity fail
start_file_id = next_file_id;
EXPECT_EQ(OB_SUCCESS, write_pool.get_fd(start_file_id, fd));
buffer.get_align_buf()[512] = 'B';
for (int64_t i = 0; i < num; i++) {
if (OB_SUCCESS != (ret = write_data(fd, buffer.get_align_buf(), blk_len, i * blk_len))) {
CLOG_LOG(ERROR, "write_data fail", KERRMSG);
}
}
pos = 0;
start_pos = 0;
next_file_id = start_file_id + 1;
//Write the trailer of file which file id is 12, include start address and next file id
EXPECT_EQ(OB_SUCCESS, trailer.build_serialized_trailer(buffer2.get_align_buf(), data_size, start_pos, next_file_id, pos));
EXPECT_EQ(OB_SUCCESS, write_data(fd, buffer2.get_align_buf(), data_size, CLOG_TRAILER_OFFSET));
if (fd >= 0) {
close(fd);
}
//--------------test case 3-----------------------
// Write file whild file id is 13, write one block, data correct,
// check meat checksum fail
start_file_id = next_file_id;
EXPECT_EQ(OB_SUCCESS, write_pool.get_fd(start_file_id, fd));
EXPECT_EQ(OB_SUCCESS, write_data(fd, buffer.get_align_buf(), blk_len, 0));
pos = 0;
start_pos = 512;
next_file_id = start_file_id + 1;
//Write the trailer of file which file id is 13, include start address and next file id
EXPECT_EQ(OB_SUCCESS, trailer.build_serialized_trailer(buffer2.get_align_buf(), data_size, start_pos, next_file_id, pos));
EXPECT_EQ(OB_SUCCESS, write_data(fd, buffer2.get_align_buf(), data_size, CLOG_TRAILER_OFFSET));
if (fd >= 0) {
close(fd);
}
//--------------test case 4-----------------------
// Write file whild file id is 14, write one block, data correct
start_file_id = next_file_id;
EXPECT_EQ(OB_SUCCESS, write_pool.get_fd(start_file_id, fd));
EXPECT_EQ(OB_SUCCESS, write_data(fd, buffer.get_align_buf(), blk_len, 0));
if (fd >= 0) {
close(fd);
}
return ret;
}
TEST_F(TestObLogPartitionMetaReader, test_read_partition_meta)
{
ObLogReadFilePool reader_pool;
ObLogDirectReader dreader;
ObLogPartitionMetaReader reader;
ObReadParam param;
ObReadRes res;
file_id_t start_file_id = 11;
param.timeout_ = 5000000; //5s
param.read_len_ = OB_MAX_LOG_BUFFER_SIZE;
param.file_id_ = start_file_id;
EXPECT_EQ(OB_NOT_INIT, reader.read_partition_meta(param, res));
EXPECT_EQ(OB_INVALID_ARGUMENT, reader.init(NULL));
EXPECT_EQ(OB_NOT_INIT, reader.read_partition_meta(param, res));
//Test read
CLOG_LOG(INFO, "begin unittest::test_ob_log_partition_meta_reader");
EXPECT_EQ(OB_SUCCESS, reader_pool.init(path));
EXPECT_EQ(OB_SUCCESS, dreader.init(&reader_pool, &cache));
EXPECT_EQ(OB_SUCCESS, reader.init(&dreader));
EXPECT_EQ(OB_INIT_TWICE, reader.init(&dreader));
EXPECT_EQ(OB_SUCCESS, write_file());
//Test read first block
param.file_id_ = start_file_id;
EXPECT_EQ(OB_SUCCESS, reader.read_partition_meta(param, res));
EXPECT_EQ((int64_t)data_size, res.data_len_);
param.file_id_ = param.file_id_ + 1;
EXPECT_EQ(OB_INVALID_DATA, reader.read_partition_meta(param, res));
param.file_id_ = param.file_id_ + 1;
EXPECT_EQ(OB_INVALID_DATA, reader.read_partition_meta(param, res));
param.file_id_ = param.file_id_ + 1;
EXPECT_EQ(OB_READ_NOTHING, reader.read_partition_meta(param, res));
//Test failure
param.reset();
EXPECT_EQ(OB_INVALID_ARGUMENT, reader.read_partition_meta(param, res));
//Test deserialize failed
param.file_id_ = start_file_id;
param.read_len_ = 1;
param.timeout_ = 5000000; //5s
EXPECT_EQ(OB_DESERIALIZE_ERROR, reader.read_partition_meta(param, res));
//Test invalid argument
param.file_id_ = start_file_id;
param.read_len_ = 1;
param.timeout_ = 0;
res.reset();
EXPECT_EQ(OB_INVALID_ARGUMENT, reader.read_partition_meta(param, res));
}
} // end namespace unittest
} // end namespace oceanbase
int main(int argc, char **argv)
{
OB_LOGGER.set_file_name("test_ob_log_partition_meta_reader.log", true);
OB_LOGGER.set_log_level("INFO");
CLOG_LOG(INFO, "begin unittest::test_ob_log_partition_meta_reader");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

875
unittest/clog/test_ob_log_writer.cpp Normal file
View File

@ -0,0 +1,875 @@
// Copyright 2014 Alibaba Inc. All Rights Reserved.
// Author:
// qiaoli.xql@alibaba-inc.com
// Owner:
// lujun.wlj@alibaba-inc.com
//
// This file tests ObLogWriter.
//
// Testing method
//1. Write 60 data blocks, each log saves the same 1K data buffer, each block 1000 logs,
// log ID is (1-1000)*i
//2. Write a confirm block after every 10,000 logs, which contains the confirm log of
// the first 10,000 logs
//3. The file is written by ObLogWriter, and the log is read iteratively by ObLogIterator
//
//Testing scenarios
//1. Write start file id is 1
//2. After writing 10 data blocks, submit a sync_flush_task task and submit a confirm block
//3. After the last confirm log data block, switch_file_task is called once
//4. Write the first 60 data blocks, at least 64M, ObLogWriter will actively cut the file once
//5. Test ObLogFileLocator:
// 1)Create the first file when the directory is empty
// 2)Find the end of the file when there is a file
// 3)The file with the largest ID happens to be the first newly created file
//6. Use ObLogReader to read the log in turn for the correctness of comparison
//
#include "clog/ob_log_writer.h"
#include "clog/ob_log_file_trailer.h"
#include "clog/ob_info_block_handler.h"
#include "clog/ob_log_common.h"
#include "clog/ob_log_type.h"
#include "lib/utility/ob_tracepoint.h"
#include "share/ob_proposal_id.h"
#include <libaio.h>
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "lib/tbsys.h"
using namespace oceanbase::common;
namespace oceanbase
{
namespace clog
{
//class MyObLogScannerUtilityImp: public ObILogScannerUtilityImp
//{
//public:
// int scan_partition_log_range(const ObPartitionKey &partition_key,
// const char *buf,
// const int64_t data_len,
// uint64_t &min_log_id,
// uint64_t &max_log_id);
//};
//
//int MyObLogScannerUtilityImp::scan_partition_log_range(const ObPartitionKey &partition_key,
// const char *buf,
// const int64_t data_len,
// uint64_t &min_log_id,
// uint64_t &max_log_id)
//{
// int ret = OB_SUCCESS;
// UNUSED(partition_key);
// UNUSED(buf);
// UNUSED(data_len);
// min_log_id = 1;
// max_log_id = 60000;
// return ret;
//}
//
//ObILogScannerUtilityImp *get_instance()
//{
// static MyObLogScannerUtilityImp instance;
// return &instance;
//}
class FlushLogTask: public IFlushLogTask
{
public:
FlushLogTask(): is_done_(false) {}
virtual ~FlushLogTask() { is_done_ = false; }
int after_flushed(const file_id_t file_id, const offset_t offset, const int error_code);
void init(char *buf, const int64_t data_len);
void reset();
bool is_done() { return is_done_; }
private:
volatile bool is_done_;
};
int FlushLogTask::after_flushed(const file_id_t file_id,
const offset_t offset,
const int error_code)
{
UNUSED(file_id);
UNUSED(offset);
UNUSED(error_code);
is_done_ = true;
return OB_SUCCESS;
}
void FlushLogTask::init(char *buf, int64_t data_len)
{
IFlushLogTask::init(buf, data_len);
is_done_ = false;
}
void FlushLogTask::reset()
{
buf_ = NULL;
data_len_ = 0;
is_done_ = false;
}
} //namespace clog
using namespace clog;
namespace unittest
{
class MyMinUsingFileIDGetter: public ObIGetMinUsingFID
{
public:
MyMinUsingFileIDGetter() {}
virtual ~MyMinUsingFileIDGetter() {}
int on_leader_revoke(const common::ObPartitionKey &partition_key)
{ UNUSED(partition_key); return OB_SUCCESS; }
int on_leader_takeover(const common::ObPartitionKey &partition_key)
{ UNUSED(partition_key); return OB_SUCCESS; }
int on_member_change_success(
const common::ObPartitionKey &partition_key,
const int64_t mc_timestamp,
const common::ObMemberList &prev_member_list,
const common::ObMemberList &curr_member_list)
{ UNUSED(partition_key); UNUSED(mc_timestamp); UNUSED(prev_member_list); UNUSED(curr_member_list); return OB_SUCCESS; }
int64_t get_min_using_file_id() const
{ return 10000; }
};
class MyMetaInfoGenerator: public ObIInfoBlockHandler
{
public:
MyMetaInfoGenerator() {}
virtual ~MyMetaInfoGenerator() {}
virtual int build_info_block(char *buf, const int64_t buf_len, int64_t &pos);
virtual int resolve_info_block(const char *buf, const int64_t buf_len, int64_t &pos);
virtual int update_info(const ObVersion &version);
};
int MyMetaInfoGenerator::build_info_block(char *buf, const int64_t buf_len, int64_t &pos)
{
UNUSED(buf_len);
memset(buf, 'Z', buf_len);
pos = 1023;
return OB_SUCCESS;
}
int MyMetaInfoGenerator::resolve_info_block(const char *buf, const int64_t buf_len, int64_t &pos)
{
UNUSED(buf);
UNUSED(buf_len);
UNUSED(pos);
return OB_SUCCESS;
}
int MyMetaInfoGenerator::update_info(const ObVersion &version)
{
UNUSED(version);
return OB_SUCCESS;
}
class TestObLogWriter: public ::testing::Test
{
public:
virtual void SetUp();
virtual void TearDown();
int write_data(const int fd, const char *buf, const int64_t data_len, const offset_t offset);
int submit_aio(struct iocb *p);
int wait_event(struct iocb *p);
int prepare_data_block(const uint64_t max_id,
char *&blk_buf,
int64_t &blk_len);
int prepare_confirm_block(const uint64_t max_id,
const uint64_t num,
char *&blk_buf,
int64_t &blk_len);
int submit_task(ObLogWriter &writer, uint64_t &max_log_id);
static const ObLogType ctype = OB_LOG_CONFIRMED;
static const ObLogType ltype = OB_LOG_SUBMIT;
static const common::ObProposalID rts;
static const int64_t gts = 201;
static const int64_t tts = 202;
static const uint64_t table_id = 5;
static const int64_t partition_idx = 20;
ObPartitionKey partition_key;
static const int64_t data_len = 1024 - 200;
char buf[data_len];
static const int64_t size = 1 << 21;
static const int64_t align_size = 512;
const char *path;
const char *dir;
int64_t file_size;
int64_t percent;
offset_t start_offset;
int64_t queue_size;
file_id_t file_id;
int64_t timeout;
io_context_t ctx_;
ObAlignedBuffer buffer;
ObAlignedBuffer confirm_buffer;
MyMetaInfoGenerator meta_gen;
ObLogReadFilePool reader_pool;
ObLogWriteFilePool write_pool;
MyMinUsingFileIDGetter min_getter;
ObReadParam param;
ObLogFileLocator<ObLogEntry, ObIRawLogIterator> locator;
ObLogReadFilePool rpool;
ObLogWriteFilePool wpool;
ObLogDirectReader dreader;
ObLogCache cache;
};
const common::ObProposalID TestObLogWriter::rts;
void TestObLogWriter::SetUp()
{
CLOG_LOG(INFO, "SetUp");
//TestObLogWriter::rts.ts_ = 200;
int task_num = 1024;
const char *label1 = "1";
const char *label2 = "2";
timeout = 10000000; //10s
file_size = CLOG_FILE_SIZE; //64M
percent = 100;
param.timeout_ = timeout;
EXPECT_EQ(OB_SUCCESS, partition_key.init(3, 8, 9));
memset(buf, 'A', data_len);
memset(&ctx_, 0, sizeof(ctx_));
path = "writer/";
dir = "writer_error";
EXPECT_EQ(OB_SUCCESS, buffer.init(size, align_size, label1));
EXPECT_EQ(OB_SUCCESS, confirm_buffer.init(size, align_size, label2));
EXPECT_EQ(0, io_setup(task_num, &ctx_));
EXPECT_LE(0, system("rm -rf writer"));
EXPECT_LE(0, system("mkdir writer"));
EXPECT_LE(0, system("rm -rf writer_error"));
EXPECT_LE(0, system("mkdir writer_error"));
EXPECT_EQ(OB_SUCCESS, reader_pool.init(path));
EXPECT_EQ(OB_SUCCESS, write_pool.init(path, file_size, percent, &min_getter));
//oceanbase::clog::log_scanner_utility_ptr = get_instance();
int fd = 0;
file_id = 1;
start_offset = 0;
queue_size = 1024;
EXPECT_EQ(OB_SUCCESS, rpool.init(dir));
EXPECT_EQ(OB_SUCCESS, wpool.init(dir, file_size, percent, &min_getter));
EXPECT_EQ(OB_SUCCESS, wpool.get_fd(file_id, fd));
if (fd >= 0) {
close(fd);
}
const int64_t bucket_num = 1024;
const int64_t max_cache_size = 1024 * 1024 * 512;
const int64_t block_size = common::OB_MALLOC_BIG_BLOCK_SIZE;
ObKVGlobalCache::get_instance().init(bucket_num, max_cache_size, block_size);
EXPECT_EQ(OB_SUCCESS, cache.init("TestObLogCache", 5));
EXPECT_EQ(OB_SUCCESS, dreader.init(&reader_pool, &cache));
}
void TestObLogWriter::TearDown()
{
CLOG_LOG(INFO, "TearDown");
if (NULL != ctx_) {
io_destroy(ctx_);
}
buffer.destroy();
confirm_buffer.destroy();
cache.destroy();
dreader.destroy();
ObKVGlobalCache::get_instance().destroy();
}
int TestObLogWriter::write_data(const int fd, const char *buf, const int64_t data_len,
const offset_t offset)
{
int ret = OB_SUCCESS;
OB_ASSERT(-1 != fd);
OB_ASSERT(NULL != buf);
OB_ASSERT(data_len > 0);
OB_ASSERT(offset >= 0);
struct iocb io;
struct iocb *p = &io;
memset(&io, 0x0, sizeof(io));
io_prep_pwrite(&io, fd, (void *)buf, data_len, offset);
EXPECT_EQ(1, io_submit(ctx_, 1, &p));
EXPECT_EQ(OB_SUCCESS, wait_event(p));
return ret;
}
int TestObLogWriter::wait_event(struct iocb *p)
{
int ret = OB_SUCCESS;
struct io_event e;
struct timespec timeout;
OB_ASSERT(p->u.c.offset != -1);
OB_ASSERT(static_cast<int64_t>(p->u.c.nbytes) >= 0);
timeout.tv_sec = 100;
timeout.tv_nsec = 0; //100s
EXPECT_EQ(1, io_getevents(ctx_, 1, 1, &e, &timeout));
EXPECT_TRUE(0 == e.res2);
EXPECT_EQ(p->u.c.nbytes, e.res);
EXPECT_EQ(p->data, e.data);
return ret;
}
int TestObLogWriter::prepare_data_block(const uint64_t max_id,
char *&blk_buf,
int64_t &blk_len)
{
ObLogEntryHeader header;
ObLogEntry entry;
ObLogBlockMetaV2::MetaContent meta;
uint64_t log_id = OB_INVALID_ID;
int64_t meta_len = meta.get_serialize_size();
int64_t pos = meta_len;
for (uint64_t i = 1; i <= max_id; i++) {
EXPECT_EQ(OB_SUCCESS, header.generate_header(ltype, partition_key, log_id, buf, data_len, gts, tts,
rts, 0, 1, 1, ObVersion()));
EXPECT_EQ(OB_SUCCESS, entry.generate_entry(header, buf));
EXPECT_EQ(OB_SUCCESS, entry.serialize(buffer.get_align_buf(), size, pos));
}
blk_len = pos - meta_len;
blk_buf = buffer.get_align_buf() + meta_len;
return OB_SUCCESS;
}
int TestObLogWriter::prepare_confirm_block(const uint64_t max_id,
const uint64_t num,
char *&blk_buf,
int64_t &blk_len)
{
ObLogEntryHeader header;
ObLogEntry entry;
ObLogBlockMetaV2::MetaContent meta;
ObConfirmedLog clog;
uint64_t log_id = OB_INVALID_ID;
int64_t meta_len = meta.get_serialize_size();
int64_t pos = meta_len;
int64_t confirm_pos = 0;
int64_t clog_pos = 0;
int64_t clog_len = 1024;
char clog_buf[clog_len];
for (uint64_t i = 1; i <= max_id; i++) {
clog_pos = 0;
log_id = i + max_id * num;
EXPECT_EQ(OB_SUCCESS, header.generate_header(ltype, partition_key, log_id, buf, data_len, gts, tts,
rts, 0, 1, 1, ObVersion()));
EXPECT_EQ(OB_SUCCESS, entry.generate_entry(header, buf));
EXPECT_EQ(OB_SUCCESS, entry.serialize(buffer.get_align_buf(), size, pos));
clog.set_log_id(log_id);
clog.set_data_checksum(header.get_data_checksum());
clog.set_epoch_id(header.get_epoch_id());
EXPECT_EQ(OB_SUCCESS, clog.serialize(clog_buf, clog_len, clog_pos));
EXPECT_EQ(OB_SUCCESS, header.generate_header(ltype, partition_key, log_id, buf, data_len, gts, tts,
rts, 0, 1, 1, ObVersion()));
EXPECT_EQ(OB_SUCCESS, entry.generate_entry(header, clog_buf));
EXPECT_EQ(OB_SUCCESS, entry.serialize(confirm_buffer.get_align_buf(), size, confirm_pos));
}
memcpy(buffer.get_align_buf() + pos, confirm_buffer.get_align_buf(), confirm_pos);
blk_len = pos + confirm_pos - meta_len;
blk_buf = buffer.get_align_buf() + meta_len;
return OB_SUCCESS;
}
int TestObLogWriter::submit_task(ObLogWriter &writer, uint64_t &max_log_id)
{
int ret = OB_SUCCESS;
uint64_t write_times = 1;
uint64_t entry_num_in_block = 60;
FlushLogTask flush_task;
char *f_buf = NULL;
int64_t f_dl = 0;
max_log_id = write_times * entry_num_in_block;
for (uint64_t i = 0; i < write_times; i++) {
flush_task.reset();
EXPECT_EQ(OB_SUCCESS, prepare_confirm_block(entry_num_in_block, i, f_buf, f_dl));
flush_task.init(f_buf, f_dl);
EXPECT_EQ(OB_SUCCESS, writer.submit_flush_log_task(&flush_task));
while (!flush_task.is_done()) {
PAUSE();
}
}
return ret;
}
TEST_F(TestObLogWriter, test_log_file_locator)
{
ObLogBlockMetaV2::MetaContent t_meta;
int fd = -1;
file_id_t file_id = OB_INVALID_FILE_ID;
offset_t offset = OB_INVALID_OFFSET;
int64_t pos = 0;
int64_t num = 1000;
int64_t meta_len = t_meta.get_serialize_size();
char *t_buf = NULL;
int64_t t_data_len = 0;;
int64_t t_blk_len = 0;
//Test invalid arguments
EXPECT_EQ(OB_INVALID_ARGUMENT, locator.get_cursor(param.timeout_, NULL, &dreader, file_id, offset));
EXPECT_LE(0, system("rm -rf writer"));
EXPECT_LE(0, system("mkdir writer"));
EXPECT_EQ(OB_SUCCESS, locator.get_cursor(param.timeout_, &reader_pool, &dreader, file_id, offset));
EXPECT_EQ(1UL, file_id);
EXPECT_EQ(0, offset);
file_id_t t_file_id = 4;
offset_t t_offset = 0;
EXPECT_EQ(OB_SUCCESS, write_pool.get_fd(t_file_id, fd));
//Test exist files, and content is valid
pos = 0;
EXPECT_EQ(OB_SUCCESS, prepare_data_block(num, t_buf, t_data_len));
EXPECT_EQ(OB_SUCCESS, t_meta.generate_block(t_buf, t_data_len, OB_DATA_BLOCK));
EXPECT_EQ(OB_SUCCESS, t_meta.serialize(t_buf - meta_len, meta_len, pos));
t_blk_len = t_meta.get_total_len();
memcpy(t_buf - meta_len + t_blk_len, ObEOFBuf::eof_flag_buf_, DIO_ALIGN_SIZE);
EXPECT_EQ(OB_SUCCESS, write_data(fd, t_buf - meta_len, t_blk_len + DIO_ALIGN_SIZE , t_offset));
EXPECT_EQ(OB_SUCCESS, locator.get_cursor(param.timeout_, &reader_pool, &dreader, file_id, offset));
EXPECT_EQ(t_file_id, file_id);
EXPECT_EQ(t_offset + t_blk_len, offset);
if (fd >= 0) {
close(fd);
}
//Test only a empty file
t_file_id = 30;
EXPECT_EQ(OB_SUCCESS, write_pool.get_fd(t_file_id, fd));
EXPECT_EQ(OB_SUCCESS, locator.get_cursor(param.timeout_, &reader_pool, &dreader, file_id, offset));
EXPECT_EQ(t_file_id, file_id);
EXPECT_EQ(0, offset);
if (fd >= 0) {
close(fd);
}
}
TEST_F(TestObLogWriter, test_ob_write_param)
{
int t_fd = 4;
file_id_t t_file_id = 7;
offset_t t_offset = 123;
int64_t t_write_len = 3333;
int64_t t_timeout = 9000;
char buf[20];
ObWriteParam param1;
ObWriteParam param2;
param1.fd_ = t_fd;
param1.file_id_ = t_file_id;
param1.offset_ = t_offset;
param1.buf_ = buf;
param1. write_len_ = t_write_len;
param1.timeout_ = t_timeout;
param2.reset();
param2.shallow_copy(param1);
EXPECT_TRUE(param1 == param2);
param1.reset();
param2.reset();
EXPECT_TRUE(param1 == param2);
CLOG_LOG(INFO, "param to string", "param", to_cstring(param1));
}
TEST_F(TestObLogWriter, test_write_data)
{
ObReadParam param;
ObLogWriter writer;
ObLogEntryHeader t_header;
ObLogEntry t_entry;
ObLogEntry r_entry;
uint64_t max_log_id = 60;
uint64_t times = 2 * 1000; //1K * 60K * 2 = 120M, switch file
int fd = -1;
param.read_len_ = OB_MAX_LOG_BUFFER_SIZE;
param.file_id_ = 1;
param.log_id_ = 1;
param.timeout_ = 10000000; //5s
param.partition_key_ = partition_key;
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf() + 100, 1 << 20);
EXPECT_EQ(OB_NOT_INIT, writer.submit_flush_log_task(&flush_task));
// Return the first file, file_id = 1
EXPECT_EQ(OB_SUCCESS, locator.get_cursor(param.timeout_, &reader_pool, &dreader, file_id, start_offset));
EXPECT_EQ(OB_SUCCESS, writer.init(file_id, start_offset, param.timeout_, queue_size, &dreader, &write_pool, &cache,
&meta_gen));
EXPECT_EQ(OB_SUCCESS, submit_task(writer, max_log_id));
CLOG_LOG(INFO, "write success");
for (uint64_t i = 1; i <= times; i++) {
param.log_id_ = i;
param.file_id_ = OB_INVALID_FILE_ID;
EXPECT_EQ(OB_SUCCESS, t_header.generate_header(ltype, partition_key, i, buf, data_len, gts, tts,
rts, 0, 1, 1, ObVersion()));
EXPECT_EQ(OB_SUCCESS, t_entry.generate_entry(t_header, buf));
EXPECT_EQ(OB_SUCCESS, submit_task(writer, max_log_id));
}
if (fd >= 0) {
close(fd);
}
writer.destroy();
}
TEST_F(TestObLogWriter, test_possible_init_b_error)
{
ObLogWriter twriter;
file_id_t file_id = 1;
int64_t timeout = 10000000; //1s
// Init failed
EXPECT_EQ(OB_SUCCESS, twriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
EXPECT_EQ(OB_INIT_TWICE, twriter.init(file_id, start_offset, param.timeout_, queue_size, &dreader, &write_pool, &cache, &meta_gen));
twriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_init_t_error)
{
ObLogWriter twriter;
file_id_t file_id = 1;
int64_t timeout = 10000000; //1s
// Init failed
TP_SET_ERROR("ob_log_writer.cpp", "init", "tinit", OB_ERROR);
EXPECT_EQ(OB_ERROR, twriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
TP_SET("ob_log_writer.cpp", "init", "tinit", NULL);
twriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_init_iosetup_error)
{
ObLogWriter twriter;
file_id_t file_id = 1;
int64_t timeout = 10000000; //1s
// Init failed
TP_SET_ERROR("ob_log_writer.cpp", "init", "iosetup", OB_ERROR);
EXPECT_EQ(OB_IO_ERROR, twriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
TP_SET("ob_log_writer.cpp", "init", "iosetup", NULL);
twriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_init_getfd_error)
{
ObLogWriter twriter;
file_id_t file_id = 1;
int64_t timeout = 10000000; //1s
// Init failed
TP_SET_ERROR("ob_log_writer.cpp", "init", "getfd", OB_ERROR);
EXPECT_EQ(OB_ERROR, twriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
TP_SET("ob_log_writer.cpp", "init", "getfd", NULL);
twriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_submit_tasks_errors)
{
// Submit flush task failed
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf(), -1);
EXPECT_EQ(OB_INVALID_ARGUMENT, ewriter.submit_flush_log_task(NULL));
EXPECT_EQ(OB_INVALID_ARGUMENT, ewriter.submit_flush_log_task(&flush_task));
flush_task.init(buffer.get_align_buf(), 1 << 20);
TP_SET_ERROR("ob_log_writer.cpp", "submit_flush_log_task", "push", OB_SIZE_OVERFLOW);
EXPECT_EQ(OB_EAGAIN, ewriter.submit_flush_log_task(&flush_task));
TP_SET("ob_log_writer.cpp", "submit_flush_log_task", "push", NULL);
ewriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_handle_flush_g_errors)
{
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf() + 100, 1 << 20);
//Handle flush task
TP_SET_ERROR("ob_log_writer.cpp", "handle_flush_log_task", "g", OB_ERROR);
EXPECT_EQ(OB_SUCCESS, ewriter.submit_flush_log_task(&flush_task));
while (!flush_task.is_done()) {
PAUSE();
}
TP_SET("ob_log_writer.cpp", "handle_flush_log_task", "g", NULL);
ewriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_handle_flush_s_errors)
{
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf() + 100, 1 << 20);
TP_SET_ERROR("ob_log_writer.cpp", "handle_flush_log_task", "s", OB_ERROR);
EXPECT_EQ(OB_SUCCESS, ewriter.submit_flush_log_task(&flush_task));
while (!flush_task.is_done()) {
PAUSE();
}
TP_SET("ob_log_writer.cpp", "handle_flush_log_task", "s", NULL);
ewriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_handle_flush_w_errors)
{
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf() + 100, 1 << 20);
TP_SET_ERROR("ob_log_writer.cpp", "handle_flush_log_task", "w", OB_ERROR);
EXPECT_EQ(OB_SUCCESS, ewriter.submit_flush_log_task(&flush_task));
while (!flush_task.is_done()) {
PAUSE();
}
TP_SET("ob_log_writer.cpp", "handle_flush_log_task", "w", NULL);
ewriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_write_data_s_errors)
{
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf() + 100, 1 << 10);
TP_SET_ERROR("ob_log_writer.cpp", "write_data", "submit", OB_ERROR);
EXPECT_EQ(OB_SUCCESS, ewriter.submit_flush_log_task(&flush_task));
while (!flush_task.is_done()) {
PAUSE();
}
TP_SET("ob_log_writer.cpp", "write_data", "submit", NULL);
ewriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_write_data_w_errors)
{
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf() + 200, 1 << 10);
TP_SET_ERROR("ob_log_writer.cpp", "write_data", "wait", OB_ERROR);
EXPECT_EQ(OB_SUCCESS, ewriter.submit_flush_log_task(&flush_task));
while (!flush_task.is_done()) {
PAUSE();
}
TP_SET("ob_log_writer.cpp", "write_data", "wait", NULL);
ewriter.destroy();
}
} // end namespace unittest
} // end namespace oceanbase
int main(int argc, char **argv)
{
OB_LOGGER.set_file_name("test_ob_log_writer.log", true);
OB_LOGGER.set_log_level("DEBUG");
CLOG_LOG(INFO, "begin unittest::test_ob_log_writer");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
EXPECT_EQ(OB_INVALID_ARGUMENT, twriter.init(file_id, -1, timeout, queue_size, &dreader, &wpool, NULL, &meta_gen));
EXPECT_EQ(OB_INVALID_ARGUMENT, twriter.init(OB_INVALID_FILE_ID, start_offset, timeout, queue_size, &dreader, &wpool, NULL, &meta_gen));
EXPECT_EQ(OB_INVALID_ARGUMENT, twriter.init(file_id, start_offset, 0, queue_size, &dreader, &wpool, NULL, &meta_gen));
TP_SET_ERROR("ob_log_writer.cpp", "init", "binit", OB_ERROR);
EXPECT_EQ(OB_ERROR, twriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
TP_SET("ob_log_writer.cpp", "init", "binit", NULL);
// Init success
EXPECT_EQ(OB_SUCCESS, twriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
EXPECT_EQ(OB_INIT_TWICE, twriter.init(file_id, start_offset, param.timeout_, queue_size, &dreader, &write_pool, &cache, &meta_gen));
twriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_init_t_error)
{
ObLogWriter twriter;
file_id_t file_id = 1;
int64_t timeout = 10000000; //1s
// Init failed
TP_SET_ERROR("ob_log_writer.cpp", "init", "tinit", OB_ERROR);
EXPECT_EQ(OB_ERROR, twriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
TP_SET("ob_log_writer.cpp", "init", "tinit", NULL);
twriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_init_iosetup_error)
{
ObLogWriter twriter;
file_id_t file_id = 1;
int64_t timeout = 10000000; //1s
// Init failed
TP_SET_ERROR("ob_log_writer.cpp", "init", "iosetup", OB_ERROR);
EXPECT_EQ(OB_IO_ERROR, twriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
TP_SET("ob_log_writer.cpp", "init", "iosetup", NULL);
twriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_init_getfd_error)
{
ObLogWriter twriter;
file_id_t file_id = 1;
int64_t timeout = 10000000; //1s
// Init failed
TP_SET_ERROR("ob_log_writer.cpp", "init", "getfd", OB_ERROR);
EXPECT_EQ(OB_ERROR, twriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
TP_SET("ob_log_writer.cpp", "init", "getfd", NULL);
twriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_submit_tasks_errors)
{
// Submit flush task failed
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf(), -1);
EXPECT_EQ(OB_INVALID_ARGUMENT, ewriter.submit_flush_log_task(NULL));
EXPECT_EQ(OB_INVALID_ARGUMENT, ewriter.submit_flush_log_task(&flush_task));
flush_task.init(buffer.get_align_buf(), 1 << 20);
TP_SET_ERROR("ob_log_writer.cpp", "submit_flush_log_task", "push", OB_SIZE_OVERFLOW);
EXPECT_EQ(OB_EAGAIN, ewriter.submit_flush_log_task(&flush_task));
TP_SET("ob_log_writer.cpp", "submit_flush_log_task", "push", NULL);
ewriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_handle_flush_g_errors)
{
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf() + 100, 1 << 20);
//Handle flush task
TP_SET_ERROR("ob_log_writer.cpp", "handle_flush_log_task", "g", OB_ERROR);
EXPECT_EQ(OB_SUCCESS, ewriter.submit_flush_log_task(&flush_task));
while (!flush_task.is_done()) {
PAUSE();
}
TP_SET("ob_log_writer.cpp", "handle_flush_log_task", "g", NULL);
ewriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_handle_flush_s_errors)
{
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf() + 100, 1 << 20);
TP_SET_ERROR("ob_log_writer.cpp", "handle_flush_log_task", "s", OB_ERROR);
EXPECT_EQ(OB_SUCCESS, ewriter.submit_flush_log_task(&flush_task));
while (!flush_task.is_done()) {
PAUSE();
}
TP_SET("ob_log_writer.cpp", "handle_flush_log_task", "s", NULL);
ewriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_handle_flush_w_errors)
{
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf() + 100, 1 << 20);
TP_SET_ERROR("ob_log_writer.cpp", "handle_flush_log_task", "w", OB_ERROR);
EXPECT_EQ(OB_SUCCESS, ewriter.submit_flush_log_task(&flush_task));
while (!flush_task.is_done()) {
PAUSE();
}
TP_SET("ob_log_writer.cpp", "handle_flush_log_task", "w", NULL);
ewriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_write_data_s_errors)
{
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf() + 100, 1 << 10);
TP_SET_ERROR("ob_log_writer.cpp", "write_data", "submit", OB_ERROR);
EXPECT_EQ(OB_SUCCESS, ewriter.submit_flush_log_task(&flush_task));
while (!flush_task.is_done()) {
PAUSE();
}
TP_SET("ob_log_writer.cpp", "write_data", "submit", NULL);
ewriter.destroy();
}
TEST_F(TestObLogWriter, test_possible_write_data_w_errors)
{
ObLogWriter ewriter;
EXPECT_EQ(OB_SUCCESS, ewriter.init(file_id, start_offset, timeout, queue_size, &dreader, &wpool, &cache, &meta_gen));
FlushLogTask flush_task;
flush_task.init(buffer.get_align_buf() + 200, 1 << 10);
TP_SET_ERROR("ob_log_writer.cpp", "write_data", "wait", OB_ERROR);
EXPECT_EQ(OB_SUCCESS, ewriter.submit_flush_log_task(&flush_task));
while (!flush_task.is_done()) {
PAUSE();
}
TP_SET("ob_log_writer.cpp", "write_data", "wait", NULL);
ewriter.destroy();
}
} // end namespace unittest
} // end namespace oceanbase
int main(int argc, char **argv)
{
OB_LOGGER.set_file_name("test_ob_log_writer.log", true);
OB_LOGGER.set_log_level("DEBUG");
CLOG_LOG(INFO, "begin unittest::test_ob_log_writer");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}