hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[PALF] PALF Cluster Test Framework

Browse Source
This commit is contained in:
BinChenn
2023-09-27 08:43:51 +00:00
committed by ob-robot
parent 6d33b0ece6
commit 5cede3d9ea
30 changed files with 5781 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
mittest/CMakeLists.txt
View File

@ -2,3 +2,4 @@ add_subdirectory(logservice)
add_subdirectory(simple_server) add_subdirectory(simple_server)
add_subdirectory(mtlenv) add_subdirectory(mtlenv)
add_subdirectory(multi_replica) add_subdirectory(multi_replica)
#add_subdirectory(palf_cluster)

0
mittest/logservice/env/ob_simple_log_cluster_env.cpp vendored Executable file → Normal file
View File

4
mittest/logservice/test_ob_simple_log_apply.cpp
View File

@ -55,6 +55,10 @@ public:
{ {
ATOMIC_INC(&failure_count_); ATOMIC_INC(&failure_count_);
} }
int replay(ObLogReplayTask *replay_task){
UNUSED(replay_task);
return OB_SUCCESS;
}
int64_t success_count_; int64_t success_count_;
int64_t failure_count_; int64_t failure_count_;
common::ObQSync ls_qs_; common::ObQSync ls_qs_;

24
mittest/palf_cluster/CMakeLists.txt Normal file
View File

@ -0,0 +1,24 @@
set(PALF_CLUSTER_TEST_SRCS
env/ob_simple_log_cluster.cpp
env/ob_simple_log_server.cpp
logservice/ob_log_client.cpp
logservice/log_service.cpp
logservice/ls_adapter.cpp
logservice/role_coordinator_handler.cpp
logservice/role_coordinator.cpp
rpc/palf_cluster_rpc_req.cpp
rpc/palf_cluster_request_handler.cpp)
add_library(palf_cluster_test ${PALF_CLUSTER_TEST_SRCS})
target_include_directories(palf_cluster_test PUBLIC
${CMAKE_SOURCE_DIR}/unittest ${CMAKE_SOURCE_DIR}/mittest)
target_link_libraries(palf_cluster_test PUBLIC oceanbase)
function(ob_unittest_clog case)
ob_unittest(${ARGV})
target_link_libraries(${case} PRIVATE gtest gmock palf_cluster_test oceanbase)
set_tests_properties(${case} PROPERTIES TIMEOUT 600)
endfunction()
ob_unittest_clog(test_palf_bench_server test_palf_bench_server.cpp)
ob_unittest_clog(test_palf_bench_client test_palf_bench_client.cpp)

77
mittest/palf_cluster/README.md Normal file
View File

@ -0,0 +1,77 @@
# A Test Framework for PALF Cluster
The framework is designed and implentmented for evaluating PALF performance in a real cluster.
## Metrics
- Throughput
- Latency
- IOPS
...
## Usage
1. Use your own IP and wrok directory
**IP**
- `mittest/palf_cluster/test_palf_bench_server.cpp:45`
- `mittest/palf_cluster/test_palf_bench_client.cpp:45`
- `mittest/palf_cluster/run_palf_bench.sh:2`
**PORT**
default 53212
- `mittest/palf_cluster/env/ob_simple_log_cluster.h:92`
**workdir**
- `mittest/palf_cluster/run_palf_bench.sh:`
2. Apply some statistic logs and RPC definitions
```
cd oceanbase/
mv mittest/palf_cluster/palf-cluster.diff ./
git apply palf-cluster.diff
```
3. Build
1. uncomment `mittest/CMakeLists.txt:5`
2. remove `OB_BUILD_CLOSE_MODULES` in `CMakeLists.txt:162` and `CMakeLists.txt:167` if you want to build in opensource mode
3. build
```
cd oceanbse/
bash build.sh release --init --make
```
4. Run test
```
cd oceanbase/
cp mittest/palf_cluster/run_palf_bench.sh build_release/mittest/palf_cluster/
cd build_release/mittest/palf_cluster/
./run_palf_bench.sh
```
5. Check Result
check workdir of the server with minimal IP:PORT
```
$tail palf_raw_result_exp_test/palf_append_1500_512_3_1000.result -n 5
[2023-09-27 11:29:25.048444] EDIAG [CLOG] try_handle_cb_queue (ob_log_apply_service.cpp:505) [62033][T1_ApplySrv5][T1][Y0-0000000000000000-0-0] [lt=51][errcode=0] result:(l_append_cnt=602962, l_log_body_size=308716544, l_rt=1465294805, avg_body_size=512, avg_rt=2430) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d8dd1 0x55807b1d8a88 0x55807b1d8886 0x55807b1d867d 0xc9e7562 0xa5f6f30 0xa5f5d86 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d
[2023-09-27 11:29:26.049100] EDIAG [CLOG] try_handle_cb_queue (ob_log_apply_service.cpp:505) [62037][T1_ApplySrv9][T1][Y0-0000000000000000-0-0] [lt=39][errcode=0] result:(l_append_cnt=590605, l_log_body_size=302389760, l_rt=1463385996, avg_body_size=512, avg_rt=2477) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d8dd1 0x55807b1d8a88 0x55807b1d8886 0x55807b1d867d 0xc9e7562 0xa5f6f30 0xa5f5d86 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d
[2023-09-27 11:29:27.050335] EDIAG [CLOG] try_handle_cb_queue (ob_log_apply_service.cpp:505) [62033][T1_ApplySrv5][T1][Y0-0000000000000000-0-0] [lt=28][errcode=0] result:(l_append_cnt=604878, l_log_body_size=309697536, l_rt=1462297116, avg_body_size=512, avg_rt=2417) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d8dd1 0x55807b1d8a88 0x55807b1d8886 0x55807b1d867d 0xc9e7562 0xa5f6f30 0xa5f5d86 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d
[2023-09-27 11:29:28.051049] EDIAG [CLOG] try_handle_cb_queue (ob_log_apply_service.cpp:505) [62033][T1_ApplySrv5][T1][Y0-0000000000000000-0-0] [lt=28][errcode=0] result:(l_append_cnt=593036, l_log_body_size=303634432, l_rt=1462893463, avg_body_size=512, avg_rt=2466) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d8dd1 0x55807b1d8a88 0x55807b1d8886 0x55807b1d867d 0xc9e7562 0xa5f6f30 0xa5f5d86 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d
[2023-09-27 11:29:29.051212] EDIAG [CLOG] try_handle_cb_queue (ob_log_apply_service.cpp:505) [62034][T1_ApplySrv6][T1][Y0-0000000000000000-0-0] [lt=28][errcode=0] result:(l_append_cnt=596583, l_log_body_size=305449984, l_rt=1464339470, avg_body_size=511, avg_rt=2454) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d8dd1 0x55807b1d8a88 0x55807b1d8886 0x55807b1d867d 0xc9e7562 0xa5f6f30 0xa5f5d86 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d
$tail palf_raw_result_exp_test/palf_io_1500_512_3_1000.result -n 5
[2023-09-27 11:29:24.910336] EDIAG [CLOG] inner_write_impl_ (log_block_handler.cpp:458) [61989][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=28][errcode=0] io result:(l_io_cnt=1438, l_log_size=333262848, avg_size=231754) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d3228 0x55807b1d2edf 0x55807b1d2cd8 0x55807b1d2ad7 0xca638f1 0xa73358f 0xa732cfa 0xa732761 0xa731caf 0xcab6358 0xa7310b4 0xa73012b 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d
[2023-09-27 11:29:25.911515] EDIAG [CLOG] inner_write_impl_ (log_block_handler.cpp:458) [61989][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=19][errcode=0] io result:(l_io_cnt=1441, l_log_size=330817536, avg_size=229574) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d3228 0x55807b1d2edf 0x55807b1d2cd8 0x55807b1d2ad7 0xca638f1 0xa73358f 0xa732cfa 0xa732761 0xa731caf 0xcab6358 0xa7310b4 0xa73012b 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d
[2023-09-27 11:29:26.911259] EDIAG [CLOG] inner_write_impl_ (log_block_handler.cpp:458) [61989][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=22][errcode=0] io result:(l_io_cnt=1425, l_log_size=331571200, avg_size=232681) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d3228 0x55807b1d2edf 0x55807b1d2cd8 0x55807b1d2ad7 0xca638f1 0xa73358f 0xa732cfa 0xa732761 0xa731caf 0xcab6358 0xa7310b4 0xa73012b 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d
[2023-09-27 11:29:27.912235] EDIAG [CLOG] inner_write_impl_ (log_block_handler.cpp:458) [61989][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=21][errcode=0] io result:(l_io_cnt=1406, l_log_size=328134656, avg_size=233381) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d3228 0x55807b1d2edf 0x55807b1d2cd8 0x55807b1d2ad7 0xca638f1 0xa73358f 0xa732cfa 0xa732761 0xa731caf 0xcab6358 0xa7310b4 0xa73012b 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d
[2023-09-27 11:29:28.912356] EDIAG [CLOG] inner_write_impl_ (log_block_handler.cpp:458) [61989][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=19][errcode=0] io result:(l_io_cnt=1413, l_log_size=332730368, avg_size=235477) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d3228 0x55807b1d2edf 0x55807b1d2cd8 0x55807b1d2ad7 0xca638f1 0xa73358f 0xa732cfa 0xa732761 0xa731caf 0xcab6358 0xa7310b4 0xa73012b 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d
$tail palf_raw_result_exp_test/palf_group_1500_512_3_1000.result -n 5
[2023-09-27 11:29:25.042978] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1056) [61992][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=41] [PALF STAT GROUP LOG INFO](palf_id=1, self="SERVER_IP1:53212", role="LEADER", total_group_log_cnt=1667, avg_log_batch_cnt=361, total_group_log_size=327750672, avg_group_log_size=196611)
[2023-09-27 11:29:26.046598] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1056) [62250][][T0][Y0-0000000000000000-0-0] [lt=44] [PALF STAT GROUP LOG INFO](palf_id=1, self="SERVER_IP1:53212", role="LEADER", total_group_log_cnt=1613, avg_log_batch_cnt=367, total_group_log_size=322325464, avg_group_log_size=199829)
[2023-09-27 11:29:27.046847] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1056) [61992][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=21] [PALF STAT GROUP LOG INFO](palf_id=1, self="SERVER_IP1:53212", role="LEADER", total_group_log_cnt=1680, avg_log_batch_cnt=360, total_group_log_size=329294128, avg_group_log_size=196008)
[2023-09-27 11:29:28.048014] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1056) [61992][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=22] [PALF STAT GROUP LOG INFO](palf_id=1, self="SERVER_IP1:53212", role="LEADER", total_group_log_cnt=1607, avg_log_batch_cnt=369, total_group_log_size=322903656, avg_group_log_size=200935)
[2023-09-27 11:29:29.048263] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1056) [61992][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=56] [PALF STAT GROUP LOG INFO](palf_id=1, self="SERVER_IP1:53212", role="LEADER", total_group_log_cnt=1611, avg_log_batch_cnt=370, total_group_log_size=324553496, avg_group_log_size=201460)
```

133
mittest/palf_cluster/env/ob_simple_log_cluster.cpp vendored Normal file
View File

@ -0,0 +1,133 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "ob_simple_log_cluster.h"
#include "common/ob_member_list.h"
#include "share/allocator/ob_tenant_mutil_allocator_mgr.h"
#include "ob_mittest_utils.h"
#include "lib/alloc/memory_dump.h"
#include "lib/alloc/alloc_func.cpp"
#include "lib/allocator/ob_mem_leak_checker.h"
#include "lib/allocator/ob_libeasy_mem_pool.h"
#include <algorithm>
namespace oceanbase
{
namespace unittest
{
ObSimpleLogServer* ObSimpleLogCluster::log_server_;
int64_t ObSimpleLogCluster::node_cnt_ = 1;
common::ObMemberList ObSimpleLogCluster::node_list_ = ObMemberList();
bool ObSimpleLogCluster::is_started_ = false;
char ObSimpleLogCluster::sig_buf_[sizeof(ObSignalWorker) + sizeof(observer::ObSignalHandle)];
ObSignalWorker *ObSimpleLogCluster::sig_worker_ = new (sig_buf_) ObSignalWorker();
observer::ObSignalHandle *ObSimpleLogCluster::signal_handle_ = new (sig_worker_ + 1) observer::ObSignalHandle();
void ObSimpleLogCluster::SetUpTestCase()
{
SERVER_LOG(INFO, "SetUpTestCase");
int ret = OB_SUCCESS;
if (!is_started_) {
ret = start();
}
ASSERT_EQ(ret, OB_SUCCESS);
}
void ObSimpleLogCluster::TearDownTestCase()
{
SERVER_LOG(INFO, "TearDownTestCase", K(node_cnt_));
int ret = OB_SUCCESS;
ret = close();
ASSERT_EQ(ret, OB_SUCCESS);
ob_delete(log_server_);
}
int ObSimpleLogCluster::start()
{
int ret = OB_SUCCESS;
ObTenantMutilAllocatorMgr::get_instance().init();
ObMemoryDump::get_instance().init();
// set easy allocator for watching easy memory holding
easy_pool_set_allocator(ob_easy_realloc);
ev_set_allocator(ob_easy_realloc);
lib::set_memory_limit(100L * 1000L * 1000L * 1000L);
if (sig_worker_ != nullptr && OB_FAIL(sig_worker_->start())) {
SERVER_LOG(ERROR, "Start signal worker error", K(ret));
} else if (signal_handle_ != nullptr && OB_FAIL(signal_handle_->start())) {
SERVER_LOG(ERROR, "Start signal handle error", K(ret));
} else if (OB_FAIL(generate_sorted_server_list_(node_cnt_))) {
} else {
for (int i = 0; OB_SUCC(ret) && i < node_cnt_; i++) {
auto svr = OB_NEW(ObSimpleLogServer, "TestBase");
common::ObAddr server;
if (OB_FAIL(node_list_.get_server_by_index(i, server))) {
} else if (OB_FAIL(svr->simple_init(test_name_, server, 1, true))) {
SERVER_LOG(WARN, "simple_init failed", K(ret), K(i), K_(node_list));
} else if (OB_FAIL(svr->simple_start(true))) {
SERVER_LOG(WARN, "simple_start failed", K(ret), K(i), K_(node_list));
} else {
log_server_ = svr;
}
}
is_started_ = true;
SERVER_LOG(INFO, "ObSimpleLogCluster started", K(ret), K_(node_cnt), K_(node_list));
}
return ret;
}
int ObSimpleLogCluster::close()
{
int ret = OB_SUCCESS;
ret = log_server_->simple_close(true);
if (OB_FAIL(ret)) {
SERVER_LOG(WARN, "simple_close failed", K(ret));
}
return ret;
}
int ObSimpleLogCluster::generate_sorted_server_list_(const int64_t node_cnt)
{
int ret = OB_SUCCESS;
// each get_rpc_port calling will get two available ports,
// so just get node_cnt / 2 + 1 times
const int64_t get_port_cnt = node_cnt / 2 + 1;
for (int i = 0; i < get_port_cnt; i++) {
int server_fd = 0;
const std::string local_ip = get_local_ip();
// const int64_t port = get_rpc_port(server_fd);
const int64_t port = RPC_PORT;
common::ObAddr addr;
if (0 == port) {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(ERROR, "get_rpc_port failed", K(ret), K(port));
break;
} else if (local_ip == "") {
ret = OB_ERR_UNEXPECTED;
} else if (false == addr.set_ip_addr(local_ip.c_str(), port)) {
SERVER_LOG(ERROR, "set_ip_addr failed", K(local_ip.c_str()), K(port), K(addr));
} else if (OB_FAIL(node_list_.add_server(addr))) {
PALF_LOG(WARN, "add_server failed", K(ret));
} else if (false == addr.set_ip_addr(local_ip.c_str(), port + 1)) {
SERVER_LOG(ERROR, "set_ip_addr failed", K(local_ip.c_str()), K(port), K(addr));
} else if (node_list_.get_member_number() < node_cnt && OB_FAIL(node_list_.add_server(addr))) {
PALF_LOG(WARN, "add_server failed", K(ret));
}
}
if (OB_SUCC(ret)) {
SERVER_LOG(INFO, "simple log cluster node_list", K_(node_list));
}
return ret;
}
} // end unittest
} // end oceanbase

111
mittest/palf_cluster/env/ob_simple_log_cluster.h vendored Normal file
View File

@ -0,0 +1,111 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#define USING_LOG_PREFIX RPC_TEST
#include <gtest/gtest.h>
#include <string>
#include "lib/hash/ob_array_hash_map.h" // ObArrayHashMap
#include "ob_simple_log_server.h"
namespace oceanbase
{
namespace unittest
{
#define SET_CASE_LOG_FILE(TEST_NAME, CASE_NAME) \
const std::string log_file_name = TEST_NAME + "/" + CASE_NAME + ".log";\
const std::string ele_log_file_name = TEST_NAME + "/" + CASE_NAME + ".election.log";\
OB_LOGGER.set_file_name(log_file_name.c_str(),\
true, false, NULL, \
ele_log_file_name.c_str(), NULL);
#define RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME) \
void *ptr = malloc(SIG_STACK_SIZE); \
abort_unless(ptr != nullptr); \
stack_t nss; \
stack_t oss; \
bzero(&nss, sizeof(nss)); \
bzero(&oss, sizeof(oss)); \
nss.ss_sp = ptr; \
nss.ss_size = SIG_STACK_SIZE; \
abort_unless(0 == sigaltstack(&nss, &oss)); \
DEFER(sigaltstack(&oss, nullptr)); \
if (OB_SUCCESS != oceanbase::observer::ObSignalHandle::change_signal_mask()) { \
} \
::oceanbase::common::g_redirect_handler = true; \
oceanbase::palf::election::GLOBAL_INIT_ELECTION_MODULE(); \
oceanbase::common::ObClockGenerator::init();\
sleep(15); \
const std::string rm_base_dir_cmd = "rm -rf " + TEST_NAME; \
const std::string rm_log_cmd = "rm -f ./" + TEST_NAME + "*log*"; \
const std::string mk_base_dir_cm = "mkdir " + TEST_NAME; \
system(rm_base_dir_cmd.c_str()); \
system(rm_log_cmd.c_str()); \
system(mk_base_dir_cm.c_str()); \
ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION); \
ObClusterVersion::get_instance().update_cluster_version(DATA_CURRENT_VERSION); \
const std::string log_file_name = TEST_NAME+"/"+TEST_NAME + ".log"; \
const std::string ele_log_file_name = TEST_NAME+"/"+TEST_NAME + ".election.log"; \
const std::string rs_log_file_name = TEST_NAME+"/"+TEST_NAME + ".rs.log"; \
OB_LOGGER.set_file_name(log_file_name.c_str(), true, false, rs_log_file_name.c_str(), \
ele_log_file_name.c_str(), NULL); \
OB_LOGGER.set_log_level("INFO"); \
OB_LOGGER.set_enable_log_limit(false); \
OB_LOGGER.set_enable_async_log(false); \
SERVER_LOG(INFO, "begin unittest"); \
::testing::InitGoogleTest(&argc, argv); \
return RUN_ALL_TESTS();
class ObSimpleLogCluster : public testing::Test
{
public:
ObSimpleLogCluster()
{
SERVER_LOG(INFO, "ObSimpleLogCluster construct");
}
virtual ~ObSimpleLogCluster() {}
static int init();
static int start();
static int close();
std::string &get_test_name() { return test_name_; }
ObSimpleLogServer *get_log_server() { return log_server_; }
private:
static int generate_sorted_server_list_(const int64_t node_cnt);
protected:
static void SetUpTestCase();
static void TearDownTestCase();
public:
static constexpr int64_t RPC_PORT = 53212;
static ObSimpleLogServer *log_server_;
static bool is_started_;
static std::string test_name_;
static ObMemberList node_list_;
static int64_t node_cnt_;
// static std::vector<ObSimpleLogServer*> cluster_;
// static ObMemberList member_list_;
// static common::ObArrayHashMap<common::ObAddr, common::ObRegion> member_region_map_;
// static int64_t member_cnt_;
// static int64_t node_idx_base_;
//thread to deal signals
static char sig_buf_[sizeof(ObSignalWorker) + sizeof(observer::ObSignalHandle)];
static ObSignalWorker *sig_worker_;
static observer::ObSignalHandle *signal_handle_;
};
} // end unittest
} // end oceanbase

814
mittest/palf_cluster/env/ob_simple_log_server.cpp vendored Normal file
View File

@ -0,0 +1,814 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define private public
#include "ob_simple_log_server.h"
#undef private
#include "lib/file/file_directory_utils.h"
#include "logservice/palf/log_define.h"
#include <iostream>
#define USING_LOG_PREFIX RPC_TEST
#include<dirent.h>
#include <memory>
#include <net/if.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include "util/easy_mod_stat.h"
#include "lib/oblog/ob_log.h"
#include "lib/oblog/ob_log_module.h"
#include "lib/random/ob_random.h"
#include "common/log/ob_log_constants.h"
#include "share/ob_io_device_helper.h"
#include "share/ob_thread_mgr.h"
#include "logservice/palf/palf_options.h"
namespace oceanbase
{
namespace unittest
{
using namespace oceanbase;
using namespace oceanbase::rpc;
using namespace oceanbase::rpc::frame;
using namespace oceanbase::obrpc;
using namespace oceanbase::common;
using namespace oceanbase::share;
using namespace palf;
using namespace palf::election;
using namespace logservice;
uint32_t get_local_addr(const char *dev_name)
{
int fd, intrface;
struct ifreq buf[16];
struct ifconf ifc;
if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
return 0;
}
ifc.ifc_len = sizeof(buf);
ifc.ifc_buf = (caddr_t)buf;
if (ioctl(fd, SIOCGIFCONF, (char *)&ifc) != 0) {
close(fd);
return 0;
}
intrface = static_cast<int>(ifc.ifc_len / sizeof(struct ifreq));
while (intrface-- > 0) {
if (ioctl(fd, SIOCGIFFLAGS, (char *)&buf[intrface]) != 0) {
continue;
}
if ((buf[intrface].ifr_flags & IFF_LOOPBACK) != 0)
continue;
if (!(buf[intrface].ifr_flags & IFF_UP))
continue;
if (dev_name != NULL && strcmp(dev_name, buf[intrface].ifr_name))
continue;
if (!(ioctl(fd, SIOCGIFADDR, (char *)&buf[intrface]))) {
close(fd);
return ((struct sockaddr_in *)(&buf[intrface].ifr_addr))->sin_addr.s_addr;
}
}
close(fd);
return 0;
}
std::string get_local_ip()
{
uint32_t ip = get_local_addr("bond0");
if (ip == 0) {
ip = get_local_addr("eth0");
}
if (ip == 0) {
return "";
}
return inet_ntoa(*(struct in_addr *)(&ip));
}
int ObSimpleLogServer::simple_init(
const std::string &cluster_name,
const common::ObAddr &addr,
const int64_t node_id,
const bool is_bootstrap = false)
{
int ret = OB_SUCCESS;
ObTimeGuard guard("simple_init", 0);
SERVER_LOG(INFO, "simple_log_server simple_init start", K(node_id), K(addr_));
tenant_base_ = OB_NEW(ObTenantBase, "TestBase", node_id);
tenant_base_->init();
// tenant_base_->set(&log_service_);
ObTenantEnv::set_tenant(tenant_base_);
// assert(&log_service_ == MTL(palfcluster::LogService*));
guard.click("init tenant_base");
node_id_ = node_id;
if (is_bootstrap && OB_FAIL(init_memory_dump_timer_())) {
SERVER_LOG(ERROR, "init_memory_dump_timer_ failed", K(ret), K_(node_id));
} else if (FALSE_IT(guard.click("init_memory_dump_timer_")) || OB_FAIL(init_network_(addr, is_bootstrap))) {
SERVER_LOG(WARN, "init_network failed", K(ret), K(addr));
} else if (FALSE_IT(guard.click("init_network_")) || OB_FAIL(init_io_(cluster_name))) {
SERVER_LOG(WARN, "init_io failed", K(ret), K(addr));
} else if (FALSE_IT(guard.click("init_io_")) || OB_FAIL(init_log_service_(cluster_name))) {
SERVER_LOG(WARN, "init_log_service failed", K(ret), K(addr));
} else {
guard.click("init_log_service_");
SERVER_LOG(INFO, "simple_log_server simple_init success", K(node_id), K(addr), K(guard));
}
return ret;
}
int ObSimpleLogServer::init_memory_dump_timer_()
{
int ret = OB_SUCCESS;
common::ObFunction<bool()> print_memory_info = [=](){
ObMallocAllocator::get_instance()->print_tenant_memory_usage(node_id_);
ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(node_id_);
ObMallocAllocator::get_instance()->print_tenant_memory_usage(OB_SERVER_TENANT_ID);
ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(OB_SERVER_TENANT_ID);
return false;};
if (OB_FAIL(timer_.init_and_start(1, 1_s, "test timer"))) {
SERVER_LOG(ERROR, "ObOccamTimer init failed", K(ret), K_(node_id));
} else if (OB_FAIL(timer_.schedule_task_repeat(timer_handle_, 1_s, print_memory_info))) {
SERVER_LOG(ERROR, "schedule_task failed", K(ret), K_(node_id));
}
return ret;
}
int ObSimpleLogServer::init_network_(const common::ObAddr &addr, const bool is_bootstrap = false)
{
int ret = OB_SUCCESS;
ObNetOptions opts;
opts.rpc_io_cnt_ = 10;
opts.tcp_user_timeout_ = 10 * 1000 * 1000; // 10s
addr_ = addr;
if (is_bootstrap && OB_FAIL(net_.init(opts))) {
SERVER_LOG(ERROR, "net init fail", K(ret));
} else if (OB_FAIL(req_handler_.init(&log_service_))) {
SERVER_LOG(ERROR, "req_handler_ init failed", K(ret));
} else if (OB_FAIL(deliver_.init())) {
SERVER_LOG(ERROR, "deliver_ init failed", K(ret));
} else if (OB_FAIL(deliver_.set_tenant_base(tenant_base_, &log_service_))) {
SERVER_LOG(ERROR, "deliver_ init failed", K(ret));
} else if (is_bootstrap && OB_FAIL(net_.add_rpc_listen(addr_.get_port(), handler_, transport_))) {
SERVER_LOG(ERROR, "net_ listen failed", K(ret));
}
return ret;
}
int ObSimpleLogServer::init_io_(const std::string &cluster_name)
{
int ret = OB_SUCCESS;
ObTimeGuard guard("init_io_", 0);
const std::string logserver_dir = cluster_name + "/port_" + std::to_string(addr_.get_port());
std::string data_dir = logserver_dir;
std::string clog_dir = logserver_dir + "/clog";
std::string slog_dir = logserver_dir + "/slog";
std::string sstable_dir = logserver_dir + "/sstable";
std::vector<std::string> dirs{data_dir, slog_dir, clog_dir, sstable_dir};
for (auto &dir : dirs) {
if (-1 == mkdir(dir.c_str(), 0777)) {
if (errno == EEXIST) {
ret = OB_SUCCESS;
SERVER_LOG(INFO, "for restart");
} else {
SERVER_LOG(WARN, "mkdir failed", K(errno));
ret = OB_IO_ERROR;
break;
}
}
}
guard.click("mkdir");
if (OB_SUCC(ret)) {
io_device_ = OB_NEW(ObLocalDevice, "TestBase");
blocksstable::ObStorageEnv storage_env;
storage_env.data_dir_ = data_dir.c_str();
storage_env.sstable_dir_ = sstable_dir.c_str();
storage_env.default_block_size_ = OB_DEFAULT_MACRO_BLOCK_SIZE;
storage_env.data_disk_size_ = 1024 * 1024 * 1024;
storage_env.data_disk_percentage_ = 0;
std::size_t found = cluster_name.find("client");
const int64_t disk_size = (found!=std::string::npos)? 10LL * 1024 * 1024 * 1024: 100LL * 1024 * 1024 * 1024;
storage_env.log_disk_size_ = disk_size;
storage_env.log_disk_percentage_ = 0;
storage_env.log_spec_.log_dir_ = slog_dir.c_str();
storage_env.log_spec_.max_log_file_size_ = ObLogConstants::MAX_LOG_FILE_SIZE;
storage_env.clog_dir_ = clog_dir.c_str();
iod_opts_.opts_ = iod_opt_array_;
iod_opt_array_[0].set("data_dir", storage_env.data_dir_);
iod_opt_array_[1].set("sstable_dir", storage_env.sstable_dir_);
iod_opt_array_[2].set("block_size", reinterpret_cast<const char*>(&storage_env.default_block_size_));
iod_opt_array_[3].set("datafile_disk_percentage", storage_env.data_disk_percentage_);
iod_opt_array_[4].set("datafile_size", storage_env.data_disk_size_);
iod_opts_.opt_cnt_ = MAX_IOD_OPT_CNT;
const int64_t DEFATULT_RESERVED_SIZE = 100 * 1024 * 1024 * 1024ul;
if (OB_FAIL(io_device_->init(iod_opts_))) {
SERVER_LOG(ERROR, "init io device fail", K(ret));
} else if (OB_FAIL(log_block_pool_.init(storage_env.clog_dir_))) {
SERVER_LOG(ERROR, "init log pool fail", K(ret));
} else {
guard.click("init io_device");
clog_dir_ = clog_dir;
SERVER_LOG(INFO, "init_io_ successs", K(ret), K(guard));
}
if (OB_SUCC(ret)) {
log_block_pool_.get_tenants_log_disk_size_func_ = [this, &storage_env](int64_t &log_disk_size) -> int
{
log_disk_size = log_block_pool_.lower_align_(storage_env.log_disk_size_);
return OB_SUCCESS;
};
if (OB_FAIL(log_block_pool_.start(storage_env.log_disk_size_))) {
LOG_ERROR("log pool start failed", KR(ret));
}
}
}
return ret;
}
int ObSimpleLogServer::init_log_service_(const std::string &cluster_name)
{
int ret = OB_SUCCESS;
// init deps of log_service
palf::PalfOptions opts;
std::size_t found = cluster_name.find("client");
const int64_t disk_size = (found!=std::string::npos)? 10 * 1024 * 1024 * 1024ul: 100 * 1024 * 1024 * 1024ul;
opts.disk_options_.log_disk_usage_limit_size_ = disk_size;
opts.disk_options_.log_disk_usage_limit_size_ = disk_size;
opts.disk_options_.log_disk_utilization_threshold_ = 80;
opts.disk_options_.log_disk_utilization_limit_threshold_ = 95;
opts.disk_options_.log_disk_throttling_percentage_ = 100;
opts.disk_options_.log_disk_throttling_maximum_duration_ = 2 * 3600 * 1000 * 1000L;
opts.disk_options_.log_writer_parallelism_ = 2;
std::string clog_dir = clog_dir_ + "/tenant_1";
allocator_ = OB_NEW(ObTenantMutilAllocator, "TestBase", node_id_);
ObMemAttr attr(1, "SimpleLog");
malloc_mgr_.set_attr(attr);
if (OB_FAIL(log_service_.init(opts, clog_dir.c_str(), addr_,
allocator_, transport_, &log_block_pool_))) {
SERVER_LOG(ERROR, "init_log_service_ fail", K(ret));
} else {
palf_env_ = log_service_.get_palf_env();
SERVER_LOG(INFO, "init_log_service_ success", K(ret));
}
return ret;
}
int ObSimpleLogServer::simple_start(const bool is_bootstrap = false)
{
int ret = OB_SUCCESS;
if (is_bootstrap && OB_FAIL(net_.start())) {
SERVER_LOG(ERROR, "net start fail", K(ret));
} else if (OB_FAIL(deliver_.start())) {
SERVER_LOG(ERROR, "deliver_ start failed", K(ret));
} else if (OB_FAIL(log_service_.start())) {
SERVER_LOG(ERROR, "arb_service start failed", K(ret));
} else {
const double tenant_unit_cpu = 10;
tenant_base_->update_thread_cnt(tenant_unit_cpu);
}
return ret;
}
int ObSimpleLogServer::simple_close(const bool is_shutdown = false)
{
int ret = OB_SUCCESS;
ObTimeGuard guard("simple_close", 0);
deliver_.destroy();
guard.click("destroy");
log_service_.destroy();
log_block_pool_.destroy();
guard.click("destroy_palf_env");
if (is_shutdown) {
net_.rpc_shutdown();
net_.stop();
net_.wait();
net_.destroy();
timer_handle_.stop_and_wait();
timer_.stop();
timer_.wait();
}
SERVER_LOG(INFO, "stop LogService success", K(ret), K(is_shutdown), K(guard));
return ret;
}
int ObSimpleLogServer::simple_restart(const std::string &cluster_name, const int64_t node_idx)
{
int ret = OB_SUCCESS;
ObTimeGuard guard("simple_restart", 0);
if (OB_FAIL(simple_close())) {
SERVER_LOG(ERROR, "simple_close failed", K(ret));
} else if (FALSE_IT(guard.click("simple_close")) || OB_FAIL(simple_init(cluster_name, addr_, node_idx))) {
SERVER_LOG(ERROR, "simple_init failed", K(ret));
} else if (FALSE_IT(guard.click("simple_init")) || OB_FAIL(simple_start())) {
SERVER_LOG(ERROR, "simple_start failed", K(ret));
} else {
guard.click("simple_start");
SERVER_LOG(INFO, "simple_restart success", K(ret), K(guard));
}
return ret;
}
void ObLogDeliver::block_net(const ObAddr &src)
{
blacklist_.set_refactored(src);
SERVER_LOG(INFO, "block_net", K(src), K(blacklist_));
}
void ObLogDeliver::unblock_net(const ObAddr &src)
{
blacklist_.erase_refactored(src);
SERVER_LOG(INFO, "unblock_net", K(src), K(blacklist_));
}
void ObLogDeliver::set_rpc_loss(const ObAddr &src, const int loss_rate)
{
// not thread safe
int old_loss_rate = -1;
bool is_exist = false;
for (int64_t i = 0; i < rpc_loss_config_.count(); ++i) {
LossConfig &tmp_config = rpc_loss_config_[i];
if (tmp_config.src_ == src) {
is_exist = true;
// src exists, update its loss_rate
old_loss_rate = tmp_config.loss_rate_;
tmp_config.loss_rate_ = loss_rate;
}
}
if (!is_exist) {
LossConfig loss_config(src, loss_rate);
rpc_loss_config_.push_back(loss_config);
}
SERVER_LOG(INFO, "set_rpc_loss", K(src), K(loss_rate), K(old_loss_rate));
}
void ObLogDeliver::reset_rpc_loss(const ObAddr &src)
{
// not thread safe
int64_t idx = -1;
for (int64_t i = 0; i < rpc_loss_config_.count(); ++i) {
LossConfig tmp_config;
int ret = rpc_loss_config_.at(i, tmp_config);
if (OB_SUCCESS != ret) {
SERVER_LOG(WARN, "at failed", K(ret), K(i), "count", rpc_loss_config_.count());
} else if (tmp_config.src_ == src) {
// src exists, update its loss_rate
idx = i;
}
}
if (idx >= 0) {
rpc_loss_config_.remove(idx);
}
SERVER_LOG(INFO, "reset_rpc_loss", K(src), K(idx));
}
void ObLogDeliver::get_loss_config(const ObAddr &src, bool &exist, LossConfig &loss_config)
{
// not thread safe
exist = false;
for (int64_t i = 0; i < rpc_loss_config_.count(); ++i) {
LossConfig tmp_config;
int ret = rpc_loss_config_.at(i, tmp_config);
if (OB_SUCCESS != ret) {
SERVER_LOG(WARN, "at failed", K(ret));
} else if (tmp_config.src_ == src) {
exist = true;
loss_config = tmp_config;
}
}
SERVER_LOG(INFO, "get_loss_config", K(src), K(loss_config));
}
int ObLogDeliver::create_queue_thread(int tg_id, const char *thread_name, observer::QueueThread *&qthread)
{
int ret = OB_SUCCESS;
qthread = OB_NEW(observer::QueueThread, ObModIds::OB_RPC, thread_name);
if (OB_ISNULL(qthread)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
} else {
qthread->queue_.set_qhandler(&qhandler_);
}
if (OB_SUCC(ret) && OB_NOT_NULL(qthread)) {
ret = TG_SET_RUNNABLE_AND_START(tg_id, qthread->thread_);
}
return ret;
}
int ObLogDeliver::init()
{
int ret = OB_SUCCESS;
// init_all_propocessor_();
if (OB_FAIL(blacklist_.create(1024))) {
SERVER_LOG(WARN, "create blacklist_ failed", K(ret));
} else if (false == blacklist_.created()) {
SERVER_LOG(WARN, "blacklist_ created failed");
} else if (OB_FAIL(create_queue_thread(lib::TGDefIDs::LogServerTest, "LogServerQue", req_queue_))) {
} else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TEST7, tg_id_))) {
SERVER_LOG(WARN, "ObSimpleThreadPool::init failed", K(ret));
} else {
is_inited_ = true;
SERVER_LOG(INFO, "ObLogDeliver init success", KP(&blacklist_));
}
return ret;
}
void ObLogDeliver::destroy()
{
if (IS_NOT_INIT) {
SERVER_LOG(INFO, "ObLogDeliver not init");
} else {
RWLock::WLockGuard guard(lock_);
stop();
wait();
is_inited_ = false;
TG_DESTROY(tg_id_);
blacklist_.destroy();
tg_id_ = 0;
SERVER_LOG(INFO, "destroy ObLogDeliver");
}
}
int ObLogDeliver::start()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
SERVER_LOG(ERROR, "ObLogDeliver not init");
} else if (OB_FAIL(TG_SET_HANDLER_AND_START(tg_id_, *this))) {
SERVER_LOG(ERROR, "start ObLogDeliver failed");
} else {
is_stopped_ = false;
SERVER_LOG(INFO, "start ObLogDeliver success");
}
return ret;
}
void ObLogDeliver::stop()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
SERVER_LOG(WARN, "ObLogDeliver stop failed");
} else {
is_stopped_ = true;
TG_STOP(tg_id_);
if (NULL != req_queue_) {
TG_STOP(lib::TGDefIDs::LogServerTest);
TG_WAIT(lib::TGDefIDs::LogServerTest);
}
SERVER_LOG(INFO, "stop ObLogDeliver success");
}
}
int ObLogDeliver::wait()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
SERVER_LOG(ERROR, "ObLogDeliver not inited!!!", K(ret));
} else {
TG_WAIT(tg_id_);
SERVER_LOG(INFO, "wait ObLogDeliver success", K(tg_id_));
}
return ret;
}
bool ObLogDeliver::need_filter_packet_by_blacklist(const ObAddr &addr)
{
return OB_HASH_EXIST == blacklist_.exist_refactored(addr);
}
bool ObLogDeliver::need_drop_by_loss_config(const ObAddr &addr)
{
bool bool_ret = false;
bool exist = false;
LossConfig loss_config;
get_loss_config(addr, exist, loss_config);
if (exist && loss_config.loss_rate_ > 0) {
const int64_t random_num = ObRandom::rand(1, 100);
bool_ret = random_num <= loss_config.loss_rate_ ? true : false;
if (bool_ret) {
SERVER_LOG(INFO, "need drop req by loss config", K(addr));
}
}
return bool_ret;
}
// int ObLogDeliver::deliver(rpc::ObRequest &req)
// {
// int ret = OB_SUCCESS;
// const ObRpcPacket &pkt = reinterpret_cast<const obrpc::ObRpcPacket&>(req.get_packet());
// SERVER_LOG(INFO, "deliver request", K(pkt), K(req.ez_req_), KPC(palf_env_impl_));
// if (IS_NOT_INIT) {
// ret = OB_NOT_INIT;
// } else {
// do {
// RWLock::RLockGuard guard(lock_);
// if (false == is_stopped_) {
// ret = TG_PUSH_TASK(tg_id_, &req);
// if (OB_SUCCESS != ret) {
// SERVER_LOG(ERROR, "deliver request failed", K(ret), K(pkt), K(req.ez_req_), KPC(palf_env_impl_));
// }
// } else {
// break;
// }
// } while (OB_EAGAIN == ret);
// }
// return ret;
// }
int ObLogDeliver::deliver(rpc::ObRequest &req)
{
int ret = OB_SUCCESS;
SERVER_LOG(INFO, "deliver request", K(req.ez_req_), KPC(palf_env_impl_));
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else {
ObReqQueue *queue = NULL;
queue = &req_queue_->queue_;
if (NULL != queue) {
if (!queue->push(&req, 1 << 18)) {
ret = OB_QUEUE_OVERFLOW;
}
}
}
return ret;
}
void ObLogDeliver::handle(void *task)
{
int ret = OB_SUCCESS;
// if (IS_NOT_INIT) {
// ret = OB_NOT_INIT;
// SERVER_LOG(WARN, "ObLogDeliver not init", K(ret));
// } else if (OB_ISNULL(task)) {
// SERVER_LOG(WARN, "invalid argument", KP(task));
// } else {
// rpc::ObRequest *req = static_cast<rpc::ObRequest *>(task);
// SERVER_LOG(INFO, "handle request", KP(req->ez_req_));
// (void) handle_req_(*req);
// }
}
// int ObLogDeliver::handle_req_(rpc::ObRequest &req)
// {
// int ret = OB_SUCCESS;
// // tenant_id of log servers are different for clear log record,
// // so set tenant_id of RpcPacket to MTL_ID
// ObTenantEnv::set_tenant(tenant_base_);
// const ObRpcPacket &pkt = dynamic_cast<const ObRpcPacket&>(req.get_packet());
// // const ObRpcPacketCode pcode = pkt.get_pcode();
// // ObFunction<bool(const ObAddr &src)> filter = [&](const ObAddr &src) -> bool {
// // if (this->need_filter_packet_by_blacklist(src)) {
// // SERVER_LOG(WARN, "need_filter_packet_by_blacklist", K(ret), K(pcode), K(src), K(blacklist_), KPC(palf_env_impl_));
// // return true;
// // } else if (this->need_drop_by_loss_config(src)) {
// // SERVER_LOG(WARN, "need_drop_by_loss_config", K(ret), K(pcode), K(src), K(rpc_loss_config_), KPC(palf_env_impl_));
// // return true;
// // }
// // return false;
// // };
// switch (pkt.get_pcode()) {
// #define PROCESS(processer) \
// processer p;\
// p.init();\
// p.set_ob_request(req);\
// p.set_log_service(log_service_);\
// p.run(); \
// break;
// case obrpc::OB_LOG_PUSH_REQ: {
// PROCESS(LogPushReqP);
// }
// case obrpc::OB_LOG_PUSH_RESP: {
// PROCESS(LogPushRespP);
// }
// case obrpc::OB_LOG_FETCH_REQ: {
// PROCESS(LogFetchReqP);
// }
// case obrpc::OB_LOG_PREPARE_REQ: {
// PROCESS(LogPrepareReqP);
// }
// case obrpc::OB_LOG_PREPARE_RESP: {
// PROCESS(LogPrepareRespP);
// }
// case obrpc::OB_LOG_CHANGE_CONFIG_META_REQ: {
// PROCESS(LogChangeConfigMetaReqP);
// }
// case obrpc::OB_LOG_CHANGE_CONFIG_META_RESP: {
// PROCESS(LogChangeConfigMetaRespP);
// }
// case obrpc::OB_LOG_CHANGE_MODE_META_REQ: {
// PROCESS(LogChangeModeMetaReqP);
// }
// case obrpc::OB_LOG_CHANGE_MODE_META_RESP: {
// PROCESS(LogChangeModeMetaRespP);
// }
// case obrpc::OB_LOG_NOTIFY_REBUILD_REQ: {
// PROCESS(LogNotifyRebuildReqP)
// }
// case obrpc::OB_LOG_COMMITTED_INFO: {
// PROCESS(CommittedInfoP)
// }
// case obrpc::OB_LOG_LEARNER_REQ: {
// PROCESS(LogLearnerReqP)
// }
// case obrpc::OB_LOG_REGISTER_PARENT_REQ: {
// PROCESS(LogRegisterParentReqP)
// }
// case obrpc::OB_LOG_REGISTER_PARENT_RESP: {
// PROCESS(LogRegisterParentRespP)
// }
// case obrpc::OB_LOG_ELECTION_PREPARE_REQUEST : {
// PROCESS(ElectionPrepareRequestMsgP)
// }
// case obrpc::OB_LOG_ELECTION_PREPARE_RESPONSE : {
// PROCESS(ElectionPrepareResponseMsgP)
// }
// case obrpc::OB_LOG_ELECTION_ACCEPT_REQUEST : {
// PROCESS(ElectionAcceptRequestMsgP)
// }
// case obrpc::OB_LOG_ELECTION_ACCEPT_RESPONSE : {
// PROCESS(ElectionAcceptResponseMsgP)
// }
// case obrpc::OB_LOG_ELECTION_CHANGE_LEADER_REQUEST : {
// PROCESS(ElectionChangeLeaderMsgP)
// }
// case obrpc::OB_LOG_GET_MC_ST: {
// PROCESS(LogGetMCStP)
// }
// case obrpc::OB_LOG_ARB_PROBE_MSG: {
// PROCESS(logservice::LogServerProbeP)
// }
// case obrpc::OB_LOG_CONFIG_CHANGE_CMD: {
// PROCESS(LogMembershipChangeP)
// }
// case obrpc::OB_LOG_GET_PALF_STAT: {
// PROCESS(LogGetPalfStatReqP)
// }
// case obrpc::OB_LOG_CHANGE_ACCESS_MODE_CMD: {
// PROCESS(LogChangeAccessModeP)
// }
// case obrpc::OB_LOG_FLASHBACK_CMD: {
// PROCESS(LogFlashbackMsgP)
// }
// case obrpc::OB_LOG_CREATE_REPLICA_CMD: {
// PROCESS(LogCreateReplicaCmdP)
// }
// case obrpc::OB_LOG_SUBMIT_LOG_CMD: {
// PROCESS(LogSubmitLogP)
// }
// case obrpc::OB_LOG_SUBMIT_LOG_CMD_RESP: {
// PROCESS(LogSubmitLogRespP)
// }
// default:
// SERVER_LOG(ERROR, "invalid req type", K(pkt.get_pcode()));
// break;
// }
// return ret;
// }
ObLogServerReqQueueHandler::~ObLogServerReqQueueHandler()
{
}
int ObLogServerReqQueueHandler::init(palfcluster::LogService *log_service)
{
int ret = OB_SUCCESS;
if (NULL == log_service) {
ret = OB_INVALID_ARGUMENT;
} else {
log_service_ = log_service;
}
return ret;
}
int ObLogServerReqQueueHandler::onThreadCreated(obsys::CThread *th)
{
UNUSED(th);
return OB_SUCCESS;
}
int ObLogServerReqQueueHandler::onThreadDestroy(obsys::CThread *th)
{
UNUSED(th);
return OB_SUCCESS;
}
bool ObLogServerReqQueueHandler::handlePacketQueue(ObRequest *req, void */* arg */)
{
// ObTenantEnv::set_tenant(tenant_base_);
const ObRpcPacket &pkt = dynamic_cast<const ObRpcPacket&>(req->get_packet());
if (NULL != log_service_) {
switch (pkt.get_pcode()) {
#define PALF_PROCESS(processer) \
processer p;\
p.init();\
p.set_palf_env_impl(log_service_->get_palf_env()->get_palf_env_impl()); \
p.set_ob_request(*req);\
p.run(); \
break;
#define LOG_SERVICE_PROCESS(processer) \
processer p;\
p.init();\
p.set_ob_request(*req);\
p.set_log_service(log_service_);\
p.run(); \
break;
case obrpc::OB_LOG_PUSH_REQ: {
PALF_PROCESS(LogPushReqP);
}
case obrpc::OB_LOG_PUSH_RESP: {
PALF_PROCESS(LogPushRespP);
}
case obrpc::OB_LOG_FETCH_REQ: {
PALF_PROCESS(LogFetchReqP);
}
case obrpc::OB_LOG_PREPARE_REQ: {
PALF_PROCESS(LogPrepareReqP);
}
case obrpc::OB_LOG_PREPARE_RESP: {
PALF_PROCESS(LogPrepareRespP);
}
case obrpc::OB_LOG_CHANGE_CONFIG_META_REQ: {
PALF_PROCESS(LogChangeConfigMetaReqP);
}
case obrpc::OB_LOG_CHANGE_CONFIG_META_RESP: {
PALF_PROCESS(LogChangeConfigMetaRespP);
}
case obrpc::OB_LOG_CHANGE_MODE_META_REQ: {
PALF_PROCESS(LogChangeModeMetaReqP);
}
case obrpc::OB_LOG_CHANGE_MODE_META_RESP: {
PALF_PROCESS(LogChangeModeMetaRespP);
}
case obrpc::OB_LOG_NOTIFY_REBUILD_REQ: {
PALF_PROCESS(LogNotifyRebuildReqP)
}
case obrpc::OB_LOG_COMMITTED_INFO: {
PALF_PROCESS(CommittedInfoP)
}
case obrpc::OB_LOG_LEARNER_REQ: {
PALF_PROCESS(LogLearnerReqP)
}
case obrpc::OB_LOG_REGISTER_PARENT_REQ: {
PALF_PROCESS(LogRegisterParentReqP)
}
case obrpc::OB_LOG_REGISTER_PARENT_RESP: {
PALF_PROCESS(LogRegisterParentRespP)
}
case obrpc::OB_LOG_ELECTION_PREPARE_REQUEST : {
PALF_PROCESS(ElectionPrepareRequestMsgP)
}
case obrpc::OB_LOG_ELECTION_PREPARE_RESPONSE : {
PALF_PROCESS(ElectionPrepareResponseMsgP)
}
case obrpc::OB_LOG_ELECTION_ACCEPT_REQUEST : {
PALF_PROCESS(ElectionAcceptRequestMsgP)
}
case obrpc::OB_LOG_ELECTION_ACCEPT_RESPONSE : {
PALF_PROCESS(ElectionAcceptResponseMsgP)
}
case obrpc::OB_LOG_ELECTION_CHANGE_LEADER_REQUEST : {
PALF_PROCESS(ElectionChangeLeaderMsgP)
}
case obrpc::OB_LOG_GET_MC_ST: {
PALF_PROCESS(LogGetMCStP)
}
case obrpc::OB_LOG_BATCH_FETCH_RESP: {
PALF_PROCESS(LogBatchFetchRespP)
}
case obrpc::OB_LOG_CREATE_REPLICA_CMD: {
LOG_SERVICE_PROCESS(palfcluster::LogCreateReplicaCmdP)
}
case obrpc::OB_LOG_SUBMIT_LOG_CMD: {
LOG_SERVICE_PROCESS(palfcluster::LogSubmitLogP)
}
case obrpc::OB_LOG_SUBMIT_LOG_CMD_RESP: {
LOG_SERVICE_PROCESS(palfcluster::LogSubmitLogRespP)
}
default:
SERVER_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "invalid req type", K(pkt.get_pcode()));
break;
}
} else {
SERVER_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "log_service_ is NULL", KP(log_service_));
}
return true;
}
} // unittest
} // oceanbase

242
mittest/palf_cluster/env/ob_simple_log_server.h vendored Normal file
View File

@ -0,0 +1,242 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#pragma once
#include "lib/hash/ob_hashset.h"
#include "lib/ob_errno.h"
#include "lib/thread/ob_simple_thread_pool.h"
#include "lib/thread/thread_mgr_interface.h"
#include "lib/signal/ob_signal_worker.h"
#include "lib/signal/ob_signal_struct.h"
#include "lib/function/ob_function.h" // ObFunction
#include "logservice/palf/log_block_pool_interface.h"
#include "observer/ob_signal_handle.h"
#include "observer/ob_srv_deliver.h"
#include "lib/utility/ob_defer.h"
#include "rpc/frame/ob_req_deliver.h"
#include "rpc/ob_request.h"
#include "rpc/frame/ob_net_easy.h"
#include "rpc/obrpc/ob_rpc_handler.h"
#include "rpc/frame/ob_req_transport.h"
#include "logservice/palf/log_rpc_macros.h"
#include "logservice/palf/log_rpc_processor.h"
#include "logservice/palf/palf_env_impl.h"
#include "logservice/palf/palf_env.h"
#include "logservice/palf/palf_handle_impl.h"
#include "logservice/ob_arbitration_service.h"
#include "lib/net/ob_addr.h"
#include "share/ob_rpc_struct.h"
#include "storage/blocksstable/ob_block_sstable_struct.h"
#include "share/allocator/ob_tenant_mutil_allocator.h"
#include "storage/tx_storage/ob_ls_map.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "logservice/palf/palf_handle_impl.h"
#include "logservice/ob_log_service.h"
#include "logservice/ob_server_log_block_mgr.h"
#include "share/ob_local_device.h"
#include "share/ob_occam_timer.h"
#include "share/resource_manager/ob_cgroup_ctrl.h"
#include "mittest/palf_cluster/rpc/palf_cluster_rpc_processor.h"
#include "mittest/palf_cluster/logservice/log_service.h"
#include <map>
namespace oceanbase
{
namespace unittest
{
using namespace oceanbase;
using namespace oceanbase::rpc;
using namespace oceanbase::rpc::frame;
using namespace oceanbase::obrpc;
using namespace oceanbase::common;
using namespace oceanbase::palf;
using namespace oceanbase::share;
uint32_t get_local_addr(const char *dev_name);
std::string get_local_ip();
struct LossConfig
{
ObAddr src_;
int loss_rate_;
LossConfig()
: src_(), loss_rate_(0)
{}
LossConfig(const ObAddr &src, const int loss_rate)
: src_(src), loss_rate_(loss_rate)
{}
TO_STRING_KV(K_(src), K_(loss_rate));
};
class ObLogServerReqQueueHandler : public rpc::frame::ObiReqQHandler
{
public:
ObLogServerReqQueueHandler()
: log_service_(NULL) { }
virtual ~ObLogServerReqQueueHandler();
int init(palfcluster::LogService *log_service);
int onThreadCreated(obsys::CThread *);
int onThreadDestroy(obsys::CThread *);
bool handlePacketQueue(ObRequest *req, void *args);
private:
palfcluster::LogService *log_service_;
};
class ObLogDeliver : public rpc::frame::ObReqQDeliver, public lib::TGTaskHandler
{
public:
ObLogDeliver(rpc::frame::ObiReqQHandler &qhandler)
: rpc::frame::ObReqQDeliver(qhandler),
palf_env_impl_(NULL),
tg_id_(0),
blacklist_(),
rpc_loss_config_(),
is_stopped_(true),
tenant_base_(NULL),
req_queue_(NULL) {}
~ObLogDeliver() { destroy(); }
int init();
int set_tenant_base(ObTenantBase *base, palfcluster::LogService *log_service)
{
tenant_base_ = base;
log_service_ = log_service;
return OB_SUCCESS;
}
void destroy();
int deliver(rpc::ObRequest &req);
int start();
void stop();
int wait();
void handle(void *task);
void set_need_drop_packet(const bool need_drop_packet) { need_drop_packet_ = need_drop_packet; }
void block_net(const ObAddr &src);
void unblock_net(const ObAddr &src);
void set_rpc_loss(const ObAddr &src, const int loss_rate);
void reset_rpc_loss(const ObAddr &src);
bool need_filter_packet_by_blacklist(const ObAddr &address);
bool need_drop_by_loss_config(const ObAddr &addr);
void get_loss_config(const ObAddr &src, bool &exist, LossConfig &loss_config);
private:
void init_all_propocessor_();
typedef std::function<int(ObReqProcessor *&)> Func;
Func funcs_[MAX_PCODE];
template <typename PROCESSOR>
void register_rpc_propocessor_(int pcode)
{
auto func = [](ObReqProcessor *&ptr) -> int {
int ret = OB_SUCCESS;
if (NULL == (ptr = OB_NEW(PROCESSOR, "SimpleLogSvr"))) {
SERVER_LOG(WARN, "allocate memory failed");
} else if (OB_FAIL(ptr->init())) {
} else {
}
return ret;
};
funcs_[pcode] = func;
SERVER_LOG(INFO, "register_rpc_propocessor_ success", K(pcode));
}
int create_queue_thread(int tg_id, const char *thread_name, observer::QueueThread *&qthread);
int handle_req_(rpc::ObRequest &req);
private:
mutable common::RWLock lock_;
bool is_inited_;
PalfEnvImpl *palf_env_impl_;
palfcluster::LogService *log_service_;
int tg_id_;
bool need_drop_packet_;
hash::ObHashSet<ObAddr> blacklist_;
common::ObSEArray<LossConfig, 4> rpc_loss_config_;
bool is_stopped_;
ObTenantBase *tenant_base_;
observer::QueueThread *req_queue_;
};
class ObSimpleLogServer
{
public:
ObSimpleLogServer()
: req_handler_(),
deliver_(req_handler_),
handler_(deliver_),
transport_(NULL)
{
}
~ObSimpleLogServer()
{
if (OB_NOT_NULL(allocator_)) {
ob_delete(allocator_);
}
if (OB_NOT_NULL(io_device_)) {
ob_delete(io_device_);
}
}
bool is_valid() {return NULL != palf_env_;}
int simple_init(const std::string &cluster_name,
const common::ObAddr &addr,
const int64_t node_id,
const bool is_bootstrap);
int simple_start(const bool is_bootstrap);
int simple_close(const bool is_shutdown);
int simple_restart(const std::string &cluster_name, const int64_t node_idx);
PalfEnv *get_palf_env() { return palf_env_; }
const std::string& get_clog_dir() const { return clog_dir_; }
common::ObAddr get_addr() const { return addr_; }
ObTenantBase *get_tenant_base() const { return tenant_base_; }
palfcluster::LogService *get_log_service() { return &log_service_; }
logservice::ObServerLogBlockMgr *get_log_block_pool() { return &log_block_pool_; }
// Nowdat, not support drop packet from specificed address
void set_need_drop_packet(const bool need_drop_packet) { deliver_.set_need_drop_packet(need_drop_packet);}
void block_net(const ObAddr &src) { deliver_.block_net(src); }
void unblock_net(const ObAddr &src) { deliver_.unblock_net(src); }
void set_rpc_loss(const ObAddr &src, const int loss_rate) { deliver_.set_rpc_loss(src, loss_rate); }
void reset_rpc_loss(const ObAddr &src) { deliver_.reset_rpc_loss(src); }
TO_STRING_KV(K_(node_id), K_(addr), KP(palf_env_));
protected:
int init_io_(const std::string &cluster_name);
int init_network_(const common::ObAddr &addr, const bool is_bootstrap);
int init_log_service_(const std::string &cluster_name);
int init_memory_dump_timer_();
private:
int64_t node_id_;
common::ObAddr addr_;
rpc::frame::ObNetEasy net_;
ObLogServerReqQueueHandler req_handler_;
ObLogDeliver deliver_;
obrpc::ObRpcHandler handler_;
ObRandom rand_;
PalfEnv *palf_env_;
ObTenantBase *tenant_base_;
ObMalloc malloc_mgr_;
ObLocalDevice *io_device_;
static const int64_t MAX_IOD_OPT_CNT = 5;
ObIODOpt iod_opt_array_[MAX_IOD_OPT_CNT];
ObIODOpts iod_opts_;
std::string clog_dir_;
ObOccamTimer timer_;
ObOccamTimerTaskRAIIHandle timer_handle_;
palfcluster::LogService log_service_;
ObTenantMutilAllocator *allocator_;
rpc::frame::ObReqTransport *transport_;
ObLSService ls_service_;
ObLocationService location_service_;
logservice::ObServerLogBlockMgr log_block_pool_;
common::ObMySQLProxy sql_proxy_;
};
} // end unittest
} // oceanbase

621
mittest/palf_cluster/logservice/log_service.cpp Normal file
View File

@ -0,0 +1,621 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX CLOG
#include "log_service.h"
#include "lib/file/file_directory_utils.h"
#include "lib/ob_errno.h"
#include "logservice/ob_server_log_block_mgr.h"
#include "logservice/palf/log_block_pool_interface.h"
#include "rpc/frame/ob_req_transport.h"
#include "share/ob_ls_id.h"
#include "share/allocator/ob_tenant_mutil_allocator.h"
#include "share/allocator/ob_tenant_mutil_allocator_mgr.h"
#include "share/ob_tenant_info_proxy.h"
#include "share/ob_unit_getter.h"
#include "share/rc/ob_tenant_base.h"
#include "share/rc/ob_tenant_module_init_ctx.h"
#include "storage/tx_storage/ob_ls_map.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "observer/ob_srv_network_frame.h"
#include "logservice/palf_handle_guard.h"
#include "storage/ob_file_system_router.h"
#include "logservice/palf/palf_env.h"
#include "logservice/palf/palf_callback.h"
#include "logservice/palf/palf_options.h"
namespace oceanbase
{
using namespace share;
using namespace common;
using namespace palf;
namespace palfcluster
{
using namespace oceanbase::share;
using namespace oceanbase::common;
LogService::LogService() :
is_inited_(false),
is_running_(false),
self_(),
palf_env_(NULL),
apply_service_(),
// replay_service_(),
role_change_service_(),
monitor_(),
rpc_proxy_(),
log_client_map_(),
ls_adapter_()
{}
LogService::~LogService()
{
destroy();
}
int LogService::mtl_init(LogService* &logservice)
{
int ret = OB_SUCCESS;
const ObAddr &self = GCTX.self_addr();
const int64_t tenant_id = MTL_ID();
observer::ObSrvNetworkFrame *net_frame = GCTX.net_frame_;
//log_disk_usage_limit_size无法主动从配置项获取, 需要在mtl初始化时作为入参传入
const palf::PalfOptions &palf_options = MTL_INIT_CTX()->palf_options_;
const char *tenant_clog_dir = MTL_INIT_CTX()->tenant_clog_dir_;
const char *clog_dir = OB_FILE_SYSTEM_ROUTER.get_clog_dir();
logservice::ObServerLogBlockMgr *log_block_mgr = GCTX.log_block_mgr_;
common::ObILogAllocator *alloc_mgr = NULL;
if (OB_FAIL(TMA_MGR_INSTANCE.get_tenant_log_allocator(tenant_id, alloc_mgr))) {
CLOG_LOG(WARN, "get_tenant_log_allocator failed", K(ret));
} else if (OB_FAIL(logservice->init(palf_options,
tenant_clog_dir,
self,
alloc_mgr,
net_frame->get_req_transport(),
log_block_mgr))) {
CLOG_LOG(ERROR, "init LogService failed", K(ret), K(tenant_clog_dir));
} else if (OB_FAIL(FileDirectoryUtils::fsync_dir(clog_dir))) {
CLOG_LOG(ERROR, "fsync_dir failed", K(ret), K(clog_dir));
} else {
CLOG_LOG(INFO, "LogService mtl_init success");
}
return ret;
}
void LogService::mtl_destroy(LogService* &logservice)
{
common::ob_delete(logservice);
logservice = nullptr;
// Free tenant_log_allocator for this tenant after destroy logservice.
const int64_t tenant_id = MTL_ID();
int ret = OB_SUCCESS;
if (OB_FAIL(TMA_MGR_INSTANCE.delete_tenant_log_allocator(tenant_id))) {
CLOG_LOG(WARN, "delete_tenant_log_allocator failed", K(ret));
}
}
int LogService::start()
{
int ret = OB_SUCCESS;
if (OB_FAIL(apply_service_.start())) {
CLOG_LOG(WARN, "failed to start apply_service_", K(ret));
// } else if (OB_FAIL(replay_service_.start())) {
// CLOG_LOG(WARN, "failed to start replay_service_", K(ret));
} else if (OB_FAIL(role_change_service_.start())) {
CLOG_LOG(WARN, "failed to start role_change_service_", K(ret));
} else {
is_running_ = true;
FLOG_INFO("LogService is started");
}
return ret;
}
void LogService::stop()
{
is_running_ = false;
CLOG_LOG(INFO, "begin to stop LogService");
(void)apply_service_.stop();
// (void)replay_service_.stop();
(void)role_change_service_.stop();
FLOG_INFO("LogService is stopped");
}
void LogService::wait()
{
apply_service_.wait();
// replay_service_.wait();
role_change_service_.wait();
}
void LogService::destroy()
{
is_inited_ = false;
self_.reset();
apply_service_.destroy();
// replay_service_.destroy();
role_change_service_.destroy();
ls_adapter_.destroy();
rpc_proxy_.destroy();
if (NULL != palf_env_) {
PalfEnv::destroy_palf_env(palf_env_);
palf_env_ = NULL;
}
FLOG_INFO("LogService is destroyed");
}
int check_and_prepare_dir(const char *dir)
{
bool is_exist = false;
int ret = OB_SUCCESS;
if (OB_FAIL(common::FileDirectoryUtils::is_exists(dir, is_exist))) {
CLOG_LOG(WARN, "chcck dir exist failed", K(ret), K(dir));
// means it's restart
} else if (is_exist == true) {
CLOG_LOG(INFO, "director exist", K(ret), K(dir));
// means it's create tenant
} else if (OB_FAIL(common::FileDirectoryUtils::create_directory(dir))) {
CLOG_LOG(WARN, "create_directory failed", K(ret), K(dir));
} else {
CLOG_LOG(INFO, "check_and_prepare_dir success", K(ret), K(dir));
}
return ret;
}
int LogService::init(const PalfOptions &options,
const char *base_dir,
const common::ObAddr &self,
common::ObILogAllocator *alloc_mgr,
rpc::frame::ObReqTransport *transport,
palf::ILogBlockPool *log_block_pool)
{
int ret = OB_SUCCESS;
const int64_t tenant_id = MTL_ID();
if (OB_FAIL(check_and_prepare_dir(base_dir))) {
CLOG_LOG(WARN, "check_and_prepare_dir failed", K(ret), K(base_dir));
} else if (is_inited_) {
ret = OB_INIT_TWICE;
CLOG_LOG(WARN, "LogService init twice", K(ret));
} else if (false == options.is_valid() || OB_ISNULL(base_dir) || OB_UNLIKELY(!self.is_valid())
|| OB_ISNULL(alloc_mgr) || OB_ISNULL(transport)
|| OB_ISNULL(log_block_pool)) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(WARN, "invalid arguments", K(ret), K(options), KP(base_dir), K(self),
KP(alloc_mgr), KP(transport), KP(log_block_pool));
} else if (OB_FAIL(PalfEnv::create_palf_env(options, base_dir, self, transport,
alloc_mgr, log_block_pool, &monitor_, palf_env_))) {
CLOG_LOG(WARN, "failed to create_palf_env", K(base_dir), K(ret));
} else if (OB_ISNULL(palf_env_)) {
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(ERROR, "palf_env_ is NULL", K(ret));
} else if (OB_FAIL(ls_adapter_.init())) {
CLOG_LOG(ERROR, "failed to init ls_adapter", K(ret));
} else if (OB_FAIL(apply_service_.init(palf_env_, &ls_adapter_))) {
CLOG_LOG(WARN, "failed to init apply_service", K(ret));
// } else if (OB_FAIL(replay_service_.init(palf_env_, &ls_adapter_, alloc_mgr))) {
// CLOG_LOG(WARN, "failed to init replay_service", K(ret));
} else if (OB_FAIL(role_change_service_.init(&log_client_map_, &apply_service_))) {
CLOG_LOG(WARN, "failed to init role_change_service_", K(ret));
} else if (OB_FAIL(rpc_proxy_.init(transport))) {
CLOG_LOG(WARN, "LogServiceRpcProxy init failed", K(ret));
} else if (OB_FAIL(log_client_map_.init("Client", MTL_ID()))) {
CLOG_LOG(WARN, "failed to init log_client_map", K(ret));
} else {
self_ = self;
is_inited_ = true;
FLOG_INFO("LogService init success", K(ret), K(base_dir), K(self), KP(transport), K(tenant_id));
}
if (OB_FAIL(ret) && OB_INIT_TWICE != ret) {
destroy();
}
return ret;
}
int LogService::create_palf_replica(const int64_t palf_id,
const common::ObMemberList &member_list,
const int64_t replica_num,
const int64_t leader_idx)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else {
share::ObLSID ls_id(palf_id);
ObLogClient *log_client = OB_NEW(ObLogClient, "client");
if (OB_FAIL(log_client->init(self_, palf_id, &rpc_proxy_, this))) {
CLOG_LOG(ERROR, "init fail", K(ret), K(palf_id));
} else if (OB_FAIL(log_client_map_.insert(ls_id, log_client))) {
CLOG_LOG(ERROR, "insert fail", K(ret), K(palf_id));
} else if (OB_FAIL(log_client->create_palf_replica(member_list, replica_num, leader_idx))) {
CLOG_LOG(ERROR, "create_replica fail", K(ret), K(palf_id));
log_client_map_.erase(ls_id);
} else {
CLOG_LOG(ERROR, "create_palf_replica successfully", K(ret), K(palf_id), K(member_list), K(replica_num), K(leader_idx));
}
}
return ret;
}
int LogService::create_log_clients(const int64_t thread_num,
const int64_t log_size,
const int64_t palf_group_num,
std::vector<common::ObAddr> leader_list)
{
int ret = OB_SUCCESS;
for (int64_t i = 0; i < thread_num; i++) {
const int64_t palf_id = (i % palf_group_num) + 1;
if (OB_FAIL(clients_[i].init_and_create(i, log_size, &rpc_proxy_, self_, leader_list[palf_id-1], palf_id))) {
CLOG_LOG(ERROR, "init_and_create fail", K(palf_id));
}
}
for (int64_t i = 0; i < thread_num; i++) {
clients_[i].join();
}
return ret;
}
palf::AccessMode LogService::get_palf_access_mode(const share::ObTenantRole &tenant_role)
{
palf::AccessMode mode = palf::AccessMode::INVALID_ACCESS_MODE;
switch (tenant_role.value()) {
case share::ObTenantRole::INVALID_TENANT:
mode = palf::AccessMode::INVALID_ACCESS_MODE;
break;
case share::ObTenantRole::PRIMARY_TENANT:
mode = palf::AccessMode::APPEND;
break;
case share::ObTenantRole::STANDBY_TENANT:
case share::ObTenantRole::RESTORE_TENANT:
mode = palf::AccessMode::RAW_WRITE;
break;
default:
mode = palf::AccessMode::INVALID_ACCESS_MODE;
break;
}
return mode;
}
int LogService::create_ls(const share::ObLSID &id,
const common::ObReplicaType &replica_type,
const share::ObTenantRole &tenant_role,
const palf::PalfBaseInfo &palf_base_info,
const bool allow_log_sync,
logservice::ObLogHandler &log_handler)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id));
} else if (!palf_base_info.is_valid()) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(WARN, "invalid arguments", K(ret), K(id), K(replica_type),
K(tenant_role), K(palf_base_info));
} else if (OB_FAIL(create_ls_(id, replica_type, tenant_role, palf_base_info, allow_log_sync,
log_handler))) {
CLOG_LOG(WARN, "create ls failed", K(ret), K(id), K(replica_type),
K(tenant_role), K(palf_base_info));
} else {
FLOG_INFO("LogService create_ls success", K(ret), K(id), K(replica_type), K(tenant_role), K(palf_base_info),
K(log_handler));
}
return ret;
}
int LogService::remove_ls(const ObLSID &id, logservice::ObLogHandler &log_handler)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id));
} else if (OB_FAIL(apply_service_.remove_ls(id))) {
CLOG_LOG(WARN, "failed to remove from apply_service", K(ret), K(id));
// } else if (OB_FAIL(replay_service_.remove_ls(id))) {
// CLOG_LOG(WARN, "failed to remove from replay_service", K(ret), K(id));
// NB: remove palf_handle lastly.
} else {
// NB: can not execute destroy, otherwise, each interface in log_handler or restore_handler
// may return OB_NOT_INIT.
// TODO by runlin: create_ls don't init logservice::ObLogHandler and ObLogRestoreHandler.
//
// In normal case(for gc), stop has been executed, this stop has no effect.
// In abnormal case(create ls failed, need remove ls directlly), there is no possibility for dead lock.
log_handler.stop();
if (OB_FAIL(palf_env_->remove(id.id()))) {
CLOG_LOG(WARN, "failed to remove from palf_env_", K(ret), K(id));
} else {
FLOG_INFO("LogService remove_ls success", K(ret), K(id));
}
}
return ret;
}
int LogService::check_palf_exist(const ObLSID &id, bool &exist) const
{
int ret = OB_SUCCESS;
PalfHandle handle;
exist = true;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
CLOG_LOG(WARN, "LogService is not inited", K(ret), K(id));
} else if (OB_FAIL(palf_env_->open(id.id(), handle))) {
if (OB_ENTRY_NOT_EXIST == ret ) {
ret = OB_SUCCESS;
exist = false;
} else {
CLOG_LOG(WARN, "open palf failed", K(ret), K(id));
}
}
if (true == handle.is_valid()) {
palf_env_->close(handle);
}
return ret;
}
int LogService::add_ls(const ObLSID &id, logservice::ObLogHandler &log_handler)
{
int ret = OB_SUCCESS;
PalfHandle palf_handle;
PalfHandle &log_handler_palf_handle = log_handler.palf_handle_;
PalfRoleChangeCb *rc_cb = &role_change_service_;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id));
} else if (OB_FAIL(palf_env_->open(id.id(), palf_handle))) {
CLOG_LOG(WARN, "failed to get palf_handle", K(ret), K(id));
} else if (OB_FAIL(apply_service_.add_ls(id))) {
CLOG_LOG(WARN, "failed to add_ls for apply_service", K(ret), K(id));
// } else if (OB_FAIL(replay_service_.add_ls(id))) {
// CLOG_LOG(WARN, "failed to add_ls for replay_service", K(ret), K(id));
// } else if (OB_FAIL(log_handler.init(id.id(), self_, &apply_service_, &replay_service_,
// &role_change_service_, palf_handle, palf_env_, loc_cache_cb, &rpc_proxy_))) {
// CLOG_LOG(WARN, "logservice::ObLogHandler init failed", K(ret), K(id), KP(palf_env_), K(palf_handle));
} else if (OB_FAIL(log_handler_palf_handle.register_role_change_cb(rc_cb))) {
CLOG_LOG(WARN, "register_role_change_cb failed", K(ret));
} else {
FLOG_INFO("add_ls success", K(ret), K(id), KP(this));
}
if (OB_FAIL(ret)) {
if (true == palf_handle.is_valid() && false == log_handler.is_valid()) {
palf_env_->close(palf_handle);
}
}
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_LS_NOT_EXIST;
}
return ret;
}
int LogService::open_palf(const share::ObLSID &id,
palf::PalfHandleGuard &palf_handle_guard)
{
int ret = OB_SUCCESS;
palf::PalfHandle palf_handle;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id));
} else if (OB_FAIL(palf_env_->open(id.id(), palf_handle))) {
CLOG_LOG(WARN, "failed to get palf_handle", K(ret), K(id));
} else if (FALSE_IT(palf_handle_guard.set(palf_handle, palf_env_))) {
} else {
CLOG_LOG(TRACE, "LogService open_palf success", K(ret), K(id));
}
if (OB_FAIL(ret)) {
if (true == palf_handle.is_valid()) {
palf_env_->close(palf_handle);
}
}
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_LS_NOT_EXIST;
}
return ret;
}
int LogService::get_palf_role(const share::ObLSID &id,
common::ObRole &role,
int64_t &proposal_id)
{
int ret = OB_SUCCESS;
palf::PalfHandleGuard palf_handle_guard;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id));
} else if (OB_FAIL(open_palf(id, palf_handle_guard))) {
CLOG_LOG(WARN, "failed to open palf", K(ret), K(id));
} else if (OB_FAIL(palf_handle_guard.get_role(role, proposal_id))) {
CLOG_LOG(WARN, "failed to get role", K(ret), K(id));
}
return ret;
}
int LogService::get_palf_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else {
ret = palf_env_->get_disk_usage(used_size_byte, total_size_byte);
}
return ret;
}
int LogService::get_palf_stable_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else {
ret = palf_env_->get_stable_disk_usage(used_size_byte, total_size_byte);
}
return ret;
}
int LogService::update_palf_options_except_disk_usage_limit_size()
{
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
ObSpinLockGuard guard(update_palf_opts_lock_);
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else if (!tenant_config.is_valid()) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(WARN, "tenant_config is not valid", K(ret), K(MTL_ID()));
} else {
PalfOptions palf_opts;
common::ObCompressorType compressor_type = LZ4_COMPRESSOR;
if (OB_FAIL(common::ObCompressorPool::get_instance().get_compressor_type(
tenant_config->log_transport_compress_func, compressor_type))) {
CLOG_LOG(ERROR, "log_transport_compress_func invalid.", K(ret));
//需要获取log_disk_usage_limit_size
} else if (OB_FAIL(palf_env_->get_options(palf_opts))) {
CLOG_LOG(WARN, "palf get_options failed", K(ret));
} else {
palf_opts.disk_options_.log_disk_utilization_threshold_ = tenant_config->log_disk_utilization_threshold;
palf_opts.disk_options_.log_disk_utilization_limit_threshold_ = tenant_config->log_disk_utilization_limit_threshold;
palf_opts.disk_options_.log_disk_throttling_percentage_ = tenant_config->log_disk_throttling_percentage;
palf_opts.disk_options_.log_disk_throttling_maximum_duration_ = tenant_config->log_disk_throttling_maximum_duration;
palf_opts.compress_options_.enable_transport_compress_ = tenant_config->log_transport_compress_all;
palf_opts.compress_options_.transport_compress_func_ = compressor_type;
palf_opts.rebuild_replica_log_lag_threshold_ = tenant_config->_rebuild_replica_log_lag_threshold;
palf_opts.disk_options_.log_writer_parallelism_ = tenant_config->_log_writer_parallelism;
if (OB_FAIL(palf_env_->update_options(palf_opts))) {
CLOG_LOG(WARN, "palf update_options failed", K(MTL_ID()), K(ret), K(palf_opts));
} else {
CLOG_LOG(INFO, "palf update_options success", K(MTL_ID()), K(ret), K(palf_opts));
}
}
}
return ret;
}
//log_disk_usage_limit_size无法主动感知,只能通过上层触发时传入
int LogService::update_log_disk_usage_limit_size(const int64_t log_disk_usage_limit_size)
{
int ret = OB_SUCCESS;
ObSpinLockGuard guard(update_palf_opts_lock_);
PalfOptions palf_opts;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else if (OB_FAIL(palf_env_->get_options(palf_opts))) {
CLOG_LOG(WARN, "palf get_options failed", K(ret));
} else if (FALSE_IT(palf_opts.disk_options_.log_disk_usage_limit_size_ = log_disk_usage_limit_size)) {
} else if (OB_FAIL(palf_env_->update_options(palf_opts))) {
CLOG_LOG(WARN, "palf update_options failed", K(ret), K(log_disk_usage_limit_size));
} else {
CLOG_LOG(INFO, "update_log_disk_usage_limit_size success", K(log_disk_usage_limit_size), K(MTL_ID()));
}
return ret;
}
int LogService::get_palf_options(palf::PalfOptions &opts)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else {
ret = palf_env_->get_options(opts);
}
return ret;
}
int LogService::iterate_palf(const ObFunction<int(const PalfHandle&)> &func)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else {
ret = palf_env_->for_each(func);
}
return ret;
}
int LogService::create_ls_(const share::ObLSID &id,
const common::ObReplicaType &replica_type,
const share::ObTenantRole &tenant_role,
const palf::PalfBaseInfo &palf_base_info,
const bool allow_log_sync,
logservice::ObLogHandler &log_handler)
{
int ret = OB_SUCCESS;
PalfHandle palf_handle;
PalfRoleChangeCb *rc_cb = &role_change_service_;
const bool is_arb_replica = (replica_type == REPLICA_TYPE_ARBITRATION);
PalfHandle &log_handler_palf_handle = log_handler.palf_handle_;
if (false == id.is_valid() ||
INVALID_TENANT_ROLE == tenant_role ||
false == palf_base_info.is_valid()) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(WARN, "invalid argument", K(ret), K(id), K(id), K(tenant_role), K(palf_base_info));
} else if (!is_arb_replica &&
OB_FAIL(palf_env_->create(id.id(), get_palf_access_mode(tenant_role), palf_base_info, palf_handle))) {
CLOG_LOG(WARN, "failed to get palf_handle", K(ret), K(id), K(replica_type));
} else if (false == allow_log_sync && OB_FAIL(palf_handle.disable_sync())) {
CLOG_LOG(WARN, "failed to disable_sync", K(ret), K(id));
} else if (OB_FAIL(apply_service_.add_ls(id))) {
CLOG_LOG(WARN, "failed to add_ls for apply engine", K(ret), K(id));
// } else if (OB_FAIL(replay_service_.add_ls(id))) {
// CLOG_LOG(WARN, "failed to add_ls", K(ret), K(id));
} else if (OB_FAIL(log_handler.init(id.id(), self_, &apply_service_, &replay_service_,
NULL, palf_handle, palf_env_, &location_adapter_, &log_service_rpc_proxy_))) {
CLOG_LOG(WARN, "logservice::ObLogHandler init failed", K(ret), KP(palf_env_), K(palf_handle));
} else if (OB_FAIL(log_handler_palf_handle.register_role_change_cb(rc_cb))) {
CLOG_LOG(WARN, "register_role_change_cb failed", K(ret), K(id));
} else {
CLOG_LOG(INFO, "LogService create_ls success", K(ret), K(id), K(log_handler));
}
if (OB_FAIL(ret)) {
if (true == palf_handle.is_valid() && false == log_handler.is_valid()) {
palf_env_->close(palf_handle);
}
}
return ret;
}
int LogService::get_io_start_time(int64_t &last_working_time)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
CLOG_LOG(WARN, "log_service is not inited", K(ret));
} else if (OB_FAIL(palf_env_->get_io_start_time(last_working_time))) {
CLOG_LOG(WARN, "palf_env get_io_start_time failed", K(ret));
} else {
// do nothing
}
return ret;
}
int LogService::check_disk_space_enough(bool &is_disk_enough)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
CLOG_LOG(WARN, "log_service is not inited", K(ret));
} else {
is_disk_enough = palf_env_->check_disk_space_enough();
}
return ret;
}
}//end of namespace logservice
}//end of namespace oceanbase

211
mittest/palf_cluster/logservice/log_service.h Normal file
View File

@ -0,0 +1,211 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_PALF_CLUSTER_OB_LOG_SERVICE_
#define OCEANBASE_PALF_CLUSTER_OB_LOG_SERVICE_
#include "common/ob_role.h"
#include "lib/ob_define.h"
#include "share/ob_tenant_info_proxy.h" // ObTenantRole
#include "logservice/applyservice/ob_log_apply_service.h"
#include "mittest/palf_cluster/rpc/palf_cluster_rpc_req.h"
#include "mittest/palf_cluster/rpc/palf_cluster_rpc_proxy.h"
#include "mittest/palf_cluster/logservice/ob_log_client.h"
#include "logservice/palf/log_block_pool_interface.h" // ILogBlockPool
#include "logservice/palf/log_define.h"
#include "logservice/ob_log_monitor.h"
#include "logservice/ob_log_handler.h"
#include "logservice/ob_log_service.h"
#include "role_coordinator.h"
#include "ls_adapter.h"
// #include "replayservice/ob_log_replay_service.h"
namespace oceanbase
{
namespace commom
{
class ObAddr;
class ObILogAllocator;
}
namespace rpc
{
namespace frame
{
class ObReqTransport;
}
}
namespace share
{
class ObLSID;
class SCN;
}
namespace palf
{
class PalfHandleGuard;
class PalfRoleChangeCb;
class PalfDiskOptions;
class PalfEnv;
}
namespace palfcluster
{
class LogService
{
public:
LogService();
virtual ~LogService();
static int mtl_init(LogService* &logservice);
static void mtl_destroy(LogService* &logservice);
int start();
void stop();
void wait();
void destroy();
public:
static palf::AccessMode get_palf_access_mode(const share::ObTenantRole &tenant_role);
int init(const palf::PalfOptions &options,
const char *base_dir,
const common::ObAddr &self,
common::ObILogAllocator *alloc_mgr,
rpc::frame::ObReqTransport *transport,
palf::ILogBlockPool *log_block_pool);
//--日志流相关接口--
//新建日志流接口，该接口会创建日志流对应的目录，新建一个以PalfBaeInfo为日志基点的日志流。
//其中包括生成并初始化对应的ObReplayStatus结构
// @param [in] id，日志流标识符
// @param [in] replica_type，日志流的副本类型
// @param [in] tenant_role, 租户角色, 以此决定Palf使用模式(APPEND/RAW_WRITE)
// @param [in] palf_base_info, 日志同步基点信息
// @param [out] log_handler，新建日志流以logservice::ObLogHandler形式返回，保证上层使用日志流时的生命周期
int create_ls(const share::ObLSID &id,
const common::ObReplicaType &replica_type,
const share::ObTenantRole &tenant_role,
const palf::PalfBaseInfo &palf_base_info,
const bool allow_log_sync,
logservice::ObLogHandler &log_handler);
//删除日志流接口:外层调用create_ls()之后，后续流程失败，需要调用remove_ls()
int remove_ls(const share::ObLSID &id,
logservice::ObLogHandler &log_handler);
int check_palf_exist(const share::ObLSID &id, bool &exist) const;
//宕机重启恢复日志流接口，包括生成并初始化对应的ObReplayStatus结构
// @param [in] id，日志流标识符
// @param [out] log_handler，新建日志流以logservice::ObLogHandler形式返回，保证上层使用日志流时的生命周期
int add_ls(const share::ObLSID &id,
logservice::ObLogHandler &log_handler);
int open_palf(const share::ObLSID &id,
palf::PalfHandleGuard &palf_handle);
// get role of current palf replica.
// NB: distinguish the difference from get_role of log_handler
// In general, get the replica role to do migration/blance/report, use this interface,
// to write log, use get_role of log_handler
int get_palf_role(const share::ObLSID &id,
common::ObRole &role,
int64_t &proposal_id);
// @brief get palf disk usage
// @param [out] used_size_byte
// @param [out] total_size_byte, if in shrinking status, total_size_byte is the value after shrinking.
// NB: total_size_byte may be smaller than used_size_byte.
int get_palf_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte);
// @brief get palf disk usage
// @param [out] used_size_byte
// @param [out] total_size_byte, if in shrinking status, total_size_byte is the value before shrinking.
int get_palf_stable_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte);
// why we need update 'log_disk_size_' and 'log_disk_util_threshold' separately.
//
// 'log_disk_size' is a member of unit config.
// 'log_disk_util_threshold' and 'log_disk_util_limit_threshold' are members of tenant parameters.
// If we just only provide 'update_disk_options', the correctness of PalfDiskOptions can not be guaranteed.
// for example, original PalfDiskOptions is that
// {
// log_disk_size = 100G,
// log_disk_util_limit_threshold = 95,
// log_disk_util_threshold = 80
// }
//
// 1. thread1 update 'log_disk_size' with 50G, and it will used 'update_disk_options' with PalfDiskOptions
// {
// log_disk_size = 50G,
// log_disk_util_limit_threshold = 95,
// log_disk_util_threshold = 80
// }
// 2. thread2 updaet 'log_disk_util_limit_threshold' with 85, and it will used 'update_disk_options' with PalfDiskOptions
// {
// log_disk_size = 100G,
// log_disk_util_limit_threshold = 85,
// log_disk_util_threshold = 80
// }.
int update_palf_options_except_disk_usage_limit_size();
int update_log_disk_usage_limit_size(const int64_t log_disk_usage_limit_size);
int get_palf_options(palf::PalfOptions &options);
int iterate_palf(const ObFunction<int(const palf::PalfHandle&)> &func);
int get_io_start_time(int64_t &last_working_time);
int check_disk_space_enough(bool &is_disk_enough);
palf::PalfEnv *get_palf_env() { return palf_env_; }
// TODO by yunlong: temp solution, will by removed after Reporter be added in MTL
// ObLogReplayService *get_log_replay_service() { return &replay_service_; }
obrpc::PalfClusterRpcProxy *get_rpc_proxy() { return &rpc_proxy_; }
ObAddr &get_self() { return self_; }
int create_palf_replica(const int64_t palf_id,
const common::ObMemberList &member_list,
const int64_t replica_num,
const int64_t leader_idx);
int create_log_clients(const int64_t thread_num,
const int64_t log_size,
const int64_t palf_group_num,
std::vector<common::ObAddr> leader_list);
public:
palfcluster::LogClientMap *get_log_client_map() { return &log_client_map_; }
static const int64_t THREAD_NUM = 2000;
palfcluster::LogRemoteClient clients_[THREAD_NUM];
private:
int create_ls_(const share::ObLSID &id,
const common::ObReplicaType &replica_type,
const share::ObTenantRole &tenant_role,
const palf::PalfBaseInfo &palf_base_info,
const bool allow_log_sync,
logservice::ObLogHandler &log_handler);
private:
bool is_inited_;
bool is_running_;
common::ObAddr self_;
palf::PalfEnv *palf_env_;
logservice::ObLogApplyService apply_service_;
logservice::ObLogReplayService replay_service_;
palfcluster::RoleCoordinator role_change_service_;
logservice::ObLogMonitor monitor_;
obrpc::PalfClusterRpcProxy rpc_proxy_;
ObSpinLock update_palf_opts_lock_;
palfcluster::LogClientMap log_client_map_;
palfcluster::MockLSAdapter ls_adapter_;
logservice::ObLocationAdapter location_adapter_;
obrpc::ObLogServiceRpcProxy log_service_rpc_proxy_;
private:
DISALLOW_COPY_AND_ASSIGN(LogService);
};
} // end namespace palfcluster
} // end namespace oceanbase
#endif // OCEANBASE_PALF_CLUSTER_OB_LOG_SERVICE_

60
mittest/palf_cluster/logservice/ls_adapter.cpp Normal file
View File

@ -0,0 +1,60 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "ls_adapter.h"
#include "logservice/replayservice/ob_replay_status.h"
namespace oceanbase
{
namespace palfcluster
{
MockLSAdapter::MockLSAdapter() :
is_inited_(false)
{}
MockLSAdapter::~MockLSAdapter()
{
destroy();
}
int MockLSAdapter::init()
{
int ret = OB_SUCCESS;
if (is_inited_) {
ret = OB_INIT_TWICE;
CLOG_LOG(WARN, "MockLSAdapter init twice", K(ret));
} else {
is_inited_ = true;
CLOG_LOG(INFO, "MockLSAdapter init success", K(ret));
}
return ret;
}
void MockLSAdapter::destroy()
{
is_inited_ = false;
}
int MockLSAdapter::replay(logservice::ObLogReplayTask *replay_task)
{
int ret = OB_SUCCESS;
return ret;
}
int MockLSAdapter::wait_append_sync(const share::ObLSID &ls_id)
{
int ret = OB_SUCCESS;
return ret;
}
} // end namespace palfcluster
} // end namespace oceanbase

47
mittest/palf_cluster/logservice/ls_adapter.h Normal file
View File

@ -0,0 +1,47 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_PALF_CLUSTER_LS_ADAPTER_H_
#define OCEANBASE_PALF_CLUSTER_LS_ADAPTER_H_
#include <stdint.h>
#include "share/ob_ls_id.h"
#include "logservice/ob_ls_adapter.h"
namespace oceanbase
{
namespace logservice
{
class ObLogReplayTask;
};
namespace palfcluster
{
class MockLSAdapter : public logservice::ObLSAdapter
{
public:
MockLSAdapter();
~MockLSAdapter();
int init();
void destroy();
public:
int replay(logservice::ObLogReplayTask *replay_task) override final;
int wait_append_sync(const share::ObLSID &ls_id) override final;
private:
bool is_inited_;
};
} // logservice
} // oceanbase
#endif

237
mittest/palf_cluster/logservice/ob_log_client.cpp Normal file
View File

@ -0,0 +1,237 @@
// Copyright (c) 2021 OceanBase
// OceanBase is licensed under Mulan PubL v2.
// You can use this software according to the terms and conditions of the Mulan PubL v2.
// You may obtain a copy of Mulan PubL v2 at:
// http://license.coscl.org.cn/MulanPubL-2.0
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PubL v2 for more details.
#define USING_LOG_PREFIX CLOG
#include "ob_log_client.h"
#include "logservice/palf/palf_env.h"
#include "logservice/palf/palf_handle.h"
#include "lib/utility/ob_macro_utils.h"
#include "lib/thread/ob_thread_name.h" // set_thread_name
#include "lib/function/ob_function.h" // ObFunction
#include "mittest/palf_cluster/rpc/palf_cluster_rpc_proxy.h" // RpcProxy
#include "mittest/palf_cluster/logservice/log_service.h" // LogService
namespace oceanbase
{
using namespace common;
using namespace palf;
namespace palfcluster
{
int64_t c_append_cnt = 0;
int64_t c_rt = 0;
ObLogClient::ObLogClient()
: self_(),
palf_id_(-1),
rpc_proxy_(NULL),
log_handler_(),
lock_(),
log_size_(-1),
election_priority_(),
total_num_(0),
is_inited_(false)
{}
ObLogClient::~ObLogClient()
{
destroy();
}
void ObLogClient::destroy()
{
if (IS_INIT) {
is_inited_ = false;
rpc_proxy_ = NULL;
palf_id_ = -1;
log_handler_.destroy();
log_size_ = -1;
}
}
int ObLogClient::init(const common::ObAddr &self,
const int64_t palf_id,
obrpc::PalfClusterRpcProxy *rpc_proxy,
LogService *log_service)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
CLOG_LOG(WARN, "ObLogClient has been inited", K(ret));
} else if (false == self.is_valid()|| OB_ISNULL(rpc_proxy) || palf_id < 0) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(WARN, "invalid argument", K(ret), K(self), K(palf_id), K(rpc_proxy));
} else {
for (int i = 0; i < REMOTE_APPEND_CB_CNT; i++) {
if (OB_FAIL(remote_append_cb_list[i].init(rpc_proxy, self))) {
CLOG_LOG(WARN, "init", K(ret), K(self), K(palf_id), K(rpc_proxy));
}
}
self_ = self;
palf_id_ = palf_id;
rpc_proxy_ = rpc_proxy;
log_service_ = log_service;
is_inited_ = true;
}
if ((OB_FAIL(ret)) && (OB_INIT_TWICE != ret)) {
destroy();
}
CLOG_LOG(INFO, "ObLogClient init finished", K(ret), K(self));
return ret;
}
int ObLogClient::create_palf_replica(const common::ObMemberList &member_list,
const int64_t replica_num,
const int64_t leader_idx)
{
int ret = OB_SUCCESS;
if(IS_NOT_INIT) {
ret = OB_NOT_INIT;
CLOG_LOG(ERROR, "ObLogClient has not been inited", K(ret));
} else if (false == member_list.is_valid() || replica_num <= 0) {
ret = OB_INVALID_ARGUMENT;
} else {
share::ObLSID ls_id(palf_id_);
share::ObTenantRole tenant_role(share::ObTenantRole::PRIMARY_TENANT);
common::ObAddr leader;
member_list.get_server_by_index(leader_idx, leader);
LSN init_lsn(0);
common::ObILogAllocator *alloc_mgr = NULL;
palf::PalfBaseInfo palf_base_info;
palf_base_info.generate_by_default();
common::GlobalLearnerList learner_list;
if (OB_FAIL(log_service_->create_ls(ls_id, ObReplicaType::REPLICA_TYPE_FULL,
tenant_role, palf_base_info, true, log_handler_))) {
CLOG_LOG(WARN, "create_ls failed", K(ret), K_(palf_id));
} else if (OB_FAIL(log_handler_.set_initial_member_list(member_list, replica_num, learner_list))) {
CLOG_LOG(WARN, "set_initial_member_list failed", K(ret), K_(palf_id));
} else {
CLOG_LOG(ERROR, "create_palf_replica success", K(ret), K_(palf_id), K(member_list), K(replica_num), K(leader_idx));
}
}
return ret;
}
static void *append_fn(void *arg)
{
ObLogClient *client = reinterpret_cast<ObLogClient *>(arg);
client->do_submit();
return (void *)0;
}
int ObLogClient::submit_append_log_task(const int64_t thread_num, const int64_t log_size)
{
int ret = OB_SUCCESS;
common::ObRole role;
int64_t unused_pid;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else if (OB_FAIL(log_handler_.get_role(role, unused_pid))) {
} else if (role != LEADER) {
CLOG_LOG(ERROR, "client is not leader");
} else if (OB_FAIL(lock_.trylock())) {
CLOG_LOG(ERROR, "another client running", K(ret));
} else {
log_size_ = log_size;
client_number_ = thread_num;
worker_number_ = (client_number_ >= MAX_THREAD_NUM)? MAX_THREAD_NUM: client_number_;
pthread_t tids[MAX_THREAD_NUM];
CLOG_LOG(ERROR, "start submit_log", K_(log_size), K(thread_num), K(worker_number_));
for (int64_t i = 0; i < worker_number_; i++) {
pthread_create(&tids[i], NULL, append_fn, this);
}
for (int64_t i = 0; i < worker_number_; i++) {
pthread_join(tids[i], NULL);
}
lock_.unlock();
}
return ret;
}
int ObLogClient::do_submit()
{
int ret = OB_SUCCESS;
const int64_t NBYTES = 40000;
char BUFFER[NBYTES];
memset(BUFFER, 'a', NBYTES);
const int64_t CB_ARRAY_NUM = (client_number_ >= worker_number_)? client_number_ / worker_number_: 1;
MockAppendCb *cb_array = new MockAppendCb[CB_ARRAY_NUM];
LSN lsn;
SCN log_scn;
while (true) {
for (int i = 0; i < CB_ARRAY_NUM; i++)
{
if (cb_array[i].is_called()) {
cb_array[i].reset();
const int64_t log_size = (log_size_ > 0)? log_size_: ObRandom::rand(100, 1024);
cb_array[i].log_size_ = log_size;
ret = log_handler_.append(BUFFER, log_size, SCN::min_scn(), true, &cb_array[i], lsn, log_scn);
if (OB_SUCCESS != ret) {
(void) cb_array[i].on_success();
}
}
}
}
while (true) {
sleep(10);
}
return ret;
}
int MockAppendCb::on_success()
{
ATOMIC_STORE(&is_called_, true);
return OB_SUCCESS;
}
int ObLogClient::submit_log(const common::ObAddr &client_addr, const int64_t client_id, const palf::LogWriteBuf &log_buf)
{
int ret = OB_SUCCESS;
LSN lsn;
SCN log_scn;
const char *buf = log_buf.write_buf_[0].buf_;
const int64_t buf_len = log_buf.write_buf_[0].buf_len_;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else if (OB_FAIL(remote_append_cb_list[client_id].pre_submit(client_addr, palf_id_, client_id))) {
// CLOG_LOG(WARN, "append_cb init failed", K(ret));
} else if (OB_FAIL(log_handler_.append(buf, buf_len, SCN::min_scn(), true, &(remote_append_cb_list[client_id]), lsn, log_scn))) {
CLOG_LOG(WARN, "append failed", K(ret));
}
return OB_SUCCESS;
}
void LogRemoteClient::has_returned()
{
const int64_t tmp_rt = common::ObTimeUtility::current_time() - last_submit_ts_;
ATOMIC_STORE(&is_returned_, true);
ObThreadCondGuard guard(cond_);
cond_.signal();
ATOMIC_INC(&c_append_cnt);
ATOMIC_FAA(&c_rt, tmp_rt);
if (REACH_TIME_INTERVAL(1000 *1000)) {
int64_t l_append_cnt = ATOMIC_LOAD(&c_append_cnt);
if (l_append_cnt == 0) l_append_cnt = 1;
int64_t l_rt = ATOMIC_LOAD(&c_rt);
ATOMIC_STORE(&c_append_cnt, 0);
ATOMIC_STORE(&c_rt, 0);
CLOG_LOG_RET(ERROR, OB_SUCCESS, "result:", K(l_append_cnt), K(l_rt), "avg_rt", l_rt/l_append_cnt);
}
}
} // end namespace palfcluster
} // end namespace oceanbase

314
mittest/palf_cluster/logservice/ob_log_client.h Normal file
View File

@ -0,0 +1,314 @@
// Copyright (c) 2021 OceanBase
// OceanBase is licensed under Mulan PubL v2.
// You can use this software according to the terms and conditions of the Mulan PubL v2.
// You may obtain a copy of Mulan PubL v2 at:
// http://license.coscl.org.cn/MulanPubL-2.0
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PubL v2 for more details.
#ifndef OCEANBASE_PALF_CLUSTER_LOG_CLIENT_H_
#define OCEANBASE_PALF_CLUSTER_LOG_CLIENT_H_
#include "share/ob_thread_pool.h"
#include "share/ob_occam_timer.h"
#include "lib/hash/ob_linear_hash_map.h" // ObLinearHashMap
#include "lib/lock/ob_tc_rwlock.h" // RWLock
#include "common/ob_member_list.h" // common::ObMemberList
#include "storage/ob_locality_manager.h" // ObLocalityManager
#include "logservice/palf/palf_handle_impl.h"
#include "logservice/ob_log_handler.h"
#include "logservice/ob_log_service.h"
#include "mittest/palf_cluster/rpc/palf_cluster_rpc_req.h" // ProbeMsg
#include "mittest/palf_cluster/rpc/palf_cluster_rpc_proxy.h" // ProbeMsg
namespace oceanbase
{
namespace palf
{
class PalfEnv;
}
namespace obrpc
{
class PalfClusterRpcProxy;
}
namespace palfcluster
{
class LogService;
class MockAppendCb : public logservice::AppendCb
{
public:
MockAppendCb()
: log_size_(0),
is_called_(true)
{ }
int on_success() override final;
int on_failure() override final
{
ATOMIC_STORE(&is_called_, true);
return OB_SUCCESS;
}
void reset()
{
ATOMIC_STORE(&is_called_, false);
}
bool is_called() const
{
return ATOMIC_LOAD(&is_called_);
}
public:
int64_t log_size_;
bool is_called_;
};
class MockRemoteAppendCb : public logservice::AppendCb
{
public:
MockRemoteAppendCb()
: rpc_proxy_(NULL),
client_addr_(),
is_called_(true)
{ }
~MockRemoteAppendCb()
{
rpc_proxy_ = NULL;
client_addr_.reset();
}
int init(obrpc::PalfClusterRpcProxy *rpc_proxy,
const common::ObAddr &self)
{
int ret = OB_SUCCESS;
if (rpc_proxy == NULL) {
ret = OB_INVALID_ARGUMENT;
} else {
rpc_proxy_ = rpc_proxy;
self_ = self;
}
return ret;
}
int pre_submit(const common::ObAddr &client_addr,
const int64_t palf_id,
const int64_t client_id)
{
int ret = OB_SUCCESS;
if (is_called()) {
reset();
client_addr_ = client_addr;
palf_id_ = palf_id;
client_id_ = client_id;
} else {
ret = OB_EAGAIN;
}
return ret;
}
int on_success() override final
{
int ret = OB_SUCCESS;
ATOMIC_STORE(&is_called_, true);
AppendCb::reset();
if (OB_NOT_NULL(rpc_proxy_) && client_addr_.is_valid()) {
// notify called
const int64_t RPC_TIMEOUT_US = 1 * 1000 * 1000;
SubmitLogCmdResp resp(self_, palf_id_, client_id_);
static obrpc::ObLogRpcCB<obrpc::OB_LOG_SUBMIT_LOG_CMD_RESP> cb; \
if (OB_FAIL(rpc_proxy_->to(client_addr_).timeout(RPC_TIMEOUT_US).trace_time(true).
max_process_handler_time(RPC_TIMEOUT_US).by(MTL_ID()).send_submit_log_resp(resp, &cb))) {
CLOG_LOG(ERROR, "send_submit_log_resp failed", KR(ret), K(resp));
}
}
return OB_SUCCESS;
}
int on_failure() override final
{
ATOMIC_STORE(&is_called_, true);
if (OB_NOT_NULL(rpc_proxy_) && client_addr_.is_valid()) {
//notify called
}
return OB_SUCCESS;
}
void reset()
{
ATOMIC_STORE(&is_called_, false);
}
bool is_called() const
{
return ATOMIC_LOAD(&is_called_);
}
obrpc::PalfClusterRpcProxy *rpc_proxy_;
common::ObAddr client_addr_;
common::ObAddr self_;
int64_t palf_id_;
int64_t client_id_;
bool is_called_;
};
class LogRemoteClient
{
public:
LogRemoteClient()
: thread_(),
th_id_(0),
log_size_(0),
rpc_proxy_(NULL),
self_(),
dst_(),
palf_id_(0),
cond_(),
is_returned_(true),
last_submit_ts_(0),
avg_rt_(-1),
is_inited_(false) {}
int init_and_create(const int64_t th_id,
const int64_t log_size,
obrpc::PalfClusterRpcProxy *rpc_proxy,
const common::ObAddr &self,
const common::ObAddr &dst,
const int64_t palf_id)
{
int ret = OB_SUCCESS;
cond_.init(ObWaitEventIds::REBALANCE_TASK_MGR_COND_WAIT);
if (IS_INIT) {
ret = OB_INIT_TWICE;
} else if (th_id < 0 || log_size <= 0 ||
rpc_proxy == NULL || false == self.is_valid()) {
ret = OB_INVALID_ARGUMENT;
} else if (0 != pthread_create(&thread_, NULL, do_submit, this)){
PALF_LOG(ERROR, "create thread fail", K(thread_));
} else {
th_id_ = th_id;
log_size_ = log_size;
rpc_proxy_ = rpc_proxy;
self_ = self;
dst_ = dst;
palf_id_ = palf_id;
}
return ret;
}
void join()
{
pthread_join(thread_, NULL);
}
static void* do_submit(void *arg)
{
int ret = OB_SUCCESS;
const int64_t NBYTES = 40000;
const int64_t RPC_TIMEOUT_US = 1 * 1000 * 1000;
char BUFFER[NBYTES];
memset(BUFFER, 'a', NBYTES);
LogRemoteClient *client = static_cast<LogRemoteClient *>(arg);
palf::LogWriteBuf write_buf;
write_buf.push_back(BUFFER, client->log_size_);
SubmitLogCmd req(client->self_, client->palf_id_, client->th_id_, write_buf);
while (true) {
// const bool is_timeout = (common::ObTimeUtility::current_time() - client->last_submit_ts_) > 500 * 1000;
// if (client->can_submit())
static obrpc::ObLogRpcCB<obrpc::OB_LOG_SUBMIT_LOG_CMD> cb;
if (OB_FAIL(client->rpc_proxy_->to(client->dst_).timeout(RPC_TIMEOUT_US).trace_time(true). \
max_process_handler_time(RPC_TIMEOUT_US).by(MTL_ID()).send_submit_log_cmd(req, &cb))) {
PALF_LOG(WARN, "send_submit_log_cmd fail", K(req));
} else {
client->has_submit();
ObThreadCondGuard guard(client->cond_);
client->cond_.wait();
}
}
return NULL;
}
bool can_submit() const
{
return ATOMIC_LOAD(&is_returned_);
}
void has_submit()
{
ATOMIC_STORE(&is_returned_, false);
last_submit_ts_ = common::ObTimeUtility::current_time();
}
void has_returned();
public:
pthread_t thread_;
int64_t th_id_;
int64_t log_size_;
obrpc::PalfClusterRpcProxy *rpc_proxy_;
common::ObAddr self_;
common::ObAddr dst_;
int64_t palf_id_;
mutable common::ObThreadCond cond_;
bool is_returned_;
int64_t last_submit_ts_;
int64_t avg_rt_;
bool is_inited_;
};
class ObLogClient
{
public:
ObLogClient();
virtual ~ObLogClient();
int init(const common::ObAddr &self,
const int64_t palf_id,
obrpc::PalfClusterRpcProxy *rpc_proxy,
palfcluster::LogService *log_service);
void destroy();
int create_palf_replica(const common::ObMemberList &member_list,
const int64_t replica_num,
const int64_t leader_idx);
int submit_append_log_task(const int64_t thread_num, const int64_t log_size);
int submit_log(const common::ObAddr &client_addr, const int64_t client_id, const palf::LogWriteBuf &log_buf);
int do_submit();
share::ObLSID get_ls_id() const { return share::ObLSID(palf_id_); }
logservice::ObLogHandler *get_log_handler() { return &log_handler_;}
TO_STRING_KV(K_(palf_id));
private:
static const int64_t MAX_THREAD_NUM = 10;
static const int64_t REMOTE_APPEND_CB_CNT = 4000;
MockRemoteAppendCb remote_append_cb_list[REMOTE_APPEND_CB_CNT];
common::ObAddr self_;
int64_t palf_id_;
obrpc::PalfClusterRpcProxy *rpc_proxy_;
logservice::ObLogHandler log_handler_;
common::ObSpinLock lock_;
int64_t log_size_;
logservice::coordinator::ElectionPriorityImpl election_priority_;
int64_t total_num_;
palfcluster::LogService *log_service_;
int64_t client_number_;
int64_t worker_number_;
bool is_inited_;
};
typedef common::ObLinearHashMap<share::ObLSID, palfcluster::ObLogClient*> LogClientMap;
} // palfcluster
} // oceanbase
#endif

747
mittest/palf_cluster/logservice/role_coordinator.cpp Normal file
View File

@ -0,0 +1,747 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "role_coordinator.h"
#include "common/ob_role.h"
#include "lib/ob_define.h"
#include "lib/ob_errno.h"
#include "lib/time/ob_time_utility.h"
#include "lib/utility/ob_macro_utils.h"
#include "lib/thread/thread_mgr.h"
#include "logservice/palf/log_define.h"
#include "share/ob_errno.h"
#include "share/ob_ls_id.h"
#include "share/ob_thread_define.h"
#include "share/rc/ob_tenant_base.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/tx_storage/ob_ls_handle.h"
#include "share/ob_occam_time_guard.h"
namespace oceanbase
{
using namespace common;
using namespace palf;
using namespace logservice;
using namespace share;
namespace palfcluster
{
RoleChangeEvent::RoleChangeEvent(const RoleChangeEventType &event_type,
const share::ObLSID &ls_id) : event_type_(event_type),
ls_id_(ls_id)
{
}
RoleChangeEvent::RoleChangeEvent(const RoleChangeEventType &event_type,
const share::ObLSID &ls_id,
const common::ObAddr &dst_addr) : event_type_(event_type),
ls_id_(ls_id),
dst_addr_(dst_addr)
{
}
bool RoleChangeEvent::is_valid() const
{
return RoleChangeEventType::INVALID_RC_EVENT_TYPE != event_type_
&& false != ls_id_.is_valid();
}
void RoleChangeEvent::reset()
{
event_type_ = RoleChangeEventType::INVALID_RC_EVENT_TYPE;
ls_id_.reset();
dst_addr_.reset();
}
bool RoleChangeEvent::operator==(const RoleChangeEvent &rhs) const
{
// for change leader event, we just check 'ls_id'.
return event_type_ == rhs.event_type_ && ls_id_ == rhs.ls_id_;
}
RoleChangeEventSet::RoleChangeEventSet()
{}
RoleChangeEventSet::~RoleChangeEventSet()
{}
int RoleChangeEventSet::insert(const RoleChangeEvent &event)
{
int ret = OB_SUCCESS;
int64_t free_idx = -1;
ObSpinLockGuard guard(lock_);
for (int64_t i = 0; i < MAX_ARRAY_SIZE; i++) {
if (event == events_[i]) {
ret = OB_ENTRY_EXIST;
}
}
for (int64_t i = 0; i < MAX_ARRAY_SIZE && -1 == free_idx && OB_SUCC(ret); i++) {
if (false == events_[i].is_valid()) {
free_idx = i;
}
}
if (OB_ENTRY_EXIST == ret) {
} else if (-1 != free_idx) {
events_[free_idx] = event;
} else {
ret = OB_SIZE_OVERFLOW;
}
CLOG_LOG(INFO, "insert event into set success", K(ret), K(event), K(free_idx));
return ret;
}
int RoleChangeEventSet::remove(const RoleChangeEvent &event)
{
int ret = OB_SUCCESS;
int64_t delete_idx = -1;
ObSpinLockGuard guard(lock_);
for (int64_t i = 0; i < MAX_ARRAY_SIZE && -1 == delete_idx; i++) {
if (event == events_[i]) {
delete_idx = i;
}
};
if (-1 != delete_idx) {
events_[delete_idx].reset();
} else {
ret = OB_ENTRY_NOT_EXIST;
}
CLOG_LOG(INFO, "remove slog from set success", K(ret), K(delete_idx), K(event));
return ret;
}
RoleCoordinator::RoleCoordinator() : log_client_map_(NULL),
apply_service_(NULL),
tg_id_(-1),
is_inited_(false)
{
}
RoleCoordinator::~RoleCoordinator()
{
if (IS_INIT) {
destroy();
}
}
int RoleCoordinator::init(palfcluster::LogClientMap *log_client_map,
logservice::ObLogApplyService *apply_service)
{
int ret = OB_SUCCESS;
const int tg_id = lib::TGDefIDs::RCService;
if (IS_INIT) {
ret = OB_INIT_TWICE;
} else if (OB_ISNULL(log_client_map) || OB_ISNULL(apply_service)) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(WARN, "invalid argument", K(ret), KP(apply_service));
} else if (OB_FAIL(TG_CREATE_TENANT(tg_id, tg_id_))) {
CLOG_LOG(WARN, "RoleCoordinator TG_CREATE failed", K(ret));
} else {
apply_service_ = apply_service;
log_client_map_ = log_client_map;
is_inited_ = true;
CLOG_LOG(INFO, "RoleCoordinator init success", K(ret), K(tg_id_), KP(apply_service));
}
return ret;
}
int RoleCoordinator::start()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else if (OB_FAIL(TG_SET_HANDLER_AND_START(tg_id_, *this))) {
CLOG_LOG(WARN, "RoleCoordinator start failed", K(ret), K(tg_id_));
} else {
CLOG_LOG(INFO, "RoleCoordinator start success", K(ret), K(tg_id_));
}
return ret;
}
void RoleCoordinator::wait()
{
if (IS_INIT) {
TG_STOP(tg_id_);
TG_WAIT(tg_id_);
}
CLOG_LOG(INFO, "RoleCoordinator wait finish", K(tg_id_));
}
void RoleCoordinator::stop()
{
if (IS_INIT) {
TG_STOP(tg_id_);
}
CLOG_LOG(INFO, "RoleCoordinator stop finish", K(tg_id_));
}
void RoleCoordinator::destroy()
{
if (IS_INIT) {
(void)stop();
(void)wait();
TG_DESTROY(tg_id_);
is_inited_ = false;
tg_id_ = -1;
apply_service_ = NULL;
CLOG_LOG(INFO, "RoleCoordinator destroy success");
}
}
void RoleCoordinator::handle(void *task)
{
int ret = OB_SUCCESS;
// When role chage service hang exceeds 30 seconds, we think there is dead lock in 'handle_role_change_event_',
// TIMEGUARD will pring lbt().
TIMEGUARD_INIT(CLOG, 30_s, 30_s);
RoleChangeEvent *event = reinterpret_cast<RoleChangeEvent*>(task);
const int64_t ls_id = event->ls_id_.id();
const int64_t start_ts = ObTimeUtility::current_time();
RetrySubmitRoleChangeEventCtx retry_ctx;
CLOG_LOG(INFO, "begin handle_role_change_event_", "sequence:", start_ts, KPC(event));
if (NULL == event) {
CLOG_LOG(WARN, "unexpected error, task is nullptr", KP(event));
} else if (OB_FAIL(handle_role_change_event_(*event, retry_ctx))) {
CLOG_LOG(WARN, "handle_role_change_event_ failed", K(ret), KPC(event), K(retry_ctx));
} else {
CLOG_LOG(INFO, "end handle_role_change_event_", "sequence:", start_ts, KPC(event));
}
if (NULL != event) {
OB_DELETE(RoleChangeEvent, "RCService", event);
}
if (retry_ctx.need_retry() && OB_FAIL(on_role_change(ls_id))) {
CLOG_LOG(WARN, "retry submit role change event failed", K(ls_id), K(retry_ctx));
}
}
int RoleCoordinator::on_role_change(const int64_t id)
{
int ret = OB_SUCCESS;
share::ObLSID ls_id(id);
RoleChangeEvent event(RoleChangeEventType::ROLE_CHANGE_CB_EVENT_TYPE, ls_id);
// TODO by runlin: if task queue has been full, push task will be failed, the role change event
// will be lost.
if (OB_FAIL(submit_role_change_event_(event))) {
CLOG_LOG(WARN, "submit_role_change_event_ failed", K(ret), K(event));
} else {
CLOG_LOG(INFO, "on_role_change success", K(ret), K(event));
}
return ret;
}
int RoleCoordinator::on_need_change_leader(const int64_t ls_id, const common::ObAddr &dst_addr)
{
int ret = OB_SUCCESS;
RoleChangeEvent event(RoleChangeEventType::CHANGE_LEADER_EVENT_TYPE, share::ObLSID(ls_id), dst_addr);
if (OB_FAIL(submit_role_change_event_(event))) {
CLOG_LOG(WARN, "submit_role_change_event_ failed", K(ret), K(event));
} else {
CLOG_LOG(INFO, "change_leader success", K(ret), K(event));
}
return ret;
}
int RoleCoordinator::submit_role_change_event_(const RoleChangeEvent &event)
{
int ret = OB_SUCCESS;
if(OB_FAIL(rc_set_.insert(event)) && OB_ENTRY_EXIST != ret) {
CLOG_LOG(ERROR, "insert into rc_set failed", K(ret), K(event));
} else if (OB_ENTRY_EXIST == ret) {
CLOG_LOG(INFO, "repeat role change event, filter it", K(ret), K(event));
ret = OB_SUCCESS;
} else if (OB_FAIL(push_event_into_queue_(event))) {
CLOG_LOG(WARN, "push_event_into_queue_ failed", K(ret), K(event));
} else {
CLOG_LOG(INFO, "submit_role_change_event_ success", K(ret), K(event));
}
return ret;
}
// TODO: use poll to avoid alloc memory failed.
int RoleCoordinator::push_event_into_queue_(const RoleChangeEvent &event)
{
int ret = OB_SUCCESS;
RoleChangeEvent *rc_event = NULL;
int64_t warn_time = OB_INVALID_TIMESTAMP;
do {
if (NULL == (rc_event =
MTL_NEW(RoleChangeEvent, "RCService", event.event_type_, event.ls_id_, event.dst_addr_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
if (palf_reach_time_interval(1 * 1000 * 1000, warn_time)) {
CLOG_LOG(WARN, "allocate memory failed", K(ret), K(event));
}
usleep(1 * 1000);
} else {
ret = OB_SUCCESS;
}
} while(OB_FAIL(ret));
if (OB_FAIL(TG_PUSH_TASK(tg_id_, rc_event))) {
CLOG_LOG(WARN, "ObRoleChangeTask push task failed", K(ret), K(event));
}
if (OB_FAIL(ret) && NULL != rc_event) {
MTL_DELETE(RoleChangeEvent, "RCService", rc_event);
}
return ret;
}
int RoleCoordinator::handle_role_change_event_(const RoleChangeEvent &event,
RetrySubmitRoleChangeEventCtx &retry_ctx)
{
int ret = OB_SUCCESS;
ObLogClient *ls = nullptr;
AccessMode curr_access_mode;
int64_t unused_mode_version;
OB_ASSERT(OB_SUCCESS == rc_set_.remove(event));
if (false == event.is_valid()) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(WARN, "invalid argument", K(event));
} else if (OB_FAIL(log_client_map_->get(event.ls_id_, ls) || OB_ISNULL(ls))) {
CLOG_LOG(ERROR, "not exist loghandler", K(ret), K(event.ls_id_), KP(ls));
ret = OB_ERR_UNEXPECTED;
} else if (OB_FAIL(ls->get_log_handler()->get_access_mode(unused_mode_version, curr_access_mode))) {
CLOG_LOG(WARN, "ObLogHandler get_access_mode failed", K(ret));
} else {
switch (event.event_type_) {
case RoleChangeEventType::CHANGE_LEADER_EVENT_TYPE:
CLOG_LOG(INFO, "begin change leader", K(curr_access_mode), K(event), KPC(ls));
if (is_append_mode(curr_access_mode)
&& OB_FAIL(handle_change_leader_event_for_log_handler_(event.dst_addr_, ls))) {
CLOG_LOG(WARN, "ObLogHandler change leader failed", K(ret), K(event), KPC(ls));
}
CLOG_LOG(INFO, "end change leader", K(ret), K(curr_access_mode), K(event), KPC(ls));
break;
case RoleChangeEventType::ROLE_CHANGE_CB_EVENT_TYPE:
CLOG_LOG(INFO, "begin log handler role change", K(curr_access_mode), K(event), KPC(ls));
if (OB_FAIL(handle_role_change_cb_event_for_log_handler_(curr_access_mode, ls, retry_ctx))) {
CLOG_LOG(WARN, "handle_role_change_cb_event_for_log_handler_ failed", K(ret),
K(curr_access_mode), KPC(ls));
}
CLOG_LOG(INFO, "end log handler role change", K(ret), K(curr_access_mode), K(event), KPC(ls), K(retry_ctx));
break;
default:
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(WARN, "unexpected role change event type", K(ret));
}
}
return ret;
}
int RoleCoordinator::handle_role_change_cb_event_for_log_handler_(
const AccessMode &curr_access_mode,
ObLogClient*ls,
RetrySubmitRoleChangeEventCtx &retry_ctx)
{
int ret = OB_SUCCESS;
const bool log_handler_is_offline = ls->get_log_handler()->is_offline();
// If log handler is offline, need execute LEADER_2_FOLLOWER or FOLLOWER_2_FOLLOWER
//
// when access mode is APPEND, log_handler need execute leader to follower or
// follower to leader. otherwise, only need execute leader to follower or follower
// to follower, therefore, we set 'need_transform_by_access_mode' to false when
// 'curr_access_mode' is APPEND.
const bool only_need_change_to_follower = !is_append_mode(curr_access_mode) || log_handler_is_offline;
RoleChangeOptType opt_type;
ObRole curr_role = ObRole::INVALID_ROLE;
ObRole new_role = ObRole::INVALID_ROLE;
bool is_pending_state = false;
int64_t curr_proposal_id = -1;
int64_t new_proposal_id = -1;
if (OB_FAIL(ls->get_log_handler()->prepare_switch_role(curr_role,
curr_proposal_id, new_role, new_proposal_id, is_pending_state))) {
CLOG_LOG(WARN, "ObLogHandler prepare_switch_role failed", K(ret), K(curr_role), K(curr_proposal_id),
K(new_role), K(new_proposal_id));
} else if (false == need_execute_role_change(curr_proposal_id, curr_role, new_proposal_id,
new_role, is_pending_state, log_handler_is_offline)) {
CLOG_LOG(INFO, "no need change role", K(ret), K(is_pending_state), K(curr_role), K(curr_proposal_id),
K(new_role), K(new_proposal_id), K(is_pending_state), K(log_handler_is_offline));
} else if (FALSE_IT(opt_type = get_role_change_opt_type_(curr_role, new_role, only_need_change_to_follower))) {
} else {
switch (opt_type) {
// leader -> follower
case RoleChangeOptType::LEADER_2_FOLLOWER:
if (OB_FAIL(switch_leader_to_follower_forcedly_(new_proposal_id, ls))) {
CLOG_LOG(WARN, "switch_leader_to_follower_forcedly_ failed", K(ret), K(curr_role),
K(curr_proposal_id), K(new_role), K(curr_access_mode), K(new_proposal_id));
}
break;
// follower -> follower
case RoleChangeOptType::FOLLOWER_2_LEADER:
if (OB_FAIL(switch_follower_to_leader_(new_proposal_id, ls, retry_ctx))) {
CLOG_LOG(WARN, "switch_follower_to_leader_ failed", K(ret), K(curr_role),
K(curr_proposal_id), K(new_role), K(curr_access_mode), K(new_proposal_id));
}
break;
// leader -> leader
case RoleChangeOptType::LEADER_2_LEADER:
if (OB_FAIL(switch_leader_to_leader_(new_proposal_id, curr_proposal_id, ls, retry_ctx))) {
CLOG_LOG(WARN, "switch_leader_to_leader_ failed", K(ret), K(curr_role),
K(curr_proposal_id), K(new_role), K(curr_access_mode), K(new_proposal_id));
}
break;
// follower -> follower
case RoleChangeOptType::FOLLOWER_2_FOLLOWER:
if (OB_FAIL(switch_follower_to_follower_(new_proposal_id, ls))) {
CLOG_LOG(WARN, "switch_follower_to_follower_ failed", K(ret), K(curr_role),
K(curr_proposal_id), K(new_role), K(curr_access_mode), K(new_proposal_id));
}
break;
default:
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(ERROR, "unexpected error, can not handle role change", K(ret), K(curr_role),
K(curr_proposal_id), K(new_role), K(new_proposal_id), KPC(ls));
}
}
return ret;
}
int RoleCoordinator::handle_change_leader_event_for_log_handler_(
const common::ObAddr &dst_addr,
ObLogClient*ls)
{
int ret = OB_SUCCESS;
ObRole curr_role = ObRole::INVALID_ROLE;
ObRole new_role = ObRole::INVALID_ROLE;
bool is_pending_state = false;
int64_t curr_proposal_id = -1;
int64_t new_proposal_id = -1;
if (OB_FAIL(ls->get_log_handler()->prepare_switch_role(curr_role,
curr_proposal_id, new_role, new_proposal_id, is_pending_state))) {
CLOG_LOG(WARN, "ObLogHandler prepare_switch_role failed", K(ret), K(curr_role), K(curr_proposal_id),
K(new_role), K(new_proposal_id));
} else if (true == is_pending_state
|| curr_proposal_id != new_proposal_id || LEADER != curr_role || LEADER != new_role) {
// when log handler is not LEDAER, we also need execute change_leader_to, otherwise, the leader can not be changed by election.
ls->get_log_handler()->change_leader_to(dst_addr);
CLOG_LOG(INFO, "no need execute switch_leader_to_follower_gracefully, change leader directlly",
K(ret), K(is_pending_state), K(curr_proposal_id), K(new_proposal_id), K(curr_role), K(new_role));
} else if (OB_FAIL(switch_leader_to_follower_gracefully_(new_proposal_id, curr_proposal_id,
dst_addr, ls))) {
CLOG_LOG(WARN, "switch_leader_to_follower_gracefully_ failed", K(ret), KPC(ls),
K(curr_role), K(curr_proposal_id), K(new_role), K(new_proposal_id));
} else {
CLOG_LOG(INFO, "handle_change_leader_event_for_log_handler_ success", K(ret), K(curr_role),
K(curr_proposal_id), K(new_role), K(new_proposal_id), K(dst_addr));
}
return ret;
}
int RoleCoordinator::switch_follower_to_leader_(
const int64_t new_proposal_id,
ObLogClient*ls,
RetrySubmitRoleChangeEventCtx &retry_ctx)
{
int ret = OB_SUCCESS;
const ObRole new_role = LEADER;
const share::ObLSID &ls_id = ls->get_ls_id();
palf::LSN end_lsn;
ObTimeGuard time_guard("switch_to_leader", EACH_ROLE_CHANGE_COST_MAX_TIME);
ObLogHandler *log_handler = ls->get_log_handler();
// ObRoleChangeHandler *role_change_handler = ls->get_role_change_handler();
if (OB_FAIL(log_handler->get_end_lsn(end_lsn))) {
CLOG_LOG(WARN, "get_end_lsn failed", K(ret), KPC(ls));
// NB: order is vital!!!
// We must guarantee that 'replay_service_' has replayed complete data, and before
// stop 'replay_service_', other components can not submit log.
} else if (FALSE_IT(time_guard.click("wait_replay_service_apply_done_"))
|| OB_FAIL(wait_replay_service_replay_done_(ls_id, end_lsn, WAIT_REPLAY_DONE_TIMEOUT_US))) {
if (need_retry_submit_role_change_event_(ret)) {
retry_ctx.set_retry_reason(RetrySubmitRoleChangeEventReason::WAIT_REPLAY_DONE_TIMEOUT);
} else {
CLOG_LOG(WARN, "wait_replay_service_replay_done_ failed", K(ret), K(end_lsn));
}
} else if (FALSE_IT(time_guard.click("apply_service->switch_to_leader"))
|| OB_FAIL(apply_service_->switch_to_leader(ls_id, new_proposal_id))) {
CLOG_LOG(WARN, "apply_service_ switch_to_leader failed", K(ret), K(new_role), K(new_proposal_id));
// } else if (FALSE_IT(time_guard.click("replay_service->switch_to_leader"))
// || OB_FAIL(replay_service_->switch_to_leader(ls_id))) {
} else if (FALSE_IT(log_handler->switch_role(new_role, new_proposal_id))) {
CLOG_LOG(WARN, "ObLogHandler switch role failed", K(ret), K(new_role), K(new_proposal_id));
// } else if (FALSE_IT(time_guard.click("role_change_handler->switch_to_leader"))
// || OB_FAIL(role_change_handler->switch_to_leader())) {
// CLOG_LOG(WARN, "ObRoleChangeHandler switch_to_leader failed", K(ret), KPC(ls));
} else {
CLOG_LOG(INFO, "switch_follower_to_leader_ success", K(ret), KPC(ls));
}
if (OB_FAIL(ret) && !retry_ctx.need_retry()) {
log_handler->revoke_leader();
CLOG_LOG(WARN, "switch_follower_to_leader_ failed", K(ret), KPC(ls));
}
return ret;
}
int RoleCoordinator::switch_leader_to_follower_forcedly_(
const int64_t new_proposal_id,
ObLogClient*ls)
{
int ret = OB_SUCCESS;
const ObRole new_role = FOLLOWER;
const share::ObLSID &ls_id = ls->get_ls_id();
ObLogHandler *log_handler = ls->get_log_handler();
// ObRoleChangeHandler *role_change_handler = ls->get_role_change_handler();
palf::LSN end_lsn;
ObTimeGuard time_guard("switch_leader_to_follower_forcedly_", EACH_ROLE_CHANGE_COST_MAX_TIME);
// Why need wait_apply_sync?
//
// when we can execute 'switch_to_follower_forcedly', means that there is no possibility to submit log via log handler successfully.
// however, the flying callback may have not been pushed into apply service, and then, 'switch_to_follower' will be executed, for trans,
// if the callback be executed after 'switch_to_follower', will cause abort.
if (OB_FAIL(apply_service_->wait_append_sync(ls_id))) {
CLOG_LOG(WARN, "wait_apply_sync failed", K(ret), K(ls_id));
} else if (FALSE_IT(time_guard.click("apply_service->wait_apply_sync"))
|| OB_FAIL(apply_service_->switch_to_follower(ls_id))) {
CLOG_LOG(WARN, "apply_service_ switch_to_follower failed", K(ret), K(new_role), K(new_proposal_id));
} else if (FALSE_IT(time_guard.click("apply_service->switch_to_follower"))
|| OB_FAIL(wait_apply_service_apply_done_(ls_id, end_lsn))) {
CLOG_LOG(WARN, "wait_apply_service_apply_done_ failed", K(ret), K(end_lsn));
} else {
time_guard.click("wait_apply_service_apply_done_");
// role_change_handler->switch_to_follower_forcedly();
// NB: order is vital
// We must guarantee that this replica will not submit any logs after switch_role.
log_handler->switch_role(new_role, new_proposal_id);
// NB: in case of leader reovke, do we no need retry.
// (void)replay_service_->switch_to_follower(ls_id, end_lsn);
CLOG_LOG(INFO, "switch_leader_to_follower_forcedly_ success", K(ret), KPC(ls));
}
if (OB_FAIL(ret)) {
log_handler->revoke_leader();
CLOG_LOG(WARN, "switch_leader_to_follower_forcedly_ failed", K(ret), K(new_proposal_id), K(new_role));
}
return ret;
}
int RoleCoordinator::switch_leader_to_follower_gracefully_(
const int64_t new_proposal_id,
const int64_t curr_proposal_id,
const common::ObAddr &dst_addr,
ObLogClient*ls)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
const ObRole new_role = FOLLOWER;
const share::ObLSID &ls_id = ls->get_ls_id();
ObLogHandler *log_handler = ls->get_log_handler();
// ObRoleChangeHandler *role_change_handler = ls->get_role_change_handler();
LSN end_lsn;
ObTimeGuard time_guard("switch_leader_to_follower_gracefully_", EACH_ROLE_CHANGE_COST_MAX_TIME);
// 1. OB_SUCCESS means execute transaction successfully, we need execute follow steps.
// 2. OB_LS_NEED_REVOKE means the transaction execute failed, and can't been rollback, need revoke LS.
// 3. OTHERS, switch_to_follower_gracefully failed, and 'role_change_handler' has rollback success,
// no need to execute follow steps.
// if (FALSE_IT(time_guard.click("role_change_handler->switch_to_follower_gracefully"))
// || OB_SUCCESS != (tmp_ret = role_change_handler->switch_to_follower_gracefully())) {
// CLOG_LOG(WARN, "switch_to_follower_gracefully failed, need revoke leader", K(tmp_ret),
// K(new_role), K(new_proposal_id), K(dst_addr));
// NB: order is vital!!!
// we must ensure that the 'end_lsn' provid by 'apply_service_' is correctly.
// just switch_role to follower firstly, avoid sync log failed because palf has changed leader.
if (FALSE_IT(log_handler->switch_role(new_role, curr_proposal_id))) {
// apply service will not update end_lsn after switch_to_follower, so wait apply done first here
} else if (FALSE_IT(time_guard.click("wait_apply_service_apply_done_when_change_leader_"))
|| OB_FAIL(wait_apply_service_apply_done_when_change_leader_(log_handler, curr_proposal_id, ls_id, end_lsn))) {
CLOG_LOG(WARN, "wait_apply_service_apply_done_when_change_leader_ failed", K(ret),
K(new_role), K(new_proposal_id), K(dst_addr));
// wait apply service done my fail, we need :
// 1. switch log handler to origin status.
// 2. resume role change handler
log_handler->switch_role(LEADER, curr_proposal_id);
// if (OB_FAIL(role_change_handler->resume_to_leader())) {
// CLOG_LOG(WARN, "resume to leader failed", K(ret), KPC(ls));
// }
// NB: the following steps mustn't be failed.
} else if (FALSE_IT(time_guard.click("apply_service->switch_to_follower"))
|| OB_FAIL(apply_service_->switch_to_follower(ls_id))) {
CLOG_LOG(WARN, "apply_service_ switch_to_follower failed", K(ret), K(new_role), K(new_proposal_id), K(dst_addr));
// } else if (FALSE_IT(time_guard.click("replay_service->switch_to_follower"))
// || OB_FAIL(replay_service_->switch_to_follower(ls_id, end_lsn))) {
// CLOG_LOG(WARN, "replay_service_ switch_to_follower failed", K(ret), KPC(ls), K(new_role), K(new_proposal_id));
// NB: execute 'change_leader_to' lastly, can make 'wait_apply_service_apply_done_when_change_leader_' finish quickly.
} else if (OB_FAIL(log_handler->change_leader_to(dst_addr))) {
CLOG_LOG(WARN, "ObLogHandler change_leader failed", K(ret), K(new_role), K(new_proposal_id), K(dst_addr));
} else {
CLOG_LOG(INFO, "switch_to_follower_gracefully success", K(ret), K(new_role), K(new_proposal_id), K(dst_addr));
}
if (OB_FAIL(ret) || OB_LS_NEED_REVOKE == tmp_ret) {
log_handler->revoke_leader();
CLOG_LOG(WARN, "switch_leader_to_follower_gracefully failed, revoke leader", K(ret), K(tmp_ret), K(dst_addr),
K(new_role), K(new_proposal_id));
ret = (OB_SUCCESS == ret ? tmp_ret : ret);
}
return ret;
}
int RoleCoordinator::switch_leader_to_leader_(
const int64_t new_proposal_id,
const int64_t curr_proposal_id,
ObLogClient*ls,
RetrySubmitRoleChangeEventCtx &retry_ctx)
{
int ret = OB_SUCCESS;
ObTimeGuard time_guard("switch_leader_to_leader", EACH_ROLE_CHANGE_COST_MAX_TIME);
if (FALSE_IT(time_guard.click("switch_leader_to_follower_forcedly_"))
|| OB_FAIL(switch_leader_to_follower_forcedly_(curr_proposal_id, ls))) {
CLOG_LOG(WARN, "switch_leader_to_leader_, switch leader to follower failed", K(ret), KPC(ls));
} else if (FALSE_IT(time_guard.click("switch_follower_to_leader_"))
|| OB_FAIL(switch_follower_to_leader_(new_proposal_id, ls, retry_ctx))) {
CLOG_LOG(WARN, "switch_follower_to_leader_ failed", K(ret), K(new_proposal_id));
} else {
CLOG_LOG(INFO, "switch_leader_to_leader_ success", K(ret), KPC(ls));
}
return ret;
}
int RoleCoordinator::switch_follower_to_follower_(const int64_t new_proposal_id, ObLogClient*ls)
{
int ret = OB_SUCCESS;
// need update proposal_id
const share::ObLSID &ls_id = ls->get_ls_id();
ObLogHandler *log_handler = ls->get_log_handler();
(void) log_handler->switch_role(common::ObRole::FOLLOWER, new_proposal_id);
CLOG_LOG(INFO, "switch_follower_to_follower_");
return ret;
}
int RoleCoordinator::wait_replay_service_replay_done_(
const share::ObLSID &ls_id,
const palf::LSN &end_lsn,
const int64_t timeout_us)
{
int ret = OB_SUCCESS;
return ret;
}
int RoleCoordinator::wait_apply_service_apply_done_(
const share::ObLSID &ls_id,
palf::LSN &end_lsn)
{
int ret = OB_SUCCESS;
bool is_done = false;
const int64_t start_ts = ObTimeUtility::current_time();
while (OB_SUCC(ret) && false == is_done) {
if (OB_FAIL(apply_service_->is_apply_done(ls_id, is_done, end_lsn))) {
CLOG_LOG(WARN, "apply_service_ is_apply_done failed", K(ret), K(is_done), K(end_lsn));
} else if (false == is_done) {
ob_usleep(5*1000);
CLOG_LOG(WARN, "wait apply done return false, need retry", K(ls_id), K(is_done), K(end_lsn), K(start_ts));
} else {
}
}
return ret;
}
int RoleCoordinator::wait_apply_service_apply_done_when_change_leader_(
const ObLogHandler *log_handler,
const int64_t proposal_id,
const share::ObLSID &ls_id,
palf::LSN &end_lsn)
{
int ret = OB_SUCCESS;
bool is_done = false;
palf::LSN max_lsn;
common::ObRole unused_curr_role;
int64_t unused_curr_proposal_id;
common::ObRole new_role;
int64_t new_proposal_id;
bool is_pending_state = false;
while (OB_SUCC(ret) && (false == is_done || end_lsn != max_lsn)) {
if (OB_FAIL(apply_service_->is_apply_done(ls_id, is_done, end_lsn))) {
CLOG_LOG(WARN, "apply_service_ is_apply_done failed", K(ret), K(is_done), K(end_lsn));
// NB: ApplyService execute on_failure only when it's FOLLOWER, therefore ApplyService my not return apply done
// when it's LEADER, we need check the role of palf when has changed.
} else if (OB_FAIL(log_handler->get_max_lsn(max_lsn))) {
CLOG_LOG(WARN, "get_end_lsn from palf failed", K(ret), K(ls_id), K(end_lsn));
} else if (OB_FAIL(log_handler->prepare_switch_role(unused_curr_role, unused_curr_proposal_id,
new_role, new_proposal_id, is_pending_state))) {
CLOG_LOG(WARN, "failed prepare_switch_role", K(ret), K(new_role), K(proposal_id), K(ls_id));
// if palf has changed role, return OB_STATE_NOT_MATCH, change leader failed.
} else if (LEADER != new_role || proposal_id != new_proposal_id) {
ret = OB_STATE_NOT_MATCH;
CLOG_LOG(WARN, "palf has changed leader, wait_apply_service_apply_done_when_change_leader_ failed", K(ret), K(proposal_id),
K(new_proposal_id));
} else if (false == is_done || end_lsn != max_lsn) {
CLOG_LOG(INFO, "wait apply done return false, need retry", K(ls_id), K(is_done),
K(end_lsn), K(max_lsn));
ob_usleep(5*1000);
} else {
}
}
return ret;
}
RoleCoordinator::RoleChangeOptType RoleCoordinator::get_role_change_opt_type_(
const ObRole &old_role,
const ObRole &new_role,
const bool need_transform_by_access_mode) const
{
RoleChangeOptType rc_opt_type = RoleChangeOptType::INVALID_RC_OPT_TYPE;
if (FOLLOWER == old_role && LEADER == new_role) {
rc_opt_type = RoleChangeOptType::FOLLOWER_2_LEADER;
} else if (LEADER == old_role && FOLLOWER == new_role) {
rc_opt_type = RoleChangeOptType::LEADER_2_FOLLOWER;
} else if (FOLLOWER == old_role && FOLLOWER == new_role) {
rc_opt_type = RoleChangeOptType::FOLLOWER_2_FOLLOWER;
} else if (LEADER == old_role && LEADER == new_role) {
rc_opt_type = RoleChangeOptType::LEADER_2_LEADER;
} else {
}
if (true == need_transform_by_access_mode) {
if (LEADER == old_role) {
rc_opt_type = RoleChangeOptType::LEADER_2_FOLLOWER;
} else {
rc_opt_type = RoleChangeOptType::FOLLOWER_2_FOLLOWER;
}
}
return rc_opt_type;
}
// NB:
// 1. when log handler is offline, need execute role change;
// 2. when palf is not in pending:
// 1. proposal_id is not same or
// 2. role is not same.(If there are no pending logs in sliding window,
// leader to follower will not advance proposal_id)
bool RoleCoordinator::need_execute_role_change(const int64_t curr_proposal_id,
const common::ObRole curr_role,
const int64_t new_proposal_id,
const common::ObRole new_role,
const bool is_pending_state,
const bool is_offline) const
{
return is_offline
|| (!is_pending_state
&& (curr_proposal_id != new_proposal_id || curr_role != new_role));
}
bool RoleCoordinator::is_append_mode(const AccessMode &mode) const
{
return (AccessMode::APPEND == mode);
}
bool RoleCoordinator::is_raw_write_or_flashback_mode(const AccessMode &mode) const
{
return (AccessMode::RAW_WRITE == mode || AccessMode::FLASHBACK == mode ||
AccessMode::PREPARE_FLASHBACK == mode);
}
bool RoleCoordinator::need_retry_submit_role_change_event_(int ret) const
{
bool bool_ret = false;
if (OB_TIMEOUT == ret) {
bool_ret = true;
}
return bool_ret;
}
} // end namespace palfcluster
} // end namespace oceanbase

189
mittest/palf_cluster/logservice/role_coordinator.h Normal file
View File

@ -0,0 +1,189 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_PALF_CLUSTER_ROLE_CHANGE_SERVICE_
#define OCEANBASE_PALF_CLUSTER_ROLE_CHANGE_SERVICE_
#include "lib/function/ob_function.h"
#include "lib/thread/thread_mgr_interface.h"
#include "lib/utility/ob_macro_utils.h"
#include "share/ob_ls_id.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "logservice/palf/palf_options.h"
#include "logservice/ob_log_handler.h"
#include "logservice/palf/palf_callback.h"
#include "logservice/applyservice/ob_log_apply_service.h"
#include "logservice/rcservice/ob_role_change_handler.h"
#include "mittest/palf_cluster/logservice/ob_log_client.h"
namespace oceanbase
{
namespace palfcluster
{
enum class RoleChangeEventType {
INVALID_RC_EVENT_TYPE = 0,
CHANGE_LEADER_EVENT_TYPE = 1,
ROLE_CHANGE_CB_EVENT_TYPE = 2,
MAX_RC_EVENT_TYPE = 3
};
struct RoleChangeEvent {
RoleChangeEvent() { reset(); }
RoleChangeEvent(const RoleChangeEventType &event_type,
const share::ObLSID &ls_id);
RoleChangeEvent(const RoleChangeEventType &event_type,
const share::ObLSID &ls_id,
const common::ObAddr &dst_addr);
bool is_valid() const;
void reset();
bool operator == (const RoleChangeEvent &event) const;
RoleChangeEventType event_type_;
share::ObLSID ls_id_;
ObAddr dst_addr_;
TO_STRING_KV(K_(event_type), K_(ls_id), K_(dst_addr));
};
class RoleChangeEventSet {
public:
RoleChangeEventSet();
~RoleChangeEventSet();
int insert(const RoleChangeEvent &event);
int remove(const RoleChangeEvent &event);
static constexpr int64_t MAX_ARRAY_SIZE = 128;
private:
// Assumed there are sixteen log streams at most.
// 64 for normal role change events.
// 64 for leader change events.
RoleChangeEvent events_[MAX_ARRAY_SIZE];
mutable ObSpinLock lock_;
DISALLOW_COPY_AND_ASSIGN(RoleChangeEventSet);
};
class RoleCoordinator : public lib::TGTaskHandler , public palf::PalfRoleChangeCb {
public:
RoleCoordinator();
~RoleCoordinator();
int init(palfcluster::LogClientMap *log_client_map,
logservice::ObLogApplyService *apply_service);
int start();
void wait();
void stop();
void destroy();
void handle(void *task);
int on_role_change(const int64_t id) final override;
int on_need_change_leader(const int64_t ls_id, const common::ObAddr &dst_addr) final override;
private:
enum class RoleChangeOptType {
INVALID_RC_OPT_TYPE = 0,
FOLLOWER_2_LEADER = 1,
LEADER_2_FOLLOWER = 2,
FOLLOWER_2_FOLLOWER = 3,
LEADER_2_LEADER = 4,
MAX_RC_OPT_TYPE = 5
};
enum class RetrySubmitRoleChangeEventReason {
INVALID_TYPE = 0,
WAIT_REPLAY_DONE_TIMEOUT = 1,
MAX_TYPE = 2
};
class RetrySubmitRoleChangeEventCtx {
public:
RetrySubmitRoleChangeEventCtx() : reason_(RetrySubmitRoleChangeEventReason::INVALID_TYPE) {}
~RetrySubmitRoleChangeEventCtx()
{
reason_ = RetrySubmitRoleChangeEventReason::INVALID_TYPE;
}
bool need_retry() const
{
return RetrySubmitRoleChangeEventReason::WAIT_REPLAY_DONE_TIMEOUT == reason_;
}
void set_retry_reason(const RetrySubmitRoleChangeEventReason &reason)
{
reason_ = reason;
}
TO_STRING_KV(K_(reason));
private:
RetrySubmitRoleChangeEventReason reason_;
};
private:
int submit_role_change_event_(const RoleChangeEvent &event);
int push_event_into_queue_(const RoleChangeEvent &event);
int handle_role_change_event_(const RoleChangeEvent &event,
RetrySubmitRoleChangeEventCtx &retry_ctx);
int handle_role_change_cb_event_for_log_handler_(const palf::AccessMode &curr_access_mode,
ObLogClient *ls,
RetrySubmitRoleChangeEventCtx &retry_ctx);
int handle_change_leader_event_for_log_handler_(const common::ObAddr &dst_addr,
ObLogClient *ls);
// retval
// - OB_SUCCESS
// - OB_TIMEOUT, means wait replay finish timeout.
int switch_follower_to_leader_(const int64_t new_proposal_id,
ObLogClient *ls,
RetrySubmitRoleChangeEventCtx &retry_ctx);
int switch_leader_to_follower_forcedly_(const int64_t new_proposal_id,
ObLogClient *ls);
int switch_leader_to_follower_gracefully_(const int64_t new_proposal_id,
const int64_t curr_proposal_id,
const common::ObAddr &dst_addr,
ObLogClient *ls);
int switch_follower_to_follower_(const int64_t new_proposal_id, ObLogClient *ls);
int switch_leader_to_leader_(const int64_t new_proposal_id,
const int64_t curr_proposal_id,
ObLogClient *ls,
RetrySubmitRoleChangeEventCtx &retry_ctx);
// wait replay finish with timeout.
int wait_replay_service_replay_done_(const share::ObLSID &ls_id,
const palf::LSN &end_lsn,
const int64_t timeout_us);
int wait_apply_service_apply_done_(const share::ObLSID &ls_id,
palf::LSN &end_lsn);
int wait_apply_service_apply_done_when_change_leader_(const logservice::ObLogHandler *log_handler,
const int64_t proposal_id,
const share::ObLSID &ls_id,
palf::LSN &end_lsn);
bool need_execute_role_change(const int64_t curr_proposal_id,
const common::ObRole curr_role,
const int64_t new_proposal_id,
const common::ObRole new_role,
const bool is_pending_state,
const bool is_offline) const;
bool is_append_mode(const palf::AccessMode &access_mode) const;
bool is_raw_write_or_flashback_mode(const palf::AccessMode &access_mode) const;
private:
RoleChangeOptType get_role_change_opt_type_(const common::ObRole &old_role,
const common::ObRole &new_role,
const bool need_transform_by_access_mode) const;
// retry submit role change event
// NB: nowdays, we only support retry submit role change event when wait replay finished
// timeout.
bool need_retry_submit_role_change_event_(int ret) const;
public:
static const int64_t MAX_THREAD_NUM = 1;
static const int64_t MAX_RC_EVENT_TASK = 1024 * 1024;
private:
DISALLOW_COPY_AND_ASSIGN(RoleCoordinator);
private:
static constexpr int64_t EACH_ROLE_CHANGE_COST_MAX_TIME = 1 * 1000 * 1000;
static constexpr int64_t WAIT_REPLAY_DONE_TIMEOUT_US = 2 * 1000 * 1000;
palfcluster::LogClientMap *log_client_map_;
logservice::ObLogApplyService *apply_service_;
RoleChangeEventSet rc_set_;
int tg_id_;
bool is_inited_;
};
} // end namespace palfcluster
} // end namespce oceanbase
#endif

197
mittest/palf_cluster/logservice/role_coordinator_handler.cpp Normal file
View File

@ -0,0 +1,197 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "role_coordinator_handler.h"
#include "lib/container/ob_fixed_array.h"
#include "lib/lock/ob_spin_lock.h"
#include "lib/ob_define.h"
#include "share/ob_errno.h"
namespace oceanbase
{
using namespace common;
using namespace logservice;
namespace palfcluster
{
ObRoleChangeHandler::ObRoleChangeHandler(): lock_(common::ObLatchIds::RCS_LOCK),
sub_role_change_handler_arr_()
{
reset();
}
ObRoleChangeHandler::~ObRoleChangeHandler()
{
reset();
}
void ObRoleChangeHandler::reset()
{
for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE; i++) {
sub_role_change_handler_arr_[i] = NULL;
}
}
int ObRoleChangeHandler::register_handler(const ObLogBaseType &type,
ObIRoleChangeSubHandler *role_change_handler)
{
int ret = OB_SUCCESS;
ObSpinLockGuard guard(lock_);
if (false == is_valid_log_base_type(type)) {
ret = OB_INVALID_ARGUMENT;
} else {
sub_role_change_handler_arr_[type] = role_change_handler;
CLOG_LOG(INFO, "register_handler success", K(ret), K(type), KP(role_change_handler));
}
return ret;
}
void ObRoleChangeHandler::unregister_handler(const ObLogBaseType &type)
{
ObSpinLockGuard guard(lock_);
if (true == is_valid_log_base_type(type)) {
sub_role_change_handler_arr_[type] = NULL;
CLOG_LOG(INFO, "unregister_handler success", K(type));
}
}
void ObRoleChangeHandler::switch_to_follower_forcedly()
{
ObSpinLockGuard guard(lock_);
for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE; i++) {
ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[i];
char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'};
ObLogBaseType base_type = static_cast<ObLogBaseType>(i);
bool has_defined_to_string = false;
if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str,
OB_LOG_BASE_TYPE_STR_MAX_LEN)) {
has_defined_to_string = true;
}
if (NULL != handler) {
handler->switch_to_follower_forcedly();
CLOG_LOG(INFO, "leader to follower forcedly, current role change handler is",
"cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string");
}
}
}
int ObRoleChangeHandler::switch_to_leader()
{
int ret = OB_SUCCESS;
ObSpinLockGuard guard(lock_);
for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE && OB_SUCC(ret); i++) {
ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[i];
char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'};
ObLogBaseType base_type = static_cast<ObLogBaseType>(i);
bool has_defined_to_string = false;
if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str,
OB_LOG_BASE_TYPE_STR_MAX_LEN)) {
has_defined_to_string = true;
}
if (NULL == handler) {
} else if (OB_FAIL(handler->switch_to_leader())) {
CLOG_LOG(WARN, "switch_to_leader failed", K(ret), KP(handler), K(i),
"cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string");
} else {
CLOG_LOG(INFO, "follower to leader, current role change handler is",
"cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string");
}
}
return ret;
}
int ObRoleChangeHandler::switch_to_follower_gracefully()
{
int ret = OB_SUCCESS;
ObSpinLockGuard guard(lock_);
int64_t cursor = 0;
const int64_t count = ObLogBaseType::MAX_LOG_BASE_TYPE;
while (cursor < count && OB_SUCC(ret)) {
char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'};
ObLogBaseType base_type = static_cast<ObLogBaseType>(cursor);
bool has_defined_to_string = false;
if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str,
OB_LOG_BASE_TYPE_STR_MAX_LEN)) {
has_defined_to_string = true;
}
ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[cursor];
if (NULL == handler) {
cursor++;
} else if (OB_FAIL(handler->switch_to_follower_gracefully()) && OB_LS_NEED_REVOKE != ret) {
CLOG_LOG(WARN, "switch_to_follower_gracefully failed, need resume other sub modules", K(ret),
KP(handler), K(cursor),
"cursor", cursor, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string");
// NB: resume_leader failed, need revoke leader.
} else if (OB_LS_NEED_REVOKE == ret) {
CLOG_LOG(WARN, "ObIRoleChangeSubHandler resume leader failed", K(ret), K(cursor));
} else {
CLOG_LOG(INFO, "leader to follower gracefully, current role change handler is",
"cursor", cursor, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string");
cursor++;
}
}
// if any sub role handler switch_to_follower_gracefully failed, and no need to revoke leader,
// we should resume other sub role handler, meanwhile, we should overrite 'ret' only if
// resume_leader_when_switch_failure_ failed.
if (OB_FAIL(ret) && OB_LS_NEED_REVOKE != ret) {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = resume_leader_when_switch_failure_(cursor))) {
CLOG_LOG(WARN, "resume_leader_when_switch_failure_ failed", K(tmp_ret), K(cursor));
ret = tmp_ret;
} else {
CLOG_LOG(WARN, "resume_leader_when_switch_failure_ success, no need to excut follower to leader gracefully",
K(ret), K(tmp_ret));
}
}
return ret;
}
int ObRoleChangeHandler::resume_to_leader()
{
int ret = OB_SUCCESS;
int cursor = ObLogBaseType::MAX_LOG_BASE_TYPE;
if (OB_FAIL(resume_leader_when_switch_failure_(cursor))) {
CLOG_LOG(WARN, "resume_leader_when_switch_failure_ failed");
} else {
CLOG_LOG(INFO, "resume_to_leader success");
}
return ret;
}
int ObRoleChangeHandler::resume_leader_when_switch_failure_(const int64_t cursor)
{
int ret = OB_SUCCESS;
for (int64_t i = cursor - 1; i >= 0 && OB_SUCC(ret); i--) {
ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[i];
ObLogBaseType base_type = static_cast<ObLogBaseType>(i);
char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'};
bool has_defined_to_string = false;
if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str,
OB_LOG_BASE_TYPE_STR_MAX_LEN)) {
has_defined_to_string = true;
}
if (NULL == handler){
CLOG_LOG(INFO, "not register into role change service", K(ret), K(i));
} else if (OB_FAIL(handler->resume_leader())) {
CLOG_LOG(WARN, "resume_leader failed", K(ret), K(i), KP(handler),
"cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string");
} else {
CLOG_LOG(INFO, "resume_leader success", K(ret), K(i), KP(handler),
"cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string");
}
}
if (OB_FAIL(ret)) {
ret = OB_LS_NEED_REVOKE;
}
return ret;
}
} // end namespace logservice
} // end namespace oceanbase

82
mittest/palf_cluster/logservice/role_coordinator_handler.h Normal file
View File

@ -0,0 +1,82 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_PALF_CLUSTER_OB_ROLE_CHANGE_HANDLER_
#define OCEANBASE_PALF_CLUSTER_OB_ROLE_CHANGE_HANDLER_
#include "lib/container/ob_fixed_array.h"
#include "lib/container/ob_se_array.h"
#include "lib/lock/ob_spin_lock.h"
#include "lib/utility/ob_print_utils.h"
#include "logservice/ob_log_base_header.h"
#include "logservice/ob_log_base_type.h"
#include "share/ob_errno.h"
namespace oceanbase
{
namespace palfcluster
{
enum TakeOverState {
INVALID_TAKE_OVER_STATE = 0,
WAIT_REPLAY_DONE = 1,
WAIT_RC_HANDLER_DONE = 2,
TAKE_OVER_FINISH = 3,
UNKNOWN_TAKE_OVER_STATE = 4,
MAX_TAKE_OVER_STATE = 5
};
static inline
int takeover_state_to_string(const TakeOverState log_type,
char *str,
const int64_t str_len)
{
int ret = OB_SUCCESS;
if (log_type == INVALID_TAKE_OVER_STATE) {
strncpy(str ,"INVALID_STATE", str_len);
} else if (log_type == WAIT_REPLAY_DONE) {
strncpy(str ,"WAIT_REPLAY_DONE", str_len);
} else if (log_type == WAIT_RC_HANDLER_DONE) {
strncpy(str ,"WAIT_RC_HANDLER_DONE", str_len);
} else if (log_type == TAKE_OVER_FINISH) {
strncpy(str ,"FINISH", str_len);
} else if (log_type == UNKNOWN_TAKE_OVER_STATE) {
strncpy(str ,"UNKNOWN", str_len);
} else {
ret = OB_INVALID_ARGUMENT;
}
return ret;
}
class ObRoleChangeHandler {
public:
ObRoleChangeHandler();
~ObRoleChangeHandler();
void reset();
int register_handler(const logservice::ObLogBaseType &type, logservice::ObIRoleChangeSubHandler *role_change_hander);
void unregister_handler(const logservice::ObLogBaseType &type);
void switch_to_follower_forcedly();
int switch_to_leader();
// @retval:
// 1. OB_SUCCESS
// 2. OB_LS_NEED_REVOKE, ObRoleChangeService need revoke this LS.
// 3. OTHERS, switch_to_follower_gracefully failed, but no need to revoke leader.
int switch_to_follower_gracefully();
int resume_to_leader();
private:
int resume_leader_when_switch_failure_(int64_t cursor);
private:
ObSpinLock lock_;
logservice::ObIRoleChangeSubHandler* sub_role_change_handler_arr_[logservice::ObLogBaseType::MAX_LOG_BASE_TYPE];
};
}
}
#endif

364
mittest/palf_cluster/palf-cluster.diff Normal file
View File

@ -0,0 +1,364 @@
diff --git a/build.sh b/build.sh
--- a/build.sh
+++ b/build.sh
@@ -188,7 +188,7 @@ function build
set -- "${BUILD_ARGS[@]}"
case "x$1" in
xrelease)
- do_build "$@" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DOB_USE_LLD=$LLD_OPTION
+ do_build "$@" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_AUTO_FDO=ON -DOB_USE_LLD=$LLD_OPTION
;;
xrelease_no_unity)
do_build "$@" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DOB_USE_LLD=$LLD_OPTION -DOB_ENABLE_UNITY=OFF
diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h
--- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h
+++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h
@@ -968,6 +968,9 @@ PCODE_DEF(OB_LOG_FORCE_CLEAR_ARB_CLUSTER_INFO, 0x1521)
PCODE_DEF(OB_LOG_GET_ARB_MEMBER_INFO, 0x1522)
#endif
PCODE_DEF(OB_LOG_BATCH_FETCH_RESP, 0X1523)
+PCODE_DEF(OB_LOG_CREATE_REPLICA_CMD, 0x1524)
+PCODE_DEF(OB_LOG_SUBMIT_LOG_CMD, 0x1525)
+PCODE_DEF(OB_LOG_SUBMIT_LOG_CMD_RESP, 0x1526)
// 1531-1550 for obesi
// PCODE_DEF(OB_ESI_IS_EXIST, 0x1531)
diff --git a/src/logservice/applyservice/ob_log_apply_service.cpp b/src/logservice/applyservice/ob_log_apply_service.cpp
--- a/src/logservice/applyservice/ob_log_apply_service.cpp
+++ b/src/logservice/applyservice/ob_log_apply_service.cpp
@@ -28,6 +28,9 @@ using namespace storage;
using namespace share;
namespace logservice
{
+int64_t p_append_cnt = 0;
+int64_t p_log_body_size = 0;
+int64_t p_rt = 0;
//---------------ObApplyFsCb---------------//
ObApplyFsCb::ObApplyFsCb()
: apply_status_(NULL)
@@ -482,6 +485,26 @@ int ObApplyStatus::try_handle_cb_queue(ObApplyServiceQueueTask *cb_queue,
scn = cb->__get_scn();
get_cb_trace_(cb, append_start_time, append_finish_time, cb_first_handle_time, cb_start_time);
CLOG_LOG(TRACE, "cb on_success", K(lsn), K(scn), KP(link->next_), KPC(cb_queue), KPC(this));
+
+ ATOMIC_INC(&p_append_cnt);
+ ATOMIC_FAA(&p_log_body_size, cb->get_log_size());
+ int64_t tmp_rt = ObTimeUtility::current_time() - cb->get_append_start_ts();
+ ATOMIC_FAA(&p_rt, tmp_rt);
+
+ if (REACH_TIME_INTERVAL(1000 *1000)) {
+ int64_t l_append_cnt = ATOMIC_LOAD(&p_append_cnt);
+ if (l_append_cnt == 0) l_append_cnt = 1;
+ int64_t l_log_body_size = ATOMIC_LOAD(&p_log_body_size);
+ int64_t l_rt = ATOMIC_LOAD(&p_rt);
+
+ ATOMIC_STORE(&p_append_cnt, 0);
+ ATOMIC_STORE(&p_log_body_size, 0);
+ ATOMIC_STORE(&p_rt, 0);
+
+ CLOG_LOG(ERROR, "result:", K(l_append_cnt), K(l_log_body_size), K(l_rt),
+ "avg_body_size", l_log_body_size/l_append_cnt, "avg_rt", l_rt/l_append_cnt);
+ }
+
if (OB_FAIL(cb->on_success())) {
// 不处理此类失败情况
CLOG_LOG(ERROR, "cb on_success failed", KP(cb), K(ret), KPC(this));
@@ -1044,7 +1067,7 @@ int ObLogApplyService::init(PalfEnv *palf_env,
CLOG_LOG(WARN, "invalid argument", K(ret), KP(palf_env), K(ls_adapter));
} else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::ApplyService, tg_id_))) {
CLOG_LOG(WARN, "fail to create thread group", K(ret));
- } else if (OB_FAIL(MTL_REGISTER_THREAD_DYNAMIC(0.5, tg_id_))) {
+ } else if (OB_FAIL(MTL_REGISTER_THREAD_DYNAMIC(1, tg_id_))) {
CLOG_LOG(WARN, "MTL_REGISTER_THREAD_DYNAMIC failed", K(ret), K(tg_id_));
} else if (OB_FAIL(apply_status_map_.init("APPLY_STATUS", MAP_TENANT_ID))) {
CLOG_LOG(WARN, "apply_status_map_ init error", K(ret));
diff --git a/src/logservice/ob_append_callback.h b/src/logservice/ob_append_callback.h
--- a/src/logservice/ob_append_callback.h
+++ b/src/logservice/ob_append_callback.h
@@ -60,6 +60,7 @@ public:
append_start_ts_ = OB_INVALID_TIMESTAMP;
append_finish_ts_ = OB_INVALID_TIMESTAMP;
cb_first_handle_ts_ = OB_INVALID_TIMESTAMP;
+ log_size_ = 0;
}
virtual int on_success() = 0;
virtual int on_failure() = 0;
@@ -69,10 +70,16 @@ public:
int64_t get_append_start_ts() const { return append_start_ts_; }
int64_t get_append_finish_ts() const { return append_finish_ts_; }
int64_t get_cb_first_handle_ts() const { return cb_first_handle_ts_; }
+ void set_log_size(const int64_t size)
+ {
+ log_size_ = size;
+ }
+ int64_t get_log_size() const { return log_size_; }
public:
int64_t append_start_ts_; //提交到palf的起始时刻
int64_t append_finish_ts_; //palf提交完成时刻,即提交到apply service起始时刻
int64_t cb_first_handle_ts_; //cb第一次被处理的时刻,不一定调用on_success
+ int64_t log_size_;
};
} // end namespace logservice
diff --git a/src/logservice/ob_log_handler.cpp b/src/logservice/ob_log_handler.cpp
--- a/src/logservice/ob_log_handler.cpp
+++ b/src/logservice/ob_log_handler.cpp
@@ -237,6 +237,7 @@ int ObLogHandler::append(const void *buffer,
cb->set_append_finish_ts(ObTimeUtility::fast_current_time());
cb->__set_lsn(lsn);
cb->__set_scn(scn);
+ cb->set_log_size(nbytes);
ret = apply_status_->push_append_cb(cb);
CLOG_LOG(TRACE, "palf_handle_ push_append_cb success", K(lsn), K(scn), K(ret), K(id_));
}
diff --git a/src/logservice/palf/log_block_handler.cpp b/src/logservice/palf/log_block_handler.cpp
--- a/src/logservice/palf/log_block_handler.cpp
+++ b/src/logservice/palf/log_block_handler.cpp
@@ -437,12 +437,26 @@ int LogBlockHandler::inner_writev_once_(const offset_t offset,
return ret;
}
+int64_t p_io_cnt = 0;
+int64_t p_log_size = 0;
+
int LogBlockHandler::inner_write_impl_(const int fd, const char *buf, const int64_t count, const int64_t offset)
{
int ret = OB_SUCCESS;
int64_t start_ts = ObTimeUtility::fast_current_time();
int64_t write_size = 0;
int64_t time_interval = OB_INVALID_TIMESTAMP;
+ ATOMIC_INC(&p_io_cnt);
+ ATOMIC_FAA(&p_log_size, count);
+ if (REACH_TIME_INTERVAL(1000 * 1000)) {
+ int64_t l_io_cnt = ATOMIC_LOAD(&p_io_cnt);
+ int64_t l_log_size = ATOMIC_LOAD(&p_log_size);
+
+ ATOMIC_STORE(&p_io_cnt, 0);
+ ATOMIC_STORE(&p_log_size, 0);
+
+ CLOG_LOG(ERROR, "io result:", K(l_io_cnt), K(l_log_size), "avg_size", l_log_size/l_io_cnt);
+ }
do {
if (count != (write_size = ob_pwrite(fd, buf, count, offset))) {
if (palf_reach_time_interval(1000 * 1000, time_interval)) {
diff --git a/src/logservice/palf/log_define.h b/src/logservice/palf/log_define.h
--- a/src/logservice/palf/log_define.h
+++ b/src/logservice/palf/log_define.h
@@ -102,7 +102,7 @@ const int64_t PALF_LEADER_RECONFIRM_SYNC_TIMEOUT_US = 10 * 1000 * 1000L; //
const int64_t PREPARE_LOG_BUFFER_SIZE = 2048;
const int64_t PALF_LEADER_ACTIVE_SYNC_TIMEOUT_US = 10 * 1000 * 1000L; // 10s
const int32_t PALF_MAX_REPLAY_TIMEOUT = 500 * 1000;
-const int32_t DEFAULT_LOG_LOOP_INTERVAL_US = 100 * 1000; // 100ms
+const int32_t DEFAULT_LOG_LOOP_INTERVAL_US = 500; // 100ms
const int32_t LOG_LOOP_INTERVAL_FOR_PERIOD_FREEZE_US = 1 * 1000; // 1ms
const int64_t PALF_SLIDING_WINDOW_SIZE = 1 << 11; // must be 2^n(n>0), default 2^11 = 2048
const int64_t PALF_MAX_LEADER_SUBMIT_LOG_COUNT = PALF_SLIDING_WINDOW_SIZE / 2; // max number of concurrent submitting group log in leader
diff --git a/src/logservice/palf/log_io_task.cpp b/src/logservice/palf/log_io_task.cpp
--- a/src/logservice/palf/log_io_task.cpp
+++ b/src/logservice/palf/log_io_task.cpp
@@ -623,7 +623,8 @@ int BatchLogIOFlushLogTask::push_flush_cb_to_thread_pool_(int tg_id, IPalfEnvImp
PALF_LOG(ERROR, "LogIOFlushMetaTask not inited!!!", K(ret), KPC(this));
} else {
const int64_t current_time = ObTimeUtility::current_time();
- for (int64_t i = 0; i < count && OB_SUCC(ret); i++) {
+ // for (int64_t i = 0; i < count && OB_SUCC(ret); i++) {
+ int64_t i = count - 1;
LogIOFlushLogTask *io_task = io_task_array_[i];
if (NULL == io_task) {
PALF_LOG(WARN, "io_task is nullptr, may be its' epoch has changed", K(ret), KP(io_task),
@@ -637,6 +638,9 @@ int BatchLogIOFlushLogTask::push_flush_cb_to_thread_pool_(int tg_id, IPalfEnvImp
// again.
io_task_array_[i] = NULL;
}
+ // }
+ for (int64_t i = 0; i < count && OB_SUCC(ret); i++) {
+ io_task_array_[i] = NULL;
}
}
return ret;
diff --git a/src/logservice/palf/log_sliding_window.cpp b/src/logservice/palf/log_sliding_window.cpp
--- a/src/logservice/palf/log_sliding_window.cpp
+++ b/src/logservice/palf/log_sliding_window.cpp
@@ -132,7 +132,8 @@ LogSlidingWindow::LogSlidingWindow()
accum_log_cnt_(0),
accum_group_log_size_(0),
last_record_group_log_id_(FIRST_VALID_LOG_ID - 1),
- freeze_mode_(FEEDBACK_FREEZE_MODE),
+ freeze_mode_(PERIOD_FREEZE_MODE),
+ avg_log_batch_cnt_(0),
is_inited_(false)
{}
@@ -448,6 +449,7 @@ int LogSlidingWindow::submit_log(const char *buf,
const int64_t start_log_id = get_start_id();
const int64_t log_id_upper_bound = start_log_id + PALF_MAX_LEADER_SUBMIT_LOG_COUNT - 1;
LSN tmp_lsn, lsn_upper_bound;
+ bool need_freeze_self = false;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else if (NULL == buf || buf_len <= 0 || buf_len > MAX_LOG_BODY_SIZE || (!ref_scn.is_valid())) {
@@ -462,7 +464,7 @@ int LogSlidingWindow::submit_log(const char *buf,
}
// sw_ cannot submit larger log
} else if (OB_FAIL(lsn_allocator_.alloc_lsn_scn(ref_scn, valid_log_size, log_id_upper_bound, lsn_upper_bound,
- tmp_lsn, log_id, scn, is_new_log, need_gen_padding_entry, padding_size))) {
+ tmp_lsn, log_id, scn, is_new_log, need_gen_padding_entry, padding_size, need_freeze_self))) {
PALF_LOG(WARN, "alloc_lsn_scn failed", K(ret), K_(palf_id), K_(self));
} else if (OB_FAIL(leader_wait_sw_slot_ready_(log_id))) {
PALF_LOG(WARN, "leader_wait_sw_slot_ready_ failed", K(ret), K_(palf_id), K_(self), K(log_id));
@@ -480,7 +482,7 @@ int LogSlidingWindow::submit_log(const char *buf,
K(padding_size), K(is_new_log), K(valid_log_size));
} else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) {
} else if (OB_FAIL(generate_new_group_log_(tmp_lsn, log_id, scn, padding_entry_body_size, LOG_PADDING, \
- NULL, padding_entry_body_size, is_need_handle))) {
+ NULL, padding_entry_body_size, need_freeze_self, is_need_handle))) {
PALF_LOG(ERROR, "generate_new_group_log_ failed", K(ret), K_(palf_id), K_(self), K(log_id), K(tmp_lsn), K(padding_size),
K(is_new_log), K(valid_log_size));
} else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) {
@@ -503,7 +505,7 @@ int LogSlidingWindow::submit_log(const char *buf,
PALF_LOG(WARN, "try_freeze_prev_log_ failed", K(ret), K_(palf_id), K_(self), K(log_id));
} else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) {
} else if (OB_FAIL(generate_new_group_log_(tmp_lsn, log_id, scn, valid_log_size, LOG_SUBMIT, \
- buf, buf_len, is_need_handle))) {
+ buf, buf_len, need_freeze_self, is_need_handle))) {
PALF_LOG(WARN, "generate_new_group_log_ failed", K(ret), K_(palf_id), K_(self), K(log_id));
} else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) {
} else {
@@ -673,6 +675,7 @@ int LogSlidingWindow::generate_new_group_log_(const LSN &lsn,
const LogType &log_type,
const char *log_data,
const int64_t data_len,
+ const bool need_freeze_self,
bool &is_need_handle)
{
int ret = OB_SUCCESS;
@@ -712,6 +715,9 @@ int LogSlidingWindow::generate_new_group_log_(const LSN &lsn,
} else if (OB_FAIL(log_task->set_initial_header_info(header_info))) {
PALF_LOG(WARN, "set_initial_header_info failed", K(ret), K_(palf_id), K_(self), K(log_id), KPC(log_task));
} else {
+ if (need_freeze_self) {
+ log_task->set_end_lsn(lsn + log_body_size + LogGroupEntryHeader::HEADER_SER_SIZE);
+ }
// The first log is responsible to try freezing self, if its end_lsn_ has been set by next log.
log_task->try_freeze_by_myself();
}
@@ -1048,6 +1054,10 @@ int LogSlidingWindow::handle_next_submit_log_(bool &is_committed_lsn_updated)
const int64_t avg_group_log_size = total_group_log_size / total_group_log_cnt;
PALF_LOG(INFO, "[PALF STAT GROUP LOG INFO]", K_(palf_id), K_(self), "role", role_to_string(role),
K(total_group_log_cnt), K(avg_log_batch_cnt), K(total_group_log_size), K(avg_group_log_size));
+ if (total_log_cnt > 0) {
+ const int64_t avg_log_size = total_group_log_size / total_log_cnt;
+ avg_log_batch_cnt_ = (avg_log_batch_cnt_ * 2 + avg_log_batch_cnt * 8)/10;
+ }
}
ATOMIC_STORE(&accum_log_cnt_, 0);
ATOMIC_STORE(&accum_group_log_size_, 0);
@@ -1235,19 +1245,34 @@ int LogSlidingWindow::check_and_switch_freeze_mode()
{
int ret = OB_SUCCESS;
int64_t total_append_cnt = 0;
- for (int i = 0; i < APPEND_CNT_ARRAY_SIZE; ++i) {
- total_append_cnt += ATOMIC_LOAD(&append_cnt_array_[i]);
- ATOMIC_STORE(&append_cnt_array_[i], 0);
- }
+ // for (int i = 0; i < APPEND_CNT_ARRAY_SIZE; ++i) {
+ // total_append_cnt += ATOMIC_LOAD(&append_cnt_array_[i]);
+ // ATOMIC_STORE(&append_cnt_array_[i], 0);
+ // }
+
+ // for (int i = 0; i < APPEND_CNT_ARRAY_SIZE; ++i) {
+ // total_append_cnt += ATOMIC_LOAD(&append_cnt_array_[i]);
+ // ATOMIC_STORE(&append_cnt_array_[i], 0);
+ // }
+ // adaptively
+ const int64_t avg_log_batch_cnt = avg_log_batch_cnt_;
+ // periodically
+ // const int64_t avg_log_batch_cnt = 20;
+ // feedback
+ // const int64_t avg_log_batch_cnt = 0;
+ const int64_t SWITCH_BARRIER = 10;
+
if (FEEDBACK_FREEZE_MODE == freeze_mode_) {
- if (total_append_cnt >= APPEND_CNT_LB_FOR_PERIOD_FREEZE) {
+ // if (total_append_cnt >= APPEND_CNT_LB_FOR_PERIOD_FREEZE) {
+ if (avg_log_batch_cnt > SWITCH_BARRIER) {
freeze_mode_ = PERIOD_FREEZE_MODE;
- PALF_LOG(INFO, "switch freeze_mode to period", K_(palf_id), K_(self), K(total_append_cnt));
+ PALF_LOG(ERROR, "switch freeze_mode to period", K_(palf_id), K_(self), K(total_append_cnt));
}
} else if (PERIOD_FREEZE_MODE == freeze_mode_) {
- if (total_append_cnt < APPEND_CNT_LB_FOR_PERIOD_FREEZE) {
+ // if (total_append_cnt < APPEND_CNT_LB_FOR_PERIOD_FREEZE) {
+ if (avg_log_batch_cnt <= SWITCH_BARRIER) {
freeze_mode_ = FEEDBACK_FREEZE_MODE;
- PALF_LOG(INFO, "switch freeze_mode to feedback", K_(palf_id), K_(self), K(total_append_cnt));
+ PALF_LOG(ERROR, "switch freeze_mode to feedback", K_(palf_id), K_(self), K(total_append_cnt));
(void) feedback_freeze_last_log_();
}
} else {}
diff --git a/src/logservice/palf/log_sliding_window.h b/src/logservice/palf/log_sliding_window.h
--- a/src/logservice/palf/log_sliding_window.h
+++ b/src/logservice/palf/log_sliding_window.h
@@ -396,6 +396,7 @@ private:
const LogType &log_type,
const char *log_data,
const int64_t data_len,
+ const bool need_freeze_self,
bool &is_need_handle);
int append_to_group_log_(const LSN &lsn,
const int64_t log_id,
@@ -608,6 +609,7 @@ private:
int64_t last_record_group_log_id_;
int64_t append_cnt_array_[APPEND_CNT_ARRAY_SIZE];
FreezeMode freeze_mode_;
+ int64_t avg_log_batch_cnt_;
bool is_inited_;
private:
DISALLOW_COPY_AND_ASSIGN(LogSlidingWindow);
diff --git a/src/logservice/palf/lsn_allocator.cpp b/src/logservice/palf/lsn_allocator.cpp
--- a/src/logservice/palf/lsn_allocator.cpp
+++ b/src/logservice/palf/lsn_allocator.cpp
@@ -297,7 +297,8 @@ int LSNAllocator::alloc_lsn_scn(const SCN &base_scn,
SCN &scn,
bool &is_new_group_log,
bool &need_gen_padding_entry,
- int64_t &padding_len)
+ int64_t &padding_len,
+ bool &need_freeze_self)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
@@ -439,6 +440,11 @@ int LSNAllocator::alloc_lsn_scn(const SCN &base_scn,
is_next_need_cut = true;
}
}
+ // freeze immediately
+ if (need_freeze_self) {
+ is_new_group_log = true;
+ is_next_need_cut = true;
+ }
if (is_new_group_log) {
tmp_next_log_id_delta++;
}
diff --git a/src/logservice/palf/lsn_allocator.h b/src/logservice/palf/lsn_allocator.h
--- a/src/logservice/palf/lsn_allocator.h
+++ b/src/logservice/palf/lsn_allocator.h
@@ -64,7 +64,8 @@ public:
share::SCN &scn,
bool &is_new_log,
bool &need_gen_padding_entry,
- int64_t &padding_len);
+ int64_t &padding_len,
+ bool &need_freeze_self);
// 更新last_lsn和log_timestamp
// receive_log/append_disk_log 时调用
int inc_update_last_log_info(const LSN &lsn, const int64_t log_id, const share::SCN &scn);
diff --git a/src/share/ob_thread_define.h b/src/share/ob_thread_define.h
--- a/src/share/ob_thread_define.h
+++ b/src/share/ob_thread_define.h
@@ -164,4 +164,5 @@ TG_DEF(SvrStartupHandler, SvrStartupHandler, QUEUE_THREAD,
TG_DEF(TenantTTLManager, TTLManager, TIMER)
TG_DEF(TenantTabletTTLMgr, TTLTabletMgr, TIMER)
TG_DEF(TntSharedTimer, TntSharedTimer, TIMER)
+TG_DEF(LogServerTest, LogServerTest, THREAD_POOL, ThreadCountPair(10, 10))
#endif

79
mittest/palf_cluster/rpc/ob_log_rpc_processor.h Normal file
View File

@ -0,0 +1,79 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_PALF_CLUSTER_RPC_PROCESSOR_H_
#define OCEANBASE_PALF_CLUSTER_RPC_PROCESSOR_H_
#include "palf_cluster_rpc_req.h"
#include "palf_cluster_request_handler.h"
#include "palf_cluster_rpc_proxy.h"
#include "lib/ob_errno.h"
namespace oceanbase
{
namespace palfcluster
{
#define DEFINE_LOGSERVICE_RPC_PROCESSOR(CLASS, PROXY, REQTYPE, PCODE) \
class CLASS : public obrpc::ObRpcProcessor<PROXY::ObRpc<PCODE>> \
{ \
public: \
CLASS() : filter_(NULL) {} \
virtual ~CLASS() { filter_ = NULL; } \
int process() \
{ \
int ret = OB_SUCCESS; \
const REQTYPE &req = arg_; \
const common::ObAddr server = req.src_; \
const uint64_t tenant_id = rpc_pkt_->get_tenant_id(); \
LogRequestHandler handler; \
handler.set_log_service(log_service_); \
if (OB_UNLIKELY(NULL != filter_ && true == (*filter_)(server))) { \
} else if (OB_FAIL(handler.handle_request(req))) { \
CLOG_LOG(WARN, "Processor handle_request failed", K(ret), K(req)); \
} else { \
CLOG_LOG(INFO, "Processor handle_request success", K(ret), K(req)); \
} \
return ret; \
} \
void set_filter(void *filter) \
{ \
filter_ = reinterpret_cast<ObFunction<bool(const ObAddr &src)> *>(filter); \
} \
void set_log_service(void *log_service) \
{ \
log_service_ = reinterpret_cast<LogService *>(log_service); \
} \
private: \
ObFunction<bool(const ObAddr &src)> *filter_; \
palfcluster::LogService *log_service_; \
}
DEFINE_LOGSERVICE_RPC_PROCESSOR(LogCreateReplicaCmdP,
obrpc::PalfClusterRpcProxy,
palfcluster::LogCreateReplicaCmd,
obrpc::OB_LOG_CREATE_REPLICA_CMD);
DEFINE_LOGSERVICE_RPC_PROCESSOR(LogSubmitLogP,
obrpc::PalfClusterRpcProxy,
palfcluster::SubmitLogCmd,
obrpc::OB_LOG_SUBMIT_LOG_CMD);
DEFINE_LOGSERVICE_RPC_PROCESSOR(LogSubmitLogRespP,
obrpc::PalfClusterRpcProxy,
SubmitLogCmdResp,
obrpc::OB_LOG_SUBMIT_LOG_CMD_RESP);
} // end namespace palfcluster
} // end namespace oceanbase
#endif

123
mittest/palf_cluster/rpc/palf_cluster_request_handler.cpp Normal file
View File

@ -0,0 +1,123 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "palf_cluster_request_handler.h"
#include "palf_cluster_rpc_req.h"
#include "palf_cluster_rpc_proxy.h"
#include "logservice/ob_log_service.h"
#include "logservice/palf/log_define.h"
namespace oceanbase
{
namespace palfcluster
{
LogRequestHandler::LogRequestHandler()
: log_service_(NULL)
{
}
LogRequestHandler::~LogRequestHandler()
{
}
int LogRequestHandler::get_self_addr_(common::ObAddr &self) const
{
int ret = OB_SUCCESS;
const common::ObAddr self_addr = GCTX.self_addr();
if (false == self_addr.is_valid()) {
ret = OB_ERR_UNEXPECTED;
} else {
self = self_addr;
}
return ret;
}
template <>
int LogRequestHandler::handle_request<LogCreateReplicaCmd>(
const LogCreateReplicaCmd &req)
{
int ret = common::OB_SUCCESS;
const int64_t palf_id = req.ls_id_;
const common::ObAddr &server = req.src_;
share::ObLSID ls_id(palf_id);
if (OB_ISNULL(log_service_)) {
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(ERROR, "get_log_service failed", K(ret));
} else if (false == req.is_valid()) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(ERROR, "Invalid argument!!!", K(ret), K(req));
} else if (OB_FAIL(log_service_->create_palf_replica(req.ls_id_, req.member_list_, req.replica_num_, req.leader_idx_))) {
CLOG_LOG(ERROR, "create_palf_replica failed", K(ret));
}
CLOG_LOG(ERROR, "create_palf_replica finish", K(ret), K(req));
return ret;
}
template <>
int LogRequestHandler::handle_request<SubmitLogCmd>(
const SubmitLogCmd &req)
{
int ret = common::OB_SUCCESS;
if (false == req.is_valid()) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(ERROR, "Invalid argument!!!", K(ret), K(req));
} else {
const int64_t palf_id = req.ls_id_;
share::ObLSID ls_id(palf_id);
const common::ObAddr &src = req.src_;
const int64_t client_id = req.client_id_;
palfcluster::LogClientMap *log_clients = nullptr;
palfcluster::ObLogClient *log_client = nullptr;
if (OB_ISNULL(log_service_)) {
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(ERROR, "get_log_service failed", K(ret));
} else if (FALSE_IT(log_clients = log_service_->get_log_client_map())) {
} else if (OB_FAIL(log_clients->get(ls_id, log_client))) {
CLOG_LOG(ERROR, "get_log_client failed", K(ret), K(palf_id));
} else if (OB_FAIL(log_client->submit_log(src, client_id, req.log_buf_))) {
CLOG_LOG(ERROR, "submit_log failed", K(ret));
}
ret = OB_SUCCESS;
}
return ret;
}
template <>
int LogRequestHandler::handle_request<SubmitLogCmdResp>(
const SubmitLogCmdResp &req)
{
int ret = common::OB_SUCCESS;
if (false == req.is_valid()) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(ERROR, "Invalid argument!!!", K(ret), K(req));
} else {
const int64_t palf_id = req.ls_id_;
share::ObLSID ls_id(palf_id);
const common::ObAddr &server = req.src_;
const int64_t client_id = req.client_id_;
palfcluster::LogClientMap *log_clients = nullptr;
palfcluster::ObLogClient *log_client = nullptr;
if (OB_ISNULL(log_service_)) {
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(ERROR, "get_log_service failed", K(ret));
} else {
log_service_->clients_[client_id].has_returned();
}
ret = OB_SUCCESS;
}
return ret;
}
} // end namespace logservice
} // end namespace oceanbase

74
mittest/palf_cluster/rpc/palf_cluster_request_handler.h Normal file
View File

@ -0,0 +1,74 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_PALF_CLUSTER_LOG_REQUEST_HANDLER_
#define OCEANBASE_PALF_CLUSTER_LOG_REQUEST_HANDLER_
#include "lib/ob_errno.h" // OB_SUCCESS...
#include "logservice/ob_reporter_adapter.h" // ObLogReporterAdapter
#include "logservice/palf_handle_guard.h" // palf::PalfHandleGuard
#include "palf_cluster_rpc_req.h" // Req...
#include "mittest/palf_cluster/logservice/log_service.h"
namespace oceanbase
{
namespace palf
{
class PalfEnv;
}
namespace obrpc
{
class PalfClusterRpcProxy;
}
namespace palfcluster
{
class ObLogHandler;
class ObLogReplayService;
class LogRequestHandler
{
public:
LogRequestHandler();
~LogRequestHandler();
template <typename ReqType, typename RespType>
int handle_sync_request(const ReqType &req, RespType &resp);
template <typename ReqType>
int handle_request(const ReqType &req);
void set_log_service(palfcluster::LogService *log_service) { log_service_ = log_service; }
private:
int get_self_addr_(common::ObAddr &self) const;
int get_rpc_proxy_(obrpc::PalfClusterRpcProxy *&rpc_proxy) const;
palfcluster::LogService *log_service_;
};
class ConfigChangeCmdHandler{
public:
explicit ConfigChangeCmdHandler(palf::PalfHandle *palf_handle)
{
if (NULL != palf_handle) {
palf_handle_ = palf_handle;
}
}
~ConfigChangeCmdHandler()
{
palf_handle_ = NULL;
}
private:
palf::PalfHandle *palf_handle_;
};
} // end namespace logservice
} // end namespace oceanbase
#endif

80
mittest/palf_cluster/rpc/palf_cluster_rpc_processor.h Normal file
View File

@ -0,0 +1,80 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_PALF_CLUSTER_RPC_PROCESSOR_H_
#define OCEANBASE_PALF_CLUSTER_RPC_PROCESSOR_H_
#include "palf_cluster_rpc_req.h"
#include "palf_cluster_request_handler.h"
#include "palf_cluster_rpc_proxy.h"
#include "lib/ob_errno.h"
namespace oceanbase
{
namespace palfcluster
{
class LogService;
#define DEFINE_LOGSERVICE_RPC_PROCESSOR(CLASS, PROXY, REQTYPE, PCODE) \
class CLASS : public obrpc::ObRpcProcessor<PROXY::ObRpc<PCODE>> \
{ \
public: \
CLASS() : filter_(NULL) {} \
virtual ~CLASS() { filter_ = NULL; } \
int process() \
{ \
int ret = OB_SUCCESS; \
const REQTYPE &req = arg_; \
const common::ObAddr server = req.src_; \
const uint64_t tenant_id = rpc_pkt_->get_tenant_id(); \
LogRequestHandler handler; \
handler.set_log_service(log_service_); \
if (OB_UNLIKELY(NULL != filter_ && true == (*filter_)(server))) { \
} else if (OB_FAIL(handler.handle_request(req))) { \
CLOG_LOG(WARN, "Processor handle_request failed", K(ret), K(req)); \
} else { \
CLOG_LOG(INFO, "Processor handle_request success", K(ret), K(req)); \
} \
return ret; \
} \
void set_filter(void *filter) \
{ \
filter_ = reinterpret_cast<ObFunction<bool(const ObAddr &src)> *>(filter); \
} \
void set_log_service(void *log_service) \
{ \
log_service_ = reinterpret_cast<palfcluster::LogService *>(log_service); \
} \
private: \
ObFunction<bool(const ObAddr &src)> *filter_; \
palfcluster::LogService *log_service_; \
}
DEFINE_LOGSERVICE_RPC_PROCESSOR(LogCreateReplicaCmdP,
obrpc::PalfClusterRpcProxy,
palfcluster::LogCreateReplicaCmd,
obrpc::OB_LOG_CREATE_REPLICA_CMD);
DEFINE_LOGSERVICE_RPC_PROCESSOR(LogSubmitLogP,
obrpc::PalfClusterRpcProxy,
palfcluster::SubmitLogCmd,
obrpc::OB_LOG_SUBMIT_LOG_CMD);
DEFINE_LOGSERVICE_RPC_PROCESSOR(LogSubmitLogRespP,
obrpc::PalfClusterRpcProxy,
SubmitLogCmdResp,
obrpc::OB_LOG_SUBMIT_LOG_CMD_RESP);
} // end namespace palfcluster
} // end namespace oceanbase
#endif

74
mittest/palf_cluster/rpc/palf_cluster_rpc_proxy.h Normal file
View File

@ -0,0 +1,74 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_PALF_CLUSTER_LOG_RPC_PROXY_
#define OCEANBASE_PALF_CLUSTER_LOG_RPC_PROXY_
#include "palf_cluster_rpc_req.h"
#include "rpc/obrpc/ob_rpc_proxy.h" // ObRpcProxy
namespace oceanbase
{
namespace obrpc
{
class PalfClusterRpcProxy : public obrpc::ObRpcProxy
{
public:
DEFINE_TO(PalfClusterRpcProxy);
RPC_AP(PR3 send_submit_log_cmd, OB_LOG_SUBMIT_LOG_CMD,
(palfcluster::SubmitLogCmd));
RPC_AP(PR3 send_submit_log_resp, OB_LOG_SUBMIT_LOG_CMD_RESP,
(palfcluster::SubmitLogCmdResp));
RPC_AP(PR3 send_create_replica_cmd, OB_LOG_CREATE_REPLICA_CMD,
(palfcluster::LogCreateReplicaCmd));
};
template <obrpc::ObRpcPacketCode pcode>
class ObLogRpcCB: public PalfClusterRpcProxy::AsyncCB<pcode>
{
public:
ObLogRpcCB() {}
~ObLogRpcCB() {}
void set_args(const typename PalfClusterRpcProxy::AsyncCB<pcode>::Request &args) {
UNUSED(args);
}
rpc::frame::ObReqTransport::AsyncCB *clone(const rpc::frame::SPAlloc &alloc) const {
void *buf = alloc(sizeof (*this));
ObLogRpcCB<pcode> *newcb = NULL;
if (NULL != buf) {
newcb = new (buf) ObLogRpcCB<pcode>();
}
return newcb;
}
int process() {
const common::ObAddr &dst = PalfClusterRpcProxy::AsyncCB<pcode>::dst_;
obrpc::ObRpcResultCode &rcode = PalfClusterRpcProxy::AsyncCB<pcode>::rcode_;
int ret = rcode.rcode_;
if (common::OB_SUCCESS != rcode.rcode_) {
PALF_LOG(WARN, "there is an rpc error in logservice", K(rcode), K(dst), K(pcode));
} else {
// do nothing
}
return common::OB_SUCCESS;
}
void on_timeout() {
const common::ObAddr &dst = PalfClusterRpcProxy::AsyncCB<pcode>::dst_;
//PALF_LOG(WARN, "logservice rpc timeout", K(dst));
}
};
} // end namespace obrpc
} // end namespace oceanbase
#endif

134
mittest/palf_cluster/rpc/palf_cluster_rpc_req.cpp Normal file
View File

@ -0,0 +1,134 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "palf_cluster_rpc_req.h"
#include "logservice/palf/log_define.h"
namespace oceanbase
{
using namespace share;
namespace palfcluster
{
// ============= LogCreateReplicaCmd start =============
LogCreateReplicaCmd::LogCreateReplicaCmd()
: src_(),
ls_id_(-1),
member_list_(),
replica_num_(-1),
leader_idx_(-1) { }
LogCreateReplicaCmd::LogCreateReplicaCmd(
const common::ObAddr &src,
const int64_t ls_id,
const common::ObMemberList &member_list,
const int64_t replica_num,
const int64_t leader_idx)
: src_(src),
ls_id_(ls_id),
member_list_(member_list),
replica_num_(replica_num),
leader_idx_(leader_idx) { }
bool LogCreateReplicaCmd::is_valid() const
{
return src_.is_valid() &&
-1 != ls_id_ &&
member_list_.is_valid() &&
replica_num_ > 0 &&
replica_num_ <= common::OB_MAX_MEMBER_NUMBER;
}
void LogCreateReplicaCmd::reset()
{
src_.reset();
ls_id_ = -1;
member_list_.reset();
replica_num_ = -1;
leader_idx_ = -1;
}
OB_SERIALIZE_MEMBER(LogCreateReplicaCmd, src_, ls_id_,
member_list_, replica_num_, leader_idx_);
// ============= LogCreateReplicaCmd end =============
// ============= SubmitLogCmd start =============
SubmitLogCmd::SubmitLogCmd()
: src_(),
ls_id_(-1),
client_id_(-1),
log_buf_()
{ }
SubmitLogCmd::SubmitLogCmd(
const common::ObAddr &src,
const int64_t ls_id,
const int64_t client_id,
const palf::LogWriteBuf &log_buf)
: src_(src),
ls_id_(ls_id),
client_id_(client_id),
log_buf_(log_buf)
{ }
bool SubmitLogCmd::is_valid() const
{
return src_.is_valid() &&
-1 != ls_id_ &&
log_buf_.is_valid();
}
void SubmitLogCmd::reset()
{
src_.reset();
ls_id_ = -1;
client_id_ = -1;
log_buf_.reset();
}
OB_SERIALIZE_MEMBER(SubmitLogCmd, src_, ls_id_, client_id_, log_buf_);
// ============= SubmitLogCmd end =============
// ============= SubmitLogCmdResp start =============
SubmitLogCmdResp::SubmitLogCmdResp()
: src_(),
ls_id_(-1),
client_id_(-1)
{ }
SubmitLogCmdResp::SubmitLogCmdResp(
const common::ObAddr &src,
const int64_t ls_id,
const int64_t client_id)
: src_(src),
ls_id_(ls_id),
client_id_(client_id)
{ }
bool SubmitLogCmdResp::is_valid() const
{
return src_.is_valid() &&
-1 != ls_id_;
}
void SubmitLogCmdResp::reset()
{
src_.reset();
ls_id_ = -1;
client_id_ = -1;
}
OB_SERIALIZE_MEMBER(SubmitLogCmdResp, src_, ls_id_, client_id_);
// ============= SubmitLogCmdResp end =============
} // end namespace logservice
}// end namespace oceanbase

94
mittest/palf_cluster/rpc/palf_cluster_rpc_req.h Normal file
View File

@ -0,0 +1,94 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_PALF_CLUSTER_RPC_REQ_H_
#define OCEANBASE_PALF_CLUSTER_RPC_REQ_H_
#include "lib/utility/ob_unify_serialize.h" // OB_UNIS_VERSION
#include "lib/utility/ob_print_utils.h" // TO_STRING_KV
#include "common/ob_member_list.h" // ObMemberList
#include "logservice/palf/palf_handle_impl.h" // PalfStat
#include "share/scn.h"
#include "logservice/palf/log_writer_utils.h" // LogWriteBuf
namespace oceanbase
{
namespace palfcluster
{
struct LogCreateReplicaCmd {
OB_UNIS_VERSION(1);
public:
LogCreateReplicaCmd();
LogCreateReplicaCmd(const common::ObAddr &src,
const int64_t ls_id,
const common::ObMemberList &member_list,
const int64_t replica_num,
const int64_t leader_idx);
~LogCreateReplicaCmd()
{
reset();
}
bool is_valid() const;
void reset();
TO_STRING_KV(K_(src), K_(ls_id), K_(member_list), K_(replica_num), K_(leader_idx));
common::ObAddr src_;
int64_t ls_id_;
common::ObMemberList member_list_;
int64_t replica_num_;
int64_t leader_idx_;
};
struct SubmitLogCmd {
OB_UNIS_VERSION(1);
public:
SubmitLogCmd();
SubmitLogCmd(const common::ObAddr &src,
const int64_t ls_id,
const int64_t client_id,
const palf::LogWriteBuf &log_buf_);
~SubmitLogCmd()
{
reset();
}
bool is_valid() const;
void reset();
TO_STRING_KV(K_(src), K_(ls_id));
common::ObAddr src_;
int64_t ls_id_;
int64_t client_id_;
palf::LogWriteBuf log_buf_;
};
struct SubmitLogCmdResp {
OB_UNIS_VERSION(1);
public:
SubmitLogCmdResp();
SubmitLogCmdResp(const common::ObAddr &src,
const int64_t ls_id,
const int64_t client_id);
~SubmitLogCmdResp()
{
reset();
}
bool is_valid() const;
void reset();
TO_STRING_KV(K_(src), K_(ls_id));
common::ObAddr src_;
int64_t ls_id_;
int64_t client_id_;
};
} // end namespace palfcluster
}// end namespace oceanbase
#endif

364
mittest/palf_cluster/run_palf_bench.sh Executable file
View File

@ -0,0 +1,364 @@
#!/bin/sh
TEST_MACHINE1='SERVER_IP1'
TEST_MACHINE2='SERVER_IP2'
TEST_MACHINE3='SERVER_IP3'
USER_NAME='USERNAME'
# use your own data dir path
TARGET_PATH='/yourworkdir'
function kill_server_process
{
ssh $USERNAME@$TEST_MACHINE1 bash -s << EOF
killall -9 test_palf_bench_server;
EOF
ssh $USERNAME@$TEST_MACHINE2 bash -s << EOF
killall -9 test_palf_bench_server;
EOF
ssh $USERNAME@$TEST_MACHINE3 bash -s << EOF
killall -9 test_palf_bench_server;
EOF
echo "kill_server_process success"
}
function cleanenv
{
ssh $USERNAME@$TEST_MACHINE1 bash -s << EOF
killall -9 test_palf_bench_server;
cd $TARGET_PATH;
rm -rf palf_cluster_bench_server/;
rm -rf test_palf_bench_server;
rm -rf *so;
EOF
ssh $USERNAME@$TEST_MACHINE2 bash -s << EOF
killall -9 test_palf_bench_server;
cd $TARGET_PATH;
rm -rf palf_cluster_bench_server/;
rm -rf test_palf_bench_server;
rm -rf *so;
EOF
ssh $USERNAME@$TEST_MACHINE3 bash -s << EOF
killall -9 test_palf_bench_server;
cd $TARGET_PATH;
rm -rf palf_cluster_bench_server/;
rm -rf test_palf_bench_server;
rm -rf *so;
EOF
echo "cleanenv success"
}
function send_server_binary
{
cleanenv
find ../../ -name *.so | xargs -I {} scp {} $USERNAME@$TEST_MACHINE1:$TARGET_PATH
scp test_palf_bench_server $USERNAME@$TEST_MACHINE1:$TARGET_PATH
ssh $USERNAME@$TEST_MACHINE1 bash -s << EOF
cd $TARGET_PATH;
scp *.so $USERNAME@$TEST_MACHINE2:$TARGET_PATH
scp test_palf_bench_server $USERNAME@$TEST_MACHINE2:$TARGET_PATH
scp *.so $USERNAME@$TEST_MACHINE3:$TARGET_PATH
scp test_palf_bench_server $USERNAME@$TEST_MACHINE3:$TARGET_PATH
EOF
echo "send_server_binary success"
}
# thread_num
# log_size
function startserver
{
ssh $USERNAME@$TEST_MACHINE1 bash -s << EOF
export LD_LIBRARY_PATH=$TARGET_PATH;
cd $TARGET_PATH;
./test_palf_bench_server $1 $2 > /dev/null 2>&1 &
EOF
ssh $USERNAME@$TEST_MACHINE2 bash -s << EOF
export LD_LIBRARY_PATH=$TARGET_PATH;
cd $TARGET_PATH;
./test_palf_bench_server $1 $2 > /dev/null 2>&1 &
EOF
ssh $USERNAME@$TEST_MACHINE3 bash -s << EOF
export LD_LIBRARY_PATH=$TARGET_PATH;
cd $TARGET_PATH;
./test_palf_bench_server $1 $2 > /dev/null 2>&1 &
EOF
sleep 5
echo "startserver success"
}
# thread_num
# log_size
# server no
function start_one_server
{
SERVER=""
if [[ $3 == '1' ]];
then
SERVER=TEST_MACHINE1
elif [[ $3 == '2' ]];
then
SERVER=TEST_MACHINE2
elif [[ $3 == '3' ]];
then
SERVER=TEST_MACHINE3
else
echo "invalid arguments"
fi;
ssh $USERNAME@$SERVER bash -s << EOF
export LD_LIBRARY_PATH=$TARGET_PATH;
cd $TARGET_PATH;
./test_palf_bench_server $1 $2 > /dev/null 2>&1
EOF
}
function start_local_client()
{
# $1 thread_number
# $2 nbytes
# $3 palf_group_number
# $4 replica_num
echo "start local client" $@
./test_palf_bench_client $1 $2 $3 $4
}
# $1 thread_number
# $2 nbytes
# $3 replica_num
# $4 freeze_us
# $5 experiment name
function generate_result
{
result_dir_name="palf_raw_result_"$5
append_result_name=$result_dir_name"/palf_append_"$1"_"$2"_"$3"_"$4".result"
io_result_name=$result_dir_name"/palf_io_"$1"_"$2"_"$3"_"$4".result"
group_result_name=$result_dir_name"/palf_group_"$1"_"$2"_"$3"_"$4".result"
ssh $USERNAME@$TEST_MACHINE1 bash -s << EOF
cd $TARGET_PATH;
mkdir -p $result_dir_name;
grep l_append palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'append_cnt=[0-9],' > $append_result_name;
grep inner_write_impl_ palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'l_io_cnt=[0-9],' > $io_result_name;
grep 'GROUP LOG INFO' palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'total_group_log_cnt=[0-9],' > $group_result_name;
EOF
ssh $USERNAME@$TEST_MACHINE2 bash -s << EOF
cd $TARGET_PATH;
mkdir -p $result_dir_name;
grep l_append palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'append_cnt=[0-9],' > $append_result_name;
grep inner_write_impl_ palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'l_io_cnt=[0-9],' > $io_result_name;
grep 'GROUP LOG INFO' palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'total_group_log_cnt=[0-9],' > $group_result_name;
EOF
ssh $USERNAME@$TEST_MACHINE3 bash -s << EOF
cd $TARGET_PATH;
mkdir -p $result_dir_name;
grep l_append palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'append_cnt=[0-9],' > $append_result_name;
grep inner_write_impl_ palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'l_io_cnt=[0-9],' > $io_result_name;
grep 'GROUP LOG INFO' palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'total_group_log_cnt=[0-9],' > $group_result_name;
EOF
}
# $1 thread_number
# $2 nbytes
# $3 palf_group_number
# $4 replica_num
# $5 freeze_us
# $6 exp1,exp2
function run_experiment_once
{
send_server_binary
echo "start experiment: "$6", thread_num: " $1 "log_size: " $2 " freeze_us: " $5
kill_server_process
startserver $1 $2
./test_palf_bench_client $1 $2 $3 $4
sleep 20
kill_server_process
generate_result $1 $2 $4 $5 $6
}
# $1 thread_number
# $2 nbytes
# $3 palf_group_number
# $4 replica_num
# $5 freeze_us
# $6 exp1,exp2
# $7 leader/follower
function run_experiment_once_with_failure
{
# send_server_binary
echo "start experiment: "$6", thread_num: " $1 "log_size: " $2 " freeze_us: " $5
kill_server_process
startserver $1 $2
./test_palf_bench_client $1 $2 $3 $4
sleep 20
FAIL_MACHINE=""
if [[ $7 == 'leader' ]];
then
FAIL_MACHINE=$TEST_MACHINE1
else
FAIL_MACHINE=$TEST_MACHINE2
fi;
ssh $USERNAME@$FAIL_MACHINE bash -s << EOF
killall -9 test_palf_bench_server;
EOF
sleep 30
kill_server_process
generate_result $1 $2 $4 $5 $6
}
# $1 thread_number
# $2 nbytes
# $3 palf_group_number
# $4 replica_num
# $5 freeze_us
# $6 exp1,exp2
# $7 leader/follower
function run_experiment_once_measure_reconfirm
{
send_server_binary
echo "start experiment: "$6", thread_num: " $1 "log_size: " $2 " freeze_us: " $5
kill_server_process
startserver $1 $2
./test_palf_bench_client $1 $2 $3 $4
loop_count=100
round_idx=0
fail_server_idx=1
while [ $round_idx -lt loop_count ];
do
sleep 10
let round_idx++
done;
FAIL_MACHINE=""
if [[ $7 == 'leader' ]];
then
FAIL_MACHINE=$TEST_MACHINE1
else
FAIL_MACHINE=$TEST_MACHINE2
fi;
ssh $USERNAME@$FAIL_MACHINE bash -s << EOF
killall -9 test_palf_bench_server;
EOF
sleep 30
kill_server_process
generate_result $1 $2 $4 $5 $6
}
function experiment1
{
log_size_array=(32 64 128 256 512 1024 2048 4096 8192)
thread_num_n_max=17
run_round=1
thread_num_n=1
while [ $thread_num_n -lt $thread_num_n_max ]
do
thread_num=1
let thread_num="500*thread_num_n"
let thread_num_n++
for log_size in ${log_size_array[@]}
do
echo "start run experiment1, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size
run_experiment_once $thread_num $log_size 1 3 1000 exp1_wo_reduce
let run_round++
done;
done;
}
# test for figure, less clients
function experiment1_less_clients
{
send_server_binary
log_size_array=(512)
thread_numbers=(1 2 5 10 20 50 100 200 500)
run_round=1
for thread_num in ${thread_numbers[@]}
do
for log_size in ${log_size_array[@]}
do
echo "start run experiment1_less_clients, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size
run_experiment_once $thread_num $log_size 1 3 1000 exp1_less_clients
let run_round++
done;
done;
}
# test freeze logs right now
function experiment3
{
log_size_array=(512)
thread_num_n_max=17
run_round=1
thread_num_n=3
while [ $thread_num_n -lt $thread_num_n_max ]
do
thread_num=1
let thread_num="500*thread_num_n"
let thread_num_n++
for log_size in ${log_size_array[@]}
do
echo "start run experiment3, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size
run_experiment_once $thread_num $log_size 1 3 1000 exp3
let run_round++
done;
break
done;
}
# freeze immediately or freeze adaptively
function experiment4
{
send_server_binary
run_round=1
log_size_array=(0)
# log_size_array=(512)
# thread_numbers=(1 2 5 10 20 50 100 200 500 1000 2000)
thread_numbers=(500)
for thread_num in ${thread_numbers[@]}
do
for log_size in ${log_size_array[@]}
do
echo "start run experiment4, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size
# run_experiment_once $thread_num $log_size 1 3 1000 exp4-noaggre
# run_experiment_once $thread_num $log_size 1 3 1000 exp4-period
run_experiment_once $thread_num $log_size 1 3 1000 exp4-adaptively
# run_experiment_once $thread_num $log_size 1 3 1000 exp4-feedback
let run_round++
done;
done;
}
function experiment5
{
send_server_binary
run_round=1
thread_num=5000
log_size=512
echo "start run experiment5, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size
run_experiment_once_with_failure $thread_num $log_size 1 3 1000 exp5_leader_failure leader
# echo "start run experiment5, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size
# run_experiment_once_with_failure $thread_num $log_size 1 3 1000 exp5_follower_failure follower
}
function main
{
# run_experiment_once 100 16 1 3 1000 exp_test_bw
# experiment1
# send_server_binary
# experiment4
# experiment1_less_clients
# experiment5
run_experiment_once 1500 512 1 3 1000 exp_test
}
main "$@"

140
mittest/palf_cluster/test_palf_bench_client.cpp Normal file
View File

@ -0,0 +1,140 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include "io/easy_connection.h"
#include "lib/file/file_directory_utils.h"
#include "lib/ob_errno.h"
#include "lib/oblog/ob_log.h"
#include "lib/utility/utility.h"
#include <cstdio>
#include <signal.h>
#include "lib/utility/ob_defer.h"
#include "share/ob_errno.h"
#define private public
#include "logservice/palf/log_define.h"
#include "logservice/palf/log_rpc_processor.h"
#include "env/ob_simple_log_cluster.h"
#undef private
const std::string TEST_NAME = "palf_cluster_bench_client";
using namespace oceanbase::common;
using namespace oceanbase;
namespace oceanbase
{
using namespace logservice;
namespace unittest
{
char *level;
int thread_num_arg = 100;
int nbytes_arg = 500;
int palf_group_number_arg = 1;
int replica_num_arg = 3;
ObAddr server1(ObAddr::VER::IPV4, "SERVER_IP1", ObSimpleLogCluster::RPC_PORT);
ObAddr server2(ObAddr::VER::IPV4, "SERVER_IP2", ObSimpleLogCluster::RPC_PORT);
ObAddr server3(ObAddr::VER::IPV4, "SERVER_IP3", ObSimpleLogCluster::RPC_PORT);
class TestObPalfClusterBench : public ObSimpleLogCluster
{
public:
std::vector<int64_t> palf_id_list_;
std::vector<common::ObAddr> palf_leader_list_;
std::vector<common::ObMemberList> palf_group_list_;
std::vector<common::ObAddr> server_list_;
public:
TestObPalfClusterBench() : ObSimpleLogCluster()
{
server_list_.push_back(server1);
server_list_.push_back(server2);
server_list_.push_back(server3);
}
int create_palf_group()
{
int ret = OB_SUCCESS;
const int64_t replica_num = oceanbase::unittest::replica_num_arg;
palfcluster::LogService *log_service = get_log_server()->get_log_service();
obrpc::PalfClusterRpcProxy *rpc_proxy = NULL;
const int64_t RPC_TIMEOUT_US = 5 * 1000 * 1000;
ObAddr self;
ObMemberList memberlist;
for (int i = 0; i < replica_num; i++) {
memberlist.add_server(server_list_[i]);
}
if (OB_ISNULL(log_service)) {
CLOG_LOG(ERROR, "get_log_service failed");
} else if (FALSE_IT(rpc_proxy = log_service->get_rpc_proxy())) {
} else if (FALSE_IT(self = log_service->get_self())) {
} else {
for (int i = 0; i < oceanbase::unittest::palf_group_number_arg; i++) {
const int64_t leader_index = i % replica_num;
ObAddr leader;
memberlist.get_server_by_index(leader_index, leader);
palf_leader_list_.push_back(leader);
for (int j = 0; j < replica_num; j++) {
palfcluster::LogCreateReplicaCmd req(self, i + 1, memberlist, replica_num, leader_index);
if (OB_FAIL(rpc_proxy->to(server_list_[j]).timeout(RPC_TIMEOUT_US).trace_time(true).
max_process_handler_time(RPC_TIMEOUT_US).by(MTL_ID()).send_create_replica_cmd(req, NULL))) {
CLOG_LOG(ERROR, "create_replica failed", KR(ret), K(req), K(server_list_[j]));
break;
}
}
}
LOG_STDOUT("create_replica success\n");
}
return ret;
}
};
std::string ObSimpleLogCluster::test_name_ = TEST_NAME;
TEST_F(TestObPalfClusterBench, multiple_replica)
{
int ret = OB_SUCCESS;
// client mode
OB_LOGGER.set_log_level("ERROR");
if (OB_FAIL(create_palf_group())) {
CLOG_LOG(ERROR, "create_palf_group failed", K(ret));
} else {
sleep(3);
if (OB_FAIL(create_palf_group())) {
CLOG_LOG(ERROR, "create_palf_group failed", K(ret));
}
}
}
} // end unittest
} // end oceanbase
// Notes: How to write a new module integrate test case in logservice?
// 1. cp test_ob_simple_log_basic_func.cpp test_ob_simple_log_xxx.cpp
// 2. modify const string TEST_NAME, class name and log file name in
// test_ob_simple_log_xxx.cpp
// 3. add ob_unittest_clog() item and set label for test_ob_simple_log_xxx in
// unittest/cluster/CMakeFiles.txt
// 4. write new TEST_F
int main(int argc, char **argv)
{
if (argc > 1) {
oceanbase::unittest::thread_num_arg = strtol(argv[1], NULL, 10);
oceanbase::unittest::nbytes_arg = strtol(argv[2], NULL, 10);
oceanbase::unittest::palf_group_number_arg = strtol(argv[3], NULL, 10);
oceanbase::unittest::replica_num_arg = strtol(argv[4], NULL, 10);
}
RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME);
}

144
mittest/palf_cluster/test_palf_bench_server.cpp Normal file
View File

@ -0,0 +1,144 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include "io/easy_connection.h"
#include "lib/file/file_directory_utils.h"
#include "lib/ob_errno.h"
#include "lib/oblog/ob_log.h"
#include "lib/utility/utility.h"
#include <cstdio>
#include <signal.h>
#include "lib/utility/ob_defer.h"
#include "share/ob_errno.h"
#define private public
#include "logservice/palf/log_define.h"
#include "logservice/palf/log_rpc_processor.h"
#include "env/ob_simple_log_cluster.h"
#undef private
const std::string TEST_NAME = "palf_cluster_bench_server";
using namespace oceanbase::common;
using namespace oceanbase;
namespace oceanbase
{
using namespace logservice;
namespace unittest
{
char *level;
int thread_num_arg = 100;
int nbytes_arg = 500;
int palf_group_number_arg = 1;
int replica_num_arg = 3;
ObAddr server1(ObAddr::VER::IPV4, "SERVER_IP1", ObSimpleLogCluster::RPC_PORT);
ObAddr server2(ObAddr::VER::IPV4, "SERVER_IP2", ObSimpleLogCluster::RPC_PORT);
ObAddr server3(ObAddr::VER::IPV4, "SERVER_IP3", ObSimpleLogCluster::RPC_PORT);
class TestPalfClusterBenchServer : public ObSimpleLogCluster
{
public:
std::vector<int64_t> palf_id_list_;
std::vector<common::ObAddr> palf_leader_list_;
std::vector<common::ObMemberList> palf_group_list_;
std::vector<common::ObAddr> server_list_;
public:
TestPalfClusterBenchServer() : ObSimpleLogCluster()
{
server_list_.push_back(server1);
server_list_.push_back(server2);
server_list_.push_back(server3);
}
int local_submit_log(const int64_t thread_num, const int64_t log_size)
{
int ret = OB_SUCCESS;
palfcluster::LogService *log_service = get_log_server()->get_log_service();
obrpc::PalfClusterRpcProxy *rpc_proxy = NULL;
const int64_t RPC_TIMEOUT_US = 500 * 1000 * 1000;
palfcluster::LogClientMap *log_clients = nullptr;
palfcluster::ObLogClient *log_client;
int64_t palf_id = 1;
const share::ObLSID ls_id(palf_id);
ObAddr self;
if (OB_ISNULL(log_service)) {
CLOG_LOG(ERROR, "get_log_service failed");
} else if (nullptr == (log_clients = log_service->get_log_client_map())) {
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(ERROR, "get_log_clients failed", K(ret));
} else {
int64_t unused_id;
common::ObRole role = ObRole::INVALID_ROLE;
while (true) {
role = ObRole::INVALID_ROLE;
if (OB_FAIL(log_clients->get(ls_id, log_client))) {
CLOG_LOG(ERROR, "get_log_client failed", K(ret), K(palf_id));
usleep(1000 * 1000);
} else if (OB_FAIL(log_client->get_log_handler()->get_role(role, unused_id))) {
CLOG_LOG(ERROR, "get_role fail", K(palf_id));
} else if (role == ObRole::LEADER) {
break;
CLOG_LOG(ERROR, "switch_to_leader success", K(palf_id));
} else {
usleep(100* 1000);
}
}
if (OB_FAIL(log_client->submit_append_log_task(thread_num, log_size))) {
CLOG_LOG(ERROR, "submit_log failed", K(ret));
}
}
PALF_LOG(INFO, "end test_palf_bench");
LOG_STDOUT("submit_log success\n");
return ret;
}
};
std::string ObSimpleLogCluster::test_name_ = TEST_NAME;
TEST_F(TestPalfClusterBenchServer, multiple_replica)
{
int ret = OB_SUCCESS;
OB_LOGGER.set_log_level("WDIAG");
// server mode
if (OB_FAIL(local_submit_log(oceanbase::unittest::thread_num_arg, oceanbase::unittest::nbytes_arg))) {
CLOG_LOG(ERROR, "local_submit_log failed");
}
while (true) {
usleep(1000 * 1000);
}
}
} // end unittest
} // end oceanbase
// Notes: How to write a new module integrate test case in logservice?
// 1. cp test_ob_simple_log_basic_func.cpp test_ob_simple_log_xxx.cpp
// 2. modify const string TEST_NAME, class name and log file name in
// test_ob_simple_log_xxx.cpp
// 3. add ob_unittest_clog() item and set label for test_ob_simple_log_xxx in
// unittest/cluster/CMakeFiles.txt
// 4. write new TEST_F
int main(int argc, char **argv)
{
if (argc > 1) {
oceanbase::unittest::thread_num_arg = strtol(argv[1], NULL, 10);
oceanbase::unittest::nbytes_arg = strtol(argv[2], NULL, 10);
}
RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME);
}