From 5cede3d9eab46c9ad7f95bc21ea933c9c84fcad9 Mon Sep 17 00:00:00 2001 From: BinChenn Date: Wed, 27 Sep 2023 08:43:51 +0000 Subject: [PATCH] [PALF] PALF Cluster Test Framework --- mittest/CMakeLists.txt | 1 + .../env/ob_simple_log_cluster_env.cpp | 0 .../logservice/test_ob_simple_log_apply.cpp | 4 + mittest/palf_cluster/CMakeLists.txt | 24 + mittest/palf_cluster/README.md | 77 ++ .../env/ob_simple_log_cluster.cpp | 133 +++ .../palf_cluster/env/ob_simple_log_cluster.h | 111 +++ .../palf_cluster/env/ob_simple_log_server.cpp | 814 ++++++++++++++++++ .../palf_cluster/env/ob_simple_log_server.h | 242 ++++++ .../palf_cluster/logservice/log_service.cpp | 621 +++++++++++++ mittest/palf_cluster/logservice/log_service.h | 211 +++++ .../palf_cluster/logservice/ls_adapter.cpp | 60 ++ mittest/palf_cluster/logservice/ls_adapter.h | 47 + .../palf_cluster/logservice/ob_log_client.cpp | 237 +++++ .../palf_cluster/logservice/ob_log_client.h | 314 +++++++ .../logservice/role_coordinator.cpp | 747 ++++++++++++++++ .../logservice/role_coordinator.h | 189 ++++ .../logservice/role_coordinator_handler.cpp | 197 +++++ .../logservice/role_coordinator_handler.h | 82 ++ mittest/palf_cluster/palf-cluster.diff | 364 ++++++++ .../palf_cluster/rpc/ob_log_rpc_processor.h | 79 ++ .../rpc/palf_cluster_request_handler.cpp | 123 +++ .../rpc/palf_cluster_request_handler.h | 74 ++ .../rpc/palf_cluster_rpc_processor.h | 80 ++ .../palf_cluster/rpc/palf_cluster_rpc_proxy.h | 74 ++ .../palf_cluster/rpc/palf_cluster_rpc_req.cpp | 134 +++ .../palf_cluster/rpc/palf_cluster_rpc_req.h | 94 ++ mittest/palf_cluster/run_palf_bench.sh | 364 ++++++++ .../palf_cluster/test_palf_bench_client.cpp | 140 +++ .../palf_cluster/test_palf_bench_server.cpp | 144 ++++ 30 files changed, 5781 insertions(+) mode change 100755 => 100644 mittest/logservice/env/ob_simple_log_cluster_env.cpp create mode 100644 mittest/palf_cluster/CMakeLists.txt create mode 100644 mittest/palf_cluster/README.md create mode 100644 mittest/palf_cluster/env/ob_simple_log_cluster.cpp create mode 100644 mittest/palf_cluster/env/ob_simple_log_cluster.h create mode 100644 mittest/palf_cluster/env/ob_simple_log_server.cpp create mode 100644 mittest/palf_cluster/env/ob_simple_log_server.h create mode 100644 mittest/palf_cluster/logservice/log_service.cpp create mode 100644 mittest/palf_cluster/logservice/log_service.h create mode 100644 mittest/palf_cluster/logservice/ls_adapter.cpp create mode 100644 mittest/palf_cluster/logservice/ls_adapter.h create mode 100644 mittest/palf_cluster/logservice/ob_log_client.cpp create mode 100644 mittest/palf_cluster/logservice/ob_log_client.h create mode 100644 mittest/palf_cluster/logservice/role_coordinator.cpp create mode 100644 mittest/palf_cluster/logservice/role_coordinator.h create mode 100644 mittest/palf_cluster/logservice/role_coordinator_handler.cpp create mode 100644 mittest/palf_cluster/logservice/role_coordinator_handler.h create mode 100644 mittest/palf_cluster/palf-cluster.diff create mode 100644 mittest/palf_cluster/rpc/ob_log_rpc_processor.h create mode 100644 mittest/palf_cluster/rpc/palf_cluster_request_handler.cpp create mode 100644 mittest/palf_cluster/rpc/palf_cluster_request_handler.h create mode 100644 mittest/palf_cluster/rpc/palf_cluster_rpc_processor.h create mode 100644 mittest/palf_cluster/rpc/palf_cluster_rpc_proxy.h create mode 100644 mittest/palf_cluster/rpc/palf_cluster_rpc_req.cpp create mode 100644 mittest/palf_cluster/rpc/palf_cluster_rpc_req.h create mode 100755 mittest/palf_cluster/run_palf_bench.sh create mode 100644 mittest/palf_cluster/test_palf_bench_client.cpp create mode 100644 mittest/palf_cluster/test_palf_bench_server.cpp diff --git a/mittest/CMakeLists.txt b/mittest/CMakeLists.txt index dbefd6a913..138d7f4f9a 100644 --- a/mittest/CMakeLists.txt +++ b/mittest/CMakeLists.txt @@ -2,3 +2,4 @@ add_subdirectory(logservice) add_subdirectory(simple_server) add_subdirectory(mtlenv) add_subdirectory(multi_replica) +#add_subdirectory(palf_cluster) diff --git a/mittest/logservice/env/ob_simple_log_cluster_env.cpp b/mittest/logservice/env/ob_simple_log_cluster_env.cpp old mode 100755 new mode 100644 diff --git a/mittest/logservice/test_ob_simple_log_apply.cpp b/mittest/logservice/test_ob_simple_log_apply.cpp index 9d94146230..51a2dc219c 100644 --- a/mittest/logservice/test_ob_simple_log_apply.cpp +++ b/mittest/logservice/test_ob_simple_log_apply.cpp @@ -55,6 +55,10 @@ public: { ATOMIC_INC(&failure_count_); } + int replay(ObLogReplayTask *replay_task){ + UNUSED(replay_task); + return OB_SUCCESS; + } int64_t success_count_; int64_t failure_count_; common::ObQSync ls_qs_; diff --git a/mittest/palf_cluster/CMakeLists.txt b/mittest/palf_cluster/CMakeLists.txt new file mode 100644 index 0000000000..c59e066944 --- /dev/null +++ b/mittest/palf_cluster/CMakeLists.txt @@ -0,0 +1,24 @@ +set(PALF_CLUSTER_TEST_SRCS + env/ob_simple_log_cluster.cpp + env/ob_simple_log_server.cpp + logservice/ob_log_client.cpp + logservice/log_service.cpp + logservice/ls_adapter.cpp + logservice/role_coordinator_handler.cpp + logservice/role_coordinator.cpp + rpc/palf_cluster_rpc_req.cpp + rpc/palf_cluster_request_handler.cpp) + +add_library(palf_cluster_test ${PALF_CLUSTER_TEST_SRCS}) +target_include_directories(palf_cluster_test PUBLIC + ${CMAKE_SOURCE_DIR}/unittest ${CMAKE_SOURCE_DIR}/mittest) +target_link_libraries(palf_cluster_test PUBLIC oceanbase) + +function(ob_unittest_clog case) + ob_unittest(${ARGV}) + target_link_libraries(${case} PRIVATE gtest gmock palf_cluster_test oceanbase) + set_tests_properties(${case} PROPERTIES TIMEOUT 600) +endfunction() + +ob_unittest_clog(test_palf_bench_server test_palf_bench_server.cpp) +ob_unittest_clog(test_palf_bench_client test_palf_bench_client.cpp) \ No newline at end of file diff --git a/mittest/palf_cluster/README.md b/mittest/palf_cluster/README.md new file mode 100644 index 0000000000..36e3f5d351 --- /dev/null +++ b/mittest/palf_cluster/README.md @@ -0,0 +1,77 @@ +# A Test Framework for PALF Cluster +The framework is designed and implentmented for evaluating PALF performance in a real cluster. + +## Metrics +- Throughput +- Latency +- IOPS +... + +## Usage + +1. Use your own IP and wrok directory + +**IP** +- `mittest/palf_cluster/test_palf_bench_server.cpp:45` +- `mittest/palf_cluster/test_palf_bench_client.cpp:45` +- `mittest/palf_cluster/run_palf_bench.sh:2` + +**PORT** +default 53212 +- `mittest/palf_cluster/env/ob_simple_log_cluster.h:92` + +**workdir** +- `mittest/palf_cluster/run_palf_bench.sh:` + +2. Apply some statistic logs and RPC definitions + +``` +cd oceanbase/ +mv mittest/palf_cluster/palf-cluster.diff ./ +git apply palf-cluster.diff +``` + +3. Build + + 1. uncomment `mittest/CMakeLists.txt:5` + 2. remove `OB_BUILD_CLOSE_MODULES` in `CMakeLists.txt:162` and `CMakeLists.txt:167` if you want to build in opensource mode + 3. build + ``` + cd oceanbse/ + bash build.sh release --init --make + ``` + +4. Run test +``` +cd oceanbase/ +cp mittest/palf_cluster/run_palf_bench.sh build_release/mittest/palf_cluster/ +cd build_release/mittest/palf_cluster/ +./run_palf_bench.sh +``` + +5. Check Result + +check workdir of the server with minimal IP:PORT +``` +$tail palf_raw_result_exp_test/palf_append_1500_512_3_1000.result -n 5 +[2023-09-27 11:29:25.048444] EDIAG [CLOG] try_handle_cb_queue (ob_log_apply_service.cpp:505) [62033][T1_ApplySrv5][T1][Y0-0000000000000000-0-0] [lt=51][errcode=0] result:(l_append_cnt=602962, l_log_body_size=308716544, l_rt=1465294805, avg_body_size=512, avg_rt=2430) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d8dd1 0x55807b1d8a88 0x55807b1d8886 0x55807b1d867d 0xc9e7562 0xa5f6f30 0xa5f5d86 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d +[2023-09-27 11:29:26.049100] EDIAG [CLOG] try_handle_cb_queue (ob_log_apply_service.cpp:505) [62037][T1_ApplySrv9][T1][Y0-0000000000000000-0-0] [lt=39][errcode=0] result:(l_append_cnt=590605, l_log_body_size=302389760, l_rt=1463385996, avg_body_size=512, avg_rt=2477) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d8dd1 0x55807b1d8a88 0x55807b1d8886 0x55807b1d867d 0xc9e7562 0xa5f6f30 0xa5f5d86 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d +[2023-09-27 11:29:27.050335] EDIAG [CLOG] try_handle_cb_queue (ob_log_apply_service.cpp:505) [62033][T1_ApplySrv5][T1][Y0-0000000000000000-0-0] [lt=28][errcode=0] result:(l_append_cnt=604878, l_log_body_size=309697536, l_rt=1462297116, avg_body_size=512, avg_rt=2417) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d8dd1 0x55807b1d8a88 0x55807b1d8886 0x55807b1d867d 0xc9e7562 0xa5f6f30 0xa5f5d86 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d +[2023-09-27 11:29:28.051049] EDIAG [CLOG] try_handle_cb_queue (ob_log_apply_service.cpp:505) [62033][T1_ApplySrv5][T1][Y0-0000000000000000-0-0] [lt=28][errcode=0] result:(l_append_cnt=593036, l_log_body_size=303634432, l_rt=1462893463, avg_body_size=512, avg_rt=2466) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d8dd1 0x55807b1d8a88 0x55807b1d8886 0x55807b1d867d 0xc9e7562 0xa5f6f30 0xa5f5d86 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d +[2023-09-27 11:29:29.051212] EDIAG [CLOG] try_handle_cb_queue (ob_log_apply_service.cpp:505) [62034][T1_ApplySrv6][T1][Y0-0000000000000000-0-0] [lt=28][errcode=0] result:(l_append_cnt=596583, l_log_body_size=305449984, l_rt=1464339470, avg_body_size=511, avg_rt=2454) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d8dd1 0x55807b1d8a88 0x55807b1d8886 0x55807b1d867d 0xc9e7562 0xa5f6f30 0xa5f5d86 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d + +$tail palf_raw_result_exp_test/palf_io_1500_512_3_1000.result -n 5 +[2023-09-27 11:29:24.910336] EDIAG [CLOG] inner_write_impl_ (log_block_handler.cpp:458) [61989][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=28][errcode=0] io result:(l_io_cnt=1438, l_log_size=333262848, avg_size=231754) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d3228 0x55807b1d2edf 0x55807b1d2cd8 0x55807b1d2ad7 0xca638f1 0xa73358f 0xa732cfa 0xa732761 0xa731caf 0xcab6358 0xa7310b4 0xa73012b 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d +[2023-09-27 11:29:25.911515] EDIAG [CLOG] inner_write_impl_ (log_block_handler.cpp:458) [61989][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=19][errcode=0] io result:(l_io_cnt=1441, l_log_size=330817536, avg_size=229574) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d3228 0x55807b1d2edf 0x55807b1d2cd8 0x55807b1d2ad7 0xca638f1 0xa73358f 0xa732cfa 0xa732761 0xa731caf 0xcab6358 0xa7310b4 0xa73012b 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d +[2023-09-27 11:29:26.911259] EDIAG [CLOG] inner_write_impl_ (log_block_handler.cpp:458) [61989][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=22][errcode=0] io result:(l_io_cnt=1425, l_log_size=331571200, avg_size=232681) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d3228 0x55807b1d2edf 0x55807b1d2cd8 0x55807b1d2ad7 0xca638f1 0xa73358f 0xa732cfa 0xa732761 0xa731caf 0xcab6358 0xa7310b4 0xa73012b 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d +[2023-09-27 11:29:27.912235] EDIAG [CLOG] inner_write_impl_ (log_block_handler.cpp:458) [61989][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=21][errcode=0] io result:(l_io_cnt=1406, l_log_size=328134656, avg_size=233381) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d3228 0x55807b1d2edf 0x55807b1d2cd8 0x55807b1d2ad7 0xca638f1 0xa73358f 0xa732cfa 0xa732761 0xa731caf 0xcab6358 0xa7310b4 0xa73012b 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d +[2023-09-27 11:29:28.912356] EDIAG [CLOG] inner_write_impl_ (log_block_handler.cpp:458) [61989][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=19][errcode=0] io result:(l_io_cnt=1413, l_log_size=332730368, avg_size=235477) BACKTRACE:0xde38230 0xdaa64e6 0x55807b1d3228 0x55807b1d2edf 0x55807b1d2cd8 0x55807b1d2ad7 0xca638f1 0xa73358f 0xa732cfa 0xa732761 0xa731caf 0xcab6358 0xa7310b4 0xa73012b 0xe111e5a 0xe10e640 0x7ff51add6e25 0x7ff51a1d4f1d + + +$tail palf_raw_result_exp_test/palf_group_1500_512_3_1000.result -n 5 +[2023-09-27 11:29:25.042978] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1056) [61992][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=41] [PALF STAT GROUP LOG INFO](palf_id=1, self="SERVER_IP1:53212", role="LEADER", total_group_log_cnt=1667, avg_log_batch_cnt=361, total_group_log_size=327750672, avg_group_log_size=196611) +[2023-09-27 11:29:26.046598] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1056) [62250][][T0][Y0-0000000000000000-0-0] [lt=44] [PALF STAT GROUP LOG INFO](palf_id=1, self="SERVER_IP1:53212", role="LEADER", total_group_log_cnt=1613, avg_log_batch_cnt=367, total_group_log_size=322325464, avg_group_log_size=199829) +[2023-09-27 11:29:27.046847] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1056) [61992][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=21] [PALF STAT GROUP LOG INFO](palf_id=1, self="SERVER_IP1:53212", role="LEADER", total_group_log_cnt=1680, avg_log_batch_cnt=360, total_group_log_size=329294128, avg_group_log_size=196008) +[2023-09-27 11:29:28.048014] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1056) [61992][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=22] [PALF STAT GROUP LOG INFO](palf_id=1, self="SERVER_IP1:53212", role="LEADER", total_group_log_cnt=1607, avg_log_batch_cnt=369, total_group_log_size=322903656, avg_group_log_size=200935) +[2023-09-27 11:29:29.048263] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1056) [61992][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=56] [PALF STAT GROUP LOG INFO](palf_id=1, self="SERVER_IP1:53212", role="LEADER", total_group_log_cnt=1611, avg_log_batch_cnt=370, total_group_log_size=324553496, avg_group_log_size=201460) +``` diff --git a/mittest/palf_cluster/env/ob_simple_log_cluster.cpp b/mittest/palf_cluster/env/ob_simple_log_cluster.cpp new file mode 100644 index 0000000000..7aed167a06 --- /dev/null +++ b/mittest/palf_cluster/env/ob_simple_log_cluster.cpp @@ -0,0 +1,133 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_simple_log_cluster.h" +#include "common/ob_member_list.h" +#include "share/allocator/ob_tenant_mutil_allocator_mgr.h" +#include "ob_mittest_utils.h" +#include "lib/alloc/memory_dump.h" +#include "lib/alloc/alloc_func.cpp" +#include "lib/allocator/ob_mem_leak_checker.h" +#include "lib/allocator/ob_libeasy_mem_pool.h" +#include + +namespace oceanbase +{ +namespace unittest +{ +ObSimpleLogServer* ObSimpleLogCluster::log_server_; +int64_t ObSimpleLogCluster::node_cnt_ = 1; +common::ObMemberList ObSimpleLogCluster::node_list_ = ObMemberList(); +bool ObSimpleLogCluster::is_started_ = false; +char ObSimpleLogCluster::sig_buf_[sizeof(ObSignalWorker) + sizeof(observer::ObSignalHandle)]; +ObSignalWorker *ObSimpleLogCluster::sig_worker_ = new (sig_buf_) ObSignalWorker(); +observer::ObSignalHandle *ObSimpleLogCluster::signal_handle_ = new (sig_worker_ + 1) observer::ObSignalHandle(); + +void ObSimpleLogCluster::SetUpTestCase() +{ + SERVER_LOG(INFO, "SetUpTestCase"); + int ret = OB_SUCCESS; + if (!is_started_) { + ret = start(); + } + ASSERT_EQ(ret, OB_SUCCESS); +} + +void ObSimpleLogCluster::TearDownTestCase() +{ + SERVER_LOG(INFO, "TearDownTestCase", K(node_cnt_)); + int ret = OB_SUCCESS; + + ret = close(); + ASSERT_EQ(ret, OB_SUCCESS); + ob_delete(log_server_); +} + +int ObSimpleLogCluster::start() +{ + int ret = OB_SUCCESS; + ObTenantMutilAllocatorMgr::get_instance().init(); + ObMemoryDump::get_instance().init(); + // set easy allocator for watching easy memory holding + easy_pool_set_allocator(ob_easy_realloc); + ev_set_allocator(ob_easy_realloc); + lib::set_memory_limit(100L * 1000L * 1000L * 1000L); + if (sig_worker_ != nullptr && OB_FAIL(sig_worker_->start())) { + SERVER_LOG(ERROR, "Start signal worker error", K(ret)); + } else if (signal_handle_ != nullptr && OB_FAIL(signal_handle_->start())) { + SERVER_LOG(ERROR, "Start signal handle error", K(ret)); + } else if (OB_FAIL(generate_sorted_server_list_(node_cnt_))) { + } else { + for (int i = 0; OB_SUCC(ret) && i < node_cnt_; i++) { + auto svr = OB_NEW(ObSimpleLogServer, "TestBase"); + common::ObAddr server; + if (OB_FAIL(node_list_.get_server_by_index(i, server))) { + } else if (OB_FAIL(svr->simple_init(test_name_, server, 1, true))) { + SERVER_LOG(WARN, "simple_init failed", K(ret), K(i), K_(node_list)); + } else if (OB_FAIL(svr->simple_start(true))) { + SERVER_LOG(WARN, "simple_start failed", K(ret), K(i), K_(node_list)); + } else { + log_server_ = svr; + } + } + is_started_ = true; + SERVER_LOG(INFO, "ObSimpleLogCluster started", K(ret), K_(node_cnt), K_(node_list)); + } + return ret; +} + +int ObSimpleLogCluster::close() +{ + int ret = OB_SUCCESS; + ret = log_server_->simple_close(true); + if (OB_FAIL(ret)) { + SERVER_LOG(WARN, "simple_close failed", K(ret)); + } + return ret; +} + +int ObSimpleLogCluster::generate_sorted_server_list_(const int64_t node_cnt) +{ + int ret = OB_SUCCESS; + // each get_rpc_port calling will get two available ports, + // so just get node_cnt / 2 + 1 times + const int64_t get_port_cnt = node_cnt / 2 + 1; + for (int i = 0; i < get_port_cnt; i++) { + int server_fd = 0; + const std::string local_ip = get_local_ip(); + // const int64_t port = get_rpc_port(server_fd); + const int64_t port = RPC_PORT; + common::ObAddr addr; + if (0 == port) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(ERROR, "get_rpc_port failed", K(ret), K(port)); + break; + } else if (local_ip == "") { + ret = OB_ERR_UNEXPECTED; + } else if (false == addr.set_ip_addr(local_ip.c_str(), port)) { + SERVER_LOG(ERROR, "set_ip_addr failed", K(local_ip.c_str()), K(port), K(addr)); + } else if (OB_FAIL(node_list_.add_server(addr))) { + PALF_LOG(WARN, "add_server failed", K(ret)); + } else if (false == addr.set_ip_addr(local_ip.c_str(), port + 1)) { + SERVER_LOG(ERROR, "set_ip_addr failed", K(local_ip.c_str()), K(port), K(addr)); + } else if (node_list_.get_member_number() < node_cnt && OB_FAIL(node_list_.add_server(addr))) { + PALF_LOG(WARN, "add_server failed", K(ret)); + } + } + if (OB_SUCC(ret)) { + SERVER_LOG(INFO, "simple log cluster node_list", K_(node_list)); + } + return ret; +} + +} // end unittest +} // end oceanbase \ No newline at end of file diff --git a/mittest/palf_cluster/env/ob_simple_log_cluster.h b/mittest/palf_cluster/env/ob_simple_log_cluster.h new file mode 100644 index 0000000000..0bf66826b0 --- /dev/null +++ b/mittest/palf_cluster/env/ob_simple_log_cluster.h @@ -0,0 +1,111 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once + +#define USING_LOG_PREFIX RPC_TEST + +#include +#include +#include "lib/hash/ob_array_hash_map.h" // ObArrayHashMap +#include "ob_simple_log_server.h" + +namespace oceanbase +{ +namespace unittest +{ +#define SET_CASE_LOG_FILE(TEST_NAME, CASE_NAME) \ + const std::string log_file_name = TEST_NAME + "/" + CASE_NAME + ".log";\ + const std::string ele_log_file_name = TEST_NAME + "/" + CASE_NAME + ".election.log";\ + OB_LOGGER.set_file_name(log_file_name.c_str(),\ + true, false, NULL, \ + ele_log_file_name.c_str(), NULL); + +#define RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME) \ + void *ptr = malloc(SIG_STACK_SIZE); \ + abort_unless(ptr != nullptr); \ + stack_t nss; \ + stack_t oss; \ + bzero(&nss, sizeof(nss)); \ + bzero(&oss, sizeof(oss)); \ + nss.ss_sp = ptr; \ + nss.ss_size = SIG_STACK_SIZE; \ + abort_unless(0 == sigaltstack(&nss, &oss)); \ + DEFER(sigaltstack(&oss, nullptr)); \ + if (OB_SUCCESS != oceanbase::observer::ObSignalHandle::change_signal_mask()) { \ + } \ + ::oceanbase::common::g_redirect_handler = true; \ + oceanbase::palf::election::GLOBAL_INIT_ELECTION_MODULE(); \ + oceanbase::common::ObClockGenerator::init();\ + sleep(15); \ + const std::string rm_base_dir_cmd = "rm -rf " + TEST_NAME; \ + const std::string rm_log_cmd = "rm -f ./" + TEST_NAME + "*log*"; \ + const std::string mk_base_dir_cm = "mkdir " + TEST_NAME; \ + system(rm_base_dir_cmd.c_str()); \ + system(rm_log_cmd.c_str()); \ + system(mk_base_dir_cm.c_str()); \ + ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION); \ + ObClusterVersion::get_instance().update_cluster_version(DATA_CURRENT_VERSION); \ + const std::string log_file_name = TEST_NAME+"/"+TEST_NAME + ".log"; \ + const std::string ele_log_file_name = TEST_NAME+"/"+TEST_NAME + ".election.log"; \ + const std::string rs_log_file_name = TEST_NAME+"/"+TEST_NAME + ".rs.log"; \ + OB_LOGGER.set_file_name(log_file_name.c_str(), true, false, rs_log_file_name.c_str(), \ + ele_log_file_name.c_str(), NULL); \ + OB_LOGGER.set_log_level("INFO"); \ + OB_LOGGER.set_enable_log_limit(false); \ + OB_LOGGER.set_enable_async_log(false); \ + SERVER_LOG(INFO, "begin unittest"); \ + ::testing::InitGoogleTest(&argc, argv); \ + return RUN_ALL_TESTS(); + +class ObSimpleLogCluster : public testing::Test +{ +public: + ObSimpleLogCluster() + { + SERVER_LOG(INFO, "ObSimpleLogCluster construct"); + } + virtual ~ObSimpleLogCluster() {} + static int init(); + static int start(); + static int close(); + std::string &get_test_name() { return test_name_; } + ObSimpleLogServer *get_log_server() { return log_server_; } + +private: + static int generate_sorted_server_list_(const int64_t node_cnt); + +protected: + static void SetUpTestCase(); + static void TearDownTestCase(); + +public: + static constexpr int64_t RPC_PORT = 53212; + + static ObSimpleLogServer *log_server_; + static bool is_started_; + static std::string test_name_; + static ObMemberList node_list_; + static int64_t node_cnt_; + // static std::vector cluster_; + // static ObMemberList member_list_; + // static common::ObArrayHashMap member_region_map_; + // static int64_t member_cnt_; + // static int64_t node_idx_base_; + //thread to deal signals + static char sig_buf_[sizeof(ObSignalWorker) + sizeof(observer::ObSignalHandle)]; + static ObSignalWorker *sig_worker_; + static observer::ObSignalHandle *signal_handle_; +}; + +} // end unittest +} // end oceanbase \ No newline at end of file diff --git a/mittest/palf_cluster/env/ob_simple_log_server.cpp b/mittest/palf_cluster/env/ob_simple_log_server.cpp new file mode 100644 index 0000000000..bedc7d6e4e --- /dev/null +++ b/mittest/palf_cluster/env/ob_simple_log_server.cpp @@ -0,0 +1,814 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define private public +#include "ob_simple_log_server.h" +#undef private + +#include "lib/file/file_directory_utils.h" +#include "logservice/palf/log_define.h" +#include +#define USING_LOG_PREFIX RPC_TEST +#include +#include +#include +#include +#include +#include "util/easy_mod_stat.h" +#include "lib/oblog/ob_log.h" +#include "lib/oblog/ob_log_module.h" +#include "lib/random/ob_random.h" +#include "common/log/ob_log_constants.h" +#include "share/ob_io_device_helper.h" +#include "share/ob_thread_mgr.h" +#include "logservice/palf/palf_options.h" + +namespace oceanbase +{ +namespace unittest +{ +using namespace oceanbase; +using namespace oceanbase::rpc; +using namespace oceanbase::rpc::frame; +using namespace oceanbase::obrpc; +using namespace oceanbase::common; +using namespace oceanbase::share; +using namespace palf; +using namespace palf::election; +using namespace logservice; + +uint32_t get_local_addr(const char *dev_name) +{ + int fd, intrface; + struct ifreq buf[16]; + struct ifconf ifc; + + if ((fd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) { + return 0; + } + + ifc.ifc_len = sizeof(buf); + ifc.ifc_buf = (caddr_t)buf; + if (ioctl(fd, SIOCGIFCONF, (char *)&ifc) != 0) { + close(fd); + return 0; + } + + intrface = static_cast(ifc.ifc_len / sizeof(struct ifreq)); + while (intrface-- > 0) { + if (ioctl(fd, SIOCGIFFLAGS, (char *)&buf[intrface]) != 0) { + continue; + } + if ((buf[intrface].ifr_flags & IFF_LOOPBACK) != 0) + continue; + if (!(buf[intrface].ifr_flags & IFF_UP)) + continue; + if (dev_name != NULL && strcmp(dev_name, buf[intrface].ifr_name)) + continue; + if (!(ioctl(fd, SIOCGIFADDR, (char *)&buf[intrface]))) { + close(fd); + return ((struct sockaddr_in *)(&buf[intrface].ifr_addr))->sin_addr.s_addr; + } + } + close(fd); + return 0; +} + +std::string get_local_ip() +{ + uint32_t ip = get_local_addr("bond0"); + if (ip == 0) { + ip = get_local_addr("eth0"); + } + if (ip == 0) { + return ""; + } + return inet_ntoa(*(struct in_addr *)(&ip)); +} + +int ObSimpleLogServer::simple_init( + const std::string &cluster_name, + const common::ObAddr &addr, + const int64_t node_id, + const bool is_bootstrap = false) +{ + int ret = OB_SUCCESS; + ObTimeGuard guard("simple_init", 0); + SERVER_LOG(INFO, "simple_log_server simple_init start", K(node_id), K(addr_)); + tenant_base_ = OB_NEW(ObTenantBase, "TestBase", node_id); + tenant_base_->init(); + // tenant_base_->set(&log_service_); + ObTenantEnv::set_tenant(tenant_base_); + // assert(&log_service_ == MTL(palfcluster::LogService*)); + guard.click("init tenant_base"); + node_id_ = node_id; + if (is_bootstrap && OB_FAIL(init_memory_dump_timer_())) { + SERVER_LOG(ERROR, "init_memory_dump_timer_ failed", K(ret), K_(node_id)); + } else if (FALSE_IT(guard.click("init_memory_dump_timer_")) || OB_FAIL(init_network_(addr, is_bootstrap))) { + SERVER_LOG(WARN, "init_network failed", K(ret), K(addr)); + } else if (FALSE_IT(guard.click("init_network_")) || OB_FAIL(init_io_(cluster_name))) { + SERVER_LOG(WARN, "init_io failed", K(ret), K(addr)); + } else if (FALSE_IT(guard.click("init_io_")) || OB_FAIL(init_log_service_(cluster_name))) { + SERVER_LOG(WARN, "init_log_service failed", K(ret), K(addr)); + } else { + guard.click("init_log_service_"); + SERVER_LOG(INFO, "simple_log_server simple_init success", K(node_id), K(addr), K(guard)); + } + return ret; +} + +int ObSimpleLogServer::init_memory_dump_timer_() +{ + int ret = OB_SUCCESS; + common::ObFunction print_memory_info = [=](){ + ObMallocAllocator::get_instance()->print_tenant_memory_usage(node_id_); + ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(node_id_); + ObMallocAllocator::get_instance()->print_tenant_memory_usage(OB_SERVER_TENANT_ID); + ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(OB_SERVER_TENANT_ID); + return false;}; + if (OB_FAIL(timer_.init_and_start(1, 1_s, "test timer"))) { + SERVER_LOG(ERROR, "ObOccamTimer init failed", K(ret), K_(node_id)); + } else if (OB_FAIL(timer_.schedule_task_repeat(timer_handle_, 1_s, print_memory_info))) { + SERVER_LOG(ERROR, "schedule_task failed", K(ret), K_(node_id)); + } + return ret; +} + +int ObSimpleLogServer::init_network_(const common::ObAddr &addr, const bool is_bootstrap = false) +{ + int ret = OB_SUCCESS; + ObNetOptions opts; + opts.rpc_io_cnt_ = 10; + opts.tcp_user_timeout_ = 10 * 1000 * 1000; // 10s + addr_ = addr; + if (is_bootstrap && OB_FAIL(net_.init(opts))) { + SERVER_LOG(ERROR, "net init fail", K(ret)); + } else if (OB_FAIL(req_handler_.init(&log_service_))) { + SERVER_LOG(ERROR, "req_handler_ init failed", K(ret)); + } else if (OB_FAIL(deliver_.init())) { + SERVER_LOG(ERROR, "deliver_ init failed", K(ret)); + } else if (OB_FAIL(deliver_.set_tenant_base(tenant_base_, &log_service_))) { + SERVER_LOG(ERROR, "deliver_ init failed", K(ret)); + } else if (is_bootstrap && OB_FAIL(net_.add_rpc_listen(addr_.get_port(), handler_, transport_))) { + SERVER_LOG(ERROR, "net_ listen failed", K(ret)); + } + return ret; +} + +int ObSimpleLogServer::init_io_(const std::string &cluster_name) +{ + int ret = OB_SUCCESS; + ObTimeGuard guard("init_io_", 0); + const std::string logserver_dir = cluster_name + "/port_" + std::to_string(addr_.get_port()); + std::string data_dir = logserver_dir; + std::string clog_dir = logserver_dir + "/clog"; + std::string slog_dir = logserver_dir + "/slog"; + std::string sstable_dir = logserver_dir + "/sstable"; + std::vector dirs{data_dir, slog_dir, clog_dir, sstable_dir}; + + for (auto &dir : dirs) { + if (-1 == mkdir(dir.c_str(), 0777)) { + if (errno == EEXIST) { + ret = OB_SUCCESS; + SERVER_LOG(INFO, "for restart"); + } else { + SERVER_LOG(WARN, "mkdir failed", K(errno)); + ret = OB_IO_ERROR; + break; + } + } + } + guard.click("mkdir"); + if (OB_SUCC(ret)) { + io_device_ = OB_NEW(ObLocalDevice, "TestBase"); + + blocksstable::ObStorageEnv storage_env; + storage_env.data_dir_ = data_dir.c_str(); + storage_env.sstable_dir_ = sstable_dir.c_str(); + storage_env.default_block_size_ = OB_DEFAULT_MACRO_BLOCK_SIZE; + storage_env.data_disk_size_ = 1024 * 1024 * 1024; + storage_env.data_disk_percentage_ = 0; + std::size_t found = cluster_name.find("client"); + const int64_t disk_size = (found!=std::string::npos)? 10LL * 1024 * 1024 * 1024: 100LL * 1024 * 1024 * 1024; + storage_env.log_disk_size_ = disk_size; + storage_env.log_disk_percentage_ = 0; + + storage_env.log_spec_.log_dir_ = slog_dir.c_str(); + storage_env.log_spec_.max_log_file_size_ = ObLogConstants::MAX_LOG_FILE_SIZE; + storage_env.clog_dir_ = clog_dir.c_str(); + iod_opts_.opts_ = iod_opt_array_; + iod_opt_array_[0].set("data_dir", storage_env.data_dir_); + iod_opt_array_[1].set("sstable_dir", storage_env.sstable_dir_); + iod_opt_array_[2].set("block_size", reinterpret_cast(&storage_env.default_block_size_)); + iod_opt_array_[3].set("datafile_disk_percentage", storage_env.data_disk_percentage_); + iod_opt_array_[4].set("datafile_size", storage_env.data_disk_size_); + iod_opts_.opt_cnt_ = MAX_IOD_OPT_CNT; + const int64_t DEFATULT_RESERVED_SIZE = 100 * 1024 * 1024 * 1024ul; + if (OB_FAIL(io_device_->init(iod_opts_))) { + SERVER_LOG(ERROR, "init io device fail", K(ret)); + } else if (OB_FAIL(log_block_pool_.init(storage_env.clog_dir_))) { + SERVER_LOG(ERROR, "init log pool fail", K(ret)); + } else { + guard.click("init io_device"); + clog_dir_ = clog_dir; + SERVER_LOG(INFO, "init_io_ successs", K(ret), K(guard)); + } + if (OB_SUCC(ret)) { + log_block_pool_.get_tenants_log_disk_size_func_ = [this, &storage_env](int64_t &log_disk_size) -> int + { + log_disk_size = log_block_pool_.lower_align_(storage_env.log_disk_size_); + return OB_SUCCESS; + }; + if (OB_FAIL(log_block_pool_.start(storage_env.log_disk_size_))) { + LOG_ERROR("log pool start failed", KR(ret)); + } + } + } + return ret; +} + +int ObSimpleLogServer::init_log_service_(const std::string &cluster_name) +{ + int ret = OB_SUCCESS; + // init deps of log_service + palf::PalfOptions opts; + std::size_t found = cluster_name.find("client"); + const int64_t disk_size = (found!=std::string::npos)? 10 * 1024 * 1024 * 1024ul: 100 * 1024 * 1024 * 1024ul; + opts.disk_options_.log_disk_usage_limit_size_ = disk_size; + opts.disk_options_.log_disk_usage_limit_size_ = disk_size; + opts.disk_options_.log_disk_utilization_threshold_ = 80; + opts.disk_options_.log_disk_utilization_limit_threshold_ = 95; + opts.disk_options_.log_disk_throttling_percentage_ = 100; + opts.disk_options_.log_disk_throttling_maximum_duration_ = 2 * 3600 * 1000 * 1000L; + opts.disk_options_.log_writer_parallelism_ = 2; + + std::string clog_dir = clog_dir_ + "/tenant_1"; + allocator_ = OB_NEW(ObTenantMutilAllocator, "TestBase", node_id_); + ObMemAttr attr(1, "SimpleLog"); + malloc_mgr_.set_attr(attr); + + if (OB_FAIL(log_service_.init(opts, clog_dir.c_str(), addr_, + allocator_, transport_, &log_block_pool_))) { + SERVER_LOG(ERROR, "init_log_service_ fail", K(ret)); + } else { + palf_env_ = log_service_.get_palf_env(); + SERVER_LOG(INFO, "init_log_service_ success", K(ret)); + } + return ret; +} + +int ObSimpleLogServer::simple_start(const bool is_bootstrap = false) +{ + int ret = OB_SUCCESS; + if (is_bootstrap && OB_FAIL(net_.start())) { + SERVER_LOG(ERROR, "net start fail", K(ret)); + } else if (OB_FAIL(deliver_.start())) { + SERVER_LOG(ERROR, "deliver_ start failed", K(ret)); + } else if (OB_FAIL(log_service_.start())) { + SERVER_LOG(ERROR, "arb_service start failed", K(ret)); + } else { + const double tenant_unit_cpu = 10; + tenant_base_->update_thread_cnt(tenant_unit_cpu); + } + return ret; +} + +int ObSimpleLogServer::simple_close(const bool is_shutdown = false) +{ + int ret = OB_SUCCESS; + ObTimeGuard guard("simple_close", 0); + deliver_.destroy(); + guard.click("destroy"); + log_service_.destroy(); + + log_block_pool_.destroy(); + guard.click("destroy_palf_env"); + + if (is_shutdown) { + net_.rpc_shutdown(); + net_.stop(); + net_.wait(); + net_.destroy(); + + timer_handle_.stop_and_wait(); + timer_.stop(); + timer_.wait(); + } + SERVER_LOG(INFO, "stop LogService success", K(ret), K(is_shutdown), K(guard)); + return ret; +} + +int ObSimpleLogServer::simple_restart(const std::string &cluster_name, const int64_t node_idx) +{ + int ret = OB_SUCCESS; + ObTimeGuard guard("simple_restart", 0); + if (OB_FAIL(simple_close())) { + SERVER_LOG(ERROR, "simple_close failed", K(ret)); + } else if (FALSE_IT(guard.click("simple_close")) || OB_FAIL(simple_init(cluster_name, addr_, node_idx))) { + SERVER_LOG(ERROR, "simple_init failed", K(ret)); + } else if (FALSE_IT(guard.click("simple_init")) || OB_FAIL(simple_start())) { + SERVER_LOG(ERROR, "simple_start failed", K(ret)); + } else { + guard.click("simple_start"); + SERVER_LOG(INFO, "simple_restart success", K(ret), K(guard)); + } + return ret; +} + +void ObLogDeliver::block_net(const ObAddr &src) +{ + blacklist_.set_refactored(src); + SERVER_LOG(INFO, "block_net", K(src), K(blacklist_)); +} + +void ObLogDeliver::unblock_net(const ObAddr &src) +{ + blacklist_.erase_refactored(src); + SERVER_LOG(INFO, "unblock_net", K(src), K(blacklist_)); +} + +void ObLogDeliver::set_rpc_loss(const ObAddr &src, const int loss_rate) +{ + // not thread safe + int old_loss_rate = -1; + bool is_exist = false; + for (int64_t i = 0; i < rpc_loss_config_.count(); ++i) { + LossConfig &tmp_config = rpc_loss_config_[i]; + if (tmp_config.src_ == src) { + is_exist = true; + // src exists, update its loss_rate + old_loss_rate = tmp_config.loss_rate_; + tmp_config.loss_rate_ = loss_rate; + } + } + if (!is_exist) { + LossConfig loss_config(src, loss_rate); + rpc_loss_config_.push_back(loss_config); + } + SERVER_LOG(INFO, "set_rpc_loss", K(src), K(loss_rate), K(old_loss_rate)); +} + +void ObLogDeliver::reset_rpc_loss(const ObAddr &src) +{ + // not thread safe + int64_t idx = -1; + for (int64_t i = 0; i < rpc_loss_config_.count(); ++i) { + LossConfig tmp_config; + int ret = rpc_loss_config_.at(i, tmp_config); + if (OB_SUCCESS != ret) { + SERVER_LOG(WARN, "at failed", K(ret), K(i), "count", rpc_loss_config_.count()); + } else if (tmp_config.src_ == src) { + // src exists, update its loss_rate + idx = i; + } + } + if (idx >= 0) { + rpc_loss_config_.remove(idx); + } + SERVER_LOG(INFO, "reset_rpc_loss", K(src), K(idx)); +} + +void ObLogDeliver::get_loss_config(const ObAddr &src, bool &exist, LossConfig &loss_config) +{ + // not thread safe + exist = false; + for (int64_t i = 0; i < rpc_loss_config_.count(); ++i) { + LossConfig tmp_config; + int ret = rpc_loss_config_.at(i, tmp_config); + if (OB_SUCCESS != ret) { + SERVER_LOG(WARN, "at failed", K(ret)); + } else if (tmp_config.src_ == src) { + exist = true; + loss_config = tmp_config; + } + } + SERVER_LOG(INFO, "get_loss_config", K(src), K(loss_config)); +} + +int ObLogDeliver::create_queue_thread(int tg_id, const char *thread_name, observer::QueueThread *&qthread) +{ + int ret = OB_SUCCESS; + qthread = OB_NEW(observer::QueueThread, ObModIds::OB_RPC, thread_name); + if (OB_ISNULL(qthread)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else { + qthread->queue_.set_qhandler(&qhandler_); + } + if (OB_SUCC(ret) && OB_NOT_NULL(qthread)) { + ret = TG_SET_RUNNABLE_AND_START(tg_id, qthread->thread_); + } + return ret; +} + +int ObLogDeliver::init() +{ + int ret = OB_SUCCESS; + // init_all_propocessor_(); + if (OB_FAIL(blacklist_.create(1024))) { + SERVER_LOG(WARN, "create blacklist_ failed", K(ret)); + } else if (false == blacklist_.created()) { + SERVER_LOG(WARN, "blacklist_ created failed"); + } else if (OB_FAIL(create_queue_thread(lib::TGDefIDs::LogServerTest, "LogServerQue", req_queue_))) { + } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TEST7, tg_id_))) { + SERVER_LOG(WARN, "ObSimpleThreadPool::init failed", K(ret)); + } else { + is_inited_ = true; + SERVER_LOG(INFO, "ObLogDeliver init success", KP(&blacklist_)); + } + return ret; +} + +void ObLogDeliver::destroy() +{ + if (IS_NOT_INIT) { + SERVER_LOG(INFO, "ObLogDeliver not init"); + } else { + RWLock::WLockGuard guard(lock_); + stop(); + wait(); + is_inited_ = false; + TG_DESTROY(tg_id_); + blacklist_.destroy(); + tg_id_ = 0; + SERVER_LOG(INFO, "destroy ObLogDeliver"); + } +} + +int ObLogDeliver::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + SERVER_LOG(ERROR, "ObLogDeliver not init"); + } else if (OB_FAIL(TG_SET_HANDLER_AND_START(tg_id_, *this))) { + SERVER_LOG(ERROR, "start ObLogDeliver failed"); + } else { + is_stopped_ = false; + SERVER_LOG(INFO, "start ObLogDeliver success"); + } + return ret; +} + +void ObLogDeliver::stop() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + SERVER_LOG(WARN, "ObLogDeliver stop failed"); + } else { + is_stopped_ = true; + TG_STOP(tg_id_); + if (NULL != req_queue_) { + TG_STOP(lib::TGDefIDs::LogServerTest); + TG_WAIT(lib::TGDefIDs::LogServerTest); + } + SERVER_LOG(INFO, "stop ObLogDeliver success"); + } +} + +int ObLogDeliver::wait() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + SERVER_LOG(ERROR, "ObLogDeliver not inited!!!", K(ret)); + } else { + TG_WAIT(tg_id_); + SERVER_LOG(INFO, "wait ObLogDeliver success", K(tg_id_)); + } + return ret; +} + +bool ObLogDeliver::need_filter_packet_by_blacklist(const ObAddr &addr) +{ + return OB_HASH_EXIST == blacklist_.exist_refactored(addr); +} + +bool ObLogDeliver::need_drop_by_loss_config(const ObAddr &addr) +{ + bool bool_ret = false; + bool exist = false; + LossConfig loss_config; + get_loss_config(addr, exist, loss_config); + if (exist && loss_config.loss_rate_ > 0) { + const int64_t random_num = ObRandom::rand(1, 100); + bool_ret = random_num <= loss_config.loss_rate_ ? true : false; + if (bool_ret) { + SERVER_LOG(INFO, "need drop req by loss config", K(addr)); + } + } + return bool_ret; +} + +// int ObLogDeliver::deliver(rpc::ObRequest &req) +// { +// int ret = OB_SUCCESS; +// const ObRpcPacket &pkt = reinterpret_cast(req.get_packet()); +// SERVER_LOG(INFO, "deliver request", K(pkt), K(req.ez_req_), KPC(palf_env_impl_)); +// if (IS_NOT_INIT) { +// ret = OB_NOT_INIT; +// } else { +// do { +// RWLock::RLockGuard guard(lock_); +// if (false == is_stopped_) { +// ret = TG_PUSH_TASK(tg_id_, &req); +// if (OB_SUCCESS != ret) { +// SERVER_LOG(ERROR, "deliver request failed", K(ret), K(pkt), K(req.ez_req_), KPC(palf_env_impl_)); +// } +// } else { +// break; +// } +// } while (OB_EAGAIN == ret); +// } +// return ret; +// } + +int ObLogDeliver::deliver(rpc::ObRequest &req) +{ + int ret = OB_SUCCESS; + SERVER_LOG(INFO, "deliver request", K(req.ez_req_), KPC(palf_env_impl_)); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else { + ObReqQueue *queue = NULL; + queue = &req_queue_->queue_; + if (NULL != queue) { + if (!queue->push(&req, 1 << 18)) { + ret = OB_QUEUE_OVERFLOW; + } + } + } + return ret; +} + +void ObLogDeliver::handle(void *task) +{ + int ret = OB_SUCCESS; + // if (IS_NOT_INIT) { + // ret = OB_NOT_INIT; + // SERVER_LOG(WARN, "ObLogDeliver not init", K(ret)); + // } else if (OB_ISNULL(task)) { + // SERVER_LOG(WARN, "invalid argument", KP(task)); + // } else { + // rpc::ObRequest *req = static_cast(task); + // SERVER_LOG(INFO, "handle request", KP(req->ez_req_)); + // (void) handle_req_(*req); + // } +} + +// int ObLogDeliver::handle_req_(rpc::ObRequest &req) +// { +// int ret = OB_SUCCESS; +// // tenant_id of log servers are different for clear log record, +// // so set tenant_id of RpcPacket to MTL_ID +// ObTenantEnv::set_tenant(tenant_base_); +// const ObRpcPacket &pkt = dynamic_cast(req.get_packet()); +// // const ObRpcPacketCode pcode = pkt.get_pcode(); +// // ObFunction filter = [&](const ObAddr &src) -> bool { +// // if (this->need_filter_packet_by_blacklist(src)) { +// // SERVER_LOG(WARN, "need_filter_packet_by_blacklist", K(ret), K(pcode), K(src), K(blacklist_), KPC(palf_env_impl_)); +// // return true; +// // } else if (this->need_drop_by_loss_config(src)) { +// // SERVER_LOG(WARN, "need_drop_by_loss_config", K(ret), K(pcode), K(src), K(rpc_loss_config_), KPC(palf_env_impl_)); +// // return true; +// // } +// // return false; +// // }; +// switch (pkt.get_pcode()) { +// #define PROCESS(processer) \ +// processer p;\ +// p.init();\ +// p.set_ob_request(req);\ +// p.set_log_service(log_service_);\ +// p.run(); \ +// break; +// case obrpc::OB_LOG_PUSH_REQ: { +// PROCESS(LogPushReqP); +// } +// case obrpc::OB_LOG_PUSH_RESP: { +// PROCESS(LogPushRespP); +// } +// case obrpc::OB_LOG_FETCH_REQ: { +// PROCESS(LogFetchReqP); +// } +// case obrpc::OB_LOG_PREPARE_REQ: { +// PROCESS(LogPrepareReqP); +// } +// case obrpc::OB_LOG_PREPARE_RESP: { +// PROCESS(LogPrepareRespP); +// } +// case obrpc::OB_LOG_CHANGE_CONFIG_META_REQ: { +// PROCESS(LogChangeConfigMetaReqP); +// } +// case obrpc::OB_LOG_CHANGE_CONFIG_META_RESP: { +// PROCESS(LogChangeConfigMetaRespP); +// } +// case obrpc::OB_LOG_CHANGE_MODE_META_REQ: { +// PROCESS(LogChangeModeMetaReqP); +// } +// case obrpc::OB_LOG_CHANGE_MODE_META_RESP: { +// PROCESS(LogChangeModeMetaRespP); +// } +// case obrpc::OB_LOG_NOTIFY_REBUILD_REQ: { +// PROCESS(LogNotifyRebuildReqP) +// } +// case obrpc::OB_LOG_COMMITTED_INFO: { +// PROCESS(CommittedInfoP) +// } +// case obrpc::OB_LOG_LEARNER_REQ: { +// PROCESS(LogLearnerReqP) +// } +// case obrpc::OB_LOG_REGISTER_PARENT_REQ: { +// PROCESS(LogRegisterParentReqP) +// } +// case obrpc::OB_LOG_REGISTER_PARENT_RESP: { +// PROCESS(LogRegisterParentRespP) +// } +// case obrpc::OB_LOG_ELECTION_PREPARE_REQUEST : { +// PROCESS(ElectionPrepareRequestMsgP) +// } +// case obrpc::OB_LOG_ELECTION_PREPARE_RESPONSE : { +// PROCESS(ElectionPrepareResponseMsgP) +// } +// case obrpc::OB_LOG_ELECTION_ACCEPT_REQUEST : { +// PROCESS(ElectionAcceptRequestMsgP) +// } +// case obrpc::OB_LOG_ELECTION_ACCEPT_RESPONSE : { +// PROCESS(ElectionAcceptResponseMsgP) +// } +// case obrpc::OB_LOG_ELECTION_CHANGE_LEADER_REQUEST : { +// PROCESS(ElectionChangeLeaderMsgP) +// } +// case obrpc::OB_LOG_GET_MC_ST: { +// PROCESS(LogGetMCStP) +// } +// case obrpc::OB_LOG_ARB_PROBE_MSG: { +// PROCESS(logservice::LogServerProbeP) +// } +// case obrpc::OB_LOG_CONFIG_CHANGE_CMD: { +// PROCESS(LogMembershipChangeP) +// } +// case obrpc::OB_LOG_GET_PALF_STAT: { +// PROCESS(LogGetPalfStatReqP) +// } +// case obrpc::OB_LOG_CHANGE_ACCESS_MODE_CMD: { +// PROCESS(LogChangeAccessModeP) +// } +// case obrpc::OB_LOG_FLASHBACK_CMD: { +// PROCESS(LogFlashbackMsgP) +// } +// case obrpc::OB_LOG_CREATE_REPLICA_CMD: { +// PROCESS(LogCreateReplicaCmdP) +// } +// case obrpc::OB_LOG_SUBMIT_LOG_CMD: { +// PROCESS(LogSubmitLogP) +// } +// case obrpc::OB_LOG_SUBMIT_LOG_CMD_RESP: { +// PROCESS(LogSubmitLogRespP) +// } +// default: +// SERVER_LOG(ERROR, "invalid req type", K(pkt.get_pcode())); +// break; +// } +// return ret; +// } + +ObLogServerReqQueueHandler::~ObLogServerReqQueueHandler() +{ +} + +int ObLogServerReqQueueHandler::init(palfcluster::LogService *log_service) +{ + int ret = OB_SUCCESS; + if (NULL == log_service) { + ret = OB_INVALID_ARGUMENT; + } else { + log_service_ = log_service; + } + return ret; +} + +int ObLogServerReqQueueHandler::onThreadCreated(obsys::CThread *th) +{ + UNUSED(th); + return OB_SUCCESS; +} + +int ObLogServerReqQueueHandler::onThreadDestroy(obsys::CThread *th) +{ + UNUSED(th); + return OB_SUCCESS; +} + +bool ObLogServerReqQueueHandler::handlePacketQueue(ObRequest *req, void */* arg */) +{ + // ObTenantEnv::set_tenant(tenant_base_); + const ObRpcPacket &pkt = dynamic_cast(req->get_packet()); + if (NULL != log_service_) { + switch (pkt.get_pcode()) { + #define PALF_PROCESS(processer) \ + processer p;\ + p.init();\ + p.set_palf_env_impl(log_service_->get_palf_env()->get_palf_env_impl()); \ + p.set_ob_request(*req);\ + p.run(); \ + break; + #define LOG_SERVICE_PROCESS(processer) \ + processer p;\ + p.init();\ + p.set_ob_request(*req);\ + p.set_log_service(log_service_);\ + p.run(); \ + break; + case obrpc::OB_LOG_PUSH_REQ: { + PALF_PROCESS(LogPushReqP); + } + case obrpc::OB_LOG_PUSH_RESP: { + PALF_PROCESS(LogPushRespP); + } + case obrpc::OB_LOG_FETCH_REQ: { + PALF_PROCESS(LogFetchReqP); + } + case obrpc::OB_LOG_PREPARE_REQ: { + PALF_PROCESS(LogPrepareReqP); + } + case obrpc::OB_LOG_PREPARE_RESP: { + PALF_PROCESS(LogPrepareRespP); + } + case obrpc::OB_LOG_CHANGE_CONFIG_META_REQ: { + PALF_PROCESS(LogChangeConfigMetaReqP); + } + case obrpc::OB_LOG_CHANGE_CONFIG_META_RESP: { + PALF_PROCESS(LogChangeConfigMetaRespP); + } + case obrpc::OB_LOG_CHANGE_MODE_META_REQ: { + PALF_PROCESS(LogChangeModeMetaReqP); + } + case obrpc::OB_LOG_CHANGE_MODE_META_RESP: { + PALF_PROCESS(LogChangeModeMetaRespP); + } + case obrpc::OB_LOG_NOTIFY_REBUILD_REQ: { + PALF_PROCESS(LogNotifyRebuildReqP) + } + case obrpc::OB_LOG_COMMITTED_INFO: { + PALF_PROCESS(CommittedInfoP) + } + case obrpc::OB_LOG_LEARNER_REQ: { + PALF_PROCESS(LogLearnerReqP) + } + case obrpc::OB_LOG_REGISTER_PARENT_REQ: { + PALF_PROCESS(LogRegisterParentReqP) + } + case obrpc::OB_LOG_REGISTER_PARENT_RESP: { + PALF_PROCESS(LogRegisterParentRespP) + } + case obrpc::OB_LOG_ELECTION_PREPARE_REQUEST : { + PALF_PROCESS(ElectionPrepareRequestMsgP) + } + case obrpc::OB_LOG_ELECTION_PREPARE_RESPONSE : { + PALF_PROCESS(ElectionPrepareResponseMsgP) + } + case obrpc::OB_LOG_ELECTION_ACCEPT_REQUEST : { + PALF_PROCESS(ElectionAcceptRequestMsgP) + } + case obrpc::OB_LOG_ELECTION_ACCEPT_RESPONSE : { + PALF_PROCESS(ElectionAcceptResponseMsgP) + } + case obrpc::OB_LOG_ELECTION_CHANGE_LEADER_REQUEST : { + PALF_PROCESS(ElectionChangeLeaderMsgP) + } + case obrpc::OB_LOG_GET_MC_ST: { + PALF_PROCESS(LogGetMCStP) + } + case obrpc::OB_LOG_BATCH_FETCH_RESP: { + PALF_PROCESS(LogBatchFetchRespP) + } + case obrpc::OB_LOG_CREATE_REPLICA_CMD: { + LOG_SERVICE_PROCESS(palfcluster::LogCreateReplicaCmdP) + } + case obrpc::OB_LOG_SUBMIT_LOG_CMD: { + LOG_SERVICE_PROCESS(palfcluster::LogSubmitLogP) + } + case obrpc::OB_LOG_SUBMIT_LOG_CMD_RESP: { + LOG_SERVICE_PROCESS(palfcluster::LogSubmitLogRespP) + } + default: + SERVER_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "invalid req type", K(pkt.get_pcode())); + break; + } + } else { + SERVER_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "log_service_ is NULL", KP(log_service_)); + } + return true; +} + +} // unittest +} // oceanbase \ No newline at end of file diff --git a/mittest/palf_cluster/env/ob_simple_log_server.h b/mittest/palf_cluster/env/ob_simple_log_server.h new file mode 100644 index 0000000000..fa93c9d5c5 --- /dev/null +++ b/mittest/palf_cluster/env/ob_simple_log_server.h @@ -0,0 +1,242 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#pragma once +#include "lib/hash/ob_hashset.h" +#include "lib/ob_errno.h" +#include "lib/thread/ob_simple_thread_pool.h" +#include "lib/thread/thread_mgr_interface.h" +#include "lib/signal/ob_signal_worker.h" +#include "lib/signal/ob_signal_struct.h" +#include "lib/function/ob_function.h" // ObFunction +#include "logservice/palf/log_block_pool_interface.h" +#include "observer/ob_signal_handle.h" +#include "observer/ob_srv_deliver.h" +#include "lib/utility/ob_defer.h" +#include "rpc/frame/ob_req_deliver.h" +#include "rpc/ob_request.h" +#include "rpc/frame/ob_net_easy.h" +#include "rpc/obrpc/ob_rpc_handler.h" +#include "rpc/frame/ob_req_transport.h" +#include "logservice/palf/log_rpc_macros.h" +#include "logservice/palf/log_rpc_processor.h" +#include "logservice/palf/palf_env_impl.h" +#include "logservice/palf/palf_env.h" +#include "logservice/palf/palf_handle_impl.h" +#include "logservice/ob_arbitration_service.h" +#include "lib/net/ob_addr.h" +#include "share/ob_rpc_struct.h" +#include "storage/blocksstable/ob_block_sstable_struct.h" +#include "share/allocator/ob_tenant_mutil_allocator.h" +#include "storage/tx_storage/ob_ls_map.h" +#include "storage/tx_storage/ob_ls_service.h" +#include "logservice/palf/palf_handle_impl.h" +#include "logservice/ob_log_service.h" +#include "logservice/ob_server_log_block_mgr.h" +#include "share/ob_local_device.h" +#include "share/ob_occam_timer.h" +#include "share/resource_manager/ob_cgroup_ctrl.h" +#include "mittest/palf_cluster/rpc/palf_cluster_rpc_processor.h" +#include "mittest/palf_cluster/logservice/log_service.h" +#include + +namespace oceanbase +{ +namespace unittest +{ +using namespace oceanbase; +using namespace oceanbase::rpc; +using namespace oceanbase::rpc::frame; +using namespace oceanbase::obrpc; +using namespace oceanbase::common; +using namespace oceanbase::palf; +using namespace oceanbase::share; + +uint32_t get_local_addr(const char *dev_name); +std::string get_local_ip(); + +struct LossConfig +{ + ObAddr src_; + int loss_rate_; + LossConfig() + : src_(), loss_rate_(0) + {} + LossConfig(const ObAddr &src, const int loss_rate) + : src_(src), loss_rate_(loss_rate) + {} + TO_STRING_KV(K_(src), K_(loss_rate)); +}; + +class ObLogServerReqQueueHandler : public rpc::frame::ObiReqQHandler +{ +public: + ObLogServerReqQueueHandler() + : log_service_(NULL) { } + virtual ~ObLogServerReqQueueHandler(); + int init(palfcluster::LogService *log_service); + + int onThreadCreated(obsys::CThread *); + int onThreadDestroy(obsys::CThread *); + + bool handlePacketQueue(ObRequest *req, void *args); +private: + palfcluster::LogService *log_service_; +}; + +class ObLogDeliver : public rpc::frame::ObReqQDeliver, public lib::TGTaskHandler +{ +public: + ObLogDeliver(rpc::frame::ObiReqQHandler &qhandler) + : rpc::frame::ObReqQDeliver(qhandler), + palf_env_impl_(NULL), + tg_id_(0), + blacklist_(), + rpc_loss_config_(), + is_stopped_(true), + tenant_base_(NULL), + req_queue_(NULL) {} + ~ObLogDeliver() { destroy(); } + int init(); + int set_tenant_base(ObTenantBase *base, palfcluster::LogService *log_service) + { + tenant_base_ = base; + log_service_ = log_service; + return OB_SUCCESS; + } + void destroy(); + int deliver(rpc::ObRequest &req); + int start(); + void stop(); + int wait(); + void handle(void *task); + void set_need_drop_packet(const bool need_drop_packet) { need_drop_packet_ = need_drop_packet; } + void block_net(const ObAddr &src); + void unblock_net(const ObAddr &src); + void set_rpc_loss(const ObAddr &src, const int loss_rate); + void reset_rpc_loss(const ObAddr &src); + bool need_filter_packet_by_blacklist(const ObAddr &address); + bool need_drop_by_loss_config(const ObAddr &addr); + void get_loss_config(const ObAddr &src, bool &exist, LossConfig &loss_config); + +private: + void init_all_propocessor_(); + typedef std::function Func; + + Func funcs_[MAX_PCODE]; + template + void register_rpc_propocessor_(int pcode) + { + auto func = [](ObReqProcessor *&ptr) -> int { + int ret = OB_SUCCESS; + if (NULL == (ptr = OB_NEW(PROCESSOR, "SimpleLogSvr"))) { + SERVER_LOG(WARN, "allocate memory failed"); + } else if (OB_FAIL(ptr->init())) { + } else { + } + return ret; + }; + funcs_[pcode] = func; + SERVER_LOG(INFO, "register_rpc_propocessor_ success", K(pcode)); + } + + int create_queue_thread(int tg_id, const char *thread_name, observer::QueueThread *&qthread); + int handle_req_(rpc::ObRequest &req); +private: + mutable common::RWLock lock_; + bool is_inited_; + PalfEnvImpl *palf_env_impl_; + palfcluster::LogService *log_service_; + int tg_id_; + bool need_drop_packet_; + hash::ObHashSet blacklist_; + common::ObSEArray rpc_loss_config_; + bool is_stopped_; + ObTenantBase *tenant_base_; + observer::QueueThread *req_queue_; +}; + +class ObSimpleLogServer +{ +public: + ObSimpleLogServer() + : req_handler_(), + deliver_(req_handler_), + handler_(deliver_), + transport_(NULL) + { + } + ~ObSimpleLogServer() + { + if (OB_NOT_NULL(allocator_)) { + ob_delete(allocator_); + } + if (OB_NOT_NULL(io_device_)) { + ob_delete(io_device_); + } + } + bool is_valid() {return NULL != palf_env_;} + int simple_init(const std::string &cluster_name, + const common::ObAddr &addr, + const int64_t node_id, + const bool is_bootstrap); + int simple_start(const bool is_bootstrap); + int simple_close(const bool is_shutdown); + int simple_restart(const std::string &cluster_name, const int64_t node_idx); + PalfEnv *get_palf_env() { return palf_env_; } + const std::string& get_clog_dir() const { return clog_dir_; } + common::ObAddr get_addr() const { return addr_; } + ObTenantBase *get_tenant_base() const { return tenant_base_; } + palfcluster::LogService *get_log_service() { return &log_service_; } + logservice::ObServerLogBlockMgr *get_log_block_pool() { return &log_block_pool_; } + // Nowdat, not support drop packet from specificed address + void set_need_drop_packet(const bool need_drop_packet) { deliver_.set_need_drop_packet(need_drop_packet);} + void block_net(const ObAddr &src) { deliver_.block_net(src); } + void unblock_net(const ObAddr &src) { deliver_.unblock_net(src); } + void set_rpc_loss(const ObAddr &src, const int loss_rate) { deliver_.set_rpc_loss(src, loss_rate); } + void reset_rpc_loss(const ObAddr &src) { deliver_.reset_rpc_loss(src); } + TO_STRING_KV(K_(node_id), K_(addr), KP(palf_env_)); +protected: + int init_io_(const std::string &cluster_name); + int init_network_(const common::ObAddr &addr, const bool is_bootstrap); + int init_log_service_(const std::string &cluster_name); + int init_memory_dump_timer_(); + +private: + int64_t node_id_; + common::ObAddr addr_; + rpc::frame::ObNetEasy net_; + ObLogServerReqQueueHandler req_handler_; + ObLogDeliver deliver_; + obrpc::ObRpcHandler handler_; + ObRandom rand_; + PalfEnv *palf_env_; + ObTenantBase *tenant_base_; + ObMalloc malloc_mgr_; + ObLocalDevice *io_device_; + static const int64_t MAX_IOD_OPT_CNT = 5; + ObIODOpt iod_opt_array_[MAX_IOD_OPT_CNT]; + ObIODOpts iod_opts_; + std::string clog_dir_; + ObOccamTimer timer_; + ObOccamTimerTaskRAIIHandle timer_handle_; + palfcluster::LogService log_service_; + ObTenantMutilAllocator *allocator_; + rpc::frame::ObReqTransport *transport_; + ObLSService ls_service_; + ObLocationService location_service_; + logservice::ObServerLogBlockMgr log_block_pool_; + common::ObMySQLProxy sql_proxy_; +}; + +} // end unittest +} // oceanbase \ No newline at end of file diff --git a/mittest/palf_cluster/logservice/log_service.cpp b/mittest/palf_cluster/logservice/log_service.cpp new file mode 100644 index 0000000000..6dde8fb8e7 --- /dev/null +++ b/mittest/palf_cluster/logservice/log_service.cpp @@ -0,0 +1,621 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX CLOG +#include "log_service.h" +#include "lib/file/file_directory_utils.h" +#include "lib/ob_errno.h" +#include "logservice/ob_server_log_block_mgr.h" +#include "logservice/palf/log_block_pool_interface.h" +#include "rpc/frame/ob_req_transport.h" +#include "share/ob_ls_id.h" +#include "share/allocator/ob_tenant_mutil_allocator.h" +#include "share/allocator/ob_tenant_mutil_allocator_mgr.h" +#include "share/ob_tenant_info_proxy.h" +#include "share/ob_unit_getter.h" +#include "share/rc/ob_tenant_base.h" +#include "share/rc/ob_tenant_module_init_ctx.h" +#include "storage/tx_storage/ob_ls_map.h" +#include "storage/tx_storage/ob_ls_service.h" +#include "observer/ob_srv_network_frame.h" +#include "logservice/palf_handle_guard.h" +#include "storage/ob_file_system_router.h" +#include "logservice/palf/palf_env.h" +#include "logservice/palf/palf_callback.h" +#include "logservice/palf/palf_options.h" + +namespace oceanbase +{ +using namespace share; +using namespace common; +using namespace palf; + +namespace palfcluster +{ +using namespace oceanbase::share; +using namespace oceanbase::common; + +LogService::LogService() : + is_inited_(false), + is_running_(false), + self_(), + palf_env_(NULL), + apply_service_(), + // replay_service_(), + role_change_service_(), + monitor_(), + rpc_proxy_(), + log_client_map_(), + ls_adapter_() + {} + +LogService::~LogService() +{ + destroy(); +} + +int LogService::mtl_init(LogService* &logservice) +{ + int ret = OB_SUCCESS; + const ObAddr &self = GCTX.self_addr(); + const int64_t tenant_id = MTL_ID(); + observer::ObSrvNetworkFrame *net_frame = GCTX.net_frame_; + //log_disk_usage_limit_size无法主动从配置项获取, 需要在mtl初始化时作为入参传入 + const palf::PalfOptions &palf_options = MTL_INIT_CTX()->palf_options_; + const char *tenant_clog_dir = MTL_INIT_CTX()->tenant_clog_dir_; + const char *clog_dir = OB_FILE_SYSTEM_ROUTER.get_clog_dir(); + logservice::ObServerLogBlockMgr *log_block_mgr = GCTX.log_block_mgr_; + common::ObILogAllocator *alloc_mgr = NULL; + if (OB_FAIL(TMA_MGR_INSTANCE.get_tenant_log_allocator(tenant_id, alloc_mgr))) { + CLOG_LOG(WARN, "get_tenant_log_allocator failed", K(ret)); + } else if (OB_FAIL(logservice->init(palf_options, + tenant_clog_dir, + self, + alloc_mgr, + net_frame->get_req_transport(), + log_block_mgr))) { + CLOG_LOG(ERROR, "init LogService failed", K(ret), K(tenant_clog_dir)); + } else if (OB_FAIL(FileDirectoryUtils::fsync_dir(clog_dir))) { + CLOG_LOG(ERROR, "fsync_dir failed", K(ret), K(clog_dir)); + } else { + CLOG_LOG(INFO, "LogService mtl_init success"); + } + return ret; +} + +void LogService::mtl_destroy(LogService* &logservice) +{ + common::ob_delete(logservice); + logservice = nullptr; + // Free tenant_log_allocator for this tenant after destroy logservice. + const int64_t tenant_id = MTL_ID(); + int ret = OB_SUCCESS; + if (OB_FAIL(TMA_MGR_INSTANCE.delete_tenant_log_allocator(tenant_id))) { + CLOG_LOG(WARN, "delete_tenant_log_allocator failed", K(ret)); + } +} + +int LogService::start() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(apply_service_.start())) { + CLOG_LOG(WARN, "failed to start apply_service_", K(ret)); + // } else if (OB_FAIL(replay_service_.start())) { + // CLOG_LOG(WARN, "failed to start replay_service_", K(ret)); + } else if (OB_FAIL(role_change_service_.start())) { + CLOG_LOG(WARN, "failed to start role_change_service_", K(ret)); + } else { + is_running_ = true; + FLOG_INFO("LogService is started"); + } + return ret; +} + +void LogService::stop() +{ + is_running_ = false; + CLOG_LOG(INFO, "begin to stop LogService"); + (void)apply_service_.stop(); + // (void)replay_service_.stop(); + (void)role_change_service_.stop(); + FLOG_INFO("LogService is stopped"); +} + +void LogService::wait() +{ + apply_service_.wait(); + // replay_service_.wait(); + role_change_service_.wait(); +} + +void LogService::destroy() +{ + is_inited_ = false; + self_.reset(); + apply_service_.destroy(); + // replay_service_.destroy(); + role_change_service_.destroy(); + ls_adapter_.destroy(); + rpc_proxy_.destroy(); + if (NULL != palf_env_) { + PalfEnv::destroy_palf_env(palf_env_); + palf_env_ = NULL; + } + FLOG_INFO("LogService is destroyed"); +} + +int check_and_prepare_dir(const char *dir) +{ + bool is_exist = false; + int ret = OB_SUCCESS; + if (OB_FAIL(common::FileDirectoryUtils::is_exists(dir, is_exist))) { + CLOG_LOG(WARN, "chcck dir exist failed", K(ret), K(dir)); + // means it's restart + } else if (is_exist == true) { + CLOG_LOG(INFO, "director exist", K(ret), K(dir)); + // means it's create tenant + } else if (OB_FAIL(common::FileDirectoryUtils::create_directory(dir))) { + CLOG_LOG(WARN, "create_directory failed", K(ret), K(dir)); + } else { + CLOG_LOG(INFO, "check_and_prepare_dir success", K(ret), K(dir)); + } + return ret; +} + +int LogService::init(const PalfOptions &options, + const char *base_dir, + const common::ObAddr &self, + common::ObILogAllocator *alloc_mgr, + rpc::frame::ObReqTransport *transport, + palf::ILogBlockPool *log_block_pool) +{ + int ret = OB_SUCCESS; + + const int64_t tenant_id = MTL_ID(); + if (OB_FAIL(check_and_prepare_dir(base_dir))) { + CLOG_LOG(WARN, "check_and_prepare_dir failed", K(ret), K(base_dir)); + } else if (is_inited_) { + ret = OB_INIT_TWICE; + CLOG_LOG(WARN, "LogService init twice", K(ret)); + } else if (false == options.is_valid() || OB_ISNULL(base_dir) || OB_UNLIKELY(!self.is_valid()) + || OB_ISNULL(alloc_mgr) || OB_ISNULL(transport) + || OB_ISNULL(log_block_pool)) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid arguments", K(ret), K(options), KP(base_dir), K(self), + KP(alloc_mgr), KP(transport), KP(log_block_pool)); + } else if (OB_FAIL(PalfEnv::create_palf_env(options, base_dir, self, transport, + alloc_mgr, log_block_pool, &monitor_, palf_env_))) { + CLOG_LOG(WARN, "failed to create_palf_env", K(base_dir), K(ret)); + } else if (OB_ISNULL(palf_env_)) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "palf_env_ is NULL", K(ret)); + } else if (OB_FAIL(ls_adapter_.init())) { + CLOG_LOG(ERROR, "failed to init ls_adapter", K(ret)); + } else if (OB_FAIL(apply_service_.init(palf_env_, &ls_adapter_))) { + CLOG_LOG(WARN, "failed to init apply_service", K(ret)); + // } else if (OB_FAIL(replay_service_.init(palf_env_, &ls_adapter_, alloc_mgr))) { + // CLOG_LOG(WARN, "failed to init replay_service", K(ret)); + } else if (OB_FAIL(role_change_service_.init(&log_client_map_, &apply_service_))) { + CLOG_LOG(WARN, "failed to init role_change_service_", K(ret)); + } else if (OB_FAIL(rpc_proxy_.init(transport))) { + CLOG_LOG(WARN, "LogServiceRpcProxy init failed", K(ret)); + } else if (OB_FAIL(log_client_map_.init("Client", MTL_ID()))) { + CLOG_LOG(WARN, "failed to init log_client_map", K(ret)); + } else { + self_ = self; + is_inited_ = true; + FLOG_INFO("LogService init success", K(ret), K(base_dir), K(self), KP(transport), K(tenant_id)); + } + + if (OB_FAIL(ret) && OB_INIT_TWICE != ret) { + destroy(); + } + return ret; +} + + +int LogService::create_palf_replica(const int64_t palf_id, + const common::ObMemberList &member_list, + const int64_t replica_num, + const int64_t leader_idx) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else { + share::ObLSID ls_id(palf_id); + ObLogClient *log_client = OB_NEW(ObLogClient, "client"); + if (OB_FAIL(log_client->init(self_, palf_id, &rpc_proxy_, this))) { + CLOG_LOG(ERROR, "init fail", K(ret), K(palf_id)); + } else if (OB_FAIL(log_client_map_.insert(ls_id, log_client))) { + CLOG_LOG(ERROR, "insert fail", K(ret), K(palf_id)); + } else if (OB_FAIL(log_client->create_palf_replica(member_list, replica_num, leader_idx))) { + CLOG_LOG(ERROR, "create_replica fail", K(ret), K(palf_id)); + log_client_map_.erase(ls_id); + } else { + CLOG_LOG(ERROR, "create_palf_replica successfully", K(ret), K(palf_id), K(member_list), K(replica_num), K(leader_idx)); + } + } + return ret; +} + +int LogService::create_log_clients(const int64_t thread_num, + const int64_t log_size, + const int64_t palf_group_num, + std::vector leader_list) +{ + int ret = OB_SUCCESS; + + for (int64_t i = 0; i < thread_num; i++) { + const int64_t palf_id = (i % palf_group_num) + 1; + if (OB_FAIL(clients_[i].init_and_create(i, log_size, &rpc_proxy_, self_, leader_list[palf_id-1], palf_id))) { + CLOG_LOG(ERROR, "init_and_create fail", K(palf_id)); + } + } + + for (int64_t i = 0; i < thread_num; i++) { + clients_[i].join(); + } + return ret; +} + +palf::AccessMode LogService::get_palf_access_mode(const share::ObTenantRole &tenant_role) +{ + palf::AccessMode mode = palf::AccessMode::INVALID_ACCESS_MODE; + switch (tenant_role.value()) { + case share::ObTenantRole::INVALID_TENANT: + mode = palf::AccessMode::INVALID_ACCESS_MODE; + break; + case share::ObTenantRole::PRIMARY_TENANT: + mode = palf::AccessMode::APPEND; + break; + case share::ObTenantRole::STANDBY_TENANT: + case share::ObTenantRole::RESTORE_TENANT: + mode = palf::AccessMode::RAW_WRITE; + break; + default: + mode = palf::AccessMode::INVALID_ACCESS_MODE; + break; + } + return mode; +} + +int LogService::create_ls(const share::ObLSID &id, + const common::ObReplicaType &replica_type, + const share::ObTenantRole &tenant_role, + const palf::PalfBaseInfo &palf_base_info, + const bool allow_log_sync, + logservice::ObLogHandler &log_handler) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id)); + } else if (!palf_base_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid arguments", K(ret), K(id), K(replica_type), + K(tenant_role), K(palf_base_info)); + } else if (OB_FAIL(create_ls_(id, replica_type, tenant_role, palf_base_info, allow_log_sync, + log_handler))) { + CLOG_LOG(WARN, "create ls failed", K(ret), K(id), K(replica_type), + K(tenant_role), K(palf_base_info)); + } else { + FLOG_INFO("LogService create_ls success", K(ret), K(id), K(replica_type), K(tenant_role), K(palf_base_info), + K(log_handler)); + } + return ret; +} + +int LogService::remove_ls(const ObLSID &id, logservice::ObLogHandler &log_handler) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id)); + } else if (OB_FAIL(apply_service_.remove_ls(id))) { + CLOG_LOG(WARN, "failed to remove from apply_service", K(ret), K(id)); + // } else if (OB_FAIL(replay_service_.remove_ls(id))) { + // CLOG_LOG(WARN, "failed to remove from replay_service", K(ret), K(id)); + // NB: remove palf_handle lastly. + } else { + // NB: can not execute destroy, otherwise, each interface in log_handler or restore_handler + // may return OB_NOT_INIT. + // TODO by runlin: create_ls don't init logservice::ObLogHandler and ObLogRestoreHandler. + // + // In normal case(for gc), stop has been executed, this stop has no effect. + // In abnormal case(create ls failed, need remove ls directlly), there is no possibility for dead lock. + log_handler.stop(); + if (OB_FAIL(palf_env_->remove(id.id()))) { + CLOG_LOG(WARN, "failed to remove from palf_env_", K(ret), K(id)); + } else { + FLOG_INFO("LogService remove_ls success", K(ret), K(id)); + } + } + + return ret; +} + +int LogService::check_palf_exist(const ObLSID &id, bool &exist) const +{ + int ret = OB_SUCCESS; + PalfHandle handle; + exist = true; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(WARN, "LogService is not inited", K(ret), K(id)); + } else if (OB_FAIL(palf_env_->open(id.id(), handle))) { + if (OB_ENTRY_NOT_EXIST == ret ) { + ret = OB_SUCCESS; + exist = false; + } else { + CLOG_LOG(WARN, "open palf failed", K(ret), K(id)); + } + } + + if (true == handle.is_valid()) { + palf_env_->close(handle); + } + return ret; +} + +int LogService::add_ls(const ObLSID &id, logservice::ObLogHandler &log_handler) +{ + int ret = OB_SUCCESS; + PalfHandle palf_handle; + PalfHandle &log_handler_palf_handle = log_handler.palf_handle_; + PalfRoleChangeCb *rc_cb = &role_change_service_; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id)); + } else if (OB_FAIL(palf_env_->open(id.id(), palf_handle))) { + CLOG_LOG(WARN, "failed to get palf_handle", K(ret), K(id)); + } else if (OB_FAIL(apply_service_.add_ls(id))) { + CLOG_LOG(WARN, "failed to add_ls for apply_service", K(ret), K(id)); + // } else if (OB_FAIL(replay_service_.add_ls(id))) { + // CLOG_LOG(WARN, "failed to add_ls for replay_service", K(ret), K(id)); + // } else if (OB_FAIL(log_handler.init(id.id(), self_, &apply_service_, &replay_service_, + // &role_change_service_, palf_handle, palf_env_, loc_cache_cb, &rpc_proxy_))) { + // CLOG_LOG(WARN, "logservice::ObLogHandler init failed", K(ret), K(id), KP(palf_env_), K(palf_handle)); + } else if (OB_FAIL(log_handler_palf_handle.register_role_change_cb(rc_cb))) { + CLOG_LOG(WARN, "register_role_change_cb failed", K(ret)); + } else { + FLOG_INFO("add_ls success", K(ret), K(id), KP(this)); + } + + if (OB_FAIL(ret)) { + if (true == palf_handle.is_valid() && false == log_handler.is_valid()) { + palf_env_->close(palf_handle); + } + } + + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_LS_NOT_EXIST; + } + + return ret; +} + +int LogService::open_palf(const share::ObLSID &id, + palf::PalfHandleGuard &palf_handle_guard) +{ + int ret = OB_SUCCESS; + palf::PalfHandle palf_handle; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id)); + } else if (OB_FAIL(palf_env_->open(id.id(), palf_handle))) { + CLOG_LOG(WARN, "failed to get palf_handle", K(ret), K(id)); + } else if (FALSE_IT(palf_handle_guard.set(palf_handle, palf_env_))) { + } else { + CLOG_LOG(TRACE, "LogService open_palf success", K(ret), K(id)); + } + + if (OB_FAIL(ret)) { + if (true == palf_handle.is_valid()) { + palf_env_->close(palf_handle); + } + } + + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_LS_NOT_EXIST; + } + return ret; +} + +int LogService::get_palf_role(const share::ObLSID &id, + common::ObRole &role, + int64_t &proposal_id) +{ + int ret = OB_SUCCESS; + palf::PalfHandleGuard palf_handle_guard; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(WARN, "log_service is not inited", K(ret), K(id)); + } else if (OB_FAIL(open_palf(id, palf_handle_guard))) { + CLOG_LOG(WARN, "failed to open palf", K(ret), K(id)); + } else if (OB_FAIL(palf_handle_guard.get_role(role, proposal_id))) { + CLOG_LOG(WARN, "failed to get role", K(ret), K(id)); + } + return ret; +} + +int LogService::get_palf_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else { + ret = palf_env_->get_disk_usage(used_size_byte, total_size_byte); + } + return ret; +} + +int LogService::get_palf_stable_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else { + ret = palf_env_->get_stable_disk_usage(used_size_byte, total_size_byte); + } + return ret; +} + +int LogService::update_palf_options_except_disk_usage_limit_size() +{ + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID())); + ObSpinLockGuard guard(update_palf_opts_lock_); + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (!tenant_config.is_valid()) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "tenant_config is not valid", K(ret), K(MTL_ID())); + } else { + PalfOptions palf_opts; + common::ObCompressorType compressor_type = LZ4_COMPRESSOR; + if (OB_FAIL(common::ObCompressorPool::get_instance().get_compressor_type( + tenant_config->log_transport_compress_func, compressor_type))) { + CLOG_LOG(ERROR, "log_transport_compress_func invalid.", K(ret)); + //需要获取log_disk_usage_limit_size + } else if (OB_FAIL(palf_env_->get_options(palf_opts))) { + CLOG_LOG(WARN, "palf get_options failed", K(ret)); + } else { + palf_opts.disk_options_.log_disk_utilization_threshold_ = tenant_config->log_disk_utilization_threshold; + palf_opts.disk_options_.log_disk_utilization_limit_threshold_ = tenant_config->log_disk_utilization_limit_threshold; + palf_opts.disk_options_.log_disk_throttling_percentage_ = tenant_config->log_disk_throttling_percentage; + palf_opts.disk_options_.log_disk_throttling_maximum_duration_ = tenant_config->log_disk_throttling_maximum_duration; + palf_opts.compress_options_.enable_transport_compress_ = tenant_config->log_transport_compress_all; + palf_opts.compress_options_.transport_compress_func_ = compressor_type; + palf_opts.rebuild_replica_log_lag_threshold_ = tenant_config->_rebuild_replica_log_lag_threshold; + palf_opts.disk_options_.log_writer_parallelism_ = tenant_config->_log_writer_parallelism; + if (OB_FAIL(palf_env_->update_options(palf_opts))) { + CLOG_LOG(WARN, "palf update_options failed", K(MTL_ID()), K(ret), K(palf_opts)); + } else { + CLOG_LOG(INFO, "palf update_options success", K(MTL_ID()), K(ret), K(palf_opts)); + } + } + } + return ret; +} + +//log_disk_usage_limit_size无法主动感知,只能通过上层触发时传入 +int LogService::update_log_disk_usage_limit_size(const int64_t log_disk_usage_limit_size) +{ + int ret = OB_SUCCESS; + ObSpinLockGuard guard(update_palf_opts_lock_); + PalfOptions palf_opts; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(palf_env_->get_options(palf_opts))) { + CLOG_LOG(WARN, "palf get_options failed", K(ret)); + } else if (FALSE_IT(palf_opts.disk_options_.log_disk_usage_limit_size_ = log_disk_usage_limit_size)) { + } else if (OB_FAIL(palf_env_->update_options(palf_opts))) { + CLOG_LOG(WARN, "palf update_options failed", K(ret), K(log_disk_usage_limit_size)); + } else { + CLOG_LOG(INFO, "update_log_disk_usage_limit_size success", K(log_disk_usage_limit_size), K(MTL_ID())); + } + return ret; +} + +int LogService::get_palf_options(palf::PalfOptions &opts) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else { + ret = palf_env_->get_options(opts); + } + return ret; +} + +int LogService::iterate_palf(const ObFunction &func) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else { + ret = palf_env_->for_each(func); + } + return ret; +} + +int LogService::create_ls_(const share::ObLSID &id, + const common::ObReplicaType &replica_type, + const share::ObTenantRole &tenant_role, + const palf::PalfBaseInfo &palf_base_info, + const bool allow_log_sync, + logservice::ObLogHandler &log_handler) +{ + int ret = OB_SUCCESS; + PalfHandle palf_handle; + PalfRoleChangeCb *rc_cb = &role_change_service_; + const bool is_arb_replica = (replica_type == REPLICA_TYPE_ARBITRATION); + PalfHandle &log_handler_palf_handle = log_handler.palf_handle_; + if (false == id.is_valid() || + INVALID_TENANT_ROLE == tenant_role || + false == palf_base_info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", K(ret), K(id), K(id), K(tenant_role), K(palf_base_info)); + } else if (!is_arb_replica && + OB_FAIL(palf_env_->create(id.id(), get_palf_access_mode(tenant_role), palf_base_info, palf_handle))) { + CLOG_LOG(WARN, "failed to get palf_handle", K(ret), K(id), K(replica_type)); + } else if (false == allow_log_sync && OB_FAIL(palf_handle.disable_sync())) { + CLOG_LOG(WARN, "failed to disable_sync", K(ret), K(id)); + } else if (OB_FAIL(apply_service_.add_ls(id))) { + CLOG_LOG(WARN, "failed to add_ls for apply engine", K(ret), K(id)); + // } else if (OB_FAIL(replay_service_.add_ls(id))) { + // CLOG_LOG(WARN, "failed to add_ls", K(ret), K(id)); + } else if (OB_FAIL(log_handler.init(id.id(), self_, &apply_service_, &replay_service_, + NULL, palf_handle, palf_env_, &location_adapter_, &log_service_rpc_proxy_))) { + CLOG_LOG(WARN, "logservice::ObLogHandler init failed", K(ret), KP(palf_env_), K(palf_handle)); + } else if (OB_FAIL(log_handler_palf_handle.register_role_change_cb(rc_cb))) { + CLOG_LOG(WARN, "register_role_change_cb failed", K(ret), K(id)); + } else { + CLOG_LOG(INFO, "LogService create_ls success", K(ret), K(id), K(log_handler)); + } + if (OB_FAIL(ret)) { + if (true == palf_handle.is_valid() && false == log_handler.is_valid()) { + palf_env_->close(palf_handle); + } + } + return ret; +} + +int LogService::get_io_start_time(int64_t &last_working_time) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(WARN, "log_service is not inited", K(ret)); + } else if (OB_FAIL(palf_env_->get_io_start_time(last_working_time))) { + CLOG_LOG(WARN, "palf_env get_io_start_time failed", K(ret)); + } else { + // do nothing + } + return ret; +} + +int LogService::check_disk_space_enough(bool &is_disk_enough) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(WARN, "log_service is not inited", K(ret)); + } else { + is_disk_enough = palf_env_->check_disk_space_enough(); + } + return ret; +} + +}//end of namespace logservice +}//end of namespace oceanbase diff --git a/mittest/palf_cluster/logservice/log_service.h b/mittest/palf_cluster/logservice/log_service.h new file mode 100644 index 0000000000..7549fcd25b --- /dev/null +++ b/mittest/palf_cluster/logservice/log_service.h @@ -0,0 +1,211 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_PALF_CLUSTER_OB_LOG_SERVICE_ +#define OCEANBASE_PALF_CLUSTER_OB_LOG_SERVICE_ + +#include "common/ob_role.h" +#include "lib/ob_define.h" +#include "share/ob_tenant_info_proxy.h" // ObTenantRole +#include "logservice/applyservice/ob_log_apply_service.h" +#include "mittest/palf_cluster/rpc/palf_cluster_rpc_req.h" +#include "mittest/palf_cluster/rpc/palf_cluster_rpc_proxy.h" +#include "mittest/palf_cluster/logservice/ob_log_client.h" +#include "logservice/palf/log_block_pool_interface.h" // ILogBlockPool +#include "logservice/palf/log_define.h" +#include "logservice/ob_log_monitor.h" +#include "logservice/ob_log_handler.h" +#include "logservice/ob_log_service.h" +#include "role_coordinator.h" +#include "ls_adapter.h" +// #include "replayservice/ob_log_replay_service.h" + +namespace oceanbase +{ +namespace commom +{ +class ObAddr; +class ObILogAllocator; +} + +namespace rpc +{ +namespace frame +{ +class ObReqTransport; +} +} + +namespace share +{ +class ObLSID; +class SCN; +} + +namespace palf +{ +class PalfHandleGuard; +class PalfRoleChangeCb; +class PalfDiskOptions; +class PalfEnv; +} + +namespace palfcluster +{ + +class LogService +{ +public: + LogService(); + virtual ~LogService(); + static int mtl_init(LogService* &logservice); + static void mtl_destroy(LogService* &logservice); + int start(); + void stop(); + void wait(); + void destroy(); +public: + static palf::AccessMode get_palf_access_mode(const share::ObTenantRole &tenant_role); + int init(const palf::PalfOptions &options, + const char *base_dir, + const common::ObAddr &self, + common::ObILogAllocator *alloc_mgr, + rpc::frame::ObReqTransport *transport, + palf::ILogBlockPool *log_block_pool); + //--日志流相关接口-- + //新建日志流接口，该接口会创建日志流对应的目录，新建一个以PalfBaeInfo为日志基点的日志流。 + //其中包括生成并初始化对应的ObReplayStatus结构 + // @param [in] id，日志流标识符 + // @param [in] replica_type，日志流的副本类型 + // @param [in] tenant_role, 租户角色, 以此决定Palf使用模式(APPEND/RAW_WRITE) + // @param [in] palf_base_info, 日志同步基点信息 + // @param [out] log_handler，新建日志流以logservice::ObLogHandler形式返回，保证上层使用日志流时的生命周期 + int create_ls(const share::ObLSID &id, + const common::ObReplicaType &replica_type, + const share::ObTenantRole &tenant_role, + const palf::PalfBaseInfo &palf_base_info, + const bool allow_log_sync, + logservice::ObLogHandler &log_handler); + + //删除日志流接口:外层调用create_ls()之后，后续流程失败，需要调用remove_ls() + int remove_ls(const share::ObLSID &id, + logservice::ObLogHandler &log_handler); + + int check_palf_exist(const share::ObLSID &id, bool &exist) const; + //宕机重启恢复日志流接口，包括生成并初始化对应的ObReplayStatus结构 + // @param [in] id，日志流标识符 + // @param [out] log_handler，新建日志流以logservice::ObLogHandler形式返回，保证上层使用日志流时的生命周期 + int add_ls(const share::ObLSID &id, + logservice::ObLogHandler &log_handler); + + int open_palf(const share::ObLSID &id, + palf::PalfHandleGuard &palf_handle); + + // get role of current palf replica. + // NB: distinguish the difference from get_role of log_handler + // In general, get the replica role to do migration/blance/report, use this interface, + // to write log, use get_role of log_handler + int get_palf_role(const share::ObLSID &id, + common::ObRole &role, + int64_t &proposal_id); + + // @brief get palf disk usage + // @param [out] used_size_byte + // @param [out] total_size_byte, if in shrinking status, total_size_byte is the value after shrinking. + // NB: total_size_byte may be smaller than used_size_byte. + int get_palf_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte); + + // @brief get palf disk usage + // @param [out] used_size_byte + // @param [out] total_size_byte, if in shrinking status, total_size_byte is the value before shrinking. + int get_palf_stable_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte); + // why we need update 'log_disk_size_' and 'log_disk_util_threshold' separately. + // + // 'log_disk_size' is a member of unit config. + // 'log_disk_util_threshold' and 'log_disk_util_limit_threshold' are members of tenant parameters. + // If we just only provide 'update_disk_options', the correctness of PalfDiskOptions can not be guaranteed. + // for example, original PalfDiskOptions is that + // { + // log_disk_size = 100G, + // log_disk_util_limit_threshold = 95, + // log_disk_util_threshold = 80 + // } + // + // 1. thread1 update 'log_disk_size' with 50G, and it will used 'update_disk_options' with PalfDiskOptions + // { + // log_disk_size = 50G, + // log_disk_util_limit_threshold = 95, + // log_disk_util_threshold = 80 + // } + // 2. thread2 updaet 'log_disk_util_limit_threshold' with 85, and it will used 'update_disk_options' with PalfDiskOptions + // { + // log_disk_size = 100G, + // log_disk_util_limit_threshold = 85, + // log_disk_util_threshold = 80 + // }. + int update_palf_options_except_disk_usage_limit_size(); + int update_log_disk_usage_limit_size(const int64_t log_disk_usage_limit_size); + int get_palf_options(palf::PalfOptions &options); + int iterate_palf(const ObFunction &func); + + int get_io_start_time(int64_t &last_working_time); + int check_disk_space_enough(bool &is_disk_enough); + + palf::PalfEnv *get_palf_env() { return palf_env_; } + // TODO by yunlong: temp solution, will by removed after Reporter be added in MTL + // ObLogReplayService *get_log_replay_service() { return &replay_service_; } + obrpc::PalfClusterRpcProxy *get_rpc_proxy() { return &rpc_proxy_; } + ObAddr &get_self() { return self_; } + + int create_palf_replica(const int64_t palf_id, + const common::ObMemberList &member_list, + const int64_t replica_num, + const int64_t leader_idx); + + int create_log_clients(const int64_t thread_num, + const int64_t log_size, + const int64_t palf_group_num, + std::vector leader_list); +public: + palfcluster::LogClientMap *get_log_client_map() { return &log_client_map_; } + static const int64_t THREAD_NUM = 2000; + palfcluster::LogRemoteClient clients_[THREAD_NUM]; +private: + int create_ls_(const share::ObLSID &id, + const common::ObReplicaType &replica_type, + const share::ObTenantRole &tenant_role, + const palf::PalfBaseInfo &palf_base_info, + const bool allow_log_sync, + logservice::ObLogHandler &log_handler); +private: + bool is_inited_; + bool is_running_; + + common::ObAddr self_; + palf::PalfEnv *palf_env_; + + logservice::ObLogApplyService apply_service_; + logservice::ObLogReplayService replay_service_; + palfcluster::RoleCoordinator role_change_service_; + logservice::ObLogMonitor monitor_; + obrpc::PalfClusterRpcProxy rpc_proxy_; + ObSpinLock update_palf_opts_lock_; + palfcluster::LogClientMap log_client_map_; + palfcluster::MockLSAdapter ls_adapter_; + logservice::ObLocationAdapter location_adapter_; + obrpc::ObLogServiceRpcProxy log_service_rpc_proxy_; +private: + DISALLOW_COPY_AND_ASSIGN(LogService); +}; +} // end namespace palfcluster +} // end namespace oceanbase +#endif // OCEANBASE_PALF_CLUSTER_OB_LOG_SERVICE_ diff --git a/mittest/palf_cluster/logservice/ls_adapter.cpp b/mittest/palf_cluster/logservice/ls_adapter.cpp new file mode 100644 index 0000000000..e6c041f6af --- /dev/null +++ b/mittest/palf_cluster/logservice/ls_adapter.cpp @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ls_adapter.h" +#include "logservice/replayservice/ob_replay_status.h" + +namespace oceanbase +{ +namespace palfcluster +{ +MockLSAdapter::MockLSAdapter() : + is_inited_(false) + {} + +MockLSAdapter::~MockLSAdapter() +{ + destroy(); +} + +int MockLSAdapter::init() +{ + int ret = OB_SUCCESS; + if (is_inited_) { + ret = OB_INIT_TWICE; + CLOG_LOG(WARN, "MockLSAdapter init twice", K(ret)); + } else { + is_inited_ = true; + CLOG_LOG(INFO, "MockLSAdapter init success", K(ret)); + } + return ret; +} + +void MockLSAdapter::destroy() +{ + is_inited_ = false; +} + +int MockLSAdapter::replay(logservice::ObLogReplayTask *replay_task) +{ + int ret = OB_SUCCESS; + return ret; +} + +int MockLSAdapter::wait_append_sync(const share::ObLSID &ls_id) +{ + int ret = OB_SUCCESS; + return ret; +} + +} // end namespace palfcluster +} // end namespace oceanbase diff --git a/mittest/palf_cluster/logservice/ls_adapter.h b/mittest/palf_cluster/logservice/ls_adapter.h new file mode 100644 index 0000000000..4aed34fc57 --- /dev/null +++ b/mittest/palf_cluster/logservice/ls_adapter.h @@ -0,0 +1,47 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_PALF_CLUSTER_LS_ADAPTER_H_ +#define OCEANBASE_PALF_CLUSTER_LS_ADAPTER_H_ + +#include +#include "share/ob_ls_id.h" +#include "logservice/ob_ls_adapter.h" + +namespace oceanbase +{ +namespace logservice +{ +class ObLogReplayTask; +}; + +namespace palfcluster +{ + +class MockLSAdapter : public logservice::ObLSAdapter +{ +public: + MockLSAdapter(); + ~MockLSAdapter(); + int init(); + void destroy(); +public: + int replay(logservice::ObLogReplayTask *replay_task) override final; + int wait_append_sync(const share::ObLSID &ls_id) override final; +private: + bool is_inited_; +}; + +} // logservice +} // oceanbase + +#endif diff --git a/mittest/palf_cluster/logservice/ob_log_client.cpp b/mittest/palf_cluster/logservice/ob_log_client.cpp new file mode 100644 index 0000000000..ba6baa469d --- /dev/null +++ b/mittest/palf_cluster/logservice/ob_log_client.cpp @@ -0,0 +1,237 @@ +// Copyright (c) 2021 OceanBase +// OceanBase is licensed under Mulan PubL v2. +// You can use this software according to the terms and conditions of the Mulan PubL v2. +// You may obtain a copy of Mulan PubL v2 at: +// http://license.coscl.org.cn/MulanPubL-2.0 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PubL v2 for more details. + +#define USING_LOG_PREFIX CLOG +#include "ob_log_client.h" +#include "logservice/palf/palf_env.h" +#include "logservice/palf/palf_handle.h" +#include "lib/utility/ob_macro_utils.h" +#include "lib/thread/ob_thread_name.h" // set_thread_name +#include "lib/function/ob_function.h" // ObFunction +#include "mittest/palf_cluster/rpc/palf_cluster_rpc_proxy.h" // RpcProxy +#include "mittest/palf_cluster/logservice/log_service.h" // LogService + +namespace oceanbase +{ +using namespace common; +using namespace palf; +namespace palfcluster +{ +int64_t c_append_cnt = 0; +int64_t c_rt = 0; + +ObLogClient::ObLogClient() + : self_(), + palf_id_(-1), + rpc_proxy_(NULL), + log_handler_(), + lock_(), + log_size_(-1), + election_priority_(), + total_num_(0), + is_inited_(false) + {} + +ObLogClient::~ObLogClient() +{ + destroy(); +} + +void ObLogClient::destroy() +{ + if (IS_INIT) { + is_inited_ = false; + rpc_proxy_ = NULL; + palf_id_ = -1; + log_handler_.destroy(); + log_size_ = -1; + } +} + +int ObLogClient::init(const common::ObAddr &self, + const int64_t palf_id, + obrpc::PalfClusterRpcProxy *rpc_proxy, + LogService *log_service) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + CLOG_LOG(WARN, "ObLogClient has been inited", K(ret)); + } else if (false == self.is_valid()|| OB_ISNULL(rpc_proxy) || palf_id < 0) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", K(ret), K(self), K(palf_id), K(rpc_proxy)); + } else { + for (int i = 0; i < REMOTE_APPEND_CB_CNT; i++) { + if (OB_FAIL(remote_append_cb_list[i].init(rpc_proxy, self))) { + CLOG_LOG(WARN, "init", K(ret), K(self), K(palf_id), K(rpc_proxy)); + } + } + self_ = self; + palf_id_ = palf_id; + rpc_proxy_ = rpc_proxy; + log_service_ = log_service; + is_inited_ = true; + } + + if ((OB_FAIL(ret)) && (OB_INIT_TWICE != ret)) { + destroy(); + } + CLOG_LOG(INFO, "ObLogClient init finished", K(ret), K(self)); + return ret; +} + +int ObLogClient::create_palf_replica(const common::ObMemberList &member_list, + const int64_t replica_num, + const int64_t leader_idx) +{ + int ret = OB_SUCCESS; + if(IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(ERROR, "ObLogClient has not been inited", K(ret)); + } else if (false == member_list.is_valid() || replica_num <= 0) { + ret = OB_INVALID_ARGUMENT; + } else { + share::ObLSID ls_id(palf_id_); + share::ObTenantRole tenant_role(share::ObTenantRole::PRIMARY_TENANT); + common::ObAddr leader; + member_list.get_server_by_index(leader_idx, leader); + + LSN init_lsn(0); + common::ObILogAllocator *alloc_mgr = NULL; + palf::PalfBaseInfo palf_base_info; + palf_base_info.generate_by_default(); + common::GlobalLearnerList learner_list; + if (OB_FAIL(log_service_->create_ls(ls_id, ObReplicaType::REPLICA_TYPE_FULL, + tenant_role, palf_base_info, true, log_handler_))) { + CLOG_LOG(WARN, "create_ls failed", K(ret), K_(palf_id)); + } else if (OB_FAIL(log_handler_.set_initial_member_list(member_list, replica_num, learner_list))) { + CLOG_LOG(WARN, "set_initial_member_list failed", K(ret), K_(palf_id)); + } else { + CLOG_LOG(ERROR, "create_palf_replica success", K(ret), K_(palf_id), K(member_list), K(replica_num), K(leader_idx)); + } + } + return ret; +} + +static void *append_fn(void *arg) +{ + ObLogClient *client = reinterpret_cast(arg); + client->do_submit(); + return (void *)0; +} + +int ObLogClient::submit_append_log_task(const int64_t thread_num, const int64_t log_size) +{ + int ret = OB_SUCCESS; + common::ObRole role; + int64_t unused_pid; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(log_handler_.get_role(role, unused_pid))) { + } else if (role != LEADER) { + CLOG_LOG(ERROR, "client is not leader"); + } else if (OB_FAIL(lock_.trylock())) { + CLOG_LOG(ERROR, "another client running", K(ret)); + } else { + log_size_ = log_size; + client_number_ = thread_num; + worker_number_ = (client_number_ >= MAX_THREAD_NUM)? MAX_THREAD_NUM: client_number_; + pthread_t tids[MAX_THREAD_NUM]; + + CLOG_LOG(ERROR, "start submit_log", K_(log_size), K(thread_num), K(worker_number_)); + for (int64_t i = 0; i < worker_number_; i++) { + pthread_create(&tids[i], NULL, append_fn, this); + } + + for (int64_t i = 0; i < worker_number_; i++) { + pthread_join(tids[i], NULL); + } + lock_.unlock(); + } + return ret; +} + +int ObLogClient::do_submit() +{ + int ret = OB_SUCCESS; + const int64_t NBYTES = 40000; + char BUFFER[NBYTES]; + memset(BUFFER, 'a', NBYTES); + const int64_t CB_ARRAY_NUM = (client_number_ >= worker_number_)? client_number_ / worker_number_: 1; + MockAppendCb *cb_array = new MockAppendCb[CB_ARRAY_NUM]; + LSN lsn; + SCN log_scn; + while (true) { + for (int i = 0; i < CB_ARRAY_NUM; i++) + { + if (cb_array[i].is_called()) { + cb_array[i].reset(); + const int64_t log_size = (log_size_ > 0)? log_size_: ObRandom::rand(100, 1024); + cb_array[i].log_size_ = log_size; + ret = log_handler_.append(BUFFER, log_size, SCN::min_scn(), true, &cb_array[i], lsn, log_scn); + if (OB_SUCCESS != ret) { + (void) cb_array[i].on_success(); + } + } + } + } + while (true) { + sleep(10); + } + return ret; +} + +int MockAppendCb::on_success() +{ + ATOMIC_STORE(&is_called_, true); + return OB_SUCCESS; +} + +int ObLogClient::submit_log(const common::ObAddr &client_addr, const int64_t client_id, const palf::LogWriteBuf &log_buf) +{ + int ret = OB_SUCCESS; + LSN lsn; + SCN log_scn; + const char *buf = log_buf.write_buf_[0].buf_; + const int64_t buf_len = log_buf.write_buf_[0].buf_len_; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(remote_append_cb_list[client_id].pre_submit(client_addr, palf_id_, client_id))) { + // CLOG_LOG(WARN, "append_cb init failed", K(ret)); + } else if (OB_FAIL(log_handler_.append(buf, buf_len, SCN::min_scn(), true, &(remote_append_cb_list[client_id]), lsn, log_scn))) { + CLOG_LOG(WARN, "append failed", K(ret)); + } + return OB_SUCCESS; +} + +void LogRemoteClient::has_returned() +{ + const int64_t tmp_rt = common::ObTimeUtility::current_time() - last_submit_ts_; + + ATOMIC_STORE(&is_returned_, true); + ObThreadCondGuard guard(cond_); + cond_.signal(); + + ATOMIC_INC(&c_append_cnt); + ATOMIC_FAA(&c_rt, tmp_rt); + + if (REACH_TIME_INTERVAL(1000 *1000)) { + int64_t l_append_cnt = ATOMIC_LOAD(&c_append_cnt); + if (l_append_cnt == 0) l_append_cnt = 1; + int64_t l_rt = ATOMIC_LOAD(&c_rt); + + ATOMIC_STORE(&c_append_cnt, 0); + ATOMIC_STORE(&c_rt, 0); + + CLOG_LOG_RET(ERROR, OB_SUCCESS, "result:", K(l_append_cnt), K(l_rt), "avg_rt", l_rt/l_append_cnt); + } +} +} // end namespace palfcluster +} // end namespace oceanbase \ No newline at end of file diff --git a/mittest/palf_cluster/logservice/ob_log_client.h b/mittest/palf_cluster/logservice/ob_log_client.h new file mode 100644 index 0000000000..fcfd21354c --- /dev/null +++ b/mittest/palf_cluster/logservice/ob_log_client.h @@ -0,0 +1,314 @@ +// Copyright (c) 2021 OceanBase +// OceanBase is licensed under Mulan PubL v2. +// You can use this software according to the terms and conditions of the Mulan PubL v2. +// You may obtain a copy of Mulan PubL v2 at: +// http://license.coscl.org.cn/MulanPubL-2.0 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PubL v2 for more details. + +#ifndef OCEANBASE_PALF_CLUSTER_LOG_CLIENT_H_ +#define OCEANBASE_PALF_CLUSTER_LOG_CLIENT_H_ + +#include "share/ob_thread_pool.h" +#include "share/ob_occam_timer.h" +#include "lib/hash/ob_linear_hash_map.h" // ObLinearHashMap +#include "lib/lock/ob_tc_rwlock.h" // RWLock +#include "common/ob_member_list.h" // common::ObMemberList +#include "storage/ob_locality_manager.h" // ObLocalityManager +#include "logservice/palf/palf_handle_impl.h" +#include "logservice/ob_log_handler.h" +#include "logservice/ob_log_service.h" +#include "mittest/palf_cluster/rpc/palf_cluster_rpc_req.h" // ProbeMsg +#include "mittest/palf_cluster/rpc/palf_cluster_rpc_proxy.h" // ProbeMsg + +namespace oceanbase +{ + +namespace palf +{ +class PalfEnv; +} + +namespace obrpc +{ +class PalfClusterRpcProxy; +} + +namespace palfcluster +{ +class LogService; + +class MockAppendCb : public logservice::AppendCb +{ +public: + MockAppendCb() + : log_size_(0), + is_called_(true) + { } + + int on_success() override final; + + int on_failure() override final + { + ATOMIC_STORE(&is_called_, true); + return OB_SUCCESS; + } + + void reset() + { + ATOMIC_STORE(&is_called_, false); + } + + bool is_called() const + { + return ATOMIC_LOAD(&is_called_); + } + +public: + int64_t log_size_; + bool is_called_; +}; + +class MockRemoteAppendCb : public logservice::AppendCb +{ +public: + MockRemoteAppendCb() + : rpc_proxy_(NULL), + client_addr_(), + is_called_(true) + { } + + ~MockRemoteAppendCb() + { + rpc_proxy_ = NULL; + client_addr_.reset(); + } + + int init(obrpc::PalfClusterRpcProxy *rpc_proxy, + const common::ObAddr &self) + { + int ret = OB_SUCCESS; + if (rpc_proxy == NULL) { + ret = OB_INVALID_ARGUMENT; + } else { + rpc_proxy_ = rpc_proxy; + self_ = self; + } + return ret; + } + + int pre_submit(const common::ObAddr &client_addr, + const int64_t palf_id, + const int64_t client_id) + { + int ret = OB_SUCCESS; + if (is_called()) { + reset(); + client_addr_ = client_addr; + palf_id_ = palf_id; + client_id_ = client_id; + } else { + ret = OB_EAGAIN; + } + return ret; + } + + int on_success() override final + { + int ret = OB_SUCCESS; + ATOMIC_STORE(&is_called_, true); + AppendCb::reset(); + if (OB_NOT_NULL(rpc_proxy_) && client_addr_.is_valid()) { + // notify called + const int64_t RPC_TIMEOUT_US = 1 * 1000 * 1000; + SubmitLogCmdResp resp(self_, palf_id_, client_id_); + static obrpc::ObLogRpcCB cb; \ + if (OB_FAIL(rpc_proxy_->to(client_addr_).timeout(RPC_TIMEOUT_US).trace_time(true). + max_process_handler_time(RPC_TIMEOUT_US).by(MTL_ID()).send_submit_log_resp(resp, &cb))) { + CLOG_LOG(ERROR, "send_submit_log_resp failed", KR(ret), K(resp)); + } + } + return OB_SUCCESS; + } + + int on_failure() override final + { + ATOMIC_STORE(&is_called_, true); + if (OB_NOT_NULL(rpc_proxy_) && client_addr_.is_valid()) { + //notify called + } + return OB_SUCCESS; + } + + void reset() + { + ATOMIC_STORE(&is_called_, false); + } + + bool is_called() const + { + return ATOMIC_LOAD(&is_called_); + } + + obrpc::PalfClusterRpcProxy *rpc_proxy_; + common::ObAddr client_addr_; + common::ObAddr self_; + int64_t palf_id_; + int64_t client_id_; + bool is_called_; +}; + +class LogRemoteClient +{ +public: + LogRemoteClient() + : thread_(), + th_id_(0), + log_size_(0), + rpc_proxy_(NULL), + self_(), + dst_(), + palf_id_(0), + cond_(), + is_returned_(true), + last_submit_ts_(0), + avg_rt_(-1), + is_inited_(false) {} + + int init_and_create(const int64_t th_id, + const int64_t log_size, + obrpc::PalfClusterRpcProxy *rpc_proxy, + const common::ObAddr &self, + const common::ObAddr &dst, + const int64_t palf_id) + { + int ret = OB_SUCCESS; + cond_.init(ObWaitEventIds::REBALANCE_TASK_MGR_COND_WAIT); + if (IS_INIT) { + ret = OB_INIT_TWICE; + } else if (th_id < 0 || log_size <= 0 || + rpc_proxy == NULL || false == self.is_valid()) { + ret = OB_INVALID_ARGUMENT; + } else if (0 != pthread_create(&thread_, NULL, do_submit, this)){ + PALF_LOG(ERROR, "create thread fail", K(thread_)); + } else { + th_id_ = th_id; + log_size_ = log_size; + rpc_proxy_ = rpc_proxy; + self_ = self; + dst_ = dst; + palf_id_ = palf_id; + } + return ret; + } + + void join() + { + pthread_join(thread_, NULL); + } + + static void* do_submit(void *arg) + { + int ret = OB_SUCCESS; + const int64_t NBYTES = 40000; + const int64_t RPC_TIMEOUT_US = 1 * 1000 * 1000; + char BUFFER[NBYTES]; + memset(BUFFER, 'a', NBYTES); + + LogRemoteClient *client = static_cast(arg); + palf::LogWriteBuf write_buf; + write_buf.push_back(BUFFER, client->log_size_); + + SubmitLogCmd req(client->self_, client->palf_id_, client->th_id_, write_buf); + + while (true) { + // const bool is_timeout = (common::ObTimeUtility::current_time() - client->last_submit_ts_) > 500 * 1000; + // if (client->can_submit()) + static obrpc::ObLogRpcCB cb; + if (OB_FAIL(client->rpc_proxy_->to(client->dst_).timeout(RPC_TIMEOUT_US).trace_time(true). \ + max_process_handler_time(RPC_TIMEOUT_US).by(MTL_ID()).send_submit_log_cmd(req, &cb))) { + PALF_LOG(WARN, "send_submit_log_cmd fail", K(req)); + } else { + client->has_submit(); + ObThreadCondGuard guard(client->cond_); + client->cond_.wait(); + } + } + return NULL; + } + + bool can_submit() const + { + return ATOMIC_LOAD(&is_returned_); + } + + void has_submit() + { + ATOMIC_STORE(&is_returned_, false); + last_submit_ts_ = common::ObTimeUtility::current_time(); + } + + void has_returned(); + +public: + pthread_t thread_; + int64_t th_id_; + int64_t log_size_; + obrpc::PalfClusterRpcProxy *rpc_proxy_; + common::ObAddr self_; + common::ObAddr dst_; + int64_t palf_id_; + mutable common::ObThreadCond cond_; + bool is_returned_; + int64_t last_submit_ts_; + int64_t avg_rt_; + bool is_inited_; +}; + +class ObLogClient +{ +public: + ObLogClient(); + virtual ~ObLogClient(); + int init(const common::ObAddr &self, + const int64_t palf_id, + obrpc::PalfClusterRpcProxy *rpc_proxy, + palfcluster::LogService *log_service); + void destroy(); + int create_palf_replica(const common::ObMemberList &member_list, + const int64_t replica_num, + const int64_t leader_idx); + int submit_append_log_task(const int64_t thread_num, const int64_t log_size); + int submit_log(const common::ObAddr &client_addr, const int64_t client_id, const palf::LogWriteBuf &log_buf); + int do_submit(); + + share::ObLSID get_ls_id() const { return share::ObLSID(palf_id_); } + logservice::ObLogHandler *get_log_handler() { return &log_handler_;} + TO_STRING_KV(K_(palf_id)); +private: + static const int64_t MAX_THREAD_NUM = 10; + static const int64_t REMOTE_APPEND_CB_CNT = 4000; + MockRemoteAppendCb remote_append_cb_list[REMOTE_APPEND_CB_CNT]; + common::ObAddr self_; + int64_t palf_id_; + obrpc::PalfClusterRpcProxy *rpc_proxy_; + logservice::ObLogHandler log_handler_; + common::ObSpinLock lock_; + int64_t log_size_; + logservice::coordinator::ElectionPriorityImpl election_priority_; + int64_t total_num_; + palfcluster::LogService *log_service_; + int64_t client_number_; + int64_t worker_number_; + bool is_inited_; +}; + + +typedef common::ObLinearHashMap LogClientMap; + +} // palfcluster +} // oceanbase + +#endif \ No newline at end of file diff --git a/mittest/palf_cluster/logservice/role_coordinator.cpp b/mittest/palf_cluster/logservice/role_coordinator.cpp new file mode 100644 index 0000000000..975ad23a9b --- /dev/null +++ b/mittest/palf_cluster/logservice/role_coordinator.cpp @@ -0,0 +1,747 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "role_coordinator.h" +#include "common/ob_role.h" +#include "lib/ob_define.h" +#include "lib/ob_errno.h" +#include "lib/time/ob_time_utility.h" +#include "lib/utility/ob_macro_utils.h" +#include "lib/thread/thread_mgr.h" +#include "logservice/palf/log_define.h" +#include "share/ob_errno.h" +#include "share/ob_ls_id.h" +#include "share/ob_thread_define.h" +#include "share/rc/ob_tenant_base.h" +#include "storage/tx_storage/ob_ls_service.h" +#include "storage/tx_storage/ob_ls_handle.h" +#include "share/ob_occam_time_guard.h" + +namespace oceanbase +{ +using namespace common; +using namespace palf; +using namespace logservice; +using namespace share; +namespace palfcluster +{ + +RoleChangeEvent::RoleChangeEvent(const RoleChangeEventType &event_type, + const share::ObLSID &ls_id) : event_type_(event_type), + ls_id_(ls_id) +{ +} +RoleChangeEvent::RoleChangeEvent(const RoleChangeEventType &event_type, + const share::ObLSID &ls_id, + const common::ObAddr &dst_addr) : event_type_(event_type), + ls_id_(ls_id), + dst_addr_(dst_addr) +{ +} + +bool RoleChangeEvent::is_valid() const +{ + return RoleChangeEventType::INVALID_RC_EVENT_TYPE != event_type_ + && false != ls_id_.is_valid(); +} + +void RoleChangeEvent::reset() +{ + event_type_ = RoleChangeEventType::INVALID_RC_EVENT_TYPE; + ls_id_.reset(); + dst_addr_.reset(); +} + +bool RoleChangeEvent::operator==(const RoleChangeEvent &rhs) const +{ + // for change leader event, we just check 'ls_id'. + return event_type_ == rhs.event_type_ && ls_id_ == rhs.ls_id_; +} + +RoleChangeEventSet::RoleChangeEventSet() +{} + +RoleChangeEventSet::~RoleChangeEventSet() +{} + +int RoleChangeEventSet::insert(const RoleChangeEvent &event) +{ + int ret = OB_SUCCESS; + int64_t free_idx = -1; + ObSpinLockGuard guard(lock_); + for (int64_t i = 0; i < MAX_ARRAY_SIZE; i++) { + if (event == events_[i]) { + ret = OB_ENTRY_EXIST; + } + } + for (int64_t i = 0; i < MAX_ARRAY_SIZE && -1 == free_idx && OB_SUCC(ret); i++) { + if (false == events_[i].is_valid()) { + free_idx = i; + } + } + if (OB_ENTRY_EXIST == ret) { + } else if (-1 != free_idx) { + events_[free_idx] = event; + } else { + ret = OB_SIZE_OVERFLOW; + } + CLOG_LOG(INFO, "insert event into set success", K(ret), K(event), K(free_idx)); + return ret; +} + +int RoleChangeEventSet::remove(const RoleChangeEvent &event) +{ + int ret = OB_SUCCESS; + int64_t delete_idx = -1; + ObSpinLockGuard guard(lock_); + for (int64_t i = 0; i < MAX_ARRAY_SIZE && -1 == delete_idx; i++) { + if (event == events_[i]) { + delete_idx = i; + } + }; + if (-1 != delete_idx) { + events_[delete_idx].reset(); + } else { + ret = OB_ENTRY_NOT_EXIST; + } + CLOG_LOG(INFO, "remove slog from set success", K(ret), K(delete_idx), K(event)); + return ret; +} + +RoleCoordinator::RoleCoordinator() : log_client_map_(NULL), + apply_service_(NULL), + tg_id_(-1), + is_inited_(false) +{ +} + +RoleCoordinator::~RoleCoordinator() +{ + if (IS_INIT) { + destroy(); + } +} + +int RoleCoordinator::init(palfcluster::LogClientMap *log_client_map, + logservice::ObLogApplyService *apply_service) +{ + int ret = OB_SUCCESS; + const int tg_id = lib::TGDefIDs::RCService; + if (IS_INIT) { + ret = OB_INIT_TWICE; + } else if (OB_ISNULL(log_client_map) || OB_ISNULL(apply_service)) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", K(ret), KP(apply_service)); + } else if (OB_FAIL(TG_CREATE_TENANT(tg_id, tg_id_))) { + CLOG_LOG(WARN, "RoleCoordinator TG_CREATE failed", K(ret)); + } else { + apply_service_ = apply_service; + log_client_map_ = log_client_map; + is_inited_ = true; + CLOG_LOG(INFO, "RoleCoordinator init success", K(ret), K(tg_id_), KP(apply_service)); + } + return ret; +} + +int RoleCoordinator::start() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(TG_SET_HANDLER_AND_START(tg_id_, *this))) { + CLOG_LOG(WARN, "RoleCoordinator start failed", K(ret), K(tg_id_)); + } else { + CLOG_LOG(INFO, "RoleCoordinator start success", K(ret), K(tg_id_)); + } + return ret; +} + +void RoleCoordinator::wait() +{ + if (IS_INIT) { + TG_STOP(tg_id_); + TG_WAIT(tg_id_); + } + CLOG_LOG(INFO, "RoleCoordinator wait finish", K(tg_id_)); +} + +void RoleCoordinator::stop() +{ + if (IS_INIT) { + TG_STOP(tg_id_); + } + CLOG_LOG(INFO, "RoleCoordinator stop finish", K(tg_id_)); +} + +void RoleCoordinator::destroy() +{ + if (IS_INIT) { + (void)stop(); + (void)wait(); + TG_DESTROY(tg_id_); + is_inited_ = false; + tg_id_ = -1; + apply_service_ = NULL; + CLOG_LOG(INFO, "RoleCoordinator destroy success"); + } +} + +void RoleCoordinator::handle(void *task) +{ + int ret = OB_SUCCESS; + // When role chage service hang exceeds 30 seconds, we think there is dead lock in 'handle_role_change_event_', + // TIMEGUARD will pring lbt(). + TIMEGUARD_INIT(CLOG, 30_s, 30_s); + RoleChangeEvent *event = reinterpret_cast(task); + const int64_t ls_id = event->ls_id_.id(); + const int64_t start_ts = ObTimeUtility::current_time(); + RetrySubmitRoleChangeEventCtx retry_ctx; + CLOG_LOG(INFO, "begin handle_role_change_event_", "sequence:", start_ts, KPC(event)); + if (NULL == event) { + CLOG_LOG(WARN, "unexpected error, task is nullptr", KP(event)); + } else if (OB_FAIL(handle_role_change_event_(*event, retry_ctx))) { + CLOG_LOG(WARN, "handle_role_change_event_ failed", K(ret), KPC(event), K(retry_ctx)); + } else { + CLOG_LOG(INFO, "end handle_role_change_event_", "sequence:", start_ts, KPC(event)); + } + if (NULL != event) { + OB_DELETE(RoleChangeEvent, "RCService", event); + } + if (retry_ctx.need_retry() && OB_FAIL(on_role_change(ls_id))) { + CLOG_LOG(WARN, "retry submit role change event failed", K(ls_id), K(retry_ctx)); + } +} + +int RoleCoordinator::on_role_change(const int64_t id) +{ + int ret = OB_SUCCESS; + share::ObLSID ls_id(id); + RoleChangeEvent event(RoleChangeEventType::ROLE_CHANGE_CB_EVENT_TYPE, ls_id); + // TODO by runlin: if task queue has been full, push task will be failed, the role change event + // will be lost. + if (OB_FAIL(submit_role_change_event_(event))) { + CLOG_LOG(WARN, "submit_role_change_event_ failed", K(ret), K(event)); + } else { + CLOG_LOG(INFO, "on_role_change success", K(ret), K(event)); + } + return ret; +} + +int RoleCoordinator::on_need_change_leader(const int64_t ls_id, const common::ObAddr &dst_addr) +{ + int ret = OB_SUCCESS; + RoleChangeEvent event(RoleChangeEventType::CHANGE_LEADER_EVENT_TYPE, share::ObLSID(ls_id), dst_addr); + if (OB_FAIL(submit_role_change_event_(event))) { + CLOG_LOG(WARN, "submit_role_change_event_ failed", K(ret), K(event)); + } else { + CLOG_LOG(INFO, "change_leader success", K(ret), K(event)); + } + return ret; +} + +int RoleCoordinator::submit_role_change_event_(const RoleChangeEvent &event) +{ + int ret = OB_SUCCESS; + if(OB_FAIL(rc_set_.insert(event)) && OB_ENTRY_EXIST != ret) { + CLOG_LOG(ERROR, "insert into rc_set failed", K(ret), K(event)); + } else if (OB_ENTRY_EXIST == ret) { + CLOG_LOG(INFO, "repeat role change event, filter it", K(ret), K(event)); + ret = OB_SUCCESS; + } else if (OB_FAIL(push_event_into_queue_(event))) { + CLOG_LOG(WARN, "push_event_into_queue_ failed", K(ret), K(event)); + } else { + CLOG_LOG(INFO, "submit_role_change_event_ success", K(ret), K(event)); + } + return ret; +} + +// TODO: use poll to avoid alloc memory failed. +int RoleCoordinator::push_event_into_queue_(const RoleChangeEvent &event) +{ + int ret = OB_SUCCESS; + RoleChangeEvent *rc_event = NULL; + + int64_t warn_time = OB_INVALID_TIMESTAMP; + do { + if (NULL == (rc_event = + MTL_NEW(RoleChangeEvent, "RCService", event.event_type_, event.ls_id_, event.dst_addr_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + if (palf_reach_time_interval(1 * 1000 * 1000, warn_time)) { + CLOG_LOG(WARN, "allocate memory failed", K(ret), K(event)); + } + usleep(1 * 1000); + } else { + ret = OB_SUCCESS; + } + } while(OB_FAIL(ret)); + + if (OB_FAIL(TG_PUSH_TASK(tg_id_, rc_event))) { + CLOG_LOG(WARN, "ObRoleChangeTask push task failed", K(ret), K(event)); + } + if (OB_FAIL(ret) && NULL != rc_event) { + MTL_DELETE(RoleChangeEvent, "RCService", rc_event); + } + return ret; +} + +int RoleCoordinator::handle_role_change_event_(const RoleChangeEvent &event, + RetrySubmitRoleChangeEventCtx &retry_ctx) +{ + int ret = OB_SUCCESS; + ObLogClient *ls = nullptr; + AccessMode curr_access_mode; + int64_t unused_mode_version; + OB_ASSERT(OB_SUCCESS == rc_set_.remove(event)); + if (false == event.is_valid()) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", K(event)); + } else if (OB_FAIL(log_client_map_->get(event.ls_id_, ls) || OB_ISNULL(ls))) { + CLOG_LOG(ERROR, "not exist loghandler", K(ret), K(event.ls_id_), KP(ls)); + ret = OB_ERR_UNEXPECTED; + } else if (OB_FAIL(ls->get_log_handler()->get_access_mode(unused_mode_version, curr_access_mode))) { + CLOG_LOG(WARN, "ObLogHandler get_access_mode failed", K(ret)); + } else { + switch (event.event_type_) { + case RoleChangeEventType::CHANGE_LEADER_EVENT_TYPE: + CLOG_LOG(INFO, "begin change leader", K(curr_access_mode), K(event), KPC(ls)); + if (is_append_mode(curr_access_mode) + && OB_FAIL(handle_change_leader_event_for_log_handler_(event.dst_addr_, ls))) { + CLOG_LOG(WARN, "ObLogHandler change leader failed", K(ret), K(event), KPC(ls)); + } + CLOG_LOG(INFO, "end change leader", K(ret), K(curr_access_mode), K(event), KPC(ls)); + break; + case RoleChangeEventType::ROLE_CHANGE_CB_EVENT_TYPE: + CLOG_LOG(INFO, "begin log handler role change", K(curr_access_mode), K(event), KPC(ls)); + if (OB_FAIL(handle_role_change_cb_event_for_log_handler_(curr_access_mode, ls, retry_ctx))) { + CLOG_LOG(WARN, "handle_role_change_cb_event_for_log_handler_ failed", K(ret), + K(curr_access_mode), KPC(ls)); + } + CLOG_LOG(INFO, "end log handler role change", K(ret), K(curr_access_mode), K(event), KPC(ls), K(retry_ctx)); + break; + default: + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "unexpected role change event type", K(ret)); + } + } + return ret; +} + +int RoleCoordinator::handle_role_change_cb_event_for_log_handler_( + const AccessMode &curr_access_mode, + ObLogClient*ls, + RetrySubmitRoleChangeEventCtx &retry_ctx) +{ + int ret = OB_SUCCESS; + const bool log_handler_is_offline = ls->get_log_handler()->is_offline(); + + // If log handler is offline, need execute LEADER_2_FOLLOWER or FOLLOWER_2_FOLLOWER + // + // when access mode is APPEND, log_handler need execute leader to follower or + // follower to leader. otherwise, only need execute leader to follower or follower + // to follower, therefore, we set 'need_transform_by_access_mode' to false when + // 'curr_access_mode' is APPEND. + const bool only_need_change_to_follower = !is_append_mode(curr_access_mode) || log_handler_is_offline; + RoleChangeOptType opt_type; + ObRole curr_role = ObRole::INVALID_ROLE; + ObRole new_role = ObRole::INVALID_ROLE; + bool is_pending_state = false; + int64_t curr_proposal_id = -1; + int64_t new_proposal_id = -1; + if (OB_FAIL(ls->get_log_handler()->prepare_switch_role(curr_role, + curr_proposal_id, new_role, new_proposal_id, is_pending_state))) { + CLOG_LOG(WARN, "ObLogHandler prepare_switch_role failed", K(ret), K(curr_role), K(curr_proposal_id), + K(new_role), K(new_proposal_id)); + } else if (false == need_execute_role_change(curr_proposal_id, curr_role, new_proposal_id, + new_role, is_pending_state, log_handler_is_offline)) { + CLOG_LOG(INFO, "no need change role", K(ret), K(is_pending_state), K(curr_role), K(curr_proposal_id), + K(new_role), K(new_proposal_id), K(is_pending_state), K(log_handler_is_offline)); + } else if (FALSE_IT(opt_type = get_role_change_opt_type_(curr_role, new_role, only_need_change_to_follower))) { + } else { + switch (opt_type) { + // leader -> follower + case RoleChangeOptType::LEADER_2_FOLLOWER: + if (OB_FAIL(switch_leader_to_follower_forcedly_(new_proposal_id, ls))) { + CLOG_LOG(WARN, "switch_leader_to_follower_forcedly_ failed", K(ret), K(curr_role), + K(curr_proposal_id), K(new_role), K(curr_access_mode), K(new_proposal_id)); + } + break; + // follower -> follower + case RoleChangeOptType::FOLLOWER_2_LEADER: + if (OB_FAIL(switch_follower_to_leader_(new_proposal_id, ls, retry_ctx))) { + CLOG_LOG(WARN, "switch_follower_to_leader_ failed", K(ret), K(curr_role), + K(curr_proposal_id), K(new_role), K(curr_access_mode), K(new_proposal_id)); + } + break; + // leader -> leader + case RoleChangeOptType::LEADER_2_LEADER: + if (OB_FAIL(switch_leader_to_leader_(new_proposal_id, curr_proposal_id, ls, retry_ctx))) { + CLOG_LOG(WARN, "switch_leader_to_leader_ failed", K(ret), K(curr_role), + K(curr_proposal_id), K(new_role), K(curr_access_mode), K(new_proposal_id)); + } + break; + // follower -> follower + case RoleChangeOptType::FOLLOWER_2_FOLLOWER: + if (OB_FAIL(switch_follower_to_follower_(new_proposal_id, ls))) { + CLOG_LOG(WARN, "switch_follower_to_follower_ failed", K(ret), K(curr_role), + K(curr_proposal_id), K(new_role), K(curr_access_mode), K(new_proposal_id)); + } + break; + default: + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "unexpected error, can not handle role change", K(ret), K(curr_role), + K(curr_proposal_id), K(new_role), K(new_proposal_id), KPC(ls)); + } + } + return ret; +} + +int RoleCoordinator::handle_change_leader_event_for_log_handler_( + const common::ObAddr &dst_addr, + ObLogClient*ls) +{ + int ret = OB_SUCCESS; + ObRole curr_role = ObRole::INVALID_ROLE; + ObRole new_role = ObRole::INVALID_ROLE; + bool is_pending_state = false; + int64_t curr_proposal_id = -1; + int64_t new_proposal_id = -1; + if (OB_FAIL(ls->get_log_handler()->prepare_switch_role(curr_role, + curr_proposal_id, new_role, new_proposal_id, is_pending_state))) { + CLOG_LOG(WARN, "ObLogHandler prepare_switch_role failed", K(ret), K(curr_role), K(curr_proposal_id), + K(new_role), K(new_proposal_id)); + } else if (true == is_pending_state + || curr_proposal_id != new_proposal_id || LEADER != curr_role || LEADER != new_role) { + // when log handler is not LEDAER, we also need execute change_leader_to, otherwise, the leader can not be changed by election. + ls->get_log_handler()->change_leader_to(dst_addr); + CLOG_LOG(INFO, "no need execute switch_leader_to_follower_gracefully, change leader directlly", + K(ret), K(is_pending_state), K(curr_proposal_id), K(new_proposal_id), K(curr_role), K(new_role)); + } else if (OB_FAIL(switch_leader_to_follower_gracefully_(new_proposal_id, curr_proposal_id, + dst_addr, ls))) { + CLOG_LOG(WARN, "switch_leader_to_follower_gracefully_ failed", K(ret), KPC(ls), + K(curr_role), K(curr_proposal_id), K(new_role), K(new_proposal_id)); + } else { + CLOG_LOG(INFO, "handle_change_leader_event_for_log_handler_ success", K(ret), K(curr_role), + K(curr_proposal_id), K(new_role), K(new_proposal_id), K(dst_addr)); + } + return ret; +} + +int RoleCoordinator::switch_follower_to_leader_( + const int64_t new_proposal_id, + ObLogClient*ls, + RetrySubmitRoleChangeEventCtx &retry_ctx) +{ + int ret = OB_SUCCESS; + const ObRole new_role = LEADER; + const share::ObLSID &ls_id = ls->get_ls_id(); + palf::LSN end_lsn; + ObTimeGuard time_guard("switch_to_leader", EACH_ROLE_CHANGE_COST_MAX_TIME); + ObLogHandler *log_handler = ls->get_log_handler(); + // ObRoleChangeHandler *role_change_handler = ls->get_role_change_handler(); + if (OB_FAIL(log_handler->get_end_lsn(end_lsn))) { + CLOG_LOG(WARN, "get_end_lsn failed", K(ret), KPC(ls)); + // NB: order is vital!!! + // We must guarantee that 'replay_service_' has replayed complete data, and before + // stop 'replay_service_', other components can not submit log. + } else if (FALSE_IT(time_guard.click("wait_replay_service_apply_done_")) + || OB_FAIL(wait_replay_service_replay_done_(ls_id, end_lsn, WAIT_REPLAY_DONE_TIMEOUT_US))) { + if (need_retry_submit_role_change_event_(ret)) { + retry_ctx.set_retry_reason(RetrySubmitRoleChangeEventReason::WAIT_REPLAY_DONE_TIMEOUT); + } else { + CLOG_LOG(WARN, "wait_replay_service_replay_done_ failed", K(ret), K(end_lsn)); + } + } else if (FALSE_IT(time_guard.click("apply_service->switch_to_leader")) + || OB_FAIL(apply_service_->switch_to_leader(ls_id, new_proposal_id))) { + CLOG_LOG(WARN, "apply_service_ switch_to_leader failed", K(ret), K(new_role), K(new_proposal_id)); + // } else if (FALSE_IT(time_guard.click("replay_service->switch_to_leader")) + // || OB_FAIL(replay_service_->switch_to_leader(ls_id))) { + } else if (FALSE_IT(log_handler->switch_role(new_role, new_proposal_id))) { + CLOG_LOG(WARN, "ObLogHandler switch role failed", K(ret), K(new_role), K(new_proposal_id)); + // } else if (FALSE_IT(time_guard.click("role_change_handler->switch_to_leader")) + // || OB_FAIL(role_change_handler->switch_to_leader())) { + // CLOG_LOG(WARN, "ObRoleChangeHandler switch_to_leader failed", K(ret), KPC(ls)); + } else { + CLOG_LOG(INFO, "switch_follower_to_leader_ success", K(ret), KPC(ls)); + } + if (OB_FAIL(ret) && !retry_ctx.need_retry()) { + log_handler->revoke_leader(); + CLOG_LOG(WARN, "switch_follower_to_leader_ failed", K(ret), KPC(ls)); + } + return ret; +} + +int RoleCoordinator::switch_leader_to_follower_forcedly_( + const int64_t new_proposal_id, + ObLogClient*ls) +{ + int ret = OB_SUCCESS; + const ObRole new_role = FOLLOWER; + const share::ObLSID &ls_id = ls->get_ls_id(); + ObLogHandler *log_handler = ls->get_log_handler(); + // ObRoleChangeHandler *role_change_handler = ls->get_role_change_handler(); + palf::LSN end_lsn; + ObTimeGuard time_guard("switch_leader_to_follower_forcedly_", EACH_ROLE_CHANGE_COST_MAX_TIME); + + // Why need wait_apply_sync? + // + // when we can execute 'switch_to_follower_forcedly', means that there is no possibility to submit log via log handler successfully. + // however, the flying callback may have not been pushed into apply service, and then, 'switch_to_follower' will be executed, for trans, + // if the callback be executed after 'switch_to_follower', will cause abort. + if (OB_FAIL(apply_service_->wait_append_sync(ls_id))) { + CLOG_LOG(WARN, "wait_apply_sync failed", K(ret), K(ls_id)); + } else if (FALSE_IT(time_guard.click("apply_service->wait_apply_sync")) + || OB_FAIL(apply_service_->switch_to_follower(ls_id))) { + CLOG_LOG(WARN, "apply_service_ switch_to_follower failed", K(ret), K(new_role), K(new_proposal_id)); + } else if (FALSE_IT(time_guard.click("apply_service->switch_to_follower")) + || OB_FAIL(wait_apply_service_apply_done_(ls_id, end_lsn))) { + CLOG_LOG(WARN, "wait_apply_service_apply_done_ failed", K(ret), K(end_lsn)); + } else { + time_guard.click("wait_apply_service_apply_done_"); + // role_change_handler->switch_to_follower_forcedly(); + // NB: order is vital + // We must guarantee that this replica will not submit any logs after switch_role. + log_handler->switch_role(new_role, new_proposal_id); + // NB: in case of leader reovke, do we no need retry. + // (void)replay_service_->switch_to_follower(ls_id, end_lsn); + CLOG_LOG(INFO, "switch_leader_to_follower_forcedly_ success", K(ret), KPC(ls)); + } + if (OB_FAIL(ret)) { + log_handler->revoke_leader(); + CLOG_LOG(WARN, "switch_leader_to_follower_forcedly_ failed", K(ret), K(new_proposal_id), K(new_role)); + } + return ret; +} + +int RoleCoordinator::switch_leader_to_follower_gracefully_( + const int64_t new_proposal_id, + const int64_t curr_proposal_id, + const common::ObAddr &dst_addr, + ObLogClient*ls) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + const ObRole new_role = FOLLOWER; + const share::ObLSID &ls_id = ls->get_ls_id(); + ObLogHandler *log_handler = ls->get_log_handler(); + // ObRoleChangeHandler *role_change_handler = ls->get_role_change_handler(); + LSN end_lsn; + ObTimeGuard time_guard("switch_leader_to_follower_gracefully_", EACH_ROLE_CHANGE_COST_MAX_TIME); + // 1. OB_SUCCESS means execute transaction successfully, we need execute follow steps. + // 2. OB_LS_NEED_REVOKE means the transaction execute failed, and can't been rollback, need revoke LS. + // 3. OTHERS, switch_to_follower_gracefully failed, and 'role_change_handler' has rollback success, + // no need to execute follow steps. + // if (FALSE_IT(time_guard.click("role_change_handler->switch_to_follower_gracefully")) + // || OB_SUCCESS != (tmp_ret = role_change_handler->switch_to_follower_gracefully())) { + // CLOG_LOG(WARN, "switch_to_follower_gracefully failed, need revoke leader", K(tmp_ret), + // K(new_role), K(new_proposal_id), K(dst_addr)); + // NB: order is vital!!! + // we must ensure that the 'end_lsn' provid by 'apply_service_' is correctly. + // just switch_role to follower firstly, avoid sync log failed because palf has changed leader. + if (FALSE_IT(log_handler->switch_role(new_role, curr_proposal_id))) { + // apply service will not update end_lsn after switch_to_follower, so wait apply done first here + } else if (FALSE_IT(time_guard.click("wait_apply_service_apply_done_when_change_leader_")) + || OB_FAIL(wait_apply_service_apply_done_when_change_leader_(log_handler, curr_proposal_id, ls_id, end_lsn))) { + CLOG_LOG(WARN, "wait_apply_service_apply_done_when_change_leader_ failed", K(ret), + K(new_role), K(new_proposal_id), K(dst_addr)); + // wait apply service done my fail, we need : + // 1. switch log handler to origin status. + // 2. resume role change handler + log_handler->switch_role(LEADER, curr_proposal_id); + // if (OB_FAIL(role_change_handler->resume_to_leader())) { + // CLOG_LOG(WARN, "resume to leader failed", K(ret), KPC(ls)); + // } + // NB: the following steps mustn't be failed. + } else if (FALSE_IT(time_guard.click("apply_service->switch_to_follower")) + || OB_FAIL(apply_service_->switch_to_follower(ls_id))) { + CLOG_LOG(WARN, "apply_service_ switch_to_follower failed", K(ret), K(new_role), K(new_proposal_id), K(dst_addr)); + // } else if (FALSE_IT(time_guard.click("replay_service->switch_to_follower")) + // || OB_FAIL(replay_service_->switch_to_follower(ls_id, end_lsn))) { + // CLOG_LOG(WARN, "replay_service_ switch_to_follower failed", K(ret), KPC(ls), K(new_role), K(new_proposal_id)); + // NB: execute 'change_leader_to' lastly, can make 'wait_apply_service_apply_done_when_change_leader_' finish quickly. + } else if (OB_FAIL(log_handler->change_leader_to(dst_addr))) { + CLOG_LOG(WARN, "ObLogHandler change_leader failed", K(ret), K(new_role), K(new_proposal_id), K(dst_addr)); + } else { + CLOG_LOG(INFO, "switch_to_follower_gracefully success", K(ret), K(new_role), K(new_proposal_id), K(dst_addr)); + } + if (OB_FAIL(ret) || OB_LS_NEED_REVOKE == tmp_ret) { + log_handler->revoke_leader(); + CLOG_LOG(WARN, "switch_leader_to_follower_gracefully failed, revoke leader", K(ret), K(tmp_ret), K(dst_addr), + K(new_role), K(new_proposal_id)); + ret = (OB_SUCCESS == ret ? tmp_ret : ret); + } + return ret; +} + +int RoleCoordinator::switch_leader_to_leader_( + const int64_t new_proposal_id, + const int64_t curr_proposal_id, + ObLogClient*ls, + RetrySubmitRoleChangeEventCtx &retry_ctx) +{ + int ret = OB_SUCCESS; + ObTimeGuard time_guard("switch_leader_to_leader", EACH_ROLE_CHANGE_COST_MAX_TIME); + if (FALSE_IT(time_guard.click("switch_leader_to_follower_forcedly_")) + || OB_FAIL(switch_leader_to_follower_forcedly_(curr_proposal_id, ls))) { + CLOG_LOG(WARN, "switch_leader_to_leader_, switch leader to follower failed", K(ret), KPC(ls)); + } else if (FALSE_IT(time_guard.click("switch_follower_to_leader_")) + || OB_FAIL(switch_follower_to_leader_(new_proposal_id, ls, retry_ctx))) { + CLOG_LOG(WARN, "switch_follower_to_leader_ failed", K(ret), K(new_proposal_id)); + } else { + CLOG_LOG(INFO, "switch_leader_to_leader_ success", K(ret), KPC(ls)); + } + return ret; +} + +int RoleCoordinator::switch_follower_to_follower_(const int64_t new_proposal_id, ObLogClient*ls) +{ + int ret = OB_SUCCESS; + // need update proposal_id + const share::ObLSID &ls_id = ls->get_ls_id(); + ObLogHandler *log_handler = ls->get_log_handler(); + (void) log_handler->switch_role(common::ObRole::FOLLOWER, new_proposal_id); + CLOG_LOG(INFO, "switch_follower_to_follower_"); + return ret; +} + +int RoleCoordinator::wait_replay_service_replay_done_( + const share::ObLSID &ls_id, + const palf::LSN &end_lsn, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + return ret; +} + +int RoleCoordinator::wait_apply_service_apply_done_( + const share::ObLSID &ls_id, + palf::LSN &end_lsn) +{ + int ret = OB_SUCCESS; + bool is_done = false; + const int64_t start_ts = ObTimeUtility::current_time(); + while (OB_SUCC(ret) && false == is_done) { + if (OB_FAIL(apply_service_->is_apply_done(ls_id, is_done, end_lsn))) { + CLOG_LOG(WARN, "apply_service_ is_apply_done failed", K(ret), K(is_done), K(end_lsn)); + } else if (false == is_done) { + ob_usleep(5*1000); + CLOG_LOG(WARN, "wait apply done return false, need retry", K(ls_id), K(is_done), K(end_lsn), K(start_ts)); + } else { + } + } + return ret; +} + +int RoleCoordinator::wait_apply_service_apply_done_when_change_leader_( + const ObLogHandler *log_handler, + const int64_t proposal_id, + const share::ObLSID &ls_id, + palf::LSN &end_lsn) +{ + int ret = OB_SUCCESS; + bool is_done = false; + palf::LSN max_lsn; + common::ObRole unused_curr_role; + int64_t unused_curr_proposal_id; + common::ObRole new_role; + int64_t new_proposal_id; + bool is_pending_state = false; + while (OB_SUCC(ret) && (false == is_done || end_lsn != max_lsn)) { + if (OB_FAIL(apply_service_->is_apply_done(ls_id, is_done, end_lsn))) { + CLOG_LOG(WARN, "apply_service_ is_apply_done failed", K(ret), K(is_done), K(end_lsn)); + // NB: ApplyService execute on_failure only when it's FOLLOWER, therefore ApplyService my not return apply done + // when it's LEADER, we need check the role of palf when has changed. + } else if (OB_FAIL(log_handler->get_max_lsn(max_lsn))) { + CLOG_LOG(WARN, "get_end_lsn from palf failed", K(ret), K(ls_id), K(end_lsn)); + } else if (OB_FAIL(log_handler->prepare_switch_role(unused_curr_role, unused_curr_proposal_id, + new_role, new_proposal_id, is_pending_state))) { + CLOG_LOG(WARN, "failed prepare_switch_role", K(ret), K(new_role), K(proposal_id), K(ls_id)); + // if palf has changed role, return OB_STATE_NOT_MATCH, change leader failed. + } else if (LEADER != new_role || proposal_id != new_proposal_id) { + ret = OB_STATE_NOT_MATCH; + CLOG_LOG(WARN, "palf has changed leader, wait_apply_service_apply_done_when_change_leader_ failed", K(ret), K(proposal_id), + K(new_proposal_id)); + } else if (false == is_done || end_lsn != max_lsn) { + CLOG_LOG(INFO, "wait apply done return false, need retry", K(ls_id), K(is_done), + K(end_lsn), K(max_lsn)); + ob_usleep(5*1000); + } else { + } + } + return ret; +} + +RoleCoordinator::RoleChangeOptType RoleCoordinator::get_role_change_opt_type_( + const ObRole &old_role, + const ObRole &new_role, + const bool need_transform_by_access_mode) const +{ + RoleChangeOptType rc_opt_type = RoleChangeOptType::INVALID_RC_OPT_TYPE; + if (FOLLOWER == old_role && LEADER == new_role) { + rc_opt_type = RoleChangeOptType::FOLLOWER_2_LEADER; + } else if (LEADER == old_role && FOLLOWER == new_role) { + rc_opt_type = RoleChangeOptType::LEADER_2_FOLLOWER; + } else if (FOLLOWER == old_role && FOLLOWER == new_role) { + rc_opt_type = RoleChangeOptType::FOLLOWER_2_FOLLOWER; + } else if (LEADER == old_role && LEADER == new_role) { + rc_opt_type = RoleChangeOptType::LEADER_2_LEADER; + } else { + } + if (true == need_transform_by_access_mode) { + if (LEADER == old_role) { + rc_opt_type = RoleChangeOptType::LEADER_2_FOLLOWER; + } else { + rc_opt_type = RoleChangeOptType::FOLLOWER_2_FOLLOWER; + } + } + return rc_opt_type; +} + +// NB: +// 1. when log handler is offline, need execute role change; +// 2. when palf is not in pending: +// 1. proposal_id is not same or +// 2. role is not same.(If there are no pending logs in sliding window, +// leader to follower will not advance proposal_id) +bool RoleCoordinator::need_execute_role_change(const int64_t curr_proposal_id, + const common::ObRole curr_role, + const int64_t new_proposal_id, + const common::ObRole new_role, + const bool is_pending_state, + const bool is_offline) const +{ + return is_offline + || (!is_pending_state + && (curr_proposal_id != new_proposal_id || curr_role != new_role)); +} + +bool RoleCoordinator::is_append_mode(const AccessMode &mode) const +{ + return (AccessMode::APPEND == mode); +} + +bool RoleCoordinator::is_raw_write_or_flashback_mode(const AccessMode &mode) const +{ + return (AccessMode::RAW_WRITE == mode || AccessMode::FLASHBACK == mode || + AccessMode::PREPARE_FLASHBACK == mode); +} + +bool RoleCoordinator::need_retry_submit_role_change_event_(int ret) const +{ + bool bool_ret = false; + if (OB_TIMEOUT == ret) { + bool_ret = true; + } + return bool_ret; +} + +} // end namespace palfcluster +} // end namespace oceanbase diff --git a/mittest/palf_cluster/logservice/role_coordinator.h b/mittest/palf_cluster/logservice/role_coordinator.h new file mode 100644 index 0000000000..e0598af87d --- /dev/null +++ b/mittest/palf_cluster/logservice/role_coordinator.h @@ -0,0 +1,189 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_PALF_CLUSTER_ROLE_CHANGE_SERVICE_ +#define OCEANBASE_PALF_CLUSTER_ROLE_CHANGE_SERVICE_ +#include "lib/function/ob_function.h" +#include "lib/thread/thread_mgr_interface.h" +#include "lib/utility/ob_macro_utils.h" +#include "share/ob_ls_id.h" +#include "storage/tx_storage/ob_ls_service.h" +#include "logservice/palf/palf_options.h" +#include "logservice/ob_log_handler.h" +#include "logservice/palf/palf_callback.h" +#include "logservice/applyservice/ob_log_apply_service.h" +#include "logservice/rcservice/ob_role_change_handler.h" +#include "mittest/palf_cluster/logservice/ob_log_client.h" +namespace oceanbase +{ +namespace palfcluster +{ +enum class RoleChangeEventType { + INVALID_RC_EVENT_TYPE = 0, + CHANGE_LEADER_EVENT_TYPE = 1, + ROLE_CHANGE_CB_EVENT_TYPE = 2, + MAX_RC_EVENT_TYPE = 3 +}; + +struct RoleChangeEvent { + RoleChangeEvent() { reset(); } + RoleChangeEvent(const RoleChangeEventType &event_type, + const share::ObLSID &ls_id); + RoleChangeEvent(const RoleChangeEventType &event_type, + const share::ObLSID &ls_id, + const common::ObAddr &dst_addr); + bool is_valid() const; + void reset(); + bool operator == (const RoleChangeEvent &event) const; + RoleChangeEventType event_type_; + share::ObLSID ls_id_; + ObAddr dst_addr_; + TO_STRING_KV(K_(event_type), K_(ls_id), K_(dst_addr)); +}; + +class RoleChangeEventSet { +public: + RoleChangeEventSet(); + ~RoleChangeEventSet(); + int insert(const RoleChangeEvent &event); + int remove(const RoleChangeEvent &event); + static constexpr int64_t MAX_ARRAY_SIZE = 128; +private: + // Assumed there are sixteen log streams at most. + // 64 for normal role change events. + // 64 for leader change events. + RoleChangeEvent events_[MAX_ARRAY_SIZE]; + mutable ObSpinLock lock_; + DISALLOW_COPY_AND_ASSIGN(RoleChangeEventSet); +}; + +class RoleCoordinator : public lib::TGTaskHandler , public palf::PalfRoleChangeCb { +public: + RoleCoordinator(); + ~RoleCoordinator(); + int init(palfcluster::LogClientMap *log_client_map, + logservice::ObLogApplyService *apply_service); + int start(); + void wait(); + void stop(); + void destroy(); + void handle(void *task); + int on_role_change(const int64_t id) final override; + int on_need_change_leader(const int64_t ls_id, const common::ObAddr &dst_addr) final override; + +private: + enum class RoleChangeOptType { + INVALID_RC_OPT_TYPE = 0, + FOLLOWER_2_LEADER = 1, + LEADER_2_FOLLOWER = 2, + FOLLOWER_2_FOLLOWER = 3, + LEADER_2_LEADER = 4, + MAX_RC_OPT_TYPE = 5 + }; + enum class RetrySubmitRoleChangeEventReason { + INVALID_TYPE = 0, + WAIT_REPLAY_DONE_TIMEOUT = 1, + MAX_TYPE = 2 + }; + class RetrySubmitRoleChangeEventCtx { + public: + RetrySubmitRoleChangeEventCtx() : reason_(RetrySubmitRoleChangeEventReason::INVALID_TYPE) {} + ~RetrySubmitRoleChangeEventCtx() + { + reason_ = RetrySubmitRoleChangeEventReason::INVALID_TYPE; + } + bool need_retry() const + { + return RetrySubmitRoleChangeEventReason::WAIT_REPLAY_DONE_TIMEOUT == reason_; + } + void set_retry_reason(const RetrySubmitRoleChangeEventReason &reason) + { + reason_ = reason; + } + TO_STRING_KV(K_(reason)); + private: + RetrySubmitRoleChangeEventReason reason_; + }; +private: + int submit_role_change_event_(const RoleChangeEvent &event); + int push_event_into_queue_(const RoleChangeEvent &event); + int handle_role_change_event_(const RoleChangeEvent &event, + RetrySubmitRoleChangeEventCtx &retry_ctx); + + int handle_role_change_cb_event_for_log_handler_(const palf::AccessMode &curr_access_mode, + ObLogClient *ls, + RetrySubmitRoleChangeEventCtx &retry_ctx); + int handle_change_leader_event_for_log_handler_(const common::ObAddr &dst_addr, + ObLogClient *ls); + + // retval + // - OB_SUCCESS + // - OB_TIMEOUT, means wait replay finish timeout. + int switch_follower_to_leader_(const int64_t new_proposal_id, + ObLogClient *ls, + RetrySubmitRoleChangeEventCtx &retry_ctx); + int switch_leader_to_follower_forcedly_(const int64_t new_proposal_id, + ObLogClient *ls); + int switch_leader_to_follower_gracefully_(const int64_t new_proposal_id, + const int64_t curr_proposal_id, + const common::ObAddr &dst_addr, + ObLogClient *ls); + int switch_follower_to_follower_(const int64_t new_proposal_id, ObLogClient *ls); + int switch_leader_to_leader_(const int64_t new_proposal_id, + const int64_t curr_proposal_id, + ObLogClient *ls, + RetrySubmitRoleChangeEventCtx &retry_ctx); + + // wait replay finish with timeout. + int wait_replay_service_replay_done_(const share::ObLSID &ls_id, + const palf::LSN &end_lsn, + const int64_t timeout_us); + int wait_apply_service_apply_done_(const share::ObLSID &ls_id, + palf::LSN &end_lsn); + int wait_apply_service_apply_done_when_change_leader_(const logservice::ObLogHandler *log_handler, + const int64_t proposal_id, + const share::ObLSID &ls_id, + palf::LSN &end_lsn); + bool need_execute_role_change(const int64_t curr_proposal_id, + const common::ObRole curr_role, + const int64_t new_proposal_id, + const common::ObRole new_role, + const bool is_pending_state, + const bool is_offline) const; + + bool is_append_mode(const palf::AccessMode &access_mode) const; + bool is_raw_write_or_flashback_mode(const palf::AccessMode &access_mode) const; +private: + RoleChangeOptType get_role_change_opt_type_(const common::ObRole &old_role, + const common::ObRole &new_role, + const bool need_transform_by_access_mode) const; + // retry submit role change event + // NB: nowdays, we only support retry submit role change event when wait replay finished + // timeout. + bool need_retry_submit_role_change_event_(int ret) const; +public: + static const int64_t MAX_THREAD_NUM = 1; + static const int64_t MAX_RC_EVENT_TASK = 1024 * 1024; +private: + DISALLOW_COPY_AND_ASSIGN(RoleCoordinator); +private: + static constexpr int64_t EACH_ROLE_CHANGE_COST_MAX_TIME = 1 * 1000 * 1000; + static constexpr int64_t WAIT_REPLAY_DONE_TIMEOUT_US = 2 * 1000 * 1000; + palfcluster::LogClientMap *log_client_map_; + logservice::ObLogApplyService *apply_service_; + RoleChangeEventSet rc_set_; + int tg_id_; + bool is_inited_; +}; +} // end namespace palfcluster +} // end namespce oceanbase +#endif diff --git a/mittest/palf_cluster/logservice/role_coordinator_handler.cpp b/mittest/palf_cluster/logservice/role_coordinator_handler.cpp new file mode 100644 index 0000000000..18572fd6f5 --- /dev/null +++ b/mittest/palf_cluster/logservice/role_coordinator_handler.cpp @@ -0,0 +1,197 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "role_coordinator_handler.h" +#include "lib/container/ob_fixed_array.h" +#include "lib/lock/ob_spin_lock.h" +#include "lib/ob_define.h" +#include "share/ob_errno.h" +namespace oceanbase +{ +using namespace common; +using namespace logservice; +namespace palfcluster +{ + +ObRoleChangeHandler::ObRoleChangeHandler(): lock_(common::ObLatchIds::RCS_LOCK), + sub_role_change_handler_arr_() +{ + reset(); +} + +ObRoleChangeHandler::~ObRoleChangeHandler() +{ + reset(); +} + +void ObRoleChangeHandler::reset() +{ + for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE; i++) { + sub_role_change_handler_arr_[i] = NULL; + } +} + +int ObRoleChangeHandler::register_handler(const ObLogBaseType &type, + ObIRoleChangeSubHandler *role_change_handler) +{ + int ret = OB_SUCCESS; + ObSpinLockGuard guard(lock_); + if (false == is_valid_log_base_type(type)) { + ret = OB_INVALID_ARGUMENT; + } else { + sub_role_change_handler_arr_[type] = role_change_handler; + CLOG_LOG(INFO, "register_handler success", K(ret), K(type), KP(role_change_handler)); + } + return ret; +} + +void ObRoleChangeHandler::unregister_handler(const ObLogBaseType &type) +{ + ObSpinLockGuard guard(lock_); + if (true == is_valid_log_base_type(type)) { + sub_role_change_handler_arr_[type] = NULL; + CLOG_LOG(INFO, "unregister_handler success", K(type)); + } +} + +void ObRoleChangeHandler::switch_to_follower_forcedly() +{ + ObSpinLockGuard guard(lock_); + for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE; i++) { + ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[i]; + char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'}; + ObLogBaseType base_type = static_cast(i); + bool has_defined_to_string = false; + if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str, + OB_LOG_BASE_TYPE_STR_MAX_LEN)) { + has_defined_to_string = true; + } + if (NULL != handler) { + handler->switch_to_follower_forcedly(); + CLOG_LOG(INFO, "leader to follower forcedly, current role change handler is", + "cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); + } + } +} + +int ObRoleChangeHandler::switch_to_leader() +{ + int ret = OB_SUCCESS; + ObSpinLockGuard guard(lock_); + for (int i = 0; i < ObLogBaseType::MAX_LOG_BASE_TYPE && OB_SUCC(ret); i++) { + ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[i]; + char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'}; + ObLogBaseType base_type = static_cast(i); + bool has_defined_to_string = false; + if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str, + OB_LOG_BASE_TYPE_STR_MAX_LEN)) { + has_defined_to_string = true; + } + if (NULL == handler) { + } else if (OB_FAIL(handler->switch_to_leader())) { + CLOG_LOG(WARN, "switch_to_leader failed", K(ret), KP(handler), K(i), + "cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); + } else { + CLOG_LOG(INFO, "follower to leader, current role change handler is", + "cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); + } + } + return ret; +} + +int ObRoleChangeHandler::switch_to_follower_gracefully() +{ + int ret = OB_SUCCESS; + ObSpinLockGuard guard(lock_); + int64_t cursor = 0; + const int64_t count = ObLogBaseType::MAX_LOG_BASE_TYPE; + while (cursor < count && OB_SUCC(ret)) { + char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'}; + ObLogBaseType base_type = static_cast(cursor); + bool has_defined_to_string = false; + if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str, + OB_LOG_BASE_TYPE_STR_MAX_LEN)) { + has_defined_to_string = true; + } + ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[cursor]; + if (NULL == handler) { + cursor++; + } else if (OB_FAIL(handler->switch_to_follower_gracefully()) && OB_LS_NEED_REVOKE != ret) { + CLOG_LOG(WARN, "switch_to_follower_gracefully failed, need resume other sub modules", K(ret), + KP(handler), K(cursor), + "cursor", cursor, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); + // NB: resume_leader failed, need revoke leader. + } else if (OB_LS_NEED_REVOKE == ret) { + CLOG_LOG(WARN, "ObIRoleChangeSubHandler resume leader failed", K(ret), K(cursor)); + } else { + CLOG_LOG(INFO, "leader to follower gracefully, current role change handler is", + "cursor", cursor, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); + cursor++; + } + } + // if any sub role handler switch_to_follower_gracefully failed, and no need to revoke leader, + // we should resume other sub role handler, meanwhile, we should overrite 'ret' only if + // resume_leader_when_switch_failure_ failed. + if (OB_FAIL(ret) && OB_LS_NEED_REVOKE != ret) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = resume_leader_when_switch_failure_(cursor))) { + CLOG_LOG(WARN, "resume_leader_when_switch_failure_ failed", K(tmp_ret), K(cursor)); + ret = tmp_ret; + } else { + CLOG_LOG(WARN, "resume_leader_when_switch_failure_ success, no need to excut follower to leader gracefully", + K(ret), K(tmp_ret)); + } + } + return ret; +} + +int ObRoleChangeHandler::resume_to_leader() +{ + int ret = OB_SUCCESS; + int cursor = ObLogBaseType::MAX_LOG_BASE_TYPE; + if (OB_FAIL(resume_leader_when_switch_failure_(cursor))) { + CLOG_LOG(WARN, "resume_leader_when_switch_failure_ failed"); + } else { + CLOG_LOG(INFO, "resume_to_leader success"); + } + return ret; +} + +int ObRoleChangeHandler::resume_leader_when_switch_failure_(const int64_t cursor) +{ + int ret = OB_SUCCESS; + for (int64_t i = cursor - 1; i >= 0 && OB_SUCC(ret); i--) { + ObIRoleChangeSubHandler *handler = sub_role_change_handler_arr_[i]; + ObLogBaseType base_type = static_cast(i); + char sub_role_change_handler_str[OB_LOG_BASE_TYPE_STR_MAX_LEN] = {'\0'}; + bool has_defined_to_string = false; + if (OB_SUCCESS == log_base_type_to_string(base_type, sub_role_change_handler_str, + OB_LOG_BASE_TYPE_STR_MAX_LEN)) { + has_defined_to_string = true; + } + if (NULL == handler){ + CLOG_LOG(INFO, "not register into role change service", K(ret), K(i)); + } else if (OB_FAIL(handler->resume_leader())) { + CLOG_LOG(WARN, "resume_leader failed", K(ret), K(i), KP(handler), + "cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); + } else { + CLOG_LOG(INFO, "resume_leader success", K(ret), K(i), KP(handler), + "cursor", i, "name", has_defined_to_string ? sub_role_change_handler_str : "hasn't define to string"); + } + } + if (OB_FAIL(ret)) { + ret = OB_LS_NEED_REVOKE; + } + return ret; +} +} // end namespace logservice +} // end namespace oceanbase diff --git a/mittest/palf_cluster/logservice/role_coordinator_handler.h b/mittest/palf_cluster/logservice/role_coordinator_handler.h new file mode 100644 index 0000000000..6095f0ece4 --- /dev/null +++ b/mittest/palf_cluster/logservice/role_coordinator_handler.h @@ -0,0 +1,82 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_PALF_CLUSTER_OB_ROLE_CHANGE_HANDLER_ +#define OCEANBASE_PALF_CLUSTER_OB_ROLE_CHANGE_HANDLER_ +#include "lib/container/ob_fixed_array.h" +#include "lib/container/ob_se_array.h" +#include "lib/lock/ob_spin_lock.h" +#include "lib/utility/ob_print_utils.h" +#include "logservice/ob_log_base_header.h" +#include "logservice/ob_log_base_type.h" +#include "share/ob_errno.h" +namespace oceanbase +{ +namespace palfcluster +{ +enum TakeOverState { + INVALID_TAKE_OVER_STATE = 0, + WAIT_REPLAY_DONE = 1, + WAIT_RC_HANDLER_DONE = 2, + TAKE_OVER_FINISH = 3, + UNKNOWN_TAKE_OVER_STATE = 4, + MAX_TAKE_OVER_STATE = 5 +}; + +static inline +int takeover_state_to_string(const TakeOverState log_type, + char *str, + const int64_t str_len) +{ + int ret = OB_SUCCESS; + if (log_type == INVALID_TAKE_OVER_STATE) { + strncpy(str ,"INVALID_STATE", str_len); + } else if (log_type == WAIT_REPLAY_DONE) { + strncpy(str ,"WAIT_REPLAY_DONE", str_len); + } else if (log_type == WAIT_RC_HANDLER_DONE) { + strncpy(str ,"WAIT_RC_HANDLER_DONE", str_len); + } else if (log_type == TAKE_OVER_FINISH) { + strncpy(str ,"FINISH", str_len); + } else if (log_type == UNKNOWN_TAKE_OVER_STATE) { + strncpy(str ,"UNKNOWN", str_len); + } else { + ret = OB_INVALID_ARGUMENT; + } + return ret; +} + +class ObRoleChangeHandler { +public: + ObRoleChangeHandler(); + ~ObRoleChangeHandler(); + void reset(); + int register_handler(const logservice::ObLogBaseType &type, logservice::ObIRoleChangeSubHandler *role_change_hander); + void unregister_handler(const logservice::ObLogBaseType &type); + + void switch_to_follower_forcedly(); + int switch_to_leader(); + // @retval: + // 1. OB_SUCCESS + // 2. OB_LS_NEED_REVOKE, ObRoleChangeService need revoke this LS. + // 3. OTHERS, switch_to_follower_gracefully failed, but no need to revoke leader. + int switch_to_follower_gracefully(); + int resume_to_leader(); + +private: + int resume_leader_when_switch_failure_(int64_t cursor); +private: + ObSpinLock lock_; + logservice::ObIRoleChangeSubHandler* sub_role_change_handler_arr_[logservice::ObLogBaseType::MAX_LOG_BASE_TYPE]; +}; +} +} +#endif diff --git a/mittest/palf_cluster/palf-cluster.diff b/mittest/palf_cluster/palf-cluster.diff new file mode 100644 index 0000000000..8187147ea9 --- /dev/null +++ b/mittest/palf_cluster/palf-cluster.diff @@ -0,0 +1,364 @@ +diff --git a/build.sh b/build.sh +--- a/build.sh ++++ b/build.sh +@@ -188,7 +188,7 @@ function build + set -- "${BUILD_ARGS[@]}" + case "x$1" in + xrelease) +- do_build "$@" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DOB_USE_LLD=$LLD_OPTION ++ do_build "$@" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_AUTO_FDO=ON -DOB_USE_LLD=$LLD_OPTION + ;; + xrelease_no_unity) + do_build "$@" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DOB_USE_LLD=$LLD_OPTION -DOB_ENABLE_UNITY=OFF +diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +--- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h ++++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +@@ -968,6 +968,9 @@ PCODE_DEF(OB_LOG_FORCE_CLEAR_ARB_CLUSTER_INFO, 0x1521) + PCODE_DEF(OB_LOG_GET_ARB_MEMBER_INFO, 0x1522) + #endif + PCODE_DEF(OB_LOG_BATCH_FETCH_RESP, 0X1523) ++PCODE_DEF(OB_LOG_CREATE_REPLICA_CMD, 0x1524) ++PCODE_DEF(OB_LOG_SUBMIT_LOG_CMD, 0x1525) ++PCODE_DEF(OB_LOG_SUBMIT_LOG_CMD_RESP, 0x1526) + + // 1531-1550 for obesi + // PCODE_DEF(OB_ESI_IS_EXIST, 0x1531) +diff --git a/src/logservice/applyservice/ob_log_apply_service.cpp b/src/logservice/applyservice/ob_log_apply_service.cpp +--- a/src/logservice/applyservice/ob_log_apply_service.cpp ++++ b/src/logservice/applyservice/ob_log_apply_service.cpp +@@ -28,6 +28,9 @@ using namespace storage; + using namespace share; + namespace logservice + { ++int64_t p_append_cnt = 0; ++int64_t p_log_body_size = 0; ++int64_t p_rt = 0; + //---------------ObApplyFsCb---------------// + ObApplyFsCb::ObApplyFsCb() + : apply_status_(NULL) +@@ -482,6 +485,26 @@ int ObApplyStatus::try_handle_cb_queue(ObApplyServiceQueueTask *cb_queue, + scn = cb->__get_scn(); + get_cb_trace_(cb, append_start_time, append_finish_time, cb_first_handle_time, cb_start_time); + CLOG_LOG(TRACE, "cb on_success", K(lsn), K(scn), KP(link->next_), KPC(cb_queue), KPC(this)); ++ ++ ATOMIC_INC(&p_append_cnt); ++ ATOMIC_FAA(&p_log_body_size, cb->get_log_size()); ++ int64_t tmp_rt = ObTimeUtility::current_time() - cb->get_append_start_ts(); ++ ATOMIC_FAA(&p_rt, tmp_rt); ++ ++ if (REACH_TIME_INTERVAL(1000 *1000)) { ++ int64_t l_append_cnt = ATOMIC_LOAD(&p_append_cnt); ++ if (l_append_cnt == 0) l_append_cnt = 1; ++ int64_t l_log_body_size = ATOMIC_LOAD(&p_log_body_size); ++ int64_t l_rt = ATOMIC_LOAD(&p_rt); ++ ++ ATOMIC_STORE(&p_append_cnt, 0); ++ ATOMIC_STORE(&p_log_body_size, 0); ++ ATOMIC_STORE(&p_rt, 0); ++ ++ CLOG_LOG(ERROR, "result:", K(l_append_cnt), K(l_log_body_size), K(l_rt), ++ "avg_body_size", l_log_body_size/l_append_cnt, "avg_rt", l_rt/l_append_cnt); ++ } ++ + if (OB_FAIL(cb->on_success())) { + // 不处理此类失败情况 + CLOG_LOG(ERROR, "cb on_success failed", KP(cb), K(ret), KPC(this)); +@@ -1044,7 +1067,7 @@ int ObLogApplyService::init(PalfEnv *palf_env, + CLOG_LOG(WARN, "invalid argument", K(ret), KP(palf_env), K(ls_adapter)); + } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::ApplyService, tg_id_))) { + CLOG_LOG(WARN, "fail to create thread group", K(ret)); +- } else if (OB_FAIL(MTL_REGISTER_THREAD_DYNAMIC(0.5, tg_id_))) { ++ } else if (OB_FAIL(MTL_REGISTER_THREAD_DYNAMIC(1, tg_id_))) { + CLOG_LOG(WARN, "MTL_REGISTER_THREAD_DYNAMIC failed", K(ret), K(tg_id_)); + } else if (OB_FAIL(apply_status_map_.init("APPLY_STATUS", MAP_TENANT_ID))) { + CLOG_LOG(WARN, "apply_status_map_ init error", K(ret)); +diff --git a/src/logservice/ob_append_callback.h b/src/logservice/ob_append_callback.h +--- a/src/logservice/ob_append_callback.h ++++ b/src/logservice/ob_append_callback.h +@@ -60,6 +60,7 @@ public: + append_start_ts_ = OB_INVALID_TIMESTAMP; + append_finish_ts_ = OB_INVALID_TIMESTAMP; + cb_first_handle_ts_ = OB_INVALID_TIMESTAMP; ++ log_size_ = 0; + } + virtual int on_success() = 0; + virtual int on_failure() = 0; +@@ -69,10 +70,16 @@ public: + int64_t get_append_start_ts() const { return append_start_ts_; } + int64_t get_append_finish_ts() const { return append_finish_ts_; } + int64_t get_cb_first_handle_ts() const { return cb_first_handle_ts_; } ++ void set_log_size(const int64_t size) ++ { ++ log_size_ = size; ++ } ++ int64_t get_log_size() const { return log_size_; } + public: + int64_t append_start_ts_; //提交到palf的起始时刻 + int64_t append_finish_ts_; //palf提交完成时刻,即提交到apply service起始时刻 + int64_t cb_first_handle_ts_; //cb第一次被处理的时刻,不一定调用on_success ++ int64_t log_size_; + }; + + } // end namespace logservice +diff --git a/src/logservice/ob_log_handler.cpp b/src/logservice/ob_log_handler.cpp +--- a/src/logservice/ob_log_handler.cpp ++++ b/src/logservice/ob_log_handler.cpp +@@ -237,6 +237,7 @@ int ObLogHandler::append(const void *buffer, + cb->set_append_finish_ts(ObTimeUtility::fast_current_time()); + cb->__set_lsn(lsn); + cb->__set_scn(scn); ++ cb->set_log_size(nbytes); + ret = apply_status_->push_append_cb(cb); + CLOG_LOG(TRACE, "palf_handle_ push_append_cb success", K(lsn), K(scn), K(ret), K(id_)); + } +diff --git a/src/logservice/palf/log_block_handler.cpp b/src/logservice/palf/log_block_handler.cpp +--- a/src/logservice/palf/log_block_handler.cpp ++++ b/src/logservice/palf/log_block_handler.cpp +@@ -437,12 +437,26 @@ int LogBlockHandler::inner_writev_once_(const offset_t offset, + return ret; + } + ++int64_t p_io_cnt = 0; ++int64_t p_log_size = 0; ++ + int LogBlockHandler::inner_write_impl_(const int fd, const char *buf, const int64_t count, const int64_t offset) + { + int ret = OB_SUCCESS; + int64_t start_ts = ObTimeUtility::fast_current_time(); + int64_t write_size = 0; + int64_t time_interval = OB_INVALID_TIMESTAMP; ++ ATOMIC_INC(&p_io_cnt); ++ ATOMIC_FAA(&p_log_size, count); ++ if (REACH_TIME_INTERVAL(1000 * 1000)) { ++ int64_t l_io_cnt = ATOMIC_LOAD(&p_io_cnt); ++ int64_t l_log_size = ATOMIC_LOAD(&p_log_size); ++ ++ ATOMIC_STORE(&p_io_cnt, 0); ++ ATOMIC_STORE(&p_log_size, 0); ++ ++ CLOG_LOG(ERROR, "io result:", K(l_io_cnt), K(l_log_size), "avg_size", l_log_size/l_io_cnt); ++ } + do { + if (count != (write_size = ob_pwrite(fd, buf, count, offset))) { + if (palf_reach_time_interval(1000 * 1000, time_interval)) { +diff --git a/src/logservice/palf/log_define.h b/src/logservice/palf/log_define.h +--- a/src/logservice/palf/log_define.h ++++ b/src/logservice/palf/log_define.h +@@ -102,7 +102,7 @@ const int64_t PALF_LEADER_RECONFIRM_SYNC_TIMEOUT_US = 10 * 1000 * 1000L; // + const int64_t PREPARE_LOG_BUFFER_SIZE = 2048; + const int64_t PALF_LEADER_ACTIVE_SYNC_TIMEOUT_US = 10 * 1000 * 1000L; // 10s + const int32_t PALF_MAX_REPLAY_TIMEOUT = 500 * 1000; +-const int32_t DEFAULT_LOG_LOOP_INTERVAL_US = 100 * 1000; // 100ms ++const int32_t DEFAULT_LOG_LOOP_INTERVAL_US = 500; // 100ms + const int32_t LOG_LOOP_INTERVAL_FOR_PERIOD_FREEZE_US = 1 * 1000; // 1ms + const int64_t PALF_SLIDING_WINDOW_SIZE = 1 << 11; // must be 2^n(n>0), default 2^11 = 2048 + const int64_t PALF_MAX_LEADER_SUBMIT_LOG_COUNT = PALF_SLIDING_WINDOW_SIZE / 2; // max number of concurrent submitting group log in leader +diff --git a/src/logservice/palf/log_io_task.cpp b/src/logservice/palf/log_io_task.cpp +--- a/src/logservice/palf/log_io_task.cpp ++++ b/src/logservice/palf/log_io_task.cpp +@@ -623,7 +623,8 @@ int BatchLogIOFlushLogTask::push_flush_cb_to_thread_pool_(int tg_id, IPalfEnvImp + PALF_LOG(ERROR, "LogIOFlushMetaTask not inited!!!", K(ret), KPC(this)); + } else { + const int64_t current_time = ObTimeUtility::current_time(); +- for (int64_t i = 0; i < count && OB_SUCC(ret); i++) { ++ // for (int64_t i = 0; i < count && OB_SUCC(ret); i++) { ++ int64_t i = count - 1; + LogIOFlushLogTask *io_task = io_task_array_[i]; + if (NULL == io_task) { + PALF_LOG(WARN, "io_task is nullptr, may be its' epoch has changed", K(ret), KP(io_task), +@@ -637,6 +638,9 @@ int BatchLogIOFlushLogTask::push_flush_cb_to_thread_pool_(int tg_id, IPalfEnvImp + // again. + io_task_array_[i] = NULL; + } ++ // } ++ for (int64_t i = 0; i < count && OB_SUCC(ret); i++) { ++ io_task_array_[i] = NULL; + } + } + return ret; +diff --git a/src/logservice/palf/log_sliding_window.cpp b/src/logservice/palf/log_sliding_window.cpp +--- a/src/logservice/palf/log_sliding_window.cpp ++++ b/src/logservice/palf/log_sliding_window.cpp +@@ -132,7 +132,8 @@ LogSlidingWindow::LogSlidingWindow() + accum_log_cnt_(0), + accum_group_log_size_(0), + last_record_group_log_id_(FIRST_VALID_LOG_ID - 1), +- freeze_mode_(FEEDBACK_FREEZE_MODE), ++ freeze_mode_(PERIOD_FREEZE_MODE), ++ avg_log_batch_cnt_(0), + is_inited_(false) + {} + +@@ -448,6 +449,7 @@ int LogSlidingWindow::submit_log(const char *buf, + const int64_t start_log_id = get_start_id(); + const int64_t log_id_upper_bound = start_log_id + PALF_MAX_LEADER_SUBMIT_LOG_COUNT - 1; + LSN tmp_lsn, lsn_upper_bound; ++ bool need_freeze_self = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (NULL == buf || buf_len <= 0 || buf_len > MAX_LOG_BODY_SIZE || (!ref_scn.is_valid())) { +@@ -462,7 +464,7 @@ int LogSlidingWindow::submit_log(const char *buf, + } + // sw_ cannot submit larger log + } else if (OB_FAIL(lsn_allocator_.alloc_lsn_scn(ref_scn, valid_log_size, log_id_upper_bound, lsn_upper_bound, +- tmp_lsn, log_id, scn, is_new_log, need_gen_padding_entry, padding_size))) { ++ tmp_lsn, log_id, scn, is_new_log, need_gen_padding_entry, padding_size, need_freeze_self))) { + PALF_LOG(WARN, "alloc_lsn_scn failed", K(ret), K_(palf_id), K_(self)); + } else if (OB_FAIL(leader_wait_sw_slot_ready_(log_id))) { + PALF_LOG(WARN, "leader_wait_sw_slot_ready_ failed", K(ret), K_(palf_id), K_(self), K(log_id)); +@@ -480,7 +482,7 @@ int LogSlidingWindow::submit_log(const char *buf, + K(padding_size), K(is_new_log), K(valid_log_size)); + } else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) { + } else if (OB_FAIL(generate_new_group_log_(tmp_lsn, log_id, scn, padding_entry_body_size, LOG_PADDING, \ +- NULL, padding_entry_body_size, is_need_handle))) { ++ NULL, padding_entry_body_size, need_freeze_self, is_need_handle))) { + PALF_LOG(ERROR, "generate_new_group_log_ failed", K(ret), K_(palf_id), K_(self), K(log_id), K(tmp_lsn), K(padding_size), + K(is_new_log), K(valid_log_size)); + } else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) { +@@ -503,7 +505,7 @@ int LogSlidingWindow::submit_log(const char *buf, + PALF_LOG(WARN, "try_freeze_prev_log_ failed", K(ret), K_(palf_id), K_(self), K(log_id)); + } else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) { + } else if (OB_FAIL(generate_new_group_log_(tmp_lsn, log_id, scn, valid_log_size, LOG_SUBMIT, \ +- buf, buf_len, is_need_handle))) { ++ buf, buf_len, need_freeze_self, is_need_handle))) { + PALF_LOG(WARN, "generate_new_group_log_ failed", K(ret), K_(palf_id), K_(self), K(log_id)); + } else if (is_need_handle && FALSE_IT(is_need_handle_next |= is_need_handle)) { + } else { +@@ -673,6 +675,7 @@ int LogSlidingWindow::generate_new_group_log_(const LSN &lsn, + const LogType &log_type, + const char *log_data, + const int64_t data_len, ++ const bool need_freeze_self, + bool &is_need_handle) + { + int ret = OB_SUCCESS; +@@ -712,6 +715,9 @@ int LogSlidingWindow::generate_new_group_log_(const LSN &lsn, + } else if (OB_FAIL(log_task->set_initial_header_info(header_info))) { + PALF_LOG(WARN, "set_initial_header_info failed", K(ret), K_(palf_id), K_(self), K(log_id), KPC(log_task)); + } else { ++ if (need_freeze_self) { ++ log_task->set_end_lsn(lsn + log_body_size + LogGroupEntryHeader::HEADER_SER_SIZE); ++ } + // The first log is responsible to try freezing self, if its end_lsn_ has been set by next log. + log_task->try_freeze_by_myself(); + } +@@ -1048,6 +1054,10 @@ int LogSlidingWindow::handle_next_submit_log_(bool &is_committed_lsn_updated) + const int64_t avg_group_log_size = total_group_log_size / total_group_log_cnt; + PALF_LOG(INFO, "[PALF STAT GROUP LOG INFO]", K_(palf_id), K_(self), "role", role_to_string(role), + K(total_group_log_cnt), K(avg_log_batch_cnt), K(total_group_log_size), K(avg_group_log_size)); ++ if (total_log_cnt > 0) { ++ const int64_t avg_log_size = total_group_log_size / total_log_cnt; ++ avg_log_batch_cnt_ = (avg_log_batch_cnt_ * 2 + avg_log_batch_cnt * 8)/10; ++ } + } + ATOMIC_STORE(&accum_log_cnt_, 0); + ATOMIC_STORE(&accum_group_log_size_, 0); +@@ -1235,19 +1245,34 @@ int LogSlidingWindow::check_and_switch_freeze_mode() + { + int ret = OB_SUCCESS; + int64_t total_append_cnt = 0; +- for (int i = 0; i < APPEND_CNT_ARRAY_SIZE; ++i) { +- total_append_cnt += ATOMIC_LOAD(&append_cnt_array_[i]); +- ATOMIC_STORE(&append_cnt_array_[i], 0); +- } ++ // for (int i = 0; i < APPEND_CNT_ARRAY_SIZE; ++i) { ++ // total_append_cnt += ATOMIC_LOAD(&append_cnt_array_[i]); ++ // ATOMIC_STORE(&append_cnt_array_[i], 0); ++ // } ++ ++ // for (int i = 0; i < APPEND_CNT_ARRAY_SIZE; ++i) { ++ // total_append_cnt += ATOMIC_LOAD(&append_cnt_array_[i]); ++ // ATOMIC_STORE(&append_cnt_array_[i], 0); ++ // } ++ // adaptively ++ const int64_t avg_log_batch_cnt = avg_log_batch_cnt_; ++ // periodically ++ // const int64_t avg_log_batch_cnt = 20; ++ // feedback ++ // const int64_t avg_log_batch_cnt = 0; ++ const int64_t SWITCH_BARRIER = 10; ++ + if (FEEDBACK_FREEZE_MODE == freeze_mode_) { +- if (total_append_cnt >= APPEND_CNT_LB_FOR_PERIOD_FREEZE) { ++ // if (total_append_cnt >= APPEND_CNT_LB_FOR_PERIOD_FREEZE) { ++ if (avg_log_batch_cnt > SWITCH_BARRIER) { + freeze_mode_ = PERIOD_FREEZE_MODE; +- PALF_LOG(INFO, "switch freeze_mode to period", K_(palf_id), K_(self), K(total_append_cnt)); ++ PALF_LOG(ERROR, "switch freeze_mode to period", K_(palf_id), K_(self), K(total_append_cnt)); + } + } else if (PERIOD_FREEZE_MODE == freeze_mode_) { +- if (total_append_cnt < APPEND_CNT_LB_FOR_PERIOD_FREEZE) { ++ // if (total_append_cnt < APPEND_CNT_LB_FOR_PERIOD_FREEZE) { ++ if (avg_log_batch_cnt <= SWITCH_BARRIER) { + freeze_mode_ = FEEDBACK_FREEZE_MODE; +- PALF_LOG(INFO, "switch freeze_mode to feedback", K_(palf_id), K_(self), K(total_append_cnt)); ++ PALF_LOG(ERROR, "switch freeze_mode to feedback", K_(palf_id), K_(self), K(total_append_cnt)); + (void) feedback_freeze_last_log_(); + } + } else {} +diff --git a/src/logservice/palf/log_sliding_window.h b/src/logservice/palf/log_sliding_window.h +--- a/src/logservice/palf/log_sliding_window.h ++++ b/src/logservice/palf/log_sliding_window.h +@@ -396,6 +396,7 @@ private: + const LogType &log_type, + const char *log_data, + const int64_t data_len, ++ const bool need_freeze_self, + bool &is_need_handle); + int append_to_group_log_(const LSN &lsn, + const int64_t log_id, +@@ -608,6 +609,7 @@ private: + int64_t last_record_group_log_id_; + int64_t append_cnt_array_[APPEND_CNT_ARRAY_SIZE]; + FreezeMode freeze_mode_; ++ int64_t avg_log_batch_cnt_; + bool is_inited_; + private: + DISALLOW_COPY_AND_ASSIGN(LogSlidingWindow); +diff --git a/src/logservice/palf/lsn_allocator.cpp b/src/logservice/palf/lsn_allocator.cpp +--- a/src/logservice/palf/lsn_allocator.cpp ++++ b/src/logservice/palf/lsn_allocator.cpp +@@ -297,7 +297,8 @@ int LSNAllocator::alloc_lsn_scn(const SCN &base_scn, + SCN &scn, + bool &is_new_group_log, + bool &need_gen_padding_entry, +- int64_t &padding_len) ++ int64_t &padding_len, ++ bool &need_freeze_self) + { + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { +@@ -439,6 +440,11 @@ int LSNAllocator::alloc_lsn_scn(const SCN &base_scn, + is_next_need_cut = true; + } + } ++ // freeze immediately ++ if (need_freeze_self) { ++ is_new_group_log = true; ++ is_next_need_cut = true; ++ } + if (is_new_group_log) { + tmp_next_log_id_delta++; + } +diff --git a/src/logservice/palf/lsn_allocator.h b/src/logservice/palf/lsn_allocator.h +--- a/src/logservice/palf/lsn_allocator.h ++++ b/src/logservice/palf/lsn_allocator.h +@@ -64,7 +64,8 @@ public: + share::SCN &scn, + bool &is_new_log, + bool &need_gen_padding_entry, +- int64_t &padding_len); ++ int64_t &padding_len, ++ bool &need_freeze_self); + // 更新last_lsn和log_timestamp + // receive_log/append_disk_log 时调用 + int inc_update_last_log_info(const LSN &lsn, const int64_t log_id, const share::SCN &scn); +diff --git a/src/share/ob_thread_define.h b/src/share/ob_thread_define.h +--- a/src/share/ob_thread_define.h ++++ b/src/share/ob_thread_define.h +@@ -164,4 +164,5 @@ TG_DEF(SvrStartupHandler, SvrStartupHandler, QUEUE_THREAD, + TG_DEF(TenantTTLManager, TTLManager, TIMER) + TG_DEF(TenantTabletTTLMgr, TTLTabletMgr, TIMER) + TG_DEF(TntSharedTimer, TntSharedTimer, TIMER) ++TG_DEF(LogServerTest, LogServerTest, THREAD_POOL, ThreadCountPair(10, 10)) + #endif diff --git a/mittest/palf_cluster/rpc/ob_log_rpc_processor.h b/mittest/palf_cluster/rpc/ob_log_rpc_processor.h new file mode 100644 index 0000000000..99c6c15e14 --- /dev/null +++ b/mittest/palf_cluster/rpc/ob_log_rpc_processor.h @@ -0,0 +1,79 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_PALF_CLUSTER_RPC_PROCESSOR_H_ +#define OCEANBASE_PALF_CLUSTER_RPC_PROCESSOR_H_ + +#include "palf_cluster_rpc_req.h" +#include "palf_cluster_request_handler.h" +#include "palf_cluster_rpc_proxy.h" +#include "lib/ob_errno.h" + +namespace oceanbase +{ +namespace palfcluster +{ + +#define DEFINE_LOGSERVICE_RPC_PROCESSOR(CLASS, PROXY, REQTYPE, PCODE) \ +class CLASS : public obrpc::ObRpcProcessor> \ +{ \ +public: \ + CLASS() : filter_(NULL) {} \ + virtual ~CLASS() { filter_ = NULL; } \ + int process() \ + { \ + int ret = OB_SUCCESS; \ + const REQTYPE &req = arg_; \ + const common::ObAddr server = req.src_; \ + const uint64_t tenant_id = rpc_pkt_->get_tenant_id(); \ + LogRequestHandler handler; \ + handler.set_log_service(log_service_); \ + if (OB_UNLIKELY(NULL != filter_ && true == (*filter_)(server))) { \ + } else if (OB_FAIL(handler.handle_request(req))) { \ + CLOG_LOG(WARN, "Processor handle_request failed", K(ret), K(req)); \ + } else { \ + CLOG_LOG(INFO, "Processor handle_request success", K(ret), K(req)); \ + } \ + return ret; \ + } \ + void set_filter(void *filter) \ + { \ + filter_ = reinterpret_cast *>(filter); \ + } \ + void set_log_service(void *log_service) \ + { \ + log_service_ = reinterpret_cast(log_service); \ + } \ +private: \ + ObFunction *filter_; \ + palfcluster::LogService *log_service_; \ +} + +DEFINE_LOGSERVICE_RPC_PROCESSOR(LogCreateReplicaCmdP, + obrpc::PalfClusterRpcProxy, + palfcluster::LogCreateReplicaCmd, + obrpc::OB_LOG_CREATE_REPLICA_CMD); + +DEFINE_LOGSERVICE_RPC_PROCESSOR(LogSubmitLogP, + obrpc::PalfClusterRpcProxy, + palfcluster::SubmitLogCmd, + obrpc::OB_LOG_SUBMIT_LOG_CMD); + +DEFINE_LOGSERVICE_RPC_PROCESSOR(LogSubmitLogRespP, + obrpc::PalfClusterRpcProxy, + SubmitLogCmdResp, + obrpc::OB_LOG_SUBMIT_LOG_CMD_RESP); +} // end namespace palfcluster +} // end namespace oceanbase + + +#endif diff --git a/mittest/palf_cluster/rpc/palf_cluster_request_handler.cpp b/mittest/palf_cluster/rpc/palf_cluster_request_handler.cpp new file mode 100644 index 0000000000..80a24d7df9 --- /dev/null +++ b/mittest/palf_cluster/rpc/palf_cluster_request_handler.cpp @@ -0,0 +1,123 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#include "palf_cluster_request_handler.h" +#include "palf_cluster_rpc_req.h" +#include "palf_cluster_rpc_proxy.h" +#include "logservice/ob_log_service.h" +#include "logservice/palf/log_define.h" + +namespace oceanbase +{ +namespace palfcluster +{ +LogRequestHandler::LogRequestHandler() + : log_service_(NULL) +{ +} + +LogRequestHandler::~LogRequestHandler() +{ +} + +int LogRequestHandler::get_self_addr_(common::ObAddr &self) const +{ + int ret = OB_SUCCESS; + const common::ObAddr self_addr = GCTX.self_addr(); + if (false == self_addr.is_valid()) { + ret = OB_ERR_UNEXPECTED; + } else { + self = self_addr; + } + return ret; +} + +template <> +int LogRequestHandler::handle_request( + const LogCreateReplicaCmd &req) +{ + int ret = common::OB_SUCCESS; + const int64_t palf_id = req.ls_id_; + const common::ObAddr &server = req.src_; + share::ObLSID ls_id(palf_id); + if (OB_ISNULL(log_service_)) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "get_log_service failed", K(ret)); + } else if (false == req.is_valid()) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(ERROR, "Invalid argument!!!", K(ret), K(req)); + } else if (OB_FAIL(log_service_->create_palf_replica(req.ls_id_, req.member_list_, req.replica_num_, req.leader_idx_))) { + CLOG_LOG(ERROR, "create_palf_replica failed", K(ret)); + } + CLOG_LOG(ERROR, "create_palf_replica finish", K(ret), K(req)); + return ret; +} + +template <> +int LogRequestHandler::handle_request( + const SubmitLogCmd &req) +{ + int ret = common::OB_SUCCESS; + if (false == req.is_valid()) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(ERROR, "Invalid argument!!!", K(ret), K(req)); + } else { + const int64_t palf_id = req.ls_id_; + share::ObLSID ls_id(palf_id); + const common::ObAddr &src = req.src_; + const int64_t client_id = req.client_id_; + + palfcluster::LogClientMap *log_clients = nullptr; + palfcluster::ObLogClient *log_client = nullptr; + + if (OB_ISNULL(log_service_)) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "get_log_service failed", K(ret)); + } else if (FALSE_IT(log_clients = log_service_->get_log_client_map())) { + } else if (OB_FAIL(log_clients->get(ls_id, log_client))) { + CLOG_LOG(ERROR, "get_log_client failed", K(ret), K(palf_id)); + } else if (OB_FAIL(log_client->submit_log(src, client_id, req.log_buf_))) { + CLOG_LOG(ERROR, "submit_log failed", K(ret)); + } + ret = OB_SUCCESS; + } + return ret; +} + +template <> +int LogRequestHandler::handle_request( + const SubmitLogCmdResp &req) +{ + int ret = common::OB_SUCCESS; + if (false == req.is_valid()) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(ERROR, "Invalid argument!!!", K(ret), K(req)); + } else { + const int64_t palf_id = req.ls_id_; + share::ObLSID ls_id(palf_id); + const common::ObAddr &server = req.src_; + const int64_t client_id = req.client_id_; + + palfcluster::LogClientMap *log_clients = nullptr; + palfcluster::ObLogClient *log_client = nullptr; + + if (OB_ISNULL(log_service_)) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "get_log_service failed", K(ret)); + } else { + log_service_->clients_[client_id].has_returned(); + } + ret = OB_SUCCESS; + } + return ret; +} +} // end namespace logservice +} // end namespace oceanbase diff --git a/mittest/palf_cluster/rpc/palf_cluster_request_handler.h b/mittest/palf_cluster/rpc/palf_cluster_request_handler.h new file mode 100644 index 0000000000..7384909934 --- /dev/null +++ b/mittest/palf_cluster/rpc/palf_cluster_request_handler.h @@ -0,0 +1,74 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_PALF_CLUSTER_LOG_REQUEST_HANDLER_ +#define OCEANBASE_PALF_CLUSTER_LOG_REQUEST_HANDLER_ + +#include "lib/ob_errno.h" // OB_SUCCESS... +#include "logservice/ob_reporter_adapter.h" // ObLogReporterAdapter +#include "logservice/palf_handle_guard.h" // palf::PalfHandleGuard +#include "palf_cluster_rpc_req.h" // Req... +#include "mittest/palf_cluster/logservice/log_service.h" + +namespace oceanbase +{ + +namespace palf +{ +class PalfEnv; +} + +namespace obrpc +{ +class PalfClusterRpcProxy; +} + +namespace palfcluster +{ +class ObLogHandler; +class ObLogReplayService; + +class LogRequestHandler +{ +public: + LogRequestHandler(); + ~LogRequestHandler(); + template + int handle_sync_request(const ReqType &req, RespType &resp); + template + int handle_request(const ReqType &req); + void set_log_service(palfcluster::LogService *log_service) { log_service_ = log_service; } +private: + int get_self_addr_(common::ObAddr &self) const; + int get_rpc_proxy_(obrpc::PalfClusterRpcProxy *&rpc_proxy) const; + palfcluster::LogService *log_service_; +}; + +class ConfigChangeCmdHandler{ +public: + explicit ConfigChangeCmdHandler(palf::PalfHandle *palf_handle) + { + if (NULL != palf_handle) { + palf_handle_ = palf_handle; + } + } + ~ConfigChangeCmdHandler() + { + palf_handle_ = NULL; + } +private: + palf::PalfHandle *palf_handle_; +}; +} // end namespace logservice +} // end namespace oceanbase + +#endif diff --git a/mittest/palf_cluster/rpc/palf_cluster_rpc_processor.h b/mittest/palf_cluster/rpc/palf_cluster_rpc_processor.h new file mode 100644 index 0000000000..328fb4c7dd --- /dev/null +++ b/mittest/palf_cluster/rpc/palf_cluster_rpc_processor.h @@ -0,0 +1,80 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_PALF_CLUSTER_RPC_PROCESSOR_H_ +#define OCEANBASE_PALF_CLUSTER_RPC_PROCESSOR_H_ + +#include "palf_cluster_rpc_req.h" +#include "palf_cluster_request_handler.h" +#include "palf_cluster_rpc_proxy.h" +#include "lib/ob_errno.h" + +namespace oceanbase +{ +namespace palfcluster +{ +class LogService; + +#define DEFINE_LOGSERVICE_RPC_PROCESSOR(CLASS, PROXY, REQTYPE, PCODE) \ +class CLASS : public obrpc::ObRpcProcessor> \ +{ \ +public: \ + CLASS() : filter_(NULL) {} \ + virtual ~CLASS() { filter_ = NULL; } \ + int process() \ + { \ + int ret = OB_SUCCESS; \ + const REQTYPE &req = arg_; \ + const common::ObAddr server = req.src_; \ + const uint64_t tenant_id = rpc_pkt_->get_tenant_id(); \ + LogRequestHandler handler; \ + handler.set_log_service(log_service_); \ + if (OB_UNLIKELY(NULL != filter_ && true == (*filter_)(server))) { \ + } else if (OB_FAIL(handler.handle_request(req))) { \ + CLOG_LOG(WARN, "Processor handle_request failed", K(ret), K(req)); \ + } else { \ + CLOG_LOG(INFO, "Processor handle_request success", K(ret), K(req)); \ + } \ + return ret; \ + } \ + void set_filter(void *filter) \ + { \ + filter_ = reinterpret_cast *>(filter); \ + } \ + void set_log_service(void *log_service) \ + { \ + log_service_ = reinterpret_cast(log_service); \ + } \ +private: \ + ObFunction *filter_; \ + palfcluster::LogService *log_service_; \ +} + +DEFINE_LOGSERVICE_RPC_PROCESSOR(LogCreateReplicaCmdP, + obrpc::PalfClusterRpcProxy, + palfcluster::LogCreateReplicaCmd, + obrpc::OB_LOG_CREATE_REPLICA_CMD); + +DEFINE_LOGSERVICE_RPC_PROCESSOR(LogSubmitLogP, + obrpc::PalfClusterRpcProxy, + palfcluster::SubmitLogCmd, + obrpc::OB_LOG_SUBMIT_LOG_CMD); + +DEFINE_LOGSERVICE_RPC_PROCESSOR(LogSubmitLogRespP, + obrpc::PalfClusterRpcProxy, + SubmitLogCmdResp, + obrpc::OB_LOG_SUBMIT_LOG_CMD_RESP); +} // end namespace palfcluster +} // end namespace oceanbase + + +#endif diff --git a/mittest/palf_cluster/rpc/palf_cluster_rpc_proxy.h b/mittest/palf_cluster/rpc/palf_cluster_rpc_proxy.h new file mode 100644 index 0000000000..35ea9f2995 --- /dev/null +++ b/mittest/palf_cluster/rpc/palf_cluster_rpc_proxy.h @@ -0,0 +1,74 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_PALF_CLUSTER_LOG_RPC_PROXY_ +#define OCEANBASE_PALF_CLUSTER_LOG_RPC_PROXY_ + +#include "palf_cluster_rpc_req.h" +#include "rpc/obrpc/ob_rpc_proxy.h" // ObRpcProxy + +namespace oceanbase +{ +namespace obrpc +{ + +class PalfClusterRpcProxy : public obrpc::ObRpcProxy +{ +public: + DEFINE_TO(PalfClusterRpcProxy); + RPC_AP(PR3 send_submit_log_cmd, OB_LOG_SUBMIT_LOG_CMD, + (palfcluster::SubmitLogCmd)); + RPC_AP(PR3 send_submit_log_resp, OB_LOG_SUBMIT_LOG_CMD_RESP, + (palfcluster::SubmitLogCmdResp)); + RPC_AP(PR3 send_create_replica_cmd, OB_LOG_CREATE_REPLICA_CMD, + (palfcluster::LogCreateReplicaCmd)); +}; + +template +class ObLogRpcCB: public PalfClusterRpcProxy::AsyncCB +{ +public: + ObLogRpcCB() {} + ~ObLogRpcCB() {} + void set_args(const typename PalfClusterRpcProxy::AsyncCB::Request &args) { + UNUSED(args); + } + rpc::frame::ObReqTransport::AsyncCB *clone(const rpc::frame::SPAlloc &alloc) const { + void *buf = alloc(sizeof (*this)); + ObLogRpcCB *newcb = NULL; + if (NULL != buf) { + newcb = new (buf) ObLogRpcCB(); + } + return newcb; + } + int process() { + const common::ObAddr &dst = PalfClusterRpcProxy::AsyncCB::dst_; + obrpc::ObRpcResultCode &rcode = PalfClusterRpcProxy::AsyncCB::rcode_; + int ret = rcode.rcode_; + + if (common::OB_SUCCESS != rcode.rcode_) { + PALF_LOG(WARN, "there is an rpc error in logservice", K(rcode), K(dst), K(pcode)); + } else { + // do nothing + } + + return common::OB_SUCCESS; + } + void on_timeout() { + const common::ObAddr &dst = PalfClusterRpcProxy::AsyncCB::dst_; + //PALF_LOG(WARN, "logservice rpc timeout", K(dst)); + } +}; +} // end namespace obrpc +} // end namespace oceanbase + +#endif diff --git a/mittest/palf_cluster/rpc/palf_cluster_rpc_req.cpp b/mittest/palf_cluster/rpc/palf_cluster_rpc_req.cpp new file mode 100644 index 0000000000..a0b767c8f3 --- /dev/null +++ b/mittest/palf_cluster/rpc/palf_cluster_rpc_req.cpp @@ -0,0 +1,134 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "palf_cluster_rpc_req.h" +#include "logservice/palf/log_define.h" + +namespace oceanbase +{ +using namespace share; +namespace palfcluster +{ + +// ============= LogCreateReplicaCmd start ============= +LogCreateReplicaCmd::LogCreateReplicaCmd() + : src_(), + ls_id_(-1), + member_list_(), + replica_num_(-1), + leader_idx_(-1) { } + +LogCreateReplicaCmd::LogCreateReplicaCmd( + const common::ObAddr &src, + const int64_t ls_id, + const common::ObMemberList &member_list, + const int64_t replica_num, + const int64_t leader_idx) + : src_(src), + ls_id_(ls_id), + member_list_(member_list), + replica_num_(replica_num), + leader_idx_(leader_idx) { } + +bool LogCreateReplicaCmd::is_valid() const +{ + return src_.is_valid() && + -1 != ls_id_ && + member_list_.is_valid() && + replica_num_ > 0 && + replica_num_ <= common::OB_MAX_MEMBER_NUMBER; +} + +void LogCreateReplicaCmd::reset() +{ + src_.reset(); + ls_id_ = -1; + member_list_.reset(); + replica_num_ = -1; + leader_idx_ = -1; +} + +OB_SERIALIZE_MEMBER(LogCreateReplicaCmd, src_, ls_id_, + member_list_, replica_num_, leader_idx_); +// ============= LogCreateReplicaCmd end ============= + +// ============= SubmitLogCmd start ============= +SubmitLogCmd::SubmitLogCmd() + : src_(), + ls_id_(-1), + client_id_(-1), + log_buf_() + { } + +SubmitLogCmd::SubmitLogCmd( + const common::ObAddr &src, + const int64_t ls_id, + const int64_t client_id, + const palf::LogWriteBuf &log_buf) + : src_(src), + ls_id_(ls_id), + client_id_(client_id), + log_buf_(log_buf) + { } + +bool SubmitLogCmd::is_valid() const +{ + return src_.is_valid() && + -1 != ls_id_ && + log_buf_.is_valid(); +} + +void SubmitLogCmd::reset() +{ + src_.reset(); + ls_id_ = -1; + client_id_ = -1; + log_buf_.reset(); +} + +OB_SERIALIZE_MEMBER(SubmitLogCmd, src_, ls_id_, client_id_, log_buf_); +// ============= SubmitLogCmd end ============= + + +// ============= SubmitLogCmdResp start ============= +SubmitLogCmdResp::SubmitLogCmdResp() + : src_(), + ls_id_(-1), + client_id_(-1) + { } + +SubmitLogCmdResp::SubmitLogCmdResp( + const common::ObAddr &src, + const int64_t ls_id, + const int64_t client_id) + : src_(src), + ls_id_(ls_id), + client_id_(client_id) + { } + +bool SubmitLogCmdResp::is_valid() const +{ + return src_.is_valid() && + -1 != ls_id_; +} + +void SubmitLogCmdResp::reset() +{ + src_.reset(); + ls_id_ = -1; + client_id_ = -1; +} + +OB_SERIALIZE_MEMBER(SubmitLogCmdResp, src_, ls_id_, client_id_); +// ============= SubmitLogCmdResp end ============= +} // end namespace logservice +}// end namespace oceanbase diff --git a/mittest/palf_cluster/rpc/palf_cluster_rpc_req.h b/mittest/palf_cluster/rpc/palf_cluster_rpc_req.h new file mode 100644 index 0000000000..71eadfce59 --- /dev/null +++ b/mittest/palf_cluster/rpc/palf_cluster_rpc_req.h @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_PALF_CLUSTER_RPC_REQ_H_ +#define OCEANBASE_PALF_CLUSTER_RPC_REQ_H_ + +#include "lib/utility/ob_unify_serialize.h" // OB_UNIS_VERSION +#include "lib/utility/ob_print_utils.h" // TO_STRING_KV +#include "common/ob_member_list.h" // ObMemberList +#include "logservice/palf/palf_handle_impl.h" // PalfStat +#include "share/scn.h" +#include "logservice/palf/log_writer_utils.h" // LogWriteBuf + +namespace oceanbase +{ +namespace palfcluster +{ + +struct LogCreateReplicaCmd { + OB_UNIS_VERSION(1); +public: + LogCreateReplicaCmd(); + LogCreateReplicaCmd(const common::ObAddr &src, + const int64_t ls_id, + const common::ObMemberList &member_list, + const int64_t replica_num, + const int64_t leader_idx); + ~LogCreateReplicaCmd() + { + reset(); + } + bool is_valid() const; + void reset(); + TO_STRING_KV(K_(src), K_(ls_id), K_(member_list), K_(replica_num), K_(leader_idx)); + common::ObAddr src_; + int64_t ls_id_; + common::ObMemberList member_list_; + int64_t replica_num_; + int64_t leader_idx_; +}; + +struct SubmitLogCmd { + OB_UNIS_VERSION(1); +public: + SubmitLogCmd(); + SubmitLogCmd(const common::ObAddr &src, + const int64_t ls_id, + const int64_t client_id, + const palf::LogWriteBuf &log_buf_); + ~SubmitLogCmd() + { + reset(); + } + bool is_valid() const; + void reset(); + TO_STRING_KV(K_(src), K_(ls_id)); + common::ObAddr src_; + int64_t ls_id_; + int64_t client_id_; + palf::LogWriteBuf log_buf_; +}; + +struct SubmitLogCmdResp { + OB_UNIS_VERSION(1); +public: + SubmitLogCmdResp(); + SubmitLogCmdResp(const common::ObAddr &src, + const int64_t ls_id, + const int64_t client_id); + ~SubmitLogCmdResp() + { + reset(); + } + bool is_valid() const; + void reset(); + TO_STRING_KV(K_(src), K_(ls_id)); + common::ObAddr src_; + int64_t ls_id_; + int64_t client_id_; +}; + +} // end namespace palfcluster +}// end namespace oceanbase + +#endif diff --git a/mittest/palf_cluster/run_palf_bench.sh b/mittest/palf_cluster/run_palf_bench.sh new file mode 100755 index 0000000000..e8ebfcf2c1 --- /dev/null +++ b/mittest/palf_cluster/run_palf_bench.sh @@ -0,0 +1,364 @@ +#!/bin/sh +TEST_MACHINE1='SERVER_IP1' +TEST_MACHINE2='SERVER_IP2' +TEST_MACHINE3='SERVER_IP3' +USER_NAME='USERNAME' +# use your own data dir path +TARGET_PATH='/yourworkdir' + +function kill_server_process +{ +ssh $USERNAME@$TEST_MACHINE1 bash -s << EOF + killall -9 test_palf_bench_server; +EOF +ssh $USERNAME@$TEST_MACHINE2 bash -s << EOF + killall -9 test_palf_bench_server; +EOF +ssh $USERNAME@$TEST_MACHINE3 bash -s << EOF + killall -9 test_palf_bench_server; +EOF + echo "kill_server_process success" +} + +function cleanenv +{ +ssh $USERNAME@$TEST_MACHINE1 bash -s << EOF + killall -9 test_palf_bench_server; + cd $TARGET_PATH; + rm -rf palf_cluster_bench_server/; + rm -rf test_palf_bench_server; + rm -rf *so; +EOF +ssh $USERNAME@$TEST_MACHINE2 bash -s << EOF + killall -9 test_palf_bench_server; + cd $TARGET_PATH; + rm -rf palf_cluster_bench_server/; + rm -rf test_palf_bench_server; + rm -rf *so; +EOF +ssh $USERNAME@$TEST_MACHINE3 bash -s << EOF + killall -9 test_palf_bench_server; + cd $TARGET_PATH; + rm -rf palf_cluster_bench_server/; + rm -rf test_palf_bench_server; + rm -rf *so; +EOF + echo "cleanenv success" +} + +function send_server_binary +{ + cleanenv + find ../../ -name *.so | xargs -I {} scp {} $USERNAME@$TEST_MACHINE1:$TARGET_PATH + scp test_palf_bench_server $USERNAME@$TEST_MACHINE1:$TARGET_PATH +ssh $USERNAME@$TEST_MACHINE1 bash -s << EOF + cd $TARGET_PATH; + scp *.so $USERNAME@$TEST_MACHINE2:$TARGET_PATH + scp test_palf_bench_server $USERNAME@$TEST_MACHINE2:$TARGET_PATH + scp *.so $USERNAME@$TEST_MACHINE3:$TARGET_PATH + scp test_palf_bench_server $USERNAME@$TEST_MACHINE3:$TARGET_PATH +EOF + echo "send_server_binary success" +} + +# thread_num +# log_size +function startserver +{ +ssh $USERNAME@$TEST_MACHINE1 bash -s << EOF + export LD_LIBRARY_PATH=$TARGET_PATH; + cd $TARGET_PATH; + ./test_palf_bench_server $1 $2 > /dev/null 2>&1 & +EOF +ssh $USERNAME@$TEST_MACHINE2 bash -s << EOF + export LD_LIBRARY_PATH=$TARGET_PATH; + cd $TARGET_PATH; + ./test_palf_bench_server $1 $2 > /dev/null 2>&1 & +EOF +ssh $USERNAME@$TEST_MACHINE3 bash -s << EOF + export LD_LIBRARY_PATH=$TARGET_PATH; + cd $TARGET_PATH; + ./test_palf_bench_server $1 $2 > /dev/null 2>&1 & +EOF + sleep 5 + echo "startserver success" +} + +# thread_num +# log_size +# server no +function start_one_server +{ + SERVER="" + if [[ $3 == '1' ]]; + then + SERVER=TEST_MACHINE1 + elif [[ $3 == '2' ]]; + then + SERVER=TEST_MACHINE2 + elif [[ $3 == '3' ]]; + then + SERVER=TEST_MACHINE3 + else + echo "invalid arguments" + fi; +ssh $USERNAME@$SERVER bash -s << EOF + export LD_LIBRARY_PATH=$TARGET_PATH; + cd $TARGET_PATH; + ./test_palf_bench_server $1 $2 > /dev/null 2>&1 +EOF +} + +function start_local_client() +{ + # $1 thread_number + # $2 nbytes + # $3 palf_group_number + # $4 replica_num + echo "start local client" $@ + ./test_palf_bench_client $1 $2 $3 $4 +} + + +# $1 thread_number +# $2 nbytes +# $3 replica_num +# $4 freeze_us +# $5 experiment name +function generate_result +{ +result_dir_name="palf_raw_result_"$5 +append_result_name=$result_dir_name"/palf_append_"$1"_"$2"_"$3"_"$4".result" +io_result_name=$result_dir_name"/palf_io_"$1"_"$2"_"$3"_"$4".result" +group_result_name=$result_dir_name"/palf_group_"$1"_"$2"_"$3"_"$4".result" + +ssh $USERNAME@$TEST_MACHINE1 bash -s << EOF + cd $TARGET_PATH; + mkdir -p $result_dir_name; + grep l_append palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'append_cnt=[0-9],' > $append_result_name; + grep inner_write_impl_ palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'l_io_cnt=[0-9],' > $io_result_name; + grep 'GROUP LOG INFO' palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'total_group_log_cnt=[0-9],' > $group_result_name; +EOF +ssh $USERNAME@$TEST_MACHINE2 bash -s << EOF + cd $TARGET_PATH; + mkdir -p $result_dir_name; + grep l_append palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'append_cnt=[0-9],' > $append_result_name; + grep inner_write_impl_ palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'l_io_cnt=[0-9],' > $io_result_name; + grep 'GROUP LOG INFO' palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'total_group_log_cnt=[0-9],' > $group_result_name; +EOF +ssh $USERNAME@$TEST_MACHINE3 bash -s << EOF + cd $TARGET_PATH; + mkdir -p $result_dir_name; + grep l_append palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'append_cnt=[0-9],' > $append_result_name; + grep inner_write_impl_ palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'l_io_cnt=[0-9],' > $io_result_name; + grep 'GROUP LOG INFO' palf_cluster_bench_server/palf_cluster_bench_server.log | grep -v 'total_group_log_cnt=[0-9],' > $group_result_name; +EOF +} + + + +# $1 thread_number +# $2 nbytes +# $3 palf_group_number +# $4 replica_num +# $5 freeze_us +# $6 exp1,exp2 +function run_experiment_once +{ + send_server_binary + echo "start experiment: "$6", thread_num: " $1 "log_size: " $2 " freeze_us: " $5 + kill_server_process + startserver $1 $2 + ./test_palf_bench_client $1 $2 $3 $4 + sleep 20 + kill_server_process + generate_result $1 $2 $4 $5 $6 +} + +# $1 thread_number +# $2 nbytes +# $3 palf_group_number +# $4 replica_num +# $5 freeze_us +# $6 exp1,exp2 +# $7 leader/follower +function run_experiment_once_with_failure +{ + # send_server_binary + echo "start experiment: "$6", thread_num: " $1 "log_size: " $2 " freeze_us: " $5 + kill_server_process + startserver $1 $2 + ./test_palf_bench_client $1 $2 $3 $4 + sleep 20 + FAIL_MACHINE="" + if [[ $7 == 'leader' ]]; + then + FAIL_MACHINE=$TEST_MACHINE1 + else + FAIL_MACHINE=$TEST_MACHINE2 + fi; +ssh $USERNAME@$FAIL_MACHINE bash -s << EOF + killall -9 test_palf_bench_server; +EOF + sleep 30 + kill_server_process + generate_result $1 $2 $4 $5 $6 +} + + +# $1 thread_number +# $2 nbytes +# $3 palf_group_number +# $4 replica_num +# $5 freeze_us +# $6 exp1,exp2 +# $7 leader/follower +function run_experiment_once_measure_reconfirm +{ + send_server_binary + echo "start experiment: "$6", thread_num: " $1 "log_size: " $2 " freeze_us: " $5 + kill_server_process + startserver $1 $2 + ./test_palf_bench_client $1 $2 $3 $4 + loop_count=100 + round_idx=0 + fail_server_idx=1 + while [ $round_idx -lt loop_count ]; + do + sleep 10 + let round_idx++ + done; + + FAIL_MACHINE="" + if [[ $7 == 'leader' ]]; + then + FAIL_MACHINE=$TEST_MACHINE1 + else + FAIL_MACHINE=$TEST_MACHINE2 + fi; +ssh $USERNAME@$FAIL_MACHINE bash -s << EOF + killall -9 test_palf_bench_server; +EOF + sleep 30 + kill_server_process + generate_result $1 $2 $4 $5 $6 +} + +function experiment1 +{ + log_size_array=(32 64 128 256 512 1024 2048 4096 8192) + + thread_num_n_max=17 + + run_round=1 + thread_num_n=1 + while [ $thread_num_n -lt $thread_num_n_max ] + do + thread_num=1 + let thread_num="500*thread_num_n" + let thread_num_n++ + for log_size in ${log_size_array[@]} + do + echo "start run experiment1, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size + run_experiment_once $thread_num $log_size 1 3 1000 exp1_wo_reduce + let run_round++ + done; + done; +} + +# test for figure, less clients +function experiment1_less_clients +{ + send_server_binary + log_size_array=(512) + thread_numbers=(1 2 5 10 20 50 100 200 500) + + run_round=1 + for thread_num in ${thread_numbers[@]} + do + for log_size in ${log_size_array[@]} + do + echo "start run experiment1_less_clients, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size + run_experiment_once $thread_num $log_size 1 3 1000 exp1_less_clients + let run_round++ + done; + done; + +} + +# test freeze logs right now +function experiment3 +{ + log_size_array=(512) + + thread_num_n_max=17 + + run_round=1 + thread_num_n=3 + while [ $thread_num_n -lt $thread_num_n_max ] + do + thread_num=1 + let thread_num="500*thread_num_n" + let thread_num_n++ + for log_size in ${log_size_array[@]} + do + echo "start run experiment3, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size + run_experiment_once $thread_num $log_size 1 3 1000 exp3 + let run_round++ + done; + break + done; +} + +# freeze immediately or freeze adaptively +function experiment4 +{ + send_server_binary + run_round=1 + + log_size_array=(0) + # log_size_array=(512) + # thread_numbers=(1 2 5 10 20 50 100 200 500 1000 2000) + thread_numbers=(500) + + for thread_num in ${thread_numbers[@]} + do + for log_size in ${log_size_array[@]} + do + echo "start run experiment4, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size + # run_experiment_once $thread_num $log_size 1 3 1000 exp4-noaggre + # run_experiment_once $thread_num $log_size 1 3 1000 exp4-period + run_experiment_once $thread_num $log_size 1 3 1000 exp4-adaptively + # run_experiment_once $thread_num $log_size 1 3 1000 exp4-feedback + let run_round++ + done; + done; +} + +function experiment5 +{ + send_server_binary + run_round=1 + + thread_num=5000 + log_size=512 + + echo "start run experiment5, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size + run_experiment_once_with_failure $thread_num $log_size 1 3 1000 exp5_leader_failure leader + + # echo "start run experiment5, round: " $run_round ", thread_num: " $thread_num "log_size: " $log_size + # run_experiment_once_with_failure $thread_num $log_size 1 3 1000 exp5_follower_failure follower +} + +function main +{ + # run_experiment_once 100 16 1 3 1000 exp_test_bw + # experiment1 + # send_server_binary + # experiment4 + # experiment1_less_clients + # experiment5 + run_experiment_once 1500 512 1 3 1000 exp_test +} + +main "$@" \ No newline at end of file diff --git a/mittest/palf_cluster/test_palf_bench_client.cpp b/mittest/palf_cluster/test_palf_bench_client.cpp new file mode 100644 index 0000000000..2a72b36c0d --- /dev/null +++ b/mittest/palf_cluster/test_palf_bench_client.cpp @@ -0,0 +1,140 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include "io/easy_connection.h" +#include "lib/file/file_directory_utils.h" +#include "lib/ob_errno.h" +#include "lib/oblog/ob_log.h" +#include "lib/utility/utility.h" +#include +#include +#include "lib/utility/ob_defer.h" +#include "share/ob_errno.h" +#define private public +#include "logservice/palf/log_define.h" +#include "logservice/palf/log_rpc_processor.h" +#include "env/ob_simple_log_cluster.h" + +#undef private + +const std::string TEST_NAME = "palf_cluster_bench_client"; +using namespace oceanbase::common; +using namespace oceanbase; +namespace oceanbase +{ +using namespace logservice; +namespace unittest +{ + +char *level; +int thread_num_arg = 100; +int nbytes_arg = 500; +int palf_group_number_arg = 1; +int replica_num_arg = 3; + +ObAddr server1(ObAddr::VER::IPV4, "SERVER_IP1", ObSimpleLogCluster::RPC_PORT); +ObAddr server2(ObAddr::VER::IPV4, "SERVER_IP2", ObSimpleLogCluster::RPC_PORT); +ObAddr server3(ObAddr::VER::IPV4, "SERVER_IP3", ObSimpleLogCluster::RPC_PORT); + + +class TestObPalfClusterBench : public ObSimpleLogCluster +{ +public: + std::vector palf_id_list_; + std::vector palf_leader_list_; + std::vector palf_group_list_; + std::vector server_list_; +public: + TestObPalfClusterBench() : ObSimpleLogCluster() + { + server_list_.push_back(server1); + server_list_.push_back(server2); + server_list_.push_back(server3); + } + + int create_palf_group() + { + int ret = OB_SUCCESS; + const int64_t replica_num = oceanbase::unittest::replica_num_arg; + palfcluster::LogService *log_service = get_log_server()->get_log_service(); + obrpc::PalfClusterRpcProxy *rpc_proxy = NULL; + const int64_t RPC_TIMEOUT_US = 5 * 1000 * 1000; + ObAddr self; + ObMemberList memberlist; + for (int i = 0; i < replica_num; i++) { + memberlist.add_server(server_list_[i]); + } + + if (OB_ISNULL(log_service)) { + CLOG_LOG(ERROR, "get_log_service failed"); + } else if (FALSE_IT(rpc_proxy = log_service->get_rpc_proxy())) { + } else if (FALSE_IT(self = log_service->get_self())) { + } else { + for (int i = 0; i < oceanbase::unittest::palf_group_number_arg; i++) { + const int64_t leader_index = i % replica_num; + ObAddr leader; + memberlist.get_server_by_index(leader_index, leader); + palf_leader_list_.push_back(leader); + for (int j = 0; j < replica_num; j++) { + palfcluster::LogCreateReplicaCmd req(self, i + 1, memberlist, replica_num, leader_index); + if (OB_FAIL(rpc_proxy->to(server_list_[j]).timeout(RPC_TIMEOUT_US).trace_time(true). + max_process_handler_time(RPC_TIMEOUT_US).by(MTL_ID()).send_create_replica_cmd(req, NULL))) { + CLOG_LOG(ERROR, "create_replica failed", KR(ret), K(req), K(server_list_[j])); + break; + } + } + } + LOG_STDOUT("create_replica success\n"); + } + return ret; + } +}; + +std::string ObSimpleLogCluster::test_name_ = TEST_NAME; + +TEST_F(TestObPalfClusterBench, multiple_replica) +{ + int ret = OB_SUCCESS; + // client mode + OB_LOGGER.set_log_level("ERROR"); + if (OB_FAIL(create_palf_group())) { + CLOG_LOG(ERROR, "create_palf_group failed", K(ret)); + } else { + sleep(3); + if (OB_FAIL(create_palf_group())) { + CLOG_LOG(ERROR, "create_palf_group failed", K(ret)); + } + } +} + +} // end unittest +} // end oceanbase + +// Notes: How to write a new module integrate test case in logservice? +// 1. cp test_ob_simple_log_basic_func.cpp test_ob_simple_log_xxx.cpp +// 2. modify const string TEST_NAME, class name and log file name in +// test_ob_simple_log_xxx.cpp +// 3. add ob_unittest_clog() item and set label for test_ob_simple_log_xxx in +// unittest/cluster/CMakeFiles.txt +// 4. write new TEST_F + +int main(int argc, char **argv) +{ + if (argc > 1) { + oceanbase::unittest::thread_num_arg = strtol(argv[1], NULL, 10); + oceanbase::unittest::nbytes_arg = strtol(argv[2], NULL, 10); + oceanbase::unittest::palf_group_number_arg = strtol(argv[3], NULL, 10); + oceanbase::unittest::replica_num_arg = strtol(argv[4], NULL, 10); + } + RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME); +} \ No newline at end of file diff --git a/mittest/palf_cluster/test_palf_bench_server.cpp b/mittest/palf_cluster/test_palf_bench_server.cpp new file mode 100644 index 0000000000..db45719d0e --- /dev/null +++ b/mittest/palf_cluster/test_palf_bench_server.cpp @@ -0,0 +1,144 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include "io/easy_connection.h" +#include "lib/file/file_directory_utils.h" +#include "lib/ob_errno.h" +#include "lib/oblog/ob_log.h" +#include "lib/utility/utility.h" +#include +#include +#include "lib/utility/ob_defer.h" +#include "share/ob_errno.h" +#define private public +#include "logservice/palf/log_define.h" +#include "logservice/palf/log_rpc_processor.h" +#include "env/ob_simple_log_cluster.h" + +#undef private + +const std::string TEST_NAME = "palf_cluster_bench_server"; +using namespace oceanbase::common; +using namespace oceanbase; +namespace oceanbase +{ +using namespace logservice; +namespace unittest +{ + +char *level; +int thread_num_arg = 100; +int nbytes_arg = 500; +int palf_group_number_arg = 1; +int replica_num_arg = 3; + +ObAddr server1(ObAddr::VER::IPV4, "SERVER_IP1", ObSimpleLogCluster::RPC_PORT); +ObAddr server2(ObAddr::VER::IPV4, "SERVER_IP2", ObSimpleLogCluster::RPC_PORT); +ObAddr server3(ObAddr::VER::IPV4, "SERVER_IP3", ObSimpleLogCluster::RPC_PORT); + +class TestPalfClusterBenchServer : public ObSimpleLogCluster +{ +public: + std::vector palf_id_list_; + std::vector palf_leader_list_; + std::vector palf_group_list_; + std::vector server_list_; +public: + TestPalfClusterBenchServer() : ObSimpleLogCluster() + { + server_list_.push_back(server1); + server_list_.push_back(server2); + server_list_.push_back(server3); + } + + int local_submit_log(const int64_t thread_num, const int64_t log_size) + { + int ret = OB_SUCCESS; + palfcluster::LogService *log_service = get_log_server()->get_log_service(); + obrpc::PalfClusterRpcProxy *rpc_proxy = NULL; + const int64_t RPC_TIMEOUT_US = 500 * 1000 * 1000; + palfcluster::LogClientMap *log_clients = nullptr; + palfcluster::ObLogClient *log_client; + int64_t palf_id = 1; + const share::ObLSID ls_id(palf_id); + ObAddr self; + + if (OB_ISNULL(log_service)) { + CLOG_LOG(ERROR, "get_log_service failed"); + } else if (nullptr == (log_clients = log_service->get_log_client_map())) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "get_log_clients failed", K(ret)); + } else { + int64_t unused_id; + common::ObRole role = ObRole::INVALID_ROLE; + while (true) { + role = ObRole::INVALID_ROLE; + if (OB_FAIL(log_clients->get(ls_id, log_client))) { + CLOG_LOG(ERROR, "get_log_client failed", K(ret), K(palf_id)); + usleep(1000 * 1000); + } else if (OB_FAIL(log_client->get_log_handler()->get_role(role, unused_id))) { + CLOG_LOG(ERROR, "get_role fail", K(palf_id)); + } else if (role == ObRole::LEADER) { + break; + CLOG_LOG(ERROR, "switch_to_leader success", K(palf_id)); + } else { + usleep(100* 1000); + } + } + + if (OB_FAIL(log_client->submit_append_log_task(thread_num, log_size))) { + CLOG_LOG(ERROR, "submit_log failed", K(ret)); + } + } + PALF_LOG(INFO, "end test_palf_bench"); + + LOG_STDOUT("submit_log success\n"); + return ret; + } + +}; + +std::string ObSimpleLogCluster::test_name_ = TEST_NAME; + +TEST_F(TestPalfClusterBenchServer, multiple_replica) +{ + int ret = OB_SUCCESS; + OB_LOGGER.set_log_level("WDIAG"); + // server mode + if (OB_FAIL(local_submit_log(oceanbase::unittest::thread_num_arg, oceanbase::unittest::nbytes_arg))) { + CLOG_LOG(ERROR, "local_submit_log failed"); + } + while (true) { + usleep(1000 * 1000); + } +} + +} // end unittest +} // end oceanbase + +// Notes: How to write a new module integrate test case in logservice? +// 1. cp test_ob_simple_log_basic_func.cpp test_ob_simple_log_xxx.cpp +// 2. modify const string TEST_NAME, class name and log file name in +// test_ob_simple_log_xxx.cpp +// 3. add ob_unittest_clog() item and set label for test_ob_simple_log_xxx in +// unittest/cluster/CMakeFiles.txt +// 4. write new TEST_F + +int main(int argc, char **argv) +{ + if (argc > 1) { + oceanbase::unittest::thread_num_arg = strtol(argv[1], NULL, 10); + oceanbase::unittest::nbytes_arg = strtol(argv[2], NULL, 10); + } + RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME); +} \ No newline at end of file