add basic rto mittest

mittest/simple_server/CMakeLists.txt

@@ -43,3 +43,4 @@ ob_unittest_observer(test_change_arb_service_status test_change_arb_service_stat
 ob_unittest_observer(test_big_tx_data test_big_tx_data.cpp)
 ob_unittest_observer(test_fast_commit_report fast_commit_report.cpp)
 ob_unittest_observer(test_mvcc_gc test_mvcc_gc.cpp)
+ob_unittest_observer(test_ob_simple_rto test_ob_simple_rto.cpp)

mittest/simple_server/test_ob_simple_rto.cpp

@@ -0,0 +1,131 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include <cstdio>
+#include <gtest/gtest.h>
+#include <signal.h>
+#define private public
+#include "share/scn.h"
+#include "storage/slog/ob_storage_logger.h"
+#include "env/ob_simple_cluster_test_base.h"
+#include "logservice/ob_log_service.h"
+#include "logservice/leader_coordinator/ob_failure_detector.h"
+#undef private
+
+const std::string TEST_NAME = "rto_func";
+const int64_t CLOG_HANG_TIME_THRESHOLD_US = 5 * 1000 * 1000;
+int64_t mock_fatal_err_ts = OB_INVALID_TIMESTAMP;
+bool mock_clog_disk_hang = false;
+bool mock_sstable_io_hang = false;
+//slog的路径全部为inline函数,暂时无法mock
+//bool mock_slog_io_hang = false;
+bool mock_clog_disk_full = false;
+using namespace oceanbase;
+namespace oceanbase
+{
+using namespace palf;
+using namespace storage;
+namespace common
+{
+int ObIOFaultDetector::get_device_health_status(ObDeviceHealthStatus &dhs,
+    int64_t &device_abnormal_time)
+{
+  if (mock_sstable_io_hang) {
+    dhs = DEVICE_HEALTH_WARNING;
+    device_abnormal_time = mock_fatal_err_ts - GCONF.data_storage_warning_tolerance_time;
+  } else {
+    dhs = DEVICE_HEALTH_NORMAL;
+    device_abnormal_time = OB_INVALID_TIMESTAMP;
+  }
+  return OB_SUCCESS;
+}
+};
+using namespace common;
+namespace logservice
+{
+int ObLogService::get_io_start_time(int64_t &last_working_time)
+{
+  last_working_time = mock_clog_disk_hang ?
+                      mock_fatal_err_ts - CLOG_HANG_TIME_THRESHOLD_US
+                      : OB_INVALID_TIMESTAMP;
+  return OB_SUCCESS;
+}
+int ObLogService::check_disk_space_enough(bool &is_disk_enough)
+{
+  is_disk_enough = !mock_clog_disk_full;
+  return OB_SUCCESS;
+}
+};
+using namespace logservice;
+using namespace coordinator;
+namespace unittest
+{
+class ObRTOTest : public ObSimpleClusterTestBase
+{
+public:
+  ObRTOTest() : ObSimpleClusterTestBase("test_ob_rto_") {}
+};
+
+TEST_F(ObRTOTest, basic_rto)
+{
+  CLOG_LOG(INFO, "test rto begin");
+  share::ObTenantSwitchGuard tguard;
+  ASSERT_EQ(OB_SUCCESS, tguard.switch_to(1));
+  //mock clog disk hang
+  mock_fatal_err_ts = ObTimeUtility::fast_current_time();
+  mock_clog_disk_hang = true;
+  while (!MTL(ObFailureDetector*)->has_add_clog_hang_event_) {
+    CLOG_LOG(INFO, "waiting detect clog disk hang");
+    usleep(100 * 1000);
+  }
+  mock_clog_disk_hang = false;
+  while (MTL(ObFailureDetector*)->has_add_clog_hang_event_) {
+    CLOG_LOG(INFO, "waiting recover clog disk hang");
+    usleep(100 * 1000);
+  }
+
+  //mock clog disk full
+  mock_clog_disk_full = true;
+  while (!MTL(ObFailureDetector*)->has_add_clog_full_event_) {
+    CLOG_LOG(INFO, "waiting detect clog disk full");
+    usleep(100 * 1000);
+  }
+  mock_clog_disk_full = false;
+  while (MTL(ObFailureDetector*)->has_add_clog_full_event_) {
+    CLOG_LOG(INFO, "waiting recover clog disk full");
+    usleep(100 * 1000);
+  }
+
+  //mock sstable io hang
+  mock_fatal_err_ts = ObTimeUtility::fast_current_time();
+  mock_sstable_io_hang = true;
+  while (!MTL(ObFailureDetector*)->has_add_sstable_hang_event_) {
+    CLOG_LOG(INFO, "waiting detect sstable io hang");
+    usleep(100 * 1000);
+  }
+  mock_sstable_io_hang = false;
+  while (MTL(ObFailureDetector*)->has_add_sstable_hang_event_) {
+    CLOG_LOG(INFO, "waiting recover sstable io hang");
+    usleep(100 * 1000);
+  }
+}
+} // unitest
+} // oceanbase
+
+int main(int argc, char **argv)
+{
+  oceanbase::unittest::init_log_and_gtest(argc, argv);
+  OB_LOGGER.set_log_level("INFO");
+  GCONF._enable_defensive_check = false;
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}