Avoid stopping writing after restart due to log disk is shrinking.

2023-07-13 09:48:23 +00:00
parent b9780d0b60
commit 6810df627b
24 changed files with 1075 additions and 231 deletions
--- a/mittest/logservice/test_ob_simple_log_disk_mgr.cpp
+++ b/mittest/logservice/test_ob_simple_log_disk_mgr.cpp
@ -0,0 +1,365 @@
+/**
+ * Copyright (c) 2021 OceanBase
+ * OceanBase CE is licensed under Mulan PubL v2.
+ * You can use this software according to the terms and conditions of the Mulan PubL v2.
+ * You may obtain a copy of Mulan PubL v2 at:
+ *          http://license.coscl.org.cn/MulanPubL-2.0
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+ * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+ * See the Mulan PubL v2 for more details.
+ */
+
+#include "lib/ob_define.h"
+#include "lib/ob_errno.h"
+#include <cstdio>
+#include <gtest/gtest.h>
+#include <signal.h>
+#include <stdexcept>
+#define private public
+#define protected public
+#include "env/ob_simple_log_cluster_env.h"
+#undef private
+#undef protected
+#include "logservice/palf/log_reader_utils.h"
+#include "logservice/palf/log_define.h"
+#include "logservice/palf/log_group_entry_header.h"
+#include "logservice/palf/log_io_worker.h"
+#include "logservice/palf/lsn.h"
+
+const std::string TEST_NAME = "log_disk_mgr";
+
+using namespace oceanbase::common;
+using namespace oceanbase;
+namespace oceanbase
+{
+using namespace logservice;
+namespace unittest
+{
+class TestObSimpleLogDiskMgr : public ObSimpleLogClusterTestEnv
+{
+public:
+  TestObSimpleLogDiskMgr() : ObSimpleLogClusterTestEnv()
+  {
+    int ret = init();
+    if (OB_SUCCESS != ret) {
+      throw std::runtime_error("TestObSimpleLogDiskMgr init failed");
+    }
+  }
+  ~TestObSimpleLogDiskMgr()
+  {
+    destroy();
+  }
+  int init()
+  {
+    return OB_SUCCESS;
+  }
+  void destroy()
+  {}
+  int64_t id_;
+};
+
+int64_t ObSimpleLogClusterTestBase::member_cnt_ = 1;
+int64_t ObSimpleLogClusterTestBase::node_cnt_ = 1;
+std::string ObSimpleLogClusterTestBase::test_name_ = TEST_NAME;
+bool ObSimpleLogClusterTestBase::need_add_arb_server_  = false;
+
+TEST_F(TestObSimpleLogDiskMgr, out_of_disk_space)
+{
+  SET_CASE_LOG_FILE(TEST_NAME, "out_of_disk_space");
+  int64_t id = ATOMIC_AAF(&palf_id_, 1);
+  int server_idx = 0;
+  PalfEnv *palf_env = NULL;
+  int64_t leader_idx = 0;
+  PalfHandleImplGuard leader;
+  share::SCN create_scn = share::SCN::base_scn();
+  EXPECT_EQ(OB_SUCCESS, get_palf_env(server_idx, palf_env));
+  EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, create_scn, leader_idx, leader));
+  update_disk_options(leader_idx, MIN_DISK_SIZE_PER_PALF_INSTANCE/PALF_PHY_BLOCK_SIZE);
+  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 6*31+1, id, MAX_LOG_BODY_SIZE));
+  LogStorage *log_storage = &leader.palf_handle_impl_->log_engine_.log_storage_;
+  while (LSN(6*PALF_BLOCK_SIZE) > log_storage->log_tail_) {
+    usleep(500);
+  }
+  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 20, id, MAX_LOG_BODY_SIZE));
+  while (LSN(6*PALF_BLOCK_SIZE + 20 * MAX_LOG_BODY_SIZE) > log_storage->log_tail_) {
+    usleep(500);
+  }
+  LSN max_lsn = leader.palf_handle_impl_->get_max_lsn();
+  wait_lsn_until_flushed(max_lsn, leader);
+  PALF_LOG(INFO, "out of disk max_lsn", K(max_lsn));
+  usleep(palf::BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS + 5*10000);
+  EXPECT_EQ(OB_LOG_OUTOF_DISK_SPACE, submit_log(leader, 1, id, MAX_LOG_BODY_SIZE));
+  usleep(ObLooper::INTERVAL_US*2);
+}
+
+TEST_F(TestObSimpleLogDiskMgr, update_disk_options_basic)
+{
+  SET_CASE_LOG_FILE(TEST_NAME, "update_disk_options_basic");
+  OB_LOGGER.set_log_level("INFO");
+  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
+  // 将日志盘空间设置为10GB
+  update_disk_options(10*1024*1024*1024ul/PALF_PHY_BLOCK_SIZE);
+  PALF_LOG(INFO, "start update_disk_options_basic", K(id));
+  int64_t leader_idx = 0;
+  PalfHandleImplGuard leader;
+  PalfEnv *palf_env = NULL;
+  EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
+  EXPECT_EQ(OB_SUCCESS, get_palf_env(leader_idx, palf_env));
+
+  // 提交1G的日志
+  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 500, id, MAX_LOG_BODY_SIZE));
+  EXPECT_EQ(OB_SUCCESS, wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader));
+
+  EXPECT_EQ(OB_SUCCESS, update_disk_options(leader_idx, 20));
+  usleep(1000*1000+palf::BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS);
+  EXPECT_EQ(PalfDiskOptionsWrapper::Status::SHRINKING_STATUS,
+            palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+  EXPECT_EQ(palf_env->palf_env_impl_.disk_options_wrapper_.disk_opts_for_recycling_blocks_.log_disk_usage_limit_size_,
+            20*PALF_PHY_BLOCK_SIZE);
+  // 可以在上一次为缩容完成之前，可以继续缩容
+  EXPECT_EQ(OB_SUCCESS, update_disk_options(leader_idx, 10));
+  usleep(1000*1000+palf::BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS);
+  EXPECT_EQ(PalfDiskOptionsWrapper::Status::SHRINKING_STATUS,
+            palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+  EXPECT_EQ(palf_env->palf_env_impl_.disk_options_wrapper_.disk_opts_for_recycling_blocks_.log_disk_usage_limit_size_,
+            10*PALF_PHY_BLOCK_SIZE);
+  // 此时日志盘依旧未完成缩容，ObSimpleLogServer维护的disk_opts_依旧是10GB
+  {
+    PalfDiskOptions opts;
+    EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+    EXPECT_EQ(opts.log_disk_usage_limit_size_, 10*1024*1024*1024ul);
+  }
+  // 可以在上一次未缩容完成之前，可以继续扩容, 同时由于扩容后日志盘依旧小于第一次缩容
+  // ，因此依旧处于缩容状态.
+  EXPECT_EQ(OB_SUCCESS, update_disk_options(leader_idx, 11));
+  usleep(1000*1000+palf::BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS);
+  EXPECT_EQ(PalfDiskOptionsWrapper::Status::SHRINKING_STATUS,
+            palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+  EXPECT_EQ(palf_env->palf_env_impl_.disk_options_wrapper_.disk_opts_for_recycling_blocks_.log_disk_usage_limit_size_,
+            11*PALF_PHY_BLOCK_SIZE);
+  // 此时日志盘依旧未完成缩容，ObSimpleLogServer维护的disk_opts_依旧是10GB
+  {
+    PalfDiskOptions opts;
+    EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+    EXPECT_EQ(opts.log_disk_usage_limit_size_, 10*1024*1024*1024ul);
+  }
+  usleep(1000*1000+100+palf::BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS);
+  EXPECT_EQ(PalfDiskOptionsWrapper::Status::SHRINKING_STATUS,
+            palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+  EXPECT_EQ(palf_env->palf_env_impl_.disk_options_wrapper_.disk_opts_for_recycling_blocks_.log_disk_usage_limit_size_,
+            11*PALF_PHY_BLOCK_SIZE);
+  EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.palf_handle_impl_->get_max_lsn()));
+  const LSN base_lsn(12*PALF_BLOCK_SIZE);
+  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->set_base_lsn(base_lsn));
+  usleep(1000);
+  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1, id, 1000));
+  EXPECT_EQ(OB_SUCCESS, wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader));
+  // wait until disk space enough
+  EXPECT_EQ(OB_SUCCESS, wait_until_disk_space_to(leader_idx, (11*PALF_PHY_BLOCK_SIZE*80+100)/100));
+  usleep(1000*1000+palf::BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS);
+  EXPECT_EQ(PalfDiskOptionsWrapper::Status::NORMAL_STATUS,
+            palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+  EXPECT_EQ(palf_env->palf_env_impl_.disk_options_wrapper_.disk_opts_for_stopping_writing_.log_disk_usage_limit_size_,
+            11*PALF_PHY_BLOCK_SIZE);
+  // 等待后台线程再次执行update_disk_options操作，预期本地持久化的disk_opts会变为11*PALF_PHY_BLOCK_SIZE
+  {
+    usleep(2*ObLooper::INTERVAL_US);
+    PalfDiskOptions opts;
+    EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+    EXPECT_EQ(opts.log_disk_usage_limit_size_, 11*PALF_PHY_BLOCK_SIZE);
+  }
+}
+
+TEST_F(TestObSimpleLogDiskMgr, update_disk_options_restart)
+{
+  disable_hot_cache_ = true;
+  SET_CASE_LOG_FILE(TEST_NAME, "update_disk_options_restart");
+  OB_LOGGER.set_log_level("INFO");
+  // 扩容操作
+  EXPECT_EQ(OB_SUCCESS, update_disk_options(10*1024*1024*1024ul/PALF_PHY_BLOCK_SIZE));
+  usleep(1000*1000+palf::BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS);
+  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
+  int64_t leader_idx = 0;
+  PalfEnv *palf_env = NULL;
+  {
+    PalfHandleImplGuard leader;
+    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
+    {
+      PalfDiskOptions opts;
+      EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+      EXPECT_EQ(opts.log_disk_usage_limit_size_, 10*1024*1024*1024ul);
+    }
+    EXPECT_EQ(OB_SUCCESS, get_palf_env(leader_idx, palf_env));
+    {
+      EXPECT_EQ(PalfDiskOptionsWrapper::Status::NORMAL_STATUS,
+                palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+      EXPECT_EQ(10*1024*1024*1024ul,
+                palf_env->palf_env_impl_.disk_options_wrapper_.disk_opts_for_recycling_blocks_.log_disk_usage_limit_size_);
+    }
+    // 产生10个文件的数据
+    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10*32, id, MAX_LOG_BODY_SIZE));
+    EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.palf_handle_impl_->get_max_lsn()));
+    // 最小的log_disk_size要求是存在8个日志文件
+    EXPECT_EQ(OB_SUCCESS, update_disk_options(leader_idx, 8));
+    usleep(1000*1000+palf::BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS);
+    // 宕机前，缩容不会正式生效，因此不会导致停写
+    EXPECT_EQ(true, palf_env->palf_env_impl_.diskspace_enough_);
+    EXPECT_EQ(PalfDiskOptionsWrapper::Status::SHRINKING_STATUS,
+              palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+    EXPECT_EQ(10*1024*1024*1024ul, palf_env->palf_env_impl_.disk_options_wrapper_.disk_opts_for_stopping_writing_.log_disk_usage_limit_size_);
+    int64_t log_disk_usage, total_log_disk_size;
+    EXPECT_EQ(OB_SUCCESS, palf_env->palf_env_impl_.get_disk_usage(log_disk_usage, total_log_disk_size));
+    PALF_LOG(INFO, "log_disk_usage:", K(log_disk_usage), K(total_log_disk_size));
+    // 缩容未成功前，log_disk_usage_limit_size_依旧保持10G.
+    // 本地持久化的log_disk_size为10G
+    // 内部表中持久化的log_disk_size为8*PALF_PHY_BLOCK_SIZE
+    PalfDiskOptions opts;
+    EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+    EXPECT_EQ(opts.log_disk_usage_limit_size_, 10*1024*1024*1024ul);
+  }
+  // 物理缩容未成功前，宕机重启预期不会停写
+  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
+  {
+    // 内部表中持久化的log_disk_size依旧是8*PALF_PHY_BLOCK_SIZE
+    // 重启后继续缩容
+    int64_t log_disk_usage, total_log_disk_size;
+    EXPECT_EQ(OB_SUCCESS, get_palf_env(leader_idx, palf_env));
+    usleep(2*ObLooper::INTERVAL_US);
+    usleep(1000*1000 + BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS);
+    EXPECT_EQ(true, palf_env->palf_env_impl_.diskspace_enough_);
+    EXPECT_EQ(PalfDiskOptionsWrapper::Status::SHRINKING_STATUS,
+              palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+    // 本地持久化的（slog）记录的依旧是10G，因此不会停写
+    EXPECT_EQ(10*1024*1024*1024ul, palf_env->palf_env_impl_.disk_options_wrapper_.disk_opts_for_stopping_writing_.log_disk_usage_limit_size_);
+    EXPECT_EQ(OB_SUCCESS, palf_env->palf_env_impl_.get_disk_usage(log_disk_usage, total_log_disk_size));
+    PALF_LOG(INFO, "log_disk_usage:", K(log_disk_usage), K(total_log_disk_size));
+    PalfDiskOptions opts;
+    EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+    EXPECT_EQ(opts.log_disk_usage_limit_size_, 10*1024*1024*1024ul);
+  }
+  {
+    PalfHandleImplGuard leader;
+    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
+    PalfDiskOptions opts;
+    EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+    EXPECT_EQ(opts.log_disk_usage_limit_size_, 10*1024*1024*1024ul);
+
+    // 物理上保证不缩容，内部表中持久化的变为16*PALF_PHY_BLOCK_SIZE, 由于palf的log_disk_size是10G,
+    // 因此本次对palf是一次缩容操作. 但下一轮GC任务运行时，发现当前的使用的日志盘空间不会导致停写,
+    // 于是日志盘变为正常状态
+    EXPECT_EQ(OB_SUCCESS, update_disk_options(leader_idx, 16));
+    // 在下一轮GC任务运行后，本地持久化的log_disk_size也会变为16*PALF_PHY_BLOCK_SIZE
+    usleep(1000*1000+palf::BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS);
+    // 经过一轮GC后，会变为NORMAL_STATUS
+    EXPECT_EQ(true, palf_env->palf_env_impl_.diskspace_enough_);
+    EXPECT_EQ(PalfDiskOptionsWrapper::Status::NORMAL_STATUS,
+              palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+    EXPECT_EQ(16*PALF_PHY_BLOCK_SIZE,
+              palf_env->palf_env_impl_.disk_options_wrapper_.disk_opts_for_recycling_blocks_.log_disk_usage_limit_size_);
+    // 后台线程会完成缩容操作，最终本地持久化的变为16*PALF_PHY_BLOCK_SIZE
+    usleep(ObLooper::INTERVAL_US*2);
+    EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+    EXPECT_EQ(opts.log_disk_usage_limit_size_, 16*PALF_PHY_BLOCK_SIZE);
+  }
+}
+
+TEST_F(TestObSimpleLogDiskMgr, overshelling)
+{
+  disable_hot_cache_ = true;
+  SET_CASE_LOG_FILE(TEST_NAME, "overshelling");
+  OB_LOGGER.set_log_level("INFO");
+  ObServerLogBlockMgr *log_pool = nullptr;
+  EXPECT_EQ(OB_SUCCESS, get_log_pool(0, log_pool));
+  ASSERT_NE(nullptr, log_pool);
+  // 验证扩缩容场景下的LogPool字段的正确性
+  EXPECT_EQ(16*PALF_PHY_BLOCK_SIZE, log_pool->min_log_disk_size_for_all_tenants_);
+  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
+  int64_t leader_idx = 0;
+  PalfEnv *palf_env = NULL;
+  {
+    PalfHandleImplGuard leader;
+    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
+  }
+  EXPECT_EQ(OB_SUCCESS, update_disk_options(leader_idx, 15));
+  int64_t log_disk_size_used_for_tenants = log_pool->min_log_disk_size_for_all_tenants_;
+  // 缩容还未成功，预期log_disk_size_used_for_tenants一定是16*PALF_PHY_BLOCK_SIZE
+  PalfDiskOptions opts;
+  EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+  if (opts.log_disk_usage_limit_size_ == 16*PALF_PHY_BLOCK_SIZE) {
+    EXPECT_EQ(16*PALF_PHY_BLOCK_SIZE, log_disk_size_used_for_tenants);
+    // 缩容不会立马生效
+    usleep(ObLooper::INTERVAL_US*2);
+    EXPECT_EQ(15*PALF_PHY_BLOCK_SIZE, log_pool->min_log_disk_size_for_all_tenants_);
+  } else {
+    PALF_LOG(INFO, "update_disk_options successfully", K(log_disk_size_used_for_tenants), K(opts));
+  }
+
+  // 扩容预期立马成功
+  EXPECT_EQ(OB_SUCCESS, update_disk_options(leader_idx, 16));
+  EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+  EXPECT_EQ(16*PALF_PHY_BLOCK_SIZE, log_disk_size_used_for_tenants);
+  EXPECT_EQ(16*PALF_PHY_BLOCK_SIZE, opts.log_disk_usage_limit_size_);
+
+  // 直接扩容为LogPool上限值
+  const int64_t limit_log_disk_size = log_pool->log_pool_meta_.curr_total_size_;
+  EXPECT_EQ(OB_SUCCESS, update_disk_options(leader_idx, limit_log_disk_size/PALF_PHY_BLOCK_SIZE));
+  EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+  log_disk_size_used_for_tenants = log_pool->min_log_disk_size_for_all_tenants_;
+  EXPECT_EQ(limit_log_disk_size, log_disk_size_used_for_tenants);
+  EXPECT_EQ(limit_log_disk_size, opts.log_disk_usage_limit_size_);
+
+  {
+    PalfHandleImplGuard leader;
+    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
+    // 生成10个文件
+    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10*32, id, MAX_LOG_BODY_SIZE));
+    EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.palf_handle_impl_->get_max_lsn()));
+    EXPECT_EQ(OB_SUCCESS, update_disk_options(leader_idx, 10));
+    // 缩容一定不会成功，租户日志盘规格依旧为上限值
+    usleep(ObLooper::INTERVAL_US * 2);
+    EXPECT_EQ(OB_SUCCESS, get_palf_env(leader_idx, palf_env));
+    EXPECT_EQ(PalfDiskOptionsWrapper::Status::SHRINKING_STATUS,
+              palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+    EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+    log_disk_size_used_for_tenants = log_pool->min_log_disk_size_for_all_tenants_;
+    EXPECT_EQ(limit_log_disk_size, log_disk_size_used_for_tenants);
+    EXPECT_EQ(limit_log_disk_size, opts.log_disk_usage_limit_size_);
+    EXPECT_EQ(OB_MACHINE_RESOURCE_NOT_ENOUGH, log_pool->create_tenant(MIN_DISK_SIZE_PER_PALF_INSTANCE));
+    const LSN base_lsn(8*PALF_BLOCK_SIZE);
+    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->set_base_lsn(base_lsn));
+    usleep(1000);
+    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1, id, 1000));
+    EXPECT_EQ(OB_SUCCESS, wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader));
+    EXPECT_EQ(OB_SUCCESS, wait_until_disk_space_to(leader_idx, (10*PALF_PHY_BLOCK_SIZE*80+100)/100));
+    usleep(ObLooper::INTERVAL_US * 2);
+    EXPECT_EQ(PalfDiskOptionsWrapper::Status::NORMAL_STATUS,
+              palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+    EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+    log_disk_size_used_for_tenants = log_pool->min_log_disk_size_for_all_tenants_;
+    EXPECT_EQ(10*PALF_PHY_BLOCK_SIZE, log_disk_size_used_for_tenants);
+    EXPECT_EQ(10*PALF_PHY_BLOCK_SIZE, opts.log_disk_usage_limit_size_);
+    EXPECT_EQ(OB_SUCCESS, log_pool->create_tenant(MIN_DISK_SIZE_PER_PALF_INSTANCE));
+    log_pool->abort_create_tenant(MIN_DISK_SIZE_PER_PALF_INSTANCE);
+
+    // 扩容预计一定成功
+    EXPECT_EQ(OB_SUCCESS, update_disk_options(leader_idx, limit_log_disk_size/PALF_PHY_BLOCK_SIZE));
+    EXPECT_EQ(PalfDiskOptionsWrapper::Status::NORMAL_STATUS,
+              palf_env->palf_env_impl_.disk_options_wrapper_.status_);
+    EXPECT_EQ(OB_SUCCESS, get_disk_options(leader_idx, opts));
+    log_disk_size_used_for_tenants = log_pool->min_log_disk_size_for_all_tenants_;
+    EXPECT_EQ(limit_log_disk_size, log_disk_size_used_for_tenants);
+    EXPECT_EQ(limit_log_disk_size, opts.log_disk_usage_limit_size_);
+    EXPECT_EQ(OB_MACHINE_RESOURCE_NOT_ENOUGH, log_pool->create_tenant(MIN_DISK_SIZE_PER_PALF_INSTANCE));
+
+  }
+
+}
+
+} // namespace unittest
+} // namespace oceanbase
+
+int main(int argc, char **argv)
+{
+  RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME);
+}