fixed hidden sys tenant pin log disk in each server.

src/logservice/ob_server_log_block_mgr.cpp

@@ -301,11 +301,13 @@ int ObServerLogBlockMgr::create_tenant(const int64_t log_disk_size)
 {
   int ret = OB_SUCCESS;
   ObSpinLockGuard guard(resize_lock_);
-  if ((min_log_disk_size_for_all_tenants_ += log_disk_size) > get_total_size_guarded_by_lock_()) {
+  int64_t tmp_log_disk_size = min_log_disk_size_for_all_tenants_;
+  if ((tmp_log_disk_size += log_disk_size) > get_total_size_guarded_by_lock_()) {
     ret = OB_MACHINE_RESOURCE_NOT_ENOUGH;
     CLOG_LOG(ERROR, "ObServerLogBlockMGR can not hold any new tenants",
         K(ret), KPC(this), K(log_disk_size));
   } else {
+    min_log_disk_size_for_all_tenants_ = tmp_log_disk_size;
     CLOG_LOG(INFO, "ObServerLogBlockMGR create_tenant success", KPC(this), K(log_disk_size));
   }
   return ret;
@@ -324,14 +326,15 @@ int ObServerLogBlockMgr::update_tenant(const int64_t old_log_disk_size, const in
 {
   int ret = OB_SUCCESS;
   ObSpinLockGuard guard(resize_lock_);
-  min_log_disk_size_for_all_tenants_ -= old_log_disk_size;
+  int64_t tmp_log_disk_size = min_log_disk_size_for_all_tenants_;
-  if (old_log_disk_size == new_log_disk_size) {
+  tmp_log_disk_size -= old_log_disk_size;
-  } else if ((min_log_disk_size_for_all_tenants_+=new_log_disk_size) > get_total_size_guarded_by_lock_()) {
+  if ((tmp_log_disk_size +=new_log_disk_size) > get_total_size_guarded_by_lock_()) {
     ret = OB_MACHINE_RESOURCE_NOT_ENOUGH;
     CLOG_LOG(ERROR, "ObServerLogBlockMGR can not hold any new tenants",
         K(ret), KPC(this),  K(old_log_disk_size), K(new_log_disk_size));
   } else {
-    CLOG_LOG(INFO, "ObServerLogBlockMGR update_tenant success", K(ret), KPC(this), K(old_log_disk_size), K(new_log_disk_size));
+    min_log_disk_size_for_all_tenants_ = tmp_log_disk_size;
+    CLOG_LOG(INFO, "ObServerLogBlockMGR update_tenant success", KPC(this), K(old_log_disk_size), K(new_log_disk_size));
   }
   return ret;
 }
@@ -346,15 +349,17 @@ void ObServerLogBlockMgr::abort_update_tenant(const int64_t old_log_disk_size, c
   CLOG_LOG(INFO, "ObServerLogBlockMGR abort_update_tenant success", KPC(this), K(old_log_disk_size), K(new_log_disk_size));
 }
 
-int ObServerLogBlockMgr::remove_tenant()
+int ObServerLogBlockMgr::remove_tenant(const int64_t log_disk_size)
 {
   int ret = OB_SUCCESS;
   ObSpinLockGuard guard(resize_lock_);
-  min_log_disk_size_for_all_tenants_ = 0;
+  if (min_log_disk_size_for_all_tenants_ - log_disk_size < 0) {
-  if (OB_FAIL(get_tenants_log_disk_size_func_(min_log_disk_size_for_all_tenants_))) {
+    ret = OB_ERR_UNEXPECTED;
-    CLOG_LOG(ERROR, "get_tenants_log_disk_size_func_ failed", K(ret), KPC(this));
+    CLOG_LOG(ERROR, "unexpected error, min_log_disk_size_for_all_tenants_ is small than zero",
+        K(ret), KPC(this), K(log_disk_size));
   } else {
-    CLOG_LOG(INFO, "remove_tenant success", K(ret), KPC(this));
+    min_log_disk_size_for_all_tenants_ -= log_disk_size;
+    CLOG_LOG(INFO, "remove tenant from ObServerLogBlockMGR success", KPC(this), K(log_disk_size));
   }
   return ret;
 }

src/logservice/ob_server_log_block_mgr.h

@@ -198,7 +198,7 @@ public:
 
   // @brief after 'del_tenant' in ObMultiTenant success, need remove it from ObServerLogBlockMgr
   // NB: accurately, when tenant not exist in 'omt_', we can remove it from ObServerLogBlockMgr
-  int remove_tenant();
+  int remove_tenant(const int64_t log_disk_size);
 
   TO_STRING_KV("dir:",
                log_pool_path_, K_(dir_fd), K_(meta_fd), K_(log_pool_meta),

src/observer/omt/ob_multi_tenant.cpp

@@ -523,6 +523,8 @@ int ObMultiTenant::create_hidden_sys_tenant()
     LOG_ERROR("fail to construct meta", K(ret));
   } else if (OB_FAIL(create_tenant(meta, true/* write_slog*/))) {
     LOG_ERROR("create hidden sys tenant failed", K(ret));
+  } else {
+    LOG_INFO("create hidden sys tenant success", K(ret), K(meta));
   }
 
   return ret;
@@ -658,10 +660,10 @@ int ObMultiTenant::convert_hidden_to_real_sys_tenant(const ObUnitInfoGetter::ObT
     LOG_WARN("must be hidden sys tenant", K(ret));
   } else if (FALSE_IT(new_super_block = tenant->get_super_block())) {
   } else if (FALSE_IT(new_super_block.is_hidden_ = false)) {
-  } else if (OB_FAIL(update_tenant_unit_no_lock(unit))) {
+  } else if (OB_FAIL(update_tenant_unit_no_lock(unit, UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT))) {
     LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(unit));
-  } else if (ObServerCheckpointSlogHandler::get_instance()
+  } else if (OB_FAIL(ObServerCheckpointSlogHandler::get_instance()
-      .write_tenant_super_block_slog(new_super_block)) {
+      .write_tenant_super_block_slog(new_super_block))) {
     LOG_WARN("fail to write_tenant_super_block_slog", K(ret), K(new_super_block));
   } else {
     tenant->set_tenant_super_block(new_super_block);
@@ -692,6 +694,7 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
   int64_t bucket_lock_idx = -1;
   const int64_t log_disk_size = meta.unit_.config_.log_disk_size();
   int64_t lock_timeout_ts = abs_timeout_us - 5000000; // reserve 5s for creating tenant
+  bool is_hidden_sys = meta.is_hidden();
 
   if (IS_NOT_INIT) {
     ret = OB_NOT_INIT;
@@ -739,6 +742,20 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
       create_step = ObTenantCreateStep::STEP_CTX_MEM_CONFIG_SETTED; // step1
     }
   }
+
+  if (OB_SUCC(ret)) {
+    if (!is_virtual_tenant_id(tenant_id)
+        && !is_hidden_sys
+        && OB_FAIL(GCTX.log_block_mgr_->create_tenant(log_disk_size))) {
+      LOG_ERROR("create_tenatn in ObServerLogBlockMgr failed", KR(ret));
+    }
+    // if create_tenant in ObServerLogBlockMGR success, the log disk size need by this tenant has been pinned,
+    // otherwise, the assigned log disk size of ObServerLogBlockMGR is origin.
+    if (OB_SUCC(ret)) {
+      create_step = ObTenantCreateStep::STEP_LOG_DISK_SIZE_PINNED;
+    }
+  }
+
   if (OB_FAIL(ret)) {
     // do nothing
   } else if (OB_ISNULL(GCTX.cgroup_ctrl_)) {
@@ -748,15 +765,14 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
     ObTenant, ObModIds::OMT, tenant_id, GCONF.workers_per_cpu_quota.get_value(), *GCTX.cgroup_ctrl_))) {
     ret = OB_ALLOCATE_MEMORY_FAILED;
     LOG_WARN("new tenant fail", K(ret));
-  } else if (FALSE_IT(create_step = ObTenantCreateStep::STEP_TENANT_NEWED)) { //step2
+  } else if (FALSE_IT(create_step = ObTenantCreateStep::STEP_TENANT_NEWED)) { //step3
-
   } else if (OB_FAIL(tenant->init_ctx())) {
     LOG_WARN("init ctx fail", K(tenant_id), K(ret));
   } else if (write_slog) {
     if (OB_FAIL(write_create_tenant_prepare_slog(meta))) {
       LOG_ERROR("fail to write create tenant prepare slog", K(ret));
     } else {
-      create_step = ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG; // step3
+      create_step = ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG; // step4
     }
   }
 
@@ -791,7 +807,7 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
       LOG_ERROR("fail to write create tenant commit slog", K(ret), K(tenant_id));
     } else {
       tenant->set_create_status(ObTenantCreateStatus::CREATE_COMMIT);
-      create_step = ObTenantCreateStep::STEP_FINISH; // step4
+      create_step = ObTenantCreateStep::STEP_FINISH; // step5
     }
   }
 
@@ -802,19 +818,9 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
     if (OB_SUCC(get_tenant_unsafe(tenant_id, tmp_tenant))) {
       ret = OB_TENANT_EXIST;
       LOG_ERROR("tenant exist", K(ret), K(tenant_id));
-    // NB: hold log disk firstly, guarded by lock_ to avoid create tenant concurrently.
-    } else if (false == is_virtual_tenant_id(tenant_id) && OB_FAIL(GCTX.log_block_mgr_->create_tenant(log_disk_size))) {
-      LOG_WARN("create_tenant in ObServerLogBlockMgr failed", K(ret), K(meta));
     } else if (OB_FAIL(tenants_.insert(tenant, iter, compare_tenant))) {
       LOG_ERROR("fail to insert tenant", K(ret), K(tenant_id));
     }
-    // Need ensure that after insert tenant into tenants_ success, means create tenant success.
-    // NB:
-    // 1. need call ObServerLogBlockMgr::abort_create_tenant after create tenant failed.
-    // 2. need guarded by lock, avoid concurrent with create_tenant or del_tenant
-    if (false == is_valid_tenant_id(tenant_id) && OB_FAIL(ret)) {
-      GCTX.log_block_mgr_->abort_create_tenant(log_disk_size);
-    }
   }
   // TODO: @lingyang 预期不能失败
   if (!is_virtual_tenant_id(tenant_id) && OB_TMP_FAIL(update_tenant_config(tenant_id))) {
@@ -854,6 +860,13 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
       }
     } while (OB_SUCCESS != tmp_ret);
 
+    do {
+      tmp_ret = OB_SUCCESS;
+      if (create_step >= ObTenantCreateStep::STEP_LOG_DISK_SIZE_PINNED) {
+        GCTX.log_block_mgr_->abort_create_tenant(log_disk_size);
+      }
+    } while (OB_SUCCESS != tmp_ret);
+
     if (create_step >= ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG) {
       if (OB_SUCCESS != (tmp_ret = write_create_tenant_abort_slog(tenant_id))) {
         LOG_ERROR("fail to write create tenant abort slog", K(tmp_ret));
@@ -869,7 +882,8 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
   return ret;
 }
 
-int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantConfig &unit)
+int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantConfig &unit,
+                                              const UpdateTenantConfigOpt &opt)
 {
   int ret = OB_SUCCESS;
 
@@ -878,6 +892,9 @@ int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantCo
   const double max_cpu = static_cast<double>(unit.config_.max_cpu());
   const uint64_t tenant_id = unit.tenant_id_;
   int64_t allowed_mem_limit = 0;
+  // when opt is UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT, the new_log_disk_size should set to zero.
+  const int64_t new_log_disk_size =
+      opt == UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT ? 0 : unit.config_.log_disk_size();
 
   if (IS_NOT_INIT) {
     ret = OB_NOT_INIT;
@@ -895,7 +912,7 @@ int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantCo
     LOG_WARN("fail to update_tenant_freezer_mem_limit", K(ret), K(tenant_id));
   } else if (OB_FAIL(tenant->update_thread_cnt(max_cpu))) {
     LOG_WARN("fail to update mtl module thread_cnt", K(ret), K(tenant_id));
-  } else if (OB_FAIL(update_tenant_log_disk_size(tenant_id, unit.config_.log_disk_size()))) {
+  } else if (OB_FAIL(update_tenant_log_disk_size(tenant_id, new_log_disk_size, opt))) {
       LOG_WARN("fail to update tenant log disk size", K(ret), K(tenant_id));
   } else if (FALSE_IT(tenant->update_memory_size(unit.config_.memory_size()))) {
     // unreachable
@@ -926,7 +943,7 @@ int ObMultiTenant::update_tenant_unit(const ObUnitInfoGetter::ObTenantConfig &un
   } else if (OB_FAIL(bucket_lock_.wrlock(bucket_lock_idx = get_tenant_lock_bucket_idx(tenant_id)))) {
     LOG_WARN("fail to try_wrlock for update tenant unit", K(ret), K(tenant_id), K(bucket_lock_idx));
   } else if (FALSE_IT(lock_succ = true)) {
-  } else if (OB_FAIL(update_tenant_unit_no_lock(unit))) {
+  } else if (OB_FAIL(update_tenant_unit_no_lock(unit, UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT))) {
     LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(unit));
   }
 
@@ -976,7 +993,8 @@ int ObMultiTenant::update_tenant_memory(const uint64_t tenant_id, const int64_t
 }
 
 int ObMultiTenant::update_tenant_log_disk_size(const uint64_t tenant_id,
-                                               const int64_t expected_log_disk_size)
+                                               const int64_t expected_log_disk_size,
+                                               const UpdateTenantConfigOpt &opt)
 {
   int ret = OB_SUCCESS;
   MAKE_TENANT_SWITCH_SCOPE_GUARD(guard);
@@ -986,23 +1004,26 @@ int ObMultiTenant::update_tenant_log_disk_size(const uint64_t tenant_id,
     ObLogService *log_service = MTL(ObLogService *);
     if (OB_ISNULL(log_service)) {
       ret = OB_ERR_UNEXPECTED;
-    } else if (OB_FAIL(log_service->get_palf_disk_usage(unused_log_disk_size, old_log_disk_size))) {
+    // when opt is UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT, the old_log_disk_size is zero.
+    } else if (opt != UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT
+               && OB_FAIL(log_service->get_palf_disk_usage(unused_log_disk_size, old_log_disk_size))) {
       LOG_WARN("failed to get_palf_disk_usage", K(ret), K(unused_log_disk_size), K(old_log_disk_size),
           K(expected_log_disk_size));
-    } else if (old_log_disk_size == expected_log_disk_size) {
+    } else if(OB_FAIL(GCTX.log_block_mgr_->update_tenant(old_log_disk_size, expected_log_disk_size))) {
-      LOG_INFO("no need to update tenant log disk size", K(tenant_id), K(old_log_disk_size), K(expected_log_disk_size));
-    } else if(false == is_valid_tenant_id(tenant_id)
-        && OB_FAIL(GCTX.log_block_mgr_->update_tenant(old_log_disk_size, expected_log_disk_size))) {
       LOG_WARN("failed to update_tenant in ObServerLogBlockMgr", K(ret), K(old_log_disk_size),
           K(expected_log_disk_size));
     } else {
-      if (OB_FAIL(log_service->update_log_disk_usage_limit_size(expected_log_disk_size))) {
+        // Only when opt is UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT update palf.
-        LOG_WARN("failed to update_log_disk_usage_limit_size", K(ret), K(tenant_id),
+        if (opt == UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT) {
-               K(expected_log_disk_size));
+         if (OB_FAIL(log_service->update_log_disk_usage_limit_size(expected_log_disk_size))) {
-      } else {
+          LOG_WARN("failed to update_log_disk_usage_limit_size", K(ret), K(tenant_id),
-        LOG_INFO("update_tenant_log_disk_size success", K(ret), K(tenant_id),
+                 K(expected_log_disk_size));
-               K(expected_log_disk_size));
+        } else {
+          LOG_INFO("update_tenant_log_disk_size success", K(ret), K(tenant_id), K(opt),
+                 K(expected_log_disk_size));
+        }
       }
+
       if (false == is_virtual_tenant_id(tenant_id) && OB_FAIL(ret)) {
         GCTX.log_block_mgr_->abort_update_tenant(expected_log_disk_size, old_log_disk_size);
       }
@@ -1382,6 +1403,11 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &try_clock_succ)
           KP(removed_tenant), K(removed_tenant->lock_));
       removed_tenant->lock_.ld_.print();
     } else {
+      const share::ObUnitInfoGetter::ObTenantConfig &config = removed_tenant->get_unit();
+      const int64_t log_disk_size = config.config_.log_disk_size();
+      if (!removed_tenant->is_hidden() && !is_virtual_tenant_id(tenant_id)) {
+        GCTX.log_block_mgr_->remove_tenant(log_disk_size);
+      }
       ObTenant *removed_tenant_tmp = nullptr;
       SpinWLockGuard guard(lock_);
 
@@ -1395,7 +1421,6 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &try_clock_succ)
     }
 
     if (OB_SUCC(ret)) {
-      GCTX.log_block_mgr_->remove_tenant();
       removed_tenant->destroy();
       ob_delete(removed_tenant);
       LOG_INFO("remove tenant success", K(tenant_id));
@@ -1594,7 +1619,7 @@ int ObMultiTenant::convert_real_to_hidden_sys_tenant()
   } else if (OB_FAIL(bucket_lock_.try_wrlock(bucket_lock_idx = get_tenant_lock_bucket_idx(tenant_id)))) {
     LOG_WARN("fail to try_wrlock for delete tenant", K(ret), K(tenant_id), K(bucket_lock_idx));
   } else if (FALSE_IT(lock_succ = true)) {
-  } else if (OB_FAIL(update_tenant_unit_no_lock(tenant_meta.unit_))) {
+  } else if (OB_FAIL(update_tenant_unit_no_lock(tenant_meta.unit_, UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT))) {
     LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(tenant_meta));
   } else {
     ObTenantSwitchGuard guard(tenant);

src/observer/omt/ob_multi_tenant.h

@@ -75,6 +75,12 @@ typedef common::ObSortedVector<ObTenant*> TenantList;
 typedef TenantList::iterator TenantIterator;
 typedef common::ObVector<uint64_t> TenantIdList;
 
+enum class UpdateTenantConfigOpt {
+  CONVERT_HIDDEN_TO_REAL_SYS_TENANT = 0,
+  CONVERT_REAL_TO_HIDDEN_SYS_TENANT = 1,
+  NORMAL_UPDATE_TENANT_CONFIG_OPT = 2
+};
+
 // This is the entry class of OMT module.
 class ObMultiTenant
     : public share::ObThreadPool
@@ -112,7 +118,8 @@ public:
   int update_tenant_cpu(const uint64_t tenant_id, const double min_cpu, const double max_cpu);
   int update_tenant_memory(const uint64_t tenant_id, const int64_t mem_limit, int64_t &allowed_mem_limit);
   int update_tenant_log_disk_size(const uint64_t tenant_id,
-                                  const int64_t expected_log_disk_size);
+                                  const int64_t expected_log_disk_size,
+                                  const UpdateTenantConfigOpt &opt);
   int modify_tenant_io(const uint64_t tenant_id, const share::ObUnitConfig &unit_config);
   int update_tenant_config(uint64_t tenant_id);
   int update_palf_config();
@@ -178,15 +185,17 @@ protected:
   int create_virtual_tenants();
   int remove_tenant(const uint64_t tenant_id, bool &lock_succ);
   uint32_t get_tenant_lock_bucket_idx(const uint64_t tenant_id);
-  int update_tenant_unit_no_lock(const share::ObUnitInfoGetter::ObTenantConfig &unit);
+  int update_tenant_unit_no_lock(const share::ObUnitInfoGetter::ObTenantConfig &unit,
+                                 const UpdateTenantConfigOpt &opt);
 
 protected:
       static const int DEL_TRY_TIMES = 30;
   enum class ObTenantCreateStep {
       STEP_BEGIN = 0, // begin
       STEP_CTX_MEM_CONFIG_SETTED = 1, // set_tenant_ctx_idle succ
-      STEP_TENANT_NEWED = 2, // new tenant succ
+      STEP_LOG_DISK_SIZE_PINNED = 2,  // pin log disk size succ
-      STEP_WRITE_PREPARE_SLOG = 3, // write_prepare_create_tenant_slog succ
+      STEP_TENANT_NEWED = 3, // new tenant succ
+      STEP_WRITE_PREPARE_SLOG = 4, // write_prepare_create_tenant_slog succ
       STEP_FINISH,
   };
 

src/observer/omt/ob_tenant.cpp

@@ -728,7 +728,7 @@ int ObTenant::construct_mtl_init_ctx(const ObTenantMeta &meta, share::ObTenantMo
 bool ObTenant::is_hidden()
 {
   TCRLockGuard guard(meta_lock_);
-  return tenant_meta_.super_block_.is_hidden_;
+  return tenant_meta_.is_hidden();
 }
 
 ObTenantCreateStatus ObTenant::get_create_status()

src/observer/omt/ob_tenant_meta.h

@@ -51,6 +51,11 @@ public:
   int build(const share::ObUnitInfoGetter::ObTenantConfig &unit,
             const storage::ObTenantSuperBlock &super_block);
 
+  bool is_hidden() const
+  {
+    return super_block_.is_hidden_;
+  }
+
 
   TO_STRING_KV(K_(unit), K_(super_block), K_(create_status));
 

unittest/logservice/test_server_log_block_mgr.cpp

@@ -130,7 +130,7 @@ public:
       if (OB_SUCC(ret)) {
         map_[key] = size;
       } else {
-        this->log_block_mgr_.abort_create_tenant(size);
+        //this->log_block_mgr_.abort_create_tenant(size);
       }
     }
     return ret;
@@ -141,8 +141,9 @@ public:
     if (map_.end() == map_.find(key)) {
       ret = OB_ENTRY_NOT_EXIST;
     } else {
+      int64 size = map_[key];
       map_.erase(key);
-      this->log_block_mgr_.remove_tenant();
+      this->log_block_mgr_.remove_tenant(size);
     }
     return ret;
   };
@@ -156,7 +157,6 @@ public:
       ret = this->log_block_mgr_.update_tenant(old_size, new_size);
       if (OB_SUCCESS != ret) {
         map_[key] = old_size;
-        this->log_block_mgr_.abort_update_tenant(new_size, old_size);
       } else {
         map_[key] = new_size;
       }