set frozen for all memtables in ObLS::offline()

src/storage/ls/ob_ls.cpp

@@ -720,8 +720,8 @@ int ObLS::offline_()
   } else if (OB_FAIL(log_handler_.offline())) {
     LOG_WARN("failed to offline log", K(ret));
   // TODO: delete it if apply sequence
-  // force release memtables and freeze their allocators to reduce active tenant_memory
-  } else if (OB_FAIL(ls_tablet_svr_.offline())) {
+  // force set allocators frozen to reduce active tenant_memory
+  } else if (OB_FAIL(ls_tablet_svr_.set_frozen_for_all_memtables())) {
     LOG_WARN("tablet service offline failed", K(ret), K(ls_meta_));
   } else if (OB_FAIL(offline_compaction_())) {
     LOG_WARN("compaction offline failed", K(ret), K(ls_meta_));

src/storage/ls/ob_ls_tablet_service.cpp

@@ -6241,6 +6241,32 @@ int ObLSTabletService::DestroyMemtableAndMemberOperator::operator()(const common
   return ret;
 }
 
+int ObLSTabletService::SetMemtableFrozenOperator::operator()(const common::ObTabletID &tablet_id)
+{
+  int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
+  ObTabletHandle handle;
+  const uint64_t tenant_id = MTL_ID();
+  cur_tablet_id_ = tablet_id;
+  if (OB_UNLIKELY(!tablet_id.is_valid()) ||
+      OB_ISNULL(tablet_svr_)) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(tablet_svr_));
+  } else if (OB_FAIL(tablet_svr_->get_tablet(tablet_id,
+                                             handle,
+                                             ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) {
+    if (OB_TABLET_NOT_EXIST == ret) {
+      LOG_WARN("failed to get tablet, skip set memtable frozen", K(ret), K(tablet_id));
+      ret = OB_SUCCESS;
+    } else {
+      LOG_ERROR("failed to get tablet", K(ret), K(tablet_id));
+    }
+  } else if (OB_FAIL(handle.get_obj()->set_frozen_for_all_memtables())) {
+    LOG_WARN("failed to set frozen for all memtables", K(tenant_id), K(tablet_id));
+  }
+  return ret;
+}
+
 void ObLSTabletService::AllowToReadMgr::disable_to_read()
 {
   AllowToReadInfo read_info;
@@ -6297,5 +6323,19 @@ int ObLSTabletService::get_all_tablet_ids(
   return ret;
 }
 
+int ObLSTabletService::set_frozen_for_all_memtables()
+{
+  int ret = OB_SUCCESS;
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not inited", K(ret), K_(is_inited));
+  } else {
+    SetMemtableFrozenOperator set_mem_frozen_op(this);
+    if (OB_FAIL(tablet_id_set_.foreach(set_mem_frozen_op))) {
+      LOG_WARN("fail to set memtables frozen", K(ret), K(set_mem_frozen_op.cur_tablet_id_));
+    }
+  }
+  return ret;
+}
 } // namespace storage
 } // namespace oceanbase

src/storage/ls/ob_ls_tablet_service.h

@@ -108,6 +108,9 @@ public:
   void destroy();
   int offline();
   int online();
+  // TODO: delete it if apply sequence
+  // set allocators frozen to reduce active tenant_memory in ObLS::offline_()
+  int set_frozen_for_all_memtables();
 public:
   class AllowToReadMgr final
   {
@@ -481,6 +484,16 @@ private:
     common::ObTabletID cur_tablet_id_;
     ObLSTabletService *tablet_svr_;
   };
+  class SetMemtableFrozenOperator final
+  {
+  public:
+    SetMemtableFrozenOperator(ObLSTabletService *tablet_svr)
+      : tablet_svr_(tablet_svr) {}
+    ~SetMemtableFrozenOperator() = default;
+    int operator()(const common::ObTabletID &tablet_id);
+    common::ObTabletID cur_tablet_id_;
+    ObLSTabletService *tablet_svr_;
+  };
 private:
   void report_tablet_to_rs(const common::ObIArray<common::ObTabletID> &tablet_id_array);
 

src/storage/ob_i_memtable_mgr.h

@@ -295,6 +295,7 @@ public:
     return OB_NOT_SUPPORTED;
   }
   virtual int remove_memtables_from_data_checkpoint() { return OB_SUCCESS; }
+  virtual int set_frozen_for_all_memtables() { return OB_SUCCESS; }
   DECLARE_VIRTUAL_TO_STRING;
 protected:
   static int64_t get_memtable_idx(const int64_t pos) { return pos & (MAX_MEMSTORE_CNT - 1); }

src/storage/tablet/ob_tablet.cpp

@@ -3351,7 +3351,7 @@ int ObTablet::remove_memtables_from_data_checkpoint()
   } else if (OB_FAIL(get_memtable_mgr(memtable_mgr))) {
     LOG_WARN("failed to get memtable mgr", K(ret));
   } else if (OB_FAIL(memtable_mgr->remove_memtables_from_data_checkpoint())){
-    LOG_WARN("failed to rmeove memtables from data_checkpoint", K(ret));
+    LOG_WARN("failed to remove memtables from data_checkpoint", K(ret));
   }
 
   return ret;
@@ -3384,5 +3384,21 @@ int ObTablet::inner_check_valid() const
   return ret;
 }
 
+int ObTablet::set_frozen_for_all_memtables()
+{
+  int ret = OB_SUCCESS;
+  ObIMemtableMgr *memtable_mgr = nullptr;
+
+  if (OB_UNLIKELY(!is_inited_)) {
+    ret = OB_NOT_INIT;
+    LOG_WARN("not inited", K(ret), K_(is_inited));
+  } else if (OB_FAIL(get_memtable_mgr(memtable_mgr))) {
+    LOG_WARN("failed to get memtable mgr", K(ret));
+  } else if (OB_FAIL(memtable_mgr->set_frozen_for_all_memtables())){
+    LOG_WARN("failed to set_frozen_for_all_memtables", K(ret));
+  }
+
+  return ret;
+}
 } // namespace storage
 } // namespace oceanbase

src/storage/tablet/ob_tablet.h

@@ -390,6 +390,7 @@ public:
       const ObMigrationTabletParam *tablet_meta);
   int clear_memtables_on_table_store(); // be careful to call this func, will destroy memtables array on table_store
   int remove_memtables_from_data_checkpoint();
+  int set_frozen_for_all_memtables();
   // different from the is_valid() function
   // typically used for check valid for migration or restore
   int check_valid() const;

src/storage/tablet/ob_tablet_memtable_mgr.cpp

@@ -1087,5 +1087,34 @@ int ObTabletMemtableMgr::get_memtable_for_multi_source_data_unit(
   return ret;
 }
 
+int ObTabletMemtableMgr::set_frozen_for_all_memtables()
+{
+  int ret = OB_SUCCESS;
+  MemMgrWLockGuard lock_guard(lock_);
+
+  if (IS_NOT_INIT) {
+    ret = OB_NOT_INIT;
+    STORAGE_LOG(WARN, "not inited", K(ret));
+  } else if (OB_ISNULL(ls_)) {
+    ret = OB_ERR_UNEXPECTED;
+    STORAGE_LOG(WARN, "ls is null", K(ret));
+  } else {
+    const share::ObLSID &ls_id = ls_->get_ls_id();
+    for (int64_t i = memtable_head_; OB_SUCC(ret) && i < memtable_tail_; ++i) {
+      // memtable that cannot be released will block memtables behind it
+      ObMemtable *memtable = static_cast<memtable::ObMemtable *>(tables_[get_memtable_idx(i)]);
+      if (OB_ISNULL(memtable)) {
+        ret = OB_ERR_UNEXPECTED;
+        STORAGE_LOG(WARN, "memtable is nullptr", K(ret), K(ls_id), KP(memtable), K(i));
+      } else {
+        STORAGE_LOG(INFO, "set frozen for offline", K(ls_id), K(i), KPC(memtable));
+        memtable->set_frozen();
+      }
+    }
+  }
+
+  return ret;
+}
+
 }  // namespace storage
 }  // namespace oceanbase

src/storage/tablet/ob_tablet_memtable_mgr.h

@@ -104,6 +104,7 @@ public:
       logservice::ObLogHandler *log_handler) override;
   virtual int reset_storage_recorder() override;
   virtual int remove_memtables_from_data_checkpoint() override;
+  virtual int set_frozen_for_all_memtables() override;
   DECLARE_VIRTUAL_TO_STRING;
 
 protected: