FREEZE_CHECKER: check whether the tenant memstore is really frozen

src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp

@@ -37,7 +37,7 @@ public:
     memtable::ObMemtable& mt = handle->mt_;
     ObLSID ls_id;
     item.protection_clock_ = handle->get_protection_clock();
-    item.is_active_ = mt.is_active_memtable();
+    item.is_active_ = handle->is_active();
     item.ls_id_ = (OB_SUCCESS == mt.get_ls_id(ls_id)) ? ls_id.id() : ObLSID::INVALID_LS_ID;
     item.tablet_id_ = mt.get_key().tablet_id_.id();
     item.scn_range_ = mt.get_scn_range();

src/share/allocator/ob_gmemstore_allocator.cpp

@@ -30,7 +30,21 @@ int FrozenMemstoreInfoLogger::operator()(ObDLink* link)
   int ret = OB_SUCCESS;
   ObGMemstoreAllocator::AllocHandle* handle = CONTAINER_OF(link, typeof(*handle), total_list_);
   memtable::ObMemtable& mt = handle->mt_;
-  if (mt.is_frozen_memtable()) {
+  if (handle->is_frozen()) {
+    if (OB_FAIL(databuff_print_obj(buf_, limit_, pos_, mt))) {
+    } else {
+      ret = databuff_printf(buf_, limit_, pos_, ",");
+    }
+  }
+  return ret;
+}
+
+int ActiveMemstoreInfoLogger::operator()(ObDLink* link)
+{
+  int ret = OB_SUCCESS;
+  ObGMemstoreAllocator::AllocHandle* handle = CONTAINER_OF(link, typeof(*handle), total_list_);
+  memtable::ObMemtable& mt = handle->mt_;
+  if (handle->is_active()) {
     if (OB_FAIL(databuff_print_obj(buf_, limit_, pos_, mt))) {
     } else {
       ret = databuff_printf(buf_, limit_, pos_, ",");

src/share/allocator/ob_gmemstore_allocator.h

@@ -33,6 +33,17 @@ struct FrozenMemstoreInfoLogger
   int64_t limit_;
   int64_t pos_;
 };
+
+struct ActiveMemstoreInfoLogger
+{
+  ActiveMemstoreInfoLogger(char* buf, int64_t limit): buf_(buf), limit_(limit), pos_(0) {}
+  ~ActiveMemstoreInfoLogger() {}
+  int operator()(ObDLink* link);
+  char* buf_;
+  int64_t limit_;
+  int64_t pos_;
+};
+
 class ObGMemstoreAllocator
 {
 public:
@@ -120,11 +131,11 @@ public:
   void* alloc(AllocHandle& handle, int64_t size);
   void set_frozen(AllocHandle& handle);
   template<typename Func>
-  int for_each(Func& f) {
+  int for_each(Func& f, const bool reverse=false) {
     int ret = common::OB_SUCCESS;
     ObDLink* iter = NULL;
     LockGuard guard(lock_);
-    while(OB_SUCC(ret) && NULL != (iter = hlist_.next(iter))) {
+    while(OB_SUCC(ret) && NULL != (iter = (reverse ? hlist_.prev(iter) : hlist_.next(iter)))) {
       ret = f(iter);
     }
     return ret;
@@ -134,12 +145,23 @@ public:
     int64_t hazard = hlist_.hazard();
     return  hazard == INT64_MAX? 0: (arena_.allocated() - hazard);
   }
+  int64_t get_frozen_memstore_pos() const {
+    int64_t hazard = hlist_.hazard();
+    return  hazard == INT64_MAX? 0: hazard;
+  }
   int64_t get_mem_total_memstore_used() const { return arena_.hold(); }
   void log_frozen_memstore_info(char* buf, int64_t limit) {
     if (NULL != buf && limit > 0) {
       FrozenMemstoreInfoLogger logger(buf, limit);
       buf[0] = 0;
-      (void)for_each(logger);
+      (void)for_each(logger, true /* reverse  */);
+    }
+  }
+  void log_active_memstore_info(char *buf, int64_t limit) {
+    if (NULL != buf && limit > 0) {
+      ActiveMemstoreInfoLogger logger(buf, limit);
+      buf[0] = 0;
+      (void)for_each(logger, true /* reverse */);
     }
   }
 public:

src/share/allocator/ob_handle_list.h

@@ -45,6 +45,16 @@ public:
       }
       return iter;
     }
+    DLink* prev(DLink* iter) {
+      if (NULL == iter) {
+        iter = &tail_;
+      }
+      iter = (DLink*)iter->prev_;
+      if (&head_ == iter) {
+        iter = NULL;
+      }
+      return iter;
+    }
     static void dlink_insert(ObDLink* cur, ObDLink* x)
     {
       ObDLink* next = (ObDLink*)cur->next_;
@@ -129,6 +139,7 @@ public:
   bool is_empty() const { return ATOMIC_LOAD(&total_count_) <= 0; }
   int64_t hazard() const { return ATOMIC_LOAD(&hazard_); }
   DLink* next(DLink* iter) { return total_list_.next(iter); }
+  DLink* prev(DLink* iter) { return total_list_.prev(iter); }
 protected:
   void set_frozen_(Handle& handle);
   int64_t alloc_id() { return ATOMIC_AAF(&id_, 1); }

src/storage/tx_storage/ob_tenant_freezer.cpp

@@ -316,6 +316,7 @@ int ObTenantFreezer::tenant_freeze_()
   ObLSService *ls_srv = MTL(ObLSService *);
   FLOG_INFO("[TenantFreezer] tenant_freeze start", KR(ret));
 
+  ObTenantFreezeGuard freeze_guard(allocator_mgr_, ret);
   if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::TXSTORAGE_MOD))) {
     LOG_WARN("[TenantFreezer] fail to get log stream iterator", KR(ret));
   } else {
@@ -1362,6 +1363,8 @@ int ObTenantFreezer::do_major_if_need_(const bool need_freeze)
     // update frozen scn
   } else if (!need_freeze) {
     // no need major
+  } else if (!is_major_freeze_turn_()) {
+    // do nothing
   } else if (OB_FAIL(get_global_frozen_scn_(frozen_scn))) {
     LOG_WARN("fail to get global frozen version", K(ret));
   } else if (0 != frozen_scn && OB_FAIL(tenant_info_.update_frozen_scn(frozen_scn))) {

src/storage/tx_storage/ob_tenant_freezer.h

@@ -153,7 +153,7 @@ private:
                             const int64_t abs_timeout_ts);
   // freeze all the ls of this tenant.
   // return the first failed code.
-  static int tenant_freeze_();
+  int tenant_freeze_();
   // we can only deal with freeze one by one.
   // set tenant freezing will prevent a new freeze.
   int set_tenant_freezing_();

src/storage/tx_storage/ob_tenant_freezer_common.cpp

@@ -10,7 +10,14 @@
  * See the Mulan PubL v2 for more details.
  */
 
+#define USING_LOG_PREFIX STORAGE
+
+#include "lib/oblog/ob_log.h"
 #include "lib/alloc/alloc_func.h"
+#include "share/allocator/ob_gmemstore_allocator.h"
+#include "share/allocator/ob_memstore_allocator_mgr.h"
+#include "share/ob_force_print_log.h"
+#include "share/rc/ob_tenant_base.h"
 #include "storage/tx_storage/ob_tenant_freezer_common.h"
 
 namespace oceanbase
@@ -18,6 +25,7 @@ namespace oceanbase
 using namespace lib;
 namespace storage
 {
+typedef ObMemstoreAllocatorMgr::TAllocator ObTenantMemstoreAllocator;
 
 DEF_TO_STRING(ObTenantFreezeArg)
 {
@@ -144,5 +152,63 @@ void ObTenantInfo::get_freeze_ctx(ObTenantFreezeCtx &ctx) const
   ctx.mem_memstore_limit_ = mem_memstore_limit_;
 }
 
+ObTenantFreezeGuard::ObTenantFreezeGuard(common::ObMemstoreAllocatorMgr *allocator_mgr,
+                                         int &err_code,
+                                         const int64_t warn_threshold)
+  : allocator_mgr_(nullptr),
+    pre_retire_pos_(0),
+    error_code_(err_code),
+    time_guard_("FREEZE_CHECKER", warn_threshold)
+{
+  int ret = OB_SUCCESS;
+  ObTenantMemstoreAllocator *tenant_allocator = NULL;
+  const uint64_t tenant_id = MTL_ID();
+  if (OB_NOT_NULL(allocator_mgr)) {
+    allocator_mgr_ = allocator_mgr;
+    if (OB_FAIL(allocator_mgr_->get_tenant_memstore_allocator(tenant_id,
+                                                              tenant_allocator))) {
+      LOG_WARN("[FREEZE_CHECKER] failed to get_tenant_memstore_allocator", KR(ret), K(tenant_id));
+    } else if (NULL == tenant_allocator) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("[FREEZE_CHECKER] tenant memstore allocator is NULL", KR(ret), K(tenant_id));
+    } else {
+      pre_retire_pos_ = tenant_allocator->get_retire_clock();
+    }
+  }
+}
+
+ObTenantFreezeGuard::~ObTenantFreezeGuard()
+{
+  int ret = OB_SUCCESS;
+  ObTenantMemstoreAllocator *tenant_allocator = NULL;
+  const uint64_t tenant_id = MTL_ID();
+  int64_t curr_frozen_pos = 0;
+  if (OB_ISNULL(allocator_mgr_)) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_WARN("[FREEZE_CHECKER]freeze guard invalid", KR(ret), K_(allocator_mgr), K(lbt()));
+  } else if (OB_FAIL(error_code_)) {
+    LOG_WARN("[FREEZE_CHECKER]tenant freeze failed, skip check frozen memstore", KR(ret));
+  } else if (OB_FAIL(allocator_mgr_->get_tenant_memstore_allocator(tenant_id,
+                                                                   tenant_allocator))) {
+    LOG_WARN("[FREEZE_CHECKER] failed to get_tenant_memstore_allocator", KR(ret), K(tenant_id));
+  } else if (NULL == tenant_allocator) {
+    ret = OB_ERR_UNEXPECTED;
+    LOG_ERROR("[FREEZE_CHECKER] tenant memstore allocator is NULL", KR(ret), K(tenant_id));
+  } else {
+    curr_frozen_pos = tenant_allocator->get_frozen_memstore_pos();
+    const bool retired_mem_frozen = (curr_frozen_pos >= pre_retire_pos_);
+    const bool has_no_active_memtable = (curr_frozen_pos == 0);
+    if (!(retired_mem_frozen || has_no_active_memtable)) {
+      ret = OB_ERR_UNEXPECTED;
+      LOG_ERROR("[FREEZE_CHECKER]there may be frequent tenant freeze", KR(ret), K(curr_frozen_pos),
+                K_(pre_retire_pos), K(retired_mem_frozen), K(has_no_active_memtable));
+      char active_mt_info[DEFAULT_BUF_LENGTH];
+      tenant_allocator->log_active_memstore_info(active_mt_info,
+                                                 sizeof(active_mt_info));
+      FLOG_INFO("[FREEZE_CHECKER] oldest active memtable", "list", active_mt_info);
+    }
+  }
+}
+
 } // storage
 } // oceanbase

src/storage/tx_storage/ob_tenant_freezer_common.h

@@ -18,6 +18,10 @@
 #define OCEABASE_STORAGE_TENANT_FREEZER_COMMON_
 namespace oceanbase
 {
+namespace common
+{
+class ObMemstoreAllocatorMgr;
+}
 namespace storage
 {
 struct ObTenantFreezeArg
@@ -116,6 +120,20 @@ private:
   DISALLOW_COPY_AND_ASSIGN(ObTenantInfo);
 };
 
+class ObTenantFreezeGuard
+{
+public:
+  ObTenantFreezeGuard(common::ObMemstoreAllocatorMgr *allocator_mgr,
+                      int &ret,
+                      const int64_t warn_threshold = 60 * 1000 * 1000 /* 1 min */);
+  ~ObTenantFreezeGuard();
+private:
+  common::ObMemstoreAllocatorMgr *allocator_mgr_;
+  int64_t pre_retire_pos_;
+  int &error_code_;
+  ObTimeGuard time_guard_;
+};
+
 } // storage
 } // oceanbase
 #endif