BUGFIX: release memtable immediately if it is failed to push into gc queue

deps/oblib/src/lib/ob_define.h

@@ -287,6 +287,7 @@ const int64_t OB_MAX_OBJECT_NAME_LENGTH = 128; //should include index_name
 const int64_t OB_MAX_ORIGINAL_NANE_LENGTH = 256; //max length of tenant_name, table_name, db_name
 
 const int64_t OB_MAX_CHAR_LEN = 3;
+const int64_t OB_MAX_POINTER_ADDR_LEN = 32;
 const int64_t OB_MAX_TRIGGER_NAME_LENGTH = 128;  // Compatible with Oracle
 const int64_t OB_MAX_WHEN_CONDITION_LENGTH = 4000;  // Compatible with Oracle
 const int64_t OB_MAX_UPDATE_COLUMNS_LENGTH = 4000;  // Compatible with Oracle

src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp

@@ -41,6 +41,8 @@ public:
     item.ls_id_ = (OB_SUCCESS == mt.get_ls_id(ls_id)) ? ls_id.id() : ObLSID::INVALID_LS_ID;
     item.tablet_id_ = mt.get_key().tablet_id_.id();
     item.scn_range_ = mt.get_scn_range();
+    item.mt_addr_ = &mt;
+    item.ref_cnt_ = mt.get_ref();
     return array_.push_back(item);
   }
   ItemArray& array_;
@@ -212,6 +214,16 @@ int ObAllVirtualTenantMemstoreAllocatorInfo::inner_get_next_row(ObNewRow *&row)
               cells[i].set_int(info.protection_clock_);
               break;
             }
+            case ADDRESS: {
+              snprintf(mt_addr_, sizeof(mt_addr_), "%p", info.mt_addr_);
+              cells[i].set_varchar(mt_addr_);
+              cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
+              break;
+            }
+            case REF_COUNT: {
+              cells[i].set_int(info.ref_cnt_);
+              break;
+            }
             default: {
               ret = OB_ERR_UNEXPECTED;
               SERVER_LOG(WARN, "unexpected column id", K(col_id), K(i), K(ret));

src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.h

@@ -27,14 +27,19 @@ struct ObMemstoreAllocatorInfo
         is_active_(false),
         ls_id_(OB_INVALID_ID),
         tablet_id_(OB_INVALID_ID),
-        scn_range_() {}
+        scn_range_(),
+        mt_addr_(NULL),
+        ref_cnt_(0) {}
   ~ObMemstoreAllocatorInfo() {}
-  TO_STRING_KV(K_(protection_clock), K_(is_active), K_(ls_id), K_(tablet_id), K_(scn_range));
+  TO_STRING_KV(K_(protection_clock), K_(is_active), K_(ls_id),
+               K_(tablet_id), K_(scn_range), K_(mt_addr), K_(ref_cnt));
   int64_t protection_clock_;
   bool is_active_;
   int64_t ls_id_;
   uint64_t tablet_id_;
   share::ObScnRange scn_range_;
+  memtable::ObMemtable *mt_addr_;
+  int64_t ref_cnt_;
 };
 class ObAllVirtualTenantMemstoreAllocatorInfo : public common::ObVirtualTableIterator
 {
@@ -58,6 +63,8 @@ private:
     IS_ACTIVE,
     RETIRE_CLOCK,
     PROTECTION_CLOCK,
+    ADDRESS,
+    REF_COUNT
   };
   int fill_tenant_ids();
   int fill_memstore_infos(const uint64_t tenant_id);
@@ -67,6 +74,7 @@ private:
   int64_t tenant_ids_idx_;
   int64_t col_count_;
   int64_t retire_clock_;
+  char mt_addr_[32];
   DISALLOW_COPY_AND_ASSIGN(ObAllVirtualTenantMemstoreAllocatorInfo);
 };
 

src/share/inner_table/ob_inner_table_schema.12001_12050.cpp

@@ -2623,6 +2623,36 @@ int ObInnerTableSchema::all_virtual_tenant_memstore_allocator_info_schema(ObTabl
       false, //is_nullable
       false); //is_autoincrement
   }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("address", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObVarcharType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      32, //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("ref_count", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
   if (OB_SUCC(ret)) {
     table_schema.get_part_option().set_part_num(1);
     table_schema.set_part_level(PARTITION_LEVEL_ONE);

src/share/inner_table/ob_inner_table_schema_def.py

@@ -9931,7 +9931,9 @@ def_table_schema(
   ('end_scn', 'uint'),
   ('is_active', 'varchar:MAX_COLUMN_YES_NO_LENGTH'),
   ('retire_clock', 'int'),
-  ('mt_protection_clock', 'int')
+  ('mt_protection_clock', 'int'),
+  ('address', 'varchar:OB_MAX_POINTER_ADDR_LEN'),
+  ('ref_count', 'int')
   ],
 
   partition_columns = ['svr_ip', 'svr_port'],

src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp

@@ -408,6 +408,9 @@ int ObTenantMetaMemMgr::push_table_into_gc_queue(ObITable *table, const ObITable
   const int64_t size = sizeof(TableGCItem);
   const ObMemAttr attr(tenant_id_, "TableGCItem");
   TableGCItem *item = nullptr;
+  static const int64_t SLEEP_TS = 100_ms;
+  static const int64_t MAX_RETRY_TIMES = 1000; // about 100s
+  int64_t retry_cnt = 0;
 
   if (OB_ISNULL(table)) {
     ret = OB_INVALID_ARGUMENT;
@@ -418,32 +421,44 @@ int ObTenantMetaMemMgr::push_table_into_gc_queue(ObITable *table, const ObITable
   } else if (OB_UNLIKELY(ObITable::is_sstable(table_type))) {
     ret = OB_INVALID_ARGUMENT;
     LOG_ERROR("should not recycle sstable", K(ret), K(table_type), KPC(table));
-  } else if (OB_ISNULL(item = (TableGCItem *)ob_malloc(size, attr))) {
-    ret = OB_ALLOCATE_MEMORY_FAILED;
-    LOG_ERROR("fail to allocate memory for TableGCItem", K(ret), K(size));
   } else {
-    if (ObITable::TableType::DATA_MEMTABLE == table_type) {
+    do {
-      ObMemtable *memtable = static_cast<ObMemtable *>(table);
+      if (OB_ISNULL(item = (TableGCItem *)ob_malloc(size, attr))) {
-      memtable::ObMtStat& mt_stat = memtable->get_mt_stat();
+        ret = OB_ALLOCATE_MEMORY_FAILED;
-      if (0 == mt_stat.push_table_into_gc_queue_time_) {
+        LOG_ERROR("fail to allocate memory for TableGCItem", K(ret), K(size));
-        mt_stat.push_table_into_gc_queue_time_ = ObTimeUtility::current_time();
+      } else {
-        if (0 != mt_stat.release_time_
+        if (ObITable::TableType::DATA_MEMTABLE == table_type) {
-            && mt_stat.push_table_into_gc_queue_time_ -
+          ObMemtable *memtable = static_cast<ObMemtable *>(table);
-               mt_stat.release_time_ >= 10 * 1000 * 1000 /*10s*/) {
+          memtable::ObMtStat& mt_stat = memtable->get_mt_stat();
-          LOG_WARN("It cost too much time to dec ref cnt", K(ret), KPC(memtable), K(lbt()));
+          if (0 == mt_stat.push_table_into_gc_queue_time_) {
+            mt_stat.push_table_into_gc_queue_time_ = ObTimeUtility::current_time();
+            if (0 != mt_stat.release_time_
+                && mt_stat.push_table_into_gc_queue_time_ -
+                   mt_stat.release_time_ >= 10 * 1000 * 1000 /*10s*/) {
+              LOG_WARN("It cost too much time to dec ref cnt", K(ret), KPC(memtable), K(lbt()));
+            }
+          }
+        }
+
+        item->table_ = table;
+        item->table_type_ = table_type;
+        if (OB_FAIL(free_tables_queue_.push((ObLink *)item))) {
+          LOG_ERROR("fail to push back into free_tables_queue_", K(ret), KPC(item));
         }
       }
-    }
 
-    item->table_ = table;
+      if (OB_FAIL(ret) && nullptr != item) {
-    item->table_type_ = table_type;
+        ob_free(item);
-    if (OB_FAIL(free_tables_queue_.push((ObLink *)item))) {
+      }
-      LOG_ERROR("fail to push back into free_tables_queue_", K(ret), KPC(item));
+      if (OB_ALLOCATE_MEMORY_FAILED == ret) {
-    }
+        ob_usleep(SLEEP_TS);
-  }
+        retry_cnt++;               // retry some times
-
+        if (retry_cnt % 100 == 0) {
-  if (OB_FAIL(ret) && nullptr != item) {
+          LOG_ERROR("push table into gc queue retry too many times", K(retry_cnt));
-    ob_free(item);
+        }
+      }
+    } while (OB_ALLOCATE_MEMORY_FAILED == ret
+             && retry_cnt < MAX_RETRY_TIMES);
   }
   LOG_DEBUG("push table into gc queue", K(ret), KP(table), K(table_type), K(common::lbt()));
   return ret;

tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result

@@ -2158,6 +2158,8 @@ end_scn	bigint(20) unsigned	NO		NULL
 is_active	varchar(3)	NO		NULL	
 retire_clock	bigint(20)	NO		NULL	
 mt_protection_clock	bigint(20)	NO		NULL	
+address	varchar(32)	NO		NULL	
+ref_count	bigint(20)	NO		NULL	
 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_tenant_memstore_allocator_info;
 IF(count(*) >= 0, 1, 0)
 1