From f457e4edea89a9721e5d690fa74d4639d4017381 Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 1 Mar 2024 12:44:39 +0000 Subject: [PATCH] BUGFIX: release memtable immediately if it is failed to push into gc queue --- deps/oblib/src/lib/ob_define.h | 1 + ...virtual_tenant_memstore_allocator_info.cpp | 12 ++++ ...l_virtual_tenant_memstore_allocator_info.h | 12 +++- .../ob_inner_table_schema.12001_12050.cpp | 30 ++++++++++ .../inner_table/ob_inner_table_schema_def.py | 4 +- .../meta_mem/ob_tenant_meta_mem_mgr.cpp | 59 ++++++++++++------- .../r/mysql/desc_virtual_table_in_sys.result | 2 + 7 files changed, 95 insertions(+), 25 deletions(-) diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index 01d2a7902e..5bd392752c 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -287,6 +287,7 @@ const int64_t OB_MAX_OBJECT_NAME_LENGTH = 128; //should include index_name const int64_t OB_MAX_ORIGINAL_NANE_LENGTH = 256; //max length of tenant_name, table_name, db_name const int64_t OB_MAX_CHAR_LEN = 3; +const int64_t OB_MAX_POINTER_ADDR_LEN = 32; const int64_t OB_MAX_TRIGGER_NAME_LENGTH = 128; // Compatible with Oracle const int64_t OB_MAX_WHEN_CONDITION_LENGTH = 4000; // Compatible with Oracle const int64_t OB_MAX_UPDATE_COLUMNS_LENGTH = 4000; // Compatible with Oracle diff --git a/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp b/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp index 319967936f..feab543543 100644 --- a/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp @@ -41,6 +41,8 @@ public: item.ls_id_ = (OB_SUCCESS == mt.get_ls_id(ls_id)) ? ls_id.id() : ObLSID::INVALID_LS_ID; item.tablet_id_ = mt.get_key().tablet_id_.id(); item.scn_range_ = mt.get_scn_range(); + item.mt_addr_ = &mt; + item.ref_cnt_ = mt.get_ref(); return array_.push_back(item); } ItemArray& array_; @@ -212,6 +214,16 @@ int ObAllVirtualTenantMemstoreAllocatorInfo::inner_get_next_row(ObNewRow *&row) cells[i].set_int(info.protection_clock_); break; } + case ADDRESS: { + snprintf(mt_addr_, sizeof(mt_addr_), "%p", info.mt_addr_); + cells[i].set_varchar(mt_addr_); + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + } + case REF_COUNT: { + cells[i].set_int(info.ref_cnt_); + break; + } default: { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "unexpected column id", K(col_id), K(i), K(ret)); diff --git a/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.h b/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.h index be333838b1..466b4616bc 100644 --- a/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.h +++ b/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.h @@ -27,14 +27,19 @@ struct ObMemstoreAllocatorInfo is_active_(false), ls_id_(OB_INVALID_ID), tablet_id_(OB_INVALID_ID), - scn_range_() {} + scn_range_(), + mt_addr_(NULL), + ref_cnt_(0) {} ~ObMemstoreAllocatorInfo() {} - TO_STRING_KV(K_(protection_clock), K_(is_active), K_(ls_id), K_(tablet_id), K_(scn_range)); + TO_STRING_KV(K_(protection_clock), K_(is_active), K_(ls_id), + K_(tablet_id), K_(scn_range), K_(mt_addr), K_(ref_cnt)); int64_t protection_clock_; bool is_active_; int64_t ls_id_; uint64_t tablet_id_; share::ObScnRange scn_range_; + memtable::ObMemtable *mt_addr_; + int64_t ref_cnt_; }; class ObAllVirtualTenantMemstoreAllocatorInfo : public common::ObVirtualTableIterator { @@ -58,6 +63,8 @@ private: IS_ACTIVE, RETIRE_CLOCK, PROTECTION_CLOCK, + ADDRESS, + REF_COUNT }; int fill_tenant_ids(); int fill_memstore_infos(const uint64_t tenant_id); @@ -67,6 +74,7 @@ private: int64_t tenant_ids_idx_; int64_t col_count_; int64_t retire_clock_; + char mt_addr_[32]; DISALLOW_COPY_AND_ASSIGN(ObAllVirtualTenantMemstoreAllocatorInfo); }; diff --git a/src/share/inner_table/ob_inner_table_schema.12001_12050.cpp b/src/share/inner_table/ob_inner_table_schema.12001_12050.cpp index 2f81357820..1b5b518796 100644 --- a/src/share/inner_table/ob_inner_table_schema.12001_12050.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12001_12050.cpp @@ -2623,6 +2623,36 @@ int ObInnerTableSchema::all_virtual_tenant_memstore_allocator_info_schema(ObTabl false, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("address", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + 32, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("ref_count", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 4223b133b8..3ace45fa27 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -9931,7 +9931,9 @@ def_table_schema( ('end_scn', 'uint'), ('is_active', 'varchar:MAX_COLUMN_YES_NO_LENGTH'), ('retire_clock', 'int'), - ('mt_protection_clock', 'int') + ('mt_protection_clock', 'int'), + ('address', 'varchar:OB_MAX_POINTER_ADDR_LEN'), + ('ref_count', 'int') ], partition_columns = ['svr_ip', 'svr_port'], diff --git a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp index d919986f87..eea9e187d9 100644 --- a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp +++ b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp @@ -408,6 +408,9 @@ int ObTenantMetaMemMgr::push_table_into_gc_queue(ObITable *table, const ObITable const int64_t size = sizeof(TableGCItem); const ObMemAttr attr(tenant_id_, "TableGCItem"); TableGCItem *item = nullptr; + static const int64_t SLEEP_TS = 100_ms; + static const int64_t MAX_RETRY_TIMES = 1000; // about 100s + int64_t retry_cnt = 0; if (OB_ISNULL(table)) { ret = OB_INVALID_ARGUMENT; @@ -418,32 +421,44 @@ int ObTenantMetaMemMgr::push_table_into_gc_queue(ObITable *table, const ObITable } else if (OB_UNLIKELY(ObITable::is_sstable(table_type))) { ret = OB_INVALID_ARGUMENT; LOG_ERROR("should not recycle sstable", K(ret), K(table_type), KPC(table)); - } else if (OB_ISNULL(item = (TableGCItem *)ob_malloc(size, attr))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("fail to allocate memory for TableGCItem", K(ret), K(size)); } else { - if (ObITable::TableType::DATA_MEMTABLE == table_type) { - ObMemtable *memtable = static_cast(table); - memtable::ObMtStat& mt_stat = memtable->get_mt_stat(); - if (0 == mt_stat.push_table_into_gc_queue_time_) { - mt_stat.push_table_into_gc_queue_time_ = ObTimeUtility::current_time(); - if (0 != mt_stat.release_time_ - && mt_stat.push_table_into_gc_queue_time_ - - mt_stat.release_time_ >= 10 * 1000 * 1000 /*10s*/) { - LOG_WARN("It cost too much time to dec ref cnt", K(ret), KPC(memtable), K(lbt())); + do { + if (OB_ISNULL(item = (TableGCItem *)ob_malloc(size, attr))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("fail to allocate memory for TableGCItem", K(ret), K(size)); + } else { + if (ObITable::TableType::DATA_MEMTABLE == table_type) { + ObMemtable *memtable = static_cast(table); + memtable::ObMtStat& mt_stat = memtable->get_mt_stat(); + if (0 == mt_stat.push_table_into_gc_queue_time_) { + mt_stat.push_table_into_gc_queue_time_ = ObTimeUtility::current_time(); + if (0 != mt_stat.release_time_ + && mt_stat.push_table_into_gc_queue_time_ - + mt_stat.release_time_ >= 10 * 1000 * 1000 /*10s*/) { + LOG_WARN("It cost too much time to dec ref cnt", K(ret), KPC(memtable), K(lbt())); + } + } + } + + item->table_ = table; + item->table_type_ = table_type; + if (OB_FAIL(free_tables_queue_.push((ObLink *)item))) { + LOG_ERROR("fail to push back into free_tables_queue_", K(ret), KPC(item)); } } - } - item->table_ = table; - item->table_type_ = table_type; - if (OB_FAIL(free_tables_queue_.push((ObLink *)item))) { - LOG_ERROR("fail to push back into free_tables_queue_", K(ret), KPC(item)); - } - } - - if (OB_FAIL(ret) && nullptr != item) { - ob_free(item); + if (OB_FAIL(ret) && nullptr != item) { + ob_free(item); + } + if (OB_ALLOCATE_MEMORY_FAILED == ret) { + ob_usleep(SLEEP_TS); + retry_cnt++; // retry some times + if (retry_cnt % 100 == 0) { + LOG_ERROR("push table into gc queue retry too many times", K(retry_cnt)); + } + } + } while (OB_ALLOCATE_MEMORY_FAILED == ret + && retry_cnt < MAX_RETRY_TIMES); } LOG_DEBUG("push table into gc queue", K(ret), KP(table), K(table_type), K(common::lbt())); return ret; diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index 0ef0ce51bb..1ed7960712 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -2158,6 +2158,8 @@ end_scn bigint(20) unsigned NO NULL is_active varchar(3) NO NULL retire_clock bigint(20) NO NULL mt_protection_clock bigint(20) NO NULL +address varchar(32) NO NULL +ref_count bigint(20) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_tenant_memstore_allocator_info; IF(count(*) >= 0, 1, 0) 1