hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Add free logic for tenant_mutil_allocator_mgr.

Browse Source
This commit is contained in:
jiadebinmary@gmail.com 2023-09-14 05:14:22 +00:00 committed by ob-robot
parent 30f03df840
commit 2bf2d711da
7 changed files with 244 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/logservice/ob_log_service.cpp
View File

@ -117,6 +117,17 @@ int ObLogService::mtl_init(ObLogService* &logservice)
return ret;
}
void ObLogService::mtl_destroy(ObLogService* &logservice)
{
common::ob_delete(logservice);
logservice = nullptr;
// Free tenant_log_allocator for this tenant after destroy logservice.
const int64_t tenant_id = MTL_ID();
int ret = OB_SUCCESS;
if (OB_FAIL(TMA_MGR_INSTANCE.delete_tenant_log_allocator(tenant_id))) {
CLOG_LOG(WARN, "delete_tenant_log_allocator failed", K(ret));
}
}
int ObLogService::start()
{
int ret = OB_SUCCESS;

1
src/logservice/ob_log_service.h
View File

@ -86,6 +86,7 @@ public:
ObLogService();
virtual ~ObLogService();
static int mtl_init(ObLogService* &logservice);
static void mtl_destroy(ObLogService* &logservice);
int start();
void stop();
void wait();

10
src/observer/omt/ob_multi_tenant.cpp
View File

@ -425,7 +425,7 @@ int ObMultiTenant::init(ObAddr myaddr,
MTL_BIND2(mtl_new_default, ObStorageLogger::mtl_init, ObStorageLogger::mtl_start, ObStorageLogger::mtl_stop, ObStorageLogger::mtl_wait, mtl_destroy_default);
MTL_BIND2(ObTenantMetaMemMgr::mtl_new, mtl_init_default, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObTransService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObLogService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObLogService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, ObLogService::mtl_destroy);
MTL_BIND2(mtl_new_default, logservice::ObGarbageCollector::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObLSService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObTenantCheckpointSlogHandler::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
@ -1646,14 +1646,6 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &remove_tenant_s
}
}
if (OB_SUCC(ret)) {
ObTenantMutilAllocator *allocator = nullptr;
if (OB_FAIL(TMA_MGR_INSTANCE.get_tenant_mutil_allocator(tenant_id, allocator))) {
LOG_ERROR("failed to get multi allocator", K(ret));
} else {
allocator->try_purge();
}
}
if (OB_SUCC(ret)) {
if (is_virtual_tenant_id(tenant_id) &&
OB_FAIL(ObVirtualTenantManager::get_instance().del_tenant(tenant_id))) {

149
src/share/allocator/ob_tenant_mutil_allocator_mgr.cpp
View File

@ -28,7 +28,7 @@ int ObTenantMutilAllocatorMgr::init()
if (is_inited_) {
ret = OB_INIT_TWICE;
} else {
for (int64_t i = 0; i < PRESERVED_TENANT_COUNT; ++i) {
for (int64_t i = 0; i < ARRAY_SIZE; ++i) {
tma_array_[i] = NULL;
}
is_inited_ = true;
@ -42,15 +42,31 @@ int ObTenantMutilAllocatorMgr::get_tenant_log_allocator(const uint64_t tenant_id
{
int ret = OB_SUCCESS;
ObTenantMutilAllocator *allocator = NULL;
if (OB_FAIL(get_tenant_mutil_allocator(tenant_id, allocator))) {
if (OB_FAIL(get_tenant_mutil_allocator_(tenant_id, allocator))) {
} else {
out_allocator = allocator;
}
return ret;
}
int ObTenantMutilAllocatorMgr::get_tenant_mutil_allocator(const uint64_t tenant_id,
ObTenantMutilAllocator *&out_allocator)
int ObTenantMutilAllocatorMgr::delete_tenant_log_allocator(const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
if (OB_FAIL(delete_tenant_mutil_allocator_(tenant_id))) {
OB_LOG(WARN, "delete_tenant_mutil_allocator_ failed", K(ret), K(tenant_id));
} else {
OB_LOG(INFO, "delete_tenant_mutil_allocator_ success", K(tenant_id));
}
return ret;
}
int64_t ObTenantMutilAllocatorMgr::get_slot_(const int64_t tenant_id) const
{
// The first slot by idx==0 won't be used.
return (tenant_id % PRESERVED_TENANT_COUNT) + 1;
}
int ObTenantMutilAllocatorMgr::get_tenant_mutil_allocator_(const uint64_t tenant_id,
TMA *&out_allocator)
{
int ret = OB_SUCCESS;
@ -59,16 +75,23 @@ int ObTenantMutilAllocatorMgr::get_tenant_mutil_allocator(const uint64_t tenant_
} else if (OB_UNLIKELY(tenant_id <= 0)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "invalid arguments", K(ret), K(tenant_id));
} else if (tenant_id < PRESERVED_TENANT_COUNT) {
// Don't need lock
if (NULL == (out_allocator = ATOMIC_LOAD(&tma_array_[tenant_id]))) {
} else if (tenant_id <= PRESERVED_TENANT_COUNT) {
// Need rlock
do {
obsys::ObRLockGuard guard(locks_[tenant_id]);
out_allocator = ATOMIC_LOAD(&tma_array_[tenant_id]);
} while(0);
if (NULL == out_allocator) {
// Need create new allocator
if (OB_FAIL(create_tenant_mutil_allocator_(tenant_id, out_allocator))) {
OB_LOG(WARN, "fail to create_tenant_mutil_allocator_", K(ret), K(tenant_id));
}
}
} else {
// Need lock
const int64_t slot = tenant_id % PRESERVED_TENANT_COUNT;
// slot must be > 0.
const int64_t slot = get_slot_(tenant_id);
bool is_need_create = false;
do {
// rdlock
@ -131,13 +154,14 @@ int ObTenantMutilAllocatorMgr::create_tenant_mutil_allocator_(const uint64_t ten
TMA *&out_allocator)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
} else if (OB_UNLIKELY(tenant_id <= 0)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "invalid arguments", K(ret), K(tenant_id));
} else if (tenant_id < PRESERVED_TENANT_COUNT) {
} else if (tenant_id <= PRESERVED_TENANT_COUNT) {
// wlock
obsys::ObWLockGuard guard(locks_[tenant_id]);
if (NULL != (out_allocator = ATOMIC_LOAD(&tma_array_[tenant_id]))) {
} else {
TMA *tmp_tma = NULL;
@ -154,26 +178,30 @@ int ObTenantMutilAllocatorMgr::create_tenant_mutil_allocator_(const uint64_t ten
}
}
} else {
const int64_t slot = tenant_id % PRESERVED_TENANT_COUNT;
if (NULL == ATOMIC_LOAD(&tma_array_[slot])) {
// slot's head node is NULL, need construct
TMA *tmp_tma = NULL;
if (OB_FAIL(construct_allocator_(slot, tmp_tma))) {
OB_LOG(WARN, "fail to construct_allocator_", K(ret), K(slot));
} else if (!ATOMIC_BCAS(&tma_array_[slot], NULL, tmp_tma)) {
if (NULL != tmp_tma) {
tmp_tma->~TMA();
ob_free(tmp_tma);
}
} else {}
}
// slot must be > 0.
const int64_t slot = get_slot_(tenant_id);
do {
// Need lock when modify slog list
obsys::ObWLockGuard guard(locks_[slot]);
if (NULL == ATOMIC_LOAD(&tma_array_[slot])) {
// slot's head node is NULL, need construct
TMA *tmp_tma = NULL;
if (OB_FAIL(construct_allocator_(slot, tmp_tma))) {
OB_LOG(WARN, "fail to construct_allocator_", K(ret), K(slot));
} else if (!ATOMIC_BCAS(&tma_array_[slot], NULL, tmp_tma)) {
if (NULL != tmp_tma) {
tmp_tma->~TMA();
ob_free(tmp_tma);
}
} else {}
}
// create tenant's allocator
if (OB_SUCC(ret)) {
bool is_need_create = false;
TMA **prev = NULL;
TMA **cur = &tma_array_[slot];
while ((NULL != cur) && (NULL != *cur) && (*cur)->get_tenant_id() < tenant_id) {
prev = cur;
cur = &((*cur)->get_next());
}
if (NULL != cur) {
@ -188,9 +216,17 @@ int ObTenantMutilAllocatorMgr::create_tenant_mutil_allocator_(const uint64_t ten
if (OB_FAIL(construct_allocator_(tenant_id, tmp_tma))) {
OB_LOG(WARN, "fail to construct_allocator_", K(ret), K(tenant_id));
} else {
OB_ASSERT(NULL != prev);
OB_ASSERT(NULL != (*prev));
OB_ASSERT(prev != cur);
// record cur's value(new next tma ptr)
TMA *next_allocator = *cur;
*cur = tmp_tma;
((*cur)->get_next()) = next_allocator;
// set cur to NULL
cur = NULL;
// Let prev->next_ points to new tma.
((*prev)->get_next()) = tmp_tma;
// Let tmp_tma->next_ points to old cur.
(tmp_tma->get_next()) = next_allocator;
out_allocator = tmp_tma;
}
}
@ -201,12 +237,60 @@ int ObTenantMutilAllocatorMgr::create_tenant_mutil_allocator_(const uint64_t ten
return ret;
}
int ObTenantMutilAllocatorMgr::delete_tenant_mutil_allocator_(const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
} else if (OB_UNLIKELY(tenant_id <= 0)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "invalid arguments", K(ret), K(tenant_id));
} else if (tenant_id <= PRESERVED_TENANT_COUNT) {
// Need wlock
obsys::ObWLockGuard guard(locks_[tenant_id]);
TMA *tma_allocator = NULL;
if (NULL != (tma_allocator = ATOMIC_LOAD(&tma_array_[tenant_id]))) {
if (NULL != tma_allocator->get_next()) {
OB_LOG(INFO, "next_ ptr is not NULL, skip deleting this allocator", K(ret), K(tenant_id));
} else {
tma_array_[tenant_id] = NULL;
tma_allocator->~TMA();
ob_free(tma_allocator);
tma_allocator = NULL;
}
}
} else {
// slot must be > 0.
const int64_t slot = get_slot_(tenant_id);
do {
// wlock
obsys::ObWLockGuard guard(locks_[slot]);
TMA *prev = NULL;
TMA *cur = tma_array_[slot];
while ((NULL != cur) && cur->get_tenant_id() < tenant_id) {
prev = cur;
cur = cur->get_next();
}
if (NULL != cur && cur->get_tenant_id() == tenant_id) {
OB_ASSERT(NULL != prev);
OB_ASSERT(prev != cur);
prev->get_next() = cur->get_next();
cur->get_next() = NULL;
ob_free(cur);
cur = NULL;
}
} while (0);
}
return ret;
}
ObTenantMutilAllocatorMgr &ObTenantMutilAllocatorMgr::get_instance()
{
static ObTenantMutilAllocatorMgr instance_;
return instance_;
}
/*
int ObTenantMutilAllocatorMgr::get_tenant_limit(const uint64_t tenant_id,
int64_t &limit)
{
@ -217,7 +301,7 @@ int ObTenantMutilAllocatorMgr::get_tenant_limit(const uint64_t tenant_id,
ret = OB_NOT_INIT;
} else if (OB_UNLIKELY(tenant_id <= 0)) {
ret = OB_INVALID_ARGUMENT;
} else if (OB_FAIL(get_tenant_mutil_allocator(tenant_id, allocator))) {
} else if (OB_FAIL(get_tenant_mutil_allocator_(tenant_id, allocator))) {
ret = OB_TENANT_NOT_EXIST;
} else {
limit = allocator->get_limit();
@ -236,7 +320,7 @@ int ObTenantMutilAllocatorMgr::set_tenant_limit(const uint64_t tenant_id,
ret = OB_NOT_INIT;
} else if (OB_UNLIKELY(tenant_id <= 0) || OB_UNLIKELY(new_limit <= 0)) {
ret = OB_INVALID_ARGUMENT;
} else if (OB_FAIL(get_tenant_mutil_allocator(tenant_id, allocator))) {
} else if (OB_FAIL(get_tenant_mutil_allocator_(tenant_id, allocator))) {
} else if (OB_ISNULL(allocator)) {
ret = OB_TENANT_NOT_EXIST;
} else {
@ -245,6 +329,7 @@ int ObTenantMutilAllocatorMgr::set_tenant_limit(const uint64_t tenant_id,
return ret;
}
*/
int ObTenantMutilAllocatorMgr::update_tenant_mem_limit(const share::TenantUnits &all_tenant_units)
{
// Update mem_limit for each tenant, called when the chane unit specifications or
@ -276,10 +361,10 @@ int ObTenantMutilAllocatorMgr::update_tenant_mem_limit(const share::TenantUnits
}
int tmp_ret = OB_SUCCESS;
ObTenantMutilAllocator *tma= NULL;
if (OB_SUCCESS != (tmp_ret = get_tenant_mutil_allocator(tenant_id, tma))) {
OB_LOG(WARN, "get_tenant_mutil_allocator failed", K(tmp_ret), K(tenant_id));
if (OB_SUCCESS != (tmp_ret = get_tenant_mutil_allocator_(tenant_id, tma))) {
OB_LOG(WARN, "get_tenant_mutil_allocator_ failed", K(tmp_ret), K(tenant_id));
} else if (NULL == tma) {
OB_LOG(WARN, "get_tenant_mutil_allocator failed", K(tenant_id));
OB_LOG(WARN, "get_tenant_mutil_allocator_ failed", K(tenant_id));
} else {
tma->set_nway(nway);
int64_t pre_tma_limit = tma->get_limit();
@ -294,7 +379,7 @@ int ObTenantMutilAllocatorMgr::update_tenant_mem_limit(const share::TenantUnits
ObGMemstoreAllocator* memstore_allocator = NULL;
if (OB_SUCCESS != (tmp_ret = ObMemstoreAllocatorMgr::get_instance().get_tenant_memstore_allocator(tenant_id, memstore_allocator))) {
} else if (OB_ISNULL(memstore_allocator)) {
OB_LOG(WARN, "get_tenant_mutil_allocator failed", K(tenant_id));
OB_LOG(WARN, "get_tenant_memstore_allocator failed", K(tenant_id));
} else if (OB_FAIL(memstore_allocator->set_memstore_threshold(tenant_id))) {
OB_LOG(WARN, "failed to set_memstore_threshold of memstore allocator", K(tenant_id), K(ret));
} else {

33
src/share/allocator/ob_tenant_mutil_allocator_mgr.h
View File

@ -37,37 +37,36 @@ public:
~ObTenantMutilAllocatorMgr()
{}
int init();
int get_tenant_mutil_allocator(const uint64_t tenant_id,
ObTenantMutilAllocator *&out_allocator);
// This interface is used by logservice module only.
int get_tenant_log_allocator(const uint64_t tenant_id,
ObILogAllocator *&out_allocator);
int get_tenant_limit(const uint64_t tenant_id, int64_t &limit);
int set_tenant_limit(const uint64_t tenant_id, const int64_t new_limit);
void *alloc_log_entry_buf(const int64_t size)
{
return clog_entry_alloc_.alloc(size);
}
void free_log_entry_buf(void *ptr)
{
if (NULL != ptr) {
clog_entry_alloc_.free(ptr);
}
}
// This interface is used by logservice module only.
int delete_tenant_log_allocator(const uint64_t tenant_id);
// int get_tenant_limit(const uint64_t tenant_id, int64_t &limit);
// int set_tenant_limit(const uint64_t tenant_id, const int64_t new_limit);
// a tricky interface, ugly but save memory
int update_tenant_mem_limit(const share::TenantUnits &all_tenant_units);
public:
static ObTenantMutilAllocatorMgr &get_instance();
private:
int64_t get_slot_(const int64_t tenant_id) const;
int get_tenant_mutil_allocator_(const uint64_t tenant_id, TMA *&out_allocator);
int delete_tenant_mutil_allocator_(const uint64_t tenant_id);
int construct_allocator_(const uint64_t tenant_id,
TMA *&out_allocator);
int create_tenant_mutil_allocator_(const uint64_t tenant_id,
TMA *&out_allocator);
private:
static const uint64_t PRESERVED_TENANT_COUNT = 10000;
// The sizeof(TMA) is about 130KB, so if the total number of tenants(including deleted ones)
// exceeds 1500, the memory used by TMA_MGR will be at least 65MB.
static const uint64_t PRESERVED_TENANT_COUNT = 1500;
static const uint64_t ARRAY_SIZE = PRESERVED_TENANT_COUNT + 1;
private:
bool is_inited_;
obsys::ObRWLock locks_[PRESERVED_TENANT_COUNT];
ObTenantMutilAllocator *tma_array_[PRESERVED_TENANT_COUNT];
obsys::ObRWLock locks_[ARRAY_SIZE];
ObTenantMutilAllocator *tma_array_[ARRAY_SIZE];
ObBlockAllocMgr clog_body_blk_alloc_;
ObVSliceAlloc clog_entry_alloc_;
private:

1
unittest/share/CMakeLists.txt
View File

@ -59,6 +59,7 @@ ob_unittest(test_ob_future)
ob_unittest(test_ob_occam_time_guard)
ob_unittest(test_cluster_version)
ob_unittest(test_scn)
ob_unittest(test_tma_mgr)
ob_unittest(test_geo_bin)
ob_unittest(test_s2adapter)
ob_unittest(test_geo_common)

97
unittest/share/test_tma_mgr.cpp Normal file
View File

@ -0,0 +1,97 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include "lib/ob_define.h"
#include "lib/random/ob_random.h"
#include "lib/alloc/ob_malloc_allocator.h"
#include "share/allocator/ob_tenant_mutil_allocator_mgr.h"
#include "share/allocator/ob_tenant_mutil_allocator.h"
#include "lib/oblog/ob_log.h"
#include "storage/ob_file_system_router.h"
namespace oceanbase
{
using namespace common;
namespace unittest
{
static const int64_t TENANT_CNT = 10000;
TEST(TestTMAMgr, test_tma_mgr)
{
ObTenantMutilAllocator *null_p = NULL;
ObTenantMutilAllocator **cur = &null_p;
PALF_LOG(INFO, "test_tma_mgr begin", "cur", (NULL == cur), "*cur", (NULL == (*cur)));
PALF_LOG(INFO, "test_tma_mgr begin", "TMA size", sizeof(ObTenantMutilAllocator));
ObMallocAllocator *malloc_allocator = ObMallocAllocator::get_instance();
ASSERT_EQ(OB_SUCCESS, TMA_MGR_INSTANCE.init());
int64_t tenant_id = 1;
for (tenant_id = 1; tenant_id <= TENANT_CNT; ++tenant_id) {
ObILogAllocator *tenant_allocator = NULL;
EXPECT_EQ(OB_SUCCESS, TMA_MGR_INSTANCE.get_tenant_log_allocator(tenant_id, tenant_allocator));
}
PALF_LOG(INFO, "after create all TMA", "TMA size", sizeof(ObTenantMutilAllocator), "500 tenant hold", \
malloc_allocator->get_tenant_hold(OB_SERVER_TENANT_ID));
// delete TMA by increasing order
for (tenant_id = 1; tenant_id <= TENANT_CNT; ++tenant_id) {
EXPECT_EQ(OB_SUCCESS, TMA_MGR_INSTANCE.delete_tenant_log_allocator(tenant_id));
}
PALF_LOG(INFO, "after delete all TMA", "TMA size", sizeof(ObTenantMutilAllocator), "500 tenant hold", \
malloc_allocator->get_tenant_hold(OB_SERVER_TENANT_ID));
for (tenant_id = 1; tenant_id <= TENANT_CNT; ++tenant_id) {
ObILogAllocator *tenant_allocator = NULL;
EXPECT_EQ(OB_SUCCESS, TMA_MGR_INSTANCE.get_tenant_log_allocator(tenant_id, tenant_allocator));
}
PALF_LOG(INFO, "after create all TMA", "TMA size", sizeof(ObTenantMutilAllocator), "500 tenant hold", \
malloc_allocator->get_tenant_hold(OB_SERVER_TENANT_ID));
// delete TMA by decreasing order
for (tenant_id = TENANT_CNT; tenant_id >= 1; --tenant_id) {
EXPECT_EQ(OB_SUCCESS, TMA_MGR_INSTANCE.delete_tenant_log_allocator(tenant_id));
}
PALF_LOG(INFO, "after delete all TMA", "TMA size", sizeof(ObTenantMutilAllocator), "500 tenant hold", \
malloc_allocator->get_tenant_hold(OB_SERVER_TENANT_ID));
for (tenant_id = 1; tenant_id <= TENANT_CNT; ++tenant_id) {
ObILogAllocator *tenant_allocator = NULL;
EXPECT_EQ(OB_SUCCESS, TMA_MGR_INSTANCE.get_tenant_log_allocator(tenant_id, tenant_allocator));
}
PALF_LOG(INFO, "after create all TMA", "TMA size", sizeof(ObTenantMutilAllocator), "500 tenant hold", \
malloc_allocator->get_tenant_hold(OB_SERVER_TENANT_ID));
ObRandom random;
// delete TMA by RANDOM order
for (int64_t loop_cnt = 1; loop_cnt <= TENANT_CNT / 2; ++loop_cnt) {
int64_t tmp_tenant_id = random.rand(1, TENANT_CNT);
EXPECT_EQ(OB_SUCCESS, TMA_MGR_INSTANCE.delete_tenant_log_allocator(tmp_tenant_id));
ObILogAllocator *tenant_allocator = NULL;
EXPECT_EQ(OB_SUCCESS, TMA_MGR_INSTANCE.get_tenant_log_allocator(tmp_tenant_id, tenant_allocator));
}
PALF_LOG(INFO, "after delete all TMA", "TMA size", sizeof(ObTenantMutilAllocator), "500 tenant hold", \
malloc_allocator->get_tenant_hold(OB_SERVER_TENANT_ID));
PALF_LOG(INFO, "test_tma_mgr end");
}
} // END of unittest
} // end of oceanbase
int main(int argc, char **argv)
{
system("rm -f ./test_tma_mgr.log*");
OB_LOGGER.set_file_name("test_tma_mgr.log", true);
OB_LOGGER.set_log_level("TRACE");
PALF_LOG(INFO, "begin unittest::test_tma_mgr");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}