hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[memtable_mgr] ObTabletMemtableMgr memtable management

Browse Source
This commit is contained in:
obdev
2023-11-27 08:00:49 +00:00
committed by ob-robot
parent b7ea58d6e6
commit fdae2b1ab4
31 changed files with 837 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
unittest/storage/CMakeLists.txt
View File

@ -91,6 +91,7 @@ storage_dml_unittest(test_partition_range_splite)
storage_dml_unittest(test_major_rows_merger)
storage_dml_unittest(test_tablet tablet/test_tablet.cpp)
storage_unittest(test_medium_list_checker compaction/test_medium_list_checker.cpp)
storage_unittest(test_protected_memtable_mgr_handle test_protected_memtable_mgr_handle.cpp)
if(OB_BUILD_CLOSE_MODULES)
storage_dml_unittest(test_compaction_policy)

31
unittest/storage/test_compaction_policy.cpp
View File

@ -318,7 +318,8 @@ int TestCompactionPolicy::mock_memtable(
ObTableHandleV2 &table_handle)
{
int ret = OB_SUCCESS;
ObTabletMemtableMgr *mt_mgr = static_cast<ObTabletMemtableMgr *>(tablet.memtable_mgr_);
SCN clog_checkpoint_scn;
ObProtectedMemtableMgrHandle *protected_handle = NULL;
ObITable::TableKey table_key;
int64_t end_border = -1;
@ -334,7 +335,18 @@ int TestCompactionPolicy::mock_memtable(
ObLSService *ls_svr = nullptr;
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr *);
if (OB_FAIL(t3m->acquire_memtable(table_handle))) {
if (OB_FAIL(tablet.get_protected_memtable_mgr_handle(protected_handle))) {
LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), K(tablet));
}
// if memtable_mgr not exist, create it
else if (OB_FAIL(protected_handle->create_tablet_memtable_mgr_(
tablet.get_tablet_meta().ls_id_, tablet.get_tablet_meta().tablet_id_, lib::Worker::CompatMode::MYSQL))) {
LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret));
}
ObTabletMemtableMgr *mt_mgr = static_cast<ObTabletMemtableMgr *>(protected_handle->memtable_mgr_handle_.get_memtable_mgr());
if (OB_FAIL(ret)) {
} else if (OB_FAIL(t3m->acquire_memtable(table_handle))) {
LOG_WARN("failed to acquire memtable", K(ret));
} else if (OB_ISNULL(memtable = static_cast<ObMemtable*>(table_handle.get_table()))) {
ret = OB_ERR_UNEXPECTED;
@ -698,7 +710,6 @@ TEST_F(TestCompactionPolicy, basic_create_tablet)
ObTablet *tablet = tablet_handle.get_obj();
ObTabletTableStore &table_store = *tablet->table_store_addr_.get_ptr();
ASSERT_EQ(true, table_store.is_valid());
ASSERT_TRUE(nullptr != tablet->memtable_mgr_);
}
TEST_F(TestCompactionPolicy, basic_create_memtable)
@ -709,7 +720,13 @@ TEST_F(TestCompactionPolicy, basic_create_memtable)
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(true, tablet_handle.is_valid());
ObTabletMemtableMgr *mt_mgr = static_cast<ObTabletMemtableMgr *>(tablet_handle.get_obj()->memtable_mgr_);
SCN clog_checkpoint_scn;
ObProtectedMemtableMgrHandle *protected_handle = NULL;
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->get_protected_memtable_mgr_handle(protected_handle));
// if memtable_mgr not exist, create it
const ObTabletMeta &tablet_meta = tablet_handle.get_obj()->get_tablet_meta();
ASSERT_EQ(OB_SUCCESS, protected_handle->create_tablet_memtable_mgr_(tablet_meta.ls_id_, tablet_meta.tablet_id_, lib::Worker::CompatMode::MYSQL));
ObTabletMemtableMgr *mt_mgr = static_cast<ObTabletMemtableMgr *>(protected_handle->memtable_mgr_handle_.get_memtable_mgr());
ASSERT_EQ(0, mt_mgr->get_memtable_count_());
ObTableHandleV2 frozen_memtable;
ret = TestCompactionPolicy::mock_memtable(1, 100, 100, *tablet_handle.get_obj(), frozen_memtable);
@ -811,7 +828,9 @@ TEST_F(TestCompactionPolicy, basic_prepare_tablet)
ASSERT_EQ(2, table_store.major_tables_.count());
ASSERT_EQ(2, table_store.minor_tables_.count());
ObTabletMemtableMgr *mt_mgr = static_cast<ObTabletMemtableMgr *>(tablet_handle_.get_obj()->memtable_mgr_);
ObProtectedMemtableMgrHandle *protected_handle = NULL;
ASSERT_EQ(OB_SUCCESS, tablet_handle_.get_obj()->get_protected_memtable_mgr_handle(protected_handle));
ObTabletMemtableMgr *mt_mgr = static_cast<ObTabletMemtableMgr *>(protected_handle->memtable_mgr_handle_.get_memtable_mgr());
ASSERT_EQ(2, mt_mgr->get_memtable_count_());
}
@ -1085,7 +1104,7 @@ TEST_F(TestCompactionPolicy, test_minor_dag_intersect)
int main(int argc, char **argv)
{
system("rm -rf test_compaction_policy.log*");
OB_LOGGER.set_file_name("test_compaction_policy.log");
OB_LOGGER.set_file_name("test_compaction_policy.log", true);
OB_LOGGER.set_log_level("INFO");
CLOG_LOG(INFO, "begin unittest: test_compaction_policy");
::testing::InitGoogleTest(&argc, argv);

120
unittest/storage/test_protected_memtable_mgr_handle.cpp Normal file
View File

@ -0,0 +1,120 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "share/ob_errno.h"
#include <gtest/gtest.h>
#include "storage/test_tablet_helper.h"
#include <lib/oblog/ob_log.h>
#include "storage/tx_storage/ob_ls_service.h"
#include "mtlenv/mock_tenant_module_env.h"
#include "init_basic_struct.h"
#define USING_LOG_PREFIX STORAGE
#define UNITTEST
namespace oceanbase
{
namespace storage
{
namespace checkpoint
{
class TestTabletMemtableMgr : public ::testing::Test
{
public:
TestTabletMemtableMgr() {}
virtual ~TestTabletMemtableMgr() = default;
static void SetUpTestCase()
{
EXPECT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init());
ObServerCheckpointSlogHandler::get_instance().is_started_ = true;
}
static void TearDownTestCase()
{
MockTenantModuleEnv::get_instance().destroy();
}
};
TEST_F(TestTabletMemtableMgr, tablet_memtable_mgr) {
ObArenaAllocator allocator;
ObLSID ls_id(2000);
ObTabletID tablet_id(500000);
ObLSHandle ls_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ObTabletMemtableMgrPool *pool = MTL(ObTabletMemtableMgrPool*);
ObCreateLSArg arg;
ObTabletHandle tablet_handle;
share::schema::ObTableSchema table_schema;
share::SCN scn1;
scn1.set_base();
share::SCN scn2 = share::SCN::scn_inc(scn1);
uint64_t table_id = 12345;
ObProtectedMemtableMgrHandle *protected_handle = NULL;
ObTableHandleV2 handle;
ASSERT_EQ(OB_SUCCESS, gen_create_ls_arg(1, ls_id, arg));
ASSERT_EQ(OB_SUCCESS, MTL(ObLSService*)->create_ls(arg));
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
ASSERT_EQ(OB_SUCCESS, build_test_schema(table_schema, table_id));
ASSERT_EQ(OB_SUCCESS, TestTabletHelper::create_tablet(ls_handle, tablet_id, table_schema, allocator));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->get_tablet(tablet_id, tablet_handle));
ASSERT_EQ(true, tablet_handle.is_valid());
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->get_protected_memtable_mgr_handle(protected_handle));
// memtable_mgr not exist
ASSERT_EQ(OB_ENTRY_NOT_EXIST, protected_handle->get_active_memtable(handle));
ASSERT_EQ(0, pool->count_);
// create memtable
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->create_memtable(1, scn1));
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->create_memtable(2, scn2));
// memtable_mgr exist
ASSERT_EQ(OB_SUCCESS, protected_handle->get_active_memtable(handle));
ASSERT_EQ(1, pool->count_);
// release a memtable, memtable count is 1
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->release_memtables(scn1));
// memtable_mgr exist yet
ASSERT_EQ(OB_SUCCESS, protected_handle->get_active_memtable(handle));
ASSERT_EQ(1, pool->count_);
// release other memtable, memtable count is 0, so release tablet_mgr
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->release_memtables());
// memtable_mgr not exist
ASSERT_EQ(OB_ENTRY_NOT_EXIST, protected_handle->get_active_memtable(handle));
ASSERT_EQ(0, pool->count_);
ASSERT_EQ(OB_SUCCESS, MTL(ObLSService*)->remove_ls(ls_id, false));
}
}
} // end namespace storage
} // end namespace oceanbase
int main(int argc, char **argv)
{
int ret = 0;
system("rm -f test_checkpoint_diagnose.log*");
OB_LOGGER.set_file_name("test_tablet_memtable_mgr.log", true);
OB_LOGGER.set_log_level("INFO");
signal(49, SIG_IGN);
testing::InitGoogleTest(&argc, argv);
ret = RUN_ALL_TESTS();
return ret;
}

20
unittest/storage/test_tablet_pointer_map.cpp
View File

@ -84,7 +84,9 @@ void TestMetaPointerMap::SetUp()
ObTenantMetaMemMgr *t3m = OB_NEW(ObTenantMetaMemMgr, ObModIds::TEST, TEST_TENANT_ID);
ASSERT_EQ(OB_SUCCESS, t3m->init());
ObTabletMemtableMgrPool *pool = OB_NEW(ObTabletMemtableMgrPool, ObModIds::TEST);
tenant_base_.set(t3m);
tenant_base_.set(pool);
ObTenantEnv::set_tenant(&tenant_base_);
ASSERT_EQ(OB_SUCCESS, tenant_base_.init());
}
@ -147,11 +149,11 @@ TEST_F(TestMetaPointerMap, test_meta_pointer_handle)
int ret = tablet_svr->init(&fake_ls);
ASSERT_EQ(common::OB_SUCCESS, ret);
ObMemtableMgrHandle memtable_mgr_hdl;
ObDDLKvMgrHandle ddl_kv_mgr_hdl;
ret = MTL(ObTenantMetaMemMgr*)->acquire_tablet_memtable_mgr(memtable_mgr_hdl);
ASSERT_EQ(common::OB_SUCCESS, ret);
ObTabletMemtableMgr *ptr = MTL(ObTabletMemtableMgrPool*)->acquire();
OB_ASSERT(NULL != ptr);
ObMemtableMgrHandle memtable_mgr_hdl(ptr, MTL(ObTabletMemtableMgrPool*));
ret = MTL(ObTenantMetaMemMgr*)->acquire_tablet_ddl_kv_mgr(ddl_kv_mgr_hdl);
ASSERT_EQ(common::OB_SUCCESS, ret);
@ -210,11 +212,11 @@ TEST_F(TestMetaPointerMap, test_meta_pointer_map)
int ret = tablet_svr->init(&fake_ls);
ASSERT_EQ(common::OB_SUCCESS, ret);
ObMemtableMgrHandle memtable_mgr_hdl;
ObDDLKvMgrHandle ddl_kv_mgr_hdl;
ret = MTL(ObTenantMetaMemMgr*)->acquire_tablet_memtable_mgr(memtable_mgr_hdl);
ASSERT_EQ(common::OB_SUCCESS, ret);
ObTabletMemtableMgr *ptr = MTL(ObTabletMemtableMgrPool*)->acquire();
OB_ASSERT(NULL != ptr);
ObMemtableMgrHandle memtable_mgr_hdl(ptr, MTL(ObTabletMemtableMgrPool*));
ret = MTL(ObTenantMetaMemMgr*)->acquire_tablet_ddl_kv_mgr(ddl_kv_mgr_hdl);
ASSERT_EQ(common::OB_SUCCESS, ret);
@ -293,11 +295,11 @@ TEST_F(TestMetaPointerMap, test_erase_and_load_concurrency)
int ret = tablet_svr->init(&fake_ls);
ASSERT_EQ(common::OB_SUCCESS, ret);
ObMemtableMgrHandle memtable_mgr_hdl;
ObDDLKvMgrHandle ddl_kv_mgr_hdl;
ret = MTL(ObTenantMetaMemMgr*)->acquire_tablet_memtable_mgr(memtable_mgr_hdl);
ASSERT_EQ(common::OB_SUCCESS, ret);
ObTabletMemtableMgr *ptr = MTL(ObTabletMemtableMgrPool*)->acquire();
OB_ASSERT(NULL != ptr);
ObMemtableMgrHandle memtable_mgr_hdl(ptr, MTL(ObTabletMemtableMgrPool*));
ret = MTL(ObTenantMetaMemMgr*)->acquire_tablet_ddl_kv_mgr(ddl_kv_mgr_hdl);
ASSERT_EQ(common::OB_SUCCESS, ret);