hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FEAT MERGE] implement Medium Compaction & adaptive Compaction Strategy

Browse Source
This commit is contained in:
obdev
2022-12-19 13:26:58 +00:00
committed by ob-robot
parent 5c19d8c8c7
commit c94062401a
177 changed files with 12721 additions and 3494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
unittest/storage/CMakeLists.txt
View File

@ -31,7 +31,8 @@ storage_unittest(test_parallel_external_sort)
storage_unittest(test_i_store)
storage_unittest(test_sstable_merge_info_mgr)
#storage_unittest(test_row_sample_iterator)
storage_unittest(test_table_store_stat_mgr)
#storage_unittest(test_table_store_stat_mgr)
storage_unittest(test_tenant_tablet_stat_mgr)
#storage_unittest(test_dag_size)
storage_unittest(test_handle_cache)
#storage_unittest(test_log_replay_engine replayengine/test_log_replay_engine.cpp)
@ -69,8 +70,10 @@ storage_unittest(test_backup_extern_info_mgr backup/test_backup_extern_info_mgr.
storage_unittest(test_simple_rows_merger)
storage_unittest(test_partition_incremental_range_spliter)
storage_unittest(test_partition_major_sstable_range_spliter)
storage_unittest(test_parallel_minor_dag)
storage_dml_unittest(test_major_rows_merger)
#storage_dml_unittest(test_table_scan_pure_index_table)
storage_unittest(test_sstable_log_ts_range_cut test_sstable_log_ts_range_cut.cpp)
storage_unittest(test_medium_compaction_mgr test_medium_compaction_mgr.cpp)

17
unittest/storage/blocksstable/ob_multi_version_sstable_test.h
View File

@ -164,7 +164,7 @@ public:
const int64_t micro_cnt,
const int64_t max_merged_trans_version = INT64_MAX - 2,
const bool contain_uncommitted = false);
void prepare_data_end(ObTableHandleV2 &handle);
void prepare_data_end(ObTableHandleV2 &handle, const ObITable::TableType &table_type = ObITable::MINI_SSTABLE);
void append_micro_block(ObMockIterator &data_iter);
protected:
@ -294,13 +294,13 @@ ObITable::TableType ObMultiVersionSSTableTest::get_merged_table_type() const
ObITable::TableType table_type = ObITable::MAX_TABLE_TYPE;
if (MAJOR_MERGE == merge_type_) {
table_type = ObITable::TableType::MAJOR_SSTABLE;
} else if (MINI_MERGE == merge_type_ || MINI_MINOR_MERGE == merge_type_) {
} else if (MINI_MERGE == merge_type_) {
table_type = ObITable::TableType::MINI_SSTABLE;
} else if (BUF_MINOR_MERGE == merge_type_) {
table_type = ObITable::TableType::BUF_MINOR_SSTABLE;
} else if (META_MAJOR_MERGE == merge_type_) {
table_type = ObITable::TableType::META_MAJOR_SSTABLE;
} else if (DDL_KV_MERGE == merge_type_) {
table_type = ObITable::TableType::KV_DUMP_SSTABLE;
} else { // MINOR_MERGE || HISTORY_MINI_MINOR_MERGE
} else { // MINOR_MERGE
table_type = ObITable::TableType::MINOR_SSTABLE;
}
return table_type;
@ -509,7 +509,9 @@ void ObMultiVersionSSTableTest::append_micro_block(ObMockIterator &data_iter)
}
}
void ObMultiVersionSSTableTest::prepare_data_end(ObTableHandleV2 &handle)
void ObMultiVersionSSTableTest::prepare_data_end(
ObTableHandleV2 &handle,
const ObITable::TableType &table_type)
{
ASSERT_EQ(OB_SUCCESS, macro_writer_.close());
ObSSTableMergeRes res;
@ -518,6 +520,7 @@ void ObMultiVersionSSTableTest::prepare_data_end(ObTableHandleV2 &handle)
ASSERT_EQ(OB_SUCCESS, root_index_builder_->close(column_cnt, res));
ObTabletCreateSSTableParam param;
table_key_.table_type_ = table_type;
param.table_key_ = table_key_;
param.schema_version_ = SCHEMA_VERSION;
param.create_snapshot_version_ = 0;
@ -548,7 +551,7 @@ void ObMultiVersionSSTableTest::prepare_data_end(ObTableHandleV2 &handle)
param.nested_size_ = res.nested_size_;
param.nested_offset_ = res.nested_offset_;
param.ddl_scn_.set_min();
if (merge_type_ == MAJOR_MERGE) {
if (table_type == ObITable::MAJOR_SSTABLE) {
ASSERT_EQ(OB_SUCCESS, ObSSTableMergeRes::fill_column_checksum_for_empty_major(param.column_cnt_, param.column_checksums_));
}

2
unittest/storage/blocksstable/test_row_reader.cpp
View File

@ -774,7 +774,7 @@ TEST_F(TestNewRowReader, test_macro_block)
ObDataStoreDesc desc;
int64_t data_version = 1;
ret = desc.init(table_schema_, data_version, 1, MINI_MINOR_MERGE, true);
ret = desc.init(table_schema_, data_version, 1, MINOR_MERGE, true);
ASSERT_EQ(OB_SUCCESS, ret);
ret = writer.open(desc, start_seq);
ASSERT_EQ(OB_SUCCESS, ret);

211
unittest/storage/test_compaction_policy.cpp
View File

@ -106,6 +106,15 @@ public:
common::ObIArray<ObTenantFreezeInfoMgr::FreezeInfo> &freeze_infos,
common::ObIArray<share::ObSnapshotInfo> &snapshots);
void prepare_schema(share::schema::ObTableSchema &table_schema);
int prepare_medium_list(
const char *snapshot_list,
ObTabletHandle &tablet_handle);
int construct_array(
const char *snapshot_list,
ObIArray<int64_t> &array);
int check_result_tables_handle(const char *end_log_ts_list, const ObGetMergeTablesResult &result);
public:
TestCompactionPolicy();
~TestCompactionPolicy() = default;
@ -129,6 +138,9 @@ public:
ObSEArray<ObTableHandleV2, 4> major_tables_;
ObSEArray<ObTableHandleV2, 4> minor_tables_;
ObSEArray<ObTableHandleV2, 4> memtables_;
ObMediumCompactionInfo medium_info_;
ObSEArray<int64_t, 10> array_;
ObArenaAllocator allocator_;
};
TestCompactionPolicy::TestCompactionPolicy()
@ -151,6 +163,15 @@ void TestCompactionPolicy::SetUp()
t3m->destroy();
ret = t3m->init();
ASSERT_EQ(OB_SUCCESS, ret);
share::schema::ObTableSchema table_schema;
prepare_schema(table_schema);
medium_info_.compaction_type_ = ObMediumCompactionInfo::MEDIUM_COMPACTION;
medium_info_.medium_snapshot_ = 100;
medium_info_.medium_scn_.convert_for_tx(100);
medium_info_.storage_schema_.init(allocator_, table_schema, lib::Worker::CompatMode::MYSQL);
}
void TestCompactionPolicy::TearDown()
@ -368,6 +389,64 @@ int TestCompactionPolicy::mock_tablet(
return ret;
}
int TestCompactionPolicy::construct_array(
const char *snapshot_list,
ObIArray<int64_t> &array)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(snapshot_list)) {
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid argument", K(ret), K(snapshot_list));
} else {
array.reset();
std::string copy(snapshot_list);
char *org = const_cast<char *>(copy.c_str());
static const char *delim = " ";
char *s = std::strtok(org, delim);
if (NULL != s) {
array.push_back(atoi(s));
while (NULL != (s= strtok(NULL, delim))) {
array.push_back(atoi(s));
}
}
}
return ret;
}
int TestCompactionPolicy::prepare_medium_list(
const char *snapshot_list,
ObTabletHandle &tablet_handle)
{
int ret = OB_SUCCESS;
ObTablet &tablet = *tablet_handle.get_obj();
construct_array(snapshot_list, array_);
tablet.medium_info_list_.reset_list();
for (int i = 0; OB_SUCC(ret) && i < array_.count(); ++i) {
medium_info_.medium_snapshot_ = array_.at(i);
ret = tablet.medium_info_list_.add_medium_compaction_info(medium_info_);
}
return ret;
}
int TestCompactionPolicy::check_result_tables_handle(
const char *end_log_ts_list,
const ObGetMergeTablesResult &result)
{
int ret = OB_SUCCESS;
construct_array(end_log_ts_list, array_);
if (array_.count() != result.handle_.get_count()) {
ret = OB_ERR_UNEXPECTED;
COMMON_LOG(WARN, "table count is not equal", K(ret), K(array_), K(result.handle_));
}
for (int i = 0; OB_SUCC(ret) && i < array_.count(); ++i) {
if (array_.at(i) != result.handle_.get_table(i)->get_end_scn().get_val_for_tx()) {
ret = OB_ERR_UNEXPECTED;
COMMON_LOG(WARN, "table is not equal", K(ret), K(i), K(array_.at(i)), KPC(result.handle_.get_table(i)));
}
}
return ret;
}
int TestCompactionPolicy::mock_table_store(
ObTabletHandle &tablet_handle,
common::ObIArray<ObTableHandleV2> &major_table_handles,
@ -547,6 +626,72 @@ int TestCompactionPolicy::prepare_freeze_info(
return ret;
}
class FakeLS : public storage::ObLS
{
public:
FakeLS() {
ls_meta_.tenant_id_ = 1001;
ls_meta_.ls_id_ = ObLSID(100);
}
int64_t get_min_reserved_snapshot() { return 10; }
};
static const int64_t TENANT_ID = 1;
static const int64_t TABLE_ID = 7777;
static const int64_t TEST_ROWKEY_COLUMN_CNT = 3;
static const int64_t TEST_COLUMN_CNT = 6;
void TestCompactionPolicy::prepare_schema(share::schema::ObTableSchema &table_schema)
{
int ret = OB_SUCCESS;
int64_t micro_block_size = 16 * 1024;
const uint64_t tenant_id = TENANT_ID;
const uint64_t table_id = TABLE_ID;
share::schema::ObColumnSchemaV2 column;
//generate data table schema
table_schema.reset();
ret = table_schema.set_table_name("test_merge_multi_version");
ASSERT_EQ(OB_SUCCESS, ret);
table_schema.set_tenant_id(tenant_id);
table_schema.set_tablegroup_id(1);
table_schema.set_database_id(1);
table_schema.set_table_id(table_id);
table_schema.set_rowkey_column_num(TEST_ROWKEY_COLUMN_CNT);
table_schema.set_max_used_column_id(TEST_COLUMN_CNT);
table_schema.set_block_size(micro_block_size);
table_schema.set_compress_func_name("none");
table_schema.set_row_store_type(FLAT_ROW_STORE);
//init column
char name[OB_MAX_FILE_NAME_LENGTH];
memset(name, 0, sizeof(name));
const int64_t column_ids[] = {16,17,20,21,22,23,24,29};
for(int64_t i = 0; i < TEST_COLUMN_CNT; ++i){
ObObjType obj_type = ObIntType;
const int64_t column_id = column_ids[i];
if (i == 1) {
obj_type = ObVarcharType;
}
column.reset();
column.set_table_id(table_id);
column.set_column_id(column_id);
sprintf(name, "test%020ld", i);
ASSERT_EQ(OB_SUCCESS, column.set_column_name(name));
column.set_data_type(obj_type);
column.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI);
column.set_data_length(10);
if (i < TEST_ROWKEY_COLUMN_CNT) {
column.set_rowkey_position(i + 1);
} else {
column.set_rowkey_position(0);
}
COMMON_LOG(INFO, "add column", K(i), K(column));
ASSERT_EQ(OB_SUCCESS, table_schema.add_column(column));
}
COMMON_LOG(INFO, "dump stable schema", LITERAL_K(TEST_ROWKEY_COLUMN_CNT), K(table_schema));
}
TEST_F(TestCompactionPolicy, basic_create_sstable)
{
@ -720,12 +865,13 @@ TEST_F(TestCompactionPolicy, check_mini_merge_basic)
ret = prepare_tablet(key_data, 150, 150);
ASSERT_EQ(OB_SUCCESS, ret);
FakeLS ls;
ObGetMergeTablesParam param;
param.merge_type_ = ObMergeType::MINI_MERGE;
ObGetMergeTablesResult result;
tablet_handle_.get_obj()->tablet_meta_.clog_checkpoint_scn_.convert_for_tx(300);
tablet_handle_.get_obj()->tablet_meta_.snapshot_version_ = 300;
ret = ObPartitionMergePolicy::get_mini_merge_tables(param, 0, *tablet_handle_.get_obj(), result);
ret = ObPartitionMergePolicy::get_mini_merge_tables(param, ls, *tablet_handle_.get_obj(), result);
ASSERT_EQ(OB_NO_NEED_MERGE, ret);
ASSERT_EQ(result.update_tablet_directly_, false);
}
@ -756,9 +902,10 @@ TEST_F(TestCompactionPolicy, check_minor_merge_basic)
ASSERT_EQ(OB_SUCCESS, ret);
ObGetMergeTablesParam param;
param.merge_type_ = ObMergeType::MINI_MINOR_MERGE;
param.merge_type_ = ObMergeType::MINOR_MERGE;
ObGetMergeTablesResult result;
ret = ObPartitionMergePolicy::get_mini_minor_merge_tables(param, 0, *tablet_handle_.get_obj(), result);
FakeLS ls;
ret = ObPartitionMergePolicy::get_minor_merge_tables(param, ls, *tablet_handle_.get_obj(), result);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(5, result.handle_.get_count());
}
@ -792,9 +939,10 @@ TEST_F(TestCompactionPolicy, check_no_need_minor_merge)
ASSERT_EQ(OB_SUCCESS, ret);
ObGetMergeTablesParam param;
param.merge_type_ = ObMergeType::MINI_MINOR_MERGE;
param.merge_type_ = ObMergeType::MINOR_MERGE;
ObGetMergeTablesResult result;
ret = ObPartitionMergePolicy::get_mini_minor_merge_tables(param, 0, *tablet_handle_.get_obj(), result);
FakeLS ls;
ret = ObPartitionMergePolicy::get_minor_merge_tables(param, ls, *tablet_handle_.get_obj(), result);
ASSERT_EQ(OB_NO_NEED_MERGE, ret);
}
@ -828,7 +976,8 @@ TEST_F(TestCompactionPolicy, check_major_merge_basic)
param.merge_type_ = ObMergeType::MAJOR_MERGE;
param.merge_version_ = 340;
ObGetMergeTablesResult result;
ret = ObPartitionMergePolicy::get_major_merge_tables(param, 0, *tablet_handle_.get_obj(), result);
FakeLS ls;
ret = ObPartitionMergePolicy::get_medium_merge_tables(param, ls, *tablet_handle_.get_obj(), result);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(6, result.handle_.get_count());
}
@ -863,10 +1012,58 @@ TEST_F(TestCompactionPolicy, check_no_need_major_merge)
param.merge_type_ = ObMergeType::MAJOR_MERGE;
param.merge_version_ = 340;
ObGetMergeTablesResult result;
ret = ObPartitionMergePolicy::get_major_merge_tables(param, 0, *tablet_handle_.get_obj(), result);
FakeLS ls;
ret = ObPartitionMergePolicy::get_medium_merge_tables(param, ls, *tablet_handle_.get_obj(), result);
ASSERT_EQ(OB_NO_NEED_MERGE, ret);
}
TEST_F(TestCompactionPolicy, test_minor_with_medium)
{
int ret = OB_SUCCESS;
ObTenantFreezeInfoMgr *mgr = MTL(ObTenantFreezeInfoMgr *);
ASSERT_TRUE(nullptr != mgr);
common::ObArray<ObTenantFreezeInfoMgr::FreezeInfo> freeze_info;
common::ObArray<share::ObSnapshotInfo> snapshots;
share::SCN scn;
ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(1, 1, 0)));
ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(140, 1, 0)));
ret = TestCompactionPolicy::prepare_freeze_info(500, freeze_info, snapshots);
ASSERT_EQ(OB_SUCCESS, ret);
const char *key_data =
"table_type start_scn end_scn max_ver upper_ver\n"
"10 0 1 1 1 \n"
"11 150 200 200 200 \n"
"11 200 250 250 250 \n"
"11 250 300 300 300 \n"
"11 300 340 340 340 \n";
ret = prepare_tablet(key_data, 340, 340);
ASSERT_EQ(OB_SUCCESS, ret);
ObGetMergeTablesParam param;
param.merge_type_ = ObMergeType::MINOR_MERGE;
param.merge_version_ = 0;
ObGetMergeTablesResult result;
FakeLS ls;
prepare_medium_list("240", tablet_handle_);
ret = ObPartitionMergePolicy::get_minor_merge_tables(param, ls, *tablet_handle_.get_obj(), result);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, check_result_tables_handle("250, 300, 340", result));
prepare_medium_list("150", tablet_handle_);
ret = ObPartitionMergePolicy::get_minor_merge_tables(param, ls, *tablet_handle_.get_obj(), result);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, check_result_tables_handle("200, 250, 300, 340", result));
prepare_medium_list("300", tablet_handle_);
ret = ObPartitionMergePolicy::get_minor_merge_tables(param, ls, *tablet_handle_.get_obj(), result);
ASSERT_EQ(OB_NO_NEED_MERGE, ret);
}
} //unittest
} //oceanbase

12
unittest/storage/test_dag_warning_history.cpp
View File

@ -50,7 +50,7 @@ TEST_F(TestDagWarningHistory, simple_add)
info->dag_ret_ = -4016;
info->dag_status_ = ObDagWarningInfo::ODS_WARNING;
info->dag_type_ = share::ObDagType::DAG_TYPE_MINOR_MERGE;
info->dag_type_ = share::ObDagType::DAG_TYPE_MERGE_EXECUTE;
strcpy(info->warning_info_, "table_id=1101710651081571, partition_id=66, mini merge error");
ObDagWarningInfo *ret_info = NULL;
@ -79,7 +79,7 @@ TEST_F(TestDagWarningHistory, simple_del_with_no_lock)
ASSERT_EQ(OB_SUCCESS, manager.alloc_and_add_with_no_lock(key, info));
info->dag_ret_ = -4016;
info->dag_status_ = ObDagWarningInfo::ODS_WARNING;
info->dag_type_ = share::ObDagType::DAG_TYPE_MINOR_MERGE;
info->dag_type_ = share::ObDagType::DAG_TYPE_MERGE_EXECUTE;
strcpy(info->warning_info_, "table_id=1101710651081571, partition_id=66, mini merge error");
ASSERT_EQ(OB_HASH_NOT_EXIST, manager.del_with_no_lock(key + 1));
@ -102,7 +102,7 @@ TEST_F(TestDagWarningHistory, simple_loop_get)
basic_info.tenant_id_ = tenant_id;
basic_info.dag_ret_ = -4016;
basic_info.dag_status_ = ObDagWarningInfo::ODS_WARNING;
basic_info.dag_type_ = share::ObDagType::DAG_TYPE_MINOR_MERGE;
basic_info.dag_type_ = share::ObDagType::DAG_TYPE_MERGE_EXECUTE;
strcpy(basic_info.warning_info_, "table_id=1101710651081571, partition_id=66, mini merge error");
const int64_t max_cnt = 20;
@ -114,7 +114,7 @@ TEST_F(TestDagWarningHistory, simple_loop_get)
info->tenant_id_ = tenant_id;
info->dag_ret_ = -4016 + i;
info->dag_status_ = ObDagWarningInfo::ODS_WARNING;
info->dag_type_ = share::ObDagType::DAG_TYPE_MINOR_MERGE;
info->dag_type_ = share::ObDagType::DAG_TYPE_MERGE_EXECUTE;
strcpy(info->warning_info_, "table_id=1101710651081571, partition_id=66, mini merge error");
}
ObDagWarningInfoIterator iterator;
@ -166,7 +166,7 @@ TEST_F(TestDagWarningHistory, test_rebuild)
basic_info.tenant_id_ = tenant_id;
basic_info.dag_ret_ = -4016;
basic_info.dag_status_ = ObDagWarningInfo::ODS_WARNING;
basic_info.dag_type_ = share::ObDagType::DAG_TYPE_MINOR_MERGE;
basic_info.dag_type_ = share::ObDagType::DAG_TYPE_MERGE_EXECUTE;
strcpy(basic_info.warning_info_, "table_id=1101710651081571, partition_id=66, mini merge error");
ObDagWarningInfo *info = NULL;
@ -177,7 +177,7 @@ TEST_F(TestDagWarningHistory, test_rebuild)
info->tenant_id_ = tenant_id;
info->dag_ret_ = -4016 + i;
info->dag_status_ = ObDagWarningInfo::ODS_WARNING;
info->dag_type_ = share::ObDagType::DAG_TYPE_MINOR_MERGE;
info->dag_type_ = share::ObDagType::DAG_TYPE_MERGE_EXECUTE;
strcpy(info->warning_info_, "table_id=1101710651081571, partition_id=66, mini merge error");
STORAGE_LOG(DEBUG, "print info", K(ret), K(i), K(key), KPC(info));
}

10
unittest/storage/test_major_rows_merger.cpp
View File

@ -140,8 +140,6 @@ void ObMajorRowsMergerTest::prepare_merge_context(const ObMergeType &merge_type,
merge_context.schema_ctx_.base_schema_version_ = table_schema_.get_schema_version();
merge_context.schema_ctx_.schema_version_ = table_schema_.get_schema_version();
merge_context.schema_ctx_.storage_schema_ = &table_merge_schema_;
merge_context.schema_ctx_.merge_schema_ = &table_merge_schema_;
merge_context.schema_ctx_.table_schema_ = &table_schema_;
merge_context.is_full_merge_ = is_full_merge;
merge_context.merge_level_ = MACRO_BLOCK_MERGE_LEVEL;
@ -225,7 +223,7 @@ TEST_F(ObMajorRowsMergerTest, tset_compare_func)
trans_version_range.multi_version_start_ = 1;
trans_version_range.base_version_ = 1;
prepare_merge_context(MINI_MINOR_MERGE, false, trans_version_range, merge_context);
prepare_merge_context(MINOR_MERGE, false, trans_version_range, merge_context);
ObMergeParameter merge_param;
OK(merge_param.init(merge_context, 0));
ObPartitionMergeIter *iter_0 = nullptr;
@ -335,7 +333,7 @@ TEST_F(ObMajorRowsMergerTest, single)
trans_version_range.multi_version_start_ = 1;
trans_version_range.base_version_ = 1;
prepare_merge_context(MINI_MINOR_MERGE, false, trans_version_range, merge_context);
prepare_merge_context(MINOR_MERGE, false, trans_version_range, merge_context);
ObMergeParameter merge_param;
OK(merge_param.init(merge_context, 0));
ObPartitionMergeIter *iter_0 = nullptr;
@ -411,7 +409,7 @@ TEST_F(ObMajorRowsMergerTest, two_iters)
prepare_table_schema(micro_data, schema_rowkey_cnt, scn_range, snapshot_version);
reset_writer(snapshot_version);
prepare_one_macro(micro_data, 1);
prepare_data_end(handle1);
prepare_data_end(handle1, storage::ObITable::MAJOR_SSTABLE);
merge_context.tables_handle_.add_table(handle1);
STORAGE_LOG(INFO, "finish prepare sstable1");
@ -502,4 +500,4 @@ int main(int argc, char **argv)
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
}

234
unittest/storage/test_medium_compaction_mgr.cpp Normal file
View File

@ -0,0 +1,234 @@
// Copyright 2019-2021 Alibaba Inc. All Rights Reserved.
// Author:
// lixia.yq@antgroup.com
// This file defines test_medium_compaction_mgr.cpp
//
#include <gtest/gtest.h>
#define protected public
#define private public
#include <string.h>
#include "storage/compaction/ob_medium_compaction_mgr.h"
#include "share/schema/ob_column_schema.h"
namespace oceanbase
{
using namespace common;
using namespace compaction;
using namespace storage;
namespace unittest
{
class TestMediumCompactionMgr : public ::testing::Test
{
public:
virtual void SetUp()
{
share::schema::ObTableSchema table_schema;
prepare_schema(table_schema);
medium_info_.compaction_type_ = ObMediumCompactionInfo::MEDIUM_COMPACTION;
medium_info_.medium_snapshot_ = 100;
medium_info_.medium_scn_.convert_for_tx(100);
medium_info_.cluster_id_ = INIT_CLUSTER_ID;
medium_info_.storage_schema_.init(allocator_, table_schema, lib::Worker::CompatMode::MYSQL);
GCONF.cluster_id = 1;
}
virtual void TearDown()
{
medium_info_.reset();
allocator_.reset();
}
int construct_list(
const char *snapshot_list,
ObMediumCompactionInfoList &list,
const int64_t cluster_id = INIT_CLUSTER_ID);
int construct_array(
const char *snapshot_list,
ObIArray<int64_t> &array);
void prepare_schema(share::schema::ObTableSchema &table_schema);
static const int64_t INIT_CLUSTER_ID = 1;
static const int64_t OTHER_CLUSTER_ID = 2;
private:
ObMediumCompactionInfo medium_info_;
ObSEArray<int64_t, 10> array_;
ObArenaAllocator allocator_;
};
int TestMediumCompactionMgr::construct_array(
const char *snapshot_list,
ObIArray<int64_t> &array)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(snapshot_list)) {
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid argument", K(ret), K(snapshot_list));
} else {
array.reset();
std::string copy(snapshot_list);
char *org = const_cast<char *>(copy.c_str());
static const char *delim = " ";
char *s = std::strtok(org, delim);
if (NULL != s) {
array.push_back(atoi(s));
while (NULL != (s= strtok(NULL, delim))) {
array.push_back(atoi(s));
}
}
}
return ret;
}
int TestMediumCompactionMgr::construct_list(
const char *snapshot_list,
ObMediumCompactionInfoList &list,
const int64_t cluster_id)
{
int ret = OB_SUCCESS;
if (!list.is_inited_ && OB_FAIL(list.init(allocator_))) {
COMMON_LOG(WARN, "failed to init list", K(ret));
}
construct_array(snapshot_list, array_);
for (int i = 0; OB_SUCC(ret) && i < array_.count(); ++i) {
medium_info_.cluster_id_ = cluster_id;
medium_info_.medium_snapshot_ = array_.at(i);
ret = list.add_medium_compaction_info(medium_info_);
}
return ret;
}
static const int64_t TENANT_ID = 1;
static const int64_t TABLE_ID = 7777;
static const int64_t TEST_ROWKEY_COLUMN_CNT = 3;
static const int64_t TEST_COLUMN_CNT = 6;
void TestMediumCompactionMgr::prepare_schema(share::schema::ObTableSchema &table_schema)
{
int ret = OB_SUCCESS;
int64_t micro_block_size = 16 * 1024;
const uint64_t tenant_id = TENANT_ID;
const uint64_t table_id = TABLE_ID;
share::schema::ObColumnSchemaV2 column;
//generate data table schema
table_schema.reset();
ret = table_schema.set_table_name("test_merge_multi_version");
ASSERT_EQ(OB_SUCCESS, ret);
table_schema.set_tenant_id(tenant_id);
table_schema.set_tablegroup_id(1);
table_schema.set_database_id(1);
table_schema.set_table_id(table_id);
table_schema.set_rowkey_column_num(TEST_ROWKEY_COLUMN_CNT);
table_schema.set_max_used_column_id(TEST_COLUMN_CNT);
table_schema.set_block_size(micro_block_size);
table_schema.set_compress_func_name("none");
table_schema.set_row_store_type(FLAT_ROW_STORE);
//init column
char name[OB_MAX_FILE_NAME_LENGTH];
memset(name, 0, sizeof(name));
const int64_t column_ids[] = {16,17,20,21,22,23,24,29};
for(int64_t i = 0; i < TEST_COLUMN_CNT; ++i){
ObObjType obj_type = ObIntType;
const int64_t column_id = column_ids[i];
if (i == 1) {
obj_type = ObVarcharType;
}
column.reset();
column.set_table_id(table_id);
column.set_column_id(column_id);
sprintf(name, "test%020ld", i);
ASSERT_EQ(OB_SUCCESS, column.set_column_name(name));
column.set_data_type(obj_type);
column.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI);
column.set_data_length(10);
if (i < TEST_ROWKEY_COLUMN_CNT) {
column.set_rowkey_position(i + 1);
} else {
column.set_rowkey_position(0);
}
COMMON_LOG(INFO, "add column", K(i), K(column));
ASSERT_EQ(OB_SUCCESS, table_schema.add_column(column));
}
COMMON_LOG(INFO, "dump stable schema", LITERAL_K(TEST_ROWKEY_COLUMN_CNT), K(table_schema));
}
TEST_F(TestMediumCompactionMgr, test_basic_init)
{
ObMediumCompactionInfoList list_1;
ASSERT_EQ(OB_SUCCESS, construct_list("300, 400, 500", list_1));
ObMediumCompactionInfoList list_2;
ASSERT_EQ(OB_SUCCESS, construct_list("100, 200", list_2));
ObMediumCompactionInfoList out_list;
ASSERT_EQ(OB_SUCCESS, out_list.init(allocator_, &list_2, &list_1, 0));
ASSERT_EQ(5, out_list.size());
ASSERT_EQ(100, out_list.get_min_medium_snapshot());
ASSERT_EQ(500, out_list.get_max_medium_snapshot());
out_list.reset();
ASSERT_EQ(OB_SUCCESS, out_list.init(allocator_, &list_2, &list_1, 400));
ASSERT_EQ(1, out_list.size());
ASSERT_EQ(500, out_list.get_min_medium_snapshot());
ASSERT_EQ(500, out_list.get_max_medium_snapshot());
}
TEST_F(TestMediumCompactionMgr, test_push_list_error)
{
ObMediumCompactionInfoList test_list;
ASSERT_EQ(OB_SUCCESS, construct_list("300, 500, 400", test_list));
ASSERT_EQ(test_list.get_max_medium_snapshot(), 500);
ASSERT_EQ(test_list.get_list().get_size(), 2);
medium_info_.medium_snapshot_ = 900;
ASSERT_EQ(OB_SUCCESS, test_list.add_medium_compaction_info(medium_info_));
ASSERT_EQ(test_list.get_list().get_size(), 3);
medium_info_.medium_snapshot_ = 700;
ASSERT_EQ(OB_SUCCESS, test_list.add_medium_compaction_info(medium_info_));
ASSERT_EQ(test_list.get_list().get_size(), 3);
ASSERT_EQ(test_list.get_max_medium_snapshot(), 900);
medium_info_.medium_snapshot_ = 1000;
ASSERT_EQ(OB_SUCCESS, test_list.add_medium_compaction_info(medium_info_));
ASSERT_EQ(test_list.get_list().get_size(), 4);
medium_info_.medium_snapshot_ = 1200;
ASSERT_EQ(OB_SUCCESS, test_list.add_medium_compaction_info(medium_info_));
ASSERT_EQ(test_list.get_list().get_size(), 5);
medium_info_.medium_snapshot_ = 100;
ASSERT_EQ(OB_SUCCESS, test_list.add_medium_compaction_info(medium_info_));
ASSERT_EQ(test_list.get_list().get_size(), 5);
medium_info_.medium_snapshot_ = 100;
ASSERT_EQ(test_list.size(), 5); // 300 500 900 1000 1200
ObMediumCompactionInfoList test_list_2;
ASSERT_EQ(OB_SUCCESS, test_list_2.init(allocator_, &test_list, nullptr, 900));
ASSERT_EQ(1000, test_list_2.get_min_medium_snapshot());
ASSERT_EQ(test_list_2.size(), 2);
const ObMediumCompactionInfo *ret_info = nullptr;
ASSERT_EQ(OB_ENTRY_NOT_EXIST, test_list_2.get_specified_scn_info(900, ret_info));
test_list_2.reset();
ASSERT_EQ(OB_SUCCESS, test_list_2.init(allocator_, &test_list, nullptr, 900));
}
}//end namespace unittest
}//end namespace oceanbase
int main(int argc, char **argv)
{
system("rm -f test_medium_compaction_mgr.log*");
OB_LOGGER.set_file_name("test_medium_compaction_mgr.log");
OB_LOGGER.set_log_level("DEBUG");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

291
unittest/storage/test_parallel_minor_dag.cpp Normal file
View File

@ -0,0 +1,291 @@
// Copyright (c) 2019-2021 Alibaba Inc. All Rights Reserved.
// Author:
// lixia.yq@antfin.com
//
#include <gtest/gtest.h>
#define private public
#define protected public
#include "storage/compaction/ob_partition_merge_policy.h"
#include "storage/ob_storage_struct.h"
#include "storage/blocksstable/ob_sstable.h"
#include "share/rc/ob_tenant_base.h"
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
namespace oceanbase
{
using namespace common;
using namespace storage;
using namespace blocksstable;
using namespace compaction;
using namespace omt;
using namespace share;
namespace unittest
{
class TestParallelMinorDag : public ::testing::Test
{
public:
TestParallelMinorDag() : allocator_(ObModIds::TEST), tenant_base_(500) {}
virtual ~TestParallelMinorDag() {}
int prepare_merge_result(const int64_t sstable_cnt, ObGetMergeTablesResult &result);
void SetUp()
{
ObTenantMetaMemMgr *t3m = OB_NEW(ObTenantMetaMemMgr, ObModIds::TEST, 500);
tenant_base_.set(t3m);
ObTenantEnv::set_tenant(&tenant_base_);
ASSERT_EQ(OB_SUCCESS, tenant_base_.init());
ASSERT_EQ(OB_SUCCESS, t3m->init());
}
share::SCN get_start_log_ts(const int64_t idx);
share::SCN get_end_log_ts(const int64_t idx);
void check_result(const int64_t sstable_cnt, const int64_t result_cnt);
static const int64_t TENANT_ID = 1;
static const int64_t TABLE_ID = 7777;
static const int64_t TEST_ROWKEY_COLUMN_CNT = 3;
static const int64_t TEST_COLUMN_CNT = 6;
static const int64_t MAX_SSTABLE_CNT = 60;
common::ObArenaAllocator allocator_;
ObTenantBase tenant_base_;
ObSSTable fake_sstables_[MAX_SSTABLE_CNT];
};
int TestParallelMinorDag::prepare_merge_result(
const int64_t sstable_cnt,
ObGetMergeTablesResult &result)
{
int ret = OB_SUCCESS;
result.reset();
result.version_range_.base_version_ = 50;
result.version_range_.snapshot_version_ = 100;
result.version_range_.multi_version_start_ = 100;
result.merge_version_ = 0;
result.base_schema_version_ = 0;
result.schema_version_ = 0;
result.create_snapshot_version_ = 0;
result.suggest_merge_type_ = MINOR_MERGE;
int64_t log_ts = 1;
for (int i = 0; OB_SUCC(ret) && i < sstable_cnt; ++i) {
fake_sstables_[i].key_.scn_range_.start_scn_.convert_for_tx(log_ts++);
fake_sstables_[i].key_.scn_range_.end_scn_.convert_for_tx(log_ts);
if (OB_FAIL(result.handle_.add_table(&fake_sstables_[i]))) {
COMMON_LOG(WARN, "failed to push table", K(ret), K(i), K(fake_sstables_[i]));
}
}
result.scn_range_.start_scn_ = fake_sstables_[0].key_.scn_range_.start_scn_;
result.scn_range_.end_scn_ = fake_sstables_[sstable_cnt - 1].key_.scn_range_.end_scn_;
return ret;
}
share::SCN TestParallelMinorDag::get_start_log_ts(const int64_t idx)
{
return fake_sstables_[idx].key_.scn_range_.start_scn_;
}
share::SCN TestParallelMinorDag::get_end_log_ts(const int64_t idx)
{
return fake_sstables_[idx].key_.scn_range_.end_scn_;
}
void check_result_valid(const ObGetMergeTablesResult &result)
{
ASSERT_EQ(result.handle_.get_table(0)->get_start_scn(), result.scn_range_.start_scn_);
ASSERT_EQ(result.handle_.get_table(result.handle_.get_count() - 1)->get_end_scn(), result.scn_range_.end_scn_);
}
void TestParallelMinorDag::check_result(const int64_t sstable_cnt, const int64_t result_cnt)
{
ObGetMergeTablesResult result;
ObArray<ObGetMergeTablesResult> result_array;
ObMinorExecuteRangeMgr minor_range_mgr;
ASSERT_EQ(OB_SUCCESS, prepare_merge_result(sstable_cnt, result));
ASSERT_EQ(OB_SUCCESS, ObPartitionMergePolicy::generate_parallel_minor_interval(result, minor_range_mgr, result_array));
COMMON_LOG(INFO, "generate_parallel_minor_interval", K(sstable_cnt), K(result_array));
ASSERT_EQ(result_array.count(), result_cnt);
int idx = 0;
int rest_cnt = sstable_cnt;
const int64_t minor_trigger = ObPartitionMergePolicy::OB_MINOR_PARALLEL_SSTABLE_CNT_IN_DAG / 2;
if (sstable_cnt < ObPartitionMergePolicy::OB_MINOR_PARALLEL_SSTABLE_CNT_TRIGGER) {
ASSERT_EQ(result_array.count(), 1);
ASSERT_EQ(result_array.at(0).handle_.get_count(), sstable_cnt);
} else {
for (int i = 0; i < result_array.count(); ++i) {
check_result_valid(result_array.at(i));
ASSERT_EQ(result_array.at(i).scn_range_.start_scn_, get_start_log_ts(idx));
if (rest_cnt > ObPartitionMergePolicy::OB_MINOR_PARALLEL_SSTABLE_CNT_IN_DAG + minor_trigger
&& sstable_cnt >= minor_trigger) {
ASSERT_EQ(result_array.at(i).handle_.get_count(), ObPartitionMergePolicy::OB_MINOR_PARALLEL_SSTABLE_CNT_IN_DAG);
idx += ObPartitionMergePolicy::OB_MINOR_PARALLEL_SSTABLE_CNT_IN_DAG;
rest_cnt -= ObPartitionMergePolicy::OB_MINOR_PARALLEL_SSTABLE_CNT_IN_DAG;
} else {
ASSERT_EQ(result_array.at(i).handle_.get_count(), rest_cnt);
idx = sstable_cnt;
}
ASSERT_EQ(result_array.at(i).scn_range_.end_scn_, get_end_log_ts(idx - 1));
}
}
}
TEST_F(TestParallelMinorDag, test_parallel_interval)
{
check_result(20, 2);
check_result(19, 1);
check_result(36, 4);
check_result(35, 3);
check_result(32, 3);
check_result(12, 1);
check_result(18, 1);
check_result(22, 2);
check_result(3, 1);
check_result(9, 1);
check_result(40, 4);
}
#define CHECK_IN_RANGE(start_log_ts, end_log_ts, flag) \
fake_sstables_[0].key_.scn_range_.start_scn_.convert_for_tx(start_log_ts); \
fake_sstables_[0].key_.scn_range_.end_scn_.convert_for_tx(end_log_ts); \
ASSERT_EQ(flag, range_mgr.in_execute_range(&fake_sstables_[0]));
ObScnRange construct_scn_range(const int64_t start_scn, const int64_t end_scn)
{
ObScnRange ret_range;
ret_range.start_scn_.convert_for_tx(start_scn);
ret_range.end_scn_.convert_for_tx(end_scn);
return ret_range;
}
TEST_F(TestParallelMinorDag, test_range_mgr)
{
ObMinorExecuteRangeMgr range_mgr;
range_mgr.exe_range_array_.push_back(construct_scn_range(60, 80));
range_mgr.exe_range_array_.push_back(construct_scn_range(50, 70));
ASSERT_EQ(OB_ERR_UNEXPECTED, range_mgr.sort_ranges());
range_mgr.reset();
range_mgr.exe_range_array_.push_back(construct_scn_range(60, 80));
range_mgr.exe_range_array_.push_back(construct_scn_range(10, 20));
range_mgr.exe_range_array_.push_back(construct_scn_range(30, 50));
ASSERT_EQ(OB_SUCCESS, range_mgr.sort_ranges());
COMMON_LOG(INFO, "success to sort ranges", K(range_mgr.exe_range_array_));
CHECK_IN_RANGE(18, 19, true);
CHECK_IN_RANGE(60, 70, true);
CHECK_IN_RANGE(22, 30, false);
CHECK_IN_RANGE(30, 50, true);
range_mgr.reset();
range_mgr.exe_range_array_.push_back(construct_scn_range(10, 20));
range_mgr.exe_range_array_.push_back(construct_scn_range(40, 80));
range_mgr.exe_range_array_.push_back(construct_scn_range(20, 40));
ASSERT_EQ(OB_SUCCESS, range_mgr.sort_ranges());
COMMON_LOG(INFO, "success to sort ranges", K(range_mgr.exe_range_array_));
CHECK_IN_RANGE(18, 19, true);
CHECK_IN_RANGE(60, 70, true);
CHECK_IN_RANGE(22, 30, true);
CHECK_IN_RANGE(30, 50, true);
CHECK_IN_RANGE(80, 85, false);
CHECK_IN_RANGE(30, 65, true);
range_mgr.reset();
range_mgr.exe_range_array_.push_back(construct_scn_range(0, 200));
range_mgr.exe_range_array_.push_back(construct_scn_range(10, 20));
range_mgr.exe_range_array_.push_back(construct_scn_range(40, 80));
range_mgr.exe_range_array_.push_back(construct_scn_range(20, 40));
ASSERT_EQ(OB_SUCCESS, range_mgr.sort_ranges());
CHECK_IN_RANGE(100, 165, true);
}
TEST_F(TestParallelMinorDag, test_parallel_with_range_mgr)
{
int64_t sstable_cnt = 40;
ObGetMergeTablesResult result;
ObArray<ObGetMergeTablesResult> result_array;
ObMinorExecuteRangeMgr minor_range_mgr;
minor_range_mgr.exe_range_array_.push_back(construct_scn_range(11, 21));
minor_range_mgr.exe_range_array_.push_back(construct_scn_range(31, 41));
ASSERT_EQ(OB_SUCCESS, prepare_merge_result(sstable_cnt, result));
ASSERT_EQ(OB_SUCCESS, ObPartitionMergePolicy::generate_parallel_minor_interval(result, minor_range_mgr, result_array));
ASSERT_EQ(result_array.count(), 2);
ASSERT_EQ(result_array.at(0).scn_range_.start_scn_.get_val_for_tx(), 1);
ASSERT_EQ(result_array.at(0).scn_range_.end_scn_.get_val_for_tx(), 11);
ASSERT_EQ(result_array.at(1).scn_range_.start_scn_.get_val_for_tx(), 21);
ASSERT_EQ(result_array.at(1).scn_range_.end_scn_.get_val_for_tx(), 31);
result_array.reset();
minor_range_mgr.reset();
minor_range_mgr.exe_range_array_.push_back(construct_scn_range(15, 19));
minor_range_mgr.exe_range_array_.push_back(construct_scn_range(37, 39));
ASSERT_EQ(OB_SUCCESS, ObPartitionMergePolicy::generate_parallel_minor_interval(result, minor_range_mgr, result_array));
COMMON_LOG(INFO, "generate_parallel_minor_interval", K(result_array));
ASSERT_EQ(result_array.count(), 2);
ASSERT_EQ(result_array.at(0).scn_range_.start_scn_.get_val_for_tx(), 1);
ASSERT_EQ(result_array.at(0).scn_range_.end_scn_.get_val_for_tx(), 15);
ASSERT_EQ(result_array.at(1).scn_range_.start_scn_.get_val_for_tx(), 19);
ASSERT_EQ(result_array.at(1).scn_range_.end_scn_.get_val_for_tx(), 37);
// two runing ranges, candidates need > 8
result_array.reset();
minor_range_mgr.reset();
minor_range_mgr.exe_range_array_.push_back(construct_scn_range(1, 17));
minor_range_mgr.exe_range_array_.push_back(construct_scn_range(18, 34));
ASSERT_EQ(OB_SUCCESS, ObPartitionMergePolicy::generate_parallel_minor_interval(result, minor_range_mgr, result_array));
COMMON_LOG(INFO, "generate_parallel_minor_interval", K(result_array));
ASSERT_EQ(result_array.count(), 0);
result_array.reset();
minor_range_mgr.reset();
minor_range_mgr.exe_range_array_.push_back(construct_scn_range(1, 17));
minor_range_mgr.exe_range_array_.push_back(construct_scn_range(17, 31));
ASSERT_EQ(OB_SUCCESS, ObPartitionMergePolicy::generate_parallel_minor_interval(result, minor_range_mgr, result_array));
COMMON_LOG(INFO, "generate_parallel_minor_interval", K(result_array));
ASSERT_EQ(result_array.count(), 1);
ASSERT_EQ(result_array.at(0).scn_range_.start_scn_.get_val_for_tx(), 31);
ASSERT_EQ(result_array.at(0).scn_range_.end_scn_.get_val_for_tx(), 41);
// one runing ranges, candidates need > 4
result_array.reset();
minor_range_mgr.reset();
minor_range_mgr.exe_range_array_.push_back(construct_scn_range(1, 34));
ASSERT_EQ(OB_SUCCESS, ObPartitionMergePolicy::generate_parallel_minor_interval(result, minor_range_mgr, result_array));
COMMON_LOG(INFO, "generate_parallel_minor_interval", K(result_array));
ASSERT_EQ(result_array.count(), 1);
ASSERT_EQ(result_array.at(0).scn_range_.start_scn_.get_val_for_tx(), 34);
ASSERT_EQ(result_array.at(0).scn_range_.end_scn_.get_val_for_tx(), 41);
}
} // namespace unittest
} // namespace oceanbase
int main(int argc, char **argv)
{
system("rm -rf test_parallel_minor_dag.log*");
OB_LOGGER.set_file_name("test_parallel_minor_dag.log");
OB_LOGGER.set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

21
unittest/storage/test_partition_incremental_range_spliter.cpp
View File

@ -624,7 +624,7 @@ public:
private:
void set_tablet_size(int64_t tablet_size)
{
table_schema_.tablet_size_ = tablet_size;
storage_schema_.tablet_size_ = tablet_size;
range_spliter_.tablet_size_ = tablet_size;
}
void set_major_sstable_meta(int64_t macro_block_count, int64_t occupy_size, int64_t row_count)
@ -659,7 +659,7 @@ private:
share::ObTenantBase tenant_base_;
ObArenaAllocator allocator_;
compaction::ObTabletMergeDagParam param_;
share::schema::ObTableSchema table_schema_;
ObStorageSchema storage_schema_;
ObTablet tablet_;
ObMockSSTableV2 major_sstable_;
ObSSTable minor_sstable_; // 增量
@ -687,13 +687,14 @@ void TestPartitionIncrementalRangeSliter::SetUp()
ASSERT_EQ(OB_SUCCESS, tenant_base_.init());
// table schema
table_schema_.tablet_size_ = 1024;
table_schema_.rowkey_column_num_ = ObMockDatumRowkey::COLUMN_NUM;
ObRowkeyColumn rowkey_col;
rowkey_col.column_id_ = 1;
rowkey_col.type_.set_int();
storage_schema_.tablet_size_ = 1024;
storage_schema_.rowkey_array_.set_allocator(&allocator_);
storage_schema_.rowkey_array_.reserve(1);
ObStorageRowkeyColumnSchema rowkey_col;
rowkey_col.column_idx_ = 1 + common::OB_APP_MIN_COLUMN_ID;
rowkey_col.meta_type_.set_int();
rowkey_col.order_ = ASC;
ASSERT_EQ(OB_SUCCESS, table_schema_.rowkey_info_.add_column(rowkey_col));
ASSERT_EQ(OB_SUCCESS, storage_schema_.rowkey_array_.push_back(rowkey_col));
// major sstable
major_sstable_.set_table_type(ObITable::MAJOR_SSTABLE);
@ -708,7 +709,7 @@ void TestPartitionIncrementalRangeSliter::SetUp()
minor_sstable_.key_.tablet_id_ = 1;
// merge ctx
merge_ctx_.schema_ctx_.merge_schema_ = &table_schema_;
merge_ctx_.schema_ctx_.storage_schema_ = &storage_schema_;
ASSERT_EQ(OB_SUCCESS, merge_ctx_.tables_handle_.add_table(&major_sstable_));
ASSERT_EQ(OB_SUCCESS, merge_ctx_.tables_handle_.add_table(&minor_sstable_));
merge_ctx_.tablet_handle_.obj_ = &tablet_;
@ -1162,7 +1163,7 @@ TEST_F(TestPartitionIncrementalRangeSliter, test_not_empty_major_sstable_split_r
int main(int argc, char **argv)
{
system("rm -f test_partition_incremental_range_spliter.log*");
oceanbase::ObLogger::get_logger().set_file_name("test_partition_incremental_range_spliter.log", true);
oceanbase::ObLogger::get_logger().set_file_name("test_partition_incremental_range_spliter.log");
oceanbase::ObLogger::get_logger().set_log_level("DEBUG");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();

415
unittest/storage/test_tenant_tablet_stat_mgr.cpp Normal file
View File

@ -0,0 +1,415 @@
/*
* test_tenant_tablet_stat_mgr.cpp
* Author: fengjingkun.fjk@antgroup.com
*/
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include <thread>
#define USING_LOG_PREFIX STORAGE
#define protected public
#define private public
#include "mtlenv/mock_tenant_module_env.h"
#include "storage/ob_tenant_tablet_stat_mgr.h"
namespace oceanbase
{
using namespace share;
using namespace common;
using namespace storage;
using namespace compaction;
class TestTenantTabletStatMgr : public ::testing::Test
{
public:
TestTenantTabletStatMgr();
virtual ~TestTenantTabletStatMgr() = default;
virtual void SetUp() override;
virtual void TearDown() override;
static void SetUpTestCase();
static void TearDownTestCase();
static void report(ObTenantTabletStatMgr *mgr, const ObTabletStat &stat);
void batch_report_stat(int64_t report_num);
private:
const uint64_t tenant_id_;
ObTenantBase tenant_base_;
ObTenantTabletStatMgr *stat_mgr_;
};
TestTenantTabletStatMgr::TestTenantTabletStatMgr()
: tenant_id_(1),
tenant_base_(tenant_id_),
stat_mgr_(nullptr)
{
}
void TestTenantTabletStatMgr::SetUpTestCase()
{
EXPECT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init());
}
void TestTenantTabletStatMgr::TearDownTestCase()
{
MockTenantModuleEnv::get_instance().destroy();
}
void TestTenantTabletStatMgr::SetUp()
{
int ret = OB_SUCCESS;
stat_mgr_ = OB_NEW(ObTenantTabletStatMgr, ObModIds::TEST);
ret = stat_mgr_->init();
ASSERT_EQ(OB_SUCCESS, ret);
tenant_base_.set(stat_mgr_);
ObTenantEnv::set_tenant(&tenant_base_);
ASSERT_EQ(OB_SUCCESS, tenant_base_.init());
ASSERT_EQ(tenant_id_, MTL_ID());
ASSERT_EQ(stat_mgr_, MTL(ObTenantTabletStatMgr *));
}
void TestTenantTabletStatMgr::TearDown()
{
stat_mgr_->destroy();
ObTenantEnv::set_tenant(nullptr);
}
void TestTenantTabletStatMgr::report(ObTenantTabletStatMgr *mgr, const ObTabletStat &stat)
{
ASSERT_TRUE(NULL != mgr);
ASSERT_TRUE(stat.is_valid());
ASSERT_EQ(OB_SUCCESS, mgr->report_stat(stat));
}
void TestTenantTabletStatMgr::batch_report_stat(int64_t report_num)
{
ASSERT_TRUE(NULL != stat_mgr_);
ASSERT_EQ(true, stat_mgr_->is_inited_);
for (int64_t i = 0; i < report_num; ++i) {
ObTabletStat curr_stat;
curr_stat.ls_id_ = 1;
curr_stat.tablet_id_ = 10001 + i;
curr_stat.query_cnt_ = 100 * (i + 1);
std::thread sub_report_thread(report, stat_mgr_, curr_stat);
if (sub_report_thread.joinable()) {
sub_report_thread.join();
}
}
}
namespace unittest
{
TEST_F(TestTenantTabletStatMgr, basic)
{
EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID());
ObTenantTabletStatMgr *stat_mgr = MTL(ObTenantTabletStatMgr *);
ASSERT_TRUE(NULL != stat_mgr);
}
TEST_F(TestTenantTabletStatMgr, basic_tablet_stat_bucket)
{
ObTabletStat tablet_stat;
tablet_stat.ls_id_ = 1;
tablet_stat.tablet_id_ = 1;
tablet_stat.query_cnt_ = 100;
tablet_stat.scan_logical_row_cnt_ = 100;
tablet_stat.scan_physical_row_cnt_ = 100;
tablet_stat.merge_physical_row_cnt_ = 100;
tablet_stat.merge_logical_row_cnt_ = 100;
{
int64_t step = 1;
ObTabletStatBucket<8> bucket(step);
ObTabletStat retired_stat;
bool has_retired = false;
for (int64_t i = 0; i < 8; ++i) {
ASSERT_EQ(i, bucket.head_idx_);
ASSERT_EQ(7 + i, bucket.curr_idx_);
bucket.add(tablet_stat);
ASSERT_EQ(100, bucket.units_[bucket.get_idx(7 + i)].query_cnt_);
bucket.refresh(retired_stat, has_retired);
ASSERT_EQ(true, has_retired);
}
bucket.refresh(retired_stat, has_retired);
ASSERT_EQ(100, retired_stat.scan_logical_row_cnt_);
}
{
int64_t step = 16;
ObTabletStatBucket<4> bucket(step);
ObTabletStat retired_stat;
bool has_retired = false;
for (int64_t i = 0; i < 64; ++i) {
ASSERT_EQ(i / step, bucket.head_idx_);
ASSERT_EQ(3 + i / step, bucket.curr_idx_);
bucket.add(tablet_stat);
bucket.refresh(retired_stat, has_retired);
if (has_retired) {
ASSERT_EQ(0, bucket.units_[bucket.get_idx(bucket.curr_idx_)].query_cnt_);
}
}
ASSERT_TRUE(has_retired);
ASSERT_EQ(1600, retired_stat.scan_logical_row_cnt_);
}
{
int64_t step = 32;
ObTabletStatBucket<4> bucket(step);
ObTabletStat retired_stat;
bool has_retired = false;
for (int64_t i = 0; i < 128; ++i) {
ASSERT_EQ(i / step, bucket.head_idx_);
ASSERT_EQ(3 + i / step, bucket.curr_idx_);
bucket.add(tablet_stat);
bucket.refresh(retired_stat, has_retired);
if (has_retired) {
ASSERT_EQ(0, bucket.units_[bucket.get_idx(bucket.curr_idx_)].query_cnt_);
}
}
ASSERT_TRUE(has_retired);
ASSERT_EQ(3200, retired_stat.scan_logical_row_cnt_);
}
}
TEST_F(TestTenantTabletStatMgr, basic_tablet_stream)
{
ObTabletStat tablet_stat;
tablet_stat.ls_id_ = 1;
tablet_stat.tablet_id_ = 1;
tablet_stat.query_cnt_ = 100;
tablet_stat.scan_logical_row_cnt_ = 100;
tablet_stat.scan_physical_row_cnt_ = 100;
tablet_stat.merge_physical_row_cnt_ = 100;
tablet_stat.merge_logical_row_cnt_ = 100;
ObTabletStream stream;
auto &curr_buckets = stream.curr_buckets_;
auto &latest_buckets = stream.latest_buckets_;
auto &past_buckets = stream.past_buckets_;
for (int64_t i = 0; i < ObTabletStream::CURR_BUCKET_CNT * ObTabletStream::CURR_BUCKET_STEP; ++i) {
stream.add_stat(tablet_stat);
stream.refresh();
}
ASSERT_EQ(8, latest_buckets.refresh_cnt_);
// retired from curr_buckets
ASSERT_EQ(100, latest_buckets.units_[latest_buckets.get_idx(latest_buckets.curr_idx_)].query_cnt_);
for (int64_t i = 0; i < ObTabletStream::LATEST_BUCKET_CNT * ObTabletStream::LATEST_BUCKET_STEP; ++i) {
stream.add_stat(tablet_stat);
stream.refresh();
}
ASSERT_EQ(72, latest_buckets.refresh_cnt_);
ASSERT_EQ(800, latest_buckets.units_[latest_buckets.get_idx(latest_buckets.curr_idx_)].query_cnt_);
ASSERT_EQ(72, past_buckets.refresh_cnt_);
ASSERT_EQ(0, past_buckets.units_[past_buckets.get_idx(past_buckets.curr_idx_)].query_cnt_);
for (int64_t i = 0; i < ObTabletStream::CURR_BUCKET_CNT * ObTabletStream::CURR_BUCKET_STEP; ++i) {
stream.add_stat(tablet_stat);
stream.refresh();
}
ASSERT_EQ(80, latest_buckets.refresh_cnt_);
ASSERT_EQ(80, past_buckets.refresh_cnt_);
ASSERT_EQ(1600, past_buckets.units_[past_buckets.get_idx(past_buckets.curr_idx_)].query_cnt_);
}
TEST_F(TestTenantTabletStatMgr, get_all_tablet_stat)
{
int ret = OB_SUCCESS;
ObTabletStat tablet_stat;
tablet_stat.ls_id_ = 1;
tablet_stat.tablet_id_ = 1;
tablet_stat.query_cnt_ = 100;
tablet_stat.scan_logical_row_cnt_ = 100;
tablet_stat.scan_physical_row_cnt_ = 100;
tablet_stat.merge_physical_row_cnt_ = 100;
tablet_stat.merge_logical_row_cnt_ = 100;
ObTabletStream stream;
auto &curr_buckets = stream.curr_buckets_;
auto &latest_buckets = stream.latest_buckets_;
auto &past_buckets = stream.past_buckets_;
ObArray<ObTabletStat> tablet_stats;
int64_t curr_bucket_size = curr_buckets.count();
for (int64_t i = 0; i < curr_bucket_size; ++i) {
curr_buckets.units_[i] += tablet_stat;
}
ret = stream.get_bucket_tablet_stat(curr_buckets, tablet_stats);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(curr_bucket_size == tablet_stats.count());
int64_t latest_bucket_size = latest_buckets.count();
for (int64_t i = 0; i < latest_bucket_size; ++i) {
latest_buckets.units_[i] += tablet_stat;
}
ret = stream.get_bucket_tablet_stat(latest_buckets, tablet_stats);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE((curr_bucket_size + latest_bucket_size) == tablet_stats.count());
int64_t past_bucket_size = past_buckets.count();
for (int64_t i = 0; i < past_bucket_size; ++i) {
past_buckets.units_[i] += tablet_stat;
}
ret = stream.get_bucket_tablet_stat(past_buckets, tablet_stats);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE((curr_bucket_size + latest_bucket_size + past_bucket_size) == tablet_stats.count());
}
TEST_F(TestTenantTabletStatMgr, basic_stream_pool)
{
int ret = OB_SUCCESS;
const int64_t max_free_list_num = 500;
const int64_t up_limit_node_num = 1000;
ObTabletStreamPool pool;
ret = pool.init(max_free_list_num, up_limit_node_num);
ASSERT_EQ(OB_SUCCESS, ret);
int64_t free_num = pool.get_free_num();
ASSERT_EQ(max_free_list_num, free_num);
ObTabletStreamNode *fixed_node = nullptr;
ret = pool.alloc(fixed_node);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(NULL != fixed_node);
ASSERT_EQ(storage::ObTabletStreamPool::NodeAllocType::FIXED_ALLOC, fixed_node->flag_);
ASSERT_EQ(max_free_list_num - 1, pool.get_free_num());
pool.free(fixed_node);
fixed_node = nullptr;
ASSERT_EQ(max_free_list_num, pool.get_free_num());
common::ObSEArray<ObTabletStreamNode *, 500> free_nodes;
for (int64_t i = 0; i < max_free_list_num; ++i) {
ObTabletStreamNode *free_node = nullptr;
ret = pool.alloc(free_node);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(NULL != free_node);
ASSERT_EQ(storage::ObTabletStreamPool::NodeAllocType::FIXED_ALLOC, free_node->flag_);
ret = free_nodes.push_back(free_node);
ASSERT_EQ(OB_SUCCESS, ret);
}
ASSERT_EQ(0, pool.get_free_num());
ObTabletStreamNode *dynamic_node = nullptr;
ret = pool.alloc(dynamic_node);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(NULL != dynamic_node);
ASSERT_EQ(storage::ObTabletStreamPool::NodeAllocType::DYNAMIC_ALLOC, dynamic_node->flag_);
pool.free(dynamic_node);
dynamic_node = nullptr;
ASSERT_EQ(0, pool.get_free_num());
for (int64_t i = 0; i < free_nodes.count(); ++i) {
ObTabletStreamNode *node = free_nodes.at(i);
ASSERT_TRUE(NULL != node);
pool.free(node);
ASSERT_EQ(i + 1, pool.get_free_num());
}
}
TEST_F(TestTenantTabletStatMgr, check_fetch_node)
{
int ret = OB_SUCCESS;
const int64_t max_free_list_num = 500;
const int64_t up_limit_node_num = 500;
ObTabletStreamPool pool;
common::ObDList<ObTabletStreamNode> lru_list_;
ret = pool.init(max_free_list_num, up_limit_node_num);
ASSERT_EQ(OB_SUCCESS, ret);
int64_t free_num = pool.get_free_num();
ASSERT_EQ(max_free_list_num, free_num);
ObTabletStreamNode *first_node = nullptr;
for (int64_t i = 0; i < max_free_list_num; ++i) {
ObTabletStreamNode *curr_node = nullptr;
ret = pool.free_list_.pop(curr_node);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(NULL == curr_node->prev_);
ASSERT_TRUE(NULL == curr_node->next_);
ASSERT_EQ(true, lru_list_.add_first(curr_node));
first_node = (NULL == first_node) ? curr_node : first_node;
}
ObTabletStreamNode *last_node = nullptr;
for (int64_t i = 0; i < max_free_list_num; ++i) {
last_node = lru_list_.get_last();
ASSERT_EQ(true, lru_list_.move_to_first(last_node));
ret = pool.free_list_.push(last_node);
ASSERT_EQ(OB_SUCCESS, ret);
}
last_node = lru_list_.get_last();
ASSERT_TRUE(last_node == first_node);
}
TEST_F(TestTenantTabletStatMgr, basic_tablet_stat_mgr)
{
int ret = OB_SUCCESS;
EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID());
ObTenantTabletStatMgr *stat_mgr = MTL(ObTenantTabletStatMgr *);
ASSERT_TRUE(NULL != stat_mgr);
ObTabletStat tablet_stat;
tablet_stat.ls_id_ = 1;
tablet_stat.tablet_id_ = 123;
tablet_stat.query_cnt_ = 100;
tablet_stat.scan_logical_row_cnt_ = 100;
tablet_stat.scan_physical_row_cnt_ = 100;
tablet_stat.merge_physical_row_cnt_ = 100;
tablet_stat.merge_logical_row_cnt_ = 100;
ret = stat_mgr_->report_stat(tablet_stat);
ASSERT_EQ(OB_SUCCESS, ret);
stat_mgr_->process_stats();
ObTabletStat res;
share::ObLSID ls_id(1);
common::ObTabletID tablet_id(123);
ret = stat_mgr_->get_latest_tablet_stat(ls_id, tablet_id, res);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(100, res.query_cnt_);
}
TEST_F(TestTenantTabletStatMgr, multi_report_tablet_stat)
{
EXPECT_EQ(OB_SYS_TENANT_ID, MTL_ID());
ObTenantTabletStatMgr *stat_mgr = MTL(ObTenantTabletStatMgr *);
ASSERT_TRUE(NULL != stat_mgr);
ASSERT_TRUE(stat_mgr->is_inited_);
int64_t report_num = 10;
batch_report_stat(report_num);
stat_mgr_->process_stats();
int64_t report_cnt = 0;
ObTenantTabletStatMgr::TabletStreamMap::iterator iter = stat_mgr_->stream_map_.begin();
for ( ; iter != stat_mgr_->stream_map_.end(); ++iter) {
++report_cnt;
}
ASSERT_TRUE(report_cnt > 5);
}
} // end unittest
} // end oceanbase
int main(int argc, char **argv)
{
::testing::InitGoogleTest(&argc, argv);
system("rm -f test_tenant_tablet_stat_mgr.log*");
OB_LOGGER.set_file_name("test_tenant_tablet_stat_mgr.log", true);
OB_LOGGER.set_log_level("INFO");
OB_LOGGER.set_max_file_size(256*1024*1024);
return RUN_ALL_TESTS();
}