hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

patch 4.0

Browse Source
This commit is contained in:
wangzelin.wzl
2022-10-24 10:34:53 +08:00
parent 4ad6e00ec3
commit 93a1074b0c
10533 changed files with 2588271 additions and 2299373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

217
unittest/storage/blocksstable/test_micro_block_reader.cpp
View File

@ -19,19 +19,23 @@
#include "storage/blocksstable/ob_row_cache.h"
#include "storage/ob_i_store.h"
#include "ob_row_generate.h"
#include "storage/blocksstable/ob_column_map.h"
#include "common/rowkey/ob_rowkey.h"
#include "share/ob_simple_mem_limit_getter.h"
namespace oceanbase {
namespace oceanbase
{
using namespace common;
using namespace blocksstable;
using namespace storage;
using namespace share::schema;
static ObSimpleMemLimitGetter getter;
#define INVALID_ITERATOR ObIMicroBlockReader::INVALID_ROW_INDEX
#define INVALID_ITERATOR ObIMicroBlockReaderInfo::INVALID_ROW_INDEX
namespace unittest {
class TestMicroBlockReader : public ::testing::Test {
namespace unittest
{
class TestMicroBlockReader : public ::testing::Test
{
public:
static const int64_t rowkey_column_count = 2;
// Every ObObjType from ObTinyIntType to ObHexStringType inclusive.
@ -39,31 +43,19 @@ public:
// can't be NULL or NOP.
static const int64_t column_num = ObHexStringType;
static const int64_t macro_block_size = 2L * 1024 * 1024L;
static const int64_t SNAPSHOT_VERSION = 2;
public:
TestMicroBlockReader() : allocator_(ObModIds::TEST)
{
reset_row();
}
TestMicroBlockReader() : allocator_(ObModIds::TEST), read_info_() { }
void SetUp();
virtual void TearDown()
{}
static void SetUpTestCase()
{}
static void TearDownTestCase()
{}
void reset_row()
{
row_.row_val_.cells_ = reinterpret_cast<ObObj*>(obj_buf_);
row_.row_val_.count_ = OB_ROW_MAX_COLUMNS_COUNT;
}
virtual void TearDown() {}
static void SetUpTestCase() {}
static void TearDownTestCase() {}
protected:
ObRowGenerate row_generate_;
ObColumnMap column_map_;
ObArenaAllocator allocator_;
ObStoreRow row_;
char obj_buf_[common::OB_ROW_MAX_COLUMNS_COUNT * sizeof(ObObj)];
ObTableReadInfo read_info_;
};
void TestMicroBlockReader::SetUp()
@ -71,19 +63,19 @@ void TestMicroBlockReader::SetUp()
const int64_t table_id = 3001;
ObTableSchema table_schema;
ObColumnSchemaV2 column;
// init table schema
//init table schema
table_schema.reset();
ASSERT_EQ(OB_SUCCESS, table_schema.set_table_name("test_row_writer"));
ASSERT_EQ(OB_SUCCESS, table_schema.set_table_name("test_row_reader"));
table_schema.set_tenant_id(1);
table_schema.set_tablegroup_id(1);
table_schema.set_database_id(1);
table_schema.set_table_id(table_id);
table_schema.set_rowkey_column_num(rowkey_column_count);
table_schema.set_max_used_column_id(column_num);
// init column
//init column
char name[OB_MAX_FILE_NAME_LENGTH];
memset(name, 0, sizeof(name));
for (int64_t i = 0; i < column_num; ++i) {
for(int64_t i = 0; i < column_num; ++i){
ObObjType obj_type = static_cast<ObObjType>(i + 1);
column.reset();
column.set_table_id(table_id);
@ -92,7 +84,7 @@ void TestMicroBlockReader::SetUp()
ASSERT_EQ(OB_SUCCESS, column.set_column_name(name));
column.set_collation_type(common::CS_TYPE_UTF8MB4_GENERAL_CI);
column.set_data_type(obj_type);
if (obj_type == common::ObIntType) {
if(obj_type == common::ObIntType){
column.set_rowkey_position(1);
} else if (obj_type == common::ObUTinyIntType) {
column.set_rowkey_position(2);
@ -101,27 +93,46 @@ void TestMicroBlockReader::SetUp()
}
ASSERT_EQ(OB_SUCCESS, table_schema.add_column(column));
}
// init ObRowGenerate
//init ObRowGenerate
ASSERT_EQ(OB_SUCCESS, row_generate_.init(table_schema));
}
static void convert_to_multi_version_row(const ObDatumRow &org_row,
const ObTableSchema &schema, const int64_t snapshot_version,
ObDatumRow &multi_row)
{
int64_t rowkey_column_count = schema.get_rowkey_column_num();
for (int64_t i = 0; i < schema.get_column_count(); ++i) {
if (i < rowkey_column_count) {
multi_row.storage_datums_[i] = org_row.storage_datums_[i];
} else {
multi_row.storage_datums_[i + 2] = org_row.storage_datums_[i];
}
}
multi_row.storage_datums_[schema.get_rowkey_column_num()].set_int(-snapshot_version);
multi_row.storage_datums_[schema.get_rowkey_column_num() + 1].set_int(0);
multi_row.row_flag_ = org_row.row_flag_;
}
TEST_F(TestMicroBlockReader, test_success)
{
int ret = OB_SUCCESS;
const int64_t test_row_num = 10;
/*** build micro block buf with ObMicroBlockWriter ***/
oceanbase::common::ObObj objs[column_num];
ObStoreRow row;
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(allocator_, column_num));
ObDatumRow multi_version_row;
ASSERT_EQ(OB_SUCCESS, multi_version_row.init(allocator_, column_num +2));
ObMicroBlockWriter writer;
ret = writer.init(macro_block_size, rowkey_column_count, column_num);
ret = writer.init(macro_block_size, rowkey_column_count, column_num + 2);
ASSERT_EQ(OB_SUCCESS, ret);
for (int64_t i = 0; i < test_row_num; ++i) {
row.row_val_.cells_ = objs;
row.row_val_.count_ = column_num;
for(int64_t i = 0; i < test_row_num; ++i){
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(row));
ASSERT_EQ(OB_SUCCESS, writer.append_row(row));
convert_to_multi_version_row(row, row_generate_.get_schema(), SNAPSHOT_VERSION, multi_version_row);
ASSERT_EQ(OB_SUCCESS, writer.append_row(multi_version_row));
}
char* buf = NULL;
char *buf = NULL;
int64_t size = 0;
ret = writer.build_block(buf, size);
ASSERT_EQ(OB_SUCCESS, ret);
@ -129,99 +140,95 @@ TEST_F(TestMicroBlockReader, test_success)
/*** init column_map ***/
ObArray<ObColDesc> columns;
ASSERT_EQ(OB_SUCCESS, row_generate_.get_schema().get_column_ids(columns));
ASSERT_EQ(OB_SUCCESS,
column_map_.init(allocator_,
row_generate_.get_schema().get_schema_version(),
row_generate_.get_schema().get_rowkey_column_num(),
column_num,
columns));
ASSERT_EQ(OB_SUCCESS, read_info_.init(
allocator_, 16000, row_generate_.get_schema().get_rowkey_column_num(), lib::is_oracle_mode(), columns));
/*** init reader ***/
ObMicroBlockReader reader;
ObMicroBlockData block(buf, size);
ASSERT_EQ(OB_SUCCESS, reader.init(block, &column_map_));
ASSERT_EQ(OB_SUCCESS, reader.init(block, read_info_));
/*** init row cache ***/
const int64_t bucket_num = 1024;
const int64_t max_cache_size = 1024 * 1024 * 512;
const int64_t block_size = common::OB_MALLOC_BIG_BLOCK_SIZE;
ObKVGlobalCache::get_instance().init(bucket_num, max_cache_size, block_size);
ObKVGlobalCache::get_instance().init(&getter, bucket_num, max_cache_size, block_size);
ObRowCache cache;
ASSERT_EQ(OB_SUCCESS, cache.init("row_cache", 1));
/*** test lower bound ***/
ObStoreRowkey rowkey;
rowkey.set_min();
int64_t iter = ObIMicroBlockReader::INVALID_ROW_INDEX;
ObDatumRowkey rowkey;
rowkey.set_min_rowkey();
int64_t iter = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX;
bool equal = false;
ret = reader.find_bound(rowkey, true, reader.begin(), reader.end(), iter, equal);
ret = reader.find_bound(rowkey, true, 0, iter, equal);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(INVALID_ITERATOR != iter);
ASSERT_EQ(reader.begin(), iter);
ASSERT_EQ(0, iter);
/*** get block row success test ***/
const ObStoreRow* generated_row = NULL;
const ObStoreRow* block_row = NULL;
for (int32_t i = 0; i < test_row_num; ++i) {
reset_row();
ASSERT_EQ(OB_SUCCESS, reader.get_row(i, row_));
block_row = &row_;
ObDatumRow block_row;
ASSERT_EQ(OB_SUCCESS, block_row.init(allocator_, column_num));
for(int32_t i = 0; i < test_row_num; ++i){
ASSERT_EQ(OB_SUCCESS, reader.get_row(i, block_row));
// check exist
bool exist = false;
ASSERT_EQ(OB_SUCCESS, row_generate_.check_one_row(*block_row, exist));
ASSERT_EQ(OB_SUCCESS, row_generate_.check_one_row(block_row, exist));
ASSERT_TRUE(exist) << "i: " << i;
// check obj equal: every obj should be same except not exist row
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i, row));
generated_row = &row;
for (int64_t j = 0; j < column_num; ++j) {
ASSERT_TRUE(block_row->row_val_.cells_[j] == generated_row->row_val_.cells_[j])
<< "\n i: " << i << " j: " << j << "\n reader: " << to_cstring(block_row->row_val_.cells_[j])
<< "\n writer: " << to_cstring(generated_row->row_val_.cells_[j]);
for (int64_t j = 0; j < column_num; ++j){
ASSERT_TRUE(block_row.storage_datums_[j] == row.storage_datums_[j])
<< "\n i: " << i << " j: " << j
<< "\n reader: "<< to_cstring(block_row.storage_datums_[j])
<< "\n writer: " << to_cstring(row.storage_datums_[j]);
}
}
/*** test batch get rows ***/
reader.reset();
ASSERT_EQ(OB_SUCCESS, reader.init(block, &column_map_));
// init batch rows
const int64_t batch_row_capacity = ObIMicroBlockReader::OB_MAX_BATCH_ROW_COUNT;
ObStoreRow batch_rows[batch_row_capacity];
void* obj_buf = allocator_.alloc(batch_row_capacity * column_num * sizeof(ObObj));
for (int i = 0; i < batch_row_capacity; ++i) {
batch_rows[i].row_val_.cells_ = reinterpret_cast<ObObj*>(obj_buf) + column_num * i;
batch_rows[i].row_val_.count_ = column_num;
}
int64_t batch_count = 0;
ASSERT_EQ(OB_SUCCESS, reader.get_rows(0, test_row_num, batch_row_capacity, batch_rows, batch_count));
ASSERT_LE(batch_count, test_row_num);
for (int64_t i = 0; i < batch_count; ++i) {
bool exist = false;
const ObStoreRow& cur_row = batch_rows[i];
ASSERT_EQ(OB_SUCCESS, row_generate_.check_one_row(cur_row, exist));
ASSERT_TRUE(exist) << "i: " << i;
// every obj should equal
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i, row));
for (int64_t j = 0; j < column_num; ++j) {
ASSERT_TRUE(row.row_val_.cells_[j] == cur_row.row_val_.cells_[j])
<< "\n i: " << i << " j: " << j << "\n writer: " << to_cstring(row.row_val_.cells_[j])
<< "\n reader: " << to_cstring(cur_row.row_val_.cells_[j]);
}
}
// reader.reset();
// ASSERT_EQ(OB_SUCCESS, reader.init(block, read_info_));
// // init batch rows
// const int64_t batch_row_capacity = ObIMicroBlockReader::OB_MAX_BATCH_ROW_COUNT;
// ObStoreRow batch_rows[batch_row_capacity];
// void *obj_buf = allocator_.alloc(batch_row_capacity * column_num * sizeof(ObObj));
// for (int i = 0; i < batch_row_capacity; ++i) {
// batch_rows[i].row_val_.cells_ = reinterpret_cast<ObObj *>(obj_buf) + column_num * i;
// batch_rows[i].row_val_.count_ = column_num;
// }
// int64_t batch_count = 0;
// ASSERT_EQ(OB_SUCCESS, reader.get_rows(0, test_row_num, batch_row_capacity, batch_rows, batch_count));
// ASSERT_LE(batch_count, test_row_num);
// for(int64_t i = 0; i < batch_count; ++ i){
// bool exist = false;
// const ObStoreRow &cur_row = batch_rows[i];
// ASSERT_EQ(OB_SUCCESS, row_generate_.check_one_row(cur_row, exist));
// ASSERT_TRUE(exist) << "i: " << i;
// //every obj should equal
// ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i, row));
// for (int64_t j = 0; j < column_num ; ++ j){
// ASSERT_TRUE(row.row_val_.cells_[j] == cur_row.row_val_.cells_[j])
// << "\n i: " << i << " j: " << j
// << "\n writer: "<< to_cstring(row.row_val_.cells_[j])
// << "\n reader: " << to_cstring(cur_row.row_val_.cells_[j]);
// }
// }
ObKVGlobalCache::get_instance().destroy();
}
TEST_F(TestMicroBlockReader, not_init)
{
int ret = OB_SUCCESS;
ObMicroBlockReader reader;
ObStoreRowkey rowkey;
rowkey.set_min();
int64_t iter = INVALID_ITERATOR;
bool equal = false;
ret = reader.find_bound(rowkey, true, reader.begin(), reader.end(), iter, equal);
ASSERT_EQ(OB_NOT_INIT, ret);
reset_row();
ASSERT_EQ(OB_NOT_INIT, reader.get_row(iter, row_));
}
//TEST_F(TestMicroBlockReader, not_init)
//{
//int ret = OB_SUCCESS;
//ObMicroBlockReader reader;
//ObStoreRowkey rowkey;
//rowkey.set_min();
//int64_t iter = INVALID_ITERATOR;
//bool equal = false;
//blocksstable::ObDatumRow block_row;
//ASSERT_EQ(OB_SUCCESS, block_row.init(allocator_, 10));
//ret = reader.find_bound(rowkey, true, reader.begin(), reader.end(), iter, equal);
//ASSERT_EQ(OB_NOT_INIT, ret);
//ASSERT_EQ(OB_NOT_INIT, reader.get_row(iter, block_row));
//}
/*TEST_F(TestMicroBlockReader, misc_function)
{
@ -307,7 +314,7 @@ TEST_F(TestMicroBlockReader, test_percise_compare)
row.row_val_.count_ = column_num;
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(row));
if(i == test_row_num - 1){//row not exist test
row.flag_ = ObActionFlag::OP_ROW_DOES_NOT_EXIST;
row.flag_.set_flag(ObDmlFlag::DF_NOT_EXIST);
}
ASSERT_EQ(OB_SUCCESS, writer.append_row(row));
}
@ -356,10 +363,10 @@ TEST_F(TestMicroBlockReader, test_percise_compare)
}
*/
} // end namespace unittest
} // end namespace oceanbase
}//end namespace unittest
}//end namespace oceanbase
int main(int argc, char** argv)
int main(int argc, char **argv)
{
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);