hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] column store ddl

Browse Source 
Co-authored-by: simonjoylet <simonjoylet@gmail.com>
Co-authored-by: Monk-Liu <1152761042@qq.com>
Co-authored-by: renju96 <fcbrenju@163.com>
This commit is contained in:
Charles0429
2024-02-07 23:09:01 +00:00
committed by ob-robot
parent fd359dc4a4
commit 1c20161f72
354 changed files with 29599 additions and 7741 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
mittest/mtlenv/mock_tenant_module_env.h
View File

@ -707,6 +707,7 @@ int MockTenantModuleEnv::init()
MTL_BIND(ObTenantSQLSessionMgr::mtl_init, ObTenantSQLSessionMgr::mtl_destroy);
MTL_BIND2(mtl_new_default, ObTenantCGReadInfoMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObDecodeResourcePool::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObTenantDirectLoadMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObEmptyReadBucket::mtl_init, nullptr, nullptr, nullptr, ObEmptyReadBucket::mtl_destroy);
MTL_BIND2(mtl_new_default, ObRebuildService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, table::ObHTableLockMgr::mtl_init, nullptr, nullptr, nullptr, table::ObHTableLockMgr::mtl_destroy);

3
mittest/mtlenv/storage/blocksstable/CMakeLists.txt
View File

@ -20,3 +20,6 @@ storage_dml_unittest(test_co_sstable_row_scanner)
storage_dml_unittest(test_pushdown_aggregate)
storage_dml_unittest(test_cg_group_by_scanner)
storage_dml_unittest(test_cs_cg_group_by_scanner)
storage_dml_unittest(test_ddl_merge_row_scanner)
storage_dml_unittest(test_ddl_merge_row_multi_scanner)
storage_dml_unittest(test_direct_load)

370
mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h
View File

@ -68,17 +68,22 @@ public:
virtual void TearDown();
virtual void prepare_schema();
virtual void prepare_data(const int64_t micro_block_size = 0);
virtual void prepare_partial_ddl_data();
virtual void prepare_cg_data();
virtual void insert_data(ObMacroBlockWriter &data_writer); // override to define data in sstable
virtual void insert_cg_data(ObMacroBlockWriter &data_writer); // override to define data in sstable
virtual void insert_partial_data(ObMacroBlockWriter &data_writer, const int64_t row_cnt); // override to define data in partial_sstable
static void convert_to_multi_version_row(const ObDatumRow &org_row, const ObTableSchema &schema,
const int64_t snapshot_version, const ObDmlFlag dml_flag, ObDatumRow &multi_row);
static void fake_freeze_info();
virtual ObITable::TableType get_merged_table_type() const;
void prepare_query_param(const bool is_reverse_scan, ObArenaAllocator *allocator = nullptr);
void destroy_query_param();
void prepare_contrastive_sstable();
void prepare_ddl_kv();
void prepare_merge_ddl_kvs();
void close_builder_and_prepare_sstable(const int64_t column_cnt);
void prepare_partial_sstable(const int64_t column_cnt);
int gen_create_tablet_arg(const int64_t tenant_id,
const share::ObLSID &ls_id,
const ObTabletID &tablet_id,
@ -96,16 +101,24 @@ protected:
static const uint64_t tablet_id_ = 200001;
static const uint64_t TEST_TABLE_ID = 200001;
static const uint64_t ls_id_ = 1001;
static const int64_t DDL_KVS_CNT = 3;
ObMergeType merge_type_;
int64_t max_row_cnt_;
int64_t max_partial_row_cnt_;
int64_t partial_kv_start_idx_;
ObTableSchema table_schema_;
ObTableSchema index_schema_;
ObRowGenerate row_generate_;
int64_t row_cnt_;
int64_t partial_sstable_row_cnt_;
ObSSTable partial_sstable_;
ObSSTable sstable_;
storage::ObDDLKV ddl_kv_;
storage::ObDDLMemtable ddl_kv_;
storage::ObDDLKVHandle ddl_kvs_;
ObSSTableIndexBuilder *root_index_builder_;
ObSSTableIndexBuilder *merge_root_index_builder_;
ObMicroBlockData root_block_data_buf_;
ObMicroBlockData merge_root_block_data_buf_;
ObRowStoreType row_store_type_;
int64_t max_row_seed_;
int64_t min_row_seed_;
@ -126,6 +139,7 @@ protected:
int64_t rows_per_mirco_block_;
int64_t mirco_blocks_per_macro_block_;
bool is_cg_data_;
bool is_ddl_merge_data_;
};
ObArenaAllocator TestIndexBlockDataPrepare::allocator_;
@ -153,6 +167,7 @@ void TestIndexBlockDataPrepare::prepare_query_param(const bool is_reverse_scan,
}
context_.store_ctx_ = &store_ctx_;
context_.ls_id_ = ls_id_;
context_.tablet_id_ = tablet_id_;
context_.allocator_ = test_allocator;
context_.stmt_allocator_ = test_allocator;
context_.limit_param_ = nullptr;
@ -180,7 +195,9 @@ TestIndexBlockDataPrepare::TestIndexBlockDataPrepare(
: merge_type_(merge_type),
max_row_cnt_(max_row_cnt),
row_cnt_(0),
partial_sstable_row_cnt_(0),
root_index_builder_(nullptr),
merge_root_index_builder_(nullptr),
row_store_type_(row_store_type),
max_row_seed_(0),
min_row_seed_(0),
@ -191,7 +208,8 @@ TestIndexBlockDataPrepare::TestIndexBlockDataPrepare(
need_agg_data_(need_aggregate_data),
rows_per_mirco_block_(rows_per_mirco_block),
mirco_blocks_per_macro_block_(mirco_blocks_per_macro_block),
is_cg_data_(false)
is_cg_data_(false),
is_ddl_merge_data_(false)
{
}
@ -283,9 +301,12 @@ void TestIndexBlockDataPrepare::SetUp()
ASSERT_EQ(OB_SUCCESS, TestTabletHelper::create_tablet(ls_handle, tablet_id, table_schema_, allocator_));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_));
sstable_.key_.table_type_ = ObITable::TableType::COLUMN_ORIENTED_SSTABLE;
partial_sstable_.key_.table_type_ = ObITable::TableType::DDL_MERGE_CO_SSTABLE;
if (is_cg_data_) {
prepare_cg_data();
} else if (is_ddl_merge_data_) {
prepare_partial_ddl_data();
} else {
prepare_data();
}
@ -293,17 +314,32 @@ void TestIndexBlockDataPrepare::SetUp()
void TestIndexBlockDataPrepare::TearDown()
{
sstable_.reset();
partial_sstable_.reset();
ddl_kv_.reset();
ddl_kvs_.reset();
cg_read_info_handle_.reset();
if (nullptr != root_block_data_buf_.buf_) {
allocator_.free((void *)root_block_data_buf_.buf_);
root_block_data_buf_.buf_ = nullptr;
}
if (nullptr != root_block_data_buf_.buf_) {
allocator_.free((void *)root_block_data_buf_.buf_);
root_block_data_buf_.buf_ = nullptr;
}
if (nullptr != merge_root_block_data_buf_.buf_) {
allocator_.free((void *)merge_root_block_data_buf_.buf_);
merge_root_block_data_buf_.buf_ = nullptr;
}
if (nullptr != root_index_builder_) {
root_index_builder_->~ObSSTableIndexBuilder();
allocator_.free((void *)root_index_builder_);
root_index_builder_ = nullptr;
}
if (nullptr != merge_root_index_builder_) {
merge_root_index_builder_->~ObSSTableIndexBuilder();
allocator_.free((void *)merge_root_index_builder_);
merge_root_index_builder_ = nullptr;
}
ObLSID ls_id(ls_id_);
ObTabletID tablet_id(tablet_id_);
ObLSHandle ls_handle;
@ -345,7 +381,7 @@ ObITable::TableType TestIndexBlockDataPrepare::get_merged_table_type() const
} else if (META_MAJOR_MERGE == merge_type_) {
table_type = ObITable::TableType::META_MAJOR_SSTABLE;
} else if (DDL_KV_MERGE == merge_type_) {
table_type = ObITable::TableType::DDL_DUMP_SSTABLE;
table_type = ObITable::TableType::DDL_MERGE_CO_SSTABLE;
} else { // MINOR_MERGE || HISTORY_MINOR_MERGE
table_type = ObITable::TableType::MINOR_SSTABLE;
}
@ -354,7 +390,7 @@ ObITable::TableType TestIndexBlockDataPrepare::get_merged_table_type() const
void TestIndexBlockDataPrepare::prepare_schema()
{
ObColumnSchemaV2 column;
ObColumnSchemaV2 column;
//init table schema
uint64_t table_id = TEST_TABLE_ID;
table_schema_.reset();
@ -502,10 +538,9 @@ void TestIndexBlockDataPrepare::close_builder_and_prepare_sstable(const int64_t
row_store_type_ = root_row_store_type;
ObITable::TableKey table_key;
int64_t table_id = 3001;
int64_t tenant_id = 1;
table_key.table_type_ = get_merged_table_type();
table_key.tablet_id_ = table_id;
table_key.tablet_id_ = tablet_id_;
table_key.version_range_.snapshot_version_ = SNAPSHOT_VERSION;
ObTabletCreateSSTableParam param;
@ -685,7 +720,7 @@ void TestIndexBlockDataPrepare::prepare_ddl_kv()
share::SCN ddl_start_scn;
ddl_start_scn.convert_from_ts(ObTimeUtility::current_time());
ASSERT_EQ(OB_SUCCESS, ddl_kv_.init(*tablet_handle.get_obj(), ddl_start_scn, sstable_.get_data_version(), ddl_start_scn, 4000));
ASSERT_EQ(OB_SUCCESS, ddl_kv_.init(*tablet_handle.get_obj(), sstable_.get_key(), ddl_start_scn, DATA_CURRENT_VERSION));
SMART_VAR(ObSSTableSecMetaIterator, meta_iter) {
ObDatumRange query_range;
@ -711,7 +746,6 @@ void TestIndexBlockDataPrepare::prepare_ddl_kv()
}
}
ASSERT_EQ(OB_ITER_END, ret);
ASSERT_EQ(OB_SUCCESS, ddl_kv_.block_meta_tree_.build_sorted_rowkeys());
}
}
@ -813,6 +847,326 @@ void TestIndexBlockDataPrepare::convert_to_multi_version_row(const ObDatumRow &o
multi_row.mvcc_row_flag_.set_last_multi_version_row(true);
}
void TestIndexBlockDataPrepare::prepare_partial_ddl_data()
{
prepare_contrastive_sstable();
ObMacroBlockWriter writer;
ObMacroDataSeq start_seq(0);
start_seq.set_data_block();
row_generate_.reset();
ObWholeDataStoreDesc desc(true/*is ddl*/);
share::SCN end_scn;
end_scn.convert_from_ts(ObTimeUtility::current_time());
ASSERT_EQ(OB_SUCCESS, desc.init(table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, end_scn));
void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder));
merge_root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder();
ASSERT_NE(nullptr, merge_root_index_builder_);
desc.get_desc().sstable_index_builder_ = merge_root_index_builder_;
ASSERT_TRUE(desc.is_valid());
if (need_agg_data_) {
ASSERT_EQ(OB_SUCCESS, desc.get_desc().col_desc_->agg_meta_array_.assign(agg_col_metas_));
}
ASSERT_EQ(OB_SUCCESS, merge_root_index_builder_->init(desc.get_desc()));
ASSERT_EQ(OB_SUCCESS, writer.open(desc.get_desc(), start_seq));
ASSERT_EQ(OB_SUCCESS, row_generate_.init(table_schema_, &allocator_));
const int64_t partial_row_cnt = max_partial_row_cnt_;
insert_partial_data(writer, partial_row_cnt);
ASSERT_EQ(OB_SUCCESS, writer.close());
// data write ctx has been moved to merge_root_index_builder_
ASSERT_EQ(writer.get_macro_block_write_ctx().get_macro_block_count(), 0);
data_macro_block_cnt_ = merge_root_index_builder_->roots_[0]->macro_metas_->count();
ASSERT_GE(data_macro_block_cnt_, 0);
int64_t column_cnt = 0;
ObTabletID tablet_id(TestIndexBlockDataPrepare::tablet_id_);
ObLSID ls_id(ls_id_);
ObLSHandle ls_handle;
ObTabletHandle tablet_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ObStorageSchema *storage_schema = nullptr;
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle));
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->load_storage_schema(allocator_, storage_schema));
ASSERT_EQ(OB_SUCCESS, storage_schema->get_stored_column_count_in_sstable(column_cnt));
prepare_partial_sstable(column_cnt);
prepare_merge_ddl_kvs();
ObTabletObjLoadHelper::free(allocator_, storage_schema);
}
void TestIndexBlockDataPrepare::prepare_partial_sstable(const int64_t column_cnt)
{
ObSSTableMergeRes res;
OK(merge_root_index_builder_->close(res));
ObIndexTreeRootBlockDesc root_desc;
root_desc = res.root_desc_;
ASSERT_TRUE(root_desc.is_valid());
ObRowStoreType root_row_store_type = res.root_row_store_type_;
char *root_buf = nullptr;
int64_t root_size = 0;
if (root_desc.addr_.is_block()) {
// read macro block
ObMacroBlockReadInfo read_info;
ObMacroBlockHandle macro_handle;
const int64_t macro_block_size = 2 * 1024 * 1024;
ASSERT_EQ(OB_SUCCESS, root_desc.addr_.get_block_addr(read_info.macro_block_id_, read_info.offset_, read_info.size_));
read_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_DATA_READ);
read_info.offset_ = 0;
read_info.size_ = macro_block_size;
read_info.io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS;
ASSERT_NE(nullptr, read_info.buf_ = reinterpret_cast<char*>(allocator_.alloc(read_info.size_)));
ASSERT_EQ(OB_SUCCESS, ObBlockManager::read_block(read_info, macro_handle));
ASSERT_NE(macro_handle.get_buffer(), nullptr);
ASSERT_EQ(macro_handle.get_data_size(), macro_block_size);
// get root block
int64_t block_offset = root_desc.addr_.offset_;
int64_t block_size = root_desc.addr_.size_;
const char *block_buf = macro_handle.get_buffer() + block_offset;
// decompress and decrypt root block
ObMicroBlockDesMeta meta(ObCompressorType::NONE_COMPRESSOR, root_row_store_type, 0, 0, nullptr);
ObMacroBlockReader reader;
const char *decomp_buf = nullptr;
int64_t decomp_size = 0;
bool is_compressed = false;
ASSERT_EQ(OB_SUCCESS, reader.decrypt_and_decompress_data(meta, block_buf, root_desc.addr_.size_,
decomp_buf, decomp_size, is_compressed, true, &allocator_));
root_buf = const_cast<char *>(decomp_buf);
root_size = decomp_size;
} else if (root_desc.is_mem_type()) {
root_buf = root_desc.buf_;
root_size = root_desc.addr_.size_;
} else {
STORAGE_LOG(INFO, "not supported root block", K(root_desc));
ASSERT_TRUE(false);
}
// deserialize micro block header in root block buf
ObMicroBlockHeader root_micro_header;
int64_t des_pos = 0;
ASSERT_EQ(OB_SUCCESS, root_micro_header.deserialize(root_buf, root_size, des_pos));
merge_root_block_data_buf_.buf_ = static_cast<char *>(allocator_.alloc(root_size));
merge_root_block_data_buf_.size_ = root_size;
int64_t copy_pos = 0;
ObMicroBlockHeader *copied_micro_header = nullptr;
ASSERT_EQ(OB_SUCCESS, root_micro_header.deep_copy(
(char *)merge_root_block_data_buf_.buf_, merge_root_block_data_buf_.size_, copy_pos, copied_micro_header));
ASSERT_TRUE(copied_micro_header->is_valid());
MEMCPY((char *)(merge_root_block_data_buf_.buf_ + copy_pos), root_buf + des_pos, root_size - des_pos);
row_store_type_ = root_row_store_type;
ObITable::TableKey table_key;
int64_t table_id = 3001;
int64_t tenant_id = 1;
table_key.table_type_ = ObITable::TableType::DDL_MERGE_CO_SSTABLE;
table_key.tablet_id_ = table_id;
table_key.version_range_.snapshot_version_ = SNAPSHOT_VERSION;
ObTabletCreateSSTableParam param;
param.table_key_ = table_key;
param.schema_version_ = 10;
param.create_snapshot_version_ = 0;
param.progressive_merge_round_ = table_schema_.get_progressive_merge_round();
param.progressive_merge_step_ = 0;
param.table_mode_ = table_schema_.get_table_mode_struct();
param.index_type_ = table_schema_.get_index_type();
if (is_cg_data_) {
param.rowkey_column_cnt_ = 0;
} else {
param.rowkey_column_cnt_ = table_schema_.get_rowkey_column_num()
+ ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt();
}
ObSSTableMergeRes::fill_addr_and_data(res.root_desc_,
param.root_block_addr_, param.root_block_data_);
ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_,
param.data_block_macro_meta_addr_, param.data_block_macro_meta_);
param.is_meta_root_ = res.data_root_desc_.is_meta_root_;
param.max_merged_trans_version_ = res.max_merged_trans_version_;
param.row_count_ = res.row_count_;
param.root_row_store_type_ = root_row_store_type;
param.latest_row_store_type_ = table_schema_.get_row_store_type();
param.data_index_tree_height_ = root_desc.height_;
param.index_blocks_cnt_ = res.index_blocks_cnt_;
param.data_blocks_cnt_ = res.data_blocks_cnt_;
param.micro_block_cnt_ = res.micro_block_cnt_;
param.use_old_macro_block_count_ = res.use_old_macro_block_count_;
param.column_cnt_= column_cnt;
param.data_checksum_ = res.data_checksum_;
param.occupy_size_ = res.occupy_size_;
param.original_size_ = res.original_size_;
param.nested_offset_ = res.nested_offset_;
param.nested_size_ = res.nested_size_;
param.compressor_type_ = ObCompressorType::NONE_COMPRESSOR;
param.data_block_ids_ = res.data_block_ids_;
param.other_block_ids_ = res.other_block_ids_;
param.ddl_scn_.convert_from_ts(ObTimeUtility::current_time());
param.filled_tx_scn_.set_min();
param.contain_uncommitted_row_ = false;
param.encrypt_id_ = res.encrypt_id_;
param.master_key_id_ = res.master_key_id_;
if (param.table_key_.is_co_sstable() && param.column_group_cnt_ <= 1) {
param.column_group_cnt_ = column_cnt + 2; /* set column group_cnt to avoid return err, cnt is calculated as each + all + default*/
}
MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH);
if (merge_type_ == MAJOR_MERGE) {
OK(ObSSTableMergeRes::fill_column_checksum_for_empty_major(param.column_cnt_, param.column_checksums_));
}
partial_sstable_.reset();
ASSERT_EQ(OB_SUCCESS, partial_sstable_.init(param, &allocator_));
STORAGE_LOG(INFO, "create partial_sstable param", K(param));
}
void TestIndexBlockDataPrepare::insert_partial_data(ObMacroBlockWriter &data_writer, const int64_t row_cnt)
{
partial_sstable_row_cnt_ = 0;
int64_t seed = min_row_seed_;
ObDatumRow row;
ObDatumRow multi_row;
ASSERT_EQ(OB_SUCCESS, row.init(allocator_, MAX_TEST_COLUMN_CNT));
ASSERT_EQ(OB_SUCCESS, multi_row.init(allocator_, MAX_TEST_COLUMN_CNT));
ObDmlFlag flags[] = {DF_INSERT, DF_UPDATE, DF_DELETE};
int64_t rows_per_mirco_block = rows_per_mirco_block_;
int64_t rows_per_macro_block = rows_per_mirco_block_ * mirco_blocks_per_macro_block_;
int64_t rows_cnt = row_cnt;
if (INT64_MAX == rows_per_mirco_block_ || INT64_MAX == mirco_blocks_per_macro_block_) {
rows_per_mirco_block = INT64_MAX;
rows_per_macro_block = INT64_MAX;
}
while (true) {
if (partial_sstable_row_cnt_ >= rows_cnt) {
break;
}
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(seed, row));
++seed;
ObDmlFlag dml = flags[partial_sstable_row_cnt_ % ARRAYSIZEOF(flags)]; // INSERT / UPDATE / DELETE
convert_to_multi_version_row(row, table_schema_, SNAPSHOT_VERSION, dml, multi_row);
ASSERT_EQ(OB_SUCCESS, data_writer.append_row(multi_row));
if (partial_sstable_row_cnt_ == 0) {
ObDatumRowkey &start_key = data_writer.last_key_;
ASSERT_EQ(OB_SUCCESS, start_key.deep_copy(start_key_, allocator_));
}
if (partial_sstable_row_cnt_ == rows_cnt - 1) {
ObDatumRowkey &end_key = data_writer.last_key_;
ASSERT_EQ(OB_SUCCESS, end_key.deep_copy(end_key_, allocator_));
}
if ((partial_sstable_row_cnt_ + 1) % rows_per_mirco_block == 0) {
OK(data_writer.build_micro_block());
}
if ((partial_sstable_row_cnt_ + 1) % rows_per_macro_block == 0) {
OK(data_writer.try_switch_macro_block());
}
++partial_sstable_row_cnt_;
}
//max_row_seed_ = seed - 1;
}
void TestIndexBlockDataPrepare::prepare_contrastive_sstable()
{
ObMacroBlockWriter writer;
ObMacroDataSeq start_seq(0);
start_seq.set_data_block();
row_generate_.reset();
ObWholeDataStoreDesc desc;
share::SCN end_scn;
end_scn.convert_from_ts(ObTimeUtility::current_time());
ASSERT_EQ(OB_SUCCESS, desc.init(table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, end_scn));
void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder));
root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder();
ASSERT_NE(nullptr, root_index_builder_);
desc.get_desc().sstable_index_builder_ = root_index_builder_;
ASSERT_TRUE(desc.is_valid());
if (need_agg_data_) {
ASSERT_EQ(OB_SUCCESS, desc.get_desc().col_desc_->agg_meta_array_.assign(agg_col_metas_));
}
ASSERT_EQ(OB_SUCCESS, root_index_builder_->init(desc.get_desc()));
ASSERT_EQ(OB_SUCCESS, writer.open(desc.get_desc(), start_seq));
ASSERT_EQ(OB_SUCCESS, row_generate_.init(table_schema_, &allocator_));
insert_data(writer);
ASSERT_EQ(OB_SUCCESS, writer.close());
// data write ctx has been moved to root_index_builder
ASSERT_EQ(writer.get_macro_block_write_ctx().get_macro_block_count(), 0);
data_macro_block_cnt_ = root_index_builder_->roots_[0]->macro_metas_->count();
ASSERT_GE(data_macro_block_cnt_, 0);
int64_t column_cnt = 0;
ObTabletID tablet_id(TestIndexBlockDataPrepare::tablet_id_);
ObLSID ls_id(ls_id_);
ObLSHandle ls_handle;
ObTabletHandle tablet_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ObStorageSchema *storage_schema = nullptr;
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle));
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->load_storage_schema(allocator_, storage_schema));
ASSERT_EQ(OB_SUCCESS, storage_schema->get_stored_column_count_in_sstable(column_cnt));
close_builder_and_prepare_sstable(column_cnt);
prepare_ddl_kv();
ObTabletObjLoadHelper::free(allocator_, storage_schema);
}
void TestIndexBlockDataPrepare::prepare_merge_ddl_kvs()
{
ddl_kvs_.reset();
ObTabletHandle tablet_handle;
int ret = OB_SUCCESS;
ObLSID ls_id(ls_id_);
ObTabletID tablet_id(tablet_id_);
ObLSHandle ls_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle));
share::SCN ddl_start_scn;
ddl_start_scn.convert_from_ts(ObTimeUtility::current_time());
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr *);
ASSERT_NE(t3m, nullptr);
ASSERT_EQ(OB_SUCCESS, t3m->acquire_ddl_kv(ddl_kvs_));
ASSERT_EQ(OB_SUCCESS, ddl_kvs_.get_obj()->init(ls_id, tablet_id, ddl_start_scn, sstable_.get_data_version(), ddl_start_scn, 4000));
ObITable::TableKey ddl_key = sstable_.get_key();
ddl_key.table_type_ = ObITable::TableType::MAJOR_SSTABLE;
for (int64_t i = 0; i < DDL_KVS_CNT; ++i) {
void *buf = allocator_.alloc(sizeof(ObDDLMemtable));
ASSERT_NE(nullptr, buf);
ObDDLMemtable *new_ddl_table = new (buf) ObDDLMemtable;
ASSERT_EQ(OB_SUCCESS, new_ddl_table->init(*tablet_handle.get_obj(), ddl_key, ddl_start_scn, 4000));
ASSERT_EQ(OB_SUCCESS, ddl_kvs_.get_obj()->get_ddl_memtables().push_back(new_ddl_table));
}
ObDDLKVHandle kv_handle;
ObDDLKvMgrHandle ddl_kv_mgr_handle;
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle, true /*CREATE*/));
ddl_kv_mgr_handle.get_obj()->set_ddl_kv(0, ddl_kvs_);
ddl_kv_mgr_handle.get_obj()->freeze_ddl_kv(ddl_start_scn, sstable_.get_data_version(), 4000, ddl_start_scn);
SMART_VAR(ObSSTableSecMetaIterator, meta_iter) {
ObDatumRange query_range;
query_range.set_whole_range();
ObDataMacroBlockMeta data_macro_meta;
ASSERT_EQ(OB_SUCCESS, meta_iter.open(query_range,
ObMacroBlockMetaType::DATA_BLOCK_META,
sstable_,
tablet_handle.get_obj()->get_rowkey_read_info(),
allocator_));
int64_t macro_idx = 0;
int64_t kv_idx = 0;
while (OB_SUCC(ret)) {
if (OB_FAIL(meta_iter.get_next(data_macro_meta))) {
if (OB_ITER_END != ret) {
STORAGE_LOG(WARN, "get data macro meta failed", K(ret));
}
} else {
STORAGE_LOG(INFO, "data_macro_meta_key", K(data_macro_meta));
++macro_idx;
ObDDLMacroHandle macro_handle;
macro_handle.set_block_id(data_macro_meta.get_macro_id());
if (macro_idx > partial_kv_start_idx_) {
ObDataMacroBlockMeta *copied_meta = nullptr;
ASSERT_EQ(OB_SUCCESS, data_macro_meta.deep_copy(copied_meta, allocator_));
ASSERT_EQ(OB_SUCCESS, ddl_kvs_.get_obj()->get_ddl_memtables().at(kv_idx)->insert_block_meta_tree(macro_handle, copied_meta));
++kv_idx;
}
}
}
ASSERT_EQ(OB_ITER_END, ret);
}
}
} // namespace blocksstable
} // namespace oceanbase

772
mittest/mtlenv/storage/blocksstable/test_ddl_merge_row_multi_scanner.cpp Normal file
View File

@ -0,0 +1,772 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#define private public
#define protected public
#include "storage/access/ob_index_tree_prefetcher.h"
#include "storage/access/ob_sstable_row_multi_scanner.h"
#include "storage/access/ob_sstable_row_multi_getter.h"
#include "ob_index_block_data_prepare.h"
namespace oceanbase
{
using namespace storage;
using namespace common;
namespace blocksstable
{
class TestDDLMergeRowMultiScanner : public TestIndexBlockDataPrepare
{
public:
TestDDLMergeRowMultiScanner();
virtual ~TestDDLMergeRowMultiScanner();
static void SetUpTestCase();
static void TearDownTestCase();
virtual void SetUp();
virtual void TearDown();
protected:
void generate_range(const int64_t start, const int64_t end, ObDatumRange &range);
public:
void test_one_case(
const ObIArray<int64_t> &start_seeds,
const int64_t count_per_range,
const bool is_reverse_scan);
void test_single_get_normal(const bool is_reverse_scan);
void test_single_get_border(const bool is_reverse_scan);
void test_multi_get_normal(const bool is_reverse_scan);
void test_multi_get_border(const bool is_reverse_scan);
void test_single_scan_normal(const bool is_reverse_scan);
void test_single_scan_border(const bool is_reverse_scan);
void test_multi_scan_multi_scan_range(
const bool is_reverse_scan,
const int64_t count_per_range);
void test_multi_scan_multi_get_with_scan(
const bool is_reverse_scan,
const int64_t count_per_range);
protected:
static const int64_t TEST_MULTI_GET_CNT = 100;
enum CacheHitMode
{
HIT_ALL = 0,
HIT_NONE,
HIT_PART,
HIT_MAX,
};
private:
ObArenaAllocator allocator_;
ObDatumRow start_row_;
ObDatumRow end_row_;
};
TestDDLMergeRowMultiScanner::TestDDLMergeRowMultiScanner()
: TestIndexBlockDataPrepare("Test sstable row multi scanner")
{
is_ddl_merge_data_ = true;
max_row_cnt_ = 150000;
max_partial_row_cnt_ = 78881;
partial_kv_start_idx_ = 3;
}
TestDDLMergeRowMultiScanner::~TestDDLMergeRowMultiScanner()
{
}
void TestDDLMergeRowMultiScanner::SetUpTestCase()
{
TestIndexBlockDataPrepare::SetUpTestCase();
}
void TestDDLMergeRowMultiScanner::TearDownTestCase()
{
TestIndexBlockDataPrepare::TearDownTestCase();
}
void TestDDLMergeRowMultiScanner::SetUp()
{
TestIndexBlockDataPrepare::SetUp();
ObLSID ls_id(ls_id_);
ObTabletID tablet_id(tablet_id_);
ObLSHandle ls_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_));
ASSERT_EQ(OB_SUCCESS, start_row_.init(allocator_, TEST_COLUMN_CNT));
ASSERT_EQ(OB_SUCCESS, end_row_.init(allocator_, TEST_COLUMN_CNT));
}
void TestDDLMergeRowMultiScanner::TearDown()
{
tablet_handle_.reset();
TestIndexBlockDataPrepare::TearDown();
}
void TestDDLMergeRowMultiScanner::generate_range(
const int64_t start,
const int64_t end,
ObDatumRange &range)
{
ObDatumRowkey tmp_rowkey;
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(start, start_row_));
tmp_rowkey.assign(start_row_.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(range.start_key_, allocator_));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(end, end_row_));
tmp_rowkey.assign(end_row_.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(range.end_key_, allocator_));
range.border_flag_.set_inclusive_start();
range.border_flag_.set_inclusive_end();
}
void TestDDLMergeRowMultiScanner::test_one_case(
const ObIArray<int64_t> &start_seeds,
const int64_t count_per_range,
const bool is_reverse_scan)
{
int ret = OB_SUCCESS;
ObDatumRange mscan_ranges[TEST_MULTI_GET_CNT];
ObSSTableRowMultiScanner<> scanner;
ObSSTableRowMultiScanner<> merge_ddl_scanner;
ObSEArray<ObDatumRange, TEST_MULTI_GET_CNT> ranges;
const ObDatumRow *prow = NULL;
const ObDatumRow *kv_prow = NULL;
ObDatumRow start_row;
ObDatumRow end_row;
ObDatumRow check_row;
ASSERT_EQ(OB_SUCCESS, start_row.init(allocator_, TEST_COLUMN_CNT));
ASSERT_EQ(OB_SUCCESS, end_row.init(allocator_, TEST_COLUMN_CNT));
ASSERT_EQ(OB_SUCCESS, check_row.init(allocator_, TEST_COLUMN_CNT));
for (int64_t i = 0; i < start_seeds.count(); ++i) {
ObDatumRowkey tmp_rowkey;
mscan_ranges[i].border_flag_.set_inclusive_start();
mscan_ranges[i].border_flag_.set_inclusive_end();
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(start_seeds.at(i), start_row));
tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mscan_ranges[i].start_key_, allocator_));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(start_seeds.at(i) + count_per_range - 1, end_row));
tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mscan_ranges[i].end_key_, allocator_));
}
for (int64_t i = 0; i < start_seeds.count(); ++i) {
ASSERT_EQ(OB_SUCCESS, ranges.push_back(mscan_ranges[i]));
}
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(iter_param_, context_, &sstable_, &ranges));
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(iter_param_, context_, &partial_sstable_, &ranges));
for (int64_t i = 0; i < start_seeds.count(); ++i) {
for (int64_t j = 0; j < count_per_range; ++j) {
const int64_t k = is_reverse_scan ? start_seeds.at(i) + count_per_range - j - 1 : start_seeds.at(i) + j;
if (k < row_cnt_ || count_per_range == 1) {
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(k, check_row));
if (k < row_cnt_) {
ASSERT_EQ(OB_SUCCESS, scanner.inner_get_next_row(prow));
ASSERT_TRUE(*prow == check_row);
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_get_next_row(kv_prow));
ASSERT_TRUE(*kv_prow == check_row);
}
}
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
scanner.reuse();
ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
merge_ddl_scanner.reuse();
}
void TestDDLMergeRowMultiScanner::test_single_get_normal(const bool is_reverse_scan)
{
int ret = OB_SUCCESS;
ObArray<int64_t> seeds;
// prepare query param and context
prepare_query_param(is_reverse_scan);
// row in first macro
ret = seeds.push_back(3);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
// row in middle macro
seeds.reset();
seeds.push_back(row_cnt_ / 2);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
// row in last macro, in cache
seeds.reset();
seeds.push_back(row_cnt_ - 3);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
destroy_query_param();
}
void TestDDLMergeRowMultiScanner::test_single_get_border(const bool is_reverse_scan)
{
int ret = OB_SUCCESS;
ObArray<int64_t> seeds;
// prepare query param and context
prepare_query_param(is_reverse_scan);
// left border rowkey
ret = seeds.push_back(0);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
// right border rowkey
seeds.reset();
ret = seeds.push_back(row_cnt_ - 1);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
// not exist
seeds.reset();
ret = seeds.push_back(row_cnt_);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
destroy_query_param();
}
void TestDDLMergeRowMultiScanner::test_multi_get_normal(const bool is_reverse_scan)
{
int ret = OB_SUCCESS;
ObArray<int64_t> seeds;
// prepare query param
prepare_query_param(is_reverse_scan);
// 2 rows exist
seeds.reuse();
for (int64_t i = 0; i < 2; ++i) {
ret = seeds.push_back(i * 11 + 2);
ASSERT_EQ(OB_SUCCESS, ret);
}
test_one_case(seeds, 1, is_reverse_scan);
// TEST_MULTI_GET_CNT rows exist
seeds.reuse();
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ret = seeds.push_back(i);
ASSERT_EQ(OB_SUCCESS, ret);
}
test_one_case(seeds, 1, is_reverse_scan);
// 2 row2 not exist
seeds.reuse();
for (int64_t i = 0; i < 2; ++i) {
ret = seeds.push_back(i * 11 + 2 + row_cnt_);
ASSERT_EQ(OB_SUCCESS, ret);
}
test_one_case(seeds, 1, is_reverse_scan);
// TEST_MULTI_GET_CNT rows not exist
seeds.reuse();
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ret = seeds.push_back(i * 11 + 2 + row_cnt_);
ASSERT_EQ(OB_SUCCESS, ret);
}
test_one_case(seeds, 1, is_reverse_scan);
// some row exist, while other rows not exist
seeds.reuse();
for (int64_t i = 0; i < 10; ++i) {
ret = seeds.push_back(i + (i % 2 ? row_cnt_ : 0));
ASSERT_EQ(OB_SUCCESS, ret);
}
test_one_case(seeds, 1, is_reverse_scan);
destroy_query_param();
}
void TestDDLMergeRowMultiScanner::test_multi_get_border(const bool is_reverse_scan)
{
int ret = OB_SUCCESS;
ObArray<int64_t> seeds;
// prepare query param
prepare_query_param(is_reverse_scan);
// first row of sstable
ret = seeds.push_back(0);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, false);
// last row of sstable
seeds.reset();
ret = seeds.push_back(row_cnt_ - 1);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, false);
// 100 rows with same rowkey
seeds.reset();
for (int64_t i = 0; i < 100; ++i) {
ret = seeds.push_back(row_cnt_ / 2);
ASSERT_EQ(OB_SUCCESS, ret);
}
test_one_case(seeds, 1, is_reverse_scan);
destroy_query_param();
}
void TestDDLMergeRowMultiScanner::test_single_scan_normal(const bool is_reverse_scan)
{
int ret = OB_SUCCESS;
ObArray<int64_t> seeds;
ObRandom random;
const int64_t random_start = random.get(0, 10000000) % row_cnt_;
const int64_t random_end = random.get(0, 100000000) % row_cnt_;
const int64_t start = std::min(random_start, random_end);
const int64_t end = std::max(random_start, random_end);
// prepare query param
prepare_query_param(is_reverse_scan);
// multiple rows exist
ret = seeds.push_back(start);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, end - start, is_reverse_scan);
// multiple rows, partial exist
seeds.reset();
ret = seeds.push_back(start);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, row_cnt_ + 10 - start, is_reverse_scan);
// single row exist
seeds.reset();
ret = seeds.push_back(row_cnt_ / 2);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
// not exist
seeds.reset();
ret = seeds.push_back(row_cnt_ + 10);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 10, is_reverse_scan);
destroy_query_param();
}
void TestDDLMergeRowMultiScanner::test_single_scan_border(const bool is_reverse_scan)
{
int ret = OB_SUCCESS;
ObArray<int64_t> seeds;
// prepare query param
prepare_query_param(is_reverse_scan);
// full table scan
ret = seeds.push_back(0);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, row_cnt_, is_reverse_scan);
// first row of sstable
seeds.reset();
ret = seeds.push_back(0);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
// last row of sstable
seeds.reset();
ret = seeds.push_back(row_cnt_ - 1);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
// not exist
seeds.reset();
ret = seeds.push_back(row_cnt_);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
destroy_query_param();
}
TEST_F(TestDDLMergeRowMultiScanner, test_single_get_normal)
{
bool is_reverse_scan = false;
test_single_get_normal(is_reverse_scan);
is_reverse_scan = true;
test_single_get_normal(is_reverse_scan);
}
TEST_F(TestDDLMergeRowMultiScanner, test_single_get_border)
{
bool is_reverse_scan = false;
test_single_get_border(is_reverse_scan);
is_reverse_scan = true;
test_single_get_border(is_reverse_scan);
}
TEST_F(TestDDLMergeRowMultiScanner, test_multi_get_normal)
{
bool is_reverse_scan = false;
test_multi_get_normal(is_reverse_scan);
is_reverse_scan = true;
test_multi_get_normal(is_reverse_scan);
}
TEST_F(TestDDLMergeRowMultiScanner, test_multi_get_border)
{
bool is_reverse_scan = false;
test_multi_get_border(is_reverse_scan);
is_reverse_scan = true;
test_multi_get_border(is_reverse_scan);
}
TEST_F(TestDDLMergeRowMultiScanner, test_single_scan_normal)
{
bool is_reverse_scan = false;
test_single_scan_normal(is_reverse_scan);
is_reverse_scan = true;
test_single_scan_normal(is_reverse_scan);
}
TEST_F(TestDDLMergeRowMultiScanner, test_single_scan_border)
{
bool is_reverse_scan = false;
test_single_scan_border(is_reverse_scan);
is_reverse_scan = true;
test_single_scan_border(is_reverse_scan);
}
void TestDDLMergeRowMultiScanner::test_multi_scan_multi_scan_range(
const bool is_reverse_scan,
const int64_t count_per_range)
{
int ret = OB_SUCCESS;
ObStoreRange range;
ObArray<ObStoreRange> ranges;
ObStoreRow row;
ObStoreRowkey rowkey;
ObArray<int64_t> seeds;
ObSSTableRowMultiScanner<> scanner;
// prepare query param
prepare_query_param(is_reverse_scan);
//left border rowkey
seeds.reset();
ret = seeds.push_back(0);
ASSERT_EQ(OB_SUCCESS, seeds.push_back(0));
test_one_case(seeds, 1, is_reverse_scan);
//right border rowkey
seeds.reset();
ret = seeds.push_back(row_cnt_ - 1);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
//not exist rowkey
seeds.reset();
ret = seeds.push_back(row_cnt_);
ASSERT_EQ(OB_SUCCESS, ret);
test_one_case(seeds, 1, is_reverse_scan);
//20 exist
seeds.reset();
for (int64_t i = 0; i < 50; ++i) {
ret = seeds.push_back(i);
ASSERT_EQ(OB_SUCCESS, ret);
}
test_one_case(seeds, count_per_range, is_reverse_scan);
//20 dup exist
seeds.reset();
for (int64_t i = 0; i < 20; ++i) {
ret = seeds.push_back(0);
ASSERT_EQ(OB_SUCCESS, ret);
}
test_one_case(seeds, count_per_range, is_reverse_scan);
// 20 not exist
seeds.reset();
for (int64_t i = 0; i < 20; ++i) {
ret = seeds.push_back(row_cnt_ + i);
ASSERT_EQ(OB_SUCCESS, ret);
}
test_one_case(seeds, count_per_range, is_reverse_scan);
// partial not exist
seeds.reset();
for (int64_t i = 0; i < 20; ++i) {
ret = seeds.push_back(i % 2 ? i : i + row_cnt_);
ASSERT_EQ(OB_SUCCESS, ret);
}
test_one_case(seeds, count_per_range, is_reverse_scan);
destroy_query_param();
}
TEST_F(TestDDLMergeRowMultiScanner, test_multi_scan)
{
bool is_reverse_scan = false;
for (int64_t i = 2; i < 20; i += 10) {
test_multi_scan_multi_scan_range(is_reverse_scan, i);
}
is_reverse_scan = true;
for (int64_t i = 2; i < 20; i += 10) {
test_multi_scan_multi_scan_range(is_reverse_scan, i);
}
}
void TestDDLMergeRowMultiScanner::test_multi_scan_multi_get_with_scan(
const bool is_reverse_scan,
const int64_t count_per_range)
{
int ret = OB_SUCCESS;
ObDatumRange range;
ObArray<ObDatumRange> ranges;
ObDatumRow row;
ObDatumRowkey rowkey;
const ObDatumRow *prow = NULL;
const ObDatumRow *kv_prow = NULL;
int64_t row_cnt = 0;
ObSSTableRowMultiScanner<> scanner;
ObSSTableRowMultiScanner<> merge_ddl_scanner;
// prepare query param
prepare_query_param(is_reverse_scan);
// multi scan interact with multi get
ObDatumRange mget_ranges[TEST_MULTI_GET_CNT];
ObDatumRow start_row;
ObDatumRow end_row;
ObDatumRow check_row;
ASSERT_EQ(OB_SUCCESS, start_row.init(allocator_, TEST_COLUMN_CNT));
ASSERT_EQ(OB_SUCCESS, end_row.init(allocator_, TEST_COLUMN_CNT));
ASSERT_EQ(OB_SUCCESS, check_row.init(allocator_, TEST_COLUMN_CNT));
ranges.reuse();
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ObDatumRowkey tmp_rowkey;
mget_ranges[i].border_flag_.set_inclusive_start();
mget_ranges[i].border_flag_.set_inclusive_end();
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i, start_row));
tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].start_key_, allocator_));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i + (i % 2 ? count_per_range - 1 : 0), end_row));
tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].end_key_, allocator_));
}
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ret = ranges.push_back(mget_ranges[i]);
ASSERT_EQ(OB_SUCCESS, ret);
}
STORAGE_LOG(INFO, "multi scan begin");
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
iter_param_,
context_,
&sstable_,
&ranges));
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
iter_param_,
context_,
&partial_sstable_,
&ranges));
row_cnt = 0;
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
const int64_t p = i;
if (p % 2) {
for (int64_t j = 0; j < count_per_range; ++j) {
const int64_t k = is_reverse_scan ? i + count_per_range - j - 1 : i + j;
ASSERT_EQ(OB_SUCCESS, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_get_next_row(kv_prow));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(k, check_row));
++row_cnt;
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
} else {
ASSERT_EQ(OB_SUCCESS, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_get_next_row(kv_prow));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(p, check_row));
++row_cnt;
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
merge_ddl_scanner.reuse();
// first half multi scan, second half multi get
ranges.reuse();
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ObDatumRowkey tmp_rowkey;
mget_ranges[i].border_flag_.set_inclusive_start();
mget_ranges[i].border_flag_.set_inclusive_end();
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i, start_row));
tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].start_key_, allocator_));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i + (i < TEST_MULTI_GET_CNT / 2 ? count_per_range - 1 : 0), end_row));
tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].end_key_, allocator_));
ASSERT_EQ(OB_SUCCESS, ranges.push_back(mget_ranges[i]));
}
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
iter_param_,
context_,
&sstable_,
&ranges));
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
iter_param_,
context_,
&partial_sstable_,
&ranges));
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
const int64_t p = i;
if (p < TEST_MULTI_GET_CNT / 2) {
for (int64_t j = 0; j < count_per_range; ++j) {
const int64_t k = is_reverse_scan ? i + count_per_range - j - 1 : i + j;
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(k, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
} else {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(p, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
merge_ddl_scanner.reuse();
// first one multi get, others multi scan
ranges.reuse();
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ObDatumRowkey tmp_rowkey;
mget_ranges[i].border_flag_.set_inclusive_start();
mget_ranges[i].border_flag_.set_inclusive_end();
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i, start_row));
tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].start_key_, allocator_));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i + (i != 0 ? count_per_range - 1 : 0), end_row));
tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].end_key_, allocator_));
ASSERT_EQ(OB_SUCCESS, ranges.push_back(mget_ranges[i]));
}
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
iter_param_,
context_,
&sstable_,
&ranges));
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
iter_param_,
context_,
&partial_sstable_,
&ranges));
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
const int64_t p = i;
if (p != 0) {
for (int64_t j = 0; j < count_per_range; ++j) {
const int64_t k = is_reverse_scan ? i + count_per_range - j - 1 : i + j;
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(k, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
} else {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(p, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
merge_ddl_scanner.reuse();
// multi scan not exist row
STORAGE_LOG(DEBUG, "multi_scan_not_exist_row");
ranges.reuse();
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
ObDatumRowkey tmp_rowkey;
mget_ranges[i].border_flag_.set_inclusive_start();
mget_ranges[i].border_flag_.set_inclusive_end();
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i + (i % 2 ? row_cnt_ : 0), start_row));
tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].start_key_, allocator_));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(i + (i % 2 ? row_cnt_ + count_per_range - 1 : 0), end_row));
tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(mget_ranges[i].end_key_, allocator_));
ASSERT_EQ(OB_SUCCESS, ranges.push_back(mget_ranges[i]));
}
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
iter_param_,
context_,
&sstable_,
&ranges));
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
iter_param_,
context_,
&partial_sstable_,
&ranges));
for (int64_t i = 0; i < TEST_MULTI_GET_CNT; ++i) {
const int64_t p = i;
if (p % 2) {
continue;
} else {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret);
ret = row_generate_.get_next_row(p, check_row);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(*prow == check_row);
ASSERT_TRUE(*kv_prow == check_row);
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
scanner.reset();
destroy_query_param();
}
TEST_F(TestDDLMergeRowMultiScanner, test_multi_get_with_scan)
{
bool is_reverse_scan = false;
for (int64_t i = 2; i < 10; i += 10) {
test_multi_scan_multi_get_with_scan(is_reverse_scan, i);
}
is_reverse_scan = true;
for (int64_t i = 2; i < 10; i += 10) {
test_multi_scan_multi_get_with_scan(is_reverse_scan, i);
}
}
}
}
int main(int argc, char **argv)
{
system("rm -f test_ddl_merge_row_multi_scanner.log*");
OB_LOGGER.set_file_name("test_ddl_merge_row_multi_scanner.log", true, true);
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

537
mittest/mtlenv/storage/blocksstable/test_ddl_merge_row_scanner.cpp Normal file
View File

@ -0,0 +1,537 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#define private public
#define protected public
#include "lib/random/ob_random.h"
#include "storage/access/ob_index_tree_prefetcher.h"
#include "storage/access/ob_sstable_row_getter.h"
#include "storage/access/ob_sstable_row_scanner.h"
#include "ob_index_block_data_prepare.h"
namespace oceanbase
{
using namespace storage;
using namespace common;
namespace blocksstable
{
class TestDDLMergeRowScanner : public TestIndexBlockDataPrepare
{
public:
TestDDLMergeRowScanner();
virtual ~TestDDLMergeRowScanner();
static void SetUpTestCase();
static void TearDownTestCase();
virtual void SetUp();
virtual void TearDown();
void generate_range(const int64_t start, const int64_t end, ObDatumRange &range);
void test_one_rowkey(const int64_t seed); //get
void test_single_case(
const ObDatumRange &range,
const int64_t start,
const int64_t end,
const bool is_reverse_scan);
void test_full_case(
const ObDatumRange &range,
const int64_t start,
const int64_t end,
const bool is_reverse_scan,
const int64_t hit_mode);
void test_border(const bool is_reverse_scan);
void test_basic(const bool is_reverse_scan);
protected:
enum CacheHitMode
{
HIT_ALL = 0,
HIT_NONE,
HIT_PART,
HIT_MAX,
};
private:
ObArenaAllocator allocator_;
ObDatumRow start_row_;
ObDatumRow end_row_;
};
TestDDLMergeRowScanner::TestDDLMergeRowScanner()
: TestIndexBlockDataPrepare("Test DDL multi row scanner")
{
is_ddl_merge_data_ = true;
max_row_cnt_ = 150000;
max_partial_row_cnt_ = 78881;
partial_kv_start_idx_ = 3;
}
TestDDLMergeRowScanner::~TestDDLMergeRowScanner()
{
}
void TestDDLMergeRowScanner::SetUpTestCase()
{
TestIndexBlockDataPrepare::SetUpTestCase();
}
void TestDDLMergeRowScanner::TearDownTestCase()
{
TestIndexBlockDataPrepare::TearDownTestCase();
}
void TestDDLMergeRowScanner::SetUp()
{
TestIndexBlockDataPrepare::SetUp();
ASSERT_EQ(OB_SUCCESS, start_row_.init(allocator_, TEST_COLUMN_CNT));
ASSERT_EQ(OB_SUCCESS, end_row_.init(allocator_, TEST_COLUMN_CNT));
ObLSID ls_id(ls_id_);
ObTabletID tablet_id(tablet_id_);
ObLSHandle ls_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_));
}
void TestDDLMergeRowScanner::TearDown()
{
tablet_handle_.reset();
TestIndexBlockDataPrepare::TearDown();
}
void TestDDLMergeRowScanner::generate_range(
const int64_t start,
const int64_t end,
ObDatumRange &range)
{
ObDatumRowkey tmp_rowkey;
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(start, start_row_));
tmp_rowkey.assign(start_row_.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(range.start_key_, allocator_));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(end, end_row_));
tmp_rowkey.assign(end_row_.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(range.end_key_, allocator_));
range.border_flag_.set_inclusive_start();
range.border_flag_.set_inclusive_end();
}
void TestDDLMergeRowScanner::test_one_rowkey(const int64_t seed)
{
ObSSTableRowGetter getter;
ObSSTableRowGetter merge_ddl_getter;
ObDatumRow query_row;
ASSERT_EQ(OB_SUCCESS, query_row.init(allocator_, TEST_COLUMN_CNT));
row_generate_.get_next_row(seed, query_row);
ObDatumRowkey query_rowkey;
query_rowkey.assign(query_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
STORAGE_LOG(INFO, "Query rowkey", K(query_row));
ASSERT_EQ(OB_SUCCESS, getter.inner_open(iter_param_, context_, &sstable_, &query_rowkey));
ASSERT_EQ(OB_SUCCESS, merge_ddl_getter.inner_open(iter_param_, context_, &partial_sstable_, &query_rowkey));
const ObDatumRow *prow = nullptr;
const ObDatumRow *kv_prow = nullptr;
ASSERT_EQ(OB_SUCCESS, getter.inner_get_next_row(prow));
ASSERT_EQ(OB_SUCCESS, merge_ddl_getter.inner_get_next_row(kv_prow));
STORAGE_LOG(INFO, "debug datum row1", KPC(prow), KPC(kv_prow));
if (seed >= row_cnt_) {
ASSERT_TRUE(prow->row_flag_.is_not_exist());
ASSERT_TRUE(kv_prow->row_flag_.is_not_exist());
} else {
ASSERT_TRUE(*prow == query_row);
if (!(*kv_prow == query_row)) {
STORAGE_LOG(INFO, "CHECK UNEUQAL", K(query_row), K(*kv_prow));
}
ASSERT_TRUE(*kv_prow == query_row);
}
ASSERT_EQ(OB_ITER_END, getter.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, merge_ddl_getter.inner_get_next_row(kv_prow));
getter.reuse();
merge_ddl_getter.reuse();
}
void TestDDLMergeRowScanner::test_single_case(
const ObDatumRange &range,
const int64_t start,
const int64_t end,
const bool is_reverse_scan)
{
int ret = OB_SUCCESS;
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(allocator_, TEST_COLUMN_CNT));
const ObDatumRow *prow = nullptr;
const ObDatumRow *kv_prow = nullptr;
ObSSTableRowScanner<> scanner;
ObSSTableRowScanner<> merge_ddl_scanner;
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
iter_param_,
context_,
&sstable_,
&range));
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
iter_param_,
context_,
&partial_sstable_,
&range));
for (int64_t i = start; i <= end; ++i) {
int64_t index = 0;
if (is_reverse_scan) {
ret = row_generate_.get_next_row(end - i + start, row);
index = end - i + start;
} else {
ret = row_generate_.get_next_row(i, row);
index = i;
}
ASSERT_EQ(OB_SUCCESS, ret);
if (index < row_cnt_) {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
<< " end: " << end << " prow: " << prow;
ASSERT_TRUE(row == *prow) << i << "index: " << index << " start: " << start
<< " end: " << end << " prow: " << prow;
ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
<< " end: " << end << " kv_prow: " << kv_prow;
ASSERT_TRUE(row == *kv_prow) << i << "index: " << index << " start: " << start
<< " end: " << end << " kv_prow: " << kv_prow;
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
merge_ddl_scanner.reuse();
}
void TestDDLMergeRowScanner::test_full_case(
const ObDatumRange &range,
const int64_t start,
const int64_t end,
const bool is_reverse_scan,
const int64_t hit_mode)
{
int ret = OB_SUCCESS;
ObDatumRow row;
ASSERT_EQ(OB_SUCCESS, row.init(allocator_, TEST_COLUMN_CNT));
const ObDatumRow *prow = nullptr;
const ObDatumRow *kv_prow = nullptr;
ObSSTableRowScanner<> scanner;
ObSSTableRowScanner<> merge_ddl_scanner;
if (HIT_PART == hit_mode) {
const int64_t part_start = start + (end - start) / 3;
const int64_t part_end = end - (end - start) / 3;
ObDatumRange part_range;
ObDatumRow start_row;
ObDatumRow end_row;
ObDatumRowkey tmp_rowkey;
ASSERT_EQ(OB_SUCCESS, start_row.init(allocator_, TEST_COLUMN_CNT));
ASSERT_EQ(OB_SUCCESS, end_row.init(allocator_, TEST_COLUMN_CNT));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(part_start, start_row));
tmp_rowkey.assign(start_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(part_range.start_key_, allocator_));
ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(part_end, end_row));
tmp_rowkey.assign(end_row.storage_datums_, TEST_ROWKEY_COLUMN_CNT);
ASSERT_EQ(OB_SUCCESS, tmp_rowkey.deep_copy(part_range.end_key_, allocator_));
part_range.border_flag_.set_inclusive_start();
part_range.border_flag_.set_inclusive_end();
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
iter_param_,
context_,
&sstable_,
&part_range));
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
iter_param_,
context_,
&partial_sstable_,
&part_range));
for (int64_t i = part_start; i <= part_end; ++i) {
if (i < row_cnt_) {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret) << "i: " << i << " part_start: " << part_start
<< " part_end: " << part_end << " prow: " << prow;
ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret) << "i: " << i << " part_start: " << part_start
<< " part_end: " << part_end << " kv_prow: " << kv_prow;
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
merge_ddl_scanner.reuse();
}
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
iter_param_,
context_,
&sstable_,
&range));
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
iter_param_,
context_,
&partial_sstable_,
&range));
for (int64_t i = start; i <= end; ++i) {
int64_t index = 0;
if (is_reverse_scan) {
ret = row_generate_.get_next_row(end - i + start, row);
index = end - i + start;
} else {
ret = row_generate_.get_next_row(i, row);
index = i;
}
ASSERT_EQ(OB_SUCCESS, ret);
if (index < row_cnt_) {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
<< " end: " << end << " prow: " << prow;
ASSERT_TRUE(row == *prow) << i << "index: " << index << " start: " << start
<< " end: " << end << " prow: " << prow;
ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
<< " end: " << end << " kv_prow: " << kv_prow;
ASSERT_TRUE(row == *kv_prow) << i << "index: " << index << " start: " << start
<< " end: " << end << " kv_prow: " << kv_prow;
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
merge_ddl_scanner.reuse();
if (HIT_ALL == hit_mode) {
int64_t index = 0;
ASSERT_EQ(OB_SUCCESS, scanner.inner_open(
iter_param_,
context_,
&sstable_,
&range));
ASSERT_EQ(OB_SUCCESS, merge_ddl_scanner.inner_open(
iter_param_,
context_,
&partial_sstable_,
&range));
for (int64_t i = start; i <= end; ++i) {
if (is_reverse_scan) {
ret = row_generate_.get_next_row(end - i + start, row);
index = end - i + start;
} else {
ret = row_generate_.get_next_row(i, row);
index = i;
}
ASSERT_EQ(OB_SUCCESS, ret);
if (index < row_cnt_) {
ret = scanner.inner_get_next_row(prow);
ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
<< " end: " << end << " prow: " << prow;
ASSERT_TRUE(row == *prow) << i << "index: " << index << " start: " << start
<< " end: " << end << " prow: " << prow;
ret = merge_ddl_scanner.inner_get_next_row(kv_prow);
ASSERT_EQ(OB_SUCCESS, ret) << i << "index: " << index << " start: " << start
<< " end: " << end << " kv_prow: " << kv_prow;
ASSERT_TRUE(row == *kv_prow) << i << "index: " << index << " start: " << start
<< " end: " << end << " kv_prow: " << kv_prow;
}
}
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, scanner.inner_get_next_row(prow));
ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
ASSERT_EQ(OB_ITER_END, merge_ddl_scanner.inner_get_next_row(kv_prow));
scanner.reuse();
merge_ddl_scanner.reuse();
}
}
void TestDDLMergeRowScanner::test_border(const bool is_reverse_scan)
{
int ret = OB_SUCCESS;
ObDatumRange range;
// prepare query param
prepare_query_param(is_reverse_scan);
// full table scan
range.set_whole_range();
for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
test_full_case(range, 0, row_cnt_ - 1, is_reverse_scan, i);
}
// the first row of sstable
generate_range(0, 0, range);
for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
test_full_case(range, 0, 0, is_reverse_scan, i);
}
// the first 100 row of sstable
generate_range(0, 100, range);
for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
test_full_case(range, 0, 100, is_reverse_scan, i);
}
// the last row of sstable
generate_range(row_cnt_ - 1, row_cnt_ - 1, range);
for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
test_full_case(range, row_cnt_ - 1, row_cnt_ - 1, is_reverse_scan, i);
}
// the last 100 row of sstable
generate_range(row_cnt_ - 100, row_cnt_ - 1, range);
for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
test_full_case(range, row_cnt_ - 100, row_cnt_ - 1, is_reverse_scan, i);
}
// not exist
generate_range(row_cnt_, row_cnt_, range);
for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
test_full_case(range, row_cnt_, row_cnt_, is_reverse_scan, i);
}
destroy_query_param();
}
void TestDDLMergeRowScanner::test_basic(const bool is_reverse_scan)
{
int ret = OB_SUCCESS;
ObDatumRange range;
prepare_query_param(is_reverse_scan);
// full table scan
range.set_whole_range();
test_single_case(range, 0, row_cnt_ - 1, is_reverse_scan);
// the first row of sstable
generate_range(0, 0, range);
test_single_case(range, 0, 0, is_reverse_scan);
// the first 100 row of sstable
generate_range(0, 100, range);
test_single_case(range, 0, 100, is_reverse_scan);
// the last row of sstable
generate_range(row_cnt_ - 1, row_cnt_ - 1, range);
test_single_case(range, row_cnt_ - 1, row_cnt_ - 1, is_reverse_scan);
// the last 100 row of sstable
generate_range(row_cnt_ - 100, row_cnt_ - 1, range);
test_single_case(range, row_cnt_ - 100, row_cnt_ - 1, is_reverse_scan);
// not exist
generate_range(row_cnt_, row_cnt_, range);
test_single_case(range, row_cnt_, row_cnt_, is_reverse_scan);
destroy_query_param();
}
TEST_F(TestDDLMergeRowScanner, test_get)
{
// forward get
prepare_query_param(false);
// left border rowkey
test_one_rowkey(0);
// end_key of left border
test_one_rowkey(15933);
// first_key of right border
test_one_rowkey(132361);
// end_key of right border
test_one_rowkey(row_cnt_ - 1);
// mid border rowkey
test_one_rowkey(row_cnt_ / 2);
// non-exist rowkey
test_one_rowkey(row_cnt_);
destroy_query_param();
// reverse get
prepare_query_param(true);
// left border rowkey
test_one_rowkey(0);
// end_key of left border
test_one_rowkey(15933);
// first_key of right border
test_one_rowkey(132361);
// end_key of right border
test_one_rowkey(row_cnt_ - 1);
// mid border rowkey
test_one_rowkey(row_cnt_ / 2);
// non-exist rowkey
test_one_rowkey(row_cnt_);
destroy_query_param();
}
TEST_F(TestDDLMergeRowScanner, test_basic_scan)
{
bool is_reverse_scan = false;
test_basic(is_reverse_scan);
is_reverse_scan = true;
test_basic(is_reverse_scan);
STORAGE_LOG(INFO, "memory usage", K(lib::get_memory_hold()), K(lib::get_memory_limit()));
ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500);
ObMallocAllocator::get_instance()->print_tenant_memory_usage(500);
ObMallocAllocator::get_instance()->print_tenant_memory_usage(1);
}
TEST_F(TestDDLMergeRowScanner, test_border_scan)
{
bool is_reverse_scan = false;
test_border(is_reverse_scan);
is_reverse_scan = true;
test_border(is_reverse_scan);
STORAGE_LOG(INFO, "memory usage", K(lib::get_memory_hold()), K(lib::get_memory_limit()));
ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500);
ObMallocAllocator::get_instance()->print_tenant_memory_usage(500);
ObMallocAllocator::get_instance()->print_tenant_memory_usage(1);
}
TEST_F(TestDDLMergeRowScanner, test_random_scan)
{
ObDatumRange range;
int64_t start = ObRandom::rand(0, row_cnt_ - 1);
int64_t end = ObRandom::rand(0, row_cnt_ - 1);
if (start > end) {
int64_t temp = start;
start = end;
end = temp;
}
generate_range(start, end, range);
bool is_reverse_scan = false;
// prepare query param
prepare_query_param(is_reverse_scan);
for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
test_full_case(range, start, end, is_reverse_scan, i);
}
destroy_query_param();
is_reverse_scan = true;
// prepare query param
prepare_query_param(is_reverse_scan);
for (int64_t i = HIT_ALL; i < HIT_MAX; ++i) {
test_full_case(range, start, end, is_reverse_scan, i);
}
destroy_query_param();
}
}
}
int main(int argc, char **argv)
{
system("rm -f test_ddl_merge_row_scanner.log*");
OB_LOGGER.set_file_name("test_ddl_merge_row_scanner.log", true, true);
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

92
mittest/mtlenv/storage/blocksstable/test_direct_load.cpp Normal file
View File

@ -0,0 +1,92 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#define private public
#define protected public
#include "ob_index_block_data_prepare.h"
#include "storage/ddl/ob_direct_insert_sstable_ctx_new.h"
namespace oceanbase
{
using namespace lib;
using namespace common;
using namespace share;
using namespace storage;
using namespace blocksstable;
class TestDirectLoad : public TestIndexBlockDataPrepare
{
public:
TestDirectLoad() : TestIndexBlockDataPrepare("Test direct load") {}
virtual ~TestDirectLoad() {}
static void SetUpTestCase() { TestIndexBlockDataPrepare::SetUpTestCase(); }
static void TearDownTestCase() { TestIndexBlockDataPrepare::TearDownTestCase(); }
virtual void SetUp();
virtual void TearDown();
};
void TestDirectLoad::SetUp()
{
TestIndexBlockDataPrepare::SetUp();
ObLSID ls_id(ls_id_);
ObTabletID tablet_id(tablet_id_);
ObLSHandle ls_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_));
prepare_query_param(true);
}
void TestDirectLoad::TearDown()
{
destroy_query_param();
tablet_handle_.reset();
TestIndexBlockDataPrepare::TearDown();
}
TEST_F(TestDirectLoad, init_ddl_table_store)
{
ObTabletFullDirectLoadMgr tablet_dl_mgr;
ObTabletDirectLoadInsertParam build_param;
build_param.common_param_.ls_id_ = ls_id_;
build_param.common_param_.tablet_id_ = tablet_id_;
build_param.common_param_.direct_load_type_ = ObDirectLoadType::DIRECT_LOAD_DDL;
build_param.common_param_.read_snapshot_ = SNAPSHOT_VERSION;
build_param.runtime_only_param_.task_cnt_ = 1;
build_param.runtime_only_param_.task_id_ = 1;
build_param.runtime_only_param_.table_id_ = TEST_TABLE_ID;
build_param.runtime_only_param_.schema_version_ = 1;
SCN ddl_start_scn;
ASSERT_EQ(OB_SUCCESS, ddl_start_scn.convert_from_ts(ObTimeUtility::current_time()));
ASSERT_EQ(OB_SUCCESS, tablet_dl_mgr.update(nullptr, build_param));
tablet_dl_mgr.start_scn_ = ddl_start_scn;
tablet_dl_mgr.data_format_version_ = DATA_VERSION_4_0_0_0;
ASSERT_EQ(OB_SUCCESS, tablet_dl_mgr.init_ddl_table_store(ddl_start_scn, SNAPSHOT_VERSION, ddl_start_scn));
}
} // namespace oceanbase
int main(int argc, char **argv)
{
system("rm -f test_direct_load.log*");
OB_LOGGER.set_file_name("test_direct_load.log", true, true);
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

2
mittest/mtlenv/storage/blocksstable/test_index_block_row_scanner.cpp
View File

@ -204,7 +204,6 @@ TEST_F(TestIndexBlockRowScanner, prefetch_and_scan)
ObIndexBlockRowHeader::DEFAULT_IDX_ROW_MACRO_ID,
*raw_block,
root_blk_header->rowkey_array_[root_row_id]));
ASSERT_EQ(idx_scanner.current_, raw_idx_scanner.current_);
idx_scanner.reuse();
raw_idx_scanner.reuse();
@ -224,7 +223,6 @@ TEST_F(TestIndexBlockRowScanner, prefetch_and_scan)
int tmp_ret = OB_SUCCESS;
while (OB_SUCCESS == tmp_ret) {
ASSERT_EQ(idx_scanner.current_, raw_idx_scanner.current_);
tmp_ret = idx_scanner.get_next(read_idx_info);
if (OB_SUCCESS == tmp_ret) {
ASSERT_EQ(tmp_ret, raw_idx_scanner.get_next(raw_read_idx_info));

8
mittest/mtlenv/storage/blocksstable/test_sstable_row_getter.cpp
View File

@ -113,7 +113,13 @@ TEST_F(TestSSTableRowGetter, get)
//left border rowkey
test_one_rowkey(0);
// right border rowkey
//end_key of left border
test_one_rowkey(15933);
//first_key of right border
test_one_rowkey(64760);
// end_key of right border
test_one_rowkey(row_cnt_ - 1);
// mid border rowkey

2
mittest/mtlenv/storage/test_co_merge.cpp
View File

@ -299,6 +299,7 @@ void TestCOMerge::prepare_scan_param(
iter_param.vectorized_enabled_ = false;
ASSERT_EQ(OB_SUCCESS,
store_ctx.init_for_read(ls_id,
iter_param.tablet_id_,
INT64_MAX, // query_expire_ts
-1, // lock_timeout_us
share::SCN::max_scn()));
@ -436,6 +437,7 @@ void TestCOMerge::prepare_query_param(const ObVersionRange &version_range)
iter_param_.vectorized_enabled_ = false;
ASSERT_EQ(OB_SUCCESS,
store_ctx_.init_for_read(ls_id,
iter_param_.tablet_id_,
INT64_MAX, // query_expire_ts
-1, // lock_timeout_us
share::SCN::max_scn()));

76
mittest/mtlenv/storage/test_index_sstable_estimator.cpp
View File

@ -31,7 +31,7 @@ namespace unittest
class TestIndexSSTableEstimator : public TestIndexBlockDataPrepare
{
public:
TestIndexSSTableEstimator() : TestIndexBlockDataPrepare("Test index sstable estimator") {}
TestIndexSSTableEstimator();
virtual ~TestIndexSSTableEstimator() {}
virtual void SetUp();
virtual void TearDown();
@ -46,6 +46,14 @@ private:
ObDatumRow end_row_;
};
TestIndexSSTableEstimator::TestIndexSSTableEstimator()
: TestIndexBlockDataPrepare("Test index sstable estimator")
{
is_ddl_merge_data_ = true;
max_row_cnt_ = 150000;
max_partial_row_cnt_ = 78881;
partial_kv_start_idx_ = 3;
}
void TestIndexSSTableEstimator::SetUpTestCase()
{
TestIndexBlockDataPrepare::SetUpTestCase();
@ -113,12 +121,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_whole_range)
range.set_whole_range();
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
ASSERT_EQ(part_est, ddl_merge_part_est);
}
TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_range)
@ -126,12 +143,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_range)
ObDatumRange range;
generate_range(100, -1, range);
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
ASSERT_EQ(part_est, ddl_merge_part_est);
}
TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_left_range)
@ -139,12 +165,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_left_range)
ObDatumRange range;
generate_range(-1, 100, range);
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
ASSERT_EQ(part_est, ddl_merge_part_est);
}
TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_right_range)
@ -152,12 +187,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_right_range)
ObDatumRange range;
generate_range(row_cnt_ - 100, -1, range);
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
ASSERT_EQ(part_est, ddl_merge_part_est);
}
TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_middle_range)
@ -165,12 +209,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_middle_range)
ObDatumRange range;
generate_range(100, row_cnt_ - 100, range);
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
ASSERT_EQ(part_est, ddl_merge_part_est);
}
TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_noexist_range)
@ -178,12 +231,21 @@ TEST_F(TestIndexSSTableEstimator, estimate_major_sstable_noexist_range)
ObDatumRange range;
generate_range(row_cnt_, row_cnt_, range);
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
ASSERT_EQ(part_est, ddl_merge_part_est);
}
} // end namespace unittest

71
mittest/mtlenv/storage/test_index_sstable_multi_estimator.cpp
View File

@ -31,7 +31,7 @@ namespace unittest
class TestMultiVersionIndexSSTableEstimator : public TestIndexBlockDataPrepare
{
public:
TestMultiVersionIndexSSTableEstimator() : TestIndexBlockDataPrepare("Test multi version index sstable estimator", compaction::MINI_MERGE) {}
TestMultiVersionIndexSSTableEstimator();
virtual ~TestMultiVersionIndexSSTableEstimator() {}
virtual void SetUp();
virtual void TearDown();
@ -48,6 +48,15 @@ private:
ObDatumRow end_row_;
};
TestMultiVersionIndexSSTableEstimator::TestMultiVersionIndexSSTableEstimator()
: TestIndexBlockDataPrepare("Test multi version index sstable estimator", MINI_MERGE)
{
is_ddl_merge_data_ = true;
max_row_cnt_ = 150000;
max_partial_row_cnt_ = 137312;
partial_kv_start_idx_ = 29;
}
void TestMultiVersionIndexSSTableEstimator::SetUpTestCase()
{
TestIndexBlockDataPrepare::SetUpTestCase();
@ -114,12 +123,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_minor_sstable_whole_range
ObDatumRange range;
range.set_whole_range();
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
}
TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_minor_sstable_range)
@ -127,12 +144,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_minor_sstable_range)
ObDatumRange range;
generate_range(100, -1, range);
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
}
TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_left_range)
@ -140,12 +165,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_left_range)
ObDatumRange range;
generate_range(-1, 100, range);
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
}
TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_right_range)
@ -153,12 +186,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_right_range
ObDatumRange range;
generate_range(row_cnt_ - 100, -1, range);
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
}
TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_middle_range)
@ -166,12 +207,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_middle_rang
ObDatumRange range;
generate_range(100, row_cnt_ - 100, range);
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
}
TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_noexist_range)
@ -179,12 +228,20 @@ TEST_F(TestMultiVersionIndexSSTableEstimator, estimate_major_sstable_noexist_ran
ObDatumRange range;
generate_range(row_cnt_, row_cnt_, range);
ObIndexSSTableEstimateContext esti_ctx(sstable_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_kv_esti_ctx(ddl_kv_, tablet_handle_, context_.query_flag_, range);
ObIndexSSTableEstimateContext ddl_merge_esti_ctx(partial_sstable_, tablet_handle_, context_.query_flag_, range);
ObPartitionEst part_est;
ObIndexBlockScanEstimator estimator(esti_ctx);
ObPartitionEst ddl_kv_part_est;
ObIndexBlockScanEstimator ddl_kv_estimator(ddl_kv_esti_ctx);
ObPartitionEst ddl_merge_part_est;
ObIndexBlockScanEstimator ddl_merge_estimator(ddl_merge_esti_ctx);
ASSERT_EQ(OB_SUCCESS, estimator.estimate_row_count(part_est));
STORAGE_LOG(INFO, "part_est", K(part_est));
ASSERT_EQ(OB_SUCCESS, ddl_kv_estimator.estimate_row_count(ddl_kv_part_est));
ASSERT_EQ(OB_SUCCESS, ddl_merge_estimator.estimate_row_count(ddl_merge_part_est));
STORAGE_LOG(INFO, "part_est", K(part_est), K(ddl_kv_part_est), K(ddl_merge_part_est));
}

204
mittest/mtlenv/storage/test_lob_meta_iterator.cpp Normal file
View File

@ -0,0 +1,204 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include <iostream>
#include <codecvt>
#include <stdio.h>
#define private public
#define protected public
#include "common/object/ob_obj_type.h"
#include "common/object/ob_object.h"
#include "share/schema/ob_table_param.h"
#include "storage/blocksstable/ob_datum_row.h"
#include "storage/lob/ob_lob_util.h"
#include "storage/lob/ob_lob_meta.h"
#include "lib/number/ob_number_v2.h"
#include "share/schema/ob_column_schema.h"
#include "share/ob_ls_id.h"
#include "mtlenv/mock_tenant_module_env.h"
namespace oceanbase
{
namespace storage
{
class TestLobMetaIterator : public ::testing::Test
{
public:
TestLobMetaIterator() = default;
virtual ~TestLobMetaIterator() = default;
static void SetUpTestCase();
static void TearDownTestCase();
int fill_lob_sstable_slice_mock(
const ObLobId &lob_id,
const transaction::ObTransID &trans_id,
const int64_t trans_version,
const int64_t sql_no,
const bool has_lob_header,
const int64_t read_snapshot,
const ObCollationType collation_type,
blocksstable::ObStorageDatum &datum);
int build_lob_data(ObObj &obj, std::string &st, common::ObArenaAllocator &allocator);
int build_lob_data_not_common(ObObj &obj, std::string &str);
};
void TestLobMetaIterator::SetUpTestCase()
{
EXPECT_EQ(OB_SUCCESS, MockTenantModuleEnv::get_instance().init());
}
void TestLobMetaIterator::TearDownTestCase()
{
MockTenantModuleEnv::get_instance().destroy();
}
int TestLobMetaIterator::fill_lob_sstable_slice_mock(
const ObLobId &lob_id,
const transaction::ObTransID &trans_id,
const int64_t trans_version,
const int64_t sql_no,
const bool has_lob_header,
const int64_t read_snapshot,
const ObCollationType collation_type,
blocksstable::ObStorageDatum &datum)
{
int ret = OB_SUCCESS;
int64_t ils_id = 1001;
int64_t itablet_id = 1001;
common::ObArenaAllocator allocator;
share::ObLSID ls_id(ils_id);
common::ObTabletID tablet_id(itablet_id);
const blocksstable::ObDatumRow *new_row = nullptr;
ObLobMetaInfo lob_meta_info;
const int64_t timeout_ts = ObTimeUtility::current_time();
ObString data = datum.get_string();
ObLobMetaWriteIter iter(data, &allocator, ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE);
ObLobMetaRowIterator row_iterator;
if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column(
allocator, nullptr, ls_id, tablet_id, lob_id, collation_type, datum, timeout_ts, has_lob_header, iter))) {
STORAGE_LOG(WARN, "fail to insert lob column", K(ret), K(ls_id), K(tablet_id), K(lob_id));
} else if (OB_FAIL(row_iterator.init(&iter, trans_id, trans_version, sql_no))) {
STORAGE_LOG(WARN, "fail to init lob meta row iterator", K(ret), K(trans_id), K(trans_version));
} else if (OB_FAIL(row_iterator.get_next_row(new_row))) {
STORAGE_LOG(WARN, "get_next_row failed", K(ret));
} else if (OB_ISNULL(new_row)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "new_row is nullptr", K(ret));
} else if (OB_FAIL(ObLobMetaUtil::transform_from_row_to_info(new_row, lob_meta_info, true))) {
STORAGE_LOG(WARN, "transform failed", K(ret));
} else if (lob_meta_info.lob_id_.lob_id_ != lob_id.lob_id_) {
//STORAGE_LOG("lob_meta_info", K(lob_meta_info));
STORAGE_LOG(WARN, "error info", K(ret), K(lob_meta_info.lob_id_.lob_id_), K(lob_id.lob_id_));
}
return ret;
}
int TestLobMetaIterator::build_lob_data(ObObj &obj, std::string &st, common::ObArenaAllocator &allocator)
{
int ret = OB_SUCCESS;
ObLobCommon *value = NULL;
void *buf = NULL;
if (OB_ISNULL(buf = allocator.alloc(sizeof(ObLobCommon) + 1000000))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "fail to allocate memory for ObLobData", K(ret));
} else {
// ObLobIndex index;
value = new (buf) ObLobCommon();
// value->version_ = 1;
// value->reserve_ = 0;
// value->is_init_ = 1;
int64_t byte_size = 1000000;
value->in_row_ = 1;
MEMCPY(value->buffer_, st.c_str(), st.length());
obj.meta_.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI);
obj.meta_.set_collation_level(CS_LEVEL_IMPLICIT);
obj.set_type(ObMediumTextType);
obj.set_lob_value(ObMediumTextType, value, value->get_handle_size(byte_size));
obj.set_has_lob_header();
}
return ret;
}
int TestLobMetaIterator::build_lob_data_not_common(ObObj &obj, std::string &str)
{
int ret = OB_SUCCESS;
obj.meta_.set_collation_type(CS_TYPE_UTF8MB4_GENERAL_CI);
obj.meta_.set_collation_level(CS_LEVEL_IMPLICIT);
obj.set_type(ObMediumTextType);
obj.set_lob_value(ObMediumTextType, str.c_str(), str.length());
return ret;
}
TEST_F(TestLobMetaIterator, test_not_lob_common)
{
for (int x = 2; x <= 10; x++) {
std::string st = "";
for (int i = 0; i < 1000000; i++) {
st += static_cast<char>(i % 26 + 'a');
}
ObDatumRow row;
ObObj obj;
ASSERT_EQ(OB_SUCCESS, build_lob_data_not_common(obj, st));
ObStorageDatum datum;
ASSERT_EQ(OB_SUCCESS, datum.from_obj_enhance(obj));
transaction::ObTransID trans_id(10);
ObLobId lob_id;
lob_id.lob_id_ = x;
lob_id.tablet_id_ = x;
int64_t sql_no = 2;
int64_t read_snapshot = 3;
ObCollationType collation_type = CS_TYPE_UTF8MB4_GENERAL_CI;
ASSERT_EQ(OB_SUCCESS, fill_lob_sstable_slice_mock(lob_id, trans_id, 100, sql_no,
false, read_snapshot, collation_type, datum));
}
}
TEST_F(TestLobMetaIterator, test_lob_common)
{
for (int x = 2; x <= 10; x++) {
std::string st = "";
for (int i = 0; i < 1000000; i++) {
st += static_cast<char>(i % 26 + 'a');
}
ObDatumRow row;
ObObj obj;
common::ObArenaAllocator allocator;
ASSERT_EQ(OB_SUCCESS, build_lob_data(obj, st, allocator));
ObStorageDatum datum;
ASSERT_EQ(OB_SUCCESS, datum.from_obj_enhance(obj));
transaction::ObTransID trans_id(10);
ObLobId lob_id;
lob_id.lob_id_ = x;
lob_id.tablet_id_ = x;
int64_t sql_no = 2;
int64_t read_snapshot = 3;
//ObLobMetaRowIterator iter;
ObCollationType collation_type = CS_TYPE_UTF8MB4_GENERAL_CI;
ASSERT_EQ(OB_SUCCESS, fill_lob_sstable_slice_mock(lob_id, trans_id, 100, sql_no,
true, read_snapshot, collation_type, datum));
}
}
}
}
int main(int argc, char **argv)
{
system("rm -f test_lob_meta_iterator.log*");
OB_LOGGER.set_file_name("test_lob_meta_iterator.log");
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

12
mittest/mtlenv/storage/test_ls_migration_param.cpp
View File

@ -239,15 +239,17 @@ TEST_F(TestLSMigrationParam, test_migrate_tablet_param)
TestSchemaUtils::prepare_data_schema(table_schema);
ObArenaAllocator schema_allocator;
ObCreateTabletSchema create_tablet_schema;
ret = create_tablet_schema.init(schema_allocator, table_schema, lib::Worker::CompatMode::MYSQL,
false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3);
false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3,
0/*tenant_data_version, default val*/, true/*need_create_empty_major*/);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletID empty_tablet_id;
SCN scn;
scn.convert_from_ts(ObTimeUtility::current_time());
ret = src_handle.get_obj()->init_for_first_time_creation(allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_,
scn, 2022, create_tablet_schema, false, ls_handle.get_ls()->get_freezer());
scn, 2022, create_tablet_schema, ls_handle.get_ls()->get_freezer());
ASSERT_EQ(common::OB_SUCCESS, ret);
ObMigrationTabletParam tablet_param;
@ -303,15 +305,17 @@ TEST_F(TestLSMigrationParam, test_migration_param_compat)
TestSchemaUtils::prepare_data_schema(table_schema);
ObArenaAllocator schema_allocator;
ObCreateTabletSchema create_tablet_schema;
ret = create_tablet_schema.init(schema_allocator, table_schema, lib::Worker::CompatMode::MYSQL,
false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3);
false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3,
0/*tenant_data_version, default val*/, true/*need_create_empty_major*/);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletID empty_tablet_id;
SCN scn;
scn.convert_from_ts(ObTimeUtility::current_time());
ret = src_handle.get_obj()->init_for_first_time_creation(allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_,
scn, 2022, create_tablet_schema, false, ls_handle.get_ls()->get_freezer());
scn, 2022, create_tablet_schema, ls_handle.get_ls()->get_freezer());
ASSERT_EQ(common::OB_SUCCESS, ret);
ObMigrationTabletParam tablet_param;

6
mittest/mtlenv/storage/test_ls_tablet_info_writer_and_reader.cpp
View File

@ -156,15 +156,17 @@ void TestLSTabletInfoWR::fill_tablet_meta()
TestSchemaUtils::prepare_data_schema(table_schema);
ObArenaAllocator schema_allocator;
ObCreateTabletSchema create_tablet_schema;
ret = create_tablet_schema.init(schema_allocator, table_schema, lib::Worker::CompatMode::MYSQL,
false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3);
false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3,
0/*tenant_data_version, default val*/, true/*need_create_empty_major*/);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletID empty_tablet_id;
SCN scn;
scn.convert_from_ts(ObTimeUtility::current_time());
ret = src_handle.get_obj()->init_for_first_time_creation(arena_allocator_, src_key.ls_id_, src_key.tablet_id_, src_key.tablet_id_,
scn, 2022, create_tablet_schema, false, ls_handle.get_ls()->get_freezer());
scn, 2022, create_tablet_schema, ls_handle.get_ls()->get_freezer());
ASSERT_EQ(common::OB_SUCCESS, ret);
ObMigrationTabletParam tablet_param;

1
mittest/mtlenv/storage/test_multi_version_merge_recycle.cpp
View File

@ -180,6 +180,7 @@ void TestMultiVersionMergeRecycle::prepare_query_param(const ObVersionRange &ver
iter_param_.vectorized_enabled_ = false;
ASSERT_EQ(OB_SUCCESS,
store_ctx_.init_for_read(ls_id,
iter_param_.tablet_id_,
INT64_MAX, // query_expire_ts
-1, // lock_timeout_us
INT64_MAX - 2));

1
mittest/mtlenv/storage/test_multi_version_sstable_merge.cpp
View File

@ -198,6 +198,7 @@ void TestMultiVersionMerge::prepare_query_param(const ObVersionRange &version_ra
iter_param_.vectorized_enabled_ = false;
ASSERT_EQ(OB_SUCCESS,
store_ctx_.init_for_read(ls_id,
iter_param_.tablet_id_,
INT64_MAX, // query_expire_ts
-1, // lock_timeout_us
share::SCN::max_scn()));

1
mittest/mtlenv/storage/test_multi_version_sstable_single_get.cpp
View File

@ -95,6 +95,7 @@ void TestMultiVersionSSTableSingleGet::prepare_query_param(
iter_param_.vectorized_enabled_ = false;
ASSERT_EQ(OB_SUCCESS,
store_ctx_.init_for_read(ls_id,
iter_param_.tablet_id_,
INT64_MAX, // query_expire_ts
-1, // lock_timeout_us
share::SCN::max_scn()));

8
mittest/mtlenv/storage/test_tablet_create_delete_helper.cpp
View File

@ -2880,8 +2880,8 @@ TEST_F(TestTabletCreateDeleteHelper, migrate_lob_tablets)
const ObSArray<ObTabletID> index_tablet_array;
const ObTabletID tablet_id(2);
const ObTabletID lob_meta_tablet_id(101);
ret = helper.do_create_tablet(lob_meta_tablet_id, tablet_id, lob_meta_tablet_id, ObTabletID(),
index_tablet_array, arg, trans_flags, table_schema1, lib::Worker::CompatMode::MYSQL, tablet_handle);
ret = helper.create_tablet(lob_meta_tablet_id, tablet_id, lob_meta_tablet_id, ObTabletID(),
index_tablet_array, arg, trans_flags, table_schema1, true /*need_create_empty_major_sstable*/, lib::Worker::CompatMode::MYSQL);
ASSERT_EQ(OB_SUCCESS, ret);
// reset tx data to normal state
tablet_handle.get_obj()->tablet_meta_.tx_data_.tx_id_ = 0;
@ -2889,8 +2889,8 @@ TEST_F(TestTabletCreateDeleteHelper, migrate_lob_tablets)
tablet_handle.get_obj()->tablet_meta_.tx_data_.tx_scn_ = share::SCN::minus(share::SCN::max_scn(), 98);
const ObTabletID lob_piece_tablet_id(102);
ret = helper.do_create_tablet(lob_piece_tablet_id, tablet_id, ObTabletID(), lob_piece_tablet_id,
index_tablet_array, arg, trans_flags, table_schema2, lib::Worker::CompatMode::MYSQL, tablet_handle);
ret = helper.create_tablet(lob_piece_tablet_id, tablet_id, ObTabletID(), lob_piece_tablet_id,
index_tablet_array, arg, trans_flags, table_schema2, true /*need_create_empty_major_sstable*/, lib::Worker::CompatMode::MYSQL);
ASSERT_EQ(OB_SUCCESS, ret);
// reset tx data to normal state
tablet_handle.get_obj()->tablet_meta_.tx_data_.tx_id_ = 0;

19
mittest/mtlenv/storage/test_tenant_meta_mem_mgr.cpp
View File

@ -226,7 +226,8 @@ void TestTenantMetaMemMgr::prepare_data_schema(
LOG_INFO("dump data table schema", LITERAL_K(TEST_ROWKEY_COLUMN_CNT), K(table_schema));
ret = create_tablet_schema.init(allocator, table_schema, lib::Worker::CompatMode::MYSQL,
false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3);
false/*skip_column_info*/, ObCreateTabletSchema::STORAGE_SCHEMA_VERSION_V3,
0/*tenant_data_version, default val*/, true/*need_create_empty_major*/);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -704,8 +705,9 @@ TEST_F(TestTenantMetaMemMgr, test_wash_tablet)
create_scn.convert_from_ts(ObTimeUtility::fast_current_time());
ObTabletID empty_tablet_id;
create_tablet_schema.set_need_create_empty_major(true);
ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
create_scn, create_scn.get_val_for_tx(), create_tablet_schema, true, &freezer);
create_scn, create_scn.get_val_for_tx(), create_tablet_schema, &freezer);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(1, tablet->get_ref());
ObTabletPersister persister;
@ -799,8 +801,9 @@ TEST_F(TestTenantMetaMemMgr, test_wash_inner_tablet)
ObTabletID empty_tablet_id;
bool make_empty_co_sstable = true;
create_tablet_schema.set_need_create_empty_major(make_empty_co_sstable);
ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
create_scn, create_scn.get_val_for_tx(), create_tablet_schema, make_empty_co_sstable, &freezer);
create_scn, create_scn.get_val_for_tx(), create_tablet_schema, &freezer);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(1, tablet->get_ref());
@ -905,9 +908,9 @@ TEST_F(TestTenantMetaMemMgr, test_wash_no_sstable_tablet)
ObTabletID empty_tablet_id;
bool make_empty_co_sstable = false;
create_tablet_schema.set_need_create_empty_major(make_empty_co_sstable);
ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
create_scn, create_scn.get_val_for_tx(), create_tablet_schema,
make_empty_co_sstable, &freezer);
create_scn, create_scn.get_val_for_tx(), create_tablet_schema, &freezer);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(1, tablet->get_ref());
@ -999,8 +1002,9 @@ TEST_F(TestTenantMetaMemMgr, test_get_tablet_with_allocator)
ObTabletID empty_tablet_id;
bool make_empty_co_sstable = true;
create_tablet_schema.set_need_create_empty_major(make_empty_co_sstable);
ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
create_scn, create_scn.get_val_for_tx(), create_tablet_schema, make_empty_co_sstable, &freezer);
create_scn, create_scn.get_val_for_tx(), create_tablet_schema, &freezer);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(1, tablet->get_ref());
@ -1123,8 +1127,9 @@ TEST_F(TestTenantMetaMemMgr, test_wash_mem_tablet)
ObTabletID empty_tablet_id;
bool make_empty_co_sstable = false;
create_tablet_schema.set_need_create_empty_major(make_empty_co_sstable);
ret = tablet->init_for_first_time_creation(allocator_, ls_id_, tablet_id, tablet_id,
create_scn, create_scn.get_val_for_tx(), create_tablet_schema, make_empty_co_sstable, &freezer);
create_scn, create_scn.get_val_for_tx(), create_tablet_schema, &freezer);
ASSERT_EQ(common::OB_SUCCESS, ret);
ASSERT_EQ(1, tablet->get_ref());