From a1ecfa86e5c7e5a1b0eb8b4f5c016c0523be0219 Mon Sep 17 00:00:00 2001 From: renju96 Date: Thu, 3 Aug 2023 13:42:26 +0000 Subject: [PATCH] skip ddl_memtable when open sec_meta_iter --- .../test_sstable_sec_meta_iterator.cpp | 203 ++++++++++++++++++ src/storage/blocksstable/ob_sstable.cpp | 2 +- .../ob_sstable_sec_meta_iterator.cpp | 13 +- src/storage/ddl/ob_tablet_ddl_kv.h | 2 +- 4 files changed, 215 insertions(+), 5 deletions(-) diff --git a/mittest/mtlenv/storage/blocksstable/test_sstable_sec_meta_iterator.cpp b/mittest/mtlenv/storage/blocksstable/test_sstable_sec_meta_iterator.cpp index f78b9d2da..5ec44d19b 100644 --- a/mittest/mtlenv/storage/blocksstable/test_sstable_sec_meta_iterator.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_sstable_sec_meta_iterator.cpp @@ -258,6 +258,209 @@ TEST_F(TestSSTableSecMetaIterator, test_basic) ASSERT_EQ(OB_ITER_END, tmp_ret); } +TEST_F(TestSSTableSecMetaIterator, test_ddl_kv) +{ + ObArray data_macro_metas; + uint64_t tenant_id = MTL_ID(); + const ObITableReadInfo &index_read_info = tablet_handle_.get_obj()->get_rowkey_read_info(); + ObSSTableSecMetaIterator meta_iter; + ObDataMacroBlockMeta data_macro_meta; + ObDatumRange range; + range.set_whole_range(); + STORAGE_LOG(INFO, "qilu debug range1", K(range)); + ASSERT_EQ(OB_SUCCESS, meta_iter.open( + range, + ObMacroBlockMetaType::DATA_BLOCK_META, + ddl_kv_, + index_read_info, + allocator_)); + int tmp_ret = OB_SUCCESS; + while (OB_SUCCESS == tmp_ret) { + tmp_ret = meta_iter.get_next(data_macro_meta); + STORAGE_LOG(DEBUG, "Got next data macro block meta", K(tmp_ret), K(data_macro_meta)); + if (OB_SUCCESS == tmp_ret) { + ObDataMacroBlockMeta *deep_copy_meta = nullptr; + ASSERT_EQ(OB_SUCCESS, data_macro_meta.deep_copy(deep_copy_meta, allocator_)); + ASSERT_EQ(OB_SUCCESS, data_macro_metas.push_back( + *(deep_copy_meta))); + } + } + ASSERT_EQ(OB_ITER_END, tmp_ret); + ASSERT_TRUE(data_macro_metas.count() > 0); + + // Iterate sample + const int64_t skip_cnt = 3; + meta_iter.reset(); + int64_t iter_cnt = 0; + STORAGE_LOG(INFO, "qilu debug range2", K(range)); + ASSERT_EQ(OB_SUCCESS, meta_iter.open( + range, + ObMacroBlockMetaType::DATA_BLOCK_META, + ddl_kv_, + index_read_info, + allocator_, + false, 3)); + tmp_ret = OB_SUCCESS; + while (OB_SUCCESS == tmp_ret) { + tmp_ret = meta_iter.get_next(data_macro_meta); + STORAGE_LOG(DEBUG, "Got skip next data macro block meta", K(tmp_ret), K(data_macro_meta)); + if (OB_SUCCESS == tmp_ret) { + ++iter_cnt; + } + } + int64_t target_scan_cnt = data_macro_metas.count() == 0 ? 0 + : 1 + (data_macro_metas.count() - 1) / skip_cnt; + ASSERT_EQ(iter_cnt, target_scan_cnt); + + + // Iterate part + meta_iter.reset(); + const int64_t pivot_rowkey_idx = 0; + ObDatumRowkey pivot_rowkey; + iter_cnt = 0; + ASSERT_EQ(OB_SUCCESS, data_macro_metas.at(pivot_rowkey_idx).get_rowkey(pivot_rowkey)); + + range.reset(); + range.start_key_.set_min_rowkey(); + range.end_key_ = pivot_rowkey; + range.border_flag_.unset_inclusive_start(); + range.border_flag_.set_inclusive_end(); + STORAGE_LOG(INFO, "qilu debug range3", K(range)); + ASSERT_EQ(OB_SUCCESS, meta_iter.open( + range, + ObMacroBlockMetaType::DATA_BLOCK_META, + ddl_kv_, + index_read_info, + allocator_)); + tmp_ret = OB_SUCCESS; + iter_cnt = 0; + while (OB_SUCCESS == tmp_ret) { + tmp_ret = meta_iter.get_next(data_macro_meta); + if (OB_SUCCESS == tmp_ret) { + ++iter_cnt; + } + } + ASSERT_EQ(OB_ITER_END, tmp_ret); + ASSERT_EQ(iter_cnt, pivot_rowkey_idx + 1); + + meta_iter.reset(); + range.reset(); + range.start_key_ = pivot_rowkey; + range.end_key_.set_max_rowkey(); + range.border_flag_.unset_inclusive_start(); + range.border_flag_.set_inclusive_end(); + STORAGE_LOG(INFO, "qilu debug range4", K(range)); + ASSERT_EQ(OB_SUCCESS, meta_iter.open( + range, + ObMacroBlockMetaType::DATA_BLOCK_META, + ddl_kv_, + index_read_info, + allocator_)); + tmp_ret = OB_SUCCESS; + iter_cnt = 0; + while (OB_SUCCESS == tmp_ret) { + tmp_ret = meta_iter.get_next(data_macro_meta); + if (OB_SUCCESS == tmp_ret) { + ++iter_cnt; + } + } + ASSERT_EQ(OB_ITER_END, tmp_ret); + ASSERT_EQ(iter_cnt, data_macro_metas.count() - 1); + + // iter range lower than first macro block endkey + ObDatumRow row; + ASSERT_EQ(OB_SUCCESS, row.init(allocator_, MAX_TEST_COLUMN_CNT)); + ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(min_row_seed_ - 1, row)); + meta_iter.reset(); + range.reset(); + range.start_key_.assign(row.storage_datums_, TEST_ROWKEY_COLUMN_CNT); + range.end_key_.assign(row.storage_datums_, TEST_ROWKEY_COLUMN_CNT); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + STORAGE_LOG(INFO, "qilu debug range5", K(range)); + ASSERT_EQ(OB_SUCCESS, meta_iter.open( + range, + ObMacroBlockMetaType::DATA_BLOCK_META, + ddl_kv_, + index_read_info, + allocator_)); + ASSERT_EQ(OB_SUCCESS, meta_iter.get_next(data_macro_meta)); + ASSERT_EQ(OB_ITER_END, meta_iter.get_next(data_macro_meta)); + + // iter range larger than sstable endkey + meta_iter.reset(); + range.reset(); + row.reuse(); + ASSERT_EQ(OB_SUCCESS, row_generate_.get_next_row(max_row_seed_ + 1, row)); + range.start_key_.assign(row.storage_datums_, TEST_ROWKEY_COLUMN_CNT); + range.end_key_.assign(row.storage_datums_, TEST_ROWKEY_COLUMN_CNT); + range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + STORAGE_LOG(INFO, "qilu debug range6", K(range)); + ASSERT_EQ(OB_SUCCESS, meta_iter.open( + range, + ObMacroBlockMetaType::DATA_BLOCK_META, + ddl_kv_, + index_read_info, + allocator_)); + + // Dual iter part + ObDualMacroMetaIterator dual_iter; + ObMacroBlockDesc macro_desc; + const int64_t pivot_rowkey_idx_2 = 3; + ObDatumRowkey pivot_rowkey_2; + iter_cnt = 0; + ASSERT_EQ(OB_SUCCESS, data_macro_metas.at(pivot_rowkey_idx_2).get_rowkey(pivot_rowkey_2)); + + range.reset(); + range.start_key_ = pivot_rowkey; + range.end_key_ = pivot_rowkey_2; + range.border_flag_.unset_inclusive_start(); + // range.border_flag_.set_inclusive_start(); + range.border_flag_.set_inclusive_end(); + macro_desc.macro_meta_ = &data_macro_meta; + STORAGE_LOG(INFO, "show query range", K(range)); + ASSERT_EQ(OB_SUCCESS, dual_iter.open(sstable_, range, index_read_info, allocator_)); + tmp_ret = OB_SUCCESS; + while (OB_SUCCESS == tmp_ret) { + tmp_ret = dual_iter.get_next_macro_block(macro_desc); + if (OB_SUCCESS == tmp_ret) { + STORAGE_LOG(DEBUG, "Got next data macro block meta", K(tmp_ret), K(macro_desc)); + } + } + ASSERT_EQ(OB_ITER_END, tmp_ret); + + // range.border_flag_.set_inclusive_start(); + range.border_flag_.unset_inclusive_start(); + range.border_flag_.unset_inclusive_end(); + dual_iter.reset(); + ASSERT_EQ(OB_SUCCESS, dual_iter.open(sstable_, range, index_read_info, allocator_)); + tmp_ret = OB_SUCCESS; + while (OB_SUCCESS == tmp_ret) { + tmp_ret = dual_iter.get_next_macro_block(macro_desc); + if (OB_SUCCESS == tmp_ret) { + STORAGE_LOG(DEBUG, "Got next data macro block meta", K(tmp_ret), K(macro_desc)); + } + } + ASSERT_EQ(OB_ITER_END, tmp_ret); + + dual_iter.reset(); + range.end_key_.set_max_rowkey(); + range.border_flag_.unset_inclusive_start(); + range.border_flag_.unset_inclusive_end(); + ASSERT_EQ(OB_SUCCESS, dual_iter.open(sstable_, range, index_read_info, allocator_)); + tmp_ret = OB_SUCCESS; + while (OB_SUCCESS == tmp_ret) { + tmp_ret = dual_iter.get_next_macro_block(macro_desc); + if (OB_SUCCESS == tmp_ret) { + STORAGE_LOG(INFO, "Got next data macro block meta", K(tmp_ret), K(macro_desc)); + } + } + ObDatumRowkey endkey; + ASSERT_EQ(OB_SUCCESS, sstable_.get_last_rowkey(allocator_, endkey)); + ASSERT_EQ(OB_ITER_END, tmp_ret); +} + TEST_F(TestSSTableSecMetaIterator, test_dual_iter) { uint64_t tenant_id = MTL_ID(); diff --git a/src/storage/blocksstable/ob_sstable.cpp b/src/storage/blocksstable/ob_sstable.cpp index 23a1f3187..42f5c3f7f 100644 --- a/src/storage/blocksstable/ob_sstable.cpp +++ b/src/storage/blocksstable/ob_sstable.cpp @@ -640,7 +640,7 @@ int ObSSTable::scan_secondary_meta( } else if (OB_FAIL(iter->open( query_range, meta_type, *this, rowkey_read_info, allocator, is_reverse_scan, sample_step))) { LOG_WARN("Fail to open secondary meta iterator with range", - K(ret), K(query_range), K(meta_type), K_(meta), K(is_reverse_scan), K(sample_step)); + K(ret), K(query_range), K(meta_type), K_(meta), K(is_reverse_scan), K(sample_step), KPC(this)); } else { meta_iter = iter; } diff --git a/src/storage/blocksstable/ob_sstable_sec_meta_iterator.cpp b/src/storage/blocksstable/ob_sstable_sec_meta_iterator.cpp index 2f5e6eb0e..088096ea3 100644 --- a/src/storage/blocksstable/ob_sstable_sec_meta_iterator.cpp +++ b/src/storage/blocksstable/ob_sstable_sec_meta_iterator.cpp @@ -72,6 +72,7 @@ int ObSSTableSecMetaIterator::open( { int ret = OB_SUCCESS; bool is_meta_root = false; + bool is_ddl_mem_sstable = false; if (IS_INIT) { ret = OB_INIT_TWICE; LOG_WARN("Fail to open sstable secondary meta iterator", K(ret)); @@ -99,6 +100,7 @@ int ObSSTableSecMetaIterator::open( if (OB_FAIL(ret) || is_prefetch_end_) { } else if (sstable.is_ddl_mem_sstable()) { + is_ddl_mem_sstable = true; const ObMicroBlockData &root_block = sstable_meta_hdl_.get_sstable_meta().get_root_info().get_block_data(); if (ObMicroBlockData::DDL_BLOCK_TREE != root_block.type_ || nullptr == root_block.buf_) { ret = OB_ERR_UNEXPECTED; @@ -111,7 +113,12 @@ int ObSSTableSecMetaIterator::open( true, //is_right_border, curr_block_start_idx_, curr_block_end_idx_))) { - LOG_WARN("locate range failed", K(ret)); + if (OB_UNLIKELY(OB_BEYOND_THE_RANGE != ret)) { + LOG_WARN("locate range failed", K(ret), K(query_range)); + } else { + curr_block_idx_ = curr_block_end_idx_ + 1; + ret = OB_SUCCESS; // return OB_ITER_END on get_next() for get + } } else { const int64_t step = max(1, sample_step); step_cnt_ = !is_reverse_scan ? step : -step; @@ -134,7 +141,7 @@ int ObSSTableSecMetaIterator::open( const int64_t request_col_cnt = rowkey_read_info.get_schema_rowkey_count() + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt() + 1; - if (OB_SUCC(ret) && !is_prefetch_end_ && !is_meta_root) { + if (OB_SUCC(ret) && !is_prefetch_end_ && !is_meta_root && !is_ddl_mem_sstable) { bool start_key_beyond_range = false; bool end_key_beyond_range = false; if (is_reverse_scan) { @@ -174,7 +181,7 @@ int ObSSTableSecMetaIterator::open( } } - if (OB_FAIL(ret)) { + if (OB_FAIL(ret) || is_ddl_mem_sstable) { // do nothing } else if (is_prefetch_end_) { is_inited_ = true; diff --git a/src/storage/ddl/ob_tablet_ddl_kv.h b/src/storage/ddl/ob_tablet_ddl_kv.h index 7d165560c..99376624a 100644 --- a/src/storage/ddl/ob_tablet_ddl_kv.h +++ b/src/storage/ddl/ob_tablet_ddl_kv.h @@ -137,7 +137,7 @@ public: void dec_pending_cnt(); bool is_pending() const { return ATOMIC_LOAD(&pending_cnt_) > 0; } int wait_pending(); - TO_STRING_KV(K_(is_inited), K_(ls_id), K_(tablet_id), K_(ddl_start_scn), K_(snapshot_version), + INHERIT_TO_STRING_KV("ObSSTable", ObSSTable, K_(is_inited), K_(ls_id), K_(tablet_id), K_(ddl_start_scn), K_(snapshot_version), K_(is_freezed), K_(is_closed), K_(last_freezed_scn), K_(min_scn), K_(max_scn), K_(freeze_scn), K_(pending_cnt), K_(data_format_version), K_(ref_cnt),