From ebf9cfed4480fc28a90d36533ceb14260623eb2d Mon Sep 17 00:00:00 2001 From: renju96 Date: Thu, 8 Feb 2024 03:48:06 +0000 Subject: [PATCH] [index_block_row_scan]add ObDDLSStableAllRangeIterator to handle ddl_merge_sstable with index_tree_height > 2 --- .../storage/blocksstable/test_cg_sstable.cpp | 3 +- .../test_index_block_tree_cursor.cpp | 2 +- .../storage/blocksstable/test_index_tree.cpp | 2 +- .../test_sstable_sec_meta_iterator.cpp | 2 +- .../ob_all_virtual_tablet_encrypt_info.h | 2 +- ...ob_all_virtual_tablet_sstable_macro_info.h | 2 +- src/storage/CMakeLists.txt | 1 + .../access/ob_index_tree_prefetcher.cpp | 6 +- .../access/ob_sstable_row_whole_scanner.h | 2 +- .../ob_ddl_index_block_row_iterator.cpp | 358 ++++++++++++++++-- .../ob_ddl_index_block_row_iterator.h | 56 ++- .../index_block/ob_index_block_builder.cpp | 2 +- .../ob_index_block_dual_meta_iterator.cpp | 109 ++++++ .../ob_index_block_dual_meta_iterator.h | 62 +++ .../ob_index_block_macro_iterator.cpp | 168 ++++---- .../ob_index_block_macro_iterator.h | 36 +- .../ob_index_block_row_scanner.cpp | 4 + .../ob_data_macro_block_merge_writer.h | 2 +- .../blocksstable/ob_macro_block_writer.cpp | 2 +- src/storage/blocksstable/ob_sstable_meta.h | 2 +- .../ob_index_block_micro_iterator.h | 2 +- .../compaction/ob_partition_merge_iter.h | 2 +- src/storage/ob_partition_range_spliter.cpp | 2 +- .../blocksstable/test_skip_index_filter.cpp | 2 +- 24 files changed, 662 insertions(+), 169 deletions(-) create mode 100644 src/storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.cpp create mode 100644 src/storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h diff --git a/mittest/mtlenv/storage/blocksstable/test_cg_sstable.cpp b/mittest/mtlenv/storage/blocksstable/test_cg_sstable.cpp index da51b20ccd..6f02f69b2e 100644 --- a/mittest/mtlenv/storage/blocksstable/test_cg_sstable.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_cg_sstable.cpp @@ -9,13 +9,12 @@ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. * See the Mulan PubL v2 for more details. */ - #include #define private public #define protected public #include "storage/blocksstable/index_block/ob_index_block_tree_cursor.h" -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "storage/blocksstable/ob_macro_block_bare_iterator.h" #include "ob_index_block_data_prepare.h" diff --git a/mittest/mtlenv/storage/blocksstable/test_index_block_tree_cursor.cpp b/mittest/mtlenv/storage/blocksstable/test_index_block_tree_cursor.cpp index 9fbd50e232..1c5a4fea25 100644 --- a/mittest/mtlenv/storage/blocksstable/test_index_block_tree_cursor.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_index_block_tree_cursor.cpp @@ -15,7 +15,7 @@ #define protected public #include "storage/blocksstable/index_block/ob_index_block_tree_cursor.h" -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "storage/blocksstable/ob_macro_block_bare_iterator.h" #include "ob_index_block_data_prepare.h" diff --git a/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp b/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp index 2a68eecca6..b40bc5f926 100644 --- a/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp @@ -32,7 +32,7 @@ #include "storage/memtable/ob_memtable_interface.h" #include "storage/ob_i_store.h" #include "storage/compaction/ob_tenant_freeze_info_mgr.h" -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "mtlenv/mock_tenant_module_env.h" #include "share/scn.h" #include "storage/blocksstable/ob_shared_macro_block_manager.h" diff --git a/mittest/mtlenv/storage/blocksstable/test_sstable_sec_meta_iterator.cpp b/mittest/mtlenv/storage/blocksstable/test_sstable_sec_meta_iterator.cpp index cbf85a2b6a..4c9d624b40 100644 --- a/mittest/mtlenv/storage/blocksstable/test_sstable_sec_meta_iterator.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_sstable_sec_meta_iterator.cpp @@ -16,7 +16,7 @@ #include "storage/ob_partition_range_spliter.h" #include "storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.h" -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "ob_index_block_data_prepare.h" namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_encrypt_info.h b/src/observer/virtual_table/ob_all_virtual_tablet_encrypt_info.h index a8f20bb2ef..1b24b93310 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_encrypt_info.h +++ b/src/observer/virtual_table/ob_all_virtual_tablet_encrypt_info.h @@ -20,7 +20,7 @@ #include "share/ob_scanner.h" #include "share/ob_virtual_table_scanner_iterator.h" #include "share/rc/ob_tenant_base.h" -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "storage/blocksstable/ob_sstable_meta.h" #include "storage/meta_mem/ob_tablet_handle.h" #include "storage/meta_mem/ob_tablet_handle.h" diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_sstable_macro_info.h b/src/observer/virtual_table/ob_all_virtual_tablet_sstable_macro_info.h index 18e7cfd399..c8e6b98e9e 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_sstable_macro_info.h +++ b/src/observer/virtual_table/ob_all_virtual_tablet_sstable_macro_info.h @@ -18,7 +18,7 @@ #include "share/ob_scanner.h" #include "share/ob_virtual_table_scanner_iterator.h" #include "share/rc/ob_tenant_base.h" -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "storage/blocksstable/ob_sstable_meta.h" #include "storage/ob_i_table.h" #include "storage/meta_mem/ob_tablet_handle.h" diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index bb8b37ab49..0ec8793957 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -123,6 +123,7 @@ ob_set_subtarget(ob_storage blocksstable_index_block blocksstable/index_block/ob_index_block_aggregator.cpp blocksstable/index_block/ob_index_block_builder.cpp blocksstable/index_block/ob_index_block_macro_iterator.cpp + blocksstable/index_block/ob_index_block_dual_meta_iterator.cpp blocksstable/index_block/ob_ddl_sstable_scan_merge.cpp blocksstable/index_block/ob_index_block_row_scanner.cpp blocksstable/index_block/ob_ddl_index_block_row_iterator.cpp diff --git a/src/storage/access/ob_index_tree_prefetcher.cpp b/src/storage/access/ob_index_tree_prefetcher.cpp index 5e57c6de2f..579d68266b 100644 --- a/src/storage/access/ob_index_tree_prefetcher.cpp +++ b/src/storage/access/ob_index_tree_prefetcher.cpp @@ -115,7 +115,8 @@ int ObIndexTreePrefetcher::init_basic_info( iter_param_ = &iter_param; datum_utils_ = &index_read_info->get_datum_utils(); data_version_ = sstable_->get_data_version(); - index_tree_height_ = sstable_meta_handle_.get_sstable_meta().get_index_tree_height(sstable.is_ddl_merge_empty_sstable()); + bool is_normal_query = !access_ctx_->query_flag_.is_daily_merge() && !access_ctx_->query_flag_.is_multi_version_minor_merge(); + index_tree_height_ = sstable_meta_handle_.get_sstable_meta().get_index_tree_height(sstable.is_ddl_merge_sstable() && is_normal_query); } return ret; } @@ -858,13 +859,14 @@ int ObIndexTreeMultiPassPrefetcher::i ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid store iterator type", K(ret), K(iter_type)); } + bool is_normal_query = !access_ctx_->query_flag_.is_daily_merge() && !access_ctx_->query_flag_.is_multi_version_minor_merge(); if (OB_FAIL(ret)) { } else if (OB_FAIL(iter_param.get_index_read_info(sstable.is_normal_cg_sstable(), index_read_info))) { LOG_WARN("failed to get index read info", KR(ret), K(sstable), K(iter_param)); } else if (FALSE_IT(datum_utils_ = &index_read_info->get_datum_utils())) { } else if (OB_FAIL(sstable.get_meta(sstable_meta_handle_))) { LOG_WARN("failed to get sstable meta handle", K(ret)); - } else if (FALSE_IT(index_tree_height_ = sstable_meta_handle_.get_sstable_meta().get_index_tree_height(sstable.is_ddl_merge_empty_sstable()))) { + } else if (FALSE_IT(index_tree_height_ = sstable_meta_handle_.get_sstable_meta().get_index_tree_height(is_normal_query && sstable.is_ddl_merge_sstable()))) { } else if (1 >= index_tree_height_ || MAX_INDEX_TREE_HEIGHT < index_tree_height_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected index tree height", K(ret), K(index_tree_height_), K(MAX_INDEX_TREE_HEIGHT)); diff --git a/src/storage/access/ob_sstable_row_whole_scanner.h b/src/storage/access/ob_sstable_row_whole_scanner.h index 13a0ad68bd..ecd3535163 100644 --- a/src/storage/access/ob_sstable_row_whole_scanner.h +++ b/src/storage/access/ob_sstable_row_whole_scanner.h @@ -13,7 +13,7 @@ #ifndef OB_STORAGE_OB_SSTABLE_ROW_WHOLE_SCANNER_H_ #define OB_STORAGE_OB_SSTABLE_ROW_WHOLE_SCANNER_H_ -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "storage/blocksstable/ob_micro_block_row_scanner.h" #include "storage/blocksstable/ob_macro_block_bare_iterator.h" #include "ob_store_row_iterator.h" diff --git a/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.cpp b/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.cpp index bedd538694..3543f3dacc 100644 --- a/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.cpp +++ b/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.cpp @@ -351,6 +351,269 @@ int ObDDLIndexBlockRowIterator::get_index_row_count(const ObDatumRange &range, return ret; } +/****************** ObDDLSStableAllRangeIterator **********************/ +ObDDLSStableAllRangeIterator::ObDDLSStableAllRangeIterator() + : is_iter_start_(false), + is_iter_finish_(true), + allocator_(nullptr), + rowkey_read_info_(nullptr), + cur_rowkey_(nullptr), + cur_header_(nullptr), + index_macro_iter_(), + iter_param_() +{ +} + +ObDDLSStableAllRangeIterator::~ObDDLSStableAllRangeIterator() +{ + reset(); +} + +void ObDDLSStableAllRangeIterator::reset() +{ + ObIndexBlockRowIterator::reset(); + is_iter_finish_ = true; + is_iter_start_ = false; + allocator_ = nullptr; + rowkey_read_info_ = nullptr; + cur_rowkey_ = nullptr; + cur_header_ = nullptr; + index_macro_iter_.reset(); + iter_param_.reset(); +} + +void ObDDLSStableAllRangeIterator::reuse() +{ + is_iter_finish_ = true; + is_iter_start_ = false; + rowkey_read_info_ = nullptr; + cur_rowkey_ = nullptr; + cur_header_ = nullptr; + index_macro_iter_.reset(); + iter_param_.reset(); +} + +int ObDDLSStableAllRangeIterator::init(const ObMicroBlockData &idx_block_data, + const ObStorageDatumUtils *datum_utils, + ObIAllocator *allocator, + const bool is_reverse_scan, + const bool set_iter_end, + const ObIndexBlockIterParam &iter_param) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(allocator) || OB_ISNULL(datum_utils) || !datum_utils->is_valid() || !iter_param.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguement", K(ret), KP(allocator), KPC(datum_utils), K(iter_param)); + } else { + ObTablet *cur_tablet = nullptr; + ObTabletHandle tmp_tablet_handle; + if (OB_ISNULL(iter_param.tablet_)) { + //get tablet handle from ls + ObLSService *ls_service = MTL(ObLSService *); + ObLSHandle ls_handle; + if (OB_FAIL(ls_service->get_ls(iter_param.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { + LOG_WARN("get ls failed", K(ret), K(iter_param.ls_id_)); + } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, + iter_param.tablet_id_, + tmp_tablet_handle, + ObMDSGetTabletMode::READ_ALL_COMMITED))) { + LOG_WARN("get tablet failed", K(ret), K(iter_param)); + } else { + cur_tablet = tmp_tablet_handle.get_obj(); + } + } else { + cur_tablet = const_cast(iter_param.tablet_); + } + + if (OB_SUCC(ret)) { + rowkey_read_info_ = &cur_tablet->get_rowkey_read_info(); + allocator_ = allocator; + iter_param_ = iter_param; + + is_reverse_scan_ = is_reverse_scan; + iter_step_ = is_reverse_scan_ ? -1 : 1; + datum_utils_ = datum_utils; + is_inited_ = true; + } + } + return ret; +} + +int ObDDLSStableAllRangeIterator::locate_key(const ObDatumRowkey &rowkey) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_UNLIKELY(!rowkey.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid rowkey", K(ret), K(rowkey)); + } else { + ObDatumRange range; + range.set_start_key(rowkey); + range.set_end_key(rowkey); + range.set_left_closed(); + range.set_right_closed(); + ObSSTable *sstable = const_cast(iter_param_.sstable_); + if (OB_FAIL(index_macro_iter_.open(*sstable, range, *rowkey_read_info_, *allocator_, is_reverse_scan_))) { + LOG_WARN("Fail to open micro block range iterator", K(ret), KPC(iter_param_.sstable_), K(range), KPC(rowkey_read_info_), K(is_reverse_scan_)); + is_iter_finish_ = true; + } else if (index_macro_iter_.is_iter_end()) { + ret = OB_BEYOND_THE_RANGE; + } else { + is_iter_start_ = true; + is_iter_finish_ = false; + } + } + LOG_TRACE("locate rowkey in ddl merge sstable", K(ret), K(rowkey), KPC(this)); + return ret; +} + +int ObDDLSStableAllRangeIterator::locate_range(const ObDatumRange &range, + const bool is_left_border, + const bool is_right_border, + const bool is_normal_cg) +{ + int ret = OB_SUCCESS; + ObSSTable *sstable = const_cast(iter_param_.sstable_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_UNLIKELY(!range.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid range", K(ret), K(range)); + } else if (OB_FAIL(index_macro_iter_.open(*sstable, range, *rowkey_read_info_, *allocator_, is_reverse_scan_))) { + is_iter_finish_ = true; + LOG_WARN("block meta tree locate range failed", K(ret), K(range)); + } else if (index_macro_iter_.is_iter_end()) { + ret = OB_BEYOND_THE_RANGE; + is_iter_finish_ = true; + } else { + is_iter_start_ = true; + is_iter_finish_ = false; + } + LOG_TRACE("Locate range in ddl merge sstable", K(ret), K(range), KPC(this)); + return ret; +} + +int ObDDLSStableAllRangeIterator::check_blockscan(const ObDatumRowkey &rowkey, bool &can_blockscan) +{ + can_blockscan = false; + return OB_SUCCESS; +} + +int ObDDLSStableAllRangeIterator::get_current(const ObIndexBlockRowHeader *&idx_row_header, + const ObDatumRowkey *&endkey) +{ + endkey = cur_rowkey_; + idx_row_header = cur_header_; + return OB_SUCCESS; +} + +int ObDDLSStableAllRangeIterator::get_next(const ObIndexBlockRowHeader *&idx_row_header, + const ObDatumRowkey *&endkey, + bool &is_scan_left_border, + bool &is_scan_right_border, + const ObIndexBlockRowMinorMetaInfo *&idx_minor_info, + const char *&agg_row_buf, + int64_t &agg_buf_size, + int64_t &row_offset) +{ + int ret = OB_SUCCESS; + idx_row_header = nullptr; + endkey = nullptr; + is_scan_left_border = false; + is_scan_right_border = false; + idx_minor_info = nullptr; + agg_row_buf = nullptr; + agg_buf_size = 0; + row_offset = 0; + + bool is_start_key = false; + bool is_end_key = false; + bool reach_cursor_end = false; + ObMicroIndexInfo tmp_idx_block_row; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_FAIL(index_macro_iter_.get_next_idx_row(tmp_idx_block_row, row_offset, reach_cursor_end))) { + LOG_WARN("fail to get next idx info", K(ret), KP(endkey), KP(idx_row_header), K(reach_cursor_end), K(index_macro_iter_)); + } else if (OB_UNLIKELY(nullptr == tmp_idx_block_row.row_header_ || nullptr == tmp_idx_block_row.endkey_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected null index block row header/endkey", K(ret), K(tmp_idx_block_row)); + } else { + idx_row_header = tmp_idx_block_row.row_header_; + endkey = tmp_idx_block_row.endkey_; + idx_minor_info = tmp_idx_block_row.minor_meta_info_; + agg_row_buf = tmp_idx_block_row.agg_row_buf_; + agg_buf_size = tmp_idx_block_row.agg_buf_size_; + cur_header_ = idx_row_header; + cur_rowkey_ = endkey; + if (is_iter_start_) { + is_start_key = true; + is_iter_start_ = false; + } + + if (OB_SUCC(ret)) { + if (reach_cursor_end) { + is_iter_finish_ = true; + is_end_key = true; + } + is_scan_left_border = is_reverse_scan_ ? is_end_key : is_start_key; + is_scan_right_border = is_reverse_scan_ ? is_start_key : is_end_key; + } + } + return ret; +} + +bool ObDDLSStableAllRangeIterator::end_of_block() const +{ + return is_iter_finish_; +} + +int ObDDLSStableAllRangeIterator::get_index_row_count(const ObDatumRange &range, + const bool is_left_border, + const bool is_right_border, + int64_t &index_row_count) +{ + // only single ddl_merge_sstable_without kv can come here, so just return real row_cnt in sstable + // todo @qilu: refine this, ddl_merge_sstable use RAW or TRANSFORM format iter + int ret = OB_SUCCESS; + index_row_count = 0; + ObIndexBlockMacroIterator tmp_index_macro_iter; + ObSSTable *sstable = const_cast(iter_param_.sstable_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_UNLIKELY(!range.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguement", K(ret), K(range)); + } else if (OB_FAIL(tmp_index_macro_iter.open(*sstable, range, *rowkey_read_info_, *allocator_, is_reverse_scan_))) { + LOG_WARN("tmp all range iter locate range failed", K(ret), K(range)); + } else { + bool tmp_reach_cursor_end = false; + ObMicroIndexInfo tmp_idx_block_row; + int64_t tmp_row_offset = 0; + while (OB_SUCC(ret)) { + if (OB_FAIL(tmp_index_macro_iter.get_next_idx_row(tmp_idx_block_row, tmp_row_offset, tmp_reach_cursor_end))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to get next idx info", K(ret), K(tmp_idx_block_row), K(tmp_reach_cursor_end), K(tmp_index_macro_iter)); + } else { + ret = OB_SUCCESS; + break; + } + } else { + ++index_row_count; + } + } + if (OB_FAIL(ret)) { + index_row_count = 0; + } + } + return ret; +} + /****************** ObDDLMergeEmptyIterator **********************/ ObDDLMergeEmptyIterator::ObDDLMergeEmptyIterator() { @@ -451,6 +714,7 @@ ObDDLMergeBlockRowIterator::ObDDLMergeBlockRowIterator() raw_iter_(nullptr), transformed_iter_(nullptr), empty_merge_iter_(nullptr), + all_range_iter_(nullptr), iters_(), consumers_(nullptr), consumer_cnt_(0), @@ -497,6 +761,13 @@ void ObDDLMergeBlockRowIterator::reset() empty_merge_iter_ = nullptr; } } + if (OB_NOT_NULL(all_range_iter_)) { + all_range_iter_->reset(); + if (OB_NOT_NULL(allocator_)) { + allocator_->free(all_range_iter_); + all_range_iter_ = nullptr; + } + } for (int64_t i = 0; i < iters_.count() - 1; ++i) { // skip sstable_iter if (OB_NOT_NULL(iters_.at(i))) { @@ -555,6 +826,9 @@ void ObDDLMergeBlockRowIterator::reuse() if (OB_NOT_NULL(empty_merge_iter_)) { empty_merge_iter_->reuse(); } + if (OB_NOT_NULL(all_range_iter_)) { + all_range_iter_->reuse(); + } for (int64_t i = 0; i < iters_.count() - 1; ++i) { // skip sstable_iter if (OB_NOT_NULL(iters_.at(i))) { @@ -617,7 +891,7 @@ int ObDDLMergeBlockRowIterator::init(const ObMicroBlockData &idx_block_data, } else if (iters_.count() > 1) { is_single_sstable_ = false; } else if (iter_param.sstable_->is_ddl_merge_empty_sstable()) { - LOG_INFO("empty sstable without ddl_kv", K(idx_block_data), K(iter_param)); + LOG_INFO("empty sstable without ddl_kv", K(ret), K(idx_block_data), K(iter_param)); } } @@ -653,11 +927,18 @@ int ObDDLMergeBlockRowIterator::init(const ObMicroBlockData &idx_block_data, } if (OB_NOT_NULL(empty_merge_iter_)) { empty_merge_iter_->reset(); - if (OB_NOT_NULL(empty_merge_iter_)) { + if (OB_NOT_NULL(allocator)) { allocator->free(empty_merge_iter_); empty_merge_iter_ = nullptr; } } + if (OB_NOT_NULL(all_range_iter_)) { + all_range_iter_->reset(); + if (OB_NOT_NULL(allocator)) { + allocator->free(all_range_iter_); + all_range_iter_ = nullptr; + } + } for (int64_t i = 0; i < iters_.count() - 1; ++i) { // skip sstable_iter if (OB_NOT_NULL(iters_.at(i))) { iters_.at(i)->reset(); @@ -688,7 +969,6 @@ int ObDDLMergeBlockRowIterator::init_sstable_index_iter(const ObMicroBlockData & ret = OB_ERR_UNEXPECTED; LOG_ERROR("last iter is null", K(ret), K(iters_)); } else { - iters_.at(iters_.count() - 1)->reuse(); iters_.pop_back(tmp_index_iter); } } @@ -701,7 +981,6 @@ int ObDDLMergeBlockRowIterator::init_sstable_index_iter(const ObMicroBlockData & } else if (iter_param.sstable_->is_ddl_merge_empty_sstable()) { // EMPTY DDL_MERGE_SSTABLE if (OB_NOT_NULL(empty_merge_iter_)) { - empty_merge_iter_->reuse(); } else if (OB_ISNULL(iter_buf = allocator->alloc(sizeof(ObDDLMergeEmptyIterator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret), K(sizeof(ObDDLMergeEmptyIterator))); @@ -712,31 +991,56 @@ int ObDDLMergeBlockRowIterator::init_sstable_index_iter(const ObMicroBlockData & sst_index_iter = empty_merge_iter_; FLOG_INFO("empty ddl merge sstable", K(iter_param), K(idx_block_data)); } - } else if (nullptr == idx_block_data.get_extra_buf()) { - // RAW - if (OB_NOT_NULL(raw_iter_)) { - raw_iter_->reuse(); - } else if (OB_ISNULL(iter_buf = allocator->alloc(sizeof(ObRAWIndexBlockRowIterator)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate memory failed", K(ret), K(sizeof(ObRAWIndexBlockRowIterator))); - } else { - raw_iter_ = new (iter_buf) ObRAWIndexBlockRowIterator; - } - if (OB_SUCC(ret)) { - sst_index_iter = raw_iter_; - } } else { - // TRANSFORMED - if (OB_NOT_NULL(transformed_iter_)) { - transformed_iter_->reuse(); - } else if (OB_ISNULL(iter_buf = allocator->alloc(sizeof(ObTFMIndexBlockRowIterator)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate memory failed", K(ret), K(sizeof(ObTFMIndexBlockRowIterator))); + ObSSTableMetaHandle sstable_meta_handle; + int64_t real_index_tree_height = 0; + if (OB_FAIL(iter_param.sstable_->get_meta(sstable_meta_handle))) { + LOG_WARN("failed to get sstable meta handle", K(ret)); } else { - transformed_iter_ = new (iter_buf) ObTFMIndexBlockRowIterator; - } - if (OB_SUCC(ret)) { - sst_index_iter = transformed_iter_; + real_index_tree_height = sstable_meta_handle.get_sstable_meta().get_index_tree_height(false); + if (real_index_tree_height < 2) { + ret= OB_ERR_UNEXPECTED; + LOG_WARN("invalid index tree_height", K(ret), KPC(iter_param.sstable_), K(real_index_tree_height)); + } else if (real_index_tree_height == 2) { // use basic iter + if (nullptr == idx_block_data.get_extra_buf()) { + // RAW + if (OB_NOT_NULL(raw_iter_)) { + } else if (OB_ISNULL(iter_buf = allocator->alloc(sizeof(ObRAWIndexBlockRowIterator)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret), K(sizeof(ObRAWIndexBlockRowIterator))); + } else { + raw_iter_ = new (iter_buf) ObRAWIndexBlockRowIterator; + } + if (OB_SUCC(ret)) { + sst_index_iter = raw_iter_; + } + } else { + // TRANSFORMED + if (OB_NOT_NULL(transformed_iter_)) { + } else if (OB_ISNULL(iter_buf = allocator->alloc(sizeof(ObTFMIndexBlockRowIterator)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret), K(sizeof(ObTFMIndexBlockRowIterator))); + } else { + transformed_iter_ = new (iter_buf) ObTFMIndexBlockRowIterator; + } + if (OB_SUCC(ret)) { + sst_index_iter = transformed_iter_; + } + } + } else { + // DDL_MERGE_SSTABLE with tree_height > 3 + if (OB_NOT_NULL(all_range_iter_)) { + } else if (OB_ISNULL(iter_buf = allocator->alloc(sizeof(ObDDLSStableAllRangeIterator)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret), K(sizeof(ObDDLSStableAllRangeIterator))); + } else { + all_range_iter_ = new (iter_buf) ObDDLSStableAllRangeIterator; + } + if (OB_SUCC(ret)) { + sst_index_iter = all_range_iter_; + FLOG_INFO("ddl merge sstable with higher tree", K(iter_param), K(idx_block_data), KPC(iter_param.sstable_), K(real_index_tree_height)); + } + } } } } diff --git a/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.h b/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.h index e3658d478c..116a4644ab 100644 --- a/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.h +++ b/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.h @@ -14,6 +14,7 @@ #define OCEANBASE_STORAGE_BLOCKSSTABLE_OB_DDL_INDEX_BLOCK_ROW_ITERATOR_H #include "storage/blocksstable/index_block/ob_index_block_row_scanner.h" +#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" namespace oceanbase { @@ -77,6 +78,56 @@ private: storage::ObBlockMetaTreeValue *cur_tree_value_; }; +// for ddl_merge_sstable with index_tree_height > 2 +class ObDDLSStableAllRangeIterator : public ObIndexBlockRowIterator +{ +public: + ObDDLSStableAllRangeIterator(); + virtual ~ObDDLSStableAllRangeIterator(); + virtual int init(const ObMicroBlockData &idx_block_data, + const ObStorageDatumUtils *datum_utils, + ObIAllocator *allocator, + const bool is_reverse_scan, + const bool set_iter_end, + const ObIndexBlockIterParam &iter_param) override; + virtual int get_current(const ObIndexBlockRowHeader *&idx_row_header, + const ObDatumRowkey *&endkey) override; + virtual int get_next(const ObIndexBlockRowHeader *&idx_row_header, + const ObDatumRowkey *&endkey, + bool &is_scan_left_border, + bool &is_scan_right_border, + const ObIndexBlockRowMinorMetaInfo *&idx_minor_info, + const char *&agg_row_buf, + int64_t &agg_buf_size, + int64_t &row_offset) override; + virtual int locate_key(const ObDatumRowkey &rowkey) override; + virtual int locate_range(const ObDatumRange &range, + const bool is_left_border, + const bool is_right_border, + const bool is_normal_cg) override; + virtual int check_blockscan(const ObDatumRowkey &rowkey, bool &can_blockscan) override; + virtual bool end_of_block() const override; + virtual int get_index_row_count(const ObDatumRange &range, + const bool is_left_border, + const bool is_right_border, + int64_t &index_row_count) override; + virtual void reuse() override; + virtual void reset() override; + INHERIT_TO_STRING_KV("base iterator:", ObIndexBlockRowIterator, "format:", "ObDDLSStableAllRangeIterator", K(is_iter_start_), K(is_iter_finish_), + KP(allocator_), KPC(rowkey_read_info_), K(index_macro_iter_), K(iter_param_), KP(cur_rowkey_), KP(cur_header_)); + +private: + bool is_iter_start_; + bool is_iter_finish_; + ObIAllocator *allocator_; + const ObITableReadInfo *rowkey_read_info_; + const blocksstable::ObDatumRowkey *cur_rowkey_; + const blocksstable::ObIndexBlockRowHeader *cur_header_; + ObIndexBlockMacroIterator index_macro_iter_; + ObIndexBlockIterParam iter_param_; +}; + +// for empty ddl_merge_sstable class ObDDLMergeEmptyIterator : public ObIndexBlockRowIterator { public: @@ -152,8 +203,8 @@ public: virtual void reuse() override; virtual int switch_context(ObStorageDatumUtils *datum_utils) override; INHERIT_TO_STRING_KV("base iterator:", ObIndexBlockRowIterator, "format:", "ObDDLMergeBlockRowIterator", - KP(raw_iter_), KP(transformed_iter_), KP(empty_merge_iter_), K(iters_), KP(allocator_), KP(consumers_), K(consumer_cnt_), - K(compare_), KPC(simple_merge_), KPC(loser_tree_), KPC(endkey_merger_), K(is_single_sstable_), + KP(raw_iter_), KP(transformed_iter_), KP(empty_merge_iter_), KP(all_range_iter_), K(iters_), KP(allocator_), KP(consumers_), + K(consumer_cnt_), K(compare_), KPC(simple_merge_), KPC(loser_tree_), KPC(endkey_merger_), K(is_single_sstable_), K(is_iter_start_), K(is_iter_finish_), K(query_range_), KP(idx_block_data_), K(first_index_item_), K(iter_param_)); struct MergeIndexItem final { @@ -228,6 +279,7 @@ private: ObRAWIndexBlockRowIterator *raw_iter_; ObTFMIndexBlockRowIterator *transformed_iter_; ObDDLMergeEmptyIterator *empty_merge_iter_; + ObDDLSStableAllRangeIterator *all_range_iter_; ObArray iters_; int64_t *consumers_; int64_t consumer_cnt_; diff --git a/src/storage/blocksstable/index_block/ob_index_block_builder.cpp b/src/storage/blocksstable/index_block/ob_index_block_builder.cpp index 1113161747..02e98c8216 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_builder.cpp +++ b/src/storage/blocksstable/index_block/ob_index_block_builder.cpp @@ -12,7 +12,7 @@ */ #include "ob_index_block_builder.h" -#include "ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "storage/blocksstable/ob_macro_block_writer.h" #include "storage/blocksstable/ob_imacro_block_flush_callback.h" #include "storage/blocksstable/cs_encoding/ob_micro_block_cs_encoder.h" diff --git a/src/storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.cpp b/src/storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.cpp new file mode 100644 index 0000000000..d71f1fccfe --- /dev/null +++ b/src/storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.cpp @@ -0,0 +1,109 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include "ob_index_block_dual_meta_iterator.h" +#include "storage/blocksstable/ob_macro_block_bare_iterator.h" + +namespace oceanbase { +using namespace storage; +namespace blocksstable { + +ObDualMacroMetaIterator::ObDualMacroMetaIterator() + : allocator_(nullptr), macro_iter_(), sec_meta_iter_(), iter_end_(false), is_inited_(false) {} + +void ObDualMacroMetaIterator::reset() +{ + macro_iter_.reset(); + sec_meta_iter_.reset(); + allocator_ = nullptr; + iter_end_ = false; + is_inited_ = false; +} + +int ObDualMacroMetaIterator::open( + ObSSTable &sstable, + const ObDatumRange &query_range, + const ObITableReadInfo &rowkey_read_info, + ObIAllocator &allocator, + const bool is_reverse_scan, + const bool need_record_micro_info) +{ + UNUSED(need_record_micro_info); + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + LOG_WARN("Init twice", K(ret)); + } else if (OB_FAIL(macro_iter_.open( + sstable, + query_range, + rowkey_read_info, + allocator, + is_reverse_scan, + true /* need record micro index info */))) { + LOG_WARN("Fail to open index block tree iterator", K(ret)); + } else if (OB_FAIL(sec_meta_iter_.open( + query_range, + blocksstable::DATA_BLOCK_META, + sstable, + rowkey_read_info, + allocator, + is_reverse_scan))) { + LOG_WARN("Fail to open secondary meta iterator", K(ret), K(rowkey_read_info)); + } else { + allocator_ = &allocator; + iter_end_ = false; + is_inited_ = true; + } + return ret; +} + +int ObDualMacroMetaIterator::get_next_macro_block(ObMacroBlockDesc &block_desc) +{ + int ret = OB_SUCCESS; + ObDataMacroBlockMeta *macro_meta = block_desc.macro_meta_; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Dual macro meta iterator not inited", K(ret)); + } else if (OB_ISNULL(macro_meta)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid null pointer to read macro meta", K(ret), K(block_desc), KP(macro_meta)); + } else if (OB_UNLIKELY(iter_end_)) { + ret = OB_ITER_END; + } else if (OB_SUCC(macro_iter_.get_next_macro_block(block_desc))) { + if (OB_FAIL(sec_meta_iter_.get_next(*macro_meta))) { + if (OB_ITER_END == ret) { + ret = OB_ERR_UNEXPECTED; + } + LOG_WARN("Fail to get next secondary meta iterator", K(ret), K_(macro_iter), K_(sec_meta_iter)); + } else if (OB_UNLIKELY(block_desc.macro_block_id_ != macro_meta->get_macro_id())) { + ret = OB_ERR_SYS; + LOG_WARN("Logic macro block id from iterated macro block and merge info not match", + K(ret), K(block_desc), KPC(macro_meta)); + } + } else if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("Fail to get next macro block from index tree", K(ret)); + } else { + int tmp_ret = sec_meta_iter_.get_next(*macro_meta); + if (OB_UNLIKELY(OB_ITER_END != tmp_ret)) { + ret = OB_ERR_SYS; + LOG_WARN("Dual macro meta iterated different range", + K(ret), K(tmp_ret), KPC(macro_meta), K(sec_meta_iter_), K(macro_iter_)); + } else { + iter_end_ = true; + } + } + return ret; +} + +} // namespace blocksstable +} // namespace oceanbase diff --git a/src/storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h b/src/storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h new file mode 100644 index 0000000000..2eb0e262de --- /dev/null +++ b/src/storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OB_INDEX_BLOCK_DUAL_META_ITERATOR_H_ +#define OB_INDEX_BLOCK_DUAL_META_ITERATOR_H_ + +#include "ob_index_block_tree_cursor.h" +#include "ob_index_block_macro_iterator.h" +#include "ob_sstable_sec_meta_iterator.h" +#include "storage/blocksstable/ob_datum_rowkey.h" + +namespace oceanbase { +namespace blocksstable { + +// Wrap-up of iterate both index block tree and secondary meta in sstable +class ObDualMacroMetaIterator final : public ObIMacroBlockIterator +{ +public: + ObDualMacroMetaIterator(); + virtual ~ObDualMacroMetaIterator() {} + + void reset() override; + virtual int open( + ObSSTable &sstable, + const ObDatumRange &query_range, + const ObITableReadInfo &rowkey_read_info, + ObIAllocator &allocator, + const bool is_reverse_scan = false, + const bool need_record_micro_info = false) override; + virtual int get_next_macro_block(blocksstable::ObMacroBlockDesc &block_desc) override; + + virtual const ObIArray &get_micro_index_infos() const + { + return macro_iter_.get_micro_index_infos(); + } + virtual const ObIArray &get_micro_endkeys() const + { + return macro_iter_.get_micro_endkeys(); + } + TO_STRING_KV(K_(iter_end), K_(is_inited), K_(macro_iter), K_(sec_meta_iter)); +private: + ObIAllocator *allocator_; // allocator for member struct and macro endkeys + ObIndexBlockMacroIterator macro_iter_; + ObSSTableSecMetaIterator sec_meta_iter_; + bool iter_end_; + bool is_inited_; + DISALLOW_COPY_AND_ASSIGN(ObDualMacroMetaIterator); +}; + +} // namespace blocksstable +} // namespace oceanbase + +#endif // OB_INDEX_BLOCK_DUAL_META_ITERATOR_H_ diff --git a/src/storage/blocksstable/index_block/ob_index_block_macro_iterator.cpp b/src/storage/blocksstable/index_block/ob_index_block_macro_iterator.cpp index 333c77c6d5..3f8d70485b 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_macro_iterator.cpp +++ b/src/storage/blocksstable/index_block/ob_index_block_macro_iterator.cpp @@ -220,6 +220,86 @@ int ObIndexBlockMacroIterator::deep_copy_rowkey(const ObDatumRowkey &src_key, Ob return ret; } +int ObIndexBlockMacroIterator::get_next_idx_row(ObMicroIndexInfo &idx_block_row, int64_t &row_offset, bool &reach_cursor_end) +{ + int ret = OB_SUCCESS; + row_offset = 0; + const ObIndexBlockRowParser *idx_row_parser = nullptr; + MacroBlockId macro_id; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Not init", K(ret)); + } else if (is_iter_end_) { + ret = OB_ITER_END; + } else if (OB_FAIL(tree_cursor_.get_macro_block_id(macro_id))) { + LOG_WARN("Fail to get macro row block id", K(ret), K(macro_id)); + } else if (OB_FAIL(tree_cursor_.get_idx_parser(idx_row_parser))) { + LOG_WARN("Fail to get idx row parser", K(ret), K_(tree_cursor)); + } else if (OB_ISNULL(idx_row_parser)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected null idx_row_parser", K(ret)); + } else if (OB_FAIL(idx_row_parser->get_header(idx_block_row.row_header_))) { + LOG_WARN("Fail to get idx row header", K(ret), KPC(idx_row_parser)); + } else if (OB_ISNULL(idx_block_row.row_header_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected null index row header", K(ret)); + } else { + if ((macro_id == begin_ && is_reverse_scan_) || (macro_id == end_ && !is_reverse_scan_)) { + is_iter_end_ = true; + } + } + + if (OB_SUCC(ret)) { + // traverse all node of this macro block and collect index info + ObDatumRowkey rowkey; + row_offset = idx_row_parser->get_row_offset(); + if (idx_block_row.row_header_->is_data_index() && !idx_block_row.row_header_->is_major_node()) { + if (OB_FAIL(idx_row_parser->get_minor_meta(idx_block_row.minor_meta_info_))) { + LOG_WARN("Fail to get minor meta info", K(ret)); + } + } else if (!idx_block_row.row_header_->is_major_node() || !idx_block_row.row_header_->is_pre_aggregated()) { + // Do not have aggregate data + } else if (OB_FAIL(idx_row_parser->get_agg_row(idx_block_row.agg_row_buf_, idx_block_row.agg_buf_size_))) { + LOG_WARN("Fail to get aggregate", K(ret)); + } + + if (OB_SUCC(ret)) { + if (need_record_micro_info_) { + micro_endkey_allocator_.reuse(); + reuse_micro_info_array(); + if (OB_FAIL(tree_cursor_.get_child_micro_infos( + *iter_range_, + micro_endkey_allocator_, + micro_endkeys_, + micro_index_infos_, + hold_item_))) { + LOG_WARN("Fail to record child micro block info", K(ret), KPC(iter_range_)); + } + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(deep_copy_rowkey(curr_key_, prev_key_, prev_key_buf_))) { + STORAGE_LOG(WARN, "Failed to save prev key", K(ret), K_(curr_key)); + } else if (OB_FAIL(tree_cursor_.get_current_endkey(rowkey))) { + LOG_WARN("Fail to get current endkey", K(ret), K_(tree_cursor)); + } else if (OB_FAIL(deep_copy_rowkey(rowkey, curr_key_, curr_key_buf_))) { + STORAGE_LOG(WARN, "Failed to save curr key", K(ret), K(rowkey)); + } else if (OB_FAIL(tree_cursor_.move_forward(is_reverse_scan_))) { + if (OB_LIKELY(OB_ITER_END == ret)) { + is_iter_end_ = true; + ret = OB_SUCCESS; + } else { + LOG_WARN("Fail to move cursor to next macro node", K(ret), K_(tree_cursor)); + } + } + if (OB_SUCC(ret)) { + idx_block_row.endkey_ = &curr_key_; + reach_cursor_end = is_iter_end_; + } + } + return ret; +} int ObIndexBlockMacroIterator::get_next_macro_block( MacroBlockId ¯o_id, int64_t &start_row_offset) @@ -247,7 +327,6 @@ int ObIndexBlockMacroIterator::get_next_macro_block( ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected null index row header", K(ret)); } else { - micro_endkey_allocator_.reuse(); if ((macro_id == begin_ && is_reverse_scan_) || (macro_id == end_ && !is_reverse_scan_)) { is_iter_end_ = true; } @@ -260,6 +339,7 @@ int ObIndexBlockMacroIterator::get_next_macro_block( // traverse all node of this macro block and collect index info ObDatumRowkey rowkey; if (need_record_micro_info_) { + micro_endkey_allocator_.reuse(); reuse_micro_info_array(); if (OB_FAIL(tree_cursor_.get_child_micro_infos( *iter_range_, @@ -464,92 +544,6 @@ void ObIndexBlockMacroIterator::reuse_micro_info_array() tree_cursor_.release_held_path_item(hold_item_); } -ObDualMacroMetaIterator::ObDualMacroMetaIterator() - : allocator_(nullptr), macro_iter_(), sec_meta_iter_(), iter_end_(false), is_inited_(false) {} - -void ObDualMacroMetaIterator::reset() -{ - macro_iter_.reset(); - sec_meta_iter_.reset(); - allocator_ = nullptr; - iter_end_ = false; - is_inited_ = false; -} - -int ObDualMacroMetaIterator::open( - ObSSTable &sstable, - const ObDatumRange &query_range, - const ObITableReadInfo &rowkey_read_info, - ObIAllocator &allocator, - const bool is_reverse_scan, - const bool need_record_micro_info) -{ - UNUSED(need_record_micro_info); - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("Init twice", K(ret)); - } else if (OB_FAIL(macro_iter_.open( - sstable, - query_range, - rowkey_read_info, - allocator, - is_reverse_scan, - true /* need record micro index info */))) { - LOG_WARN("Fail to open index block tree iterator", K(ret)); - } else if (OB_FAIL(sec_meta_iter_.open( - query_range, - blocksstable::DATA_BLOCK_META, - sstable, - rowkey_read_info, - allocator, - is_reverse_scan))) { - LOG_WARN("Fail to open secondary meta iterator", K(ret), K(rowkey_read_info)); - } else { - allocator_ = &allocator; - iter_end_ = false; - is_inited_ = true; - } - return ret; -} - -int ObDualMacroMetaIterator::get_next_macro_block(ObMacroBlockDesc &block_desc) -{ - int ret = OB_SUCCESS; - ObDataMacroBlockMeta *macro_meta = block_desc.macro_meta_; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("Dual macro meta iterator not inited", K(ret)); - } else if (OB_ISNULL(macro_meta)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("Invalid null pointer to read macro meta", K(ret), K(block_desc), KP(macro_meta)); - } else if (OB_UNLIKELY(iter_end_)) { - ret = OB_ITER_END; - } else if (OB_SUCC(macro_iter_.get_next_macro_block(block_desc))) { - if (OB_FAIL(sec_meta_iter_.get_next(*macro_meta))) { - if (OB_ITER_END == ret) { - ret = OB_ERR_UNEXPECTED; - } - LOG_WARN("Fail to get next secondary meta iterator", K(ret), K_(macro_iter), K_(sec_meta_iter)); - } else if (OB_UNLIKELY(block_desc.macro_block_id_ != macro_meta->get_macro_id())) { - ret = OB_ERR_SYS; - LOG_WARN("Logic macro block id from iterated macro block and merge info not match", - K(ret), K(block_desc), KPC(macro_meta)); - } - } else if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("Fail to get next macro block from index tree", K(ret)); - } else { - int tmp_ret = sec_meta_iter_.get_next(*macro_meta); - if (OB_UNLIKELY(OB_ITER_END != tmp_ret)) { - ret = OB_ERR_SYS; - LOG_WARN("Dual macro meta iterated different range", - K(ret), K(tmp_ret), KPC(macro_meta), K(sec_meta_iter_), K(macro_iter_)); - } else { - iter_end_ = true; - } - } - return ret; -} } // namespace blocksstable } // namespace oceanbase diff --git a/src/storage/blocksstable/index_block/ob_index_block_macro_iterator.h b/src/storage/blocksstable/index_block/ob_index_block_macro_iterator.h index e361ead5b4..70ab3c80ef 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_macro_iterator.h +++ b/src/storage/blocksstable/index_block/ob_index_block_macro_iterator.h @@ -14,7 +14,6 @@ #define OB_INDEX_BLOCK_MACRO_ITERATOR_H_ #include "ob_index_block_tree_cursor.h" -#include "ob_sstable_sec_meta_iterator.h" #include "storage/blocksstable/ob_datum_rowkey.h" namespace oceanbase { @@ -100,6 +99,7 @@ public: const bool need_record_micro_info = false) override; int get_next_macro_block(MacroBlockId ¯o_block_id, int64_t &start_row_offset); int get_next_macro_block(blocksstable::ObMacroBlockDesc &block_desc); + int get_next_idx_row(ObMicroIndexInfo &idx_block_row, int64_t &row_offset, bool &reach_cursor_end); int get_cs_range( const ObITableReadInfo &rowkey_read_info, const bool is_start, @@ -156,40 +156,6 @@ private: DISALLOW_COPY_AND_ASSIGN(ObIndexBlockMacroIterator); }; -// Wrap-up of iterate both index block tree and secondary meta in sstable -class ObDualMacroMetaIterator final : public ObIMacroBlockIterator -{ -public: - ObDualMacroMetaIterator(); - virtual ~ObDualMacroMetaIterator() {} - - void reset() override; - virtual int open( - ObSSTable &sstable, - const ObDatumRange &query_range, - const ObITableReadInfo &rowkey_read_info, - ObIAllocator &allocator, - const bool is_reverse_scan = false, - const bool need_record_micro_info = false) override; - virtual int get_next_macro_block(blocksstable::ObMacroBlockDesc &block_desc) override; - - virtual const ObIArray &get_micro_index_infos() const - { - return macro_iter_.get_micro_index_infos(); - } - virtual const ObIArray &get_micro_endkeys() const - { - return macro_iter_.get_micro_endkeys(); - } - TO_STRING_KV(K_(iter_end), K_(is_inited), K_(macro_iter), K_(sec_meta_iter)); -private: - ObIAllocator *allocator_; // allocator for member struct and macro endkeys - ObIndexBlockMacroIterator macro_iter_; - ObSSTableSecMetaIterator sec_meta_iter_; - bool iter_end_; - bool is_inited_; - DISALLOW_COPY_AND_ASSIGN(ObDualMacroMetaIterator); -}; } // namespace blocksstable } // namespace oceanbase diff --git a/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp b/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp index 1eb96f69c4..12118cded9 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp +++ b/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp @@ -1571,6 +1571,7 @@ int ObIndexBlockRowScanner::init_by_micro_data(const ObMicroBlockData &idx_block if (ObMicroBlockData::DDL_MERGE_INDEX_BLOCK == idx_block_data.type_ && is_ddl_merge_type() && is_normal_query_) { if (OB_NOT_NULL(ddl_merge_iter_)) { iter_ = ddl_merge_iter_; + index_format_ = ObIndexFormat::DDL_MERGE; } else { if (OB_ISNULL(iter_buf = allocator_->alloc(sizeof(ObDDLMergeBlockRowIterator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -1585,6 +1586,7 @@ int ObIndexBlockRowScanner::init_by_micro_data(const ObMicroBlockData &idx_block if (nullptr == idx_block_data.get_extra_buf()) { if (OB_NOT_NULL(raw_iter_)) { iter_ = raw_iter_; + index_format_ = ObIndexFormat::RAW_DATA; } else { if (OB_ISNULL(iter_buf = allocator_->alloc(sizeof(ObRAWIndexBlockRowIterator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -1598,6 +1600,7 @@ int ObIndexBlockRowScanner::init_by_micro_data(const ObMicroBlockData &idx_block } else { if (OB_NOT_NULL(transformed_iter_)) { iter_ = transformed_iter_; + index_format_ = ObIndexFormat::TRANSFORMED; } else { if (OB_ISNULL(iter_buf = allocator_->alloc(sizeof(ObTFMIndexBlockRowIterator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -1613,6 +1616,7 @@ int ObIndexBlockRowScanner::init_by_micro_data(const ObMicroBlockData &idx_block } else if (ObMicroBlockData::DDL_BLOCK_TREE == idx_block_data.type_) { if (OB_NOT_NULL(ddl_iter_)) { iter_ = ddl_iter_; + index_format_ = ObIndexFormat::BLOCK_TREE; } else { if (OB_ISNULL(iter_buf = allocator_->alloc(sizeof(ObDDLIndexBlockRowIterator)))) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/storage/blocksstable/ob_data_macro_block_merge_writer.h b/src/storage/blocksstable/ob_data_macro_block_merge_writer.h index c6b4a88820..bec46a7904 100644 --- a/src/storage/blocksstable/ob_data_macro_block_merge_writer.h +++ b/src/storage/blocksstable/ob_data_macro_block_merge_writer.h @@ -14,7 +14,7 @@ #define OCEANBASE_STORAGE_BLOCKSSTABLE_DATA_MACRO_BLOCK_MERGE_WRITER_H_ #include "ob_macro_block_writer.h" #include "ob_macro_block.h" -#include "index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "ob_macro_block.h" #include "ob_datum_row.h" diff --git a/src/storage/blocksstable/ob_macro_block_writer.cpp b/src/storage/blocksstable/ob_macro_block_writer.cpp index 7a88b0849c..d8788cb50d 100644 --- a/src/storage/blocksstable/ob_macro_block_writer.cpp +++ b/src/storage/blocksstable/ob_macro_block_writer.cpp @@ -21,7 +21,7 @@ #include "share/ob_task_define.h" #include "share/schema/ob_table_schema.h" #include "storage/blocksstable/index_block/ob_index_block_builder.h" -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "storage/blocksstable/index_block/ob_index_block_row_struct.h" #include "storage/blocksstable/ob_macro_block_writer.h" #include "storage/blocksstable/cs_encoding/ob_micro_block_cs_encoder.h" diff --git a/src/storage/blocksstable/ob_sstable_meta.h b/src/storage/blocksstable/ob_sstable_meta.h index 3f1aa6cd9c..ef34f1819b 100644 --- a/src/storage/blocksstable/ob_sstable_meta.h +++ b/src/storage/blocksstable/ob_sstable_meta.h @@ -166,7 +166,7 @@ public: return basic_meta_.column_cnt_ - ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); } - OB_INLINE int16_t get_index_tree_height(const bool is_ddl_merge_empty_sstable) const { return is_ddl_merge_empty_sstable ? 2 : basic_meta_.data_index_tree_height_; } + OB_INLINE int16_t get_index_tree_height(const bool is_ddl_merge_sstable) const { return is_ddl_merge_sstable ? 2 : basic_meta_.data_index_tree_height_; } OB_INLINE ObSSTableStatus get_status() const { return static_cast(basic_meta_.status_); diff --git a/src/storage/compaction/ob_index_block_micro_iterator.h b/src/storage/compaction/ob_index_block_micro_iterator.h index c6336990bf..41f3f76868 100644 --- a/src/storage/compaction/ob_index_block_micro_iterator.h +++ b/src/storage/compaction/ob_index_block_micro_iterator.h @@ -20,7 +20,7 @@ #include "storage/blocksstable/ob_block_manager.h" #include "storage/blocksstable/index_block/ob_index_block_tree_cursor.h" #include "storage/blocksstable/ob_macro_block_reader.h" -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "storage/blocksstable/ob_block_manager.h" namespace oceanbase diff --git a/src/storage/compaction/ob_partition_merge_iter.h b/src/storage/compaction/ob_partition_merge_iter.h index 8b81db82aa..43623f675d 100644 --- a/src/storage/compaction/ob_partition_merge_iter.h +++ b/src/storage/compaction/ob_partition_merge_iter.h @@ -21,7 +21,7 @@ #include "storage/access/ob_table_access_param.h" #include "storage/access/ob_table_access_context.h" #include "storage/compaction/ob_index_block_micro_iterator.h" -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.h" #include "storage/blocksstable/ob_datum_row.h" #include "storage/access/ob_table_access_param.h" diff --git a/src/storage/ob_partition_range_spliter.cpp b/src/storage/ob_partition_range_spliter.cpp index c48b44eec8..0b5c71dd76 100644 --- a/src/storage/ob_partition_range_spliter.cpp +++ b/src/storage/ob_partition_range_spliter.cpp @@ -20,7 +20,7 @@ #include "access/ob_multiple_scan_merge.h" #include "storage/tablet/ob_tablet.h" #include "storage/column_store/ob_column_oriented_sstable.h" -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" namespace oceanbase { diff --git a/unittest/storage/blocksstable/test_skip_index_filter.cpp b/unittest/storage/blocksstable/test_skip_index_filter.cpp index b400d01976..d3ac904df7 100644 --- a/unittest/storage/blocksstable/test_skip_index_filter.cpp +++ b/unittest/storage/blocksstable/test_skip_index_filter.cpp @@ -29,7 +29,7 @@ #include "storage/memtable/ob_memtable_interface.h" #include "storage/ob_i_store.h" #include "storage/compaction/ob_tenant_freeze_info_mgr.h" -#include "storage/blocksstable/index_block/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/index_block/ob_index_block_dual_meta_iterator.h" #include "mtlenv/mock_tenant_module_env.h" #include "storage/blocksstable/index_block/ob_agg_row_struct.h" #include "storage/blocksstable/index_block/ob_skip_index_filter_executor.h"