diff --git a/src/storage/column_store/ob_cg_bitmap.cpp b/src/storage/column_store/ob_cg_bitmap.cpp index 8779ca331f..5d409dbf9e 100644 --- a/src/storage/column_store/ob_cg_bitmap.cpp +++ b/src/storage/column_store/ob_cg_bitmap.cpp @@ -34,12 +34,15 @@ int ObCGBitmap::get_first_valid_idx(const ObCSRange &range, ObCSRowId &row_idx) ret = OB_INVALID_ARGUMENT; LOG_WARN("Invalid argument", K(ret), K(range), K_(start_row_id), K(bitmap_.size())); } else if (is_all_false(range)) { + } else if (filter_constant_type_.is_always_true()) { + valid_offset = is_reverse_scan_ ? bitmap_.size() - 1 : 0; } else if (OB_FAIL(bitmap_.next_valid_idx(range.start_row_id_ - start_row_id_, range.get_row_count(), is_reverse_scan_, valid_offset))){ LOG_WARN("Fail to get next valid idx", K(ret), K(range), KPC(this)); - } else if (-1 != valid_offset) { + } + if (OB_SUCC(ret) && -1 != valid_offset) { row_idx = start_row_id_ + valid_offset; } return ret; @@ -51,6 +54,10 @@ int ObCGBitmap::set_bitmap(const ObCSRowId start, const int64_t row_count, const if (OB_UNLIKELY(start < start_row_id_ || row_count != bitmap.size())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("Invalid argument", K(ret), K(start), K_(start_row_id), K(row_count), K(bitmap)); + } else if (filter_constant_type_.is_constant()) { + if (OB_FAIL(bitmap.set_bitmap_batch(0, row_count, filter_constant_type_.is_always_true()))) { + LOG_WARN("Fail to set bitmap batch", K(ret), K(row_count)); + } } else if (!is_reverse) { if (OB_FAIL(bitmap.copy_from(bitmap_, start - start_row_id_, row_count))) { LOG_WARN("Fail to copy bitmap", K(ret), K(start), K(row_count), KPC(this)); diff --git a/src/storage/column_store/ob_cg_bitmap.h b/src/storage/column_store/ob_cg_bitmap.h index dc7537a68e..a5bd1d89ac 100644 --- a/src/storage/column_store/ob_cg_bitmap.h +++ b/src/storage/column_store/ob_cg_bitmap.h @@ -48,6 +48,7 @@ public: } else if (bitmap.filter_constant_type_.is_constant()) { max_filter_constant_id_ = bitmap.max_filter_constant_id_; filter_constant_type_ = bitmap.filter_constant_type_; + bitmap_.reuse(filter_constant_type_.is_always_true()); } else { if (OB_FAIL(bitmap_.copy_from(bitmap.bitmap_, 0, bitmap.bitmap_.size()))) { STORAGE_LOG(WARN, "Fail to set bitmap", K(ret), K(bitmap)); @@ -98,6 +99,9 @@ public: OB_INLINE int switch_context(const uint64_t count, const bool is_reverse) { is_reverse_scan_ = is_reverse; + filter_constant_type_.set_uncertain(); + max_filter_constant_id_ = OB_INVALID_CS_ROW_ID; + start_row_id_ = OB_INVALID_CS_ROW_ID; return bitmap_.reserve(count); } @@ -199,17 +203,18 @@ public: { OB_ASSERT(range.is_valid() && range.end_row_id_ >= start_row_id_); return filter_constant_type_.is_always_true() || - bitmap_.is_all_true(MAX(range.start_row_id_ - start_row_id_, 0), - MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1)); - + (filter_constant_type_.is_uncertain() && + bitmap_.is_all_true(MAX(range.start_row_id_ - start_row_id_, 0), + MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1))); } OB_INLINE bool is_all_false(const ObCSRange &range) const { OB_ASSERT(range.is_valid() && range.end_row_id_ >= start_row_id_); return filter_constant_type_.is_always_false() || - bitmap_.is_all_false(MAX(range.start_row_id_ - start_row_id_, 0), - MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1)); + (filter_constant_type_.is_uncertain() && + bitmap_.is_all_false(MAX(range.start_row_id_ - start_row_id_, 0), + MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1))); } OB_INLINE void set_all_true() diff --git a/src/storage/column_store/ob_cg_scanner.cpp b/src/storage/column_store/ob_cg_scanner.cpp index 0b7dfb0f8f..d7397a904b 100644 --- a/src/storage/column_store/ob_cg_scanner.cpp +++ b/src/storage/column_store/ob_cg_scanner.cpp @@ -315,6 +315,8 @@ int ObCGScanner::apply_filter( ret = OB_SUCCESS; } LOG_TRACE("[COLUMNSTORE] apply filter info in cg", K(ret), K_(query_index_range), K(row_count), + "tablet_id", iter_param_->tablet_id_, "cg_idx", iter_param_->cg_idx_, + "filter_type", filter_info.filter_->get_type(), "filter_constant_type", result_bitmap.get_filter_constant_type(), "filter_constant_id", result_bitmap.get_filter_constant_id(), "bitmap_size", result_bitmap.size(), "popcnt", result_bitmap.popcnt()); diff --git a/src/storage/column_store/ob_co_sstable_rows_filter.cpp b/src/storage/column_store/ob_co_sstable_rows_filter.cpp index 18d46c419a..dc342ff3b0 100644 --- a/src/storage/column_store/ob_co_sstable_rows_filter.cpp +++ b/src/storage/column_store/ob_co_sstable_rows_filter.cpp @@ -221,6 +221,8 @@ int ObCOSSTableRowsFilter::apply(const ObCSRange &range) adjust_batch_size(); prepared_ = false; LOG_DEBUG("[COLUMNSTORE] apply filter info", K(range), + "tablet_id", iter_param_->tablet_id_, + "cg_idx", iter_param_->cg_idx_, "filter_constant_type", bitmap_buffer_[0]->get_filter_constant_type(), "filter_constant_id", bitmap_buffer_[0]->get_filter_constant_id(), "bitmap_size", bitmap_buffer_[0]->size(), diff --git a/unittest/storage/CMakeLists.txt b/unittest/storage/CMakeLists.txt index 9373b9a10e..dbad7abece 100644 --- a/unittest/storage/CMakeLists.txt +++ b/unittest/storage/CMakeLists.txt @@ -142,6 +142,7 @@ if(OB_BUILD_CLOSE_MODULES) storage_unittest(test_block_gc_handler) endif() storage_unittest(test_sstable_log_ts_range_cut test_sstable_log_ts_range_cut.cpp) +storage_unittest(test_cg_bitmap column_store/test_cg_bitmap.cpp) storage_unittest(test_co_sstable column_store/test_co_sstable.cpp) storage_unittest(test_co_sstable_rows_filter column_store/test_co_sstable_rows_filter.cpp) storage_unittest(test_compaction_iter compaction/test_compaction_iter.cpp) diff --git a/unittest/storage/column_store/test_cg_bitmap.cpp b/unittest/storage/column_store/test_cg_bitmap.cpp new file mode 100644 index 0000000000..ab40bd7902 --- /dev/null +++ b/unittest/storage/column_store/test_cg_bitmap.cpp @@ -0,0 +1,152 @@ +/** + * Copyright (c) 2022 OceanBase + * OceanBase is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX STORAGE +#include +#include + +#define private public +#define protected public + +#include +#define private public +#define OK(ass) ASSERT_EQ(OB_SUCCESS, (ass)) +#include "common/ob_target_specific.h" +#include "lib/container/ob_bitmap.h" +#include "lib/allocator/page_arena.h" +#include "storage/ob_storage_struct.h" +#include "storage/column_store/ob_cg_bitmap.h" + + +namespace oceanbase +{ +using namespace common; +using namespace storage; +using namespace blocksstable; +using namespace share; + +namespace unittest +{ + +class TestCGBitmap: public ::testing::Test +{ +public: + TestCGBitmap(); + ~TestCGBitmap() = default; +public: + virtual void SetUp() {}; + virtual void TearDown() {}; + static void SetUpTestCase() + { + LOG_INFO("Supported cpu instructions", + K(common::is_arch_supported(ObTargetArch::Default)), + K(common::is_arch_supported(ObTargetArch::SSE42)), + K(common::is_arch_supported(ObTargetArch::AVX)), + K(common::is_arch_supported(ObTargetArch::AVX2)), + K(common::is_arch_supported(ObTargetArch::AVX512))); + } + static void TearDownTestCase() {}; +public: + ModulePageAllocator allocator_; +}; + +TestCGBitmap::TestCGBitmap() + : allocator_() +{ +} + +TEST_F(TestCGBitmap, test_constant) +{ + ObCGBitmap bitmap(allocator_); + bitmap.init(1000, false); + ASSERT_EQ(false, bitmap.get_filter_constant_type().is_constant()); + + bitmap.reuse(0); + ASSERT_EQ(false, bitmap.is_all_true()); + + sql::ObBoolMask bmt; + bmt.set_always_false(); + bitmap.set_constant_filter_info(bmt, 0); + ASSERT_EQ(true, bitmap.is_all_false()); + + bmt.set_always_true(); + bitmap.set_constant_filter_info(bmt, 0); + ASSERT_EQ(true, bitmap.is_all_true()); + + bitmap.bit_not(); + ASSERT_EQ(false, bitmap.is_all_true()); +} + +TEST_F(TestCGBitmap, set_batch) +{ + ObCGBitmap bitmap(allocator_); + bitmap.init(1000, false); + ASSERT_EQ(false, bitmap.get_filter_constant_type().is_constant()); + + bitmap.reuse(0, true); + ASSERT_EQ(true, bitmap.is_all_true()); + + ObCSRange cs_range(0, 200); + bitmap.set_bitmap_batch(cs_range.start_row_id_, cs_range.end_row_id_, 0); + ASSERT_EQ(false, bitmap.is_all_true()); + ASSERT_EQ(true, bitmap.get_filter_constant_type().is_uncertain()); + ASSERT_EQ(true, bitmap.is_all_false(cs_range)); +} + +TEST_F(TestCGBitmap, copy_from) +{ + ObCGBitmap bitmap1(allocator_); + bitmap1.init(1000, false); + ASSERT_EQ(false, bitmap1.get_filter_constant_type().is_constant()); + + ModulePageAllocator allocator2_; + ObCGBitmap bitmap2(allocator2_); + bitmap2.init(500, false); + ASSERT_EQ(false, bitmap2.get_filter_constant_type().is_constant()); + + bitmap1.reuse(0, true); + ASSERT_EQ(true, bitmap1.is_all_true()); + + bitmap2.reuse(0); + ASSERT_EQ(true, bitmap2.is_all_false()); + + bitmap1.copy_from(bitmap2); + ASSERT_EQ(true, bitmap1.is_all_false()); + ASSERT_EQ(500, bitmap1.size()); + + sql::ObBoolMask bmt; + bmt.set_always_true(); + bitmap2.set_constant_filter_info(bmt, 0); + ASSERT_EQ(true, bitmap2.is_all_true()); + + bitmap1.copy_from(bitmap2); + ASSERT_EQ(true, bitmap1.is_all_true()); + ASSERT_EQ(500, bitmap1.size()); + + bitmap2.set_filter_uncertain(); + bitmap1.copy_from(bitmap2); + ASSERT_EQ(false, bitmap1.get_filter_constant_type().is_constant()); +} + +} //namespace unittest +} //namespace oceanbase + + +int main(int argc, char **argv) +{ + system("rm -rf test_cg_bitmap.log"); + OB_LOGGER.set_file_name("test_cg_bitmap.log", true); + OB_LOGGER.set_log_level("INFO"); + CLOG_LOG(INFO, "begin unittest: test_cg_bitmap"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/unittest/storage/column_store/test_co_sstable.cpp b/unittest/storage/column_store/test_co_sstable.cpp index 2c879c6587..ba913b4098 100644 --- a/unittest/storage/column_store/test_co_sstable.cpp +++ b/unittest/storage/column_store/test_co_sstable.cpp @@ -478,7 +478,7 @@ int main(int argc, char **argv) { system("rm -rf test_co_sstable.log"); OB_LOGGER.set_file_name("test_co_sstable.log", true); - OB_LOGGER.set_log_level("DEBUG"); + OB_LOGGER.set_log_level("INFO"); CLOG_LOG(INFO, "begin unittest: test_co_sstable"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS();