diff --git a/mittest/mtlenv/storage/blocksstable/test_cg_group_by_scanner.cpp b/mittest/mtlenv/storage/blocksstable/test_cg_group_by_scanner.cpp index e40210387..2ce364653 100644 --- a/mittest/mtlenv/storage/blocksstable/test_cg_group_by_scanner.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_cg_group_by_scanner.cpp @@ -652,8 +652,8 @@ TEST_F(TestCGGroupByScanner, test_calc_aggregate_group_by_with_bitmap) ASSERT_EQ(500, group_size); ObCGBitmap bitmap(allocator_); - bitmap.init(group_size); - bitmap.reuse(0); + bitmap.init(group_size, false); + bitmap.reuse(0, false); for (int64_t i = 0; i < group_size; i++) { if (i % 5 == 0 || i % 5 == 2 || i % 5 == 4) { bitmap.set(i); diff --git a/mittest/mtlenv/storage/blocksstable/test_cg_scanner.cpp b/mittest/mtlenv/storage/blocksstable/test_cg_scanner.cpp index 420f68d80..5302743f8 100644 --- a/mittest/mtlenv/storage/blocksstable/test_cg_scanner.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_cg_scanner.cpp @@ -418,7 +418,7 @@ TEST_F(TestCGScanner, test_large_micro_selected) int64_t sql_batch_size = 256; int64_t total_cnt = row_cnt_ / 10; ObCGBitmap bitmap(allocator_); - bitmap.init(row_cnt_); + bitmap.init(row_cnt_, false); bitmap.reuse(0); for (int64_t i = 0; i < total_cnt; i++) { bitmap.set(i * 10); @@ -592,8 +592,8 @@ TEST_F(TestCGScanner, test_filter) int64_t locate_count = 25; ObCGBitmap parent_bitmap(allocator_); ObCGBitmap bitmap(allocator_); - bitmap.init(locate_count); - bitmap.reuse(start, false); + bitmap.init(locate_count, false); + bitmap.reuse(start); ASSERT_EQ(OB_SUCCESS, cg_scanner->locate(ObCSRange(start, locate_count))); ASSERT_EQ(OB_SUCCESS, cg_scanner->apply_filter(nullptr, pd_filter, locate_count, &parent_bitmap, bitmap)); diff --git a/mittest/mtlenv/storage/blocksstable/test_cs_cg_group_by_scanner.cpp b/mittest/mtlenv/storage/blocksstable/test_cs_cg_group_by_scanner.cpp index 24dcf0b83..4062be362 100644 --- a/mittest/mtlenv/storage/blocksstable/test_cs_cg_group_by_scanner.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_cs_cg_group_by_scanner.cpp @@ -654,8 +654,8 @@ TEST_F(TestCSCGGroupByScanner, test_calc_aggregate_group_by_with_bitmap) ASSERT_EQ(500, group_size); ObCGBitmap bitmap(allocator_); - bitmap.init(group_size); - bitmap.reuse(0); + bitmap.init(group_size, false); + bitmap.reuse(0, false); for (int64_t i = 0; i < group_size; i++) { if (i % 5 == 0 || i % 5 == 2 || i % 5 == 4) { bitmap.set(i); diff --git a/mittest/mtlenv/storage/blocksstable/test_pushdown_aggregate.cpp b/mittest/mtlenv/storage/blocksstable/test_pushdown_aggregate.cpp index 9e6c0fb19..368ab91ca 100644 --- a/mittest/mtlenv/storage/blocksstable/test_pushdown_aggregate.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_pushdown_aggregate.cpp @@ -516,8 +516,8 @@ TEST_F(TestPushdownAggregate, test_decide_use_group_by2) int64_t true_count = row_count / ObGroupByCell::USE_GROUP_BY_FILTER_FACTOR - 2; ObCGBitmap bitmap(allocator_); - bitmap.init(row_count); - bitmap.reuse(0); + bitmap.init(row_count, false); + bitmap.reuse(0, false); for (int64_t i = 0; i < row_count; i++) { if (i < true_count) { bitmap.set(i); @@ -529,7 +529,7 @@ TEST_F(TestPushdownAggregate, test_decide_use_group_by2) ASSERT_TRUE(nullptr == group_by_cell.distinct_projector_buf_); ASSERT_TRUE(nullptr == group_by_cell.tmp_group_by_datum_buf_); - bitmap.reuse(0); + bitmap.reuse(0, false); true_count = row_count / ObGroupByCell::USE_GROUP_BY_FILTER_FACTOR + 2; for (int64_t i = 0; i < row_count; i++) { if (i < true_count) { diff --git a/src/sql/engine/basic/ob_pushdown_filter.h b/src/sql/engine/basic/ob_pushdown_filter.h index e682938a9..55e5135f4 100644 --- a/src/sql/engine/basic/ob_pushdown_filter.h +++ b/src/sql/engine/basic/ob_pushdown_filter.h @@ -147,6 +147,11 @@ struct ObBoolMask { return ObBoolMaskType::PROBABILISTIC == bmt_; } + OB_INLINE bool is_constant() const + { + return ObBoolMaskType::ALWAYS_TRUE == bmt_ || + ObBoolMaskType::ALWAYS_FALSE == bmt_; + } OB_INLINE void set_always_true() { bmt_ = ObBoolMaskType::ALWAYS_TRUE; @@ -159,6 +164,10 @@ struct ObBoolMask { bmt_ = ObBoolMaskType::PROBABILISTIC; } + OB_INLINE void set(ObBoolMaskType bmt) + { + bmt_ = bmt; + } TO_STRING_KV(K_(bmt)); ObBoolMaskType bmt_; diff --git a/src/storage/blocksstable/index_block/ob_index_block_row_struct.h b/src/storage/blocksstable/index_block/ob_index_block_row_struct.h index 34b1e790f..87aa2aac1 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_row_struct.h +++ b/src/storage/blocksstable/index_block/ob_index_block_row_struct.h @@ -615,6 +615,11 @@ public: { return sql::ObBoolMaskType::PROBABILISTIC == static_cast(filter_constant_type_); } + OB_INLINE bool is_filter_constant() const + { + return sql::ObBoolMaskType::ALWAYS_TRUE == static_cast(filter_constant_type_) || + sql::ObBoolMaskType::ALWAYS_FALSE == static_cast(filter_constant_type_); + } OB_INLINE void set_filter_constant_type(const sql::ObBoolMaskType type) { filter_constant_type_ = static_cast(type); diff --git a/src/storage/column_store/ob_cg_bitmap.cpp b/src/storage/column_store/ob_cg_bitmap.cpp index 2040596b6..8779ca331 100644 --- a/src/storage/column_store/ob_cg_bitmap.cpp +++ b/src/storage/column_store/ob_cg_bitmap.cpp @@ -25,7 +25,7 @@ using namespace common; namespace storage { -int ObCGBitmap::get_first_valid_idx(const ObCSRange &range, const bool is_reverse_scan, ObCSRowId &row_idx) const +int ObCGBitmap::get_first_valid_idx(const ObCSRange &range, ObCSRowId &row_idx) const { int ret = OB_SUCCESS; int64_t valid_offset = -1; @@ -36,7 +36,7 @@ int ObCGBitmap::get_first_valid_idx(const ObCSRange &range, const bool is_revers } else if (is_all_false(range)) { } else if (OB_FAIL(bitmap_.next_valid_idx(range.start_row_id_ - start_row_id_, range.get_row_count(), - is_reverse_scan, + is_reverse_scan_, valid_offset))){ LOG_WARN("Fail to get next valid idx", K(ret), K(range), KPC(this)); } else if (-1 != valid_offset) { @@ -418,5 +418,86 @@ int ObCGBitmap::get_row_ids( return ret; } +int ObCGBitmap::bit_and(const ObCGBitmap &right) +{ + int ret = OB_SUCCESS; + const sql::ObBoolMask right_constant_type = right.get_filter_constant_type(); + const ObCSRowId right_constant_id = right.get_filter_constant_id(); + if (start_row_id_ != right.start_row_id_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", K(ret), K(start_row_id_), K(right.start_row_id_)); + } else if (filter_constant_type_.is_always_true()) { + if (right_constant_type.is_always_true()) { + max_filter_constant_id_ = is_reverse_scan_ ? MAX(max_filter_constant_id_, right_constant_id) : MIN(max_filter_constant_id_, right_constant_id); + } else if (right_constant_type.is_always_false()) { + max_filter_constant_id_ = right_constant_id; + filter_constant_type_.set_always_false(); + } else { + if (OB_FAIL(bitmap_.copy_from(right.bitmap_, 0, right.bitmap_.size()))) { + LOG_WARN("Fail to copy bitmap", K(ret), KPC(this), K(right)); + } else { + max_filter_constant_id_ = OB_INVALID_CS_ROW_ID; + filter_constant_type_.set_uncertain(); + } + } + } else if (filter_constant_type_.is_always_false()) { + if (right_constant_type.is_always_false()) { + max_filter_constant_id_ = is_reverse_scan_ ? MIN(max_filter_constant_id_, right_constant_id) : MAX(max_filter_constant_id_, right_constant_id); + } // else always_true/uncertain + } else { + // uncertain + if (right_constant_type.is_always_true()) { + } else if (right_constant_type.is_always_false()) { + max_filter_constant_id_ = right_constant_id; + filter_constant_type_.set_always_false(); + } else { + if (OB_FAIL(bitmap_.bit_and(right.bitmap_))) { + LOG_WARN("Fail to bit and", K(ret), KPC(this), K(right)); + } + } + } + return ret; +} + +int ObCGBitmap::bit_or(const ObCGBitmap &right) +{ + int ret = OB_SUCCESS; + const sql::ObBoolMask right_constant_type = right.get_filter_constant_type(); + const ObCSRowId right_constant_id = right.get_filter_constant_id(); + if (start_row_id_ != right.start_row_id_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", K(ret), K(start_row_id_), K(right.start_row_id_)); + } else if (filter_constant_type_.is_always_true()) { + if (right_constant_type.is_always_true()) { + max_filter_constant_id_ = is_reverse_scan_ ? MIN(max_filter_constant_id_, right_constant_id) : MAX(max_filter_constant_id_, right_constant_id); + } // else always_false/uncertain + } else if (filter_constant_type_.is_always_false()) { + if (right_constant_type.is_always_true()) { + max_filter_constant_id_ = right_constant_id; + filter_constant_type_.set_always_true(); + } else if (right_constant_type.is_always_false()) { + max_filter_constant_id_ = is_reverse_scan_ ? MAX(max_filter_constant_id_, right_constant_id) : MIN(max_filter_constant_id_, right_constant_id); + } else { + if (OB_FAIL(bitmap_.copy_from(right.bitmap_, 0, right.bitmap_.size()))) { + LOG_WARN("Fail to copy bitmap", K(ret), KPC(this), K(right)); + } else { + max_filter_constant_id_ = OB_INVALID_CS_ROW_ID; + filter_constant_type_.set_uncertain(); + } + } + } else { + if (right_constant_type.is_always_true()) { + max_filter_constant_id_ = right_constant_id; + filter_constant_type_.set_always_true(); + } else if (right_constant_type.is_always_false()) { + } else { + if (OB_FAIL(bitmap_.bit_or(right.bitmap_))) { + LOG_WARN("Fail to bit_or bitmap", K(ret), KPC(this), K(right)); + } + } + } + return ret; +} + } } diff --git a/src/storage/column_store/ob_cg_bitmap.h b/src/storage/column_store/ob_cg_bitmap.h index 38e7b4f44..dc7537a68 100644 --- a/src/storage/column_store/ob_cg_bitmap.h +++ b/src/storage/column_store/ob_cg_bitmap.h @@ -25,14 +25,17 @@ class ObCGBitmap { public: ObCGBitmap(ObIAllocator &allocator) : + is_reverse_scan_(false), bitmap_(allocator), - start_row_id_(OB_INVALID_CS_ROW_ID) + start_row_id_(OB_INVALID_CS_ROW_ID), + max_filter_constant_id_(OB_INVALID_CS_ROW_ID), + filter_constant_type_() {} virtual ~ObCGBitmap() {} bool is_valid() const { - return start_row_id_ != OB_INVALID_CS_ROW_ID; + return filter_constant_type_.is_constant() || start_row_id_ != OB_INVALID_CS_ROW_ID; } int copy_from(const ObCGBitmap &bitmap) { @@ -42,26 +45,43 @@ public: STORAGE_LOG(WARN, "Invalid argument", K(ret), K(bitmap)); } else if (OB_FAIL(bitmap_.reserve(bitmap.bitmap_.size()))) { STORAGE_LOG(WARN, "Fail to expand bitmap", K(ret), K(bitmap.bitmap_.size())); - } else if (OB_FAIL(bitmap_.copy_from(bitmap.bitmap_, 0, bitmap.bitmap_.size()))) { - STORAGE_LOG(WARN, "Fail to set bitmap", K(ret), K(bitmap)); + } else if (bitmap.filter_constant_type_.is_constant()) { + max_filter_constant_id_ = bitmap.max_filter_constant_id_; + filter_constant_type_ = bitmap.filter_constant_type_; } else { + if (OB_FAIL(bitmap_.copy_from(bitmap.bitmap_, 0, bitmap.bitmap_.size()))) { + STORAGE_LOG(WARN, "Fail to set bitmap", K(ret), K(bitmap)); + } else { + max_filter_constant_id_ = OB_INVALID_CS_ROW_ID; + filter_constant_type_.set_uncertain(); + } + } + + if (OB_SUCC(ret)) { + is_reverse_scan_ = bitmap.is_reverse_scan_; start_row_id_ = bitmap.start_row_id_; } return ret; } - int init( - const uint64_t count, - const ObCSRowId start_row_id = OB_INVALID_CS_ROW_ID, + int init(const uint64_t count, const bool is_reverse) + { + is_reverse_scan_ = is_reverse; + return bitmap_.init(count); + } + + void reuse( + const ObCSRowId start_row_id, const bool is_all_true = false) { start_row_id_ = start_row_id; - return bitmap_.init(count, is_all_true); - } - - void reuse(const ObCSRowId start_row_id, const bool is_all_true = false) - { - start_row_id_ = start_row_id; + if (is_all_true) { + filter_constant_type_.set_always_true(); + max_filter_constant_id_ = is_reverse_scan_ ? start_row_id : start_row_id + bitmap_.size() - 1; + } else { + filter_constant_type_.set_always_false(); + max_filter_constant_id_ = is_reverse_scan_ ? start_row_id : start_row_id + bitmap_.size() - 1; + } bitmap_.reuse(is_all_true); } @@ -75,33 +95,48 @@ public: return start_row_id_; } - OB_INLINE int reserve(uint64_t count) + OB_INLINE int switch_context(const uint64_t count, const bool is_reverse) { + is_reverse_scan_ = is_reverse; return bitmap_.reserve(count); } OB_INLINE int set(const ObCSRowId row_idx, const bool value = true) { OB_ASSERT(row_idx >= start_row_id_); + filter_constant_type_.set_uncertain(); return bitmap_.set(row_idx - start_row_id_, value); } OB_INLINE int wipe(const ObCSRowId row_idx) { OB_ASSERT(row_idx >= start_row_id_); + filter_constant_type_.set_uncertain(); return bitmap_.wipe(row_idx - start_row_id_); } OB_INLINE bool test(const ObCSRowId row_idx) const { OB_ASSERT(row_idx >= start_row_id_); - return bitmap_.test(row_idx - start_row_id_); + bool res; + if (filter_constant_type_.is_constant()) { + res = filter_constant_type_.is_always_true(); + } else { + res = bitmap_.test(row_idx - start_row_id_); + } + return res; } OB_INLINE bool operator[](const ObCSRowId row_idx) const { OB_ASSERT(row_idx >= start_row_id_); - return bitmap_.test(row_idx - start_row_id_); + bool res; + if (filter_constant_type_.is_constant()) { + res = filter_constant_type_.is_always_true(); + } else { + res = bitmap_.test(row_idx - start_row_id_); + } + return res; } OB_INLINE int64_t capacity() const @@ -116,86 +151,110 @@ public: OB_INLINE uint64_t popcnt() const { - return bitmap_.popcnt(); + uint64_t cnt = 0; + if (filter_constant_type_.is_constant()) { + cnt = filter_constant_type_.is_always_true() ? bitmap_.size() : 0; + } else { + cnt = bitmap_.popcnt(); + } + return cnt; } OB_INLINE bool is_all_true() const { - return bitmap_.is_all_true(); + return filter_constant_type_.is_always_true() || + (filter_constant_type_.is_uncertain() && bitmap_.is_all_true()); } OB_INLINE bool is_all_false() const { - return bitmap_.is_all_false(); + return filter_constant_type_.is_always_false() || + (filter_constant_type_.is_uncertain() && bitmap_.is_all_false()); } - OB_INLINE int bit_and(const ObCGBitmap &right) - { - int ret = OB_SUCCESS; - if (start_row_id_ != right.start_row_id_) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument", K(ret), K(start_row_id_), K(right.start_row_id_)); - } else if (OB_FAIL(bitmap_.bit_and(right.bitmap_))) { - STORAGE_LOG(WARN, "Fail to bit and", K(ret), KPC(this), K(right)); - } - return ret; - } + OB_INLINE int bit_and(const ObCGBitmap &right); - OB_INLINE int bit_or(const ObCGBitmap &right) - { - int ret = OB_SUCCESS; - if (start_row_id_ != right.start_row_id_) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "Invalid argument", K(ret), K(start_row_id_), K(right.start_row_id_)); - } else if (OB_FAIL(bitmap_.bit_or(right.bitmap_))) { - STORAGE_LOG(WARN, "Fail to bit or", K(ret), KPC(this), K(right)); - } - return ret; - } + OB_INLINE int bit_or(const ObCGBitmap &right); OB_INLINE int bit_not() { - return bitmap_.bit_not(); + int ret = OB_SUCCESS; + if (filter_constant_type_.is_always_true()) { + filter_constant_type_.set_always_false(); + } else if (filter_constant_type_.is_always_false()) { + filter_constant_type_.set_always_true(); + } else { + ret = bitmap_.bit_not(); + } + return ret; } int append_bitmap(const ObBitmap &bitmap, const uint32_t offset, const bool is_reverse) { + filter_constant_type_.set_uncertain(); return bitmap_.append_bitmap(bitmap, offset, is_reverse); } OB_INLINE bool is_all_true(const ObCSRange &range) const { OB_ASSERT(range.is_valid() && range.end_row_id_ >= start_row_id_); - return bitmap_.is_all_true(MAX(range.start_row_id_ - start_row_id_, 0), - MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1)); + return filter_constant_type_.is_always_true() || + bitmap_.is_all_true(MAX(range.start_row_id_ - start_row_id_, 0), + MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1)); } OB_INLINE bool is_all_false(const ObCSRange &range) const { OB_ASSERT(range.is_valid() && range.end_row_id_ >= start_row_id_); - return bitmap_.is_all_false(MAX(range.start_row_id_ - start_row_id_, 0), - MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1)); + return filter_constant_type_.is_always_false() || + bitmap_.is_all_false(MAX(range.start_row_id_ - start_row_id_, 0), + MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1)); } OB_INLINE void set_all_true() { - bitmap_.reuse(true); + filter_constant_type_.set_always_true(); + max_filter_constant_id_ = is_reverse_scan_ ? start_row_id_ : start_row_id_ + bitmap_.size() - 1; } OB_INLINE void set_all_false() { - bitmap_.reuse(false); + filter_constant_type_.set_always_false(); + max_filter_constant_id_ = is_reverse_scan_ ? start_row_id_ : start_row_id_ + bitmap_.size() - 1; } + OB_INLINE int set_bitmap_batch(ObCSRowId start, ObCSRowId end, const bool value) { int64_t offset = MAX(start - start_row_id_, 0); int64_t count = MIN(end - start + 1, bitmap_.size() - offset); + filter_constant_type_.set_uncertain(); return bitmap_.set_bitmap_batch(offset, count, value); } + OB_INLINE void set_constant_filter_info(const sql::ObBoolMask filter_constant_type, const ObCSRowId filter_constant_id) + { + filter_constant_type_ = filter_constant_type; + max_filter_constant_id_ = filter_constant_id; + } + + OB_INLINE sql::ObBoolMask get_filter_constant_type() const + { + return filter_constant_type_; + } + + OB_INLINE ObCSRowId get_filter_constant_id() const + { + return max_filter_constant_id_; + } + + OB_INLINE void set_filter_uncertain() + { + return filter_constant_type_.set_uncertain(); + } + int set_bitmap(const ObCSRowId start, const int64_t row_count, const bool is_reverse, ObBitmap &bitmap) const; - int get_first_valid_idx(const ObCSRange &range, const bool is_reverse_scan, ObCSRowId &row_idx) const; + int get_first_valid_idx(const ObCSRange &range, ObCSRowId &row_idx) const; int get_row_ids(int32_t *row_ids, int64_t &row_cap, ObCSRowId ¤t, @@ -203,11 +262,15 @@ public: const ObCSRange &data_range, const int64_t batch_size, const bool is_reverse); - TO_STRING_KV(K_(bitmap), K_(start_row_id)); - + TO_STRING_KV(K_(is_reverse_scan), K_(start_row_id), K_(max_filter_constant_id), + K_(filter_constant_type), K_(bitmap)); private: + bool is_reverse_scan_; common::ObBitmap bitmap_; ObCSRowId start_row_id_; + // max_filter_constant_id_ may greater/less than offset+size + ObCSRowId max_filter_constant_id_; + sql::ObBoolMask filter_constant_type_; }; int convert_bitmap_to_cs_index(int32_t *row_ids, diff --git a/src/storage/column_store/ob_cg_prefetcher.cpp b/src/storage/column_store/ob_cg_prefetcher.cpp index c2c42a978..3d22f0491 100644 --- a/src/storage/column_store/ob_cg_prefetcher.cpp +++ b/src/storage/column_store/ob_cg_prefetcher.cpp @@ -31,6 +31,8 @@ void ObCGPrefetcher::reset() micro_data_prewarm_idx_ = 0; cur_micro_data_read_idx_ = -1; agg_group_ = nullptr; + filter_constant_type_.set_uncertain(); + max_filter_constant_id_ = OB_INVALID_CS_ROW_ID; ObIndexTreeMultiPassPrefetcher::reset(); } @@ -44,6 +46,8 @@ void ObCGPrefetcher::reuse() filter_bitmap_ = nullptr; micro_data_prewarm_idx_ = 0; cur_micro_data_read_idx_ = -1; + filter_constant_type_.set_uncertain(); + max_filter_constant_id_ = OB_INVALID_CS_ROW_ID; ObIndexTreeMultiPassPrefetcher::reuse(); } @@ -209,10 +213,19 @@ int ObCGPrefetcher::locate_in_prefetched_data(bool &found) { int ret = OB_SUCCESS; found = false; - const ObCSRowId start_row_idx = is_reverse_scan_ ? query_index_range_.end_row_id_ : query_index_range_.start_row_id_; - int64_t max_data_prefetched_idx = MAX(micro_data_prewarm_idx_, micro_data_prefetch_idx_); - if (max_data_prefetched_idx > 0 && max_data_prefetched_idx > cur_micro_data_read_idx_) { + if (filter_constant_type_.is_constant() && + ((is_reverse_scan_ && max_filter_constant_id_ <= query_index_range_.end_row_id_) || + (!is_reverse_scan_ && max_filter_constant_id_ >= query_index_range_.start_row_id_))) { + found = true; + is_prefetch_end_ = + (is_reverse_scan_ && max_filter_constant_id_ <= query_index_range_.start_row_id_) || + (!is_reverse_scan_ && max_filter_constant_id_ >= query_index_range_.end_row_id_); + } else { + filter_constant_type_.set_uncertain(); + max_filter_constant_id_ = is_reverse_scan_ ? query_index_range_.end_row_id_ + 1 : query_index_range_.start_row_id_ - 1; int cmp_ret = -1; + int64_t max_data_prefetched_idx = MAX(micro_data_prewarm_idx_, micro_data_prefetch_idx_); + const ObCSRowId start_row_idx = is_reverse_scan_ ? query_index_range_.end_row_id_ : query_index_range_.start_row_id_; for (int64_t micro_data_idx = MAX(0, cur_micro_data_read_idx_); OB_SUCC(ret) && cmp_ret < 0 && micro_data_idx < max_data_prefetched_idx; micro_data_idx++) { ObMicroIndexInfo µ_info = micro_data_infos_[micro_data_idx % max_micro_handle_cnt_]; const ObCSRange µ_range = micro_info.get_row_range(); @@ -231,10 +244,12 @@ int ObCGPrefetcher::locate_in_prefetched_data(bool &found) found = true; } } + } - if (OB_SUCC(ret) && found) { - cmp_ret = 1; - is_prefetch_end_ = false; + if (OB_SUCC(ret) && found) { + if (!is_prefetch_end_) { + int cmp_ret = 1; + int64_t max_data_prefetched_idx = MAX(micro_data_prewarm_idx_, micro_data_prefetch_idx_); micro_data_prefetch_idx_ = max_data_prefetched_idx; const ObCSRowId end_row_id = is_reverse_scan_ ? query_index_range_.start_row_id_ : query_index_range_.end_row_id_; for (int64_t micro_data_idx = max_data_prefetched_idx - 1; OB_SUCC(ret) && cmp_ret > 0 && micro_data_idx > cur_micro_data_fetch_idx_; micro_data_idx--) { @@ -296,11 +311,19 @@ bool ObCGPrefetcher::locate_back(const ObCSRange &locate_range) { bool is_locate_back = false; if (0 < query_index_range_.end_row_id_) { - int micro_index = MAX(0, cur_micro_data_read_idx_); - if (is_reverse_scan_) { - is_locate_back = locate_range.end_row_id_ > micro_data_infos_[micro_index % max_micro_handle_cnt_].get_row_range().end_row_id_; + if (filter_constant_type_.is_constant()) { + if (is_reverse_scan_) { + is_locate_back = locate_range.end_row_id_ > query_index_range_.end_row_id_; + } else { + is_locate_back = locate_range.start_row_id_ < query_index_range_.start_row_id_; + } } else { - is_locate_back = locate_range.start_row_id_ < micro_data_infos_[micro_index % max_micro_handle_cnt_].get_row_range().start_row_id_; + int micro_index = MAX(0, cur_micro_data_read_idx_); + if (is_reverse_scan_) { + is_locate_back = locate_range.end_row_id_ > micro_data_infos_[micro_index % max_micro_handle_cnt_].get_row_range().end_row_id_; + } else { + is_locate_back = locate_range.start_row_id_ < micro_data_infos_[micro_index % max_micro_handle_cnt_].get_row_range().start_row_id_; + } } } return is_locate_back; @@ -323,7 +346,10 @@ int ObCGPrefetcher::locate(const ObCSRange &range, const ObCGBitmap *bitmap) ObIndexTreeMultiPassPrefetcher::reuse(); micro_data_prewarm_idx_ = 0; cur_micro_data_read_idx_ = -1; + filter_constant_type_.set_uncertain(); + max_filter_constant_id_ = is_reverse_scan_ ? query_index_range_.end_row_id_ + 1 : query_index_range_.start_row_id_ - 1; update_query_range(range); + cur_level_ = 0; read_handles_[0].row_state_ = ObSSTableRowState::IN_BLOCK; if (OB_FAIL(open_index_root())) { LOG_WARN("Fail to open index root", K(ret)); @@ -345,6 +371,48 @@ int ObCGPrefetcher::locate(const ObCSRange &range, const ObCGBitmap *bitmap) LOG_WARN("Fail to refresh index tree", K(ret), K_(query_index_range)); } } + + if (OB_SUCC(ret) && OB_FAIL(refresh_constant_filter_info())) { + LOG_WARN("Fail to refresh constant filter info", K(ret)); + } + return ret; +} + +// update filter_constant_type_ and max_filter_constant_id_ +int ObCGPrefetcher::refresh_constant_filter_info() +{ + int ret = OB_SUCCESS; + if (!is_cg_scanner() || nullptr == sstable_index_filter_ || filter_constant_type_.is_constant()) { + // skip check when filter_constant_type_ is constant + } else { + // update max row id for constant filtered result + bool need_check = true; + if (need_check && cur_level_ == index_tree_height_ - 1) { + int64_t max_data_prefetched_idx = MAX(micro_data_prewarm_idx_, micro_data_prefetch_idx_); + for (int64_t micro_data_idx = cur_micro_data_fetch_idx_ + 1; need_check && micro_data_idx < max_data_prefetched_idx; micro_data_idx++) { + const ObMicroIndexInfo &index_info = micro_data_infos_[micro_data_idx % max_micro_handle_cnt_]; + if (!check_and_update_constant_filter(index_info)) { + need_check = false; + } + } + } + + for (int64_t level = 0; need_check && level <= cur_level_; level++) { + ObIndexTreeLevelHandle &tree_handle = tree_handles_[level]; + for (int64_t idx = tree_handle.read_idx_; need_check && idx <= tree_handle.prefetch_idx_; idx++) { + const ObMicroIndexInfo &index_info = tree_handle.index_block_read_handles_[idx % INDEX_TREE_PREFETCH_DEPTH].index_info_; + if (is_constant_filter_continuous(index_info)) { + if (filter_constant_type_.is_uncertain() || + index_info.get_filter_constant_type() == filter_constant_type_.bmt_) { + filter_constant_type_.set(index_info.get_filter_constant_type()); + max_filter_constant_id_ = is_reverse_scan_ ? index_info.get_row_range().start_row_id_ : index_info.get_row_range().end_row_id_; + } else { + need_check = false; + } + } + } + } + } return ret; } @@ -396,7 +464,9 @@ int ObCGPrefetcher::prefetch_micro_data() break; } else { // read index leaf and prefetch micro data - while (OB_SUCC(ret) && !is_prefetch_end_ && prefetched_cnt < prefetch_depth) { + bool hit_constant_filter = false; + while (OB_SUCC(ret) && (!is_prefetch_end_ || hit_constant_filter) && prefetched_cnt < prefetch_depth) { + hit_constant_filter = false; prefetch_micro_idx = micro_data_prefetch_idx_ % max_micro_handle_cnt_; ObMicroIndexInfo &block_info = micro_data_infos_[prefetch_micro_idx]; bool can_agg = false; @@ -415,9 +485,6 @@ int ObCGPrefetcher::prefetch_micro_data() ret = OB_SUCCESS; break; } - } else if (compare_range(block_info.get_row_range()) > 0) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Unexpected prefetch index info", K(ret), K(block_info), KPC(this)); } else if (!contain_rows(block_info.get_row_range())) { // continue if no rows contained } else if (nullptr != sstable_index_filter_ @@ -426,12 +493,33 @@ int ObCGPrefetcher::prefetch_micro_data() iter_param_->read_info_, block_info, *(access_ctx_->allocator_), iter_param_->vectorized_enabled_))) { LOG_WARN("Fail to check if can skip prefetch", K(ret), K(block_info)); - } else if (nullptr != sstable_index_filter_ - && (block_info.is_filter_always_false() || block_info.is_filter_always_true())) { - // TODO: skip data block which is always_false/always_true and record the result in filter bitmap - prefetched_cnt++; - micro_data_prefetch_idx_++; - tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++; + // exist only in filter iter + } else if (nullptr != sstable_index_filter_ && block_info.is_filter_constant()) { + if (check_and_update_constant_filter(block_info)) { + hit_constant_filter = true; + } else { + prefetched_cnt++; + tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++; + if (0 < compare_range(block_info.get_row_range())) { + micro_data_prewarm_idx_ = micro_data_prefetch_idx_ + 1; + } else { + micro_data_prefetch_idx_++; + } + } + } else if (0 < compare_range(block_info.get_row_range())) { + if (!is_cg_scanner() || nullptr == sstable_index_filter_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected prefetch range", K(ret), K_(cg_iter_type), K_(query_index_range), K(block_info.get_row_range())); + } else if (OB_FAIL(prefetch_data_block( + micro_data_prefetch_idx_, + block_info, + micro_data_handles_[prefetch_micro_idx]))) { + LOG_WARN("fail to prefetch_block_data", K(ret), K(block_info)); + } else { + prefetched_cnt++; + micro_data_prewarm_idx_ = micro_data_prefetch_idx_ + 1; + tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++; + } } else if (OB_FAIL(can_agg_micro_index(block_info, can_agg))) { LOG_WARN("fail to check can agg index info", K(ret), K(block_info), KPC_(agg_group)); } else if (can_agg) { @@ -451,7 +539,7 @@ int ObCGPrefetcher::prefetch_micro_data() tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++; } - if (OB_SUCC(ret) && (0 == compare_range(block_info.get_row_range()))) { + if (OB_SUCC(ret) && (0 <= compare_range(block_info.get_row_range()))) { is_prefetch_end_ = true; } } @@ -561,7 +649,7 @@ int ObCGPrefetcher::ObCSIndexTreeLevelHandle::prefetch( *(prefetcher.access_ctx_->allocator_), prefetcher.iter_param_->vectorized_enabled_))) { LOG_WARN("Fail to check if can skip prefetch", K(ret), K(index_info)); - // TODO: skip data block which is always_false/always_true and record the result in filter bitmap + } else if (prefetcher.check_and_update_constant_filter(index_info)) { } else if (OB_FAIL(prefetcher.can_agg_micro_index(index_info, can_agg))) { LOG_WARN("fail to check index info", K(ret), K(index_info), KPC(prefetcher.agg_group_)); } else if (can_agg) { @@ -584,7 +672,7 @@ int ObCGPrefetcher::ObCSIndexTreeLevelHandle::prefetch( } } } - if (OB_SUCC(ret) && !is_prefetch_end_ && 0 == prefetcher.compare_range(index_info.get_row_range())) { + if (OB_SUCC(ret) && !is_prefetch_end_ && 0 <= prefetcher.compare_range(index_info.get_row_range())) { is_prefetch_end_ = true; } } @@ -764,12 +852,14 @@ int ObCGPrefetcher::prewarm() iter_param_->read_info_, block_info, *(access_ctx_->allocator_), iter_param_->vectorized_enabled_))) { LOG_WARN("Fail to check if can skip prefetch", K(ret), K(block_info)); - } else if (nullptr != sstable_index_filter_ - && (block_info.is_filter_always_false() || block_info.is_filter_always_true())) { - // TODO: skip data block which is always_false/always_true and record the result in filter bitmap - prefetched_cnt++; - micro_data_prewarm_idx_++; - tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++; + } else if (nullptr != sstable_index_filter_ && block_info.is_filter_constant()) { + if (check_and_update_constant_filter(block_info)) { + prefetched_cnt++; + } else { + prefetched_cnt++; + micro_data_prewarm_idx_++; + tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++; + } } else if (OB_FAIL(prefetch_data_block( micro_data_prewarm_idx_, block_info, diff --git a/src/storage/column_store/ob_cg_prefetcher.h b/src/storage/column_store/ob_cg_prefetcher.h index 20e1ceb66..cd79c289b 100644 --- a/src/storage/column_store/ob_cg_prefetcher.h +++ b/src/storage/column_store/ob_cg_prefetcher.h @@ -31,6 +31,8 @@ public: leaf_query_range_(), filter_bitmap_(nullptr), micro_data_prewarm_idx_(0), + filter_constant_type_(), + max_filter_constant_id_(OB_INVALID_CS_ROW_ID), cur_micro_data_read_idx_(-1), agg_group_(nullptr), sstable_index_filter_(nullptr) @@ -73,18 +75,47 @@ public: { return index_info.has_agg_data() && index_info.is_filter_uncertain(); } + OB_INLINE bool is_cg_scanner() const + { return ObICGIterator::OB_CG_SCANNER == cg_iter_type_; } void recycle_block_data(); void set_agg_group(ObAggGroupBase *agg_group) { agg_group_ = agg_group; } void set_project_type(const bool project_without_filter) { is_project_without_filter_ = project_without_filter; } + OB_INLINE sql::ObBoolMask get_filter_constant_type() const + { + return filter_constant_type_; + } + OB_INLINE ObCSRowId get_max_filter_constant_id() const + { + return max_filter_constant_id_; + } INHERIT_TO_STRING_KV("ObCGPrefetcher", ObIndexTreeMultiPassPrefetcher, K_(is_reverse_scan), K_(is_project_without_filter), K_(need_prewarm), K_(query_index_range), K_(query_range), K_(cg_iter_type), K_(micro_data_prewarm_idx), K_(cur_micro_data_read_idx), KP_(filter_bitmap), - KP_(agg_group), KP_(sstable_index_filter)); + KP_(agg_group), KP_(sstable_index_filter), K_(filter_constant_type), K_(max_filter_constant_id)); protected: int get_prefetch_depth(int64_t &depth, const int64_t prefetching_idx); private: int prewarm(); + int refresh_constant_filter_info(); + bool is_constant_filter_continuous(const ObMicroIndexInfo &index_info) + { + return index_info.is_filter_constant() && + ((is_reverse_scan_ && 0 == index_info.get_row_range().compare(max_filter_constant_id_ - 1)) || + (!is_reverse_scan_ && 0 == index_info.get_row_range().compare(max_filter_constant_id_ + 1))); + } + bool check_and_update_constant_filter(const ObMicroIndexInfo &index_info) + { + bool can_continuous = false; + if (is_constant_filter_continuous(index_info) && + (filter_constant_type_.is_uncertain() || + index_info.get_filter_constant_type() == filter_constant_type_.bmt_)) { + filter_constant_type_.set(index_info.get_filter_constant_type()); + max_filter_constant_id_ = is_reverse_scan_ ? index_info.get_row_range().start_row_id_ : index_info.get_row_range().end_row_id_; + can_continuous = true; + } + return can_continuous; + } struct ObCSIndexTreeLevelHandle : public ObIndexTreeLevelHandle { public: int prefetch(const int64_t level, ObCGPrefetcher &prefetcher); @@ -120,6 +151,12 @@ private: ObDatumRange leaf_query_range_; const ObCGBitmap *filter_bitmap_; int64_t micro_data_prewarm_idx_; + // filter_constant_type_ and max_filter_constant_id_ only avaliable in filter scanner + // when filter_constant_type_ is always_true, max_filter_constant_id_ means the max continuous rowid that selected by the filter; + // when filter_constant_type_ is always_false, max_filter_constant_id_ means the max continuose rowid that filtered; + // when filter_constant_type_ is probablistic, means nothing + sql::ObBoolMask filter_constant_type_; + ObCSRowId max_filter_constant_id_; public: int64_t cur_micro_data_read_idx_; ObAggGroupBase *agg_group_; diff --git a/src/storage/column_store/ob_cg_scanner.cpp b/src/storage/column_store/ob_cg_scanner.cpp index 420bdbbf6..0b7dfb0f8 100644 --- a/src/storage/column_store/ob_cg_scanner.cpp +++ b/src/storage/column_store/ob_cg_scanner.cpp @@ -193,7 +193,7 @@ int ObCGScanner::locate( if (nullptr != bitmap) { if (bitmap->is_all_true()) { locate_bitmap = nullptr; - } else if (OB_FAIL(bitmap->get_first_valid_idx(range, is_reverse_scan_, current_))) { + } else if (OB_FAIL(bitmap->get_first_valid_idx(range, current_))) { LOG_WARN("Fail to get first valid idx", K(ret), K_(is_reverse_scan), K(range)); } else { if (is_reverse_scan_) { @@ -315,6 +315,8 @@ int ObCGScanner::apply_filter( ret = OB_SUCCESS; } LOG_TRACE("[COLUMNSTORE] apply filter info in cg", K(ret), K_(query_index_range), K(row_count), + "filter_constant_type", result_bitmap.get_filter_constant_type(), + "filter_constant_id", result_bitmap.get_filter_constant_id(), "bitmap_size", result_bitmap.size(), "popcnt", result_bitmap.popcnt()); return ret; } @@ -330,32 +332,62 @@ int ObCGScanner::get_next_valid_block(sql::ObPushdownFilterExecutor *parent, LOG_DEBUG("Calc to end of prefetched data", K(ret), K(prefetcher_.cur_micro_data_fetch_idx_), K(prefetcher_.micro_data_prefetch_idx_)); } else { - if (prefetcher_.cur_micro_data_fetch_idx_ > -1) { - prefetcher_.current_micro_handle().reset(); + if (result_bitmap.get_filter_constant_type().is_constant()) { + result_bitmap.set_filter_uncertain(); + ObCSRowId prefetch_constant_id = prefetcher_.get_max_filter_constant_id(); + sql::ObBoolMask prefetch_constant_type = prefetcher_.get_filter_constant_type(); + // reset bitmap when bits not consistent with filter constant type after set by prefetcher + // filter info + // bitmap set value could be decided by the logic op type of filter root in the iter param + bool need_reset = is_reverse_scan_ ? + prefetch_constant_id <= query_index_range_.end_row_id_ : + prefetch_constant_id >= query_index_range_.start_row_id_; + if (!need_reset) { + } else if (is_reverse_scan_) { + if (OB_FAIL(result_bitmap.set_bitmap_batch( + MAX(query_index_range_.start_row_id_, prefetch_constant_id), + query_index_range_.end_row_id_, + prefetch_constant_type.is_always_true()))) { + LOG_WARN("Fail to set bitmap batch", K(ret), K_(query_index_range), K(prefetch_constant_id)); + } + } else { + if (OB_FAIL(result_bitmap.set_bitmap_batch( + query_index_range_.start_row_id_, + MIN(query_index_range_.end_row_id_, prefetch_constant_id), + prefetch_constant_type.is_always_true()))) { + LOG_WARN("Fail to set bitmap batch", K(ret), K_(query_index_range), K(prefetch_constant_id)); + } + } } - ++prefetcher_.cur_micro_data_fetch_idx_; - ++prefetcher_.cur_micro_data_read_idx_; - const ObMicroIndexInfo &index_info = prefetcher_.current_micro_info(); - const ObCSRange &row_range = index_info.get_row_range(); - if (index_info.is_filter_always_false()) { - if (OB_FAIL(result_bitmap.set_bitmap_batch( - MAX(query_index_range_.start_row_id_, row_range.start_row_id_), - MIN(query_index_range_.end_row_id_, row_range.end_row_id_), - false))) { - LOG_WARN("Fail to set bitmap batch", K(ret), K(row_range)); + + if (OB_SUCC(ret)) { + if (prefetcher_.cur_micro_data_fetch_idx_ > -1) { + prefetcher_.current_micro_handle().reset(); } - } else if (index_info.is_filter_always_true()) { - if (OB_FAIL(result_bitmap.set_bitmap_batch( - MAX(query_index_range_.start_row_id_, row_range.start_row_id_), - MIN(query_index_range_.end_row_id_, row_range.end_row_id_), - true))) { - LOG_WARN("Fail to set bitmap batch", K(ret), K(row_range)); + ++prefetcher_.cur_micro_data_fetch_idx_; + ++prefetcher_.cur_micro_data_read_idx_; + const ObMicroIndexInfo &index_info = prefetcher_.current_micro_info(); + const ObCSRange &row_range = index_info.get_row_range(); + if (index_info.is_filter_always_false()) { + if (OB_FAIL(result_bitmap.set_bitmap_batch( + MAX(query_index_range_.start_row_id_, row_range.start_row_id_), + MIN(query_index_range_.end_row_id_, row_range.end_row_id_), + false))) { + LOG_WARN("Fail to set bitmap batch", K(ret), K(row_range)); + } + } else if (index_info.is_filter_always_true()) { + if (OB_FAIL(result_bitmap.set_bitmap_batch( + MAX(query_index_range_.start_row_id_, row_range.start_row_id_), + MIN(query_index_range_.end_row_id_, row_range.end_row_id_), + true))) { + LOG_WARN("Fail to set bitmap batch", K(ret), K(row_range)); + } + } else if (nullptr != parent && ObCGScanner::can_skip_filter( + *parent, *parent_bitmap, prefetcher_.current_micro_info().get_row_range())) { + continue; + } else { + break; } - } else if (nullptr != parent && ObCGScanner::can_skip_filter( - *parent, *parent_bitmap, prefetcher_.current_micro_info().get_row_range())) { - continue; - } else { - break; } } } @@ -379,38 +411,48 @@ int ObCGScanner::inner_filter( int ret = OB_SUCCESS; int64_t access_count = 0; const common::ObBitmap *bitmap = nullptr; + sql::ObBoolMask prefetch_constant_type = prefetcher_.get_filter_constant_type(); + ObCSRowId prefetch_constant_id = prefetcher_.get_max_filter_constant_id(); while (OB_SUCC(ret)) { if (end_of_scan()) { ret = OB_ITER_END; - } else if (is_new_range_ || OB_ITER_END == micro_scanner_->end_of_block()) { - if (OB_FAIL(get_next_valid_block(parent, parent_bitmap, result_bitmap))) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("Fail to get next valid index", K(ret)); - } - } else if (OB_FAIL(open_cur_data_block())) { - if (OB_UNLIKELY(OB_ITER_END != ret)) { - LOG_WARN("Fail to open cur data block", K(ret)); + } else if (prefetch_constant_type.is_constant() && + ((is_reverse_scan_ && prefetch_constant_id <= current_) || + (!is_reverse_scan_ && prefetch_constant_id >= current_))) { + current_ = is_reverse_scan_ ? prefetch_constant_id - 1 : prefetch_constant_id + 1; + result_bitmap.set_constant_filter_info(prefetch_constant_type, prefetch_constant_id); + LOG_DEBUG("Set constant filter info", K(ret), K(prefetch_constant_type), K(prefetch_constant_id)); + } else { + if (is_new_range_ || OB_ITER_END == micro_scanner_->end_of_block()) { + if (OB_FAIL(get_next_valid_block(parent, parent_bitmap, result_bitmap))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("Fail to get next valid index", K(ret)); + } + } else if (OB_FAIL(open_cur_data_block())) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("Fail to open cur data block", K(ret)); + } } } - } - if (OB_SUCC(ret)) { - ObMicroIndexInfo &index_info = prefetcher_.current_micro_info(); - const ObCSRange &row_range = index_info.get_row_range(); - uint32_t offset = row_range.start_row_id_ > query_index_range_.start_row_id_ ? (row_range.start_row_id_ - query_index_range_.start_row_id_) : 0; + if (OB_SUCC(ret)) { + ObMicroIndexInfo &index_info = prefetcher_.current_micro_info(); + const ObCSRange &row_range = index_info.get_row_range(); + uint32_t offset = row_range.start_row_id_ > query_index_range_.start_row_id_ ? (row_range.start_row_id_ - query_index_range_.start_row_id_) : 0; - index_info.pre_process_filter(*filter_info.filter_); - if (OB_FAIL(micro_scanner_->filter_micro_block_in_cg( - parent, filter_info, parent_bitmap, row_range.start_row_id_, access_count))) { - LOG_WARN("Fail to apply filter", K(ret)); - } else if (OB_ISNULL(bitmap = filter_info.filter_->get_result()) || bitmap->size() != access_count) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Unexpected null filter bitmap", K(ret), KPC(filter_info.filter_), K(access_count)); - } else if (OB_FAIL(result_bitmap.append_bitmap(*bitmap, offset, false))) { - LOG_WARN("Fail to append bitmap", K(ret), K(offset), KPC(bitmap), K(result_bitmap)); - } else { - current_ = is_reverse_scan_ ? current_ - access_count : current_ + access_count; + index_info.pre_process_filter(*filter_info.filter_); + if (OB_FAIL(micro_scanner_->filter_micro_block_in_cg( + parent, filter_info, parent_bitmap, row_range.start_row_id_, access_count))) { + LOG_WARN("Fail to apply filter", K(ret)); + } else if (OB_ISNULL(bitmap = filter_info.filter_->get_result()) || bitmap->size() != access_count) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected null filter bitmap", K(ret), KPC(filter_info.filter_), K(access_count)); + } else if (OB_FAIL(result_bitmap.append_bitmap(*bitmap, offset, false))) { + LOG_WARN("Fail to append bitmap", K(ret), K(offset), KPC(bitmap), K(result_bitmap)); + } else { + current_ = is_reverse_scan_ ? current_ - access_count : current_ + access_count; + } + index_info.post_process_filter(*filter_info.filter_); } - index_info.post_process_filter(*filter_info.filter_); } } return ret; diff --git a/src/storage/column_store/ob_co_sstable_row_scanner.cpp b/src/storage/column_store/ob_co_sstable_row_scanner.cpp index 911c67dbf..baaa21d79 100644 --- a/src/storage/column_store/ob_co_sstable_row_scanner.cpp +++ b/src/storage/column_store/ob_co_sstable_row_scanner.cpp @@ -639,6 +639,7 @@ int ObCOSSTableRowScanner::filter_rows_with_limit(BlockScanState &blockscan_stat LOG_WARN("Fail to inner filter", K(ret)); } } else if (nullptr != result_bitmap && result_bitmap->is_all_false()) { + // TODO: @dengzhi.ldz opt for cg scanner with limit update_current(group_size_); } else { int64_t begin_idx = begin; @@ -682,7 +683,15 @@ int ObCOSSTableRowScanner::filter_rows_without_limit(BlockScanState &blockscan_s ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected result bitmap", K(ret), KPC(rows_filter_)); } else if (nullptr != result_bitmap && result_bitmap->is_all_false()) { - update_current(group_size_); + if (result_bitmap->get_filter_constant_type().is_always_false()) { + if (reverse_scan_) { + current_ = MAX(result_bitmap->get_filter_constant_id(), end_) - 1; + } else { + current_ = MIN(result_bitmap->get_filter_constant_id(), end_) + 1; + } + } else { + update_current(group_size_); + } current_start_row_id = current_; } else { need_do_filter = false; @@ -717,7 +726,7 @@ int ObCOSSTableRowScanner::filter_rows_without_limit(BlockScanState &blockscan_s result_bitmap))) { LOG_WARN("Fail to locate", K(ret), K(current_), K(group_size_), KP(result_bitmap)); } - LOG_DEBUG("COScanner filter_rows_with_limit end", K(ret), K_(state), K_(blockscan_state), + LOG_DEBUG("COScanner filter_rows_without_limit end", K(ret), K_(state), K_(blockscan_state), K_(current), K_(group_size), K_(end)); return ret; } @@ -748,8 +757,7 @@ int ObCOSSTableRowScanner::inner_filter( ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected result bitmap", K(ret), KPC(rows_filter_)); } else { - int64_t select_cnt = result_bitmap->popcnt(); - EVENT_ADD(ObStatEventIds::PUSHDOWN_STORAGE_FILTER_ROW_CNT, select_cnt); + EVENT_ADD(ObStatEventIds::PUSHDOWN_STORAGE_FILTER_ROW_CNT, result_bitmap->popcnt()); } } else { EVENT_ADD(ObStatEventIds::PUSHDOWN_STORAGE_FILTER_ROW_CNT, group_size); @@ -759,7 +767,9 @@ int ObCOSSTableRowScanner::inner_filter( access_ctx_->table_store_stat_.logical_read_cnt_ += group_size; access_ctx_->table_store_stat_.physical_read_cnt_ += group_size; LOG_TRACE("[COLUMNSTORE] COSSTableRowScanner inner filter", K(ret), "begin", begin, "count", group_size, - "filtered", nullptr == rows_filter_ ? 0 : 1, "popcnt", nullptr == result_bitmap ? group_size : result_bitmap->popcnt()); + "filtered", nullptr == rows_filter_ ? 0 : 1, "popcnt", nullptr == result_bitmap ? group_size : result_bitmap->popcnt(), + "filter_constant_type", nullptr == result_bitmap ? sql::ObBoolMaskType::ALWAYS_TRUE : result_bitmap->get_filter_constant_type(), + "filter_constant_id", nullptr == result_bitmap ? (begin + group_size - 1) : result_bitmap->get_filter_constant_id()); } return ret; } @@ -787,18 +797,32 @@ int ObCOSSTableRowScanner::update_continuous_range( rows_filter_ == nullptr ? true : rows_filter_->can_continuous_filter(); if (group_is_true && filter_tree_can_continuous) { // current group is true, continue do filter if not reach end + sql::ObBoolMask filter_constant_type; + if (nullptr != result_bitmap) { + filter_constant_type = result_bitmap->get_filter_constant_type(); + } if (reverse_scan_) { - continuous_end_row_id = current_start_row_id; + // continuous_end_row_id = current_start_row_id; + continuous_end_row_id = filter_constant_type.is_always_true() ? MAX(result_bitmap->get_filter_constant_id(), end_) : current_start_row_id; current_start_row_id = continuous_end_row_id - 1; continue_filter = current_start_row_id >= end_; } else { - continuous_end_row_id = current_start_row_id + current_group_size - 1; + // continuous_end_row_id = current_start_row_id + current_group_size - 1; + continuous_end_row_id = filter_constant_type.is_always_true() ? MIN(result_bitmap->get_filter_constant_id(), end_) : current_start_row_id + current_group_size - 1; current_start_row_id = continuous_end_row_id + 1; continue_filter = current_start_row_id <= end_; } } else if ((nullptr != result_bitmap && result_bitmap->is_all_false()) && OB_INVALID_CS_ROW_ID == continuous_end_row_id) { // current group is false and no continuous true range before, skip this group and continue do filter - update_current(current_group_size); + if (result_bitmap->get_filter_constant_type().is_always_false()) { + if (reverse_scan_) { + current_ = MAX(result_bitmap->get_filter_constant_id(), end_) - 1; + } else { + current_ = MIN(result_bitmap->get_filter_constant_id(), end_) + 1; + } + } else { + update_current(current_group_size); + } current_start_row_id = current_; } else { continue_filter = false; @@ -809,6 +833,8 @@ int ObCOSSTableRowScanner::update_continuous_range( // no continuous true range before, will project current group } } + LOG_DEBUG("Filter continuous range info", K(ret), K(current_start_row_id), K(current_group_size), + K(continuous_end_row_id), K(continue_filter), K_(pending_end_row_id), K_(current), KPC(result_bitmap)); return ret; } diff --git a/src/storage/column_store/ob_co_sstable_rows_filter.cpp b/src/storage/column_store/ob_co_sstable_rows_filter.cpp index faa555bcb..18d46c419 100644 --- a/src/storage/column_store/ob_co_sstable_rows_filter.cpp +++ b/src/storage/column_store/ob_co_sstable_rows_filter.cpp @@ -221,7 +221,10 @@ int ObCOSSTableRowsFilter::apply(const ObCSRange &range) adjust_batch_size(); prepared_ = false; LOG_DEBUG("[COLUMNSTORE] apply filter info", K(range), - K(bitmap_buffer_[0]->size()), K(bitmap_buffer_[0]->popcnt())); + "filter_constant_type", bitmap_buffer_[0]->get_filter_constant_type(), + "filter_constant_id", bitmap_buffer_[0]->get_filter_constant_id(), + "bitmap_size", bitmap_buffer_[0]->size(), + "popcnt", bitmap_buffer_[0]->popcnt()); } return ret; } @@ -269,6 +272,7 @@ int ObCOSSTableRowsFilter::apply_filter( // Parent prepare_skip_filter can not be called here. } else if (filter->is_sample_node()) { ObSampleFilterExecutor *sample_executor = static_cast(filter); + result->set_filter_uncertain(); if (OB_FAIL(sample_executor->apply_sample_filter(range, *result->get_inner_bitmap()))) { LOG_WARN("Failed to apply sample filter", K(ret), K(range), KP(result), KPC(sample_executor)); } @@ -353,8 +357,9 @@ int ObCOSSTableRowsFilter::post_apply_filter( is_skip = true; } } - LOG_DEBUG("[COLUMNSTORE] post apply filter info", "is_and", filter.is_logic_and_node(), K(result.size()), - K(result.popcnt()), K(child_result.size()), K(child_result.popcnt())); + LOG_DEBUG("[COLUMNSTORE] post apply filter info", K(ret), K(is_skip), "is_and", filter.is_logic_and_node(), + K(result.size()), K(result.popcnt()), K(result.get_filter_constant_type()), K(result.get_filter_constant_id()), + K(child_result.size()), K(child_result.popcnt()), K(child_result.get_filter_constant_type()), K(child_result.get_filter_constant_id())); return ret; } @@ -660,7 +665,7 @@ int ObCOSSTableRowsFilter::init_bitmap_buffer(uint32_t bitmap_buffer_count) ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Failed to alloc memory for filter bitmap", K(ret)); } else if (FALSE_IT(filter_bitmap = new (buf) ObCGBitmap(*allocator_))) { - } else if (OB_FAIL(filter_bitmap->init(batch_size_))) { + } else if (OB_FAIL(filter_bitmap->init(batch_size_, access_ctx_->query_flag_.is_reverse_scan()))) { LOG_WARN("Failed to init bitmap", K(ret), K_(batch_size)); } else if (OB_FAIL(bitmap_buffer_.push_back(filter_bitmap))) { LOG_WARN("Failed to push_back filter bitmap", K(ret), KP(filter_bitmap)); @@ -690,7 +695,7 @@ int ObCOSSTableRowsFilter::prepare_bitmap_buffer( if (OB_ISNULL(result)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected nullptr filter_bitmap", K(ret), K(depth)); - } else if (OB_FAIL(result->reserve(range.get_row_count()))) { + } else if (OB_FAIL(result->switch_context(range.get_row_count(), access_ctx_->query_flag_.is_reverse_scan()))) { LOG_WARN("Failed to expand size for filter bitmap", K(ret), K(range)); } else if (0 == depth) { result->reuse(range.start_row_id_, filter.is_logic_and_node()); diff --git a/src/storage/column_store/ob_virtual_cg_scanner.cpp b/src/storage/column_store/ob_virtual_cg_scanner.cpp index 0d3c6ef92..7f73187ea 100644 --- a/src/storage/column_store/ob_virtual_cg_scanner.cpp +++ b/src/storage/column_store/ob_virtual_cg_scanner.cpp @@ -489,6 +489,7 @@ int ObDefaultCGScanner::apply_filter( } else { filter_result_ = result; filter_ = filter; + // TODO: set the max contant row id if (result) { result_bitmap.set_all_true(); } else { diff --git a/unittest/storage/column_store/test_co_sstable_rows_filter.cpp b/unittest/storage/column_store/test_co_sstable_rows_filter.cpp index be2b759c8..292493783 100644 --- a/unittest/storage/column_store/test_co_sstable_rows_filter.cpp +++ b/unittest/storage/column_store/test_co_sstable_rows_filter.cpp @@ -289,6 +289,7 @@ void TestCOSSTableRowsFilter::init_single_white_filter() ASSERT_FALSE(nullptr == filter_); co_filter_.filter_ = filter_; co_filter_.allocator_ = &allocator_; + co_filter_.access_ctx_ = &context_; } void TestCOSSTableRowsFilter::init_single_black_filter() @@ -300,6 +301,7 @@ void TestCOSSTableRowsFilter::init_single_black_filter() ASSERT_FALSE(nullptr == filter_); co_filter_.filter_ = filter_; co_filter_.allocator_ = &allocator_; + co_filter_.access_ctx_ = &context_; } void TestCOSSTableRowsFilter::init_multi_white_filter(bool is_common) @@ -322,6 +324,7 @@ void TestCOSSTableRowsFilter::init_multi_white_filter(bool is_common) filter_->set_childs(3, childs); co_filter_.filter_ = filter_; co_filter_.allocator_ = &allocator_; + co_filter_.access_ctx_ = &context_; } void TestCOSSTableRowsFilter::init_multi_black_filter(bool is_common) @@ -351,6 +354,7 @@ void TestCOSSTableRowsFilter::init_multi_black_filter(bool is_common) filter_->set_childs(3, childs); co_filter_.filter_ = filter_; co_filter_.allocator_ = &allocator_; + co_filter_.access_ctx_ = &context_; } void TestCOSSTableRowsFilter::init_multi_white_and_black_filter_case_one() @@ -383,6 +387,7 @@ void TestCOSSTableRowsFilter::init_multi_white_and_black_filter_case_one() filter_->set_childs(2, childs); co_filter_.filter_ = filter_; co_filter_.allocator_ = &allocator_; + co_filter_.access_ctx_ = &context_; } void TestCOSSTableRowsFilter::init_multi_white_and_black_filter_case_two() @@ -434,6 +439,7 @@ void TestCOSSTableRowsFilter::init_multi_white_and_black_filter_case_two() filter_->set_childs(3, childs); co_filter_.filter_ = filter_; co_filter_.allocator_ = &allocator_; + co_filter_.access_ctx_ = &context_; } void TestCOSSTableRowsFilter::reset_filter()