hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Opt skip index filter

Browse Source
This commit is contained in:
DengzhiLiu 2024-09-30 07:46:27 +00:00 committed by ob-robot
parent 13f9f3dea8
commit a14e4547ae
15 changed files with 520 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mittest/mtlenv/storage/blocksstable/test_cg_group_by_scanner.cpp
View File

@ -652,8 +652,8 @@ TEST_F(TestCGGroupByScanner, test_calc_aggregate_group_by_with_bitmap)
ASSERT_EQ(500, group_size);
ObCGBitmap bitmap(allocator_);
bitmap.init(group_size);
bitmap.reuse(0);
bitmap.init(group_size, false);
bitmap.reuse(0, false);
for (int64_t i = 0; i < group_size; i++) {
if (i % 5 == 0 || i % 5 == 2 || i % 5 == 4) {
bitmap.set(i);

6
mittest/mtlenv/storage/blocksstable/test_cg_scanner.cpp
View File

@ -418,7 +418,7 @@ TEST_F(TestCGScanner, test_large_micro_selected)
int64_t sql_batch_size = 256;
int64_t total_cnt = row_cnt_ / 10;
ObCGBitmap bitmap(allocator_);
bitmap.init(row_cnt_);
bitmap.init(row_cnt_, false);
bitmap.reuse(0);
for (int64_t i = 0; i < total_cnt; i++) {
bitmap.set(i * 10);
@ -592,8 +592,8 @@ TEST_F(TestCGScanner, test_filter)
int64_t locate_count = 25;
ObCGBitmap parent_bitmap(allocator_);
ObCGBitmap bitmap(allocator_);
bitmap.init(locate_count);
bitmap.reuse(start, false);
bitmap.init(locate_count, false);
bitmap.reuse(start);
ASSERT_EQ(OB_SUCCESS, cg_scanner->locate(ObCSRange(start, locate_count)));
ASSERT_EQ(OB_SUCCESS, cg_scanner->apply_filter(nullptr, pd_filter, locate_count, &parent_bitmap, bitmap));

4
mittest/mtlenv/storage/blocksstable/test_cs_cg_group_by_scanner.cpp
View File

@ -654,8 +654,8 @@ TEST_F(TestCSCGGroupByScanner, test_calc_aggregate_group_by_with_bitmap)
ASSERT_EQ(500, group_size);
ObCGBitmap bitmap(allocator_);
bitmap.init(group_size);
bitmap.reuse(0);
bitmap.init(group_size, false);
bitmap.reuse(0, false);
for (int64_t i = 0; i < group_size; i++) {
if (i % 5 == 0 || i % 5 == 2 || i % 5 == 4) {
bitmap.set(i);

6
mittest/mtlenv/storage/blocksstable/test_pushdown_aggregate.cpp
View File

@ -516,8 +516,8 @@ TEST_F(TestPushdownAggregate, test_decide_use_group_by2)
int64_t true_count = row_count / ObGroupByCell::USE_GROUP_BY_FILTER_FACTOR - 2;
ObCGBitmap bitmap(allocator_);
bitmap.init(row_count);
bitmap.reuse(0);
bitmap.init(row_count, false);
bitmap.reuse(0, false);
for (int64_t i = 0; i < row_count; i++) {
if (i < true_count) {
bitmap.set(i);
@ -529,7 +529,7 @@ TEST_F(TestPushdownAggregate, test_decide_use_group_by2)
ASSERT_TRUE(nullptr == group_by_cell.distinct_projector_buf_);
ASSERT_TRUE(nullptr == group_by_cell.tmp_group_by_datum_buf_);
bitmap.reuse(0);
bitmap.reuse(0, false);
true_count = row_count / ObGroupByCell::USE_GROUP_BY_FILTER_FACTOR + 2;
for (int64_t i = 0; i < row_count; i++) {
if (i < true_count) {

9
src/sql/engine/basic/ob_pushdown_filter.h
View File

@ -147,6 +147,11 @@ struct ObBoolMask
{
return ObBoolMaskType::PROBABILISTIC == bmt_;
}
OB_INLINE bool is_constant() const
{
return ObBoolMaskType::ALWAYS_TRUE == bmt_ ||
ObBoolMaskType::ALWAYS_FALSE == bmt_;
}
OB_INLINE void set_always_true()
{
bmt_ = ObBoolMaskType::ALWAYS_TRUE;
@ -159,6 +164,10 @@ struct ObBoolMask
{
bmt_ = ObBoolMaskType::PROBABILISTIC;
}
OB_INLINE void set(ObBoolMaskType bmt)
{
bmt_ = bmt;
}
TO_STRING_KV(K_(bmt));
ObBoolMaskType bmt_;

5
src/storage/blocksstable/index_block/ob_index_block_row_struct.h
View File

@ -615,6 +615,11 @@ public:
{
return sql::ObBoolMaskType::PROBABILISTIC == static_cast<sql::ObBoolMaskType>(filter_constant_type_);
}
OB_INLINE bool is_filter_constant() const
{
return sql::ObBoolMaskType::ALWAYS_TRUE == static_cast<sql::ObBoolMaskType>(filter_constant_type_) ||
sql::ObBoolMaskType::ALWAYS_FALSE == static_cast<sql::ObBoolMaskType>(filter_constant_type_);
}
OB_INLINE void set_filter_constant_type(const sql::ObBoolMaskType type)
{
filter_constant_type_ = static_cast<uint16_t>(type);

85
src/storage/column_store/ob_cg_bitmap.cpp
View File

@ -25,7 +25,7 @@ using namespace common;
namespace storage
{
int ObCGBitmap::get_first_valid_idx(const ObCSRange &range, const bool is_reverse_scan, ObCSRowId &row_idx) const
int ObCGBitmap::get_first_valid_idx(const ObCSRange &range, ObCSRowId &row_idx) const
{
int ret = OB_SUCCESS;
int64_t valid_offset = -1;
@ -36,7 +36,7 @@ int ObCGBitmap::get_first_valid_idx(const ObCSRange &range, const bool is_revers
} else if (is_all_false(range)) {
} else if (OB_FAIL(bitmap_.next_valid_idx(range.start_row_id_ - start_row_id_,
range.get_row_count(),
is_reverse_scan,
is_reverse_scan_,
valid_offset))){
LOG_WARN("Fail to get next valid idx", K(ret), K(range), KPC(this));
} else if (-1 != valid_offset) {
@ -418,5 +418,86 @@ int ObCGBitmap::get_row_ids(
return ret;
}
int ObCGBitmap::bit_and(const ObCGBitmap &right)
{
int ret = OB_SUCCESS;
const sql::ObBoolMask right_constant_type = right.get_filter_constant_type();
const ObCSRowId right_constant_id = right.get_filter_constant_id();
if (start_row_id_ != right.start_row_id_) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("Invalid argument", K(ret), K(start_row_id_), K(right.start_row_id_));
} else if (filter_constant_type_.is_always_true()) {
if (right_constant_type.is_always_true()) {
max_filter_constant_id_ = is_reverse_scan_ ? MAX(max_filter_constant_id_, right_constant_id) : MIN(max_filter_constant_id_, right_constant_id);
} else if (right_constant_type.is_always_false()) {
max_filter_constant_id_ = right_constant_id;
filter_constant_type_.set_always_false();
} else {
if (OB_FAIL(bitmap_.copy_from(right.bitmap_, 0, right.bitmap_.size()))) {
LOG_WARN("Fail to copy bitmap", K(ret), KPC(this), K(right));
} else {
max_filter_constant_id_ = OB_INVALID_CS_ROW_ID;
filter_constant_type_.set_uncertain();
}
}
} else if (filter_constant_type_.is_always_false()) {
if (right_constant_type.is_always_false()) {
max_filter_constant_id_ = is_reverse_scan_ ? MIN(max_filter_constant_id_, right_constant_id) : MAX(max_filter_constant_id_, right_constant_id);
} // else always_true/uncertain
} else {
// uncertain
if (right_constant_type.is_always_true()) {
} else if (right_constant_type.is_always_false()) {
max_filter_constant_id_ = right_constant_id;
filter_constant_type_.set_always_false();
} else {
if (OB_FAIL(bitmap_.bit_and(right.bitmap_))) {
LOG_WARN("Fail to bit and", K(ret), KPC(this), K(right));
}
}
}
return ret;
}
int ObCGBitmap::bit_or(const ObCGBitmap &right)
{
int ret = OB_SUCCESS;
const sql::ObBoolMask right_constant_type = right.get_filter_constant_type();
const ObCSRowId right_constant_id = right.get_filter_constant_id();
if (start_row_id_ != right.start_row_id_) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("Invalid argument", K(ret), K(start_row_id_), K(right.start_row_id_));
} else if (filter_constant_type_.is_always_true()) {
if (right_constant_type.is_always_true()) {
max_filter_constant_id_ = is_reverse_scan_ ? MIN(max_filter_constant_id_, right_constant_id) : MAX(max_filter_constant_id_, right_constant_id);
} // else always_false/uncertain
} else if (filter_constant_type_.is_always_false()) {
if (right_constant_type.is_always_true()) {
max_filter_constant_id_ = right_constant_id;
filter_constant_type_.set_always_true();
} else if (right_constant_type.is_always_false()) {
max_filter_constant_id_ = is_reverse_scan_ ? MAX(max_filter_constant_id_, right_constant_id) : MIN(max_filter_constant_id_, right_constant_id);
} else {
if (OB_FAIL(bitmap_.copy_from(right.bitmap_, 0, right.bitmap_.size()))) {
LOG_WARN("Fail to copy bitmap", K(ret), KPC(this), K(right));
} else {
max_filter_constant_id_ = OB_INVALID_CS_ROW_ID;
filter_constant_type_.set_uncertain();
}
}
} else {
if (right_constant_type.is_always_true()) {
max_filter_constant_id_ = right_constant_id;
filter_constant_type_.set_always_true();
} else if (right_constant_type.is_always_false()) {
} else {
if (OB_FAIL(bitmap_.bit_or(right.bitmap_))) {
LOG_WARN("Fail to bit_or bitmap", K(ret), KPC(this), K(right));
}
}
}
return ret;
}
}
}

165
src/storage/column_store/ob_cg_bitmap.h
View File

@ -25,14 +25,17 @@ class ObCGBitmap
{
public:
ObCGBitmap(ObIAllocator &allocator) :
is_reverse_scan_(false),
bitmap_(allocator),
start_row_id_(OB_INVALID_CS_ROW_ID)
start_row_id_(OB_INVALID_CS_ROW_ID),
max_filter_constant_id_(OB_INVALID_CS_ROW_ID),
filter_constant_type_()
{}
virtual ~ObCGBitmap()
{}
bool is_valid() const
{
return start_row_id_ != OB_INVALID_CS_ROW_ID;
return filter_constant_type_.is_constant() || start_row_id_ != OB_INVALID_CS_ROW_ID;
}
int copy_from(const ObCGBitmap &bitmap)
{
@ -42,26 +45,43 @@ public:
STORAGE_LOG(WARN, "Invalid argument", K(ret), K(bitmap));
} else if (OB_FAIL(bitmap_.reserve(bitmap.bitmap_.size()))) {
STORAGE_LOG(WARN, "Fail to expand bitmap", K(ret), K(bitmap.bitmap_.size()));
} else if (OB_FAIL(bitmap_.copy_from(bitmap.bitmap_, 0, bitmap.bitmap_.size()))) {
STORAGE_LOG(WARN, "Fail to set bitmap", K(ret), K(bitmap));
} else if (bitmap.filter_constant_type_.is_constant()) {
max_filter_constant_id_ = bitmap.max_filter_constant_id_;
filter_constant_type_ = bitmap.filter_constant_type_;
} else {
if (OB_FAIL(bitmap_.copy_from(bitmap.bitmap_, 0, bitmap.bitmap_.size()))) {
STORAGE_LOG(WARN, "Fail to set bitmap", K(ret), K(bitmap));
} else {
max_filter_constant_id_ = OB_INVALID_CS_ROW_ID;
filter_constant_type_.set_uncertain();
}
}
if (OB_SUCC(ret)) {
is_reverse_scan_ = bitmap.is_reverse_scan_;
start_row_id_ = bitmap.start_row_id_;
}
return ret;
}
int init(
const uint64_t count,
const ObCSRowId start_row_id = OB_INVALID_CS_ROW_ID,
int init(const uint64_t count, const bool is_reverse)
{
is_reverse_scan_ = is_reverse;
return bitmap_.init(count);
}
void reuse(
const ObCSRowId start_row_id,
const bool is_all_true = false)
{
start_row_id_ = start_row_id;
return bitmap_.init(count, is_all_true);
}
void reuse(const ObCSRowId start_row_id, const bool is_all_true = false)
{
start_row_id_ = start_row_id;
if (is_all_true) {
filter_constant_type_.set_always_true();
max_filter_constant_id_ = is_reverse_scan_ ? start_row_id : start_row_id + bitmap_.size() - 1;
} else {
filter_constant_type_.set_always_false();
max_filter_constant_id_ = is_reverse_scan_ ? start_row_id : start_row_id + bitmap_.size() - 1;
}
bitmap_.reuse(is_all_true);
}
@ -75,33 +95,48 @@ public:
return start_row_id_;
}
OB_INLINE int reserve(uint64_t count)
OB_INLINE int switch_context(const uint64_t count, const bool is_reverse)
{
is_reverse_scan_ = is_reverse;
return bitmap_.reserve(count);
}
OB_INLINE int set(const ObCSRowId row_idx, const bool value = true)
{
OB_ASSERT(row_idx >= start_row_id_);
filter_constant_type_.set_uncertain();
return bitmap_.set(row_idx - start_row_id_, value);
}
OB_INLINE int wipe(const ObCSRowId row_idx)
{
OB_ASSERT(row_idx >= start_row_id_);
filter_constant_type_.set_uncertain();
return bitmap_.wipe(row_idx - start_row_id_);
}
OB_INLINE bool test(const ObCSRowId row_idx) const
{
OB_ASSERT(row_idx >= start_row_id_);
return bitmap_.test(row_idx - start_row_id_);
bool res;
if (filter_constant_type_.is_constant()) {
res = filter_constant_type_.is_always_true();
} else {
res = bitmap_.test(row_idx - start_row_id_);
}
return res;
}
OB_INLINE bool operator[](const ObCSRowId row_idx) const
{
OB_ASSERT(row_idx >= start_row_id_);
return bitmap_.test(row_idx - start_row_id_);
bool res;
if (filter_constant_type_.is_constant()) {
res = filter_constant_type_.is_always_true();
} else {
res = bitmap_.test(row_idx - start_row_id_);
}
return res;
}
OB_INLINE int64_t capacity() const
@ -116,86 +151,110 @@ public:
OB_INLINE uint64_t popcnt() const
{
return bitmap_.popcnt();
uint64_t cnt = 0;
if (filter_constant_type_.is_constant()) {
cnt = filter_constant_type_.is_always_true() ? bitmap_.size() : 0;
} else {
cnt = bitmap_.popcnt();
}
return cnt;
}
OB_INLINE bool is_all_true() const
{
return bitmap_.is_all_true();
return filter_constant_type_.is_always_true() ||
(filter_constant_type_.is_uncertain() && bitmap_.is_all_true());
}
OB_INLINE bool is_all_false() const
{
return bitmap_.is_all_false();
return filter_constant_type_.is_always_false() ||
(filter_constant_type_.is_uncertain() && bitmap_.is_all_false());
}
OB_INLINE int bit_and(const ObCGBitmap &right)
{
int ret = OB_SUCCESS;
if (start_row_id_ != right.start_row_id_) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument", K(ret), K(start_row_id_), K(right.start_row_id_));
} else if (OB_FAIL(bitmap_.bit_and(right.bitmap_))) {
STORAGE_LOG(WARN, "Fail to bit and", K(ret), KPC(this), K(right));
}
return ret;
}
OB_INLINE int bit_and(const ObCGBitmap &right);
OB_INLINE int bit_or(const ObCGBitmap &right)
{
int ret = OB_SUCCESS;
if (start_row_id_ != right.start_row_id_) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument", K(ret), K(start_row_id_), K(right.start_row_id_));
} else if (OB_FAIL(bitmap_.bit_or(right.bitmap_))) {
STORAGE_LOG(WARN, "Fail to bit or", K(ret), KPC(this), K(right));
}
return ret;
}
OB_INLINE int bit_or(const ObCGBitmap &right);
OB_INLINE int bit_not()
{
return bitmap_.bit_not();
int ret = OB_SUCCESS;
if (filter_constant_type_.is_always_true()) {
filter_constant_type_.set_always_false();
} else if (filter_constant_type_.is_always_false()) {
filter_constant_type_.set_always_true();
} else {
ret = bitmap_.bit_not();
}
return ret;
}
int append_bitmap(const ObBitmap &bitmap, const uint32_t offset, const bool is_reverse)
{
filter_constant_type_.set_uncertain();
return bitmap_.append_bitmap(bitmap, offset, is_reverse);
}
OB_INLINE bool is_all_true(const ObCSRange &range) const
{
OB_ASSERT(range.is_valid() && range.end_row_id_ >= start_row_id_);
return bitmap_.is_all_true(MAX(range.start_row_id_ - start_row_id_, 0),
MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1));
return filter_constant_type_.is_always_true() ||
bitmap_.is_all_true(MAX(range.start_row_id_ - start_row_id_, 0),
MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1));
}
OB_INLINE bool is_all_false(const ObCSRange &range) const
{
OB_ASSERT(range.is_valid() && range.end_row_id_ >= start_row_id_);
return bitmap_.is_all_false(MAX(range.start_row_id_ - start_row_id_, 0),
MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1));
return filter_constant_type_.is_always_false() ||
bitmap_.is_all_false(MAX(range.start_row_id_ - start_row_id_, 0),
MIN(range.end_row_id_ - start_row_id_, bitmap_.size() - 1));
}
OB_INLINE void set_all_true()
{
bitmap_.reuse(true);
filter_constant_type_.set_always_true();
max_filter_constant_id_ = is_reverse_scan_ ? start_row_id_ : start_row_id_ + bitmap_.size() - 1;
}
OB_INLINE void set_all_false()
{
bitmap_.reuse(false);
filter_constant_type_.set_always_false();
max_filter_constant_id_ = is_reverse_scan_ ? start_row_id_ : start_row_id_ + bitmap_.size() - 1;
}
OB_INLINE int set_bitmap_batch(ObCSRowId start, ObCSRowId end, const bool value)
{
int64_t offset = MAX(start - start_row_id_, 0);
int64_t count = MIN(end - start + 1, bitmap_.size() - offset);
filter_constant_type_.set_uncertain();
return bitmap_.set_bitmap_batch(offset, count, value);
}
OB_INLINE void set_constant_filter_info(const sql::ObBoolMask filter_constant_type, const ObCSRowId filter_constant_id)
{
filter_constant_type_ = filter_constant_type;
max_filter_constant_id_ = filter_constant_id;
}
OB_INLINE sql::ObBoolMask get_filter_constant_type() const
{
return filter_constant_type_;
}
OB_INLINE ObCSRowId get_filter_constant_id() const
{
return max_filter_constant_id_;
}
OB_INLINE void set_filter_uncertain()
{
return filter_constant_type_.set_uncertain();
}
int set_bitmap(const ObCSRowId start, const int64_t row_count, const bool is_reverse, ObBitmap &bitmap) const;
int get_first_valid_idx(const ObCSRange &range, const bool is_reverse_scan, ObCSRowId &row_idx) const;
int get_first_valid_idx(const ObCSRange &range, ObCSRowId &row_idx) const;
int get_row_ids(int32_t *row_ids,
int64_t &row_cap,
ObCSRowId &current,
@ -203,11 +262,15 @@ public:
const ObCSRange &data_range,
const int64_t batch_size,
const bool is_reverse);
TO_STRING_KV(K_(bitmap), K_(start_row_id));
TO_STRING_KV(K_(is_reverse_scan), K_(start_row_id), K_(max_filter_constant_id),
K_(filter_constant_type), K_(bitmap));
private:
bool is_reverse_scan_;
common::ObBitmap bitmap_;
ObCSRowId start_row_id_;
// max_filter_constant_id_ may greater/less than offset+size
ObCSRowId max_filter_constant_id_;
sql::ObBoolMask filter_constant_type_;
};
int convert_bitmap_to_cs_index(int32_t *row_ids,

148
src/storage/column_store/ob_cg_prefetcher.cpp
View File

@ -31,6 +31,8 @@ void ObCGPrefetcher::reset()
micro_data_prewarm_idx_ = 0;
cur_micro_data_read_idx_ = -1;
agg_group_ = nullptr;
filter_constant_type_.set_uncertain();
max_filter_constant_id_ = OB_INVALID_CS_ROW_ID;
ObIndexTreeMultiPassPrefetcher::reset();
}
@ -44,6 +46,8 @@ void ObCGPrefetcher::reuse()
filter_bitmap_ = nullptr;
micro_data_prewarm_idx_ = 0;
cur_micro_data_read_idx_ = -1;
filter_constant_type_.set_uncertain();
max_filter_constant_id_ = OB_INVALID_CS_ROW_ID;
ObIndexTreeMultiPassPrefetcher::reuse();
}
@ -209,10 +213,19 @@ int ObCGPrefetcher::locate_in_prefetched_data(bool &found)
{
int ret = OB_SUCCESS;
found = false;
const ObCSRowId start_row_idx = is_reverse_scan_ ? query_index_range_.end_row_id_ : query_index_range_.start_row_id_;
int64_t max_data_prefetched_idx = MAX(micro_data_prewarm_idx_, micro_data_prefetch_idx_);
if (max_data_prefetched_idx > 0 && max_data_prefetched_idx > cur_micro_data_read_idx_) {
if (filter_constant_type_.is_constant() &&
((is_reverse_scan_ && max_filter_constant_id_ <= query_index_range_.end_row_id_) ||
(!is_reverse_scan_ && max_filter_constant_id_ >= query_index_range_.start_row_id_))) {
found = true;
is_prefetch_end_ =
(is_reverse_scan_ && max_filter_constant_id_ <= query_index_range_.start_row_id_) ||
(!is_reverse_scan_ && max_filter_constant_id_ >= query_index_range_.end_row_id_);
} else {
filter_constant_type_.set_uncertain();
max_filter_constant_id_ = is_reverse_scan_ ? query_index_range_.end_row_id_ + 1 : query_index_range_.start_row_id_ - 1;
int cmp_ret = -1;
int64_t max_data_prefetched_idx = MAX(micro_data_prewarm_idx_, micro_data_prefetch_idx_);
const ObCSRowId start_row_idx = is_reverse_scan_ ? query_index_range_.end_row_id_ : query_index_range_.start_row_id_;
for (int64_t micro_data_idx = MAX(0, cur_micro_data_read_idx_); OB_SUCC(ret) && cmp_ret < 0 && micro_data_idx < max_data_prefetched_idx; micro_data_idx++) {
ObMicroIndexInfo &micro_info = micro_data_infos_[micro_data_idx % max_micro_handle_cnt_];
const ObCSRange &micro_range = micro_info.get_row_range();
@ -231,10 +244,12 @@ int ObCGPrefetcher::locate_in_prefetched_data(bool &found)
found = true;
}
}
}
if (OB_SUCC(ret) && found) {
cmp_ret = 1;
is_prefetch_end_ = false;
if (OB_SUCC(ret) && found) {
if (!is_prefetch_end_) {
int cmp_ret = 1;
int64_t max_data_prefetched_idx = MAX(micro_data_prewarm_idx_, micro_data_prefetch_idx_);
micro_data_prefetch_idx_ = max_data_prefetched_idx;
const ObCSRowId end_row_id = is_reverse_scan_ ? query_index_range_.start_row_id_ : query_index_range_.end_row_id_;
for (int64_t micro_data_idx = max_data_prefetched_idx - 1; OB_SUCC(ret) && cmp_ret > 0 && micro_data_idx > cur_micro_data_fetch_idx_; micro_data_idx--) {
@ -296,11 +311,19 @@ bool ObCGPrefetcher::locate_back(const ObCSRange &locate_range)
{
bool is_locate_back = false;
if (0 < query_index_range_.end_row_id_) {
int micro_index = MAX(0, cur_micro_data_read_idx_);
if (is_reverse_scan_) {
is_locate_back = locate_range.end_row_id_ > micro_data_infos_[micro_index % max_micro_handle_cnt_].get_row_range().end_row_id_;
if (filter_constant_type_.is_constant()) {
if (is_reverse_scan_) {
is_locate_back = locate_range.end_row_id_ > query_index_range_.end_row_id_;
} else {
is_locate_back = locate_range.start_row_id_ < query_index_range_.start_row_id_;
}
} else {
is_locate_back = locate_range.start_row_id_ < micro_data_infos_[micro_index % max_micro_handle_cnt_].get_row_range().start_row_id_;
int micro_index = MAX(0, cur_micro_data_read_idx_);
if (is_reverse_scan_) {
is_locate_back = locate_range.end_row_id_ > micro_data_infos_[micro_index % max_micro_handle_cnt_].get_row_range().end_row_id_;
} else {
is_locate_back = locate_range.start_row_id_ < micro_data_infos_[micro_index % max_micro_handle_cnt_].get_row_range().start_row_id_;
}
}
}
return is_locate_back;
@ -323,7 +346,10 @@ int ObCGPrefetcher::locate(const ObCSRange &range, const ObCGBitmap *bitmap)
ObIndexTreeMultiPassPrefetcher::reuse();
micro_data_prewarm_idx_ = 0;
cur_micro_data_read_idx_ = -1;
filter_constant_type_.set_uncertain();
max_filter_constant_id_ = is_reverse_scan_ ? query_index_range_.end_row_id_ + 1 : query_index_range_.start_row_id_ - 1;
update_query_range(range);
cur_level_ = 0;
read_handles_[0].row_state_ = ObSSTableRowState::IN_BLOCK;
if (OB_FAIL(open_index_root())) {
LOG_WARN("Fail to open index root", K(ret));
@ -345,6 +371,48 @@ int ObCGPrefetcher::locate(const ObCSRange &range, const ObCGBitmap *bitmap)
LOG_WARN("Fail to refresh index tree", K(ret), K_(query_index_range));
}
}
if (OB_SUCC(ret) && OB_FAIL(refresh_constant_filter_info())) {
LOG_WARN("Fail to refresh constant filter info", K(ret));
}
return ret;
}
// update filter_constant_type_ and max_filter_constant_id_
int ObCGPrefetcher::refresh_constant_filter_info()
{
int ret = OB_SUCCESS;
if (!is_cg_scanner() || nullptr == sstable_index_filter_ || filter_constant_type_.is_constant()) {
// skip check when filter_constant_type_ is constant
} else {
// update max row id for constant filtered result
bool need_check = true;
if (need_check && cur_level_ == index_tree_height_ - 1) {
int64_t max_data_prefetched_idx = MAX(micro_data_prewarm_idx_, micro_data_prefetch_idx_);
for (int64_t micro_data_idx = cur_micro_data_fetch_idx_ + 1; need_check && micro_data_idx < max_data_prefetched_idx; micro_data_idx++) {
const ObMicroIndexInfo &index_info = micro_data_infos_[micro_data_idx % max_micro_handle_cnt_];
if (!check_and_update_constant_filter(index_info)) {
need_check = false;
}
}
}
for (int64_t level = 0; need_check && level <= cur_level_; level++) {
ObIndexTreeLevelHandle &tree_handle = tree_handles_[level];
for (int64_t idx = tree_handle.read_idx_; need_check && idx <= tree_handle.prefetch_idx_; idx++) {
const ObMicroIndexInfo &index_info = tree_handle.index_block_read_handles_[idx % INDEX_TREE_PREFETCH_DEPTH].index_info_;
if (is_constant_filter_continuous(index_info)) {
if (filter_constant_type_.is_uncertain() ||
index_info.get_filter_constant_type() == filter_constant_type_.bmt_) {
filter_constant_type_.set(index_info.get_filter_constant_type());
max_filter_constant_id_ = is_reverse_scan_ ? index_info.get_row_range().start_row_id_ : index_info.get_row_range().end_row_id_;
} else {
need_check = false;
}
}
}
}
}
return ret;
}
@ -396,7 +464,9 @@ int ObCGPrefetcher::prefetch_micro_data()
break;
} else {
// read index leaf and prefetch micro data
while (OB_SUCC(ret) && !is_prefetch_end_ && prefetched_cnt < prefetch_depth) {
bool hit_constant_filter = false;
while (OB_SUCC(ret) && (!is_prefetch_end_ || hit_constant_filter) && prefetched_cnt < prefetch_depth) {
hit_constant_filter = false;
prefetch_micro_idx = micro_data_prefetch_idx_ % max_micro_handle_cnt_;
ObMicroIndexInfo &block_info = micro_data_infos_[prefetch_micro_idx];
bool can_agg = false;
@ -415,9 +485,6 @@ int ObCGPrefetcher::prefetch_micro_data()
ret = OB_SUCCESS;
break;
}
} else if (compare_range(block_info.get_row_range()) > 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected prefetch index info", K(ret), K(block_info), KPC(this));
} else if (!contain_rows(block_info.get_row_range())) {
// continue if no rows contained
} else if (nullptr != sstable_index_filter_
@ -426,12 +493,33 @@ int ObCGPrefetcher::prefetch_micro_data()
iter_param_->read_info_, block_info,
*(access_ctx_->allocator_), iter_param_->vectorized_enabled_))) {
LOG_WARN("Fail to check if can skip prefetch", K(ret), K(block_info));
} else if (nullptr != sstable_index_filter_
&& (block_info.is_filter_always_false() || block_info.is_filter_always_true())) {
// TODO: skip data block which is always_false/always_true and record the result in filter bitmap
prefetched_cnt++;
micro_data_prefetch_idx_++;
tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++;
// exist only in filter iter
} else if (nullptr != sstable_index_filter_ && block_info.is_filter_constant()) {
if (check_and_update_constant_filter(block_info)) {
hit_constant_filter = true;
} else {
prefetched_cnt++;
tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++;
if (0 < compare_range(block_info.get_row_range())) {
micro_data_prewarm_idx_ = micro_data_prefetch_idx_ + 1;
} else {
micro_data_prefetch_idx_++;
}
}
} else if (0 < compare_range(block_info.get_row_range())) {
if (!is_cg_scanner() || nullptr == sstable_index_filter_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected prefetch range", K(ret), K_(cg_iter_type), K_(query_index_range), K(block_info.get_row_range()));
} else if (OB_FAIL(prefetch_data_block(
micro_data_prefetch_idx_,
block_info,
micro_data_handles_[prefetch_micro_idx]))) {
LOG_WARN("fail to prefetch_block_data", K(ret), K(block_info));
} else {
prefetched_cnt++;
micro_data_prewarm_idx_ = micro_data_prefetch_idx_ + 1;
tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++;
}
} else if (OB_FAIL(can_agg_micro_index(block_info, can_agg))) {
LOG_WARN("fail to check can agg index info", K(ret), K(block_info), KPC_(agg_group));
} else if (can_agg) {
@ -451,7 +539,7 @@ int ObCGPrefetcher::prefetch_micro_data()
tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++;
}
if (OB_SUCC(ret) && (0 == compare_range(block_info.get_row_range()))) {
if (OB_SUCC(ret) && (0 <= compare_range(block_info.get_row_range()))) {
is_prefetch_end_ = true;
}
}
@ -561,7 +649,7 @@ int ObCGPrefetcher::ObCSIndexTreeLevelHandle::prefetch(
*(prefetcher.access_ctx_->allocator_),
prefetcher.iter_param_->vectorized_enabled_))) {
LOG_WARN("Fail to check if can skip prefetch", K(ret), K(index_info));
// TODO: skip data block which is always_false/always_true and record the result in filter bitmap
} else if (prefetcher.check_and_update_constant_filter(index_info)) {
} else if (OB_FAIL(prefetcher.can_agg_micro_index(index_info, can_agg))) {
LOG_WARN("fail to check index info", K(ret), K(index_info), KPC(prefetcher.agg_group_));
} else if (can_agg) {
@ -584,7 +672,7 @@ int ObCGPrefetcher::ObCSIndexTreeLevelHandle::prefetch(
}
}
}
if (OB_SUCC(ret) && !is_prefetch_end_ && 0 == prefetcher.compare_range(index_info.get_row_range())) {
if (OB_SUCC(ret) && !is_prefetch_end_ && 0 <= prefetcher.compare_range(index_info.get_row_range())) {
is_prefetch_end_ = true;
}
}
@ -764,12 +852,14 @@ int ObCGPrefetcher::prewarm()
iter_param_->read_info_, block_info,
*(access_ctx_->allocator_), iter_param_->vectorized_enabled_))) {
LOG_WARN("Fail to check if can skip prefetch", K(ret), K(block_info));
} else if (nullptr != sstable_index_filter_
&& (block_info.is_filter_always_false() || block_info.is_filter_always_true())) {
// TODO: skip data block which is always_false/always_true and record the result in filter bitmap
prefetched_cnt++;
micro_data_prewarm_idx_++;
tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++;
} else if (nullptr != sstable_index_filter_ && block_info.is_filter_constant()) {
if (check_and_update_constant_filter(block_info)) {
prefetched_cnt++;
} else {
prefetched_cnt++;
micro_data_prewarm_idx_++;
tree_handles_[cur_level_].current_block_read_handle().end_prefetched_row_idx_++;
}
} else if (OB_FAIL(prefetch_data_block(
micro_data_prewarm_idx_,
block_info,

39
src/storage/column_store/ob_cg_prefetcher.h
View File

@ -31,6 +31,8 @@ public:
leaf_query_range_(),
filter_bitmap_(nullptr),
micro_data_prewarm_idx_(0),
filter_constant_type_(),
max_filter_constant_id_(OB_INVALID_CS_ROW_ID),
cur_micro_data_read_idx_(-1),
agg_group_(nullptr),
sstable_index_filter_(nullptr)
@ -73,18 +75,47 @@ public:
{
return index_info.has_agg_data() && index_info.is_filter_uncertain();
}
OB_INLINE bool is_cg_scanner() const
{ return ObICGIterator::OB_CG_SCANNER == cg_iter_type_; }
void recycle_block_data();
void set_agg_group(ObAggGroupBase *agg_group) { agg_group_ = agg_group; }
void set_project_type(const bool project_without_filter) { is_project_without_filter_ = project_without_filter; }
OB_INLINE sql::ObBoolMask get_filter_constant_type() const
{
return filter_constant_type_;
}
OB_INLINE ObCSRowId get_max_filter_constant_id() const
{
return max_filter_constant_id_;
}
INHERIT_TO_STRING_KV("ObCGPrefetcher", ObIndexTreeMultiPassPrefetcher,
K_(is_reverse_scan), K_(is_project_without_filter), K_(need_prewarm),
K_(query_index_range), K_(query_range), K_(cg_iter_type),
K_(micro_data_prewarm_idx), K_(cur_micro_data_read_idx), KP_(filter_bitmap),
KP_(agg_group), KP_(sstable_index_filter));
KP_(agg_group), KP_(sstable_index_filter), K_(filter_constant_type), K_(max_filter_constant_id));
protected:
int get_prefetch_depth(int64_t &depth, const int64_t prefetching_idx);
private:
int prewarm();
int refresh_constant_filter_info();
bool is_constant_filter_continuous(const ObMicroIndexInfo &index_info)
{
return index_info.is_filter_constant() &&
((is_reverse_scan_ && 0 == index_info.get_row_range().compare(max_filter_constant_id_ - 1)) ||
(!is_reverse_scan_ && 0 == index_info.get_row_range().compare(max_filter_constant_id_ + 1)));
}
bool check_and_update_constant_filter(const ObMicroIndexInfo &index_info)
{
bool can_continuous = false;
if (is_constant_filter_continuous(index_info) &&
(filter_constant_type_.is_uncertain() ||
index_info.get_filter_constant_type() == filter_constant_type_.bmt_)) {
filter_constant_type_.set(index_info.get_filter_constant_type());
max_filter_constant_id_ = is_reverse_scan_ ? index_info.get_row_range().start_row_id_ : index_info.get_row_range().end_row_id_;
can_continuous = true;
}
return can_continuous;
}
struct ObCSIndexTreeLevelHandle : public ObIndexTreeLevelHandle {
public:
int prefetch(const int64_t level, ObCGPrefetcher &prefetcher);
@ -120,6 +151,12 @@ private:
ObDatumRange leaf_query_range_;
const ObCGBitmap *filter_bitmap_;
int64_t micro_data_prewarm_idx_;
// filter_constant_type_ and max_filter_constant_id_ only avaliable in filter scanner
// when filter_constant_type_ is always_true, max_filter_constant_id_ means the max continuous rowid that selected by the filter;
// when filter_constant_type_ is always_false, max_filter_constant_id_ means the max continuose rowid that filtered;
// when filter_constant_type_ is probablistic, means nothing
sql::ObBoolMask filter_constant_type_;
ObCSRowId max_filter_constant_id_;
public:
int64_t cur_micro_data_read_idx_;
ObAggGroupBase *agg_group_;

140
src/storage/column_store/ob_cg_scanner.cpp
View File

@ -193,7 +193,7 @@ int ObCGScanner::locate(
if (nullptr != bitmap) {
if (bitmap->is_all_true()) {
locate_bitmap = nullptr;
} else if (OB_FAIL(bitmap->get_first_valid_idx(range, is_reverse_scan_, current_))) {
} else if (OB_FAIL(bitmap->get_first_valid_idx(range, current_))) {
LOG_WARN("Fail to get first valid idx", K(ret), K_(is_reverse_scan), K(range));
} else {
if (is_reverse_scan_) {
@ -315,6 +315,8 @@ int ObCGScanner::apply_filter(
ret = OB_SUCCESS;
}
LOG_TRACE("[COLUMNSTORE] apply filter info in cg", K(ret), K_(query_index_range), K(row_count),
"filter_constant_type", result_bitmap.get_filter_constant_type(),
"filter_constant_id", result_bitmap.get_filter_constant_id(),
"bitmap_size", result_bitmap.size(), "popcnt", result_bitmap.popcnt());
return ret;
}
@ -330,32 +332,62 @@ int ObCGScanner::get_next_valid_block(sql::ObPushdownFilterExecutor *parent,
LOG_DEBUG("Calc to end of prefetched data", K(ret), K(prefetcher_.cur_micro_data_fetch_idx_),
K(prefetcher_.micro_data_prefetch_idx_));
} else {
if (prefetcher_.cur_micro_data_fetch_idx_ > -1) {
prefetcher_.current_micro_handle().reset();
if (result_bitmap.get_filter_constant_type().is_constant()) {
result_bitmap.set_filter_uncertain();
ObCSRowId prefetch_constant_id = prefetcher_.get_max_filter_constant_id();
sql::ObBoolMask prefetch_constant_type = prefetcher_.get_filter_constant_type();
// reset bitmap when bits not consistent with filter constant type after set by prefetcher
// filter info
// bitmap set value could be decided by the logic op type of filter root in the iter param
bool need_reset = is_reverse_scan_ ?
prefetch_constant_id <= query_index_range_.end_row_id_ :
prefetch_constant_id >= query_index_range_.start_row_id_;
if (!need_reset) {
} else if (is_reverse_scan_) {
if (OB_FAIL(result_bitmap.set_bitmap_batch(
MAX(query_index_range_.start_row_id_, prefetch_constant_id),
query_index_range_.end_row_id_,
prefetch_constant_type.is_always_true()))) {
LOG_WARN("Fail to set bitmap batch", K(ret), K_(query_index_range), K(prefetch_constant_id));
}
} else {
if (OB_FAIL(result_bitmap.set_bitmap_batch(
query_index_range_.start_row_id_,
MIN(query_index_range_.end_row_id_, prefetch_constant_id),
prefetch_constant_type.is_always_true()))) {
LOG_WARN("Fail to set bitmap batch", K(ret), K_(query_index_range), K(prefetch_constant_id));
}
}
}
++prefetcher_.cur_micro_data_fetch_idx_;
++prefetcher_.cur_micro_data_read_idx_;
const ObMicroIndexInfo &index_info = prefetcher_.current_micro_info();
const ObCSRange &row_range = index_info.get_row_range();
if (index_info.is_filter_always_false()) {
if (OB_FAIL(result_bitmap.set_bitmap_batch(
MAX(query_index_range_.start_row_id_, row_range.start_row_id_),
MIN(query_index_range_.end_row_id_, row_range.end_row_id_),
false))) {
LOG_WARN("Fail to set bitmap batch", K(ret), K(row_range));
if (OB_SUCC(ret)) {
if (prefetcher_.cur_micro_data_fetch_idx_ > -1) {
prefetcher_.current_micro_handle().reset();
}
} else if (index_info.is_filter_always_true()) {
if (OB_FAIL(result_bitmap.set_bitmap_batch(
MAX(query_index_range_.start_row_id_, row_range.start_row_id_),
MIN(query_index_range_.end_row_id_, row_range.end_row_id_),
true))) {
LOG_WARN("Fail to set bitmap batch", K(ret), K(row_range));
++prefetcher_.cur_micro_data_fetch_idx_;
++prefetcher_.cur_micro_data_read_idx_;
const ObMicroIndexInfo &index_info = prefetcher_.current_micro_info();
const ObCSRange &row_range = index_info.get_row_range();
if (index_info.is_filter_always_false()) {
if (OB_FAIL(result_bitmap.set_bitmap_batch(
MAX(query_index_range_.start_row_id_, row_range.start_row_id_),
MIN(query_index_range_.end_row_id_, row_range.end_row_id_),
false))) {
LOG_WARN("Fail to set bitmap batch", K(ret), K(row_range));
}
} else if (index_info.is_filter_always_true()) {
if (OB_FAIL(result_bitmap.set_bitmap_batch(
MAX(query_index_range_.start_row_id_, row_range.start_row_id_),
MIN(query_index_range_.end_row_id_, row_range.end_row_id_),
true))) {
LOG_WARN("Fail to set bitmap batch", K(ret), K(row_range));
}
} else if (nullptr != parent && ObCGScanner::can_skip_filter(
*parent, *parent_bitmap, prefetcher_.current_micro_info().get_row_range())) {
continue;
} else {
break;
}
} else if (nullptr != parent && ObCGScanner::can_skip_filter(
*parent, *parent_bitmap, prefetcher_.current_micro_info().get_row_range())) {
continue;
} else {
break;
}
}
}
@ -379,38 +411,48 @@ int ObCGScanner::inner_filter(
int ret = OB_SUCCESS;
int64_t access_count = 0;
const common::ObBitmap *bitmap = nullptr;
sql::ObBoolMask prefetch_constant_type = prefetcher_.get_filter_constant_type();
ObCSRowId prefetch_constant_id = prefetcher_.get_max_filter_constant_id();
while (OB_SUCC(ret)) {
if (end_of_scan()) {
ret = OB_ITER_END;
} else if (is_new_range_ || OB_ITER_END == micro_scanner_->end_of_block()) {
if (OB_FAIL(get_next_valid_block(parent, parent_bitmap, result_bitmap))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("Fail to get next valid index", K(ret));
}
} else if (OB_FAIL(open_cur_data_block())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("Fail to open cur data block", K(ret));
} else if (prefetch_constant_type.is_constant() &&
((is_reverse_scan_ && prefetch_constant_id <= current_) ||
(!is_reverse_scan_ && prefetch_constant_id >= current_))) {
current_ = is_reverse_scan_ ? prefetch_constant_id - 1 : prefetch_constant_id + 1;
result_bitmap.set_constant_filter_info(prefetch_constant_type, prefetch_constant_id);
LOG_DEBUG("Set constant filter info", K(ret), K(prefetch_constant_type), K(prefetch_constant_id));
} else {
if (is_new_range_ || OB_ITER_END == micro_scanner_->end_of_block()) {
if (OB_FAIL(get_next_valid_block(parent, parent_bitmap, result_bitmap))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("Fail to get next valid index", K(ret));
}
} else if (OB_FAIL(open_cur_data_block())) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("Fail to open cur data block", K(ret));
}
}
}
}
if (OB_SUCC(ret)) {
ObMicroIndexInfo &index_info = prefetcher_.current_micro_info();
const ObCSRange &row_range = index_info.get_row_range();
uint32_t offset = row_range.start_row_id_ > query_index_range_.start_row_id_ ? (row_range.start_row_id_ - query_index_range_.start_row_id_) : 0;
if (OB_SUCC(ret)) {
ObMicroIndexInfo &index_info = prefetcher_.current_micro_info();
const ObCSRange &row_range = index_info.get_row_range();
uint32_t offset = row_range.start_row_id_ > query_index_range_.start_row_id_ ? (row_range.start_row_id_ - query_index_range_.start_row_id_) : 0;
index_info.pre_process_filter(*filter_info.filter_);
if (OB_FAIL(micro_scanner_->filter_micro_block_in_cg(
parent, filter_info, parent_bitmap, row_range.start_row_id_, access_count))) {
LOG_WARN("Fail to apply filter", K(ret));
} else if (OB_ISNULL(bitmap = filter_info.filter_->get_result()) || bitmap->size() != access_count) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected null filter bitmap", K(ret), KPC(filter_info.filter_), K(access_count));
} else if (OB_FAIL(result_bitmap.append_bitmap(*bitmap, offset, false))) {
LOG_WARN("Fail to append bitmap", K(ret), K(offset), KPC(bitmap), K(result_bitmap));
} else {
current_ = is_reverse_scan_ ? current_ - access_count : current_ + access_count;
index_info.pre_process_filter(*filter_info.filter_);
if (OB_FAIL(micro_scanner_->filter_micro_block_in_cg(
parent, filter_info, parent_bitmap, row_range.start_row_id_, access_count))) {
LOG_WARN("Fail to apply filter", K(ret));
} else if (OB_ISNULL(bitmap = filter_info.filter_->get_result()) || bitmap->size() != access_count) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected null filter bitmap", K(ret), KPC(filter_info.filter_), K(access_count));
} else if (OB_FAIL(result_bitmap.append_bitmap(*bitmap, offset, false))) {
LOG_WARN("Fail to append bitmap", K(ret), K(offset), KPC(bitmap), K(result_bitmap));
} else {
current_ = is_reverse_scan_ ? current_ - access_count : current_ + access_count;
}
index_info.post_process_filter(*filter_info.filter_);
}
index_info.post_process_filter(*filter_info.filter_);
}
}
return ret;

42
src/storage/column_store/ob_co_sstable_row_scanner.cpp
View File

@ -639,6 +639,7 @@ int ObCOSSTableRowScanner::filter_rows_with_limit(BlockScanState &blockscan_stat
LOG_WARN("Fail to inner filter", K(ret));
}
} else if (nullptr != result_bitmap && result_bitmap->is_all_false()) {
// TODO: @dengzhi.ldz opt for cg scanner with limit
update_current(group_size_);
} else {
int64_t begin_idx = begin;
@ -682,7 +683,15 @@ int ObCOSSTableRowScanner::filter_rows_without_limit(BlockScanState &blockscan_s
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected result bitmap", K(ret), KPC(rows_filter_));
} else if (nullptr != result_bitmap && result_bitmap->is_all_false()) {
update_current(group_size_);
if (result_bitmap->get_filter_constant_type().is_always_false()) {
if (reverse_scan_) {
current_ = MAX(result_bitmap->get_filter_constant_id(), end_) - 1;
} else {
current_ = MIN(result_bitmap->get_filter_constant_id(), end_) + 1;
}
} else {
update_current(group_size_);
}
current_start_row_id = current_;
} else {
need_do_filter = false;
@ -717,7 +726,7 @@ int ObCOSSTableRowScanner::filter_rows_without_limit(BlockScanState &blockscan_s
result_bitmap))) {
LOG_WARN("Fail to locate", K(ret), K(current_), K(group_size_), KP(result_bitmap));
}
LOG_DEBUG("COScanner filter_rows_with_limit end", K(ret), K_(state), K_(blockscan_state),
LOG_DEBUG("COScanner filter_rows_without_limit end", K(ret), K_(state), K_(blockscan_state),
K_(current), K_(group_size), K_(end));
return ret;
}
@ -748,8 +757,7 @@ int ObCOSSTableRowScanner::inner_filter(
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected result bitmap", K(ret), KPC(rows_filter_));
} else {
int64_t select_cnt = result_bitmap->popcnt();
EVENT_ADD(ObStatEventIds::PUSHDOWN_STORAGE_FILTER_ROW_CNT, select_cnt);
EVENT_ADD(ObStatEventIds::PUSHDOWN_STORAGE_FILTER_ROW_CNT, result_bitmap->popcnt());
}
} else {
EVENT_ADD(ObStatEventIds::PUSHDOWN_STORAGE_FILTER_ROW_CNT, group_size);
@ -759,7 +767,9 @@ int ObCOSSTableRowScanner::inner_filter(
access_ctx_->table_store_stat_.logical_read_cnt_ += group_size;
access_ctx_->table_store_stat_.physical_read_cnt_ += group_size;
LOG_TRACE("[COLUMNSTORE] COSSTableRowScanner inner filter", K(ret), "begin", begin, "count", group_size,
"filtered", nullptr == rows_filter_ ? 0 : 1, "popcnt", nullptr == result_bitmap ? group_size : result_bitmap->popcnt());
"filtered", nullptr == rows_filter_ ? 0 : 1, "popcnt", nullptr == result_bitmap ? group_size : result_bitmap->popcnt(),
"filter_constant_type", nullptr == result_bitmap ? sql::ObBoolMaskType::ALWAYS_TRUE : result_bitmap->get_filter_constant_type(),
"filter_constant_id", nullptr == result_bitmap ? (begin + group_size - 1) : result_bitmap->get_filter_constant_id());
}
return ret;
}
@ -787,18 +797,32 @@ int ObCOSSTableRowScanner::update_continuous_range(
rows_filter_ == nullptr ? true : rows_filter_->can_continuous_filter();
if (group_is_true && filter_tree_can_continuous) {
// current group is true, continue do filter if not reach end
sql::ObBoolMask filter_constant_type;
if (nullptr != result_bitmap) {
filter_constant_type = result_bitmap->get_filter_constant_type();
}
if (reverse_scan_) {
continuous_end_row_id = current_start_row_id;
// continuous_end_row_id = current_start_row_id;
continuous_end_row_id = filter_constant_type.is_always_true() ? MAX(result_bitmap->get_filter_constant_id(), end_) : current_start_row_id;
current_start_row_id = continuous_end_row_id - 1;
continue_filter = current_start_row_id >= end_;
} else {
continuous_end_row_id = current_start_row_id + current_group_size - 1;
// continuous_end_row_id = current_start_row_id + current_group_size - 1;
continuous_end_row_id = filter_constant_type.is_always_true() ? MIN(result_bitmap->get_filter_constant_id(), end_) : current_start_row_id + current_group_size - 1;
current_start_row_id = continuous_end_row_id + 1;
continue_filter = current_start_row_id <= end_;
}
} else if ((nullptr != result_bitmap && result_bitmap->is_all_false()) && OB_INVALID_CS_ROW_ID == continuous_end_row_id) {
// current group is false and no continuous true range before, skip this group and continue do filter
update_current(current_group_size);
if (result_bitmap->get_filter_constant_type().is_always_false()) {
if (reverse_scan_) {
current_ = MAX(result_bitmap->get_filter_constant_id(), end_) - 1;
} else {
current_ = MIN(result_bitmap->get_filter_constant_id(), end_) + 1;
}
} else {
update_current(current_group_size);
}
current_start_row_id = current_;
} else {
continue_filter = false;
@ -809,6 +833,8 @@ int ObCOSSTableRowScanner::update_continuous_range(
// no continuous true range before, will project current group
}
}
LOG_DEBUG("Filter continuous range info", K(ret), K(current_start_row_id), K(current_group_size),
K(continuous_end_row_id), K(continue_filter), K_(pending_end_row_id), K_(current), KPC(result_bitmap));
return ret;
}

15
src/storage/column_store/ob_co_sstable_rows_filter.cpp
View File

@ -221,7 +221,10 @@ int ObCOSSTableRowsFilter::apply(const ObCSRange &range)
adjust_batch_size();
prepared_ = false;
LOG_DEBUG("[COLUMNSTORE] apply filter info", K(range),
K(bitmap_buffer_[0]->size()), K(bitmap_buffer_[0]->popcnt()));
"filter_constant_type", bitmap_buffer_[0]->get_filter_constant_type(),
"filter_constant_id", bitmap_buffer_[0]->get_filter_constant_id(),
"bitmap_size", bitmap_buffer_[0]->size(),
"popcnt", bitmap_buffer_[0]->popcnt());
}
return ret;
}
@ -269,6 +272,7 @@ int ObCOSSTableRowsFilter::apply_filter(
// Parent prepare_skip_filter can not be called here.
} else if (filter->is_sample_node()) {
ObSampleFilterExecutor *sample_executor = static_cast<ObSampleFilterExecutor *>(filter);
result->set_filter_uncertain();
if (OB_FAIL(sample_executor->apply_sample_filter(range, *result->get_inner_bitmap()))) {
LOG_WARN("Failed to apply sample filter", K(ret), K(range), KP(result), KPC(sample_executor));
}
@ -353,8 +357,9 @@ int ObCOSSTableRowsFilter::post_apply_filter(
is_skip = true;
}
}
LOG_DEBUG("[COLUMNSTORE] post apply filter info", "is_and", filter.is_logic_and_node(), K(result.size()),
K(result.popcnt()), K(child_result.size()), K(child_result.popcnt()));
LOG_DEBUG("[COLUMNSTORE] post apply filter info", K(ret), K(is_skip), "is_and", filter.is_logic_and_node(),
K(result.size()), K(result.popcnt()), K(result.get_filter_constant_type()), K(result.get_filter_constant_id()),
K(child_result.size()), K(child_result.popcnt()), K(child_result.get_filter_constant_type()), K(child_result.get_filter_constant_id()));
return ret;
}
@ -660,7 +665,7 @@ int ObCOSSTableRowsFilter::init_bitmap_buffer(uint32_t bitmap_buffer_count)
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("Failed to alloc memory for filter bitmap", K(ret));
} else if (FALSE_IT(filter_bitmap = new (buf) ObCGBitmap(*allocator_))) {
} else if (OB_FAIL(filter_bitmap->init(batch_size_))) {
} else if (OB_FAIL(filter_bitmap->init(batch_size_, access_ctx_->query_flag_.is_reverse_scan()))) {
LOG_WARN("Failed to init bitmap", K(ret), K_(batch_size));
} else if (OB_FAIL(bitmap_buffer_.push_back(filter_bitmap))) {
LOG_WARN("Failed to push_back filter bitmap", K(ret), KP(filter_bitmap));
@ -690,7 +695,7 @@ int ObCOSSTableRowsFilter::prepare_bitmap_buffer(
if (OB_ISNULL(result)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected nullptr filter_bitmap", K(ret), K(depth));
} else if (OB_FAIL(result->reserve(range.get_row_count()))) {
} else if (OB_FAIL(result->switch_context(range.get_row_count(), access_ctx_->query_flag_.is_reverse_scan()))) {
LOG_WARN("Failed to expand size for filter bitmap", K(ret), K(range));
} else if (0 == depth) {
result->reuse(range.start_row_id_, filter.is_logic_and_node());

1
src/storage/column_store/ob_virtual_cg_scanner.cpp
View File

@ -489,6 +489,7 @@ int ObDefaultCGScanner::apply_filter(
} else {
filter_result_ = result;
filter_ = filter;
// TODO: set the max contant row id
if (result) {
result_bitmap.set_all_true();
} else {

6
unittest/storage/column_store/test_co_sstable_rows_filter.cpp
View File

@ -289,6 +289,7 @@ void TestCOSSTableRowsFilter::init_single_white_filter()
ASSERT_FALSE(nullptr == filter_);
co_filter_.filter_ = filter_;
co_filter_.allocator_ = &allocator_;
co_filter_.access_ctx_ = &context_;
}
void TestCOSSTableRowsFilter::init_single_black_filter()
@ -300,6 +301,7 @@ void TestCOSSTableRowsFilter::init_single_black_filter()
ASSERT_FALSE(nullptr == filter_);
co_filter_.filter_ = filter_;
co_filter_.allocator_ = &allocator_;
co_filter_.access_ctx_ = &context_;
}
void TestCOSSTableRowsFilter::init_multi_white_filter(bool is_common)
@ -322,6 +324,7 @@ void TestCOSSTableRowsFilter::init_multi_white_filter(bool is_common)
filter_->set_childs(3, childs);
co_filter_.filter_ = filter_;
co_filter_.allocator_ = &allocator_;
co_filter_.access_ctx_ = &context_;
}
void TestCOSSTableRowsFilter::init_multi_black_filter(bool is_common)
@ -351,6 +354,7 @@ void TestCOSSTableRowsFilter::init_multi_black_filter(bool is_common)
filter_->set_childs(3, childs);
co_filter_.filter_ = filter_;
co_filter_.allocator_ = &allocator_;
co_filter_.access_ctx_ = &context_;
}
void TestCOSSTableRowsFilter::init_multi_white_and_black_filter_case_one()
@ -383,6 +387,7 @@ void TestCOSSTableRowsFilter::init_multi_white_and_black_filter_case_one()
filter_->set_childs(2, childs);
co_filter_.filter_ = filter_;
co_filter_.allocator_ = &allocator_;
co_filter_.access_ctx_ = &context_;
}
void TestCOSSTableRowsFilter::init_multi_white_and_black_filter_case_two()
@ -434,6 +439,7 @@ void TestCOSSTableRowsFilter::init_multi_white_and_black_filter_case_two()
filter_->set_childs(3, childs);
co_filter_.filter_ = filter_;
co_filter_.allocator_ = &allocator_;
co_filter_.access_ctx_ = &context_;
}
void TestCOSSTableRowsFilter::reset_filter()