From 2019bb387007c89952cdb4fd665fbb26bb88e991 Mon Sep 17 00:00:00 2001 From: TengJianPing <18241664+jacktengg@users.noreply.github.com> Date: Wed, 9 Aug 2023 18:31:24 +0800 Subject: [PATCH] [fix](bitmap) fix wrong result of bitmap intersect functions (#22735) * [fix](bitmap) fix wrong result of bitmap intersect functions * fix test case --- .../aggregate_function_orthogonal_bitmap.h | 3 + .../bitmap_functions/test_bitmap_function.out | 39 +++++ .../test_bitmap_function.groovy | 142 ++++++++++++++++++ 3 files changed, 184 insertions(+) diff --git a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h index c632169c55..0204c08e02 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h +++ b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h @@ -105,6 +105,7 @@ public: return; } result |= rhs.result; + AggOrthBitmapBaseData::first_init = false; } void write(BufferWritable& buf) { @@ -139,6 +140,7 @@ public: return; } AggOrthBitmapBaseData::bitmap.merge(rhs.bitmap); + AggOrthBitmapBaseData::first_init = false; } void write(BufferWritable& buf) { @@ -174,6 +176,7 @@ public: return; } result += rhs.result; + AggOrthBitmapBaseData::first_init = false; } void write(BufferWritable& buf) { diff --git a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out index 8dc638b78a..4eeae0ccd9 100644 --- a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out +++ b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out @@ -492,3 +492,42 @@ true \N \N +-- !sql_bitmap_intersect_check0 -- +1 + +-- !sql_bitmap_intersect_check1 -- +1 + +-- !sql_bitmap_intersect_check2 -- +1 + +-- !sql_bitmap_intersect_nereids0 -- +1 1 + +-- !sql_bitmap_intersect_nereids1 -- +1 1 + +-- !sql_bitmap_intersect_no_nereids0 -- +1 1 + +-- !sql_bitmap_intersect_no_nereids1 -- +1 1 + +-- !sql_orthogonal_bitmap_intersect_nereids1 -- +1 1 + +-- !sql_orthogonal_bitmap_intersect_not_nereids1 -- +1 1 + +-- !sql_orthogonal_bitmap_intersect_count_nereids0 -- +1 1 + +-- !sql_orthogonal_bitmap_intersect_count_nereids1 -- +1 1 + +-- !sql_orthogonal_bitmap_intersect_count_not_nereids0 -- +1 1 + +-- !sql_orthogonal_bitmap_intersect_count_not_nereids1 -- +1 1 + diff --git a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy index a14267a395..2b3fbb2031 100644 --- a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy @@ -690,4 +690,146 @@ suite("test_bitmap_function") { test_bitmap1; """ + sql """ + drop TABLE if exists test_bitmap_intersect; + """ + + sql """ + CREATE TABLE test_bitmap_intersect ( + dt1 date NOT NULL, + dt2 date NOT NULL, + id varchar(256) NULL, + type smallint(6) MAX NULL, + id_bitmap bitmap BITMAP_UNION NULL + ) ENGINE = OLAP AGGREGATE KEY(dt1, dt2, id) PARTITION BY RANGE(dt1) ( + PARTITION p20230725 + VALUES + [('2023-07-25'), ('2023-07-26'))) + DISTRIBUTED BY HASH(dt1, dt2) BUCKETS 1 properties("replication_num"="1"); + """ + + sql """ + insert into test_bitmap_intersect + select + str_to_date('2023-07-25','%Y-%m-%d') as dt1, + str_to_date('2023-07-25','%Y-%m-%d') as dt2, + 'aaaaaaaaaa' as id, + 1 as type, + BITMAP_HASH64('aaaaaaaaaa') as id_bitmap; + """ + qt_sql_bitmap_intersect_check0 """ + select intersect_count(id_bitmap, type, 1) as count2_bitmap from test_bitmap_intersect; + """ + qt_sql_bitmap_intersect_check1 """ + select bitmap_count(orthogonal_bitmap_intersect(id_bitmap, type, 1)) as count2_bitmap from test_bitmap_intersect; + """ + qt_sql_bitmap_intersect_check2 """ + select orthogonal_bitmap_intersect_count(id_bitmap, type, 1) as count2_bitmap from test_bitmap_intersect; + """ + + // test function intersect_count + // test nereids + sql """ set experimental_enable_nereids_planner=true; """ + // test pipeline + sql """ set experimental_enable_pipeline_engine=true; """ + qt_sql_bitmap_intersect_nereids0 """ + select count(distinct if(type=1, id,null)) as count1, + intersect_count(id_bitmap, type, 1) as count2_bitmap from test_bitmap_intersect; + """ + sql """ set experimental_enable_pipeline_engine=false; """ + qt_sql_bitmap_intersect_nereids1 """ + select count(distinct if(type=1, id,null)) as count1, + intersect_count(id_bitmap, type, 1) as count2_bitmap from test_bitmap_intersect; + """ + + // test not nereids + sql """ set experimental_enable_nereids_planner=false; """ + // test pipeline + sql """ set experimental_enable_pipeline_engine=true; """ + qt_sql_bitmap_intersect_no_nereids0 """ + select count(distinct if(type=1, id,null)) as count1, + intersect_count(id_bitmap, type, 1) as count2_bitmap from test_bitmap_intersect; + """ + sql """ set experimental_enable_pipeline_engine=false; """ + qt_sql_bitmap_intersect_no_nereids1 """ + select count(distinct if(type=1, id,null)) as count1, + intersect_count(id_bitmap, type, 1) as count2_bitmap from test_bitmap_intersect; + """ + + sql """ + drop TABLE if exists test_orthog_bitmap_intersect; + """ + sql """ + CREATE TABLE test_orthog_bitmap_intersect ( + tag int NOT NULL, + hid int NOT NULL, + id_bitmap bitmap BITMAP_UNION NULL + ) ENGINE = OLAP AGGREGATE KEY(tag, hid) + DISTRIBUTED BY HASH(hid) BUCKETS 1 properties("replication_num"="1"); + """ + + sql """ + insert into test_orthog_bitmap_intersect + select 0, 1, to_bitmap(1) as id_bitmap; + """ + // test function orthogonal_bitmap_intersect + // test nereids + sql """ set experimental_enable_nereids_planner=true; """ + // TODO: case will stuck untile timeout, enable it when pipeline bug is fixed + // test pipeline + // sql """ set experimental_enable_pipeline_engine=true; """ + // qt_sql_orthogonal_bitmap_intersect_nereids0 """ + // select count(distinct if(type=1, id,null)) as count1, + // bitmap_count(orthogonal_bitmap_intersect(id_bitmap, type, 1)) as count2_bitmap from test_orthog_bitmap_intersect; + // """ + sql """ set experimental_enable_pipeline_engine=false; """ + qt_sql_orthogonal_bitmap_intersect_nereids1 """ + select count(distinct tag) as count1, + bitmap_count(orthogonal_bitmap_intersect(id_bitmap, tag, 0)) as count2_bitmap from test_orthog_bitmap_intersect; + """ + + // test not nereids + sql """ set experimental_enable_nereids_planner=false; """ + // TODO: case will stuck untile timeout, enable it when pipeline bug is fixed + // test pipeline + // sql """ set experimental_enable_pipeline_engine=true; """ + // qt_sql_orthogonal_bitmap_intersect_not_nereids0 """ + // select count(distinct if(type=1, id,null)) as count1, + // bitmap_count(orthogonal_bitmap_intersect(id_bitmap, type, 1)) as count2_bitmap from test_orthog_bitmap_intersect; + // """ + sql """ set experimental_enable_pipeline_engine=false; """ + qt_sql_orthogonal_bitmap_intersect_not_nereids1 """ + select count(distinct tag) as count1, + bitmap_count(orthogonal_bitmap_intersect(id_bitmap, tag, 0)) as count2_bitmap from test_orthog_bitmap_intersect; + """ + + // test function orthogonal_bitmap_intersect_count + // test nereids + sql """ set experimental_enable_nereids_planner=true; """ + // test pipeline + sql """ set experimental_enable_pipeline_engine=true; """ + qt_sql_orthogonal_bitmap_intersect_count_nereids0 """ + select count(distinct tag) as count1, + orthogonal_bitmap_intersect_count(id_bitmap, tag, 0) as count2_bitmap from test_orthog_bitmap_intersect; + """ + sql """ set experimental_enable_pipeline_engine=false; """ + qt_sql_orthogonal_bitmap_intersect_count_nereids1 """ + select count(distinct tag) as count1, + orthogonal_bitmap_intersect_count(id_bitmap, tag, 0) as count2_bitmap from test_orthog_bitmap_intersect; + """ + + // test not nereids + sql """ set experimental_enable_nereids_planner=false; """ + // test pipeline + sql """ set experimental_enable_pipeline_engine=true; """ + // TODO: enable this case after issue #22771 if solved + // qt_sql_orthogonal_bitmap_intersect_count_not_nereids0 """ + // select count(distinct tag) as count1, + // orthogonal_bitmap_intersect_count(id_bitmap, tag, 0) as count2_bitmap from test_orthog_bitmap_intersect; + // """ + sql """ set experimental_enable_pipeline_engine=false; """ + qt_sql_orthogonal_bitmap_intersect_count_not_nereids1 """ + select count(distinct tag) as count1, + orthogonal_bitmap_intersect_count(id_bitmap, tag, 0) as count2_bitmap from test_orthog_bitmap_intersect; + """ }