From eea6d770d769d6f229c3257db79ba4b2a6edc750 Mon Sep 17 00:00:00 2001 From: TengJianPing <18241664+jacktengg@users.noreply.github.com> Date: Wed, 8 Mar 2023 16:29:01 +0800 Subject: [PATCH] [fix](bitmap) fix wrong result of bitmap_or for null (#17456) Result of select bitmap_to_string(bitmap_or(to_bitmap(1), null)) should be 1 instead of null. This PR fix logic of bitmap_or and bitmap_or_count. Other count related funcitons should also be checked and fix, they will be fixed in another PR. --- .../functions/function_bitmap_variadic.cpp | 178 ++++++++++--- be/test/vec/function/function_bitmap_test.cpp | 2 +- .../bitmap_functions/test_bitmap_function.out | 134 +++++++++- .../test_bitmap_function_nereids.out | 50 +--- .../bitmap_functions/test_bitmap_function.out | 134 +++++++++- .../test_bitmap_function.groovy | 250 +++++++++++++++++- .../test_bitmap_function_nereids.groovy | 24 +- .../test_bitmap_function.groovy | 247 ++++++++++++++++- 8 files changed, 877 insertions(+), 142 deletions(-) diff --git a/be/src/vec/functions/function_bitmap_variadic.cpp b/be/src/vec/functions/function_bitmap_variadic.cpp index d9679c3819..bbc0309808 100644 --- a/be/src/vec/functions/function_bitmap_variadic.cpp +++ b/be/src/vec/functions/function_bitmap_variadic.cpp @@ -25,49 +25,104 @@ namespace doris::vectorized { -#define BITMAP_FUNCTION_VARIADIC(CLASS, FUNCTION_NAME, OP) \ - struct CLASS { \ - static constexpr auto name = #FUNCTION_NAME; \ - using ResultDataType = DataTypeBitMap; \ - static Status vector_vector(ColumnPtr argument_columns[], size_t col_size, \ - size_t input_rows_count, std::vector& res) { \ - auto& mid_data = \ - assert_cast(argument_columns[0].get())->get_data(); \ - for (size_t row = 0; row < input_rows_count; ++row) { \ - res[row] = mid_data[row]; \ - } \ - for (size_t col = 1; col < col_size; ++col) { \ - auto& col_data = \ - assert_cast(argument_columns[col].get())->get_data(); \ - for (size_t row = 0; row < input_rows_count; ++row) { \ - res[row] OP col_data[row]; \ - } \ - } \ - return Status::OK(); \ - } \ +// currently only bitmap_or and bitmap_or_count will call this function, +// other bitmap functions will use default implementation for nulls +#define BITMAP_OR_NULLABLE(nullable, input_rows_count, res, op) \ + const auto& nested_col_ptr = nullable->get_nested_column_ptr(); \ + const auto* __restrict null_map_data = nullable->get_null_map_data().data(); \ + const auto& mid_data = assert_cast(nested_col_ptr.get())->get_data(); \ + for (size_t row = 0; row < input_rows_count; ++row) { \ + if (!null_map_data[row]) { \ + res[row] op mid_data[row]; \ + } \ } -#define BITMAP_FUNCTION_COUNT_VARIADIC(CLASS, FUNCTION_NAME, OP) \ - struct CLASS { \ - static constexpr auto name = #FUNCTION_NAME; \ - using ResultDataType = DataTypeInt64; \ - using TData = std::vector; \ - using ResTData = typename ColumnVector::Container; \ - static Status vector_vector(ColumnPtr argument_columns[], size_t col_size, \ - size_t input_rows_count, ResTData& res) { \ - TData vals = assert_cast(argument_columns[0].get())->get_data(); \ - for (size_t col = 1; col < col_size; ++col) { \ - auto& col_data = \ - assert_cast(argument_columns[col].get())->get_data(); \ - for (size_t row = 0; row < input_rows_count; ++row) { \ - vals[row] OP col_data[row]; \ - } \ - } \ - for (size_t row = 0; row < input_rows_count; ++row) { \ - res[row] = vals[row].cardinality(); \ - } \ - return Status::OK(); \ - } \ +#define BITMAP_FUNCTION_VARIADIC(CLASS, FUNCTION_NAME, OP) \ + struct CLASS { \ + static constexpr auto name = #FUNCTION_NAME; \ + using ResultDataType = DataTypeBitMap; \ + static Status vector_vector(ColumnPtr argument_columns[], size_t col_size, \ + size_t input_rows_count, std::vector& res, \ + IColumn* res_nulls) { \ + const ColumnUInt8::value_type* null_map_datas[col_size]; \ + int nullable_cols_count = 0; \ + ColumnUInt8::value_type* __restrict res_nulls_data = nullptr; \ + if (res_nulls) { \ + res_nulls_data = assert_cast(res_nulls)->get_data().data(); \ + } \ + if (auto* nullable = check_and_get_column(*argument_columns[0])) { \ + null_map_datas[nullable_cols_count++] = nullable->get_null_map_data().data(); \ + BITMAP_OR_NULLABLE(nullable, input_rows_count, res, =); \ + } else { \ + const auto& mid_data = \ + assert_cast(argument_columns[0].get())->get_data(); \ + for (size_t row = 0; row < input_rows_count; ++row) { \ + res[row] = mid_data[row]; \ + } \ + } \ + for (size_t col = 1; col < col_size; ++col) { \ + if (auto* nullable = \ + check_and_get_column(*argument_columns[col])) { \ + null_map_datas[nullable_cols_count++] = nullable->get_null_map_data().data(); \ + BITMAP_OR_NULLABLE(nullable, input_rows_count, res, OP); \ + } else { \ + const auto& col_data = \ + assert_cast(argument_columns[col].get()) \ + ->get_data(); \ + for (size_t row = 0; row < input_rows_count; ++row) { \ + res[row] OP col_data[row]; \ + } \ + } \ + } \ + if (res_nulls_data && nullable_cols_count == col_size) { \ + const auto* null_map_data = null_map_datas[0]; \ + for (size_t row = 0; row < input_rows_count; ++row) { \ + res_nulls_data[row] = null_map_data[row]; \ + } \ + for (int i = 1; i < nullable_cols_count; ++i) { \ + const auto* null_map_data = null_map_datas[i]; \ + for (size_t row = 0; row < input_rows_count; ++row) { \ + res_nulls_data[row] &= null_map_data[row]; \ + } \ + } \ + } \ + return Status::OK(); \ + } \ + } + +#define BITMAP_FUNCTION_COUNT_VARIADIC(CLASS, FUNCTION_NAME, OP) \ + struct CLASS { \ + static constexpr auto name = #FUNCTION_NAME; \ + using ResultDataType = DataTypeInt64; \ + using TData = std::vector; \ + using ResTData = typename ColumnVector::Container; \ + static Status vector_vector(ColumnPtr argument_columns[], size_t col_size, \ + size_t input_rows_count, ResTData& res, IColumn* res_nulls) { \ + TData vals; \ + if (auto* nullable = check_and_get_column(*argument_columns[0])) { \ + vals.resize(input_rows_count); \ + BITMAP_OR_NULLABLE(nullable, input_rows_count, vals, =); \ + } else { \ + vals = assert_cast(argument_columns[0].get())->get_data(); \ + } \ + for (size_t col = 1; col < col_size; ++col) { \ + if (auto* nullable = \ + check_and_get_column(*argument_columns[col])) { \ + BITMAP_OR_NULLABLE(nullable, input_rows_count, vals, OP); \ + } else { \ + const auto& col_data = \ + assert_cast(argument_columns[col].get()) \ + ->get_data(); \ + for (size_t row = 0; row < input_rows_count; ++row) { \ + vals[row] OP col_data[row]; \ + } \ + } \ + } \ + for (size_t row = 0; row < input_rows_count; ++row) { \ + res[row] = vals[row].cardinality(); \ + } \ + return Status::OK(); \ + } \ } BITMAP_FUNCTION_VARIADIC(BitmapOr, bitmap_or, |=); @@ -92,10 +147,31 @@ public: DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { using ResultDataType = typename Impl::ResultDataType; - return std::make_shared(); + if (std::is_same_v || std::is_same_v) { + bool return_nullable = false; + // result is nullable only when any columns is nullable for bitmap_or and bitmap_or_count + for (size_t i = 0; i < arguments.size(); ++i) { + if (arguments[i]->is_nullable()) { + return_nullable = true; + break; + } + } + auto result_type = std::make_shared(); + return return_nullable ? make_nullable(result_type) : result_type; + } else { + return std::make_shared(); + } } bool use_default_implementation_for_constants() const override { return true; } + bool use_default_implementation_for_nulls() const override { + // result is null only when all columns is null for bitmap_or and bitmap_or_count + if (std::is_same_v || std::is_same_v) { + return false; + } else { + return true; + } + } Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) override { @@ -113,13 +189,27 @@ public: std::conditional_t, ColumnComplexType, ColumnVector>; typename ColVecResult::MutablePtr col_res = nullptr; + + typename ColumnUInt8::MutablePtr col_res_nulls; + auto& result_info = block.get_by_position(result); + // special case for bitmap_or and bitmap_or_count + if (!use_default_implementation_for_nulls() && result_info.type->is_nullable()) { + col_res_nulls = ColumnUInt8::create(input_rows_count, 0); + } + col_res = ColVecResult::create(); auto& vec_res = col_res->get_data(); vec_res.resize(input_rows_count); - Impl::vector_vector(argument_columns, argument_size, input_rows_count, vec_res); - block.replace_by_position(result, std::move(col_res)); + Impl::vector_vector(argument_columns, argument_size, input_rows_count, vec_res, + col_res_nulls); + if (!use_default_implementation_for_nulls() && result_info.type->is_nullable()) { + block.replace_by_position( + result, ColumnNullable::create(std::move(col_res), std::move(col_res_nulls))); + } else { + block.replace_by_position(result, std::move(col_res)); + } return Status::OK(); } }; diff --git a/be/test/vec/function/function_bitmap_test.cpp b/be/test/vec/function/function_bitmap_test.cpp index b0d725d0b1..fadb858242 100644 --- a/be/test/vec/function/function_bitmap_test.cpp +++ b/be/test/vec/function/function_bitmap_test.cpp @@ -118,7 +118,7 @@ TEST(function_bitmap_test, function_bitmap_or_count) { DataSet data_set = {{{&bitmap1, &bitmap2, &empty_bitmap}, (int64_t)5}, //0,1,33,1024,2019 {{&bitmap1, &bitmap2, &bitmap3}, (int64_t)7}, //0,1,5,33,1024,2019,18446744073709551615 - {{&bitmap1, &empty_bitmap, Null()}, Null()}, + {{&bitmap1, &empty_bitmap, Null()}, (int64_t)3}, {{&bitmap1, &bitmap3, &bitmap3}, (int64_t)6}}; //1,5,33,1024,2019,18446744073709551615 diff --git a/regression-test/data/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.out b/regression-test/data/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.out index 2f5c7a4850..29e1234dca 100644 --- a/regression-test/data/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.out +++ b/regression-test/data/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.out @@ -65,24 +65,130 @@ false -- !sql_bitmap_hash64_3 -- 0 --- !sql -- +-- !sql_bitmap_or1 -- 2 --- !sql -- +-- !sql_bitmap_or2 -- 1 --- !sql -- +-- !sql_bitmap_or3 -- 1,2 --- !sql -- -\N - --- !sql -- +-- !sql_bitmap_or4 -- 0,1,2,10 --- !sql -- +-- !sql_bitmap_or5 -- +0,1,2,10 + +-- !sql_bitmap_or6 -- 1,2,3,4,5,10 +-- !sql_bitmap_or7 -- +1 + +-- !sql_bitmap_or8 -- +1 2 +2 2 +3 1 +4 1 + +-- !sql_bitmap_or_count6 -- +1 2 +2 2 +3 1 +4 1 + +-- !sql_bitmap_or9 -- +1 2 +2 2 + +-- !sql_bitmap_or_count7 -- +1 2 +2 2 + +-- !sql_bitmap_or10 -- +1 11,111 +2 22,222 +3 33 +4 44 + +-- !sql_bitmap_or11 -- +1 11,111 +2 22,222 + +-- !sql_bitmap_or12 -- +1 2 +2 2 +3 1 +4 1 + +-- !sql_bitmap_or_count8 -- +1 2 +2 2 +3 1 +4 1 + +-- !sql_bitmap_or13 -- +1 2 +2 2 + +-- !sql_bitmap_or_count9 -- +1 2 +2 2 + +-- !sql_bitmap_or14 -- +1 11,111 +2 22,222 +3 33 +4 44 + +-- !sql_bitmap_or15 -- +1 11,111 +2 22,222 + +-- !sql_bitmap_or16_0 -- +false + +-- !sql_bitmap_or16_1 -- +true + +-- !sql_bitmap_or16 -- +true + +-- !sql_bitmap_or17 -- +0 + +-- !sql_bitmap_or_count10 -- +0 + +-- !sql_bitmap_or18 -- +\N + +-- !sql_bitmap_or19 -- +true + +-- !sql_bitmap_or21 -- +\N + +-- !sql_bitmap_or22_0 -- +3 1 0 +4 1 0 + +-- !sql_bitmap_or22_1 -- +5 0 1 + +-- !sql_bitmap_or22 -- +3 \N \N 5 true +4 \N \N 5 true + +-- !sql_bitmap_or_count11 -- +3 \N \N 5 0 +4 \N \N 5 0 + +-- !sql_bitmap_or23 -- +3 \N \N 5 \N +4 \N \N 5 \N + -- !sql -- 0 @@ -101,20 +207,20 @@ false -- !sql -- \N --- !sql -- +-- !sql_bitmap_or_count1 -- 3 --- !sql -- +-- !sql_bitmap_or_count2 -- 3 --- !sql -- +-- !sql_bitmap_or_count3 -- 5 --- !sql -- +-- !sql_bitmap_or_count4 -- 6 --- !sql -- -\N +-- !sql_bitmap_or_count5 -- +6 -- !sql -- 2 diff --git a/regression-test/data/nereids_syntax_p0/test_bitmap_function_nereids.out b/regression-test/data/nereids_syntax_p0/test_bitmap_function_nereids.out index ed771ea325..acbc116f38 100644 --- a/regression-test/data/nereids_syntax_p0/test_bitmap_function_nereids.out +++ b/regression-test/data/nereids_syntax_p0/test_bitmap_function_nereids.out @@ -65,22 +65,22 @@ false -- !sql_bitmap_hash64_3 -- 0 --- !sql -- +-- !sql_bitmap_or1 -- 2 --- !sql -- +-- !sql_bitmap_or2 -- 1 --- !sql -- +-- !sql_bitmap_or3 -- 1,2 --- !sql -- -\N - --- !sql -- +-- !sql_bitmap_or4 -- 0,1,2,10 --- !sql -- +-- !sql_bitmap_or5 -- +0,1,2,10 + +-- !sql_bitmap_or6 -- 1,2,3,4,5,10 -- !sql -- @@ -101,20 +101,20 @@ false -- !sql -- \N --- !sql -- +-- !sql_bitmap_or_count1 -- 3 --- !sql -- +-- !sql_bitmap_or_count2 -- 3 --- !sql -- +-- !sql_bitmap_or_count3 -- 5 --- !sql -- +-- !sql_bitmap_or_count4 -- 6 --- !sql -- -\N +-- !sql_bitmap_or_count5 -- +6 -- !sql -- 2 @@ -249,25 +249,3 @@ false -- !sql -- 0 --- !sql -- -[1, 2, 3] -[1, 2, 3, 4, 5] - --- !sql -- -[] - --- !sql -- -[3, 4, 100, 200] - --- !sql -- -1,2,3 - --- !sql -- -1 - --- !sql -- -100 - --- !sql -- -20221103 - diff --git a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out index d7b68d929b..c3cccfbba5 100644 --- a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out +++ b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out @@ -65,24 +65,130 @@ false -- !sql_bitmap_hash64_3 -- 0 --- !sql -- +-- !sql_bitmap_or1 -- 2 --- !sql -- +-- !sql_bitmap_or2 -- 1 --- !sql -- +-- !sql_bitmap_or3 -- 1,2 --- !sql -- -\N - --- !sql -- +-- !sql_bitmap_or4 -- 0,1,2,10 --- !sql -- +-- !sql_bitmap_or5 -- +0,1,2,10 + +-- !sql_bitmap_or6 -- 1,2,3,4,5,10 +-- !sql_bitmap_or7 -- +1 + +-- !sql_bitmap_or8 -- +1 2 +2 2 +3 1 +4 1 + +-- !sql_bitmap_or_count6 -- +1 2 +2 2 +3 1 +4 1 + +-- !sql_bitmap_or9 -- +1 2 +2 2 + +-- !sql_bitmap_or_count7 -- +1 2 +2 2 + +-- !sql_bitmap_or10 -- +1 11,111 +2 22,222 +3 33 +4 44 + +-- !sql_bitmap_or11 -- +1 11,111 +2 22,222 + +-- !sql_bitmap_or12 -- +1 2 +2 2 +3 1 +4 1 + +-- !sql_bitmap_or_count8 -- +1 2 +2 2 +3 1 +4 1 + +-- !sql_bitmap_or13 -- +1 2 +2 2 + +-- !sql_bitmap_or_count9 -- +1 2 +2 2 + +-- !sql_bitmap_or14 -- +1 11,111 +2 22,222 +3 33 +4 44 + +-- !sql_bitmap_or15 -- +1 11,111 +2 22,222 + +-- !sql_bitmap_or16_0 -- +false + +-- !sql_bitmap_or16_1 -- +true + +-- !sql_bitmap_or16 -- +true + +-- !sql_bitmap_or17 -- +0 + +-- !sql_bitmap_or_count10 -- +0 + +-- !sql_bitmap_or18 -- +\N + +-- !sql_bitmap_or19 -- +true + +-- !sql_bitmap_or21 -- +\N + +-- !sql_bitmap_or22_0 -- +3 1 0 +4 1 0 + +-- !sql_bitmap_or22_1 -- +5 0 1 + +-- !sql_bitmap_or22 -- +3 \N \N 5 true +4 \N \N 5 true + +-- !sql_bitmap_or_count11 -- +3 \N \N 5 0 +4 \N \N 5 0 + +-- !sql_bitmap_or23 -- +3 \N \N 5 \N +4 \N \N 5 \N + -- !sql -- 0 @@ -101,20 +207,20 @@ false -- !sql -- \N --- !sql -- +-- !sql_bitmap_or_count1 -- 3 --- !sql -- +-- !sql_bitmap_or_count2 -- 3 --- !sql -- +-- !sql_bitmap_or_count3 -- 5 --- !sql -- +-- !sql_bitmap_or_count4 -- 6 --- !sql -- -\N +-- !sql_bitmap_or_count5 -- +6 -- !sql -- 2 diff --git a/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy b/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy index e544e2734c..8ca8227b2a 100644 --- a/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy +++ b/regression-test/suites/nereids_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy @@ -58,12 +58,239 @@ suite("test_bitmap_function") { qt_sql_bitmap_hash64_3 """ select bitmap_count(bitmap_hash64(null)) """ // BITMAP_OR - qt_sql """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt """ - qt_sql """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), NULL)) """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), bitmap_empty())) """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(10), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'))) """ + qt_sql_bitmap_or1 """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt """ + qt_sql_bitmap_or2 """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt """ + qt_sql_bitmap_or3 """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) """ + // TODO: fix constant fold of bitmap_or and enable this case + // qt_sql_bitmap_or4 """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), NULL)) """ + qt_sql_bitmap_or5 """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), bitmap_empty())) """ + qt_sql_bitmap_or6 """ select bitmap_to_string(bitmap_or(to_bitmap(10), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'))) """ + // TODO: fix constant fold of bitmap_or and enable this case + // qt_sql_bitmap_or7 """ select bitmap_count(bitmap_or(to_bitmap(1), null)) cnt """ + + // bitmap_or of all nullable column + sql """ DROP TABLE IF EXISTS test_bitmap_or1 """ + sql """ DROP TABLE IF EXISTS test_bitmap_or2 """ + sql """ + CREATE TABLE test_bitmap_or1 ( + dt INT(11) NULL, + id bitmap BITMAP_UNION NULL + ) ENGINE=OLAP + AGGREGATE KEY(dt) + DISTRIBUTED BY HASH(dt) BUCKETS 2 + properties ( + "replication_num" = "1" + ); + """ + sql """ + insert into + test_bitmap_or1 + values + (1, to_bitmap(11)), + (2, to_bitmap(22)), + (3, to_bitmap(33)), + (4, to_bitmap(44)); + """ + sql """ + CREATE TABLE test_bitmap_or2 ( + dt INT(11) NULL, + id bitmap BITMAP_UNION NULL + ) ENGINE=OLAP + AGGREGATE KEY(dt) + DISTRIBUTED BY HASH(dt) BUCKETS 2 + properties ( + "replication_num" = "1" + ); + """ + sql """ + insert into + test_bitmap_or2 + values + (1, to_bitmap(111)), + (2, to_bitmap(222)), + (5, to_bitmap(555)); + """ + qt_sql_bitmap_or8 """ + select + l.dt, + bitmap_count(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt, count + """ + qt_sql_bitmap_or_count6 """ + select + l.dt, + bitmap_or_count(l.id, r.id) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt, count + """ + qt_sql_bitmap_or9 """ + select + l.dt, + bitmap_count(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt, count + """ + qt_sql_bitmap_or_count7 """ + select + l.dt, + bitmap_or_count(l.id, r.id) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt, count + """ + qt_sql_bitmap_or10 """ + select + l.dt, + bitmap_to_string(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt + """ + qt_sql_bitmap_or11 """ + select + l.dt, + bitmap_to_string(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt + """ + + // bitmap_or of NOT NULLABLE column and nullable column + sql """ DROP TABLE IF EXISTS test_bitmap_or1 """ + sql """ + CREATE TABLE test_bitmap_or1 ( + dt INT(11) NULL, + id bitmap BITMAP_UNION NOT NULL + ) ENGINE=OLAP + AGGREGATE KEY(dt) + DISTRIBUTED BY HASH(dt) BUCKETS 2 + properties ( + "replication_num" = "1" + ); + """ + sql """ + insert into + test_bitmap_or1 + values + (1, to_bitmap(11)), + (2, to_bitmap(22)), + (3, to_bitmap(33)), + (4, to_bitmap(44)); + """ + qt_sql_bitmap_or12 """ + select + l.dt, + bitmap_count(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt, count + """ + qt_sql_bitmap_or_count8 """ + select + l.dt, + bitmap_or_count(l.id, r.id) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt, count + """ + qt_sql_bitmap_or13 """ + select + l.dt, + bitmap_count(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt, count + """ + qt_sql_bitmap_or_count9 """ + select + l.dt, + bitmap_or_count(l.id, r.id) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt, count + """ + qt_sql_bitmap_or14 """ + select + l.dt, + bitmap_to_string(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt + """ + qt_sql_bitmap_or15 """ + select + l.dt, + bitmap_to_string(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt + """ + + qt_sql_bitmap_or16_0 """ select bitmap_from_string("1") is null """ + qt_sql_bitmap_or16_1 """ select bitmap_from_string("a") is null """ + qt_sql_bitmap_or16 """ select bitmap_or(bitmap_from_string("a"), bitmap_from_string("b")) is null""" + qt_sql_bitmap_or17 """ select bitmap_count(bitmap_or(bitmap_from_string("a"), bitmap_from_string("b"))) """ + qt_sql_bitmap_or_count10 """ select bitmap_or_count(bitmap_from_string("a"), bitmap_from_string("b")) """ + qt_sql_bitmap_or18 """ select bitmap_to_string(bitmap_or(bitmap_from_string("a"), bitmap_from_string("b"))) """ + qt_sql_bitmap_or19 """ select bitmap_or(null, null) is null""" + // qt_sql_bitmap_or20 """ select bitmap_count(bitmap_or(null, null))""" + qt_sql_bitmap_or21 """ select bitmap_to_string(bitmap_or(null, null))""" + + sql """ drop view if exists v1 """ + sql """ drop view if exists v2 """ + sql """ + create view v1 as + (select + l.dt ldt, + l.id lid, + r.dt rdt, + r.id rid + from + test_bitmap_or1 l + left join test_bitmap_or2 r on l.dt = r.dt + where r.id is null); + """ + sql """ + create view v2 as + (select + l.dt ldt, + l.id lid, + r.dt rdt, + r.id rid + from + test_bitmap_or1 l + right join test_bitmap_or2 r on l.dt = r.dt + where l.id is null); + """ + + // test bitmap_or of all non-const null column values + qt_sql_bitmap_or22_0 """ select ldt, bitmap_count(lid), bitmap_count(rid) from v1 where rid is null order by ldt; """ + qt_sql_bitmap_or22_1 """ select rdt, bitmap_count(lid), bitmap_count(rid) from v2 where lid is null order by rdt; """ + qt_sql_bitmap_or22 """ select v1.ldt, v1.rdt, v2.ldt, v2.rdt, bitmap_or(v1.rid, v2.lid) is null from v1, v2 order by v1.ldt, v2.rdt; """ + qt_sql_bitmap_or_count11 """ select v1.ldt, v1.rdt, v2.ldt, v2.rdt, bitmap_or_count(v1.rid, v2.lid) from v1, v2 order by v1.ldt, v2.rdt; """ + qt_sql_bitmap_or23 """ select v1.ldt, v1.rdt, v2.ldt, v2.rdt, bitmap_to_string(bitmap_or(v1.rid, v2.lid)) from v1, v2 order by v1.ldt, v2.rdt; """ // bitmap_and_count qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ @@ -74,11 +301,12 @@ suite("test_bitmap_function") { qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'), NULL) """ // bitmap_or_count - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3'))""" - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """ - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), bitmap_empty()) """ - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), NULL) """ + qt_sql_bitmap_or_count1 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ + qt_sql_bitmap_or_count2 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3'))""" + qt_sql_bitmap_or_count3 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """ + qt_sql_bitmap_or_count4 """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), bitmap_empty()) """ + // TODO: fix constant fold of bitmap_or and enable this case + // qt_sql_bitmap_or_count5 """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), NULL) """ // BITMAP_XOR qt_sql """ select bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt """ diff --git a/regression-test/suites/nereids_syntax_p0/test_bitmap_function_nereids.groovy b/regression-test/suites/nereids_syntax_p0/test_bitmap_function_nereids.groovy index 311b7eba69..0ca43b64a2 100644 --- a/regression-test/suites/nereids_syntax_p0/test_bitmap_function_nereids.groovy +++ b/regression-test/suites/nereids_syntax_p0/test_bitmap_function_nereids.groovy @@ -57,12 +57,13 @@ suite("test_bitmap_function_nereids") { qt_sql_bitmap_hash64_3 """ select bitmap_count(bitmap_hash64(null)) """ // BITMAP_OR - qt_sql """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt """ - qt_sql """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), NULL)) """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), bitmap_empty())) """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(10), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'))) """ + qt_sql_bitmap_or1 """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt """ + qt_sql_bitmap_or2 """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt """ + qt_sql_bitmap_or3 """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) """ + // TODO: fix constant fold of bitmap_or and enable this case + // qt_sql_bitmap_or4 """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), NULL)) """ + qt_sql_bitmap_or5 """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), bitmap_empty())) """ + qt_sql_bitmap_or6 """ select bitmap_to_string(bitmap_or(to_bitmap(10), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'))) """ // bitmap_and_count qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ @@ -73,11 +74,12 @@ suite("test_bitmap_function_nereids") { qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'), NULL) """ // bitmap_or_count - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3'))""" - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """ - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), bitmap_empty()) """ - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), NULL) """ + qt_sql_bitmap_or_count1 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ + qt_sql_bitmap_or_count2 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3'))""" + qt_sql_bitmap_or_count3 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """ + qt_sql_bitmap_or_count4 """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), bitmap_empty()) """ + // TODO: fix constant fold of bitmap_or and enable this case + // qt_sql_bitmap_or_count5 """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), NULL) """ // BITMAP_XOR qt_sql """ select bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt """ diff --git a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy index d31d96bd1a..b958871c94 100644 --- a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy @@ -55,12 +55,237 @@ suite("test_bitmap_function") { qt_sql_bitmap_hash64_3 """ select bitmap_count(bitmap_hash64(null)) """ // BITMAP_OR - qt_sql """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt """ - qt_sql """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), NULL)) """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), bitmap_empty())) """ - qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(10), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'))) """ + qt_sql_bitmap_or1 """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt """ + qt_sql_bitmap_or2 """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt """ + qt_sql_bitmap_or3 """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) """ + qt_sql_bitmap_or4 """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), NULL)) """ + qt_sql_bitmap_or5 """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), bitmap_empty())) """ + qt_sql_bitmap_or6 """ select bitmap_to_string(bitmap_or(to_bitmap(10), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'))) """ + qt_sql_bitmap_or7 """ select bitmap_count(bitmap_or(to_bitmap(1), null)) cnt """ + + // bitmap_or of all nullable column + sql """ DROP TABLE IF EXISTS test_bitmap_or1 """ + sql """ DROP TABLE IF EXISTS test_bitmap_or2 """ + sql """ + CREATE TABLE test_bitmap_or1 ( + dt INT(11) NULL, + id bitmap BITMAP_UNION NULL + ) ENGINE=OLAP + AGGREGATE KEY(dt) + DISTRIBUTED BY HASH(dt) BUCKETS 2 + properties ( + "replication_num" = "1" + ); + """ + sql """ + insert into + test_bitmap_or1 + values + (1, to_bitmap(11)), + (2, to_bitmap(22)), + (3, to_bitmap(33)), + (4, to_bitmap(44)); + """ + sql """ + CREATE TABLE test_bitmap_or2 ( + dt INT(11) NULL, + id bitmap BITMAP_UNION NULL + ) ENGINE=OLAP + AGGREGATE KEY(dt) + DISTRIBUTED BY HASH(dt) BUCKETS 2 + properties ( + "replication_num" = "1" + ); + """ + sql """ + insert into + test_bitmap_or2 + values + (1, to_bitmap(111)), + (2, to_bitmap(222)), + (5, to_bitmap(555)); + """ + qt_sql_bitmap_or8 """ + select + l.dt, + bitmap_count(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt, count + """ + qt_sql_bitmap_or_count6 """ + select + l.dt, + bitmap_or_count(l.id, r.id) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt, count + """ + qt_sql_bitmap_or9 """ + select + l.dt, + bitmap_count(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt, count + """ + qt_sql_bitmap_or_count7 """ + select + l.dt, + bitmap_or_count(l.id, r.id) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt, count + """ + qt_sql_bitmap_or10 """ + select + l.dt, + bitmap_to_string(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt + """ + qt_sql_bitmap_or11 """ + select + l.dt, + bitmap_to_string(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt + """ + + // bitmap_or of NOT NULLABLE column and nullable column + sql """ DROP TABLE IF EXISTS test_bitmap_or1 """ + sql """ + CREATE TABLE test_bitmap_or1 ( + dt INT(11) NULL, + id bitmap BITMAP_UNION NOT NULL + ) ENGINE=OLAP + AGGREGATE KEY(dt) + DISTRIBUTED BY HASH(dt) BUCKETS 2 + properties ( + "replication_num" = "1" + ); + """ + sql """ + insert into + test_bitmap_or1 + values + (1, to_bitmap(11)), + (2, to_bitmap(22)), + (3, to_bitmap(33)), + (4, to_bitmap(44)); + """ + qt_sql_bitmap_or12 """ + select + l.dt, + bitmap_count(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt, count + """ + qt_sql_bitmap_or_count8 """ + select + l.dt, + bitmap_or_count(l.id, r.id) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt, count + """ + qt_sql_bitmap_or13 """ + select + l.dt, + bitmap_count(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt, count + """ + qt_sql_bitmap_or_count9 """ + select + l.dt, + bitmap_or_count(l.id, r.id) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt, count + """ + qt_sql_bitmap_or14 """ + select + l.dt, + bitmap_to_string(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + order by l.dt + """ + qt_sql_bitmap_or15 """ + select + l.dt, + bitmap_to_string(bitmap_or(l.id, r.id)) count + from + test_bitmap_or1 l left join test_bitmap_or2 r + on l.dt = r.dt + where r.id is not null + order by l.dt + """ + + qt_sql_bitmap_or16_0 """ select bitmap_from_string("1") is null """ + qt_sql_bitmap_or16_1 """ select bitmap_from_string("a") is null """ + qt_sql_bitmap_or16 """ select bitmap_or(bitmap_from_string("a"), bitmap_from_string("b")) is null""" + qt_sql_bitmap_or17 """ select bitmap_count(bitmap_or(bitmap_from_string("a"), bitmap_from_string("b"))) """ + qt_sql_bitmap_or_count10 """ select bitmap_or_count(bitmap_from_string("a"), bitmap_from_string("b")) """ + qt_sql_bitmap_or18 """ select bitmap_to_string(bitmap_or(bitmap_from_string("a"), bitmap_from_string("b"))) """ + qt_sql_bitmap_or19 """ select bitmap_or(null, null) is null""" + // qt_sql_bitmap_or20 """ select bitmap_count(bitmap_or(null, null))""" + qt_sql_bitmap_or21 """ select bitmap_to_string(bitmap_or(null, null))""" + + sql """ drop view if exists v1 """ + sql """ drop view if exists v2 """ + sql """ + create view v1 as + (select + l.dt ldt, + l.id lid, + r.dt rdt, + r.id rid + from + test_bitmap_or1 l + left join test_bitmap_or2 r on l.dt = r.dt + where r.id is null); + """ + sql """ + create view v2 as + (select + l.dt ldt, + l.id lid, + r.dt rdt, + r.id rid + from + test_bitmap_or1 l + right join test_bitmap_or2 r on l.dt = r.dt + where l.id is null); + """ + + // test bitmap_or of all non-const null column values + qt_sql_bitmap_or22_0 """ select ldt, bitmap_count(lid), bitmap_count(rid) from v1 where rid is null order by ldt; """ + qt_sql_bitmap_or22_1 """ select rdt, bitmap_count(lid), bitmap_count(rid) from v2 where lid is null order by rdt; """ + qt_sql_bitmap_or22 """ select v1.ldt, v1.rdt, v2.ldt, v2.rdt, bitmap_or(v1.rid, v2.lid) is null from v1, v2 order by v1.ldt, v2.rdt; """ + qt_sql_bitmap_or_count11 """ select v1.ldt, v1.rdt, v2.ldt, v2.rdt, bitmap_or_count(v1.rid, v2.lid) from v1, v2 order by v1.ldt, v2.rdt; """ + qt_sql_bitmap_or23 """ select v1.ldt, v1.rdt, v2.ldt, v2.rdt, bitmap_to_string(bitmap_or(v1.rid, v2.lid)) from v1, v2 order by v1.ldt, v2.rdt; """ // bitmap_and_count qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ @@ -71,11 +296,11 @@ suite("test_bitmap_function") { qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'), NULL) """ // bitmap_or_count - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3'))""" - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """ - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), bitmap_empty()) """ - qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), NULL) """ + qt_sql_bitmap_or_count1 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_empty()) """ + qt_sql_bitmap_or_count2 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3'))""" + qt_sql_bitmap_or_count3 """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """ + qt_sql_bitmap_or_count4 """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), bitmap_empty()) """ + qt_sql_bitmap_or_count5 """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), NULL) """ // BITMAP_XOR qt_sql """ select bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt """