From 5fe3342aa3e872175f5c2646fd1b414a86ecf47d Mon Sep 17 00:00:00 2001 From: zhangstar333 <87313068+zhangstar333@users.noreply.github.com> Date: Thu, 3 Nov 2022 14:29:28 +0800 Subject: [PATCH] [Vectorized](function) support bitmap_to_array function (#13926) --- be/src/util/bitmap_value.h | 21 +++++- be/src/vec/functions/function_bitmap.cpp | 54 +++++++++++++- .../bitmap-functions/bitmap_to_array.md | 70 +++++++++++++++++++ docs/sidebars.json | 1 + .../bitmap-functions/bitmap_to_array.md | 70 +++++++++++++++++++ gensrc/script/doris_builtins_functions.py | 1 + .../bitmap_functions/test_bitmap_function.out | 22 ++++-- .../test_bitmap_function.groovy | 4 +- 8 files changed, 234 insertions(+), 9 deletions(-) create mode 100644 docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap_to_array.md create mode 100644 docs/zh-CN/docs/sql-manual/sql-functions/bitmap-functions/bitmap_to_array.md diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 69e1233dd1..2cbc0c00f9 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -34,7 +34,8 @@ #include "common/logging.h" #include "udf/udf.h" #include "util/coding.h" - +#include "vec/common/pod_array.h" +#include "vec/common/pod_array_fwd.h" namespace doris { // serialized bitmap := TypeCode(1), Payload @@ -1745,6 +1746,24 @@ public: return count; } + //for function bitmap_to_array + void to_array(vectorized::PaddedPODArray& data) const { + switch (_type) { + case EMPTY: + break; + case SINGLE: { + data.emplace_back(_sv); + break; + } + case BITMAP: { + for (auto it = _bitmap.begin(); it != _bitmap.end(); ++it) { + data.emplace_back(*it); + } + break; + } + } + } + void clear() { _type = EMPTY; _bitmap.clear(); diff --git a/be/src/vec/functions/function_bitmap.cpp b/be/src/vec/functions/function_bitmap.cpp index 3db2c1033a..b4e6382d15 100644 --- a/be/src/vec/functions/function_bitmap.cpp +++ b/be/src/vec/functions/function_bitmap.cpp @@ -21,13 +21,15 @@ #include "gutil/strings/numbers.h" #include "gutil/strings/split.h" #include "util/string_parser.hpp" +#include "vec/columns/columns_number.h" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_number.h" #include "vec/functions/function_always_not_nullable.h" #include "vec/functions/function_bitmap_min_or_max.h" #include "vec/functions/function_const.h" #include "vec/functions/function_string.h" #include "vec/functions/function_totype.h" #include "vec/functions/simple_function_factory.h" - namespace doris::vectorized { struct BitmapEmpty { @@ -582,6 +584,55 @@ public: } }; +class FunctionBitmapToArray : public IFunction { +public: + static constexpr auto name = "bitmap_to_array"; + + String get_name() const override { return name; } + + static FunctionPtr create() { return std::make_shared(); } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + auto nested_type = make_nullable(std::make_shared()); + return std::make_shared(nested_type); + } + + size_t get_number_of_arguments() const override { return 1; } + + bool use_default_implementation_for_nulls() const override { return true; } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + size_t result, size_t input_rows_count) override { + auto return_nested_type = make_nullable(std::make_shared()); + auto dest_array_column_ptr = ColumnArray::create(return_nested_type->create_column(), + ColumnArray::ColumnOffsets::create()); + + IColumn* dest_nested_column = &dest_array_column_ptr->get_data(); + ColumnNullable* dest_nested_nullable_col = + reinterpret_cast(dest_nested_column); + dest_nested_column = dest_nested_nullable_col->get_nested_column_ptr(); + auto& dest_nested_null_map = dest_nested_nullable_col->get_null_map_column().get_data(); + + auto arg_col = + block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); + auto bitmap_col = assert_cast(arg_col.get()); + const auto& bitmap_col_data = bitmap_col->get_data(); + auto& nested_column_data = + assert_cast*>(dest_nested_column)->get_data(); + auto& dest_offsets = dest_array_column_ptr->get_offsets(); + dest_offsets.reserve(input_rows_count); + + for (int i = 0; i < input_rows_count; ++i) { + bitmap_col_data[i].to_array(nested_column_data); + dest_nested_null_map.resize_fill(nested_column_data.size(), 0); + dest_offsets.push_back(nested_column_data.size()); + } + + block.replace_by_position(result, std::move(dest_array_column_ptr)); + return Status::OK(); + } +}; + using FunctionBitmapEmpty = FunctionConst; using FunctionToBitmap = FunctionAlwaysNotNullable; using FunctionToBitmapWithCheck = FunctionAlwaysNotNullable; @@ -631,6 +682,7 @@ void register_function_bitmap(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); } } // namespace doris::vectorized diff --git a/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap_to_array.md b/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap_to_array.md new file mode 100644 index 0000000000..a369b73074 --- /dev/null +++ b/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap_to_array.md @@ -0,0 +1,70 @@ +--- +{ + "title": "bitmap_to_array", + "language": "en" +} +--- + + + +## bitmap_to_array + +### description +#### Syntax + +`ARRAY_BIGINT bitmap_to_array(BITMAP input)` + +Convert a input BITMAP to Array. +If input is null, return null. + +### example + +``` +mysql> select bitmap_to_array(null); ++------------------------+ +| bitmap_to_array(NULL) | ++------------------------+ +| NULL | ++------------------------+ + +mysql> select bitmap_to_array(bitmap_empty()); ++---------------------------------+ +| bitmap_to_array(bitmap_empty()) | ++---------------------------------+ +| [] | ++---------------------------------+ + +mysql> select bitmap_to_array(to_bitmap(1)); ++-------------------------------+ +| bitmap_to_array(to_bitmap(1)) | ++-------------------------------+ +| [1] | ++-------------------------------+ + +mysql> select bitmap_to_array(bitmap_from_string('1,2,3,4,5')); ++--------------------------------------------------+ +| bitmap_to_array(bitmap_from_string('1,2,3,4,5')) | ++--------------------------------------------------+ +| [1, 2, 3, 4, 5] | ++-------------------------------------------------- + +``` + +### keywords + + BITMAP_TO_ARRAY,BITMAP diff --git a/docs/sidebars.json b/docs/sidebars.json index 1f91df2d29..8aa6f4f6fd 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -459,6 +459,7 @@ "sql-manual/sql-functions/bitmap-functions/bitmap_hash", "sql-manual/sql-functions/bitmap-functions/bitmap_from_string", "sql-manual/sql-functions/bitmap-functions/bitmap_to_string", + "sql-manual/sql-functions/bitmap-functions/bitmap_to_array", "sql-manual/sql-functions/bitmap-functions/bitmap_empty", "sql-manual/sql-functions/bitmap-functions/bitmap_or", "sql-manual/sql-functions/bitmap-functions/bitmap_and", diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/bitmap-functions/bitmap_to_array.md b/docs/zh-CN/docs/sql-manual/sql-functions/bitmap-functions/bitmap_to_array.md new file mode 100644 index 0000000000..4ecd51de69 --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-functions/bitmap-functions/bitmap_to_array.md @@ -0,0 +1,70 @@ +--- +{ + "title": "bitmap_to_array", + "language": "zh-CN" +} +--- + + + +## bitmap_to_array + +### description +#### Syntax + +`ARRAY_BIGINT bitmap_to_array(BITMAP input)` + +将一个bitmap转化成一个array 数组。 +输入是null的话会返回null。 + +### example + +``` +mysql> select bitmap_to_array(null); ++------------------------+ +| bitmap_to_array(NULL) | ++------------------------+ +| NULL | ++------------------------+ + +mysql> select bitmap_to_array(bitmap_empty()); ++---------------------------------+ +| bitmap_to_array(bitmap_empty()) | ++---------------------------------+ +| [] | ++---------------------------------+ + +mysql> select bitmap_to_array(to_bitmap(1)); ++-------------------------------+ +| bitmap_to_array(to_bitmap(1)) | ++-------------------------------+ +| [1] | ++-------------------------------+ + +mysql> select bitmap_to_array(bitmap_from_string('1,2,3,4,5')); ++--------------------------------------------------+ +| bitmap_to_array(bitmap_from_string('1,2,3,4,5')) | ++--------------------------------------------------+ +| [1, 2, 3, 4, 5] | ++-------------------------------------------------- + +``` + +### keywords + + BITMAP_TO_ARRAY,BITMAP diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 940ee6b2f3..982506f3d4 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -2597,6 +2597,7 @@ visible_functions = [ [['sub_bitmap'], 'BITMAP', ['BITMAP', 'BIGINT', 'BIGINT'], '_ZN5doris15BitmapFunctions10sub_bitmapEPN9doris_udf15FunctionContextERKNS1_9StringValERKNS1_9BigIntValES9_', '', '', 'vec', 'ALWAYS_NULLABLE'], + [['bitmap_to_array'], 'ARRAY_BIGINT', ['BITMAP'], '', '', '', 'vec', ''], # quantile_function [['to_quantile_state'], 'QUANTILE_STATE', ['VARCHAR', 'FLOAT'], '_ZN5doris22QuantileStateFunctions17to_quantile_stateEPN9doris_udf15FunctionContextERKNS1_9StringValE', diff --git a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out index 6661a40fb8..74ea0e50ff 100644 --- a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out +++ b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out @@ -47,22 +47,22 @@ true -- !sql -- false --- !sql_bitmap_hash1 -- +-- !sql_bitmap_hash1 -- 1 --- !sql_bitmap_hash2 -- +-- !sql_bitmap_hash2 -- 1 --- !sql_bitmap_hash3 -- +-- !sql_bitmap_hash3 -- 0 --- !sql_bitmap_hash64_1 -- +-- !sql_bitmap_hash64_1 -- 1 --- !sql_bitmap_hash64_2 -- +-- !sql_bitmap_hash64_2 -- 1 --- !sql_bitmap_hash64_3 -- +-- !sql_bitmap_hash64_3 -- 0 -- !sql -- @@ -249,3 +249,13 @@ false -- !sql -- 0 +-- !sql -- +[1, 2, 3] +[1, 2, 3, 4, 5] + +-- !sql -- +[] + +-- !sql -- +[3, 4, 100, 200] + diff --git a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy index d13bffdc7c..b429b30668 100644 --- a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy @@ -183,5 +183,7 @@ suite("test_bitmap_function") { qt_sql """ select orthogonal_bitmap_intersect_count(members, tag_group, 1150000, 1150001, 390006) from ${arthogonalBitmapTable} where tag_group in ( 1150000, 1150001, 390006); """ qt_sql """ select orthogonal_bitmap_union_count(members) from ${arthogonalBitmapTable} where tag_group in ( 1150000, 1150001, 390006); """ - + qt_sql """ select bitmap_to_array(user_id) from ${intersectCountTable}; """ + qt_sql """ select bitmap_to_array(bitmap_empty()); """ + qt_sql """ select bitmap_to_array(bitmap_from_string('100,200,3,4')); """ }