From a1808c1a71b69d625303e5dccf961fba3c63634d Mon Sep 17 00:00:00 2001 From: HappenLee Date: Sun, 7 Feb 2021 22:39:17 +0800 Subject: [PATCH] [Function] Add BE udf bitmap_not (#5346) (#5357) this function will return the not result of inputs two bitmap. --- be/src/exprs/bitmap_function.cpp | 19 +++++++ be/src/exprs/bitmap_function.h | 1 + be/src/util/bitmap_value.h | 41 ++++++++++++++ be/test/exprs/bitmap_function_test.cpp | 39 +++++++++++++ docs/.vuepress/sidebar/en.js | 2 + docs/.vuepress/sidebar/zh-CN.js | 2 + .../bitmap-functions/bitmap_not.md | 55 +++++++++++++++++++ .../bitmap-functions/bitmap_not.md | 55 +++++++++++++++++++ gensrc/script/doris_builtins_functions.py | 2 + 9 files changed, 216 insertions(+) create mode 100644 docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_not.md create mode 100644 docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_not.md diff --git a/be/src/exprs/bitmap_function.cpp b/be/src/exprs/bitmap_function.cpp index f0c0e6ff6d..9e0cb9815e 100644 --- a/be/src/exprs/bitmap_function.cpp +++ b/be/src/exprs/bitmap_function.cpp @@ -497,6 +497,25 @@ StringVal BitmapFunctions::bitmap_xor(FunctionContext* ctx, const StringVal& lhs return serialize(ctx, &bitmap); } +StringVal BitmapFunctions::bitmap_not(FunctionContext* ctx, const StringVal& lhs, + const StringVal& rhs) { + if (lhs.is_null || rhs.is_null) { + return StringVal::null(); + } + BitmapValue bitmap; + if (lhs.len == 0) { + bitmap |= *reinterpret_cast(lhs.ptr); + } else { + bitmap |= BitmapValue((char*)lhs.ptr); + } + + if (rhs.len == 0) { + bitmap -= *reinterpret_cast(rhs.ptr); + } else { + bitmap -= BitmapValue((char*)rhs.ptr); + } + return serialize(ctx, &bitmap); +} StringVal BitmapFunctions::bitmap_to_string(FunctionContext* ctx, const StringVal& input) { if (input.is_null) { diff --git a/be/src/exprs/bitmap_function.h b/be/src/exprs/bitmap_function.h index 21fbd89791..5fd0d9613a 100644 --- a/be/src/exprs/bitmap_function.h +++ b/be/src/exprs/bitmap_function.h @@ -62,6 +62,7 @@ public: static StringVal bitmap_or(FunctionContext* ctx, const StringVal& src, const StringVal& dst); static StringVal bitmap_xor(FunctionContext* ctx, const StringVal& src, const StringVal& dst); static StringVal bitmap_and(FunctionContext* ctx, const StringVal& src, const StringVal& dst); + static StringVal bitmap_not(FunctionContext* ctx, const StringVal& src, const StringVal& dst); static StringVal bitmap_to_string(FunctionContext* ctx, const StringVal& input); // Convert a comma separated string to a Bitmap // Example: diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index f1ff0d1fbd..a187e00dc8 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1006,6 +1006,47 @@ public: } } + void remove(uint64_t value) { + switch (_type) { + case EMPTY: + break; + case SINGLE: + //there is need to convert the type if two variables are equal + if (_sv == value) { + _type = EMPTY; + } + break; + case BITMAP: + _bitmap.remove(value); + _convert_to_smaller_type(); + } + } + + // Compute the union between the current bitmap and the provided bitmap. + BitmapValue& operator-=(const BitmapValue& rhs) { + switch (rhs._type) { + case EMPTY: + break; + case SINGLE: + remove(rhs._sv); + break; + case BITMAP: + switch (_type) { + case EMPTY: + break; + case SINGLE: + remove(rhs._sv); + break; + case BITMAP: + _bitmap -= rhs._bitmap; + _convert_to_smaller_type(); + break; + } + break; + } + return *this; + } + // Compute the union between the current bitmap and the provided bitmap. // Possible type transitions are: // EMPTY -> SINGLE diff --git a/be/test/exprs/bitmap_function_test.cpp b/be/test/exprs/bitmap_function_test.cpp index 62ab0431f3..6c4a11cb31 100644 --- a/be/test/exprs/bitmap_function_test.cpp +++ b/be/test/exprs/bitmap_function_test.cpp @@ -321,6 +321,45 @@ TEST_F(BitmapFunctionsTest, bitmap_and) { BigIntVal expected(1); ASSERT_EQ(expected, result); } + +TEST_F(BitmapFunctionsTest, bitmap_not) { + // result is bitmap + BitmapValue bitmap1({1024, 1, 2019}); + BitmapValue bitmap2({33, 44, 2019}); + + StringVal bitmap_src = convert_bitmap_to_string(ctx, bitmap1); + StringVal bitmap_dst = convert_bitmap_to_string(ctx, bitmap2); + + StringVal bitmap_str = BitmapFunctions::bitmap_not(ctx, bitmap_src, bitmap_dst); + BigIntVal result = BitmapFunctions::bitmap_count(ctx, bitmap_str); + BigIntVal expected(2); + ASSERT_EQ(expected, result); + + // result is single + bitmap1 = BitmapValue({1024, 1, 2019}); + bitmap2 = BitmapValue({33, 1024, 2019}); + + bitmap_src = convert_bitmap_to_string(ctx, bitmap1); + bitmap_dst = convert_bitmap_to_string(ctx, bitmap2); + + bitmap_str = BitmapFunctions::bitmap_not(ctx, bitmap_src, bitmap_dst); + result = BitmapFunctions::bitmap_count(ctx, bitmap_str); + expected = BigIntVal(1); + ASSERT_EQ(expected, result); + + // result is empty + bitmap1 = BitmapValue({1024, 1, 2019}); + bitmap2 = BitmapValue({1, 1024, 2019}); + + bitmap_src = convert_bitmap_to_string(ctx, bitmap1); + bitmap_dst = convert_bitmap_to_string(ctx, bitmap2); + + bitmap_str = BitmapFunctions::bitmap_not(ctx, bitmap_src, bitmap_dst); + result = BitmapFunctions::bitmap_count(ctx, bitmap_str); + expected = BigIntVal(0); + ASSERT_EQ(expected, result); +} + TEST_F(BitmapFunctionsTest, bitmap_contains) { BitmapValue bitmap({4, 5}); StringVal bitmap_str = convert_bitmap_to_string(ctx, bitmap); diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js index bbbc2c63c0..ec68f3a47b 100644 --- a/docs/.vuepress/sidebar/en.js +++ b/docs/.vuepress/sidebar/en.js @@ -337,6 +337,8 @@ module.exports = [ "bitmap_has_any", "bitmap_hash", "bitmap_or", + "bitmap_xor", + "bitmap_not", "bitmap_to_string", "to_bitmap", "bitmap_intersect", diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js index 9a4b8d7e19..b6e0584bf4 100644 --- a/docs/.vuepress/sidebar/zh-CN.js +++ b/docs/.vuepress/sidebar/zh-CN.js @@ -343,6 +343,8 @@ module.exports = [ "bitmap_has_any", "bitmap_hash", "bitmap_or", + "bitmap_xor", + "bitmap_not", "bitmap_to_string", "to_bitmap", "bitmap_intersect", diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_not.md b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_not.md new file mode 100644 index 0000000000..9ec99383ed --- /dev/null +++ b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_not.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_not", + "language": "en" +} +--- + + + +# bitmap_not +## description +### Syntax + +`BITMAP BITMAP_NOT(BITMAP lhs, BITMAP rhs)` + +Calculate the set after lhs minus rhs, return the new bitmap. + +## example + +``` +mysql> select bitmap_count(bitmap_not(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt; ++------+ +| cnt | ++------+ +| 0 | ++------+ + +mysql> select bitmap_to_string(bitmap_not(bitmap_from_string('2,3,5'),bitmap_from_string('1,2,3,4'))); ++----------------------------------------------------------------------------------------+ +| bitmap_to_string(bitmap_xor(bitmap_from_string('2,3,5'), bitmap_from_string('1,2,3,4'))) | ++----------------------------------------------------------------------------------------+ +| 5 | ++----------------------------------------------------------------------------------------+ +``` + +## keyword + + BITMAP_NOT,BITMAP diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_not.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_not.md new file mode 100644 index 0000000000..28838941d3 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_not.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_not", + "language": "zh-CN" +} +--- + + + +# bitmap_not +## description +### Syntax + +`BITMAP BITMAP_NOT(BITMAP lhs, BITMAP rhs)` + +计算lhs减去rhs之后的集合，返回新的bitmap. + +## example + +``` +mysql> select bitmap_count(bitmap_not(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt; ++------+ +| cnt | ++------+ +| 0 | ++------+ + +mysql> select bitmap_to_string(bitmap_not(bitmap_from_string('2,3,5'),bitmap_from_string('1,2,3,4'))); ++----------------------------------------------------------------------------------------+ +| bitmap_to_string(bitmap_xor(bitmap_from_string('2,3,5'), bitmap_from_string('1,2,3,4'))) | ++----------------------------------------------------------------------------------------+ +| 5 | ++----------------------------------------------------------------------------------------+ +``` + +## keyword + + BITMAP_NOT,BITMAP diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 67742ba9ac..26d4e3c07e 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -802,6 +802,8 @@ visible_functions = [ '_ZN5doris15BitmapFunctions9bitmap_orEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], [['bitmap_xor'], 'BITMAP', ['BITMAP','BITMAP'], '_ZN5doris15BitmapFunctions10bitmap_xorEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + [['bitmap_not'], 'BITMAP', ['BITMAP','BITMAP'], + '_ZN5doris15BitmapFunctions10bitmap_notEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], [['bitmap_and'], 'BITMAP', ['BITMAP','BITMAP'], '_ZN5doris15BitmapFunctions10bitmap_andEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], [['bitmap_to_string'], 'VARCHAR', ['BITMAP'],