database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[function](bitmap) support bitmap_to_base64 and bitmap_from_base64 (#23759)

Browse Source
This commit is contained in:
TengJianPing
2023-09-02 00:58:48 +08:00
committed by GitHub
parent e0efda1234
commit 75e2bc8a25
11 changed files with 627 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
be/test/vec/function/function_bitmap_test.cpp
View File

@ -14,12 +14,16 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include <stdint.h>
#include <cstdint>
#include <limits>
#include <numeric>
#include <string>
#include <vector>
#include "common/config.h"
#include "common/status.h"
#include "function_test_util.h"
#include "gtest/gtest_pred_impl.h"
@ -27,6 +31,7 @@
#include "testutil/any_type.h"
#include "util/bitmap_value.h"
#include "vec/core/types.h"
#include "vec/data_types/data_type_bitmap.h"
#include "vec/data_types/data_type_nullable.h"
#include "vec/data_types/data_type_number.h"
#include "vec/data_types/data_type_string.h"
@ -77,6 +82,172 @@ TEST(function_bitmap_test, function_bitmap_to_string_test) {
check_function<DataTypeString, true>(func_name, input_types, data_set);
}
namespace doris {
namespace config {
DECLARE_Bool(enable_set_in_bitmap_value);
}
} // namespace doris
TEST(function_bitmap_test, function_bitmap_to_base64) {
config::Register::Field field("bool", "enable_set_in_bitmap_value",
&config::enable_set_in_bitmap_value, "false", false);
config::Register::_s_field_map->insert(
std::make_pair(std::string("enable_set_in_bitmap_value"), field));
std::string func_name = "bitmap_to_base64";
InputTypeSet input_types = {TypeIndex::BitMap};
EXPECT_TRUE(config::set_config("enable_set_in_bitmap_value", "false", false, true).ok());
std::vector<uint64_t> bits32 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31, 32}; // SET_TYPE_THRESHOLD + 1
std::vector<uint64_t> bits64 {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
22, 23, 24, 25, 26, 27, 28, 29, 30, 31, (uint64_t)4294967296}; // SET_TYPE_THRESHOLD + 1
BitmapValue bitmap32_1(1); // single
BitmapValue bitmap32_2({1, 9999999}); // bitmap
BitmapValue bitmap32_3(bits32); // bitmap
BitmapValue bitmap64_1((uint64_t)4294967296); // single
BitmapValue bitmap64_2({1, (uint64_t)4294967296}); // bitmap
BitmapValue bitmap64_3(bits64); // bitmap
BitmapValue empty_bitmap;
EXPECT_EQ(bitmap32_1.get_type_code(), BitmapTypeCode::SINGLE32);
EXPECT_EQ(bitmap32_2.get_type_code(), BitmapTypeCode::BITMAP32);
EXPECT_EQ(bitmap32_3.get_type_code(), BitmapTypeCode::BITMAP32);
EXPECT_EQ(bitmap64_1.get_type_code(), BitmapTypeCode::SINGLE64);
EXPECT_EQ(bitmap64_2.get_type_code(), BitmapTypeCode::BITMAP64);
EXPECT_EQ(bitmap64_3.get_type_code(), BitmapTypeCode::BITMAP64);
DataSet data_set = {
{{&bitmap32_1}, std::string("AQEAAAA=")},
{{&bitmap32_2}, std::string("AjowAAACAAAAAAAAAJgAAAAYAAAAGgAAAAEAf5Y=")},
{{&bitmap32_3}, std::string("AjswAAABAAAgAAEAAAAgAA==")},
{{&bitmap64_1}, std::string("AwAAAAABAAAA")},
{{&bitmap64_2},
std::string("BAIAAAAAOjAAAAEAAAAAAAAAEAAAAAEAAQAAADowAAABAAAAAAAAABAAAAAAAA==")},
{{&bitmap64_3},
std::string("BAIAAAAAOzAAAAEAAB8AAQAAAB8AAQAAADowAAABAAAAAAAAABAAAAAAAA==")},
{{&empty_bitmap}, std::string("AA==")},
{{Null()}, Null()}};
check_function<DataTypeString, true>(func_name, input_types, data_set);
EXPECT_TRUE(config::set_config("enable_set_in_bitmap_value", "true", false, true).ok());
bitmap32_1 = BitmapValue(1); // single
bitmap32_2 = BitmapValue({1, 9999999}); // set
bitmap32_3 = BitmapValue(bits32); // bitmap
bitmap64_1 = BitmapValue((uint64_t)4294967296); // single
bitmap64_2 = BitmapValue({1, (uint64_t)4294967296}); // set
bitmap64_3 = BitmapValue(bits64); // bitmap
EXPECT_EQ(bitmap32_1.get_type_code(), BitmapTypeCode::SINGLE32);
EXPECT_EQ(bitmap32_2.get_type_code(), BitmapTypeCode::SET);
EXPECT_EQ(bitmap32_3.get_type_code(), BitmapTypeCode::BITMAP32);
EXPECT_EQ(bitmap64_1.get_type_code(), BitmapTypeCode::SINGLE64);
EXPECT_EQ(bitmap64_2.get_type_code(), BitmapTypeCode::SET);
EXPECT_EQ(bitmap64_3.get_type_code(), BitmapTypeCode::BITMAP64);
DataSet data_set2 = {
{{&bitmap32_1}, std::string("AQEAAAA=")},
{{&bitmap32_2}, std::string("BQIBAAAAAAAAAH+WmAAAAAAA")},
{{&bitmap32_3}, std::string("AjswAAABAAAgAAEAAAAgAA==")},
{{&bitmap64_1}, std::string("AwAAAAABAAAA")},
{{&bitmap64_2}, std::string("BQIAAAAAAQAAAAEAAAAAAAAA")},
{{&bitmap64_3},
std::string("BAIAAAAAOzAAAAEAAB8AAQAAAB8AAQAAADowAAABAAAAAAAAABAAAAAAAA==")},
{{&empty_bitmap}, std::string("AA==")},
{{Null()}, Null()}};
check_function<DataTypeString, true>(func_name, input_types, data_set2);
}
TEST(function_bitmap_test, function_bitmap_from_base64) {
config::Register::Field field("bool", "enable_set_in_bitmap_value",
&config::enable_set_in_bitmap_value, "false", false);
config::Register::_s_field_map->insert(
std::make_pair(std::string("enable_set_in_bitmap_value"), field));
std::string func_name = "bitmap_from_base64";
InputTypeSet input_types = {TypeIndex::String};
EXPECT_TRUE(config::set_config("enable_set_in_bitmap_value", "false", false, true).ok());
std::string bitmap32_base64_1("AQEAAAA=");
std::string bitmap32_base64_2("AjowAAACAAAAAAAAAJgAAAAYAAAAGgAAAAEAf5Y=");
std::string bitmap32_base64_3("AjswAAABAAAgAAEAAAAgAA==");
std::string bitmap64_base64_1("AwAAAAABAAAA");
std::string bitmap64_base64_2(
"BAIAAAAAOjAAAAEAAAAAAAAAEAAAAAEAAQAAADowAAABAAAAAAAAABAAAAAAAA==");
std::string bitmap64_base64_3("BAIAAAAAOzAAAAEAAB8AAQAAAB8AAQAAADowAAABAAAAAAAAABAAAAAAAA==");
std::string base64_empty("AA==");
std::vector<uint64_t> bits32 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31, 32}; // SET_TYPE_THRESHOLD + 1
std::vector<uint64_t> bits64 {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
22, 23, 24, 25, 26, 27, 28, 29, 30, 31, (uint64_t)4294967296}; // SET_TYPE_THRESHOLD + 1
BitmapValue bitmap32_1(1); // single
BitmapValue bitmap32_2({1, 9999999}); // bitmap
BitmapValue bitmap32_3(bits32); // bitmap
BitmapValue bitmap64_1((uint64_t)4294967296); // single
BitmapValue bitmap64_2({1, (uint64_t)4294967296}); // bitmap
BitmapValue bitmap64_3(bits64); // bitmap
BitmapValue empty_bitmap;
DataSet data_set = {{{bitmap32_base64_1}, bitmap32_1}, {{bitmap32_base64_2}, bitmap32_2},
{{bitmap32_base64_3}, bitmap32_3}, {{bitmap64_base64_1}, bitmap64_1},
{{bitmap64_base64_2}, bitmap64_2}, {{bitmap64_base64_3}, bitmap64_3},
{{base64_empty}, empty_bitmap}, {{Null()}, Null()}};
check_function<DataTypeBitMap, true>(func_name, input_types, data_set);
EXPECT_TRUE(config::set_config("enable_set_in_bitmap_value", "true", false, true).ok());
bitmap32_base64_1 = ("AQEAAAA=");
bitmap32_base64_2 = ("BQIBAAAAAAAAAH");
bitmap32_base64_3 = ("AjswAAABAAAgAAEAAAAgAA==");
bitmap64_base64_1 = ("AwAAAAABAAAA");
bitmap64_base64_2 = ("BQIAAAAAAQAAAAEAAAAAAAAA");
bitmap64_base64_3 = ("BAIAAAAAOzAAAAEAAB8AAQAAAB8AAQAAADowAAABAAAAAAAAABAAAAAAAA==");
check_function<DataTypeBitMap, true>(func_name, input_types, data_set);
/* sr
mysql [(none)]>select bitmap_to_base64(bitmap_from_string("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32"));
+----------------------------------------------------------------------------------------------------------------------------------+
| bitmap_to_base64(bitmap_from_string('0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32')) |
+----------------------------------------------------------------------------------------------------------------------------------+
| AjowAAABAAAAAAAgABAAAAAAAAEAAgADAAQABQAGAAcACAAJAAoACwAMAA0ADgAPABAAEQASABMAFAAVABYAFwAYABkAGgAbABwAHQAeAB8AIAA= |
+----------------------------------------------------------------------------------------------------------------------------------+
mysql [(none)]>select bitmap_to_base64(bitmap_from_string("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,4294967296"));
+--------------------------------------------------------------------------------------------------------------------------------------------------+
| bitmap_to_base64(bitmap_from_string('0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,4294967296')) |
+--------------------------------------------------------------------------------------------------------------------------------------------------+
| BAIAAAAAOjAAAAEAAAAAAB8AEAAAAAAAAQACAAMABAAFAAYABwAIAAkACgALAAwADQAOAA8AEAARABIAEwAUABUAFgAXABgAGQAaABsAHAAdAB4AHwABAAAAOjAAAAEAAAAAAAAAEAAAAAAA |
+--------------------------------------------------------------------------------------------------------------------------------------------------+
*/
bitmap32_base64_3 =
("AjowAAABAAAAAAAgABAAAAAAAAEAAgADAAQABQAGAAcACAAJAAoACwAMAA0ADgAPABAAEQASABMAFAAVABYAF"
"wAYABkAGgAbABwAHQAeAB8AIAA=");
bitmap64_base64_3 =
("BAIAAAAAOjAAAAEAAAAAAB8AEAAAAAAAAQACAAMABAAFAAYABwAIAAkACgALAAwADQAOAA8AEAARABIAEwAUA"
"BUAFgAXABgAGQAaABsAHAAdAB4AHwABAAAAOjAAAAEAAAAAAAAAEAAAAAAA");
data_set = {{{bitmap32_base64_3}, bitmap32_3}, {{bitmap64_base64_3}, bitmap64_3}};
check_function<DataTypeBitMap, true>(func_name, input_types, data_set);
}
TEST(function_bitmap_test, function_bitmap_and_count) {
std::string func_name = "bitmap_and_count";
InputTypeSet input_types = {TypeIndex::BitMap, TypeIndex::BitMap};