diff --git a/be/src/exprs/bitmap_function.cpp b/be/src/exprs/bitmap_function.cpp index bc03ab24e3..52231b92f8 100644 --- a/be/src/exprs/bitmap_function.cpp +++ b/be/src/exprs/bitmap_function.cpp @@ -22,6 +22,226 @@ #include "util/string_parser.hpp" namespace doris { + +namespace detail { + +const int DATETIME_PACKED_TIME_BYTE_SIZE = 8; +const int DATETIME_TYPE_BYTE_SIZE = 4; + +const int DECIMAL_BYTE_SIZE = 16; + +// get_val start +template +T get_val(const ValType& x) { + DCHECK(!x.is_null); + return x.val; +} + +template<> +StringValue get_val(const StringVal& x) { + DCHECK(!x.is_null); + return StringValue::from_string_val(x); +} + +template<> +DateTimeValue get_val(const DateTimeVal& x) { + return DateTimeValue::from_datetime_val(x); +} + +template<> +DecimalV2Value get_val(const DecimalV2Val& x) { + return DecimalV2Value::from_decimal_val(x); +} +// get_val end + +// serialize_size start +template +int32_t serialize_size(const T& v) { + return sizeof(T); +} + +template<> +int32_t serialize_size(const DateTimeValue& v) { + return DATETIME_PACKED_TIME_BYTE_SIZE + DATETIME_TYPE_BYTE_SIZE; +} + +template<> +int32_t serialize_size(const DecimalV2Value& v) { + return DECIMAL_BYTE_SIZE; +} + +template<> +int32_t serialize_size(const StringValue& v) { + return v.len + 4; +} +// serialize_size end + +// write_to start +template +char* write_to(const T& v, char* dest) { + size_t type_size = sizeof(T); + memcpy(dest, &v, type_size); + dest += type_size; + return dest; +} + +template<> +char* write_to(const DateTimeValue& v, char* dest) { + DateTimeVal value; + v.to_datetime_val(&value); + *(int64_t*)dest = value.packed_time; + dest += DATETIME_PACKED_TIME_BYTE_SIZE; + *(int*)dest = value.type; + dest += DATETIME_TYPE_BYTE_SIZE; + return dest; +} + +template<> +char* write_to(const DecimalV2Value& v, char* dest) { + __int128 value = v.value(); + memcpy(dest, &value, DECIMAL_BYTE_SIZE); + dest += DECIMAL_BYTE_SIZE; + return dest; +} + +template<> +char* write_to(const StringValue& v, char* dest) { + *(int32_t*)dest = v.len; + dest += 4; + memcpy(dest, v.ptr, v.len); + dest += v.len; + return dest; +} +// write_to end + +// read_from start +template +void read_from(const char** src, T* result) { + size_t type_size = sizeof(T); + memcpy(result, *src, type_size); + *src += type_size; +} + +template<> +void read_from(const char** src, DateTimeValue* result) { + DateTimeVal value; + value.is_null = false; + value.packed_time = *(int64_t*)(*src); + *src += DATETIME_PACKED_TIME_BYTE_SIZE; + value.type = *(int*)(*src); + *src += DATETIME_TYPE_BYTE_SIZE; + *result = DateTimeValue::from_datetime_val(value);; +} + +template<> +void read_from(const char** src, DecimalV2Value* result) { + __int128 v = 0; + memcpy(&v, *src, DECIMAL_BYTE_SIZE); + *src += DECIMAL_BYTE_SIZE; + *result = DecimalV2Value(v); +} + +template<> +void read_from(const char** src, StringValue* result) { + int32_t length = *(int32_t*)(*src); + *src += 4; + *result = StringValue((char *)*src, length); + *src += length; +} +// read_from end + +} // namespace detail + +// Calculate the intersection of two or more bitmaps +// Usage: intersect_count(bitmap_column_to_count, filter_column, filter_values ...) +// Example: intersect_count(user_id, event, 'A', 'B', 'C'), meaning find the intersect count of user_id in all A/B/C 3 bitmaps +// Todo(kks) Use Array type instead of variable arguments +template +struct BitmapIntersect { +public: + BitmapIntersect() {} + + explicit BitmapIntersect(const char* src) { + deserialize(src); + } + + void add_key(const T key) { + RoaringBitmap empty_bitmap; + _bitmaps[key] = empty_bitmap; + } + + void update(const T& key, const RoaringBitmap& bitmap) { + if (_bitmaps.find(key) != _bitmaps.end()) { + _bitmaps[key].merge(bitmap); + } + } + + void merge(const BitmapIntersect& other) { + for (auto& kv: other._bitmaps) { + if (_bitmaps.find(kv.first) != _bitmaps.end()) { + _bitmaps[kv.first].merge(kv.second); + } else { + _bitmaps[kv.first] = kv.second; + } + } + } + + // calculate the intersection for _bitmaps's bitmap values + int64_t intersect_count() const { + if (_bitmaps.empty()) { + return 0; + } + + RoaringBitmap result; + auto it = _bitmaps.begin(); + result.merge(it->second); + it++; + for (;it != _bitmaps.end(); it++) { + result.intersect(it->second); + } + + return result.cardinality(); + } + + // the serialize size + size_t size() { + size_t size = 4; + for (auto& kv: _bitmaps) { + size += detail::serialize_size(kv.first);; + size += kv.second.size(); + } + return size; + } + + //must call size() first + void serialize(char* dest) { + char* writer = dest; + *(int32_t*)writer = _bitmaps.size(); + writer += 4; + for (auto& kv: _bitmaps) { + writer = detail::write_to(kv.first, writer); + kv.second.serialize(writer); + writer += kv.second.size(); + } + } + + void deserialize(const char* src) { + const char* reader = src; + int32_t bitmaps_size = *(int32_t*)reader; + reader += 4; + for (int32_t i = 0; i < bitmaps_size; i++) { + T key; + detail::read_from(&reader, &key); + RoaringBitmap bitmap(reader); + reader += bitmap.size(); + _bitmaps[key] = bitmap; + } + } + +private: + std::map _bitmaps; +}; + void BitmapFunctions::init() { } @@ -117,6 +337,58 @@ StringVal BitmapFunctions::bitmap_serialize(FunctionContext* ctx, const StringVa return result; } +template +void BitmapFunctions::bitmap_intersect_init(FunctionContext* ctx, StringVal* dst) { + dst->is_null = false; + dst->len = sizeof(BitmapIntersect); + auto intersect = new BitmapIntersect(); + + // constant args start from index 2 + for (int i = 2; i < ctx->get_num_constant_args(); ++i) { + DCHECK(ctx->is_arg_constant(i)); + ValType* arg = reinterpret_cast(ctx->get_constant_arg(i)); + intersect->add_key(detail::get_val(*arg)); + } + + dst->ptr = (uint8_t*)intersect; +} + +template +void BitmapFunctions::bitmap_intersect_update(FunctionContext* ctx, const StringVal& src, const ValType& key, + int num_key, const ValType* keys, const StringVal* dst) { + auto* dst_bitmap = reinterpret_cast*>(dst->ptr); + // zero size means the src input is a agg object + if (src.len == 0) { + dst_bitmap->update(detail::get_val(key), *reinterpret_cast(src.ptr)); + } else { + dst_bitmap->update(detail::get_val(key), RoaringBitmap((char*)src.ptr)); + } +} + +template +void BitmapFunctions::bitmap_intersect_merge(FunctionContext* ctx, const StringVal& src, const StringVal* dst) { + auto* dst_bitmap = reinterpret_cast*>(dst->ptr); + dst_bitmap->merge(BitmapIntersect((char*)src.ptr)); +} + +template +StringVal BitmapFunctions::bitmap_intersect_serialize(FunctionContext* ctx, const StringVal& src) { + auto* src_bitmap = reinterpret_cast*>(src.ptr); + StringVal result(ctx, src_bitmap->size()); + src_bitmap->serialize((char*)result.ptr); + delete src_bitmap; + return result; +} + +template +BigIntVal BitmapFunctions::bitmap_intersect_finalize(FunctionContext* ctx, const StringVal& src) { + auto* src_bitmap = reinterpret_cast*>(src.ptr); + BigIntVal result = BigIntVal(src_bitmap->intersect_count()); + delete src_bitmap; + return result; +} + + template void BitmapFunctions::bitmap_update_int( FunctionContext* ctx, const TinyIntVal& src, StringVal* dst); template void BitmapFunctions::bitmap_update_int( @@ -124,4 +396,122 @@ template void BitmapFunctions::bitmap_update_int( template void BitmapFunctions::bitmap_update_int( FunctionContext* ctx, const IntVal& src, StringVal* dst); + +template void BitmapFunctions::bitmap_intersect_init( + FunctionContext* ctx, StringVal* dst); +template void BitmapFunctions::bitmap_intersect_init( + FunctionContext* ctx, StringVal* dst); +template void BitmapFunctions::bitmap_intersect_init( + FunctionContext* ctx, StringVal* dst); +template void BitmapFunctions::bitmap_intersect_init( + FunctionContext* ctx, StringVal* dst); +template void BitmapFunctions::bitmap_intersect_init<__int128, LargeIntVal>( + FunctionContext* ctx, StringVal* dst); +template void BitmapFunctions::bitmap_intersect_init( + FunctionContext* ctx, StringVal* dst); +template void BitmapFunctions::bitmap_intersect_init( + FunctionContext* ctx, StringVal* dst); +template void BitmapFunctions::bitmap_intersect_init( + FunctionContext* ctx, StringVal* dst); +template void BitmapFunctions::bitmap_intersect_init( + FunctionContext* ctx, StringVal* dst); +template void BitmapFunctions::bitmap_intersect_init( + FunctionContext* ctx, StringVal* dst); + + +template void BitmapFunctions::bitmap_intersect_update( + FunctionContext* ctx, const StringVal& src, const TinyIntVal& key, + int num_key, const TinyIntVal* keys, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_update( + FunctionContext* ctx, const StringVal& src, const SmallIntVal& key, + int num_key, const SmallIntVal* keys, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_update( + FunctionContext* ctx, const StringVal& src, const IntVal& key, + int num_key, const IntVal* keys, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_update( + FunctionContext* ctx, const StringVal& src, const BigIntVal& key, + int num_key, const BigIntVal* keys, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_update<__int128, LargeIntVal>( + FunctionContext* ctx, const StringVal& src, const LargeIntVal& key, + int num_key, const LargeIntVal* keys, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_update( + FunctionContext* ctx, const StringVal& src, const FloatVal& key, + int num_key, const FloatVal* keys, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_update( + FunctionContext* ctx, const StringVal& src, const DoubleVal& key, + int num_key, const DoubleVal* keys, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_update( + FunctionContext* ctx, const StringVal& src, const DateTimeVal& key, + int num_key, const DateTimeVal* keys, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_update( + FunctionContext* ctx, const StringVal& src, const DecimalV2Val& key, + int num_key, const DecimalV2Val* keys, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_update( + FunctionContext* ctx, const StringVal& src, const StringVal& key, + int num_key, const StringVal* keys, const StringVal* dst); + + +template void BitmapFunctions::bitmap_intersect_merge( + FunctionContext* ctx, const StringVal& src, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_merge( + FunctionContext* ctx, const StringVal& src, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_merge( + FunctionContext* ctx, const StringVal& src, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_merge( + FunctionContext* ctx, const StringVal& src, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_merge<__int128>( + FunctionContext* ctx, const StringVal& src, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_merge( + FunctionContext* ctx, const StringVal& src, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_merge( + FunctionContext* ctx, const StringVal& src, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_merge( + FunctionContext* ctx, const StringVal& src, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_merge( + FunctionContext* ctx, const StringVal& src, const StringVal* dst); +template void BitmapFunctions::bitmap_intersect_merge( + FunctionContext* ctx, const StringVal& src, const StringVal* dst); + +template StringVal BitmapFunctions::bitmap_intersect_serialize( + FunctionContext* ctx, const StringVal& src); +template StringVal BitmapFunctions::bitmap_intersect_serialize( + FunctionContext* ctx, const StringVal& src); +template StringVal BitmapFunctions::bitmap_intersect_serialize( + FunctionContext* ctx, const StringVal& src); +template StringVal BitmapFunctions::bitmap_intersect_serialize( + FunctionContext* ctx, const StringVal& src); +template StringVal BitmapFunctions::bitmap_intersect_serialize<__int128>( + FunctionContext* ctx, const StringVal& src); +template StringVal BitmapFunctions::bitmap_intersect_serialize( + FunctionContext* ctx, const StringVal& src); +template StringVal BitmapFunctions::bitmap_intersect_serialize( + FunctionContext* ctx, const StringVal& src); +template StringVal BitmapFunctions::bitmap_intersect_serialize( + FunctionContext* ctx, const StringVal& src); +template StringVal BitmapFunctions::bitmap_intersect_serialize( + FunctionContext* ctx, const StringVal& src); +template StringVal BitmapFunctions::bitmap_intersect_serialize( + FunctionContext* ctx, const StringVal& src); + +template BigIntVal BitmapFunctions::bitmap_intersect_finalize( + FunctionContext* ctx, const StringVal& src); +template BigIntVal BitmapFunctions::bitmap_intersect_finalize( + FunctionContext* ctx, const StringVal& src); +template BigIntVal BitmapFunctions::bitmap_intersect_finalize( + FunctionContext* ctx, const StringVal& src); +template BigIntVal BitmapFunctions::bitmap_intersect_finalize( + FunctionContext* ctx, const StringVal& src); +template BigIntVal BitmapFunctions::bitmap_intersect_finalize<__int128>( + FunctionContext* ctx, const StringVal& src); +template BigIntVal BitmapFunctions::bitmap_intersect_finalize( + FunctionContext* ctx, const StringVal& src); +template BigIntVal BitmapFunctions::bitmap_intersect_finalize( + FunctionContext* ctx, const StringVal& src); +template BigIntVal BitmapFunctions::bitmap_intersect_finalize( + FunctionContext* ctx, const StringVal& src); +template BigIntVal BitmapFunctions::bitmap_intersect_finalize( + FunctionContext* ctx, const StringVal& src); +template BigIntVal BitmapFunctions::bitmap_intersect_finalize( + FunctionContext* ctx, const StringVal& src); + } diff --git a/be/src/exprs/bitmap_function.h b/be/src/exprs/bitmap_function.h index aef5016dd2..9c313842d2 100644 --- a/be/src/exprs/bitmap_function.h +++ b/be/src/exprs/bitmap_function.h @@ -40,6 +40,20 @@ public: static StringVal bitmap_serialize(FunctionContext* ctx, const StringVal& src); static StringVal to_bitmap(FunctionContext* ctx, const StringVal& src); static StringVal bitmap_hash(FunctionContext* ctx, const StringVal& src); + + + // bitmap_intersect + template + static void bitmap_intersect_init(FunctionContext* ctx, StringVal* dst); + template + static void bitmap_intersect_update(FunctionContext* ctx, const StringVal& src, const ValType& key, + int num_key, const ValType* keys, const StringVal* dst); + template + static void bitmap_intersect_merge(FunctionContext* ctx, const StringVal& src, const StringVal* dst); + template + static StringVal bitmap_intersect_serialize(FunctionContext* ctx, const StringVal& src); + template + static BigIntVal bitmap_intersect_finalize(FunctionContext* ctx, const StringVal& src); }; } #endif //DORIS_BE_SRC_QUERY_EXPRS_BITMAP_FUNCTION_H diff --git a/be/src/exprs/new_agg_fn_evaluator.cc b/be/src/exprs/new_agg_fn_evaluator.cc index 293d3cc239..f51ef33bb8 100644 --- a/be/src/exprs/new_agg_fn_evaluator.cc +++ b/be/src/exprs/new_agg_fn_evaluator.cc @@ -282,7 +282,7 @@ void NewAggFnEvaluator::SetDstSlot(const AnyVal* src, const SlotDescriptor& dst_ *reinterpret_cast(src)); return; case TYPE_DECIMALV2: - *reinterpret_cast(slot) = + *reinterpret_cast(slot) = reinterpret_cast(src)->val; return; default: @@ -388,7 +388,7 @@ inline void NewAggFnEvaluator::set_any_val( return; case TYPE_DECIMALV2: - reinterpret_cast(dst)->val = + reinterpret_cast(dst)->val = reinterpret_cast(slot)->value; return; diff --git a/be/src/udf/udf.h b/be/src/udf/udf.h index 80656271d1..80eb07e196 100755 --- a/be/src/udf/udf.h +++ b/be/src/udf/udf.h @@ -204,6 +204,9 @@ public: // argument). int get_num_args() const; + // Returns _constant_args size + int get_num_constant_args() const; + // Returns the type information for the arg_idx-th argument (0-indexed, not including // the FunctionContext* argument). Returns NULL if arg_idx is invalid. const TypeDesc* get_arg_type(int arg_idx) const; diff --git a/be/src/udf/udf_ir.cpp b/be/src/udf/udf_ir.cpp index 35225f27d5..7d477d93ee 100644 --- a/be/src/udf/udf_ir.cpp +++ b/be/src/udf/udf_ir.cpp @@ -36,6 +36,10 @@ int FunctionContext::get_num_args() const { return _impl->_arg_types.size(); } +int FunctionContext::get_num_constant_args() const { + return _impl->_constant_args.size(); +} + const FunctionContext::TypeDesc& FunctionContext::get_return_type() const { return _impl->_return_type; } diff --git a/be/src/util/bitmap.h b/be/src/util/bitmap.h index 2d0c1de062..8547819975 100644 --- a/be/src/util/bitmap.h +++ b/be/src/util/bitmap.h @@ -269,6 +269,7 @@ public: // the src is the serialized bitmap data, the type could be EMPTY, SINGLE or BITMAP explicit RoaringBitmap(const char* src) { _type = (BitmapDataType)src[0]; + DCHECK(_type >= 0 && _type <= 2); switch (_type) { case EMPTY: break; @@ -327,6 +328,49 @@ public: } } + // the _type maybe change: + // EMPTY -> EMPTY + // SINGLE -> EMPTY, SINGLE + // BITMAP -> EMPTY, SINGLE, BITMAP + void intersect(const RoaringBitmap& bitmap) { + switch(bitmap._type) { + case EMPTY: + _type = EMPTY; + return; + case SINGLE: + switch (_type) { + case EMPTY: + break; + case SINGLE: + if (_int_value != bitmap._int_value) { + _type = EMPTY; + } + break; + case BITMAP: + if (!_roaring.contains(bitmap._int_value)) { + _type = EMPTY; + } else { + _type = SINGLE; + _int_value = bitmap._int_value; + } + } + return; + case BITMAP: + switch (_type) { + case EMPTY: + break; + case SINGLE: + if (!bitmap._roaring.contains(_int_value)) { + _type = EMPTY; + } + break; + case BITMAP: + _roaring &= bitmap._roaring; + } + return; + } + } + int64_t cardinality() const { switch (_type) { case EMPTY: @@ -366,7 +410,7 @@ public: } } - std::string toString() const { + std::string to_string() const { switch (_type) { case EMPTY: return {}; diff --git a/be/test/exprs/bitmap_function_test.cpp b/be/test/exprs/bitmap_function_test.cpp index b8492f2ad8..fa79afd855 100644 --- a/be/test/exprs/bitmap_function_test.cpp +++ b/be/test/exprs/bitmap_function_test.cpp @@ -17,7 +17,7 @@ #include "exprs/aggregate_functions.h" #include "exprs/anyval_util.h" -#include "exprs/bitmap_function.h" +#include "exprs/bitmap_function.cpp" #include #include #include "testutil/function_utils.h" @@ -35,6 +35,14 @@ StringVal convert_bitmap_to_string(FunctionContext* ctx, RoaringBitmap& bitmap) return AnyValUtil::from_string_temp(ctx, buf); } +template +StringVal convert_bitmap_intersect_to_string(FunctionContext* ctx, BitmapIntersect& intersect) { + std::string buf; + buf.resize(intersect.size()); + intersect.serialize((char*)buf.c_str()); + return AnyValUtil::from_string_temp(ctx, buf); +} + class BitmapFunctionsTest : public testing::Test { public: BitmapFunctionsTest() = default; @@ -150,6 +158,96 @@ TEST_F(BitmapFunctionsTest, bitmap_count) { ASSERT_EQ(expected, result); } +template +void test_bitmap_intersect(FunctionContext* ctx, ValType key1, ValType key2) { + StringVal bitmap_column("placeholder"); + StringVal filter_column("placeholder"); + std::vector const_vals; + const_vals.push_back(&bitmap_column); + const_vals.push_back(&filter_column); + const_vals.push_back(&key1); + const_vals.push_back(&key2); + ctx->impl()->set_constant_args(const_vals); + + StringVal dst; + BitmapFunctions::bitmap_intersect_init(ctx, &dst); + + RoaringBitmap bitmap1(1024); + bitmap1.update(1025); + bitmap1.update(1026); + StringVal src1 = convert_bitmap_to_string(ctx, bitmap1); + BitmapFunctions::bitmap_intersect_update( + ctx, src1, key1, 1, nullptr, &dst); + + RoaringBitmap bitmap2(1024); + bitmap2.update(1023); + bitmap2.update(1022); + StringVal src2 = convert_bitmap_to_string(ctx, bitmap2); + BitmapFunctions::bitmap_intersect_update( + ctx, src2, key2, 2, nullptr, &dst); + + StringVal intersect1 = BitmapFunctions::bitmap_intersect_serialize(ctx, dst); + + BitmapIntersect intersect2; + for(size_t i = 2; i < const_vals.size(); i++) { + ValType* arg = reinterpret_cast(const_vals[i]); + intersect2.add_key(detail::get_val(*arg)); + } + intersect2.update(detail::get_val(key1), bitmap1); + intersect2.update(detail::get_val(key2), bitmap2); + StringVal expected = convert_bitmap_intersect_to_string(ctx, intersect2); + ASSERT_EQ(expected, intersect1); + + BitmapIntersect intersect2_serde((char *)expected.ptr); + ASSERT_EQ(1, intersect2_serde.intersect_count()); + + StringVal dst2; + BitmapFunctions::bitmap_intersect_init(ctx, &dst2); + BitmapFunctions::bitmap_intersect_merge(ctx, intersect1, &dst2); + BigIntVal result = BitmapFunctions::bitmap_intersect_finalize(ctx, dst2); + BigIntVal expected_count(1); + + ASSERT_EQ(expected_count, result); +} + +TEST_F(BitmapFunctionsTest, test_bitmap_intersect) { + test_bitmap_intersect( + ctx, TinyIntVal(101), TinyIntVal(102)); + test_bitmap_intersect( + ctx, SmallIntVal((int16_t)65535), SmallIntVal((int16_t)65534)); + test_bitmap_intersect( + ctx, IntVal(20191211), IntVal(20191212)); + test_bitmap_intersect( + ctx, BigIntVal(20191211), BigIntVal(20191212)); + test_bitmap_intersect( + ctx, LargeIntVal(20191211), LargeIntVal(20191212)); + test_bitmap_intersect( + ctx, FloatVal(1.01), FloatVal(1.02)); + test_bitmap_intersect( + ctx, DoubleVal(1.01), DoubleVal(1.02)); + + DecimalV2Val v1; + DecimalV2Value("1.01").to_decimal_val(&v1); + DecimalV2Val v2; + DecimalV2Value("1.02").to_decimal_val(&v2); + test_bitmap_intersect( + ctx, v1, v2); + + DateTimeVal datatime1; + DateTimeValue date_time_value; + date_time_value.from_date_int64(19880201); + date_time_value.to_datetime_val(&datatime1); + DateTimeVal datatime2; + date_time_value.from_date_int64(19880202); + date_time_value.to_datetime_val(&datatime2); + test_bitmap_intersect( + ctx, datatime1, datatime2); + + test_bitmap_intersect( + ctx, StringVal("20191211"), StringVal("20191212")); +} + + } int main(int argc, char** argv) { diff --git a/be/test/util/bitmap_test.cpp b/be/test/util/bitmap_test.cpp index bbfdd4504e..64a1d9e960 100644 --- a/be/test/util/bitmap_test.cpp +++ b/be/test/util/bitmap_test.cpp @@ -131,6 +131,142 @@ TEST_F(BitMapTest, iterator) { ASSERT_EQ(2000 - 500 - 200, run); } +TEST_F(BitMapTest, roaring_bitmap_union) { + RoaringBitmap empty; + RoaringBitmap single(1024); + RoaringBitmap bitmap; + bitmap.update(1024); + bitmap.update(1025); + bitmap.update(1026); + + ASSERT_EQ(0, empty.cardinality()); + ASSERT_EQ(1, single.cardinality()); + ASSERT_EQ(3, bitmap.cardinality()); + + RoaringBitmap empty2; + empty2.merge(empty); + ASSERT_EQ(0, empty2.cardinality()); + empty2.merge(single); + ASSERT_EQ(1, empty2.cardinality()); + RoaringBitmap empty3; + empty3.merge(bitmap); + ASSERT_EQ(3, empty3.cardinality()); + + RoaringBitmap single2(1025); + single2.merge(empty); + ASSERT_EQ(1, single2.cardinality()); + single2.merge(single); + ASSERT_EQ(2, single2.cardinality()); + RoaringBitmap single3(1027); + single3.merge(bitmap); + ASSERT_EQ(4, single3.cardinality()); + + RoaringBitmap bitmap2; + bitmap2.update(1024); + bitmap2.update(2048); + bitmap2.update(4096); + bitmap2.merge(empty); + ASSERT_EQ(3, bitmap2.cardinality()); + bitmap2.merge(single); + ASSERT_EQ(3, bitmap2.cardinality()); + bitmap2.merge(bitmap); + ASSERT_EQ(5, bitmap2.cardinality()); +} + +TEST_F(BitMapTest, roaring_bitmap_intersect) { + RoaringBitmap empty; + RoaringBitmap single(1024); + RoaringBitmap bitmap; + bitmap.update(1024); + bitmap.update(1025); + bitmap.update(1026); + + RoaringBitmap empty2; + empty2.intersect(empty); + ASSERT_EQ(0, empty2.cardinality()); + empty2.intersect(single); + ASSERT_EQ(0, empty2.cardinality()); + empty2.intersect(bitmap); + ASSERT_EQ(0, empty2.cardinality()); + + RoaringBitmap single2(1025); + single2.intersect(empty); + ASSERT_EQ(0, single2.cardinality()); + + RoaringBitmap single4(1025); + single4.intersect(single); + ASSERT_EQ(0, single4.cardinality()); + + RoaringBitmap single3(1024); + single3.intersect(single); + ASSERT_EQ(1, single3.cardinality()); + + single3.intersect(bitmap); + ASSERT_EQ(1, single3.cardinality()); + + RoaringBitmap single5(2048); + single5.intersect(bitmap); + ASSERT_EQ(0, single5.cardinality()); + + RoaringBitmap bitmap2; + bitmap2.update(1024); + bitmap2.update(2048); + bitmap2.intersect(empty); + ASSERT_EQ(0, bitmap2.cardinality()); + + RoaringBitmap bitmap3; + bitmap3.update(1024); + bitmap3.update(2048); + bitmap3.intersect(single); + ASSERT_EQ(1, bitmap3.cardinality()); + + RoaringBitmap bitmap4; + bitmap4.update(2049); + bitmap4.update(2048); + bitmap4.intersect(single); + ASSERT_EQ(0, bitmap4.cardinality()); + + RoaringBitmap bitmap5; + bitmap5.update(2049); + bitmap5.update(2048); + bitmap5.intersect(bitmap); + ASSERT_EQ(0, bitmap5.cardinality()); + + RoaringBitmap bitmap6; + bitmap6.update(1024); + bitmap6.update(1025); + bitmap6.intersect(bitmap); + ASSERT_EQ(2, bitmap6.cardinality()); +} + +std::string convert_bitmap_to_string(RoaringBitmap& bitmap) { + std::string buf; + buf.resize(bitmap.size()); + bitmap.serialize((char*)buf.c_str()); + return buf; +} + +TEST_F(BitMapTest, roaring_bitmap_serde) { + RoaringBitmap empty; + RoaringBitmap single(1024); + RoaringBitmap bitmap; + bitmap.update(1024); + bitmap.update(1025); + bitmap.update(1026); + + std::string buffer = convert_bitmap_to_string(empty); + RoaringBitmap empty_serde((char*)buffer.c_str()); + ASSERT_EQ(0, empty_serde.cardinality()); + + buffer = convert_bitmap_to_string(single); + RoaringBitmap single_serde((char*)buffer.c_str()); + ASSERT_EQ(1, single_serde.cardinality()); + + buffer = convert_bitmap_to_string(bitmap); + RoaringBitmap bitmap_serde((char*)buffer.c_str()); + ASSERT_EQ(3, bitmap_serde.cardinality()); +} + } int main(int argc, char** argv) { diff --git a/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 5eb9d6dec8..247d3514ad 100644 --- a/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -401,6 +401,30 @@ public class FunctionCallExpr extends Expr { return; } + if (fnName.getFunction().equalsIgnoreCase(FunctionSet.INTERSECT_COUNT)) { + if (children.size() <= 2) { + throw new AnalysisException("intersect_count(bitmap_column, column_to_filter, filter_values) " + + "function requires at least three parameters"); + } + + if (getChild(0) instanceof SlotRef) { + SlotRef slotRef = (SlotRef) getChild(0); + Column column = slotRef.getDesc().getColumn(); + if (column != null && column.getAggregationType() != AggregateType.BITMAP_UNION) { + throw new AnalysisException("intersect_count function first arg must be bitmap column"); + } + } else { + throw new AnalysisException("intersect_count function first arg must be bitmap column"); + } + + for(int i = 2; i < children.size(); i++) { + if (!getChild(i).isConstant()) { + throw new AnalysisException("intersect_count function filter_values arg must be constant"); + } + } + return; + } + if ((fnName.getFunction().equalsIgnoreCase(FunctionSet.BITMAP_COUNT))) { if (children.size() != 1) { throw new AnalysisException("BITMAP_COUNT function could only have one child"); diff --git a/fe/src/main/java/org/apache/doris/catalog/AggregateFunction.java b/fe/src/main/java/org/apache/doris/catalog/AggregateFunction.java index 1b09fbb289..24b03144b5 100644 --- a/fe/src/main/java/org/apache/doris/catalog/AggregateFunction.java +++ b/fe/src/main/java/org/apache/doris/catalog/AggregateFunction.java @@ -116,6 +116,27 @@ public class AggregateFunction extends Function { returnsNonNullOnEmpty = false; } + public AggregateFunction(FunctionName fnName, List argTypes, + Type retType, Type intermediateType, boolean hasVarArgs, + HdfsURI location, String updateFnSymbol, String initFnSymbol, + String serializeFnSymbol, String mergeFnSymbol, String getValueFnSymbol, + String removeFnSymbol, String finalizeFnSymbol) { + super(fnName, argTypes, retType, hasVarArgs); + setLocation(location); + this.intermediateType = (intermediateType.equals(retType)) ? null : intermediateType; + this.updateFnSymbol = updateFnSymbol; + this.initFnSymbol = initFnSymbol; + this.serializeFnSymbol = serializeFnSymbol; + this.mergeFnSymbol = mergeFnSymbol; + this.getValueFnSymbol = getValueFnSymbol; + this.removeFnSymbol = removeFnSymbol; + this.finalizeFnSymbol = finalizeFnSymbol; + ignoresDistinct = false; + isAnalyticFn = false; + isAggregateFn = true; + returnsNonNullOnEmpty = false; + } + public static AggregateFunction createBuiltin(String name, List argTypes, Type retType, Type intermediateType, String initFnSymbol, String updateFnSymbol, String mergeFnSymbol, @@ -132,8 +153,20 @@ public class AggregateFunction extends Function { String serializeFnSymbol, String getValueFnSymbol, String removeFnSymbol, String finalizeFnSymbol, boolean ignoresDistinct, boolean isAnalyticFn, boolean returnsNonNullOnEmpty) { + return createBuiltin(name, argTypes, retType, intermediateType, false, + initFnSymbol, updateFnSymbol, mergeFnSymbol, + serializeFnSymbol, getValueFnSymbol, removeFnSymbol, + finalizeFnSymbol, ignoresDistinct, isAnalyticFn,returnsNonNullOnEmpty); + } + + public static AggregateFunction createBuiltin(String name, + List argTypes, Type retType, Type intermediateType, boolean hasVarArgs, + String initFnSymbol, String updateFnSymbol, String mergeFnSymbol, + String serializeFnSymbol, String getValueFnSymbol, String removeFnSymbol, + String finalizeFnSymbol, boolean ignoresDistinct, boolean isAnalyticFn, + boolean returnsNonNullOnEmpty) { AggregateFunction fn = new AggregateFunction(new FunctionName(name), - argTypes, retType, intermediateType, null, updateFnSymbol, initFnSymbol, + argTypes, retType, intermediateType, hasVarArgs, null, updateFnSymbol, initFnSymbol, serializeFnSymbol, mergeFnSymbol, getValueFnSymbol, removeFnSymbol, finalizeFnSymbol); fn.setBinaryType(TFunctionBinaryType.BUILTIN); diff --git a/fe/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/src/main/java/org/apache/doris/catalog/FunctionSet.java index 5b94c72451..3213bd5738 100644 --- a/fe/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -516,9 +516,9 @@ public class FunctionSet { public static final String BITMAP_UNION_COUNT = "bitmap_union_count"; public static final String BITMAP_UNION_INT = "bitmap_union_int"; public static final String BITMAP_COUNT = "bitmap_count"; - + public static final String INTERSECT_COUNT = "intersect_count"; public static final Set BITMAP_LOAD_FNS = new ImmutableSortedSet.Builder(String.CASE_INSENSITIVE_ORDER) - .add("to_bitmap", "bitmap_hash", "bitmap_empty").build(); + .add("to_bitmap", "bitmap_hash", "bitmap_empty").build(); private static final Map BITMAP_UNION_INT_SYMBOL = ImmutableMap.builder() @@ -530,6 +530,146 @@ public class FunctionSet { "_ZN5doris15BitmapFunctions17bitmap_update_intIN9doris_udf6IntValEEEvPNS2_15FunctionContextERKT_PNS2_9StringValE") .build(); + private static final Map BITMAP_INTERSECT_INIT_SYMBOL = + ImmutableMap.builder() + .put(Type.TINYINT, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initIaN9doris_udf10TinyIntValEEEvPNS2_15FunctionContextEPNS2_9StringValE") + .put(Type.SMALLINT, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initIsN9doris_udf11SmallIntValEEEvPNS2_15FunctionContextEPNS2_9StringValE") + .put(Type.INT, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initIiN9doris_udf6IntValEEEvPNS2_15FunctionContextEPNS2_9StringValE") + .put(Type.BIGINT, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initIlN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPNS2_9StringValE") + .put(Type.LARGEINT, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initInN9doris_udf11LargeIntValEEEvPNS2_15FunctionContextEPNS2_9StringValE") + .put(Type.FLOAT, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initIfN9doris_udf8FloatValEEEvPNS2_15FunctionContextEPNS2_9StringValE") + .put(Type.DOUBLE, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initIdN9doris_udf9DoubleValEEEvPNS2_15FunctionContextEPNS2_9StringValE") + .put(Type.DATE, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initINS_13DateTimeValueEN9doris_udf11DateTimeValEEEvPNS3_15FunctionContextEPNS3_9StringValE") + .put(Type.DATETIME, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initINS_13DateTimeValueEN9doris_udf11DateTimeValEEEvPNS3_15FunctionContextEPNS3_9StringValE") + .put(Type.DECIMALV2, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initINS_14DecimalV2ValueEN9doris_udf12DecimalV2ValEEEvPNS3_15FunctionContextEPNS3_9StringValE") + .put(Type.CHAR, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initINS_11StringValueEN9doris_udf9StringValEEEvPNS3_15FunctionContextEPS4_") + .put(Type.VARCHAR, + "_ZN5doris15BitmapFunctions21bitmap_intersect_initINS_11StringValueEN9doris_udf9StringValEEEvPNS3_15FunctionContextEPS4_") + .build(); + + private static final Map BITMAP_INTERSECT_UPDATE_SYMBOL = + ImmutableMap.builder() + .put(Type.TINYINT, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateIaN9doris_udf10TinyIntValEEEvPNS2_15FunctionContextERKNS2_9StringValERKT0_iPSA_PS7_") + .put(Type.SMALLINT, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateIsN9doris_udf11SmallIntValEEEvPNS2_15FunctionContextERKNS2_9StringValERKT0_iPSA_PS7_") + .put(Type.INT, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateIiN9doris_udf6IntValEEEvPNS2_15FunctionContextERKNS2_9StringValERKT0_iPSA_PS7_") + .put(Type.BIGINT, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateIlN9doris_udf9BigIntValEEEvPNS2_15FunctionContextERKNS2_9StringValERKT0_iPSA_PS7_") + .put(Type.LARGEINT, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateInN9doris_udf11LargeIntValEEEvPNS2_15FunctionContextERKNS2_9StringValERKT0_iPSA_PS7_") + .put(Type.FLOAT, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateIfN9doris_udf8FloatValEEEvPNS2_15FunctionContextERKNS2_9StringValERKT0_iPSA_PS7_") + .put(Type.DOUBLE, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateIdN9doris_udf9DoubleValEEEvPNS2_15FunctionContextERKNS2_9StringValERKT0_iPSA_PS7_") + .put(Type.DATE, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateINS_13DateTimeValueEN9doris_udf11DateTimeValEEEvPNS3_15FunctionContextERKNS3_9StringValERKT0_iPSB_PS8_") + .put(Type.DATETIME, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateINS_13DateTimeValueEN9doris_udf11DateTimeValEEEvPNS3_15FunctionContextERKNS3_9StringValERKT0_iPSB_PS8_") + .put(Type.DECIMALV2, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateINS_14DecimalV2ValueEN9doris_udf12DecimalV2ValEEEvPNS3_15FunctionContextERKNS3_9StringValERKT0_iPSB_PS8_") + .put(Type.CHAR, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateINS_11StringValueEN9doris_udf9StringValEEEvPNS3_15FunctionContextERKS4_RKT0_iPSA_PS7_") + .put(Type.VARCHAR, + "_ZN5doris15BitmapFunctions23bitmap_intersect_updateINS_11StringValueEN9doris_udf9StringValEEEvPNS3_15FunctionContextERKS4_RKT0_iPSA_PS7_") + .build(); + + private static final Map BITMAP_INTERSECT_MERGE_SYMBOL = + ImmutableMap.builder() + .put(Type.TINYINT, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeIaEEvPN9doris_udf15FunctionContextERKNS2_9StringValEPS6_") + .put(Type.SMALLINT, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeIsEEvPN9doris_udf15FunctionContextERKNS2_9StringValEPS6_") + .put(Type.INT, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeIiEEvPN9doris_udf15FunctionContextERKNS2_9StringValEPS6_") + .put(Type.BIGINT, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeIlEEvPN9doris_udf15FunctionContextERKNS2_9StringValEPS6_") + .put(Type.LARGEINT, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeInEEvPN9doris_udf15FunctionContextERKNS2_9StringValEPS6_") + .put(Type.FLOAT, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeIfEEvPN9doris_udf15FunctionContextERKNS2_9StringValEPS6_") + .put(Type.DOUBLE, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeIdEEvPN9doris_udf15FunctionContextERKNS2_9StringValEPS6_") + .put(Type.DATE, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeINS_13DateTimeValueEEEvPN9doris_udf15FunctionContextERKNS3_9StringValEPS7_") + .put(Type.DATETIME, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeINS_13DateTimeValueEEEvPN9doris_udf15FunctionContextERKNS3_9StringValEPS7_") + .put(Type.DECIMALV2, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeINS_14DecimalV2ValueEEEvPN9doris_udf15FunctionContextERKNS3_9StringValEPS7_") + .put(Type.CHAR, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeINS_11StringValueEEEvPN9doris_udf15FunctionContextERKNS3_9StringValEPS7_") + .put(Type.VARCHAR, + "_ZN5doris15BitmapFunctions22bitmap_intersect_mergeINS_11StringValueEEEvPN9doris_udf15FunctionContextERKNS3_9StringValEPS7_") + .build(); + + private static final Map BITMAP_INTERSECT_SERIALIZE_SYMBOL = + ImmutableMap.builder() + .put(Type.TINYINT, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeIaEEN9doris_udf9StringValEPNS2_15FunctionContextERKS3_") + .put(Type.SMALLINT, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeIsEEN9doris_udf9StringValEPNS2_15FunctionContextERKS3_") + .put(Type.INT, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeIiEEN9doris_udf9StringValEPNS2_15FunctionContextERKS3_") + .put(Type.BIGINT, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeIlEEN9doris_udf9StringValEPNS2_15FunctionContextERKS3_") + .put(Type.LARGEINT, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeInEEN9doris_udf9StringValEPNS2_15FunctionContextERKS3_") + .put(Type.FLOAT, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeIfEEN9doris_udf9StringValEPNS2_15FunctionContextERKS3_") + .put(Type.DOUBLE, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeIdEEN9doris_udf9StringValEPNS2_15FunctionContextERKS3_") + .put(Type.DATE, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeINS_13DateTimeValueEEEN9doris_udf9StringValEPNS3_15FunctionContextERKS4_") + .put(Type.DATETIME, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeINS_13DateTimeValueEEEN9doris_udf9StringValEPNS3_15FunctionContextERKS4_") + .put(Type.DECIMALV2, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeINS_14DecimalV2ValueEEEN9doris_udf9StringValEPNS3_15FunctionContextERKS4_") + .put(Type.CHAR, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeINS_11StringValueEEEN9doris_udf9StringValEPNS3_15FunctionContextERKS4_") + .put(Type.VARCHAR, + "_ZN5doris15BitmapFunctions26bitmap_intersect_serializeINS_11StringValueEEEN9doris_udf9StringValEPNS3_15FunctionContextERKS4_") + .build(); + + private static final Map BITMAP_INTERSECT_FINALIZE_SYMBOL = + ImmutableMap.builder() + .put(Type.TINYINT, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeIaEEN9doris_udf9BigIntValEPNS2_15FunctionContextERKNS2_9StringValE") + .put(Type.SMALLINT, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeIsEEN9doris_udf9BigIntValEPNS2_15FunctionContextERKNS2_9StringValE") + .put(Type.INT, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeIiEEN9doris_udf9BigIntValEPNS2_15FunctionContextERKNS2_9StringValE") + .put(Type.BIGINT, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeIlEEN9doris_udf9BigIntValEPNS2_15FunctionContextERKNS2_9StringValE") + .put(Type.LARGEINT, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeInEEN9doris_udf9BigIntValEPNS2_15FunctionContextERKNS2_9StringValE") + .put(Type.FLOAT, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeIfEEN9doris_udf9BigIntValEPNS2_15FunctionContextERKNS2_9StringValE") + .put(Type.DOUBLE, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeIdEEN9doris_udf9BigIntValEPNS2_15FunctionContextERKNS2_9StringValE") + .put(Type.DATE, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeINS_13DateTimeValueEEEN9doris_udf9BigIntValEPNS3_15FunctionContextERKNS3_9StringValE") + .put(Type.DATETIME, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeINS_13DateTimeValueEEEN9doris_udf9BigIntValEPNS3_15FunctionContextERKNS3_9StringValE") + .put(Type.DECIMALV2, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeINS_14DecimalV2ValueEEEN9doris_udf9BigIntValEPNS3_15FunctionContextERKNS3_9StringValE") + .put(Type.CHAR, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeINS_11StringValueEEEN9doris_udf9BigIntValEPNS3_15FunctionContextERKNS3_9StringValE") + .put(Type.VARCHAR, + "_ZN5doris15BitmapFunctions25bitmap_intersect_finalizeINS_11StringValueEEEN9doris_udf9BigIntValEPNS3_15FunctionContextERKNS3_9StringValE") + .build(); + public Function getFunction(Function desc, Function.CompareMode mode) { List fns = functions.get(desc.functionName()); if (fns == null) { @@ -723,17 +863,6 @@ public class FunctionSet { null, prefix + "30count_distinct_string_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", false, true, true)); - - addBuiltin(AggregateFunction.createBuiltin(BITMAP_UNION, Lists.newArrayList(t), - Type.VARCHAR, - Type.VARCHAR, - "_ZN5doris15BitmapFunctions11bitmap_initEPN9doris_udf15FunctionContextEPNS1_9StringValE", - "_ZN5doris15BitmapFunctions12bitmap_unionEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", - "_ZN5doris15BitmapFunctions12bitmap_unionEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", - "_ZN5doris15BitmapFunctions16bitmap_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", - "_ZN5doris15BitmapFunctions16bitmap_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", - true, false, true)); - } else if (t == Type.TINYINT || t == Type.SMALLINT || t == Type.INT || t == Type.BIGINT || t == Type.LARGEINT || t == Type.DOUBLE) { addBuiltin(AggregateFunction.createBuiltin("multi_distinct_count", Lists.newArrayList(t), @@ -863,6 +992,18 @@ public class FunctionSet { "_ZN5doris15BitmapFunctions15bitmap_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE", true, false, true)); + // INTERSECT_COUNT + addBuiltin(AggregateFunction.createBuiltin(INTERSECT_COUNT, + Lists.newArrayList(Type.BITMAP, t, t), Type.BIGINT, Type.VARCHAR, true, + BITMAP_INTERSECT_INIT_SYMBOL.get(t), + BITMAP_INTERSECT_UPDATE_SYMBOL.get(t), + BITMAP_INTERSECT_MERGE_SYMBOL.get(t), + BITMAP_INTERSECT_SERIALIZE_SYMBOL.get(t), + null, + null, + BITMAP_INTERSECT_FINALIZE_SYMBOL.get(t), + true, false, true)); + // HLL_UNION_AGG addBuiltin(AggregateFunction.createBuiltin("hll_union_agg", Lists.newArrayList(t), Type.BIGINT, Type.VARCHAR, @@ -994,6 +1135,16 @@ public class FunctionSet { null, false, true, false)); } + addBuiltin(AggregateFunction.createBuiltin(BITMAP_UNION, Lists.newArrayList(Type.BITMAP), + Type.VARCHAR, + Type.VARCHAR, + "_ZN5doris15BitmapFunctions11bitmap_initEPN9doris_udf15FunctionContextEPNS1_9StringValE", + "_ZN5doris15BitmapFunctions12bitmap_unionEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", + "_ZN5doris15BitmapFunctions12bitmap_unionEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", + "_ZN5doris15BitmapFunctions16bitmap_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + "_ZN5doris15BitmapFunctions16bitmap_serializeEPN9doris_udf15FunctionContextERKNS1_9StringValE", + true, false, true)); + addBuiltin(AggregateFunction.createBuiltin(BITMAP_UNION_COUNT, Lists.newArrayList(Type.BITMAP), Type.BIGINT, Type.VARCHAR,