diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index f6fc98908e..b42833845d 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -44,6 +44,7 @@ #include "exprs/math_functions.h" #include "exprs/encryption_functions.h" #include "exprs/es_functions.h" +#include "exprs/hash_functions.h" #include "exprs/timestamp_functions.h" #include "exprs/decimal_operators.h" #include "exprs/decimalv2_operators.h" @@ -275,6 +276,7 @@ void init_daemon(int argc, char** argv, const std::vector& paths) { TimezoneDatabase::init(); BitmapFunctions::init(); HllFunctions::init(); + HashFunctions::init(); pthread_t tc_malloc_pid; pthread_create(&tc_malloc_pid, NULL, tcmalloc_gc_thread, NULL); diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt index 6ff2d11e5d..30768acf79 100644 --- a/be/src/exprs/CMakeLists.txt +++ b/be/src/exprs/CMakeLists.txt @@ -39,6 +39,7 @@ add_library(Exprs decimalv2_operators.cpp time_operators.cpp es_functions.cpp + hash_functions.cpp literal.cpp expr.cpp expr_ir.cpp diff --git a/be/src/exprs/bitmap_function.cpp b/be/src/exprs/bitmap_function.cpp index 71fe25c7a9..4868c974bf 100644 --- a/be/src/exprs/bitmap_function.cpp +++ b/be/src/exprs/bitmap_function.cpp @@ -20,6 +20,8 @@ #include "exprs/anyval_util.h" #include "util/bitmap.h" #include "util/string_parser.hpp" +#include "gutil/strings/split.h" +#include "gutil/strings/numbers.h" namespace doris { @@ -425,7 +427,40 @@ StringVal BitmapFunctions::bitmap_and(FunctionContext* ctx, const StringVal& lhs bitmap.intersect(RoaringBitmap((char*)rhs.ptr)); } - StringVal result(ctx,bitmap.size()); + StringVal result(ctx, bitmap.size()); + bitmap.serialize((char*)result.ptr); + return result; +} + +StringVal BitmapFunctions::bitmap_to_string(FunctionContext* ctx, const StringVal& input) { + if (input.is_null) { + return StringVal::null(); + } + RoaringBitmap bitmap_obj; + RoaringBitmap* bitmap = &bitmap_obj; + if (input.len == 0) { + bitmap = (RoaringBitmap*)input.ptr; + } else { + bitmap_obj.deserialize((const char*)input.ptr); + } + + std::string str = bitmap->to_string(); + return AnyValUtil::from_string_temp(ctx, str); +} + +StringVal BitmapFunctions::bitmap_from_string(FunctionContext* ctx, const StringVal& input) { + if (input.is_null) { + return StringVal::null(); + } + + std::vector bits; + if (!SplitStringAndParse({(const char*)input.ptr, input.len}, ",", &safe_strtou32, &bits)) { + return StringVal::null(); + } + + RoaringBitmap bitmap(bits); + + StringVal result(ctx, bitmap.size()); bitmap.serialize((char*)result.ptr); return result; } diff --git a/be/src/exprs/bitmap_function.h b/be/src/exprs/bitmap_function.h index fead82a4de..bf00ba259b 100644 --- a/be/src/exprs/bitmap_function.h +++ b/be/src/exprs/bitmap_function.h @@ -55,6 +55,13 @@ public: static StringVal bitmap_hash(FunctionContext* ctx, const StringVal& src); static StringVal bitmap_or(FunctionContext* ctx, const StringVal& src,const StringVal& dst); static StringVal bitmap_and(FunctionContext* ctx, const StringVal& src,const StringVal& dst); + static StringVal bitmap_to_string(FunctionContext* ctx, const StringVal& input); + // Convert a comma separated string to a Bitmap + // Example: + // "" will be converted to an empty Bitmap + // "1,2,3" will be converted to Bitmap with its Bit 1, 2, 3 set. + // "-1, 1" will get NULL, because -1 is not a valid bit for Bitmap + static StringVal bitmap_from_string(FunctionContext* ctx, const StringVal& input); // bitmap_intersect template diff --git a/be/src/exprs/cast_functions.cpp b/be/src/exprs/cast_functions.cpp index 8dda9f57d0..9c89f52485 100644 --- a/be/src/exprs/cast_functions.cpp +++ b/be/src/exprs/cast_functions.cpp @@ -24,6 +24,7 @@ #include "runtime/string_value.h" #include "util/string_parser.hpp" #include "string_functions.h" +#include "util/mysql_global.h" namespace doris { @@ -161,13 +162,26 @@ StringVal CastFunctions::cast_to_string_val(FunctionContext* ctx, const LargeInt return AnyValUtil::from_buffer_temp(ctx, d, len); } +template +int float_to_string(T value, char* buf); + +template<> +int float_to_string(float value, char* buf) { + return FloatToBuffer(value, MAX_FLOAT_STR_LENGTH + 2, buf); +} + +template<> +int float_to_string(double value, char* buf) { + return DoubleToBuffer(value, MAX_DOUBLE_STR_LENGTH + 2, buf); +} + #define CAST_FLOAT_TO_STRING(float_type, format) \ StringVal CastFunctions::cast_to_string_val(FunctionContext* ctx, const float_type& val) { \ if (val.is_null) return StringVal::null(); \ /* val.val could be -nan, return "nan" instead */ \ if (std::isnan(val.val)) return StringVal("nan"); \ /* Add 1 to MAX_FLOAT_CHARS since snprintf adds a trailing '\0' */ \ - StringVal sv(ctx, MAX_FLOAT_CHARS + 1); \ + StringVal sv(ctx, MAX_DOUBLE_STR_LENGTH + 2); \ if (UNLIKELY(sv.is_null)) { \ return sv; \ } \ @@ -178,11 +192,9 @@ StringVal CastFunctions::cast_to_string_val(FunctionContext* ctx, const LargeInt DCHECK_LE(sv.len, MAX_FLOAT_CHARS); \ AnyValUtil::TruncateIfNecessary(returnType, &sv); \ } else if (returnType.len == -1) { \ - std::stringstream ss; \ - ss << val.val; \ - std::string str = ss.str(); \ - sv.len = str.length(); \ - memcpy(sv.ptr, str.c_str(), str.length()); \ + char buf[MAX_DOUBLE_STR_LENGTH + 2]; \ + sv.len = float_to_string(val.val, buf); \ + memcpy(sv.ptr, buf, sv.len); \ } else { \ DCHECK(false); \ } \ diff --git a/be/src/exprs/hash_functions.cpp b/be/src/exprs/hash_functions.cpp new file mode 100644 index 0000000000..79f3e36e0b --- /dev/null +++ b/be/src/exprs/hash_functions.cpp @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/hash_functions.h" + +#include "udf/udf.h" +#include "util/hash_util.hpp" + +namespace doris { + +using doris_udf::FunctionContext; +using doris_udf::IntVal; +using doris_udf::StringVal; + +void HashFunctions::init() { } + +IntVal HashFunctions::murmur_hash3_32( + FunctionContext* ctx, int num_children, const StringVal* inputs) { + uint32_t seed = HashUtil::MURMUR3_32_SEED; + for (int i = 0; i < num_children; ++i) { + if (inputs[i].is_null) { + return IntVal::null(); + } + seed = HashUtil::murmur_hash3_32(inputs[i].ptr, inputs[i].len, seed); + } + return seed; +} + +} diff --git a/be/src/exprs/hash_functions.h b/be/src/exprs/hash_functions.h new file mode 100644 index 0000000000..1a0c77f099 --- /dev/null +++ b/be/src/exprs/hash_functions.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace doris_udf { +class FunctionContext; +class IntVal; +class StringVal; +} + +namespace doris { + +class HashFunctions { +public: + static void init(); + static doris_udf::IntVal murmur_hash3_32( + doris_udf::FunctionContext* ctx, int num_children, + const doris_udf::StringVal* inputs); +}; + +} diff --git a/be/src/util/bitmap.cpp b/be/src/util/bitmap.cpp index a6ef2903bf..c132a95651 100644 --- a/be/src/util/bitmap.cpp +++ b/be/src/util/bitmap.cpp @@ -151,4 +151,37 @@ std::string BitmapToString(const uint8_t *bitmap, size_t num_bits) { return s; } +std::string RoaringBitmap::to_string() const { + std::stringstream ss; + switch (_type) { + case EMPTY: + break; + case SINGLE: + ss << _int_value; + break; + case BITMAP: { + struct IterCtx { + std::stringstream* ss = nullptr; + bool first = true; + } iter_ctx; + iter_ctx.ss = &ss; + _roaring.iterate( + [] (uint32_t value, void *inner_iter_data) -> bool { + IterCtx* ctx = (IterCtx*)inner_iter_data; + if (ctx->first) { + ctx->first = false; + } else { + (*ctx->ss) << ","; + } + (*ctx->ss) << value; + return true; + }, &iter_ctx); + break; + } + default: + break; + } + return ss.str(); +} + } diff --git a/be/src/util/bitmap.h b/be/src/util/bitmap.h index 8547819975..5784e252c1 100644 --- a/be/src/util/bitmap.h +++ b/be/src/util/bitmap.h @@ -268,16 +268,22 @@ public: // the src is the serialized bitmap data, the type could be EMPTY, SINGLE or BITMAP explicit RoaringBitmap(const char* src) { - _type = (BitmapDataType)src[0]; - DCHECK(_type >= 0 && _type <= 2); - switch (_type) { - case EMPTY: - break; - case SINGLE: - _int_value = decode_fixed32_le(reinterpret_cast(src + 1)); - break; - case BITMAP: - _roaring = Roaring::read(src + 1); + deserialize(src); + } + + // construct a bitmap from given bits + explicit RoaringBitmap(const std::vector& bits) { + switch (bits.size()) { + case 0: + _type = EMPTY; + break; + case 1: + _type = SINGLE; + _int_value = bits[0]; + break; + default: + _type = BITMAP; + _roaring.addMany(bits.size(), &bits[0]); } } @@ -371,6 +377,19 @@ public: } } + // check if value x is present + bool contains(uint32_t x) { + switch (_type) { + case EMPTY: + return false; + case SINGLE: + return _int_value == x; + case BITMAP: + return _roaring.contains(x); + } + return false; + } + int64_t cardinality() const { switch (_type) { case EMPTY: @@ -410,18 +429,28 @@ public: } } - std::string to_string() const { + // deserialize a bitmap from input serialized binary. + // the src is the serialized bitmap data, the type could be EMPTY, SINGLE or BITMAP + bool deserialize(const char* src) { + _type = (BitmapDataType)src[0]; + DCHECK(_type >= 0 && _type <= 2); switch (_type) { case EMPTY: - return {}; + break; case SINGLE: - return std::to_string(_int_value); + _int_value = decode_fixed32_le(reinterpret_cast(src + 1)); + break; case BITMAP: - return _roaring.toString(); + _roaring = Roaring::read(src + 1); + break; + default: + return false; } - return {}; + return true; } + std::string to_string() const; + private: enum BitmapDataType { EMPTY = 0, @@ -430,7 +459,7 @@ private: }; Roaring _roaring; - uint32_t _int_value; + uint32_t _int_value = 0; BitmapDataType _type; }; diff --git a/be/src/util/mysql_global.h b/be/src/util/mysql_global.h index da51d90335..d933dff930 100644 --- a/be/src/util/mysql_global.h +++ b/be/src/util/mysql_global.h @@ -15,8 +15,7 @@ // specific language governing permissions and limitations // under the License. -#ifndef DORIS_BE_SRC_QUERY_MYSQL_MYSQL_GLOBAL_H -#define DORIS_BE_SRC_QUERY_MYSQL_MYSQL_GLOBAL_H +#pragma once #include #include @@ -54,5 +53,4 @@ typedef unsigned char uchar; /* -[digits].[frac] */ #define MAX_DECIMAL_STR_LENGTH 29 -#endif //_DORIS_BE_SRC_QUERY_MYSQL_MYSQL_GLOBAL_H_ } diff --git a/be/test/util/bitmap_test.cpp b/be/test/util/bitmap_test.cpp index 64a1d9e960..ac3b81e2e0 100644 --- a/be/test/util/bitmap_test.cpp +++ b/be/test/util/bitmap_test.cpp @@ -21,6 +21,8 @@ #include #include "common/logging.h" +#include "testutil/function_utils.h" +#include "exprs/bitmap_function.h" namespace doris { @@ -267,6 +269,40 @@ TEST_F(BitMapTest, roaring_bitmap_serde) { ASSERT_EQ(3, bitmap_serde.cardinality()); } +TEST_F(BitMapTest, bitmap_to_string) { + RoaringBitmap empty; + ASSERT_STREQ("", empty.to_string().c_str()); + empty.update(1); + ASSERT_STREQ("1", empty.to_string().c_str()); + empty.update(2); + ASSERT_STREQ("1,2", empty.to_string().c_str()); +} + +TEST_F(BitMapTest, bitmap_from_string) { + FunctionUtils utils; + + { + StringVal val = StringVal("0,1,2"); + auto bitmap_str = BitmapFunctions::bitmap_from_string(utils.get_fn_ctx(), val); + ASSERT_FALSE(bitmap_str.is_null); + RoaringBitmap bitmap((const char*)bitmap_str.ptr); + bitmap.contains(0); + bitmap.contains(1); + bitmap.contains(2); + + } + { + StringVal val = StringVal("a,b,1,2"); + auto bitmap_str = BitmapFunctions::bitmap_from_string(utils.get_fn_ctx(), val); + ASSERT_TRUE(bitmap_str.is_null); + } + { + StringVal val = StringVal("-1,1,2"); + auto bitmap_str = BitmapFunctions::bitmap_from_string(utils.get_fn_ctx(), val); + ASSERT_TRUE(bitmap_str.is_null); + } +} + } int main(int argc, char** argv) { diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md new file mode 100644 index 0000000000..261da29983 --- /dev/null +++ b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md @@ -0,0 +1,56 @@ + + +# bitmap_from_string + +## description +### Syntax + +`BITMAP BITMAP_FROM_STRING(VARCHAR input)` + +将一个字符串转化为一个BITAMP，字符串是由逗号分隔的一组UINT32数字组成. +比如"0, 1, 2"字符串会转化为一个Bitmap，其中的第0, 1, 2位被设置. +当输入字段不合法时，返回NULL + +## example + +``` +mysql> select bitmap_to_string(bitmap_empty()); ++----------------------------------+ +| bitmap_to_string(bitmap_empty()) | ++----------------------------------+ +| | ++----------------------------------+ + +mysql> select bitmap_to_string(bitmap_from_string("0, 1, 2")); ++-------------------------------------------------+ +| bitmap_to_string(bitmap_from_string('0, 1, 2')) | ++-------------------------------------------------+ +| 0,1,2 | ++-------------------------------------------------+ + +mysql> select bitmap_from_string("-1, 0, 1, 2"); ++-----------------------------------+ +| bitmap_from_string('-1, 0, 1, 2') | ++-----------------------------------+ +| NULL | ++-----------------------------------+ +``` + +## keyword + + BITMAP_FROM_STRING,BITMAP diff --git a/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md new file mode 100644 index 0000000000..e7a25ebc73 --- /dev/null +++ b/docs/documentation/cn/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md @@ -0,0 +1,62 @@ + + +# bitmap_to_string + +## description +### Syntax + +`VARCHAR BITMAP_TO_STRING(BITMAP input)` + +将一个bitmap转化成一个逗号分隔的字符串，字符串中包含所有设置的BIT位。输入是null的话会返回null。 + +## example + +``` +mysql> select bitmap_to_string(null); ++------------------------+ +| bitmap_to_string(NULL) | ++------------------------+ +| NULL | ++------------------------+ + +mysql> select bitmap_to_string(bitmap_empty()); ++----------------------------------+ +| bitmap_to_string(bitmap_empty()) | ++----------------------------------+ +| | ++----------------------------------+ + +mysql> select bitmap_to_string(to_bitmap(1)); ++--------------------------------+ +| bitmap_to_string(to_bitmap(1)) | ++--------------------------------+ +| 1 | ++--------------------------------+ + +mysql> select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))); ++---------------------------------------------------------+ +| bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) | ++---------------------------------------------------------+ +| 1,2 | ++---------------------------------------------------------+ + +``` + +## keyword + + BITMAP_TO_STRING,BITMAP diff --git a/docs/documentation/cn/sql-reference/sql-functions/hash-functions/index.rst b/docs/documentation/cn/sql-reference/sql-functions/hash-functions/index.rst new file mode 100644 index 0000000000..b0556ff3b2 --- /dev/null +++ b/docs/documentation/cn/sql-reference/sql-functions/hash-functions/index.rst @@ -0,0 +1,8 @@ +============= +Hash函数 +============= + +.. toctree:: + :glob: + + * diff --git a/docs/documentation/cn/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md b/docs/documentation/cn/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md new file mode 100644 index 0000000000..cf0cc98da7 --- /dev/null +++ b/docs/documentation/cn/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md @@ -0,0 +1,54 @@ + + +# murmur_hash3_32 + +## description +### Syntax + +`INT MURMUR_HASH3_32(VARCHAR input, ...)` + +返回输入字符串的32位murmur3 hash值 + +## example + +``` +mysql> select murmur_hash3_32(null); ++-----------------------+ +| murmur_hash3_32(NULL) | ++-----------------------+ +| NULL | ++-----------------------+ + +mysql> select murmur_hash3_32("hello"); ++--------------------------+ +| murmur_hash3_32('hello') | ++--------------------------+ +| 1321743225 | ++--------------------------+ + +mysql> select murmur_hash3_32("hello", "world"); ++-----------------------------------+ +| murmur_hash3_32('hello', 'world') | ++-----------------------------------+ +| 984713481 | ++-----------------------------------+ +``` + +## keyword + + MURMUR_HASH3_32,HASH diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md new file mode 100644 index 0000000000..76c72c375a --- /dev/null +++ b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_from_string.md @@ -0,0 +1,56 @@ + + +# bitmap_from_string + +## description +### Syntax + +`BITMAP BITMAP_FROM_STRING(VARCHAR input)` + +Convert a string into a bitmap. The input string should be a comma separated UNIT32. +For example: input string "0, 1, 2" will be converted to a Bitmap with bit 0, 1, 2 set. +If input string is invalid, return NULL. + +## example + +``` +mysql> select bitmap_to_string(bitmap_empty()); ++----------------------------------+ +| bitmap_to_string(bitmap_empty()) | ++----------------------------------+ +| | ++----------------------------------+ + +mysql> select bitmap_to_string(bitmap_from_string("0, 1, 2")); ++-------------------------------------------------+ +| bitmap_to_string(bitmap_from_string('0, 1, 2')) | ++-------------------------------------------------+ +| 0,1,2 | ++-------------------------------------------------+ + +mysql> select bitmap_from_string("-1, 0, 1, 2"); ++-----------------------------------+ +| bitmap_from_string('-1, 0, 1, 2') | ++-----------------------------------+ +| NULL | ++-----------------------------------+ +``` + +## keyword + + BITMAP_FROM_STRING,BITMAP diff --git a/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md new file mode 100644 index 0000000000..fbaf4b28a7 --- /dev/null +++ b/docs/documentation/en/sql-reference/sql-functions/bitmap-functions/bitmap_to_string.md @@ -0,0 +1,63 @@ + + +# bitmap_to_string + +## description +### Syntax + +`VARCHAR BITMAP_TO_STRING(BITMAP input)` + +Convert a input BITMAP to a string. The string is a separated string, contains all set bits in Bitmap. +If input is null, return null. + +## example + +``` +mysql> select bitmap_to_string(null); ++------------------------+ +| bitmap_to_string(NULL) | ++------------------------+ +| NULL | ++------------------------+ + +mysql> select bitmap_to_string(bitmap_empty()); ++----------------------------------+ +| bitmap_to_string(bitmap_empty()) | ++----------------------------------+ +| | ++----------------------------------+ + +mysql> select bitmap_to_string(to_bitmap(1)); ++--------------------------------+ +| bitmap_to_string(to_bitmap(1)) | ++--------------------------------+ +| 1 | ++--------------------------------+ + +mysql> select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))); ++---------------------------------------------------------+ +| bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) | ++---------------------------------------------------------+ +| 1,2 | ++---------------------------------------------------------+ + +``` + +## keyword + + BITMAP_TO_STRING,BITMAP diff --git a/docs/documentation/en/sql-reference/sql-functions/hash-functions/index.rst b/docs/documentation/en/sql-reference/sql-functions/hash-functions/index.rst new file mode 100644 index 0000000000..5de92cdae4 --- /dev/null +++ b/docs/documentation/en/sql-reference/sql-functions/hash-functions/index.rst @@ -0,0 +1,8 @@ +======================= +Hash Functions +======================= + +.. toctree:: + :glob: + + * diff --git a/docs/documentation/en/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md b/docs/documentation/en/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md new file mode 100644 index 0000000000..1be7ff0699 --- /dev/null +++ b/docs/documentation/en/sql-reference/sql-functions/hash-functions/murmur_hash3_32.md @@ -0,0 +1,54 @@ + + +# murmur_hash3_32 + +## description +### Syntax + +`INT MURMUR_HASH3_32(VARCHAR input, ...)` + +Return the 32 bits murmur3 hash of input string. + +## example + +``` +mysql> select murmur_hash3_32(null); ++-----------------------+ +| murmur_hash3_32(NULL) | ++-----------------------+ +| NULL | ++-----------------------+ + +mysql> select murmur_hash3_32("hello"); ++--------------------------+ +| murmur_hash3_32('hello') | ++--------------------------+ +| 1321743225 | ++--------------------------+ + +mysql> select murmur_hash3_32("hello", "world"); ++-----------------------------------+ +| murmur_hash3_32('hello', 'world') | ++-----------------------------------+ +| 984713481 | ++-----------------------------------+ +``` + +## keyword + + MURMUR_HASH3_32,HASH diff --git a/fe/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/src/main/java/org/apache/doris/catalog/PrimitiveType.java index 3520803746..1e1fdc5e46 100644 --- a/fe/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -83,6 +83,7 @@ public enum PrimitiveType { builder.put(NULL_TYPE, DECIMALV2); builder.put(NULL_TYPE, CHAR); builder.put(NULL_TYPE, VARCHAR); + builder.put(NULL_TYPE, BITMAP); builder.put(NULL_TYPE, TIME); // Boolean builder.put(BOOLEAN, BOOLEAN); @@ -377,6 +378,7 @@ public enum PrimitiveType { compatibilityMatrix[NULL_TYPE.ordinal()][DECIMAL.ordinal()] = DECIMAL; compatibilityMatrix[NULL_TYPE.ordinal()][DECIMALV2.ordinal()] = DECIMALV2; compatibilityMatrix[NULL_TYPE.ordinal()][TIME.ordinal()] = TIME; + compatibilityMatrix[NULL_TYPE.ordinal()][BITMAP.ordinal()] = BITMAP; compatibilityMatrix[BOOLEAN.ordinal()][BOOLEAN.ordinal()] = BOOLEAN; compatibilityMatrix[BOOLEAN.ordinal()][TINYINT.ordinal()] = TINYINT; diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 6cdf7090fd..83b0eccee0 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -614,6 +614,14 @@ visible_functions = [ '_ZN5doris15BitmapFunctions9bitmap_orEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], [['bitmap_and'], 'BITMAP', ['BITMAP','BITMAP'], '_ZN5doris15BitmapFunctions10bitmap_andEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + [['bitmap_to_string'], 'VARCHAR', ['BITMAP'], + '_ZN5doris15BitmapFunctions16bitmap_to_stringEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + [['bitmap_from_string'], 'BITMAP', ['VARCHAR'], + '_ZN5doris15BitmapFunctions18bitmap_from_stringEPN9doris_udf15FunctionContextERKNS1_9StringValE'], + + # hash functions + [['murmur_hash3_32'], 'INT', ['VARCHAR', '...'], + '_ZN5doris13HashFunctions15murmur_hash3_32EPN9doris_udf15FunctionContextEiPKNS1_9StringValE'], # aes and base64 function [['aes_encrypt'], 'VARCHAR', ['VARCHAR', 'VARCHAR'],