From f33ef32d92b2fa348abe9566fb569657e94c8b13 Mon Sep 17 00:00:00 2001 From: Pxl Date: Sat, 28 May 2022 17:33:55 +0800 Subject: [PATCH] [Bug] [Bitmap] change to_bitmap to always_not_nullable (#9716) --- be/src/exec/tablet_sink.cpp | 5 +++-- be/src/exprs/bitmap_function.cpp | 20 ++++++++++---------- be/src/vec/functions/function_bitmap.cpp | 10 ++++------ be/test/exprs/bitmap_function_test.cpp | 6 +++--- gensrc/script/doris_builtins_functions.py | 4 ++-- 5 files changed, 22 insertions(+), 23 deletions(-) diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp index 1aa1bc5d5a..3166736e7d 100644 --- a/be/src/exec/tablet_sink.cpp +++ b/be/src/exec/tablet_sink.cpp @@ -1102,8 +1102,9 @@ Status OlapTableSink::_convert_batch(RuntimeState* state, RowBatch* input_batch, []() -> std::string { return ""; }, [&]() -> std::string { fmt::memory_buffer buf; - fmt::format_to(buf, "null value for not null column, column={}", - slot_desc->col_name()); + fmt::format_to( + buf, "null value for not null column, column={}, type={}", + slot_desc->col_name(), slot_desc->type().debug_string()); return fmt::to_string(buf); }, &stop_processing)); diff --git a/be/src/exprs/bitmap_function.cpp b/be/src/exprs/bitmap_function.cpp index ed9c7b2825..d49863ab73 100644 --- a/be/src/exprs/bitmap_function.cpp +++ b/be/src/exprs/bitmap_function.cpp @@ -366,17 +366,17 @@ BigIntVal BitmapFunctions::bitmap_min(FunctionContext* ctx, const StringVal& src StringVal BitmapFunctions::to_bitmap(doris_udf::FunctionContext* ctx, const doris_udf::StringVal& src) { - if (src.is_null) { - return StringVal::null(); - } - StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS; - uint64_t int_value = StringParser::string_to_unsigned_int( - reinterpret_cast(src.ptr), src.len, &parse_result); - if (UNLIKELY(parse_result != StringParser::PARSE_SUCCESS)) { - return StringVal::null(); - } BitmapValue bitmap; - bitmap.add(int_value); + + if (!src.is_null) { + StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS; + uint64_t int_value = StringParser::string_to_unsigned_int( + reinterpret_cast(src.ptr), src.len, &parse_result); + if (parse_result == StringParser::PARSE_SUCCESS) { + bitmap.add(int_value); + } + } + return serialize(ctx, &bitmap); } diff --git a/be/src/vec/functions/function_bitmap.cpp b/be/src/vec/functions/function_bitmap.cpp index f4752a3052..2c3b4f02dc 100644 --- a/be/src/vec/functions/function_bitmap.cpp +++ b/be/src/vec/functions/function_bitmap.cpp @@ -45,18 +45,16 @@ struct ToBitmap { auto size = offsets.size(); res.reserve(size); for (size_t i = 0; i < size; ++i) { + res.emplace_back(); + const char* raw_str = reinterpret_cast(&data[offsets[i - 1]]); size_t str_size = offsets[i] - offsets[i - 1] - 1; StringParser::ParseResult parse_result = StringParser::PARSE_SUCCESS; uint64_t int_value = StringParser::string_to_unsigned_int(raw_str, str_size, &parse_result); - if (UNLIKELY(parse_result != StringParser::PARSE_SUCCESS)) { - res.emplace_back(); - null_map[i] = 1; - continue; + if (LIKELY(parse_result == StringParser::PARSE_SUCCESS)) { + res.back().add(int_value); } - res.emplace_back(); - res.back().add(int_value); } return Status::OK(); } diff --git a/be/test/exprs/bitmap_function_test.cpp b/be/test/exprs/bitmap_function_test.cpp index 69c465ec86..ddf8a52ea3 100644 --- a/be/test/exprs/bitmap_function_test.cpp +++ b/be/test/exprs/bitmap_function_test.cpp @@ -86,19 +86,19 @@ TEST_F(BitmapFunctionsTest, to_bitmap_null) { StringVal input = StringVal::null(); StringVal result = BitmapFunctions::to_bitmap(ctx, input); - EXPECT_EQ(StringVal::null(), result); + EXPECT_EQ(BitmapFunctions::bitmap_empty(ctx), result); } TEST_F(BitmapFunctionsTest, to_bitmap_invalid_argument) { StringVal input = AnyValUtil::from_string_temp(ctx, std::string("-1")); StringVal result = BitmapFunctions::to_bitmap(ctx, input); - EXPECT_EQ(StringVal::null(), result); + EXPECT_EQ(BitmapFunctions::bitmap_empty(ctx), result); } TEST_F(BitmapFunctionsTest, to_bitmap_out_of_range) { StringVal input = AnyValUtil::from_string_temp(ctx, std::string("18446744073709551616")); StringVal result = BitmapFunctions::to_bitmap(ctx, input); - EXPECT_EQ(StringVal::null(), result); + EXPECT_EQ(BitmapFunctions::bitmap_empty(ctx), result); } TEST_F(BitmapFunctionsTest, bitmap_union_int) { diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 2d2d7d1bbf..1831e09cf6 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -1175,13 +1175,13 @@ visible_functions = [ [['to_bitmap'], 'BITMAP', ['VARCHAR'], '_ZN5doris15BitmapFunctions9to_bitmapEPN9doris_udf15FunctionContextERKNS1_9StringValE', - '', '', 'vec', 'ALWAYS_NULLABLE'], + '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], [['bitmap_hash'], 'BITMAP', ['VARCHAR'], '_ZN5doris15BitmapFunctions11bitmap_hashEPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], [['to_bitmap'], 'BITMAP', ['STRING'], '_ZN5doris15BitmapFunctions9to_bitmapEPN9doris_udf15FunctionContextERKNS1_9StringValE', - '', '', 'vec', 'ALWAYS_NULLABLE'], + '', '', 'vec', 'ALWAYS_NOT_NULLABLE'], [['bitmap_hash'], 'BITMAP', ['STRING'], '_ZN5doris15BitmapFunctions11bitmap_hashEPN9doris_udf15FunctionContextERKNS1_9StringValE', '', '', 'vec', 'ALWAYS_NOT_NULLABLE'],