From 3ee000c13cde28e5ae073dd6b2257f59bfdd5c99 Mon Sep 17 00:00:00 2001 From: Pxl <952130278@qq.com> Date: Sun, 30 Jan 2022 16:47:22 +0800 Subject: [PATCH] [chore] support build with libc++ && add some build config (#7903) support LIBCPP/LDD/BUILD_META_TOOL for build.sh --- be/CMakeLists.txt | 11 +++++++- be/src/agent/cgroups_mgr.h | 2 -- be/src/common/configbase.cpp | 4 +-- be/src/exprs/grouping_sets_functions.h | 2 -- be/src/exprs/json_functions.cpp | 20 +++++++------- be/src/exprs/math_functions.cpp | 16 +++++------ be/src/gutil/strings/memutil.h | 2 -- be/src/gutil/strtoint.cc | 3 +-- be/src/gutil/strtoint.h | 3 +-- be/src/gutil/sysinfo.cc | 2 +- be/src/http/http_parser.h | 2 +- be/src/olap/delete_handler.h | 1 - be/src/olap/olap_server.cpp | 6 ++++- be/src/olap/tablet_schema.h | 1 - be/src/runtime/data_stream_sender.cpp | 7 ++--- be/src/udf/udf_ir.cpp | 7 +++++ be/src/util/encryption_util.cpp | 7 ++--- be/src/util/errno.cpp | 1 + be/src/util/error_util.cc | 2 +- be/src/util/error_util.h | 1 - be/src/util/md5.h | 2 +- be/src/util/mysql_load_error_hub.h | 1 + be/src/util/string_parser.hpp | 35 ++++++++++++++----------- be/src/util/string_util.h | 11 ++++++++ be/src/util/utf8_check.cpp | 2 -- be/src/vec/functions/function_json.cpp | 6 ++--- be/src/vec/functions/modulo.cpp | 7 ++--- be/src/vec/olap/vcollect_iterator.h | 9 +++++++ be/src/vec/sink/vdata_stream_sender.cpp | 31 +++++++++++++--------- build.sh | 15 +++++++++++ run-be-ut.sh | 2 +- 31 files changed, 138 insertions(+), 83 deletions(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 74fc2666ab..9a9f0f182a 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -342,6 +342,15 @@ set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED") # Enable the cpu and heap profile of brpc set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBRPC_ENABLE_CPU_PROFILER") +if (USE_LDD) + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fuse-ld=lld") +endif () + +if (USE_LIBCPP AND COMPILER_CLANG) + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++ -lstdc++") + add_definitions(-DUSE_LIBCPP) +endif() + if (COMPILER_GCC) # Avoid GCC 11 false alarm # https://stackoverflow.com/questions/67584073/gcc-11-false-array-subscript-is-partly-outside-array-bounds-warning @@ -760,7 +769,7 @@ add_subdirectory(${SRC_DIR}/runtime) add_subdirectory(${SRC_DIR}/service) add_subdirectory(${SRC_DIR}/udf) -if (${MAKE_TEST} STREQUAL "OFF") +if (${BUILD_META_TOOL} STREQUAL "ON") add_subdirectory(${SRC_DIR}/tools) endif() diff --git a/be/src/agent/cgroups_mgr.h b/be/src/agent/cgroups_mgr.h index 4c30f190ba..de46a47eb2 100644 --- a/be/src/agent/cgroups_mgr.h +++ b/be/src/agent/cgroups_mgr.h @@ -18,8 +18,6 @@ #ifndef DORIS_BE_SRC_AGENT_CGROUPS_MGR_H #define DORIS_BE_SRC_AGENT_CGROUPS_MGR_H -#include - #include #include #include diff --git a/be/src/common/configbase.cpp b/be/src/common/configbase.cpp index 7593f76d60..3258da3e91 100644 --- a/be/src/common/configbase.cpp +++ b/be/src/common/configbase.cpp @@ -45,12 +45,12 @@ std::mutex mutable_string_config_lock; // trim string std::string& trim(std::string& s) { // rtrim - s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun(std::isspace))) + s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char c) { return !std::isspace(c); }) .base(), s.end()); // ltrim s.erase(s.begin(), - std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun(std::isspace)))); + std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isspace(c); })); return s; } diff --git a/be/src/exprs/grouping_sets_functions.h b/be/src/exprs/grouping_sets_functions.h index 05bb807657..524692bc50 100644 --- a/be/src/exprs/grouping_sets_functions.h +++ b/be/src/exprs/grouping_sets_functions.h @@ -21,8 +21,6 @@ namespace doris { -class Expr; - class GroupingSetsFunctions { public: static void init(); diff --git a/be/src/exprs/json_functions.cpp b/be/src/exprs/json_functions.cpp index 55e11aa204..bfe26c2d20 100644 --- a/be/src/exprs/json_functions.cpp +++ b/be/src/exprs/json_functions.cpp @@ -25,7 +25,6 @@ #include #include -#include #include #include #include @@ -41,6 +40,7 @@ #include "runtime/string_value.h" #include "runtime/tuple_row.h" #include "udf/udf.h" +#include "util/string_util.h" namespace doris { @@ -180,7 +180,9 @@ rapidjson::Value JsonFunctions::parse_str_with_flag(const StringVal& arg, const val.SetDouble(number); } else if (*(flag.ptr + num) == '4' || *(flag.ptr + num) == '5') { StringPiece str((char*)arg.ptr, arg.len); - if (*(flag.ptr + num) == '4') str = str.substr(1, str.length() - 2); + if (*(flag.ptr + num) == '4') { + str = str.substr(1, str.length() - 2); + } val.SetString(str.data(), str.length(), allocator); } else { DCHECK(false) << "parse json type error with unknown type"; @@ -310,24 +312,24 @@ rapidjson::Value* JsonFunctions::get_json_object(FunctionContext* context, // '$.text#abc.xyz' -> [$, text#abc, xyz] // '$."text.abc".xyz' -> [$, text.abc, xyz] // '$."text.abc"[1].xyz' -> [$, text.abc[1], xyz] - JsonState* json_state; + JsonState* json_state = nullptr; JsonState tmp_json_state; + #ifndef BE_TEST - json_state = reinterpret_cast(context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); + json_state = reinterpret_cast( + context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); if (json_state == nullptr) { json_state = &tmp_json_state; } if (json_state->json_paths.size() == 0) { - boost::tokenizer> tok( - path_string, boost::escaped_list_separator("\\", ".", "\"")); + auto tok = get_json_token(path_string); std::vector paths(tok.begin(), tok.end()); get_parsed_paths(paths, &json_state->json_paths); } #else json_state = &tmp_json_state; - boost::tokenizer> tok( - path_string, boost::escaped_list_separator("\\", ".", "\"")); + auto tok = get_json_token(path_string); std::vector paths(tok.begin(), tok.end()); get_parsed_paths(paths, &json_state->json_paths); #endif @@ -353,7 +355,7 @@ rapidjson::Value* JsonFunctions::get_json_object(FunctionContext* context, //rapidjson::Document document; if (UNLIKELY(document->HasParseError())) { VLOG_CRITICAL << "Error at offset " << document->GetErrorOffset() << ": " - << GetParseError_En(document->GetParseError()); + << GetParseError_En(document->GetParseError()); document->SetNull(); return document; } diff --git a/be/src/exprs/math_functions.cpp b/be/src/exprs/math_functions.cpp index 00cf3bd9ea..1bdd83f429 100644 --- a/be/src/exprs/math_functions.cpp +++ b/be/src/exprs/math_functions.cpp @@ -27,8 +27,8 @@ #include "common/compiler_util.h" #include "exprs/anyval_util.h" #include "runtime/decimalv2_value.h" -#include "util/string_parser.hpp" #include "util/simd/vstring_function.h" +#include "util/string_parser.hpp" namespace doris { @@ -113,7 +113,7 @@ DecimalV2Val MathFunctions::abs(FunctionContext* ctx, const doris_udf::DecimalV2 if (UNLIKELY(val.val == MIN_INT128)) { return DecimalV2Val::null(); } else { - return DecimalV2Val(::abs(val.val)); + return DecimalV2Val(val.val > 0 ? val.val : -val.val); } } @@ -124,7 +124,7 @@ LargeIntVal MathFunctions::abs(FunctionContext* ctx, const doris_udf::LargeIntVa if (UNLIKELY(val.val == MIN_INT128)) { return LargeIntVal::null(); } else { - return LargeIntVal(::abs(val.val)); + return LargeIntVal(val.val > 0 ? val.val : -val.val); } } @@ -132,28 +132,28 @@ LargeIntVal MathFunctions::abs(FunctionContext* ctx, const doris_udf::BigIntVal& if (val.is_null) { return LargeIntVal::null(); } - return LargeIntVal(::abs(__int128(val.val))); + return LargeIntVal(__int128(std::abs(val.val))); } BigIntVal MathFunctions::abs(FunctionContext* ctx, const doris_udf::IntVal& val) { if (val.is_null) { return BigIntVal::null(); } - return BigIntVal(::abs(int64_t(val.val))); + return BigIntVal(int64_t(std::abs(val.val))); } IntVal MathFunctions::abs(FunctionContext* ctx, const doris_udf::SmallIntVal& val) { if (val.is_null) { return IntVal::null(); } - return IntVal(::abs(int32_t(val.val))); + return IntVal(int32_t(std::abs(val.val))); } SmallIntVal MathFunctions::abs(FunctionContext* ctx, const doris_udf::TinyIntVal& val) { if (val.is_null) { return SmallIntVal::null(); } - return SmallIntVal(::abs(int16_t(val.val))); + return SmallIntVal(int16_t(std::abs(val.val))); } // Generates a UDF that always calls FN() on the input val and returns it. @@ -352,7 +352,7 @@ StringVal MathFunctions::hex_string(FunctionContext* ctx, const StringVal& s) { } StringVal result = StringVal::create_temp_string_val(ctx, s.len * 2); - simd::VStringFunctions::hex_encode(s.ptr, s.len, reinterpret_cast(result.ptr)); + simd::VStringFunctions::hex_encode(s.ptr, s.len, reinterpret_cast(result.ptr)); return result; } diff --git a/be/src/gutil/strings/memutil.h b/be/src/gutil/strings/memutil.h index 83425da9b5..ee0eeb774b 100644 --- a/be/src/gutil/strings/memutil.h +++ b/be/src/gutil/strings/memutil.h @@ -56,8 +56,6 @@ #include #include // to get the POSIX mem*() routines -#include "gutil/port.h" // disable some warnings on Windows - inline char* memcat(char* dest, size_t destlen, const char* src, size_t srclen) { return reinterpret_cast(memcpy(dest + destlen, src, srclen)); } diff --git a/be/src/gutil/strtoint.cc b/be/src/gutil/strtoint.cc index b3b711f323..d0aeff5c39 100644 --- a/be/src/gutil/strtoint.cc +++ b/be/src/gutil/strtoint.cc @@ -7,8 +7,7 @@ #include "gutil/strtoint.h" #include - -#include "gutil/port.h" +#include // Replacement strto[u]l functions that have identical overflow and underflow // characteristics for both ILP-32 and LP-64 platforms, including errno diff --git a/be/src/gutil/strtoint.h b/be/src/gutil/strtoint.h index fb839034df..c253b93503 100644 --- a/be/src/gutil/strtoint.h +++ b/be/src/gutil/strtoint.h @@ -31,12 +31,11 @@ #define BASE_STRTOINT_H_ #include // For strtol* functions. - #include + using std::string; #include "gutil/integral_types.h" #include "gutil/macros.h" -#include "gutil/port.h" // Adapter functions for handling overflow and errno. int32 strto32_adapter(const char* nptr, char** endptr, int base); diff --git a/be/src/gutil/sysinfo.cc b/be/src/gutil/sysinfo.cc index ce43f5fa06..cfd325c483 100644 --- a/be/src/gutil/sysinfo.cc +++ b/be/src/gutil/sysinfo.cc @@ -35,6 +35,7 @@ #include #include // for open() #include // for read() +#include #if defined __MACH__ // Mac OS X, almost certainly #include // how we figure out numcpu's on OS X @@ -50,7 +51,6 @@ #endif #include - #include #include // for errno #include // for snprintf(), sscanf() diff --git a/be/src/http/http_parser.h b/be/src/http/http_parser.h index c211cae437..6d4b1d445c 100644 --- a/be/src/http/http_parser.h +++ b/be/src/http/http_parser.h @@ -19,7 +19,7 @@ #define DORIS_BE_SRC_COMMON_UTIL_HTTP_PARSER_H #include - +#include #include namespace doris { diff --git a/be/src/olap/delete_handler.h b/be/src/olap/delete_handler.h index 0f2693a27b..d95f0f9ac8 100644 --- a/be/src/olap/delete_handler.h +++ b/be/src/olap/delete_handler.h @@ -26,7 +26,6 @@ #include "olap/block_column_predicate.h" #include "olap/column_predicate.h" #include "olap/olap_define.h" -#include "olap/tablet_schema.h" namespace doris { diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index fafbde5c07..7acca7080b 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include "agent/cgroups_mgr.h" #include "common/status.h" @@ -412,7 +413,10 @@ std::vector StorageEngine::_generate_compaction_tasks( std::vector tablets_compaction; uint32_t max_compaction_score = 0; - std::random_shuffle(data_dirs.begin(), data_dirs.end()); + + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(data_dirs.begin(), data_dirs.end(), g); // Copy _tablet_submitted_xxx_compaction map so that we don't need to hold _tablet_submitted_compaction_mutex // when travesing the data dir diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index ffd8c34bc4..becee2e64b 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -22,7 +22,6 @@ #include "gen_cpp/olap_file.pb.h" #include "olap/olap_define.h" -#include "olap/tablet_schema.h" #include "olap/types.h" namespace doris { diff --git a/be/src/runtime/data_stream_sender.cpp b/be/src/runtime/data_stream_sender.cpp index efd92606e3..8c306de4ed 100644 --- a/be/src/runtime/data_stream_sender.cpp +++ b/be/src/runtime/data_stream_sender.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include "common/config.h" @@ -377,9 +378,9 @@ Status DataStreamSender::prepare(RuntimeState* state) { state->instance_mem_tracker()); if (_part_type == TPartitionType::UNPARTITIONED || _part_type == TPartitionType::RANDOM) { - // Randomize the order we open/transmit to channels to avoid thundering herd problems. - srand(reinterpret_cast(this)); - random_shuffle(_channels.begin(), _channels.end()); + std::random_device rd; + std::mt19937 g(rd()); + shuffle(_channels.begin(), _channels.end(), g); } else if (_part_type == TPartitionType::HASH_PARTITIONED || _part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED) { RETURN_IF_ERROR(Expr::prepare(_partition_expr_ctxs, state, _row_desc, _expr_mem_tracker)); diff --git a/be/src/udf/udf_ir.cpp b/be/src/udf/udf_ir.cpp index a1ceb02a3c..416a52025a 100644 --- a/be/src/udf/udf_ir.cpp +++ b/be/src/udf/udf_ir.cpp @@ -15,7 +15,14 @@ // specific language governing permissions and limitations // under the License. +#include + #include "udf/udf_internal.h" +#include "udf/udf.h" + +namespace doris { +class ColumnPtrWrapper; +} // namespace doris namespace doris_udf { bool FunctionContext::is_arg_constant(int i) const { diff --git a/be/src/util/encryption_util.cpp b/be/src/util/encryption_util.cpp index 5854a7ff25..eb95cf55bc 100644 --- a/be/src/util/encryption_util.cpp +++ b/be/src/util/encryption_util.cpp @@ -17,14 +17,11 @@ #include "util/encryption_util.h" -#include #include #include - +#include +#include #include -#include -#include -#include namespace doris { diff --git a/be/src/util/errno.cpp b/be/src/util/errno.cpp index df72495bce..3be386ddab 100644 --- a/be/src/util/errno.cpp +++ b/be/src/util/errno.cpp @@ -17,6 +17,7 @@ #include "util/errno.h" +#include #include #include "gutil/dynamic_annotations.h" diff --git a/be/src/util/error_util.cc b/be/src/util/error_util.cc index a7c76fd864..031bf8c109 100644 --- a/be/src/util/error_util.cc +++ b/be/src/util/error_util.cc @@ -18,9 +18,9 @@ #include "util/error_util.h" #include - #include #include +#include using std::string; using std::stringstream; diff --git a/be/src/util/error_util.h b/be/src/util/error_util.h index 4c6bfabcd0..ff1e0786d0 100644 --- a/be/src/util/error_util.h +++ b/be/src/util/error_util.h @@ -19,7 +19,6 @@ #define DORIS_BE_SRC_UTIL_ERROR_UTIL_H #include -#include // #include "gen-cpp/CatalogObjects_types.h" // #include "gen-cpp/ErrorCodes_types.h" diff --git a/be/src/util/md5.h b/be/src/util/md5.h index 130fc1c099..064040c859 100644 --- a/be/src/util/md5.h +++ b/be/src/util/md5.h @@ -18,7 +18,7 @@ #pragma once #include - +#include #include namespace doris { diff --git a/be/src/util/mysql_load_error_hub.h b/be/src/util/mysql_load_error_hub.h index aaa31208f7..0b51ebb15d 100644 --- a/be/src/util/mysql_load_error_hub.h +++ b/be/src/util/mysql_load_error_hub.h @@ -18,6 +18,7 @@ #ifndef DORIS_BE_SRC_UTIL_MYSQL_LOAD_ERROR_HUB_H #define DORIS_BE_SRC_UTIL_MYSQL_LOAD_ERROR_HUB_H +#include #include #include #include diff --git a/be/src/util/string_parser.hpp b/be/src/util/string_parser.hpp index cc1110c7a2..cfb9a487ef 100644 --- a/be/src/util/string_parser.hpp +++ b/be/src/util/string_parser.hpp @@ -134,10 +134,9 @@ public: return string_to_bool_internal(s + i, len - i, result); } - static inline __int128 string_to_decimal(const char* s, int len, - int type_precision, int type_scale, ParseResult* result); + static inline __int128 string_to_decimal(const char* s, int len, int type_precision, + int type_scale, ParseResult* result); - template static Status split_string_to_map(const std::string& base, const T element_separator, const T key_value_separator, @@ -146,23 +145,23 @@ public: int key_end; int val_pos; int val_end; - + while ((key_end = base.find(key_value_separator, key_pos)) != std::string::npos) { - if ((val_pos = base.find_first_not_of(key_value_separator, key_end)) - == std::string::npos) { + if ((val_pos = base.find_first_not_of(key_value_separator, key_end)) == + std::string::npos) { break; } - if ((val_end = base.find(element_separator, val_pos)) == std::string::npos) { + if ((val_end = base.find(element_separator, val_pos)) == std::string::npos) { val_end = base.size(); } - result->insert(std::make_pair(base.substr(key_pos, key_end - key_pos), - base.substr(val_pos, val_end - val_pos))); + result->insert(std::make_pair(base.substr(key_pos, key_end - key_pos), + base.substr(val_pos, val_end - val_pos))); key_pos = val_end; if (key_pos != std::string::npos) { - ++key_pos; + ++key_pos; } } - + return Status::OK(); } private: @@ -659,7 +658,9 @@ inline __int128 StringParser::get_scale_multiplier(int scale) { static_cast<__int128>(1000000000000000000ll) * 100000000000000000ll * 10ll, static_cast<__int128>(1000000000000000000ll) * 100000000000000000ll * 100ll, static_cast<__int128>(1000000000000000000ll) * 100000000000000000ll * 1000ll}; - if (scale >= 0 && scale < 39) return values[scale]; + if (scale >= 0 && scale < 39) { + return values[scale]; + } return -1; // Overflow } @@ -773,7 +774,9 @@ inline __int128 StringParser::string_to_decimal(const char* s, int len, } // Ex: 0.001, at this point would have precision 1 and scale 3 since leading zeros // were ignored during previous parsing. - if (scale > precision) precision = scale; + if (scale > precision) { + precision = scale; + } // Microbenchmarks show that beyond this point, returning on parse failure is slower // than just letting the function run out. @@ -783,13 +786,15 @@ inline __int128 StringParser::string_to_decimal(const char* s, int len, } else if (UNLIKELY(scale > type_scale)) { *result = StringParser::PARSE_UNDERFLOW; int shift = scale - type_scale; - if (UNLIKELY(truncated_digit_count > 0)) shift -= truncated_digit_count; + if (UNLIKELY(truncated_digit_count > 0)) { + shift -= truncated_digit_count; + } if (shift > 0) { __int128 divisor = get_scale_multiplier(shift); if (LIKELY(divisor >= 0)) { value /= divisor; __int128 remainder = value % divisor; - if (abs(remainder) >= (divisor >> 1)) { + if ((remainder > 0 ? remainder : -remainder) >= (divisor >> 1)) { value += 1; } } else { diff --git a/be/src/util/string_util.h b/be/src/util/string_util.h index d5c7fe822c..7fce0a8500 100644 --- a/be/src/util/string_util.h +++ b/be/src/util/string_util.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -129,4 +130,14 @@ template using StringCaseUnorderedMap = std::unordered_map; +inline auto get_json_token(const std::string_view& path_string) { +#ifdef USE_LIBCPP + return boost::tokenizer>( + std::string(path_string), boost::escaped_list_separator("\\", ".", "\"")); +#else + return boost::tokenizer>( + path_string, boost::escaped_list_separator("\\", ".", "\"")); +#endif +} + } // namespace doris diff --git a/be/src/util/utf8_check.cpp b/be/src/util/utf8_check.cpp index 0d58c50e10..5355b90142 100644 --- a/be/src/util/utf8_check.cpp +++ b/be/src/util/utf8_check.cpp @@ -9,8 +9,6 @@ #include "util/utf8_check.h" -#include - #if defined(__i386) || defined(__x86_64__) #include "util/simdutf8check.h" #elif defined(__aarch64__) diff --git a/be/src/vec/functions/function_json.cpp b/be/src/vec/functions/function_json.cpp index fdfab43f64..dd2da73b05 100644 --- a/be/src/vec/functions/function_json.cpp +++ b/be/src/vec/functions/function_json.cpp @@ -18,10 +18,11 @@ #include #include #include + #include -#include #include "exprs/json_functions.h" +#include "util/string_util.h" #include "vec/columns/column_nullable.h" #include "vec/columns/column_string.h" #include "vec/columns/column_vector.h" @@ -185,8 +186,7 @@ rapidjson::Value* get_json_object(const std::string_view& json_string, std::vector* parsed_paths; std::vector tmp_parsed_paths; - boost::tokenizer> tok( - path_string, boost::escaped_list_separator("\\", ".", "\"")); + auto tok = get_json_token(path_string); std::vector paths(tok.begin(), tok.end()); get_parsed_paths(paths, &tmp_parsed_paths); parsed_paths = &tmp_parsed_paths; diff --git a/be/src/vec/functions/modulo.cpp b/be/src/vec/functions/modulo.cpp index 94a02d01ba..e0d2e6dd7d 100644 --- a/be/src/vec/functions/modulo.cpp +++ b/be/src/vec/functions/modulo.cpp @@ -39,7 +39,7 @@ struct ModuloImpl { static inline Result apply(A a, B b, NullMap& null_map, size_t index) { if constexpr (std::is_floating_point_v) { null_map[index] = 0; - return fmod(a, b); + return std::fmod((double)a, (double)b); } else { null_map[index] = b == 0; return typename NumberTraits::ToInteger::Type(a) % @@ -47,7 +47,8 @@ struct ModuloImpl { } } - static inline DecimalV2Value apply(DecimalV2Value a, DecimalV2Value b, NullMap& null_map, size_t index) { + static inline DecimalV2Value apply(DecimalV2Value a, DecimalV2Value b, NullMap& null_map, + size_t index) { null_map[index] = b == DecimalV2Value(0); return a % (b + DecimalV2Value(b == DecimalV2Value(0))); } @@ -64,7 +65,7 @@ struct ModuloByConstantImpl : BinaryOperationImplBase> { static void vector_constant(const PaddedPODArray& a, B b, PaddedPODArray& c) { // TODO: Support return NULL in the future if (UNLIKELY(b == 0)) { -// throw Exception("Division by zero", TStatusCode::VEC_ILLEGAL_DIVISION); + // throw Exception("Division by zero", TStatusCode::VEC_ILLEGAL_DIVISION); memset(c.data(), 0, sizeof(ResultType) * c.size()); return; } diff --git a/be/src/vec/olap/vcollect_iterator.h b/be/src/vec/olap/vcollect_iterator.h index 6cf9d625be..6c525bef58 100644 --- a/be/src/vec/olap/vcollect_iterator.h +++ b/be/src/vec/olap/vcollect_iterator.h @@ -17,7 +17,11 @@ #pragma once +#ifdef USE_LIBCPP +#include +#else #include +#endif #include "olap/olap_define.h" #include "olap/reader.h" @@ -106,8 +110,13 @@ private: int _sequence; }; +#ifdef USE_LIBCPP + using MergeHeap = std::priority_queue, + LevelIteratorComparator>; +#else using MergeHeap = __gnu_pbds::priority_queue; +#endif // Iterate from rowset reader. This Iterator usually like a leaf node class Level0Iterator : public LevelIterator { diff --git a/be/src/vec/sink/vdata_stream_sender.cpp b/be/src/vec/sink/vdata_stream_sender.cpp index 8afa66b2d7..e3e081df0a 100644 --- a/be/src/vec/sink/vdata_stream_sender.cpp +++ b/be/src/vec/sink/vdata_stream_sender.cpp @@ -20,6 +20,8 @@ #include #include +#include + #include "runtime/client_cache.h" #include "runtime/dpp_sink_internal.h" #include "runtime/exec_env.h" @@ -69,11 +71,11 @@ Status VDataStreamSender::Channel::init(RuntimeState* state) { } Status VDataStreamSender::Channel::send_current_block(bool eos) { -// TODO: Now, local exchange will cause the performance problem is in a multi-threaded scenario -// so this feature is turned off here. We need to re-examine this logic -// if (is_local()) { -// return send_local_block(eos); -// } + // TODO: Now, local exchange will cause the performance problem is in a multi-threaded scenario + // so this feature is turned off here. We need to re-examine this logic + // if (is_local()) { + // return send_local_block(eos); + // } { SCOPED_TIMER(_parent->_serialize_batch_timer); _pb_block.Clear(); @@ -184,8 +186,9 @@ Status VDataStreamSender::Channel::add_rows(Block* block, const std::vector const int* begin = &rows[0]; while (row_wait_add > 0) { - int row_add, max_add = batch_size - _mutable_block->rows(); - if (row_wait_add >= max_add) { + int row_add = 0; + int max_add = batch_size - _mutable_block->rows(); + if (row_wait_add >= max_add) { row_add = max_add; } else { row_add = row_wait_add; @@ -343,9 +346,9 @@ Status VDataStreamSender::prepare(RuntimeState* state) { state->instance_mem_tracker()); if (_part_type == TPartitionType::UNPARTITIONED || _part_type == TPartitionType::RANDOM) { - // Randomize the order we open/transmit to channels to avoid thundering herd problems. - srand(reinterpret_cast(this)); - random_shuffle(_channels.begin(), _channels.end()); + std::random_device rd; + std::mt19937 g(rd()); + shuffle(_channels.begin(), _channels.end(), g); } else if (_part_type == TPartitionType::HASH_PARTITIONED || _part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED) { RETURN_IF_ERROR(VExpr::prepare(_partition_expr_ctxs, state, _row_desc, _expr_mem_tracker)); @@ -446,7 +449,7 @@ Status VDataStreamSender::send(RuntimeState* state, Block* block) { } } - // channel2rows' subscript means channel id + // channel2rows' subscript means channel id std::vector hash_vals(rows); for (int i = 0; i < rows; i++) { hash_vals[i] = siphashs[i].get64(); @@ -479,12 +482,14 @@ Status VDataStreamSender::send(RuntimeState* state, Block* block) { hash_vals[i] = RawValue::zlib_crc32(&INT_VALUE, INT_TYPE, hash_vals[i]); } else { hash_vals[i] = RawValue::zlib_crc32(val.data, val.size, - _partition_expr_ctxs[j]->root()->type(), hash_vals[i]); + _partition_expr_ctxs[j]->root()->type(), + hash_vals[i]); } } } - RETURN_IF_ERROR(channel_add_rows(_channel_shared_ptrs, num_channels, hash_vals, rows, block)); + RETURN_IF_ERROR( + channel_add_rows(_channel_shared_ptrs, num_channels, hash_vals, rows, block)); } else { // Range partition // 1. caculate range diff --git a/build.sh b/build.sh index df1f7edcee..a4dd472ecf 100755 --- a/build.sh +++ b/build.sh @@ -170,6 +170,15 @@ fi if [[ -z ${WITH_LZO} ]]; then WITH_LZO=OFF fi +if [[ -z ${USE_LIBCPP} ]]; then + USE_LIBCPP=OFF +fi +if [[ -z ${BUILD_META_TOOL} ]]; then + BUILD_META_TOOL=ON +fi +if [[ -z ${USE_LDD} ]]; then + USE_LDD=OFF +fi echo "Get params: BUILD_BE -- $BUILD_BE @@ -182,6 +191,9 @@ echo "Get params: WITH_LZO -- $WITH_LZO GLIBC_COMPATIBILITY -- $GLIBC_COMPATIBILITY USE_AVX2 -- $USE_AVX2 + USE_LIBCPP -- $USE_LIBCPP + BUILD_META_TOOL -- $BUILD_META_TOOL + USE_LDD -- $USE_LDD " # Clean and build generated code @@ -211,6 +223,9 @@ if [ ${BUILD_BE} -eq 1 ] ; then ${CMAKE_USE_CCACHE} \ -DWITH_MYSQL=${WITH_MYSQL} \ -DWITH_LZO=${WITH_LZO} \ + -DUSE_LIBCPP=${USE_LIBCPP} \ + -DBUILD_META_TOOL=${BUILD_META_TOOL} \ + -DUSE_LDD=${USE_LDD} \ -DUSE_AVX2=${USE_AVX2} \ -DGLIBC_COMPATIBILITY=${GLIBC_COMPATIBILITY} ../ ${BUILD_SYSTEM} -j ${PARALLEL} diff --git a/run-be-ut.sh b/run-be-ut.sh index 66b4b67835..1aa420b541 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -127,7 +127,7 @@ else fi cd ${CMAKE_BUILD_DIR} -${CMAKE_CMD} -G "${GENERATOR}" ../ -DWITH_MYSQL=OFF -DMAKE_TEST=ON -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ +${CMAKE_CMD} -G "${GENERATOR}" ../ -DWITH_MYSQL=OFF -DMAKE_TEST=ON -DBUILD_META_TOOL=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DGLIBC_COMPATIBILITY=${GLIBC_COMPATIBILITY} ${CMAKE_USE_CCACHE} ${BUILD_SYSTEM} -j ${PARALLEL} $RUN_FILE