diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index a7b726c55f..192ee3f406 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -258,6 +258,9 @@ set_target_properties(bitshuffle PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/ add_library(roaring STATIC IMPORTED) set_target_properties(roaring PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libroaring.a) +add_library(fmt STATIC IMPORTED) +set_target_properties(fmt PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libfmt.a) + add_library(orc STATIC IMPORTED) set_target_properties(orc PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/liborc.a) @@ -478,6 +481,7 @@ set(DORIS_DEPENDENCIES leveldb bitshuffle roaring + fmt double-conversion jemalloc brotlicommon diff --git a/be/src/runtime/mysql_result_writer.cpp b/be/src/runtime/mysql_result_writer.cpp index 86e4675c59..d56e8acc83 100644 --- a/be/src/runtime/mysql_result_writer.cpp +++ b/be/src/runtime/mysql_result_writer.cpp @@ -66,7 +66,6 @@ void MysqlResultWriter::_init_profile() { } Status MysqlResultWriter::_add_one_row(TupleRow* row) { - SCOPED_TIMER(_convert_tuple_timer); _row_buffer->reset(); int num_columns = _output_expr_ctxs.size(); int buf_ret = 0; @@ -213,18 +212,22 @@ Status MysqlResultWriter::append_row_batch(const RowBatch* batch) { int num_rows = batch->num_rows(); result->result_batch.rows.resize(num_rows); - for (int i = 0; status.ok() && i < num_rows; ++i) { - TupleRow* row = batch->get_row(i); - status = _add_one_row(row); + { + SCOPED_TIMER(_convert_tuple_timer); + for (int i = 0; status.ok() && i < num_rows; ++i) { + TupleRow* row = batch->get_row(i); + status = _add_one_row(row); - if (status.ok()) { - result->result_batch.rows[i].assign(_row_buffer->buf(), _row_buffer->length()); - } else { - LOG(WARNING) << "convert row to mysql result failed."; - break; + if (status.ok()) { + result->result_batch.rows[i].assign(_row_buffer->buf(), _row_buffer->length()); + } else { + LOG(WARNING) << "convert row to mysql result failed."; + break; + } } } + if (status.ok()) { SCOPED_TIMER(_result_send_timer); // push this batch to back diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp index d5bbd93881..51b2bcee85 100644 --- a/be/src/util/mysql_row_buffer.cpp +++ b/be/src/util/mysql_row_buffer.cpp @@ -24,6 +24,8 @@ #include "gutil/strings/numbers.h" #include "util/mysql_global.h" +#include + namespace doris { // the first byte: @@ -97,6 +99,16 @@ int MysqlRowBuffer::reserve(int size) { return 0; } +template +static char* add_int(T data, char* pos) +{ + auto fi = fmt::format_int(data); + int length = fi.size(); + int1store(pos++, length); + memcpy(pos, fi.data(), length); + return pos + length; +} + int MysqlRowBuffer::push_tinyint(int8_t data) { // 1 for string trail, 1 for length, 1 for sign, other for digits int ret = reserve(3 + MAX_TINYINT_WIDTH); @@ -106,15 +118,7 @@ int MysqlRowBuffer::push_tinyint(int8_t data) { return ret; } - int length = snprintf(_pos + 1, MAX_TINYINT_WIDTH + 2, "%d", data); - - if (length < 0) { - LOG(ERROR) << "snprintf failed. data = " << data; - return length; - } - - int1store(_pos, length); - _pos += length + 1; + _pos = add_int(data, _pos); return 0; } @@ -127,15 +131,7 @@ int MysqlRowBuffer::push_smallint(int16_t data) { return ret; } - int length = snprintf(_pos + 1, MAX_SMALLINT_WIDTH + 2, "%d", data); - - if (length < 0) { - LOG(ERROR) << "snprintf failed. data = " << data; - return length; - } - - int1store(_pos, length); - _pos += length + 1; + _pos = add_int(data, _pos); return 0; } @@ -148,15 +144,7 @@ int MysqlRowBuffer::push_int(int32_t data) { return ret; } - int length = snprintf(_pos + 1, MAX_INT_WIDTH + 2, "%d", data); - - if (length < 0) { - LOG(ERROR) << "snprintf failed. data = " << data; - return length; - } - - int1store(_pos, length); - _pos += length + 1; + _pos = add_int(data, _pos); return 0; } @@ -169,15 +157,7 @@ int MysqlRowBuffer::push_bigint(int64_t data) { return ret; } - int length = snprintf(_pos + 1, MAX_BIGINT_WIDTH + 2, "%ld", data); - - if (length < 0) { - LOG(ERROR) << "snprintf failed. data = " << data; - return length; - } - - int1store(_pos, length); - _pos += length + 1; + _pos = add_int(data, _pos); return 0; } diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 98b28b7e91..ca0ae841a6 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -671,6 +671,17 @@ build_croaringbitmap() { -DENABLE_ROARING_TESTS=OFF .. ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } + +# fmt +build_fmt() { + check_if_source_exist $FMT_SOURCE + cd $TP_SOURCE_DIR/$FMT_SOURCE + mkdir -p $BUILD_DIR && cd $BUILD_DIR + rm -rf CMakeCache.txt CMakeFiles/ + $CMAKE_CMD -G "${GENERATOR}" -v -DBUILD_SHARED_LIBS=FALSE -DFMT_TEST=OFF -DFMT_DOC=OFF -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR .. + ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install +} + #orc build_orc() { check_if_source_exist $ORC_SOURCE @@ -825,6 +836,7 @@ build_arrow build_s2 build_bitshuffle build_croaringbitmap +build_fmt build_orc build_cctz build_tsan_header diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index e8318c0846..5a6ce3981b 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -258,6 +258,13 @@ CROARINGBITMAP_DOWNLOAD="https://github.com/RoaringBitmap/CRoaring/archive/v0.2. CROARINGBITMAP_NAME=CRoaring-0.2.60.tar.gz CROARINGBITMAP_SOURCE=CRoaring-0.2.60 CROARINGBITMAP_MD5SUM="29602918e6890ffdeed84cb171857046" + +# fmt +FMT_DOWNLOAD="https://github.com/fmtlib/fmt/archive/7.1.3.tar.gz" +FMT_NAME="fmt-7.1.3.tar.gz" +FMT_SOURCE="fmt-7.1.3" +FMT_MD5SUM="2522ec65070c0bda0ca288677ded2831" + # ORC ORC_DOWNLOAD="https://archive.apache.org/dist/orc/orc-1.5.8/orc-1.5.8.tar.gz" ORC_NAME=orc-1.5.8.tar.gz @@ -377,6 +384,7 @@ ZSTD S2 BITSHUFFLE CROARINGBITMAP +FMT ORC JEMALLOC CCTZ