database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add fmt library to speed up mysql text result serialization (#5554)

Browse Source 
* Add fmt library to speed up mysql text result serialization

* use BUILD_SYSTEM instead of make

Co-authored-by: gaodayue <gaodayue@bytedance.com>
This commit is contained in:
Dayue Gao
2021-04-08 09:16:58 +08:00
committed by GitHub
parent b423274f17
commit 2b5e4dc951
5 changed files with 52 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
be/CMakeLists.txt
View File

@ -258,6 +258,9 @@ set_target_properties(bitshuffle PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/
add_library(roaring STATIC IMPORTED)
set_target_properties(roaring PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libroaring.a)
add_library(fmt STATIC IMPORTED)
set_target_properties(fmt PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libfmt.a)
add_library(orc STATIC IMPORTED)
set_target_properties(orc PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/liborc.a)
@ -478,6 +481,7 @@ set(DORIS_DEPENDENCIES
leveldb
bitshuffle
roaring
fmt
double-conversion
jemalloc
brotlicommon

21
be/src/runtime/mysql_result_writer.cpp
View File

@ -66,7 +66,6 @@ void MysqlResultWriter::_init_profile() {
}
Status MysqlResultWriter::_add_one_row(TupleRow* row) {
SCOPED_TIMER(_convert_tuple_timer);
_row_buffer->reset();
int num_columns = _output_expr_ctxs.size();
int buf_ret = 0;
@ -213,18 +212,22 @@ Status MysqlResultWriter::append_row_batch(const RowBatch* batch) {
int num_rows = batch->num_rows();
result->result_batch.rows.resize(num_rows);
for (int i = 0; status.ok() && i < num_rows; ++i) {
TupleRow* row = batch->get_row(i);
status = _add_one_row(row);
{
SCOPED_TIMER(_convert_tuple_timer);
for (int i = 0; status.ok() && i < num_rows; ++i) {
TupleRow* row = batch->get_row(i);
status = _add_one_row(row);
if (status.ok()) {
result->result_batch.rows[i].assign(_row_buffer->buf(), _row_buffer->length());
} else {
LOG(WARNING) << "convert row to mysql result failed.";
break;
if (status.ok()) {
result->result_batch.rows[i].assign(_row_buffer->buf(), _row_buffer->length());
} else {
LOG(WARNING) << "convert row to mysql result failed.";
break;
}
}
}
if (status.ok()) {
SCOPED_TIMER(_result_send_timer);
// push this batch to back

52
be/src/util/mysql_row_buffer.cpp
View File

@ -24,6 +24,8 @@
#include "gutil/strings/numbers.h"
#include "util/mysql_global.h"
#include <fmt/format.h>
namespace doris {
// the first byte:
@ -97,6 +99,16 @@ int MysqlRowBuffer::reserve(int size) {
return 0;
}
template<typename T>
static char* add_int(T data, char* pos)
{
auto fi = fmt::format_int(data);
int length = fi.size();
int1store(pos++, length);
memcpy(pos, fi.data(), length);
return pos + length;
}
int MysqlRowBuffer::push_tinyint(int8_t data) {
// 1 for string trail, 1 for length, 1 for sign, other for digits
int ret = reserve(3 + MAX_TINYINT_WIDTH);
@ -106,15 +118,7 @@ int MysqlRowBuffer::push_tinyint(int8_t data) {
return ret;
}
int length = snprintf(_pos + 1, MAX_TINYINT_WIDTH + 2, "%d", data);
if (length < 0) {
LOG(ERROR) << "snprintf failed. data = " << data;
return length;
}
int1store(_pos, length);
_pos += length + 1;
_pos = add_int(data, _pos);
return 0;
}
@ -127,15 +131,7 @@ int MysqlRowBuffer::push_smallint(int16_t data) {
return ret;
}
int length = snprintf(_pos + 1, MAX_SMALLINT_WIDTH + 2, "%d", data);
if (length < 0) {
LOG(ERROR) << "snprintf failed. data = " << data;
return length;
}
int1store(_pos, length);
_pos += length + 1;
_pos = add_int(data, _pos);
return 0;
}
@ -148,15 +144,7 @@ int MysqlRowBuffer::push_int(int32_t data) {
return ret;
}
int length = snprintf(_pos + 1, MAX_INT_WIDTH + 2, "%d", data);
if (length < 0) {
LOG(ERROR) << "snprintf failed. data = " << data;
return length;
}
int1store(_pos, length);
_pos += length + 1;
_pos = add_int(data, _pos);
return 0;
}
@ -169,15 +157,7 @@ int MysqlRowBuffer::push_bigint(int64_t data) {
return ret;
}
int length = snprintf(_pos + 1, MAX_BIGINT_WIDTH + 2, "%ld", data);
if (length < 0) {
LOG(ERROR) << "snprintf failed. data = " << data;
return length;
}
int1store(_pos, length);
_pos += length + 1;
_pos = add_int(data, _pos);
return 0;
}

12
thirdparty/build-thirdparty.sh vendored
View File

@ -671,6 +671,17 @@ build_croaringbitmap() {
-DENABLE_ROARING_TESTS=OFF ..
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
# fmt
build_fmt() {
check_if_source_exist $FMT_SOURCE
cd $TP_SOURCE_DIR/$FMT_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
rm -rf CMakeCache.txt CMakeFiles/
$CMAKE_CMD -G "${GENERATOR}" -v -DBUILD_SHARED_LIBS=FALSE -DFMT_TEST=OFF -DFMT_DOC=OFF -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR ..
${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
}
#orc
build_orc() {
check_if_source_exist $ORC_SOURCE
@ -825,6 +836,7 @@ build_arrow
build_s2
build_bitshuffle
build_croaringbitmap
build_fmt
build_orc
build_cctz
build_tsan_header

8
thirdparty/vars.sh vendored
View File

@ -258,6 +258,13 @@ CROARINGBITMAP_DOWNLOAD="https://github.com/RoaringBitmap/CRoaring/archive/v0.2.
CROARINGBITMAP_NAME=CRoaring-0.2.60.tar.gz
CROARINGBITMAP_SOURCE=CRoaring-0.2.60
CROARINGBITMAP_MD5SUM="29602918e6890ffdeed84cb171857046"
# fmt
FMT_DOWNLOAD="https://github.com/fmtlib/fmt/archive/7.1.3.tar.gz"
FMT_NAME="fmt-7.1.3.tar.gz"
FMT_SOURCE="fmt-7.1.3"
FMT_MD5SUM="2522ec65070c0bda0ca288677ded2831"
# ORC
ORC_DOWNLOAD="https://archive.apache.org/dist/orc/orc-1.5.8/orc-1.5.8.tar.gz"
ORC_NAME=orc-1.5.8.tar.gz
@ -377,6 +384,7 @@ ZSTD
S2
BITSHUFFLE
CROARINGBITMAP
FMT
ORC
JEMALLOC
CCTZ