database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](mysql result format) use new serde framework to tuple convert (#25006)

Browse Source
This commit is contained in:
zhiqiang
2023-10-14 06:46:42 -05:00
committed by GitHub
parent b946521a56
commit e5ef0aa6d4
15 changed files with 160 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/cmake/thirdparty.cmake
View File

@ -164,6 +164,8 @@ add_thirdparty(krb5)
add_thirdparty(com_err)
add_thirdparty(k5crypto)
add_thirdparty(gssapi_krb5)
add_thirdparty(dragonbox_to_chars LIB64)
target_include_directories(dragonbox_to_chars INTERFACE "${THIRDPARTY_DIR}/include/dragonbox-1.1.3")
if (OS_MACOSX)
add_thirdparty(bfd)

2
be/src/gutil/CMakeLists.txt
View File

@ -57,6 +57,8 @@ add_library(Gutil STATIC ${SOURCE_FILES})
set_target_properties(Gutil PROPERTIES COMPILE_FLAGS "-funsigned-char -Wno-deprecated")
target_compile_options(Gutil PRIVATE -Wno-char-subscripts -Wno-implicit-fallthrough)
target_link_libraries(Gutil PRIVATE dragonbox_to_chars)
# target_link_libraries(Gutil glog protobuf rt)
#set(GUTIL_LIBS

13
be/src/gutil/strings/numbers.cc
View File

@ -27,6 +27,7 @@ using std::string;
#include "common/logging.h"
#include <fmt/format.h>
#include "dragonbox/dragonbox_to_chars.h"
#include "gutil/gscoped_ptr.h"
#include "gutil/int128.h"
#include "gutil/integral_types.h"
@ -1274,7 +1275,11 @@ int FloatToBuffer(float value, int width, char* buffer) {
return snprintf_result;
}
int FastDoubleToBuffer(double value, char* buffer) {
int FastDoubleToBuffer(double value, char* buffer, bool faster_float_convert) {
if (faster_float_convert) {
return jkj::dragonbox::to_chars_n(value, buffer) - buffer;
}
auto end = fmt::format_to(buffer, "{:.15g}", value);
*end = '\0';
if (strtod(buffer, nullptr) != value) {
@ -1283,7 +1288,11 @@ int FastDoubleToBuffer(double value, char* buffer) {
return end - buffer;
}
int FastFloatToBuffer(float value, char* buffer) {
int FastFloatToBuffer(float value, char* buffer, bool faster_float_convert) {
if (faster_float_convert) {
return jkj::dragonbox::to_chars_n(value, buffer) - buffer;
}
auto end = fmt::format_to(buffer, "{:.6g}", value);
*end = '\0';
#ifdef _MSC_VER // has no strtof()

4
be/src/gutil/strings/numbers.h
View File

@ -443,8 +443,8 @@ int FloatToBuffer(float i, int width, char* buffer);
char* DoubleToBuffer(double i, char* buffer);
char* FloatToBuffer(float i, char* buffer);
int FastDoubleToBuffer(double i, char* buffer);
int FastFloatToBuffer(float i, char* buffer);
int FastDoubleToBuffer(double i, char* buffer, bool faster_float_convert = false);
int FastFloatToBuffer(float i, char* buffer, bool faster_float_convert = false);
// In practice, doubles should never need more than 24 bytes and floats
// should never need more than 14 (including null terminators), but we
// overestimate to be safe.

4
be/src/runtime/runtime_state.h
View File

@ -151,6 +151,10 @@ public:
_query_options.enable_common_expr_pushdown;
}
bool enable_faster_float_convert() const {
return _query_options.__isset.faster_float_convert && _query_options.faster_float_convert;
}
Status query_status();
// Appends error to the _error_log if there is space

10
be/src/util/mysql_row_buffer.cpp
View File

@ -174,12 +174,12 @@ static char* add_largeint(int128_t data, char* pos, bool dynamic_mode) {
}
template <typename T>
char* add_float(T data, char* pos, bool dynamic_mode) {
char* add_float(T data, char* pos, bool dynamic_mode, bool faster_float_convert = false) {
int length = 0;
if constexpr (std::is_same_v<T, float>) {
length = FastFloatToBuffer(data, pos + !dynamic_mode);
length = FastFloatToBuffer(data, pos + !dynamic_mode, faster_float_convert);
} else if constexpr (std::is_same_v<T, double>) {
length = FastDoubleToBuffer(data, pos + !dynamic_mode);
length = FastDoubleToBuffer(data, pos + !dynamic_mode, faster_float_convert);
}
if (!dynamic_mode) {
int1store(pos++, length);
@ -385,7 +385,7 @@ int MysqlRowBuffer<is_binary_format>::push_float(float data) {
return ret;
}
_pos = add_float(data, _pos, _dynamic_mode);
_pos = add_float(data, _pos, _dynamic_mode, _faster_float_convert);
return 0;
}
@ -405,7 +405,7 @@ int MysqlRowBuffer<is_binary_format>::push_double(double data) {
return ret;
}
_pos = add_float(data, _pos, _dynamic_mode);
_pos = add_float(data, _pos, _dynamic_mode, _faster_float_convert);
return 0;
}

4
be/src/util/mysql_row_buffer.h
View File

@ -124,6 +124,8 @@ public:
*/
void close_dynamic_mode();
void set_faster_float_convert(bool faster) { _faster_float_convert = faster; }
private:
int reserve(int64_t size);
@ -142,6 +144,8 @@ private:
uint64_t _len_pos;
uint32_t _field_pos = 0;
uint32_t _field_count = 0;
bool _faster_float_convert = false;
};
} // namespace doris

3
be/src/vec/sink/vmysql_result_writer.cpp
View File

@ -90,6 +90,8 @@ Status VMysqlResultWriter<is_binary_format>::init(RuntimeState* state) {
}
set_output_object_data(state->return_object_data_as_binary());
_is_dry_run = state->query_options().dry_run_query;
_enable_faster_float_convert = state->enable_faster_float_convert();
return Status::OK();
}
@ -126,6 +128,7 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
{
SCOPED_TIMER(_convert_tuple_timer);
MysqlRowBuffer<is_binary_format> row_buffer;
row_buffer.set_faster_float_convert(_enable_faster_float_convert);
if constexpr (is_binary_format) {
row_buffer.start_binary_row(_output_vexpr_ctxs.size());
}

2
be/src/vec/sink/vmysql_result_writer.h
View File

@ -89,6 +89,8 @@ private:
bool _is_dry_run = false;
uint64_t _bytes_sent = 0;
bool _enable_faster_float_convert = false;
};
} // namespace vectorized
} // namespace doris

58
be/test/util/mysql_row_buffer_test.cpp
View File

@ -118,4 +118,62 @@ TEST(MysqlRowBufferTest, dynamic_mode) {
EXPECT_EQ(0, strncmp(buf + 43, "test", 4));
}
TEST(MysqlRowBufferTest /*unused*/, faster_float_convert /*unused*/) {
MysqlRowBuffer mrb;
mrb.set_faster_float_convert(true);
mrb.push_float(0);
mrb.push_float(1.0);
mrb.push_float(-1.0);
mrb.push_float(56.45);
mrb.push_double(10.12);
const char* buf = mrb.buf();
// mem: size-data-size-data
// 3-'0E0'-3-'1E0'-4-'-1E0'-7-'5.645E1'-7-'1.012E1'
// 1b-3b---1b-3b---1b-4b----1b-7b-------1b-7b------
// 0 1 4 5 8 9 13 14 21 22
EXPECT_EQ(29, mrb.length());
EXPECT_EQ(3, *((int8_t*)(buf + 0)));
EXPECT_EQ(0, strncmp(buf + 1, "0E0", 3));
EXPECT_EQ(3, *((int8_t*)(buf + 4)));
EXPECT_EQ(0, strncmp(buf + 5, "1E0", 3));
EXPECT_EQ(4, *((int8_t*)(buf + 8)));
EXPECT_EQ(0, strncmp(buf + 9, "-1E0", 4));
EXPECT_EQ(7, *((int8_t*)(buf + 13)));
EXPECT_EQ(0, strncmp(buf + 14, "5.645E1", 7));
EXPECT_EQ(7, *((int8_t*)(buf + 21)));
EXPECT_EQ(0, strncmp(buf + 22, "1.012E1", 7));
}
TEST(MysqlRowBufferTest /*unused*/, faster_float_convert_dynamic /*unused*/) {
MysqlRowBuffer mrb;
mrb.set_faster_float_convert(true);
mrb.open_dynamic_mode();
mrb.push_float(0);
mrb.push_float(1.0);
mrb.push_float(-1.0);
mrb.push_float(56.45);
mrb.push_double(10.12);
const char* buf = mrb.buf();
EXPECT_EQ(33, mrb.length());
EXPECT_EQ(254, *((uint8_t*)(buf)));
EXPECT_EQ(0, strncmp(buf + 9, "0E0", 3));
EXPECT_EQ(0, strncmp(buf + 12, "1E0", 3));
EXPECT_EQ(0, strncmp(buf + 15, "-1E0", 4));
EXPECT_EQ(0, strncmp(buf + 19, "5.645E1", 7));
EXPECT_EQ(0, strncmp(buf + 26, "1.012E1", 7));
}
} // namespace doris

2
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -253,6 +253,7 @@ public class Coordinator implements CoordInterface {
private boolean enablePipelineEngine = false;
private boolean enablePipelineXEngine = false;
private boolean fasterFloatConvert = false;
// Runtime filter merge instance address and ID
public TNetworkAddress runtimeFilterMergeAddr;
@ -324,6 +325,7 @@ public class Coordinator implements CoordInterface {
&& (fragments.size() > 0);
this.enablePipelineXEngine = context.getSessionVariable().getEnablePipelineXEngine()
&& (fragments.size() > 0);
this.fasterFloatConvert = context.getSessionVariable().fasterFloatConvert();
initQueryOptions(context);

12
fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View File

@ -416,6 +416,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_FULL_AUTO_ANALYZE = "enable_full_auto_analyze";
public static final String FASTER_FLOAT_CONVERT = "faster_float_convert";
public static final List<String> DEBUG_VARIABLES = ImmutableList.of(
SKIP_DELETE_PREDICATE,
SKIP_DELETE_BITMAP,
@ -1224,6 +1226,10 @@ public class SessionVariable implements Serializable, Writable {
flag = VariableMgr.GLOBAL)
public boolean enableFullAutoAnalyze = true;
@VariableMgr.VarAttr(name = FASTER_FLOAT_CONVERT,
description = {"是否启用更快的浮点数转换算法，注意会影响输出格式", "Set true to enable faster float pointer number convert"})
public boolean fasterFloatConvert = false;
// If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables,
// not the default value set in the code.
public void initFuzzyModeVariables() {
@ -2345,6 +2351,8 @@ public class SessionVariable implements Serializable, Writable {
tResult.setInvertedIndexConjunctionOptThreshold(invertedIndexConjunctionOptThreshold);
tResult.setFasterFloatConvert(fasterFloatConvert);
return tResult;
}
@ -2679,4 +2687,8 @@ public class SessionVariable implements Serializable, Writable {
public int getProfileLevel() {
return this.profileLevel;
}
public boolean fasterFloatConvert() {
return this.fasterFloatConvert;
}
}

4
gensrc/thrift/PaloInternalService.thrift
View File

@ -246,7 +246,9 @@ struct TQueryOptions {
// use is_report_success any more
84: optional bool enable_profile = false;
85: optional bool enable_page_cache = false;
86: optional i32 analyze_timeout = 43200
86: optional i32 analyze_timeout = 43200;
87: optional bool faster_float_convert = false;
}

10
regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out Normal file
View File

@ -0,0 +1,10 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select --
1 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878]
-- !select --
1 0.33 3.1415925 [3.3E-1, 6.7E-1] [3.1415926E0, 8.78787878E-1]
-- !select --
1 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878]
2 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878]

40
regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy Normal file
View File

@ -0,0 +1,40 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("mysql_tuple_convert_test") {
testTable = "doule_faster_convert_test"
sql "DROP TABLE IF EXISTS ${testTable}"
sql """
CREATE TABLE IF NOT EXISTS ${testTable} (
`col_0` INT,
`col_1` DOUBLE,
`col_2` FLOAT,
`col_3` ARRAY<FLOAT>,
`col_4` ARRAY<DOUBLE>)
ENGINE=OLAP DUPLICATE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`) BUCKETS 1
PROPERTIES("replication_num" = "1");
"""
sql """INSERT INTO ${testTable} VALUES (1, 0.33, 3.1415926, [0.33, 0.67], [3.1415926, 0.878787878]);"""
qt_select """SELECT /*+SET_VAR(faster_float_convert=false)*/ * FROM ${testTable};"""
qt_select """SELECT /*+SET_VAR(faster_float_convert=true)*/ * FROM ${testTable};"""
// make sure we can convert number from string to value
sql """INSERT INTO ${testTable} SELECT 2, col_1, col_2, col_3, col_4 from ${testTable} where col_0 = 1;"""
qt_select """SELECT * FROM ${testTable};"""
sql """SET faster_float_convert=false;"""
sql "DROP TABLE IF EXISTS ${testTable}"
}