database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](json) fix json int128 overflow (#22917)

Browse Source 
* support int128 in jsonb

* fix jsonb int128 write

* fix jsonb to json int128

* fix json functions for int128

* add nereids function jsonb_extract_largeint

* add testcase for json int128

* change docs for json int128

* add nereids function jsonb_extract_largeint

* clang format

* fix check style

* using int128_t = __int128_t for all int128

* use fmt::format_to instead of snprintf digit by digit for int128

* clang format

* delete useless check

* add warn log

* clang format
This commit is contained in:
Kang
2023-08-25 11:40:30 +08:00
committed by GitHub
parent 372f83df5c
commit 8ef6b4d996
20 changed files with 2972 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
be/src/vec/functions/function_cast.h
View File

@ -616,6 +616,8 @@ struct ConvertImplNumberToJsonb {
writer.writeInt32(data[i]);
} else if constexpr (std::is_same_v<ColumnInt64, ColumnType>) {
writer.writeInt64(data[i]);
} else if constexpr (std::is_same_v<ColumnInt128, ColumnType>) {
writer.writeInt128(data[i]);
} else if constexpr (std::is_same_v<ColumnFloat64, ColumnType>) {
writer.writeDouble(data[i]);
} else {
@ -721,7 +723,7 @@ struct ConvertImplFromJsonb {
}
} else if constexpr (type_index == TypeIndex::Int8) {
if (value->isInt8()) {
res[i] = ((const JsonbIntVal*)value)->val();
res[i] = (int8_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
@ -743,7 +745,15 @@ struct ConvertImplFromJsonb {
} else if constexpr (type_index == TypeIndex::Int64) {
if (value->isInt8() || value->isInt16() || value->isInt32() ||
value->isInt64()) {
res[i] = ((const JsonbIntVal*)value)->val();
res[i] = (int64_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Int128) {
if (value->isInt8() || value->isInt16() || value->isInt32() ||
value->isInt64() || value->isInt128()) {
res[i] = (int128_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
@ -1793,6 +1803,8 @@ private:
return &ConvertImplFromJsonb<TypeIndex::Int32, ColumnInt32>::execute;
case TypeIndex::Int64:
return &ConvertImplFromJsonb<TypeIndex::Int64, ColumnInt64>::execute;
case TypeIndex::Int128:
return &ConvertImplFromJsonb<TypeIndex::Int128, ColumnInt128>::execute;
case TypeIndex::Float64:
return &ConvertImplFromJsonb<TypeIndex::Float64, ColumnFloat64>::execute;
default:
@ -1817,6 +1829,8 @@ private:
return &ConvertImplNumberToJsonb<ColumnInt32>::execute;
case TypeIndex::Int64:
return &ConvertImplNumberToJsonb<ColumnInt64>::execute;
case TypeIndex::Int128:
return &ConvertImplNumberToJsonb<ColumnInt128>::execute;
case TypeIndex::Float64:
return &ConvertImplNumberToJsonb<ColumnFloat64>::execute;
case TypeIndex::String:

25
be/src/vec/functions/function_jsonb.cpp
View File

@ -756,7 +756,15 @@ private:
}
} else if constexpr (std::is_same_v<int64_t, typename ValueType::T>) {
if (value->isInt8() || value->isInt16() || value->isInt32() || value->isInt64()) {
res[i] = ((const JsonbIntVal*)value)->val();
res[i] = (int64_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (std::is_same_v<int128_t, typename ValueType::T>) {
if (value->isInt8() || value->isInt16() || value->isInt32() || value->isInt64() ||
value->isInt128()) {
res[i] = (int128_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
@ -892,6 +900,13 @@ struct JsonbTypeInt64 {
static const bool only_check_exists = false;
};
struct JsonbTypeInt128 {
using T = int128_t;
using ReturnType = DataTypeInt128;
using ColumnType = ColumnVector<T>;
static const bool only_check_exists = false;
};
struct JsonbTypeDouble {
using T = double;
using ReturnType = DataTypeFloat64;
@ -948,6 +963,11 @@ struct JsonbExtractBigInt : public JsonbExtractImpl<JsonbTypeInt64> {
static constexpr auto alias = "jsonb_extract_bigint";
};
struct JsonbExtractLargeInt : public JsonbExtractImpl<JsonbTypeInt128> {
static constexpr auto name = "json_extract_largeint";
static constexpr auto alias = "jsonb_extract_largeint";
};
struct JsonbExtractDouble : public JsonbExtractImpl<JsonbTypeDouble> {
static constexpr auto name = "json_extract_double";
static constexpr auto alias = "jsonb_extract_double";
@ -975,6 +995,7 @@ using FunctionJsonbExtractIsnull = FunctionJsonbExtract<JsonbExtractIsnull>;
using FunctionJsonbExtractBool = FunctionJsonbExtract<JsonbExtractBool>;
using FunctionJsonbExtractInt = FunctionJsonbExtract<JsonbExtractInt>;
using FunctionJsonbExtractBigInt = FunctionJsonbExtract<JsonbExtractBigInt>;
using FunctionJsonbExtractLargeInt = FunctionJsonbExtract<JsonbExtractLargeInt>;
using FunctionJsonbExtractDouble = FunctionJsonbExtract<JsonbExtractDouble>;
using FunctionJsonbExtractString = FunctionJsonbExtract<JsonbExtractString>;
using FunctionJsonbExtractJsonb = FunctionJsonbExtract<JsonbExtractJsonb>;
@ -1027,6 +1048,8 @@ void register_function_jsonb(SimpleFunctionFactory& factory) {
factory.register_alias(FunctionJsonbExtractInt::name, FunctionJsonbExtractInt::alias);
factory.register_function<FunctionJsonbExtractBigInt>();
factory.register_alias(FunctionJsonbExtractBigInt::name, FunctionJsonbExtractBigInt::alias);
factory.register_function<FunctionJsonbExtractLargeInt>();
factory.register_alias(FunctionJsonbExtractLargeInt::name, FunctionJsonbExtractLargeInt::alias);
factory.register_function<FunctionJsonbExtractDouble>();
factory.register_alias(FunctionJsonbExtractDouble::name, FunctionJsonbExtractDouble::alias);
factory.register_function<FunctionJsonbExtractString>();