database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug](serde) fix serialize column to jsonb when meet boolean and decimal_v3 (#19011)

Browse Source 
* [Bug](serde) fix serialize column to jsonb when meet boolean and decimal_v3

* add comment to explain why use uint8
This commit is contained in:
lihangyu
2023-04-25 10:48:13 +08:00
committed by GitHub
parent 4ef43f5374
commit d555bae290
5 changed files with 50 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
be/src/vec/data_types/serde/data_type_decimal_serde.h
View File

@ -107,7 +107,8 @@ void DataTypeDecimalSerDe<T>::write_one_cell_to_jsonb(const IColumn& column, Jso
*reinterpret_cast<const Decimal128::NativeType*>(data_ref.data);
result.writeInt128(val);
} else if constexpr (std::is_same_v<T, Decimal<Int128I>>) {
Decimal64::NativeType val = *reinterpret_cast<const Decimal64::NativeType*>(data_ref.data);
Decimal128I::NativeType val =
*reinterpret_cast<const Decimal128I::NativeType*>(data_ref.data);
result.writeInt128(val);
} else if constexpr (std::is_same_v<T, Decimal<Int32>>) {
Decimal32::NativeType val = *reinterpret_cast<const Decimal32::NativeType*>(data_ref.data);

8
be/src/vec/data_types/serde/data_type_number_serde.h
View File

@ -185,7 +185,7 @@ template <typename T>
void DataTypeNumberSerDe<T>::read_one_cell_from_jsonb(IColumn& column,
const JsonbValue* arg) const {
auto& col = reinterpret_cast<ColumnType&>(column);
if constexpr (std::is_same_v<T, Int8>) {
if constexpr (std::is_same_v<T, Int8> || std::is_same_v<T, UInt8>) {
col.insert_value(static_cast<const JsonbInt8Val*>(arg)->val());
} else if constexpr (std::is_same_v<T, Int16> || std::is_same_v<T, UInt16>) {
col.insert_value(static_cast<const JsonbInt16Val*>(arg)->val());
@ -210,7 +210,11 @@ void DataTypeNumberSerDe<T>::write_one_cell_to_jsonb(const IColumn& column,
int row_num) const {
result.writeKey(col_id);
StringRef data_ref = column.get_data_at(row_num);
if constexpr (std::is_same_v<T, Int8>) {
// TODO: Casting unsigned integers to signed integers may result in loss of data precision.
// However, as Doris currently does not support unsigned integers, only the boolean type uses
// uint8_t for representation, making the cast acceptable. In the future, we should add support for
// both unsigned integers in Doris types and the JSONB types.
if constexpr (std::is_same_v<T, Int8> || std::is_same_v<T, UInt8>) {
int8_t val = *reinterpret_cast<const int8_t*>(data_ref.data);
result.writeInt8(val);
} else if constexpr (std::is_same_v<T, Int16> || std::is_same_v<T, UInt16>) {

4
be/src/vec/jsonb/serialize.cpp
View File

@ -79,6 +79,10 @@ void JsonbSerializeUtil::jsonb_to_block(const TupleDescriptor& desc, const char*
SlotDescriptor* slot = desc.slots()[j];
JsonbValue* slot_value = doc->find(slot->col_unique_id());
MutableColumnPtr dst_column = dst.get_by_position(j).column->assume_mutable();
if (!slot_value || slot_value->isNull()) {
dst_column->insert_default();
continue;
}
dst.get_data_type(j)->get_serde()->read_one_cell_from_jsonb(*dst_column, slot_value);
}
}

46
regression-test/data/point_query_p0/test_point_query.out
View File

@ -1,24 +1,24 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !point_select --
1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01
1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 false 1.111
-- !point_select --
1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01
1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 false 1.111
-- !point_select --
1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01
1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374
-- !point_select --
251 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01T12:36:38 251.0 7022-01-01
251 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01T12:36:38 251.0 7022-01-01 true 90696620686827832.374
-- !point_select --
252 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01T12:36:38 252.0 7022-01-01
252 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01T12:36:38 252.0 7022-01-01 false 90696620686827832.374
-- !point_select --
298 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01T12:36:38 298.0 7022-01-01
298 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01T12:36:38 298.0 7022-01-01 true 90696620686827832.374
-- !point_select --
1235 991129292901.111380000 dd \N 2120-01-02 2020-01-01T12:36:38 652.692 5022-01-01
1235 991129292901.111380000 dd \N 2120-01-02 2020-01-01T12:36:38 652.692 5022-01-01 false 90696620686827832.374
-- !point_select --
646464 6C616F6F71
@ -30,50 +30,50 @@
646464 6C616F6F71
-- !point_select --
1235 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 123
1235 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 true 1.000 1
-- !point_select --
1235 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 123
1235 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 true 1.000 1
-- !point_select --
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 123
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 true 0.000 1929111
-- !point_select --
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 123
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 true 0.000 1929111
-- !point_select --
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 123
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 true 0.000 1929111
-- !point_select --
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 123 \N
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 true 0.000 1929111 \N
-- !point_select --
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 123 \N
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 true 0.000 1929111 \N
-- !point_select --
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 123
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 true 0.000 1929111
-- !point_select --
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 123
1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 true 0.000 1929111
-- !sql --
1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N
1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 false 1.111 \N
-- !sql --
1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 \N
1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 \N
-- !sql --
6120202020646464 6C616F6F71 32.922000640
6120202020646464 6C616F6F71 32.92200050354004
-- !sql --
1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N
1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 false 1.111 \N
-- !sql --
1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 \N
1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 \N
-- !sql --
1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N
1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 false 1.111 \N
-- !sql --
1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 \N
1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 \N

28
regression-test/suites/point_query_p0/test_point_query.groovy
View File

@ -73,7 +73,9 @@ suite("test_point_query") {
`k5` date NULL COMMENT "",
`k6` datetime NULL COMMENT "",
`k7` float NULL COMMENT "",
`k8` datev2 NULL COMMENT ""
`k8` datev2 NULL COMMENT "",
`k9` boolean NULL COMMENT "",
`k10` decimalv3(20, 3) NULL COMMENT ""
) ENGINE=OLAP
UNIQUE KEY(`k1`, `k2`, `k3`)
DISTRIBUTED BY HASH(`k1`, k2, k3) BUCKETS 1
@ -85,16 +87,16 @@ suite("test_point_query") {
"storage_format" = "V2"
)
"""
sql """ INSERT INTO ${tableName} VALUES(1231, 119291.11, "ddd", "laooq", null, "2020-01-01 12:36:38", null, "1022-01-01 11:30:38") """
sql """ INSERT INTO ${tableName} VALUES(1232, 12222.99121135, "xxx", "laooq", "2023-01-02", "2020-01-01 12:36:38", 522.762, "2022-01-01 11:30:38") """
sql """ INSERT INTO ${tableName} VALUES(1233, 1.392932911, "yyy", "laooq", "2024-01-02", "2020-01-01 12:36:38", 52.862, "3022-01-01 11:30:38") """
sql """ INSERT INTO ${tableName} VALUES(1234, 12919291.129191137, "xxddd", "laooq", "2025-01-02", "2020-01-01 12:36:38", 552.872, "4022-01-01 11:30:38") """
sql """ INSERT INTO ${tableName} VALUES(1235, 991129292901.11138, "dd", null, "2120-01-02", "2020-01-01 12:36:38", 652.692, "5022-01-01 11:30:38") """
sql """ INSERT INTO ${tableName} VALUES(1236, 100320.11139, "laa ddd", "laooq", "2220-01-02", "2020-01-01 12:36:38", 2.7692, "6022-01-01 11:30:38") """
sql """ INSERT INTO ${tableName} VALUES(1237, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38") """
sql """ INSERT INTO ${tableName} VALUES(251, 120939.11130, "${generateString(251)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 251, "7022-01-01 11:30:38") """
sql """ INSERT INTO ${tableName} VALUES(252, 120939.11130, "${generateString(252)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 252, "7022-01-01 11:30:38") """
sql """ INSERT INTO ${tableName} VALUES(298, 120939.11130, "${generateString(298)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 298, "7022-01-01 11:30:38") """
sql """ INSERT INTO ${tableName} VALUES(1231, 119291.11, "ddd", "laooq", null, "2020-01-01 12:36:38", null, "1022-01-01 11:30:38", 0, 1.111112) """
sql """ INSERT INTO ${tableName} VALUES(1232, 12222.99121135, "xxx", "laooq", "2023-01-02", "2020-01-01 12:36:38", 522.762, "2022-01-01 11:30:38", 1, 212.111) """
sql """ INSERT INTO ${tableName} VALUES(1233, 1.392932911, "yyy", "laooq", "2024-01-02", "2020-01-01 12:36:38", 52.862, "3022-01-01 11:30:38", 0, 5973903488739435.668) """
sql """ INSERT INTO ${tableName} VALUES(1234, 12919291.129191137, "xxddd", "laooq", "2025-01-02", "2020-01-01 12:36:38", 552.872, "4022-01-01 11:30:38", 1, 5973903488739435.668) """
sql """ INSERT INTO ${tableName} VALUES(1235, 991129292901.11138, "dd", null, "2120-01-02", "2020-01-01 12:36:38", 652.692, "5022-01-01 11:30:38", 0, 90696620686827832.374) """
sql """ INSERT INTO ${tableName} VALUES(1236, 100320.11139, "laa ddd", "laooq", "2220-01-02", "2020-01-01 12:36:38", 2.7692, "6022-01-01 11:30:38", 1, 23698.299) """
sql """ INSERT INTO ${tableName} VALUES(1237, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", 0, 90696620686827832.374) """
sql """ INSERT INTO ${tableName} VALUES(251, 120939.11130, "${generateString(251)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 251, "7022-01-01 11:30:38", 1, 90696620686827832.374) """
sql """ INSERT INTO ${tableName} VALUES(252, 120939.11130, "${generateString(252)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 252, "7022-01-01 11:30:38", 0, 90696620686827832.374) """
sql """ INSERT INTO ${tableName} VALUES(298, 120939.11130, "${generateString(298)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 298, "7022-01-01 11:30:38", 1, 90696620686827832.374) """
def nprep_sql = {sql_str->
def url_without_prep ="jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb
@ -151,13 +153,13 @@ suite("test_point_query") {
ALTER table ${tableName} ADD COLUMN new_column0 INT default "0";
"""
sleep(1);
nprep_sql """ INSERT INTO ${tableName} VALUES(1235, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", 123) """
nprep_sql """ INSERT INTO ${tableName} VALUES(1235, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", 123, 1, 1.1111111299) """
stmt.setBigDecimal(1, new BigDecimal("120939.11130"))
stmt.setString(2, "a ddd")
qe_point_select stmt
qe_point_select stmt
// invalidate cache
nprep_sql """ INSERT INTO ${tableName} VALUES(1235, 120939.11130, "a ddd", "xxxxxx", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", 123) """
nprep_sql """ INSERT INTO ${tableName} VALUES(1235, 120939.11130, "a ddd", "xxxxxx", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", 123, 0, 1929111.1111) """
qe_point_select stmt
qe_point_select stmt
qe_point_select stmt